using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
class RemoveDuplicateWordsFreeText
{
/*
* SplitWords
*
* Splits free text into words using Unicode-aware regex.
*
* Regex explanation:
* \P{L}+ → any sequence of NON-letter characters
* \p{L} → any Unicode letter (Hebrew, Arabic, Latin, Cyrillic, etc.)
*
* This gives correct splitting for multilingual free text.
*/
static List<string> SplitWords(string text)
{
text = text.Trim();
// Unicode-aware split on non-letter sequences
string[] parts = Regex.Split(text, @"\P{L}+");
return new List<string>(parts);
}
/*
* RemoveDuplicateWords
*
* Removes duplicate words while preserving:
* - original order
* - original casing of first occurrence
* - case-insensitive comparison
*
* Uses HashSet for O(1) average lookup time.
*/
static string RemoveDuplicateWords(string text)
{
List<string> words = SplitWords(text);
HashSet<string> seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
List<string> unique = new List<string>();
foreach (string word in words)
{
if (word == "") continue;
// Case-insensitive check via HashSet
if (!seen.Contains(word)) {
seen.Add(word);
unique.Add(word); // preserve original casing
}
}
// Reassemble into a space-separated string
return string.Join(" ", unique);
}
static void Main()
{
string input =
"Hello, hello! This is a test. A TEST, hello universe... " +
"UNIVERSE! Hello; *** Is Anybody There?";
string output = RemoveDuplicateWords(input);
Console.WriteLine(output);
}
}
/*
run:
Hello This is a test universe Anybody There
*/
