Question

How to remove duplicate words with Unicode characters from free‑text in C#

Answer 1

using System;
using System.Collections.Generic;
using System.Text;

/*
    This program removes duplicate words from free text containing Unicode characters.
    It uses:
      - C#'s built-in Unicode support
      - Lowercasing + punctuation stripping for comparison
      - Cleaned original (no punctuation, original casing) for output
      - Dictionary to preserve first occurrence order
*/

class Program
{
	// Remove punctuation but keep original casing
	static string CleanPreserveCase(string s)
	{
		var sb = new StringBuilder();
		foreach (char c in s) {
			// Keep letters, digits, and all non-ASCII Unicode characters
			if (char.IsLetterOrDigit(c) || c > 127)
				sb.Append(c);
		}

		return sb.ToString();
	}

	// Normalize a word: remove punctuation + lowercase (for comparison)
	static string NormalizeWord(string s)
	{
		var sb = new StringBuilder();
		foreach (char c in s) {
			if (char.IsLetterOrDigit(c) || c > 127)
				sb.Append(char.ToLowerInvariant(c));
		}
		
		return sb.ToString();
	}

	static void Main()
    {
        string input =
            "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";

        // Dictionary preserves insertion order in modern .NET
        var unique = new Dictionary<string, string>();

        // Split on whitespace
        foreach (string word in input.Split((char[])null, StringSplitOptions.RemoveEmptyEntries))
        {
            string normalized = NormalizeWord(word);
            string cleaned    = CleanPreserveCase(word);

            if (normalized.Length > 0 && !unique.ContainsKey(normalized))
                unique[normalized] = cleaned;
        }

        // Print result
        var output = new StringBuilder();
        foreach (string original in unique.Values)
            output.Append(original).Append(" ");

        Console.WriteLine(output.ToString().Trim());
    }
}


/*
run:

Hello こんにちは Bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals

Answer 2

using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;

public class WordDeduplicator
{
    /// <summary>
    /// Removes duplicate words from a free-text string containing Unicode characters.
    /// Preserves word order and the case of the first occurrence.
    /// </summary>
    /// <param name="input">The input free-text string containing punctuation and Unicode words.</param>
    /// <returns>A space-separated string of unique words.</returns>
    public static string RemoveDuplicateWords(string input)
    {
        if (string.IsNullOrWhiteSpace(input))
        {
            return string.Empty;
        }

        // 1. \w+ matches any sequence of Unicode word characters (letters, digits, connector punctuation).
        //    Punctuation like !, ;, *, and spaces are automatically filtered out.
        //    Regex engine natively supports full Unicode character sets (Latin, Japanese, Greek, etc.).
        MatchCollection matches = Regex.Matches(input, @"\w+");

        // 2. Use a HashSet with OrdinalIgnoreCase to track seen words in O(1) time without extra allocations.
        var seenWords = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        // 3. StringBuilder for efficient O(N) string construction.
        var resultBuilder = new StringBuilder();

        foreach (Match match in matches)
        {
            string word = match.Value;

            // Add returns true if the element was added (meaning it's the first time we've seen it)
            if (seenWords.Add(word))
            {
                if (resultBuilder.Length > 0)
                {
                    resultBuilder.Append(' ');
                }
                resultBuilder.Append(word);
            }
        }

        return resultBuilder.ToString();
    }

    public static void Main()
    {
        string input = "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";
        
        string result = RemoveDuplicateWords(input);
        
        Console.WriteLine(result);
    }
}


/*
run:

Hello こんにちは Bună ziua Γεια σας

*/

 

70+ SQL courses for beginners and professionals