Question

How to remove duplicate words with Unicode characters from free‑text in VB.NET

Answer 1

Imports System
Imports System.Collections.Generic
Imports System.Text

' This program removes duplicate words from free text containing Unicode characters.
' It uses:
'   - VB.NET's built-in Unicode support
'   - Lowercasing + punctuation stripping for comparison
'   - Cleaned original (no punctuation, original casing) for output
'   - LinkedHash-like behavior using Dictionary(Of String, String)
'   - First occurrence wins, order preserved

Module RemoveUnicodeDuplicates

    ' Remove punctuation but keep original casing
    Function CleanPreserveCase(s As String) As String
        Dim sb As New StringBuilder()
        For Each c As Char In s
            ' Keep letters, digits, and all non-ASCII Unicode characters
            If Char.IsLetterOrDigit(c) Or Convert.ToInt32(c) > 127 Then
                sb.Append(c)
            End If
        Next
        Return sb.ToString()
    End Function

    ' Normalize a word: remove punctuation + lowercase (for comparison)
    Function NormalizeWord(s As String) As String
        Dim sb As New StringBuilder()
        For Each c As Char In s
            If Char.IsLetterOrDigit(c) Or Convert.ToInt32(c) > 127 Then
                sb.Append(Char.ToLowerInvariant(c))
            End If
        Next
        Return sb.ToString()
    End Function

    Sub Main()
        Dim input As String =
            "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας"

        ' Dictionary preserves insertion order in .NET Core / .NET 5+
        Dim unique As New Dictionary(Of String, String)()

        ' Split on whitespace
        For Each word As String In input.Split({" "}, StringSplitOptions.RemoveEmptyEntries)

            Dim normalized As String = NormalizeWord(word)
            Dim cleaned As String = CleanPreserveCase(word)

            If normalized.Length > 0 AndAlso Not unique.ContainsKey(normalized) Then
                unique(normalized) = cleaned
            End If
        Next

        ' Print result
        Dim output As New StringBuilder()
        For Each original As String In unique.Values
            output.Append(original).Append(" ")
        Next

        Console.WriteLine(output.ToString().Trim())
    End Sub

End Module



' run:
'
' Hello こんにちは Bună ziua Γεια σας
'

 

70+ SQL courses for beginners and professionals

Answer 2

Imports System
Imports System.Collections.Generic
Imports System.Text
Imports System.Text.RegularExpressions

Module WordDeduplicator

    '------------------------------------------------------------
    ' Removes duplicate words from free text containing Unicode.
    ' - Preserves original casing of the first occurrence
    ' - Removes punctuation automatically via Regex \w+
    ' - Uses HashSet for O(1) duplicate detection
    '------------------------------------------------------------
    Function RemoveDuplicateWords(input As String) As String
        If String.IsNullOrWhiteSpace(input) Then
            Return String.Empty
        End If

        ' \w+ matches Unicode letters, digits, and connector punctuation.
        ' Regex engine is fully Unicode-aware.
        Dim matches As MatchCollection = Regex.Matches(input, "\w+")

        ' Case-insensitive deduplication (OrdinalIgnoreCase = fast + Unicode-safe)
        Dim seen As New HashSet(Of String)(StringComparer.OrdinalIgnoreCase)

        Dim sb As New StringBuilder()

        For Each m As Match In matches
            Dim word As String = m.Value

            If seen.Add(word) Then
                If sb.Length > 0 Then
                    sb.Append(" ")
                End If
                sb.Append(word)
            End If
        Next

        Return sb.ToString()
    End Function

    Sub Main()
        Dim input As String =
            "Hello! こんにちは,  ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας"

        Dim result As String = RemoveDuplicateWords(input)

        Console.WriteLine(result)
    End Sub

End Module



' run:
'
' Hello こんにちは Bună ziua Γεια σας
'

 

70+ SQL courses for beginners and professionals