package main
import (
"fmt"
"regexp"
"strings"
)
// RemoveDuplicateWords removes duplicate words from a free-text string containing Unicode characters.
// It preserves word order and the case of the first occurrence.
func RemoveDuplicateWords(input string) string {
if strings.TrimSpace(input) == "" {
return ""
}
// 1. Compile Unicode word matching regex:
// \p{L} matches any Unicode letter, \p{N} matches digits.
// This works seamlessly across scripts (Japanese, Greek, Latin, etc.).
wordRegex := regexp.MustCompile(`[\p{L}\p{N}_]+`)
words := wordRegex.FindAllString(input, -1)
// 2. Map initialized with empty struct{} serves as an O(1) memory-efficient Set.
seenWords := make(map[string]struct{})
uniqueWords := make([]string, 0, len(words))
// 3. Filter duplicates while maintaining original case & order.
for _, word := range words {
lowerWord := strings.ToLower(word)
if _, exists := seenWords[lowerWord]; !exists {
seenWords[lowerWord] = struct{}{}
uniqueWords = append(uniqueWords, word)
}
}
// 4. Join unique words with a single space.
return strings.Join(uniqueWords, " ")
}
func main() {
input := "Hello! こんにちは, ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας"
result := RemoveDuplicateWords(input)
fmt.Println(result)
}
/*
run:
Hello こんにちは Bună ziua Γεια σας
*/
