package main
import (
"fmt"
"regexp"
"strings"
)
/*
splitWords
Splits free text into words using a Unicode-aware regex.
Go supports Unicode character classes:
\p{L} → any Unicode letter
\P{L} → any NON-letter
So we split on ANY sequence of non-letters:
[^\p{L}]+
*/
func splitWords(text string) []string {
trimmed := strings.TrimSpace(text)
// Regex: split on any sequence of non-letter characters
re := regexp.MustCompile(`[^\p{L}]+`)
parts := re.Split(trimmed, -1)
return parts
}
/*
removeDuplicateWords
Removes duplicate words while preserving:
- original order
- original casing of first occurrence
- case-insensitive comparison
Uses a map[string]bool for O(1) lookup.
*/
func removeDuplicateWords(text string) string {
words := splitWords(text)
seen := make(map[string]bool)
unique := make([]string, 0)
for _, word := range words {
if word == "" {
continue
}
key := strings.ToLower(word) // Unicode-aware lowercase
if !seen[key] {
seen[key] = true
unique = append(unique, word) // preserve original casing
}
}
// Reassemble into a space-separated string
result := strings.Join(unique, " ")
return result
}
func main() {
input := "Hello, hello! This is a test. A TEST, hello universe... " +
"UNIVERSE! Hello; *** Is Anybody There?"
output := removeDuplicateWords(input)
fmt.Println(output)
}
/*
run:
Hello This is a test universe Anybody There
*/
