use std::collections::HashSet;
/// Removes duplicate words from a free-text string containing Unicode characters.
/// Preserves word order and the case of the first occurrence.
fn remove_duplicate_words(input: &str) -> String {
if input.trim().is_empty() {
return String::new();
}
// HashSet for O(1) duplicate checks on normalized lowercase strings.
let mut seen_words = HashSet::new();
// Split input into words using built-in Unicode char classification:
// A word boundary is defined by any non-alphanumeric character (excluding '_').
let unique_words: Vec<&str> = input
.split(|c: char| !c.is_alphanumeric() && c != '_')
.filter(|s| !s.is_empty())
.filter(|&word| {
// HashSet::insert returns true if the value was NOT present in the set
seen_words.insert(word.to_lowercase())
})
.collect();
// Join unique words with a single space.
unique_words.join(" ")
}
fn main() {
let input = "Hello! こんにちは, ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";
let result = remove_duplicate_words(input);
println!("{}", result);
}
/*
run:
Hello こんにちは Bună ziua Γεια σας
*/
