/**
* removeDuplicateWordsFreeTextUnicode
*
* Removes duplicate words from free text with full Unicode support:
* - Case‑insensitive (Unicode‑aware)
* - Preserves original casing of first occurrence
* - Splits on ANY non‑letter (Unicode) sequence
* - Trims whitespace
* - Preserves original order
*
* Uses Unicode regex classes:
* \p{L} = any letter in any language
* \p{M} = combining marks (accents, diacritics)
*
* @param string $text Input free text
* @return string Cleaned text with duplicates removed
*/
function removeDuplicateWordsFreeTextUnicode(string $text): string
{
// Normalize whitespace at edges
$text = trim($text);
/**
* Split on ANY sequence of characters that are NOT letters or combining marks.
*
* Regex explanation:
* /[^\p{L}\p{M}]+/u
* ^ → negation
* \p{L} → any Unicode letter (Hebrew, Arabic, Latin, Cyrillic, etc.)
* \p{M} → combining marks (accents)
* + → one or more
* u → Unicode mode
*/
$words = preg_split('/[^\p{L}\p{M}]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
$seen = [];
$unique = [];
foreach ($words as $word) {
// Unicode‑aware lowercase
$key = mb_strtolower($word, 'UTF-8');
if (!isset($seen[$key])) {
$seen[$key] = true;
$unique[] = $word; // preserve original casing
}
}
// Reassemble into a space‑separated string
return implode(' ', $unique);
}
// Free text with multiple languages + punctuation
$input = "Hello! こんにちは, ,hello こんにちは Bună ziua; Γεια σας Bună ziua *HELLO* Γεια σας";
$output = removeDuplicateWordsFreeTextUnicode($input);
echo $output;
/*
run:
Hello こんにちは Bună ziua Γεια σας
*/
