Question

How to remove duplicate case‑insensitive words separated by multiple delimiters from a string in Rust

Answer 1

use regex::Regex;
use std::collections::HashSet;

/*
    Remove duplicate case‑insensitive words separated by multiple delimiters.

    Features:
    - Case‑insensitive comparison (to_lowercase)
    - Preserves original casing of the first occurrence
    - Trims whitespace around tokens
    - Supports ANY number of delimiters (including multi‑character)
    - Preserves original order
    - Efficient O(n) lookup using HashSet<String>
*/

/* ---------------------------------------------------------------
   Build a regex that matches ANY delimiter
--------------------------------------------------------------- */
fn build_delimiter_regex(delimiters: &[&str]) -> Regex {
    // Escape delimiters so characters like "|" or "*" are treated literally
    let escaped: Vec<String> = delimiters
        .iter()
        .map(|d| regex::escape(d))
        .collect();

    let pattern = format!("(?:{})+", escaped.join("|"));
    
    Regex::new(&pattern).unwrap()
}

/* ---------------------------------------------------------------
   Split input string using multiple delimiters
--------------------------------------------------------------- */
fn split_by_delimiters(input: &str, delimiters: &[&str]) -> Vec<String> {
    let regex = build_delimiter_regex(delimiters);

    regex
        .split(input)
        .map(|t| t.trim().to_string())
        .filter(|t| !t.is_empty())
        .collect()
}

/* ---------------------------------------------------------------
   Remove duplicates (case‑insensitive)
--------------------------------------------------------------- */
fn remove_duplicates_case_insensitive(tokens: &[String]) -> Vec<String> {
    let mut seen: HashSet<String> = HashSet::new();
    let mut unique: Vec<String> = Vec::new();

    for token in tokens {
        let key = token.to_lowercase();
        if !seen.contains(&key) {
            seen.insert(key);
            unique.push(token.clone());
        }
    }

    unique
}

/* ---------------------------------------------------------------
   Join tokens with a chosen delimiter
--------------------------------------------------------------- */
fn join_tokens(tokens: &[String], delimiter: &str) -> String {
    tokens.join(delimiter)
}

/* ---------------------------------------------------------------
   Main function
--------------------------------------------------------------- */
fn remove_duplicates_multi_delimiter_ci(
    input: &str,
    delimiters: &[&str],
    output_delimiter: &str,
) -> String {
    let tokens = split_by_delimiters(input, delimiters);
    let unique = remove_duplicates_case_insensitive(&tokens);
    
    join_tokens(&unique, output_delimiter)
}

/* ---------------------------------------------------------------
   Main
--------------------------------------------------------------- */
fn main() {
    let s = "AAA | aaa ,   aAA * aaA | AAa | AAA   | BBB | ccc ---- CCC | AAA ; aaa | bbb";

    let delimiters = ["  ", "|", ",", "*", "-", ";"];

    let result = remove_duplicates_multi_delimiter_ci(s, &delimiters, " | ");

    println!("{}", result);
}



/*
run:

AAA | BBB | ccc

*/

 

70+ SQL courses for beginners and professionals