Question

How to remove duplicate case‑insensitive words separated by multiple delimiters from a string in C++

Answer 1

#include <iostream>
#include <string>
#include <vector>
#include <unordered_set>
#include <algorithm>
#include <sstream>

/*
    removeDuplicatesMultiDelimiterCI

    Removes duplicate words separated by MULTIPLE delimiters.
    Features:
    - Case‑insensitive comparison (ASCII)
    - Preserves original casing of first occurrence
    - Trims whitespace around tokens
    - Supports ANY number of delimiters, including multi‑character ones
    - Preserves original order
    - Efficient O(n) hashing with std::unordered_set

    Algorithm:
    1. Replace all delimiters with a single sentinel delimiter.
    2. Split by that sentinel.
    3. Trim each token.
    4. Convert to lowercase for comparison.
    5. Keep only first occurrence.
    6. Reassemble using a chosen delimiter.
*/

// Trim whitespace from both ends
std::string trim(const std::string& s) {
    const char* ws = " \t\n\r";
    size_t start = s.find_first_not_of(ws);
    if (start == std::string::npos) return "";
    size_t end = s.find_last_not_of(ws);
    return s.substr(start, end - start + 1);
}

// Lowercase (ASCII)
std::string toLower(const std::string& s) {
    std::string out = s;
    std::transform(out.begin(), out.end(), out.begin(),
                   [](unsigned char c){ return std::tolower(c); });
    return out;
}

std::string removeDuplicatesMultiDelimiterCI(
    const std::string& input,
    const std::vector<std::string>& delimiters,
    const std::string& outputDelimiter)
{
    // Step 1: Normalize all delimiters into a single sentinel
    std::string normalized = input;
    const std::string sentinel = "\n"; // safe delimiter unlikely to appear

    for (const auto& d : delimiters) {
        size_t pos = 0;
        while ((pos = normalized.find(d, pos)) != std::string::npos) {
            normalized.replace(pos, d.size(), sentinel);
            pos += sentinel.size();
        }
    }

    // Step 2: Split by sentinel
    std::vector<std::string> tokens;
    {
        size_t start = 0, pos = 0;
        while ((pos = normalized.find(sentinel, start)) != std::string::npos) {
            tokens.push_back(trim(normalized.substr(start, pos - start)));
            start = pos + sentinel.size();
        }
        tokens.push_back(trim(normalized.substr(start)));
    }

    // Step 3: Remove duplicates (case‑insensitive)
    std::unordered_set<std::string> seen;
    std::vector<std::string> unique;

    for (const auto& token : tokens) {
        if (token.empty()) continue;
        std::string key = toLower(token);
        if (seen.insert(key).second) {
            unique.push_back(token);
        }
    }

    // Step 4: Reassemble
    std::ostringstream out;
    for (size_t i = 0; i < unique.size(); ++i) {
        if (i > 0) out << outputDelimiter;
        out << unique[i];
    }

    return out.str();
}

int main() {
    std::string s =
        "AAA | aaa ,   aAA * aaA | AAa | AAA   | BBB | ccc ---- CCC | AAA ; aaa | bbb";

    // Your updated delimiter list
    std::vector<std::string> delimiters = {
        "  ", "|", ",", "*", "-", ";"
    };

    std::string result =
        removeDuplicatesMultiDelimiterCI(s, delimiters, " | ");

    std::cout << result << "\n";
}


/*
run:

AAA | BBB | ccc

*/

 

70+ SQL courses for beginners and professionals