Question

How to remove duplicate case‑insensitive words separated by multiple delimiters from a string in Go

Answer 1

package main

import (
    "fmt"
    "regexp"
    "strings"
)

/*
    Remove duplicate case‑insensitive words separated by multiple delimiters.

    Features:
    - Case‑insensitive comparison (strings.ToLower)
    - Preserves original casing of the first occurrence
    - Trims whitespace around tokens
    - Supports ANY number of delimiters (including multi‑character)
    - Preserves original order
    - Efficient O(n) lookup using map[string]bool
*/

/* ---------------------------------------------------------------
   Build a regex that matches ANY delimiter
--------------------------------------------------------------- */
func buildDelimiterRegex(delimiters []string) *regexp.Regexp {
    escaped := make([]string, len(delimiters))

    // Escape delimiters so characters like "|" or "*" are treated literally
    for i, d := range delimiters {
        escaped[i] = regexp.QuoteMeta(d)
    }

    // Combine into one regex: (delim1|delim2|...)
    pattern := "(?:" + strings.Join(escaped, "|") + ")+"
    return regexp.MustCompile(pattern)
}

/* ---------------------------------------------------------------
   Split input string using multiple delimiters
--------------------------------------------------------------- */
func splitByDelimiters(input string, delimiters []string) []string {
    regex := buildDelimiterRegex(delimiters)

    // Split by any delimiter sequence
    rawTokens := regex.Split(input, -1)

    // Trim and filter empty tokens
    tokens := make([]string, 0, len(rawTokens))
    for _, t := range rawTokens {
        trimmed := strings.TrimSpace(t)
        if trimmed != "" {
            tokens = append(tokens, trimmed)
        }
    }
    return tokens
}

/* ---------------------------------------------------------------
   Remove duplicates (case‑insensitive)
--------------------------------------------------------------- */
func removeDuplicatesCaseInsensitive(tokens []string) []string {
    seen := make(map[string]bool)
    unique := make([]string, 0, len(tokens))

    for _, token := range tokens {
        key := strings.ToLower(token)
        if !seen[key] {
            seen[key] = true
            unique = append(unique, token)
        }
    }
    return unique
}

/* ---------------------------------------------------------------
   Join tokens with a chosen delimiter
--------------------------------------------------------------- */
func joinTokens(tokens []string, delimiter string) string {
    return strings.Join(tokens, delimiter)
}

/* ---------------------------------------------------------------
    Remove duplicates (Multi Delimiter case‑insensitive)
--------------------------------------------------------------- */
func removeDuplicatesMultiDelimiterCI(
    input string,
    delimiters []string,
    outputDelimiter string,
) string {

    tokens := splitByDelimiters(input, delimiters)
    unique := removeDuplicatesCaseInsensitive(tokens)
    return joinTokens(unique, outputDelimiter)
}

/* ---------------------------------------------------------------
  Main
--------------------------------------------------------------- */
func main() {
    s := "AAA | aaa ,   aAA * aaA | AAa | AAA   | BBB | ccc ---- CCC | AAA ; aaa | bbb"

    delimiters := []string{"  ", "|", ",", "*", "-", ";"}

    result := removeDuplicatesMultiDelimiterCI(s, delimiters, " | ")

    fmt.Println(result)
}



/*
run:

AAA | BBB | ccc

*/

 

70+ SQL courses for beginners and professionals