How to use scripts or regexes to remove duplicate entries when creating an index?

Something like this could work. You could then use built in Sort Paragraph script to sort. The new text frame would be added in the upper left corner of the page with the text frame you select. The styling of the grafs probaly could be solve with some negative indentation/tab reconfiguration. Might need to sort the page numbers per custom object array. 

var main = function() {
    var o = {};
    try {
        var sel = app.selection[0];
        alert(sel.constructor.name);
        var ps = sel.parentStory;
    } catch(e) { 
        alert("Select a text frame and try again.");
        return; 
    }
    var pars = ps.paragraphs;
    var i = 0;
    var n = pars.length;
    var sp;
    for (i; i < n; i++) {
        sp = pars[i].contents.split("\t");

        if (!o.hasOwnProperty(sp[0])) {
            o[sp[0]] = [];
        }
        o[sp[0]].push(sp[1].contents.replace(/\r|\n/g,""));
    }
    var tf = sel.parentPage.textFrames.add({})
    for (var p in o) {
        tf.parentStory.contents += p + "\t" + o[p].join("\u2003") + "\r;
    }
}
main();