Merge Layers within LayerSet

A code you may play with to get that you want: How to merge multiple groups and keep them on their separate layers?

Here is other approach to the same problem if you have CS6 (however it works also in higher Adobe Photoshop versions):

function adl(v1, v2, v3) {
     eval("(" + v1 + " = new ActionReference()).put" + (!v3 ? "Enumerated" : "Index")
     + "(sTT(v2), " + (!v3 ? "sTT('ordinal'), sTT('targetEnum')" : v3) + ")")
}
function sec(v1, v2, v3) {
     lS = 'layerSection'; return typeIDToStringID(eval((v1 + (v1 ? ' = ' : '')
     + 'executeActionGet(v2)')).getEnumerationValue(sTT(lS))) != lS + v3
}
function rnm() {
     adl('ref1', 'layer'), dsc1.putReference(sTT('null'), ref1),
     (dsc2 = new ActionDescriptor()).putString(sTT('name'), nme)
     dsc1.putObject(sTT('to'), sTT('layer'), dsc2), eA('set')
}
function eA(v1, v2) {
     executeAction(sTT(v1), v2 ? new ActionDescriptor() : dsc1, DialogModes.NO)
}
function sTT(v) {return stringIDToTypeID(v)} $.level = 0
if (adl('ref1', 'layer'), sec('dsc1', ref1, 'Content')) {
     try{
          (ref = new ActionReference()).putIndex(sTT('layer'), 0)
          bG = (executeActionGet(ref)).getBoolean(sTT('background'))
     }
     catch (err) {bG = 0}
     adl('ref2', 'layer', dsc1.getInteger(sTT('itemIndex')) - 1 - bG)
     if (sec('', ref2, 'End')) {
          nme = dsc1.getString(sTT('name'))
          dsc1.putReference(sTT('null'), ref1)
          eA('ungroupLayersEvent');
          (lst = new ActionList()).putReference(ref1)
          dsc1.putList(sTT('null'), lst), eA('show')
          adl('ref', 'document'), tL = sTT('targetLayers')
          if ((dsc = executeActionGet(ref)).hasKey(tL) &&
          dsc.getList(tL).count > 1) eA('mergeLayersNew', true)
          rnm(), (ref2 = new ActionReference())
          .putClass(sTT(lS)), dsc1.putReference(sTT('null'), ref2)
          dsc1.putReference(sTT('from'), ref1), eA('make'), rnm()
     }
}

I noticed difference in 'targetLayers' working between CS6 and CC 2018. In CS6 that works for over one selected layer while in CC 2018 it works for one and more selected layers. So basicly this script is good for CS6 until I make a tweak for CC 2018 !

Tweak has been implemented by changing:

adl('ref', 'document'); if (executeActionGet(ref).hasKey
(sTT('targetLayers'))) eA('mergeLayersNew', true)

to:

adl('ref', 'document'), tL = sTT('targetLayers')
if ((dsc = executeActionGet(ref)).hasKey(tL) &&
dsc.getList(tL).count > 1) eA('mergeLayersNew', true)

This is awesome! Thanks for the feedback, guys! I'll give these a try and let you know how they work!

I will assume that you need this. (Works only in c2018 because of the "collapseAllGroupsEvent" command)

var myLayerSet = app.activeDocument.activeLayer;
var mergeLayerName = myLayerSet.name;
executeAction(stringIDToTypeID("collapseAllGroupsEvent"), undefined, DialogModes.NO); 
app.activeDocument.activeLayer = myLayerSet.layers[0];
select_back(myLayerSet.layers.length-1)
executeAction(stringIDToTypeID("mergeLayersNew"), undefined, DialogModes.NO);
myLayerSet.layers[0].name = mergeLayerName;
function select_back(count)
    {
    try {
        var d = new ActionDescriptor();
        var r = new ActionReference();
        r.putEnumerated(stringIDToTypeID("layer"), stringIDToTypeID("ordinal"), stringIDToTypeID("backwardEnum"));
        d.putReference(stringIDToTypeID("null"), r);
        d.putEnumerated(stringIDToTypeID("selectionModifier"), stringIDToTypeID("selectionModifierType"), stringIDToTypeID("addToSelection"));
        d.putBoolean(stringIDToTypeID("makeVisible"), false);
        for (var i = 0; i < count; i++) executeAction(stringIDToTypeID("select"), d, DialogModes.NO); 
        }
    catch (e) { alert(e); throw(e); }
    }

You mentioned being able to:

...get it to the point where it will merge the whole group to a layer and rename that, but that won't work for the application I'm trying to accommodate...

What if you do this, and then create a new LayerSet (with the same name) and place the merged one into it? Does that accomplish what you want?

var docRef = app.activeDocument;
var myLayerSet = docRef.layerSets.getByName("exampleLayerSet");
var mergeLayerName = myLayerSet.name;
var mergeLayerSet = myLayerSet.merge();
mergeLayerSet.name = mergeLayerName;
var newLayerSet = docRef.layerSets.add();
newLayerSet.name = mergeLayerName;
mergeLayerSet.move(newLayerSet, ElementPlacement.INSIDE);

If you are after a fast result, then the following action will get the job done fast for a selected group:

The key step being to hold down the opt/alt key when recording the visibility click on the group layer.

If this is a “simple” exercise to try to learn scripting, then it sounds like a good choice!

EDIT: Reading your post again, perhaps you wish to merge all layers within the set and name them after the set, but also retain the layer set as well? It is not clear to me which is required.