Get selection from layer mask by Index, without activating the layer

[danielsian.com]

I'm trying to get a selection from a layer mask without activating the proper layer.
My starting point is this code, but it's not working properly.
If there is any active layer with mask, the script works, otherwise, it returns an error.
How can I make this code work flawless?

function getSelectionFromLayerMask(idx) {
  var desc = new ActionDescriptor();
  var ref = new ActionReference();
  ref.putProperty(charIDToTypeID("Chnl"), charIDToTypeID("fsel"));
  desc.putReference(charIDToTypeID("null"), ref);
  var ref1 = new ActionReference();
  ref1.putEnumerated(charIDToTypeID("Chnl"), charIDToTypeID("Chnl"), charIDToTypeID("Msk "));
  ref1.putIndex(charIDToTypeID("Lyr "), idx);
  desc.putReference( charIDToTypeID("T "), ref1);
  executeAction(charIDToTypeID("setd"), desc, DialogModes.NO)
}

[jazz-y]

Your code contains an error, line:

desc.putReference( charIDToTypeID("T "), ref1);

should look like this:

desc.putReference( charIDToTypeID( "T   "), ref1);

(perhaps something happened while loading the message, and initially this line was correct)

Selection cannot be created if the layer palette does not have active (any) layer. Solving the problem is quite simple - before creating a selection, you need to check whether at least one layer is selected on the palette. If not, select it with a script and continue (if necessary, after creating the selection, you can restore (that is, remove) the selection), for example:

var ref = new ActionReference()
ref.putProperty(s2t("property"), s2t("targetLayers"))
ref.putEnumerated(s2t("document"), s2t("ordinal"), s2t("targetEnum"))
if (!executeActionGet(ref).hasKey(s2t("targetLayers"))) {
    var ref = new ActionReference()
    ref.putIndex(s2t("layer"), 1)
    var desc = new ActionDescriptor()
    desc.putReference(s2t("null"), ref)
    executeAction(s2t("select"), desc, DialogModes.NO)

    // your code here

    var ref = new ActionReference()
    ref.putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"))
    var desc = new ActionDescriptor()
    desc.putReference(s2t("null"), ref)
    executeAction(s2t("selectNoLayers"), desc, DialogModes.NO)
}

function s2t(s) { return app.stringIDToTypeID(s) }

Also may be useful to pre-check if the desired layer has user mask:

function hasLayerUserMask (idx) {
    var ref = new ActionReference()
    ref.putProperty(s2t("property"), s2t("hasUserMask"))
    ref.putIndex(s2t("layer"), idx)
    return (executeActionGet(ref).getBoolean(s2t("hasUserMask")))
}

function s2t(s) { return app.stringIDToTypeID(s) }

[danielsian.com]

That was a typo or like you said, an error while loading, because in my master code is correct. Actually, I'm now using stringIDToTypeID.

I was activating the layer before getting the selection, like your suggestion. The problem doing this is the loss of performance.

So the solution I've found so far is getting the layer mask by ID. It's working perfectly without any layer pre-activated.
I'm getting the ID with:

stringIDToTypeID('layerID')

This is my updated working version:

function getSelectionFromLayerMask(ID) {
    var desc = new ActionDescriptor();
    var ref = new ActionReference();
    ref.putProperty(charIDToTypeID("Chnl"), charIDToTypeID("fsel"));
    desc.putReference(charIDToTypeID("null"), ref);
    var ref1 = new ActionReference();
    ref1.putEnumerated(charIDToTypeID("Chnl"), charIDToTypeID("Chnl"), charIDToTypeID("Msk "));
    ref1.putIdentifier(charIDToTypeID("Lyr "), ID);
    desc.putReference( charIDToTypeID("T   "), ref1);
    executeAction(charIDToTypeID("setd"), desc, DialogModes.NO)
}

 Thanks

[jazz-y]

It’s strange. Your new example on my Photoshop 21.1.1 works just like the first one - the selection is created only if the layer is pre-activated on the palette. If no layer is selected, or a background layer is selected, then I get an error. What is your version of Photoshop?

In the example above, I activated the layer only in one case - if there were no selected layers on the palette. In other cases, the state of the active layer did not change. I changed my code a bit - now it works in almost all cases:

getSelectionFromLayerMask(2)

function getSelectionFromLayerMask(idx) {
        s2t = stringIDToTypeID;

        (docRef = new ActionReference()).putProperty(s2t("property"), s2t("targetLayers"))
        docRef.putEnumerated(s2t("document"), s2t("ordinal"), s2t("targetEnum"))
        var targetLayers = executeActionGet(docRef).hasKey(s2t("targetLayers"));

        (docRef = new ActionReference()).putProperty(s2t("property"), s2t("numberOfLayers"))
        docRef.putEnumerated(s2t("document"), s2t("ordinal"), s2t("targetEnum"))
        var numberOfLayers = executeActionGet(docRef).getInteger(s2t("numberOfLayers"));

        (lrRef = new ActionReference()).putProperty(s2t("property"), s2t("background"))
        lrRef.putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"))
        var background = executeActionGet(lrRef).getBoolean(s2t("background"));

        if (numberOfLayers == 1 && background) return;

        (lrRef = new ActionReference()).putProperty(s2t("property"), s2t("hasUserMask"))
        lrRef.putIndex(s2t("layer"), idx)
        if (!executeActionGet(lrRef).getBoolean(s2t("hasUserMask"))) return;

        if (!targetLayers || background) {
                (ref = new ActionReference()).putIndex(s2t("layer"), 1);
                (desc = new ActionDescriptor()).putReference(s2t("null"), ref);
                executeAction(s2t("select"), desc, DialogModes.NO)
        }
        makeSelectionFromLayerMask(idx)

        if (!targetLayers) {
                (ref = new ActionReference()).putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
                (desc = new ActionDescriptor()).putReference(s2t("null"), ref);
                executeAction(s2t("selectNoLayers"), desc, DialogModes.NO)
        }

        if (background) {
                (ref = new ActionReference()).putIndex(s2t("layer"), 0);
                (desc = new ActionDescriptor()).putReference(s2t("null"), ref);
                executeAction(s2t("select"), desc, DialogModes.NO)
        }

        function makeSelectionFromLayerMask(idx) {
                var desc = new ActionDescriptor();
                var ref = new ActionReference();
                ref.putProperty(charIDToTypeID("Chnl"), charIDToTypeID("fsel"));
                desc.putReference(charIDToTypeID("null"), ref);
                var ref1 = new ActionReference();
                ref1.putEnumerated(charIDToTypeID("Chnl"), charIDToTypeID("Chnl"), charIDToTypeID("Msk "));
                ref1.putIndex(charIDToTypeID("Lyr "), idx);
                desc.putReference(charIDToTypeID("T   "), ref1);
                executeAction(charIDToTypeID("setd"), desc, DialogModes.NO)
        }
}

The most time-consuming operation is directly creating a selection. The time to get the layer attributes is practically zero and can be neglected (especially since if you do not take care of additional checks, this will lead to an error).

[danielsian.com]

I'm testing in a Photoshop CC 2015.5.0.
But after reading the problem you've got, I have a bad new for you, mate... Actually really interesting bad news.
I was able to test the exactly same script in a Photoshop 2020 21.1.1, like yours, and it's working flawless in all scenarios (no active layer, only Background selected, other layer with and without layer mask and finally, multiple layers selected). It works.

I got the ID and then used it in my main function like you see here:

function getSelectionFromLayerMask(ID) { 
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putProperty( stringIDToTypeID( "channel" ), stringIDToTypeID( "selection" ) );
	desc.putReference( stringIDToTypeID( "null" ), ref );
	var ref1 = new ActionReference();
	ref1.putEnumerated( stringIDToTypeID( "channel" ), stringIDToTypeID( "channel" ), stringIDToTypeID( "mask" ) );
	if ( ID != undefined ) ref1.putIdentifier(stringIDToTypeID( "layer" ), ID);
	desc.putReference( stringIDToTypeID( "to" ), ref1 );
	executeAction( stringIDToTypeID( "set" ), desc, DialogModes.NO );
}
function getLayerID(idx){
	var ref = new ActionReference();
	if ( idx != undefined ) ref.putIndex(stringIDToTypeID('layer'), idx);
	else ref.putEnumerated( stringIDToTypeID('layer'), stringIDToTypeID('ordinal'), stringIDToTypeID('targetEnum') );
	var desc = executeActionGet(ref);
	var layerID = desc.getInteger(stringIDToTypeID('layerID'));
	$.write("ID: "+layerID+"\n")
	return layerID;
}

//~ $.write(getLayerID()+"\n")
getSelectionFromLayerMask(15)

[jazz-y]

This is strange. I tried several versions of Photoshop on the virtual machine (starting with cs6) and they all returned an error in the situation I described. Perhaps the result is affected by the fact that I'm testing only this one function without taking into account the preliminary processing of the document that your code can do. To a lesser extent, the result depends on the contents of the document.

[danielsian.com]

"the result depends on the contents of the document."

You are absolutely correct.
As I said earlier, I'm using this function in another script, with many other functions.
And at the stage where I was asking the question in my first comment, the function was working perfectly with everything else in a dummy PSD test file.
But when the things started to get more complex in my script and in dummy PSD, the function got a selection from another layer mask, different from the one specified by the ID. I have to say it was really hard to debug because there was no errors, only a wrong result.
The funny thing is that this "mistake" is happening only with 1 layer mask of a total of more than 50 layers, where the function is getting the selection from each one at a time in a loop.

It's a really strange behavior, and it is related to the PSD content along with the script actions.
I just give up on using the ID in that function.
It's 100% safer to just select the layer mask and get its selection.

[jazz-y]

Did you make sure that the problem layer ID was obtained correctly? If you get an ID based on the index of the layer, then I hope you take into account that the numbering of the indices of the layers in Action Manager starts with one (the exception is the background layer, if it is, then the numbering starts from zero)

[danielsian.com]

This is how I'm getting the ID, before start processing each layer in a loop:

var numberOfLayers = getNumberOfLayers();
$.write("numberOfLayers="+numberOfLayers+"\n")

//===== collect layer details
for ( var idx = 1; idx <= numberOfLayers; idx++ ) {
	var layerDetails = {
		idx: idx
		,ID: getLayerID(idx)
		,name: getLayerName(idx)
		,blendMode: getBlendMode(idx)
		,opacity: getOpacity(idx)
		,fillOpacity: getFillOpacity(idx)
		,selected: false
		,empty: isEmptyLayer(idx)
		,kind: getLayerKind(idx)
		,type: getLayerType(idx)
		,whiteMask: hasWhiteLayerMask(idx)
		,blackMask: hasBlackLayerMask(idx)
		,hasMask: hasLayerMask(idx)
		,maskEnabled: isLayerMaskEnabled(idx)
		,feather: hasFeather(idx)
		,iconColor: getIconColor(idx)
		,visible: isVisible(idx)
		,clipped: isClipped(idx)
	}
	layers.push(layerDetails);

[jazz-y]

In this case, the list of layers also includes closing groups that are not visible on the palette (layerSectionEnd with a backslash in the name). They have their own index & ID. It is unlikely that you could make a mistake and try to load the selection using them, but it would be logical to immediately exclude them from the layers array like that:

if (t2s(desc.getEnumerationValue(s2t('layerSection')))=="layerSectionEnd")  continue;

[danielsian.com]

This is a good idea, but I'm using the layerSectionEnd to find each end of each group in the layer stack. I need to know the index of this "layer" in order to find the first layerSectionContent of each layerSet.
The "mistake" the code is making is with just a few layers. Everything else was working perfectly.
Thanks

[JJMack]

With Action Manager code you most likely can not loads a layer mask as a selection without fist targeting the layer mask you want to load as a selection.  Targeting the layer mask is done in the layers palette and targeting a layer mask will make the layer its on the active layer. The way Photoshop works you target what you want to use or work on first.  A side effect of targeting a layer mask  is Photoshop make its layer the active layer.  If you use Alpha channels instead of Layer mask Channels the active layer will not change. Alpha channel are not attached to layer they  are targeted in the Channels Palette  not the layers palette.  However Photoshop only supports 53 Alpha channels where you can have up to 8,000 layer and 8,000 Layer mask channels

[danielsian.com]

Actually this is the big advantage of using AM, where you can do stuff without activating the layer or layer mask. I'm not sure if this is possible in DOM.
DmitryEgorov seems to have problems with this code in his machine, but in my Photoshop it works like a charm:

function getSelectionFromLayerMask(ID) { 
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putProperty( stringIDToTypeID( "channel" ), stringIDToTypeID( "selection" ) );
	desc.putReference( stringIDToTypeID( "null" ), ref );
	var ref1 = new ActionReference();
	ref1.putEnumerated( stringIDToTypeID( "channel" ), stringIDToTypeID( "channel" ), stringIDToTypeID( "mask" ) );
	if ( ID != undefined ) ref1.putIdentifier(stringIDToTypeID( "layer" ), ID);
	desc.putReference( stringIDToTypeID( "to" ), ref1 );
	executeAction( stringIDToTypeID( "set" ), desc, DialogModes.NO );
}

[jazz-y]

Windows or MacOS?

[danielsian.com]

Windows 10 in both machines

[JJMack]

I have to admit the Action Manage Code is not easy to read or undetstand and to know exactly what it does. In the code you posted  I see Layer and ID to target the layer so its layer mask will be accessable.  There is no Layer mask in the channels palette unless a layer with an layer mask is the Active Layer. Then its Laye Mask is put in the Channels palette and is accessable.  So I believe the code does activate the layer first.  Using the function will cause Photoshop the change the layer with the passed ID to be the active layer the targeted layer. 

function getSelectionFromLayerMask(ID) { 
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putProperty( stringIDToTypeID( "channel" ), stringIDToTypeID( "selection" ) );
	desc.putReference( stringIDToTypeID( "null" ), ref );
	var ref1 = new ActionReference();
	ref1.putEnumerated( stringIDToTypeID( "channel" ), stringIDToTypeID( "channel" ), stringIDToTypeID( "mask" ) );
	if ( ID != undefined ) ref1.putIdentifier(stringIDToTypeID( "layer" ), ID);
	desc.putReference( stringIDToTypeID( "to" ), ref1 );
	executeAction( stringIDToTypeID( "set" ), desc, DialogModes.NO );
}

[danielsian.com]

"So I believe the code does activate the layer first".
If it does, it's only internally in some way.

But I like your train of thoughts.