Using Extendscript, how can I get the bounds of a layer mask's selection?

[reubenlara]

Hello! I'm trying to retrieve the bounds of the selection of a layer's mask. User @c.pfaffenbichler already helped me with this amazing code and it works perfectly (thank you!):

// 2024, use it at your own risk;
if (app.documents.length > 0) {
// set to pixels;
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.PIXELS;
var myDocument = app.activeDocument;
var theLayers = collectSelectedLayersBounds ();
// process layers;
var theCount = 0; 
for (var m = theLayers.length-1; m >= 0; m--) {
var thisOne = theLayers[m];
alert (thisOne[0]+"\nbounds\n"+thisOne[2].join("\n")+"\nbounds with mask\n"+thisOne[3].join("\n"));
theCount++
};
// reset;
app.preferences.rulerUnits = originalRulerUnits;
};
////////////////////////////////////
////// collect bounds of selected layers //////
function collectSelectedLayersBounds () {
// set to pixels;
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.PIXELS;
// get selected layers;
var selectedLayers = new Array;
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Dcmn"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var desc = executeActionGet(ref);
if (desc.getBoolean(stringIDToTypeID("hasBackgroundLayer")) == true) {var theAdd =0}
else {var theAdd = 1};
if( desc.hasKey( stringIDToTypeID( 'targetLayers' ) ) ){
desc = desc.getList( stringIDToTypeID( 'targetLayers' ));
var c = desc.count;
var selectedLayers = new Array();
// run through selected layers;
for(var i=0;i<c;i++){
var theIndex = desc.getReference( i ).getIndex()+theAdd;
// get id for solid color layers;
try {
var ref = new ActionReference();
ref.putIndex( charIDToTypeID("Lyr "), theIndex ); 
var layerDesc = executeActionGet(ref);
var theName = layerDesc.getString(stringIDToTypeID('name'));
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
var theBoundsNoMask = layerDesc.getObjectValue(stringIDToTypeID("boundsNoMask"));
var theseBoundsNoMask = [theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("left")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("top")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("right")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("bottom"))];
var theBounds = layerDesc.getObjectValue(stringIDToTypeID("bounds"));
var theseBounds = [theBounds.getUnitDoubleValue(stringIDToTypeID("left")), theBounds.getUnitDoubleValue(stringIDToTypeID("top")), theBounds.getUnitDoubleValue(stringIDToTypeID("right")), theBounds.getUnitDoubleValue(stringIDToTypeID("bottom"))];
selectedLayers.push([theName, theIdentifier, theseBoundsNoMask, theseBounds]);
} catch (e) {};
};
// if only one:
}else{
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var layerDesc = executeActionGet(ref);
try {
var theName = layerDesc.getString(stringIDToTypeID('name'));
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
var theBounds = layerDesc.getObjectValue(stringIDToTypeID("bounds"));
var theseBounds = [theBounds.getUnitDoubleValue(stringIDToTypeID("left")), theBounds.getUnitDoubleValue(stringIDToTypeID("top")), theBounds.getUnitDoubleValue(stringIDToTypeID("right")), theBounds.getUnitDoubleValue(stringIDToTypeID("bottom"))];
selectedLayers = [[theName, theIdentifier, theseBounds]]
} catch (e) {};
};
// reset;
app.preferences.rulerUnits = originalRulerUnits;
return selectedLayers;
};

My only problem is that I need it to act on a layer that already exists in an array I've collected elsewhere in the script. So for example, assume I've already successfully retrieved my target layers in the following array:

var layers = getSelectedLayers();

I'd like to be able to execute the above code in a statement like this, where the function would just send back the info for the layer passed in an argument, like this:

var theLayers = collectSelectedLayersBounds(layers[3]);

What would i need to adjust on the "collectSelectedLayersBounds" function in order to implement this into my existing code? 

[reubenlara]

BTW here is my "getSelectedLayers()" function, but in practice it would also need to work with the standard "doc.layers" object:

function getSelectedLayers() {
    var A = [];
    var desc11 = new ActionDescriptor();
    var ref9 = new ActionReference();
    ref9.putClass( stringIDToTypeID('layerSection') );
    desc11.putReference( charIDToTypeID('null'), ref9 );
    var ref10 = new ActionReference();
    ref10.putEnumerated( charIDToTypeID('Lyr '), charIDToTypeID('Ordn'), charIDToTypeID('Trgt') );
    desc11.putReference( charIDToTypeID('From'), ref10 );
    executeAction( charIDToTypeID('Mk  '), desc11, DialogModes.NO );
    var gL = activeDocument.activeLayer.layers;


    for(var i = 0; i < gL.length; i++) { 
        A.push(gL[i]); 
    } 
    executeAction( charIDToTypeID('undo'), undefined, DialogModes.NO );
    return A;
};

I've tried my hand at trying to understand Action Descriptors but I'm convinced that it is like a lost ancient native hieroglyphic language that ony superhero archeologist coders can speak. Thank you for your service.

[c.pfaffenbichler]

var aaa = getSelectedLayers();
for (var m = 0; m < aaa.length; m++) {
    activeDocument.activeLayer = aaa[m];
    var theBounds = getBoundsOfActiveLayer();
    alert (theBounds.join("\n\n"));
};
////// 
function getBoundsOfActiveLayer () {
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") ); 
var layerDesc = executeActionGet(ref);
var theName = layerDesc.getString(stringIDToTypeID('name'));
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
var theBoundsNoMask = layerDesc.getObjectValue(stringIDToTypeID("boundsNoMask"));
var theseBoundsNoMask = [theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("left")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("top")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("right")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("bottom"))];
var theBounds = layerDesc.getObjectValue(stringIDToTypeID("bounds"));
var theseBounds = [theBounds.getUnitDoubleValue(stringIDToTypeID("left")), theBounds.getUnitDoubleValue(stringIDToTypeID("top")), theBounds.getUnitDoubleValue(stringIDToTypeID("right")), theBounds.getUnitDoubleValue(stringIDToTypeID("bottom"))];
return [theName, theIdentifier, theseBoundsNoMask, theseBounds]
};
function getSelectedLayers() {
    var A = [];
    var desc11 = new ActionDescriptor();
    var ref9 = new ActionReference();
    ref9.putClass( stringIDToTypeID('layerSection') );
    desc11.putReference( charIDToTypeID('null'), ref9 );
    var ref10 = new ActionReference();
    ref10.putEnumerated( charIDToTypeID('Lyr '), charIDToTypeID('Ordn'), charIDToTypeID('Trgt') );
    desc11.putReference( charIDToTypeID('From'), ref10 );
    executeAction( charIDToTypeID('Mk  '), desc11, DialogModes.NO );
    var gL = activeDocument.activeLayer.layers;


    for(var i = 0; i < gL.length; i++) { 
        A.push(gL[i]); 
    } 
    executeAction( charIDToTypeID('undo'), undefined, DialogModes.NO );
    return A;
};

It would seem more efficient, though, not to collect the Layers as DOM-objects at all but create an array with the AM-relevant infromation straight away. 

[reubenlara]

This works perfectly thank you! I agree about your obeservation on effeciency. Likely if you saw my whole script you could make it 100% better. But this gets me out of the most immediate challenge. I will keep attempting to optimize the script as you have suggested. thank you so much for your time!

[reubenlara]

Hello again @c.pfaffenbichler ! After further testing, this script only works on layer groups with masks, but not on single layers with masks. Is there a way to adjust it so that it also retrieves the mask bounds on single layers? Currently the result send back the same image bounds to both variables. 

[c.pfaffenbichler]

It seems the function getSelectedLayers is the cause of the problem. 

And instead of changing that it might be better to replace it with a function that collects the bounds etc. right away and subsequently address the Layers via the AM identifiers instead of the DOM Layers. 

[c.pfaffenbichler]

I changed the function getSelectedLayers to include Groups and Layers via a mixture of AM and DOm code. 

var aaa = getSelectedLayers();
for (var m = 0; m < aaa.length; m++) {
    activeDocument.activeLayer = aaa[m];
    var theBounds = getBoundsOfActiveLayer();
    alert (theBounds.join("\n\n"));
};
////// bounds of active layer //////
function getBoundsOfActiveLayer () {
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") ); 
var layerDesc = executeActionGet(ref);
var theName = layerDesc.getString(stringIDToTypeID('name'));
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
var theBoundsNoMask = layerDesc.getObjectValue(stringIDToTypeID("boundsNoMask"));
var theseBoundsNoMask = [theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("left")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("top")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("right")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("bottom"))];
var theBounds = layerDesc.getObjectValue(stringIDToTypeID("bounds"));
var theseBounds = [theBounds.getUnitDoubleValue(stringIDToTypeID("left")), theBounds.getUnitDoubleValue(stringIDToTypeID("top")), theBounds.getUnitDoubleValue(stringIDToTypeID("right")), theBounds.getUnitDoubleValue(stringIDToTypeID("bottom"))];
return [theName, theIdentifier, theseBoundsNoMask, theseBounds]
};
////// collect bounds of selected layers //////
function getSelectedLayers () {
// set to pixels;
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.PIXELS;
// get selected layers;
var selectedLayers = new Array;
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Dcmn"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var desc = executeActionGet(ref);
if (desc.getBoolean(stringIDToTypeID("hasBackgroundLayer")) == true) {var theAdd =0}
else {var theAdd = 1};
if( desc.hasKey( stringIDToTypeID( 'targetLayers' ) ) ){
desc = desc.getList( stringIDToTypeID( 'targetLayers' ));
var c = desc.count;
var selectedLayers = new Array();
// run through selected layers;
for(var i=0;i<c;i++){
var theIndex = desc.getReference( i ).getIndex()+theAdd;
// get id for solid color layers;
try {
var ref = new ActionReference();
ref.putIndex( charIDToTypeID("Lyr "), theIndex ); 
var layerDesc = executeActionGet(ref);
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
selectedLayers.push(theIdentifier);
} catch (e) {};
};
// if only one:
}else{
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var layerDesc = executeActionGet(ref);
try {
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
selectedLayers = [theIdentifier]
} catch (e) {};
};
//
var theResult = new Array;
for (var m = 0; m < selectedLayers.length; m++) {
selectLayerByID(selectedLayers[m], false);
theResult.push (activeDocument.activeLayer)
};
// reset;
app.preferences.rulerUnits = originalRulerUnits;
return theResult;
};
// based on code by mike hale, via paul riggott;
function selectLayerByID(id,add){ 
add = undefined ? add = false:add 
var ref = new ActionReference();
ref.putIdentifier(charIDToTypeID("Lyr "), id);
var desc = new ActionDescriptor();
desc.putReference(charIDToTypeID("null"), ref );
if(add) desc.putEnumerated( stringIDToTypeID( "selectionModifier" ), stringIDToTypeID( "selectionModifierType" ), stringIDToTypeID( "addToSelection" ) ); 
desc.putBoolean( charIDToTypeID( "MkVs" ), false ); 
try{
executeAction(charIDToTypeID("slct"), desc, DialogModes.NO );
}catch(e){
alert(e.message); 
}
};

[reubenlara]

Thank you! The getSelectedLayers() is much more effective. However, it's still only returning mask bounds on Layer Groups and not on single layers. Here's a quick screen recording of what's happening on my end:

https://www.loom.com/share/f1526d7c363b4d8dbac18a82defcffed?sid=8d7e96f7-9dba-4150-8e98-5afecad9e2a1

[c.pfaffenbichler]

Seems to be working here, please provide a file for testing. 

[reubenlara]

Here's my file: https://www.dropbox.com/scl/fi/xkan4jr9mft4rj47f774b/superheroes.psd?rlkey=e5diszac19gxjjytdpsakl4qo...

Did you try a single layer that is not inside a layer group?

[c.pfaffenbichler · Accepted solution]

Sorry, I misunderstood. 

This goes back to my previous question about the unmasked bounds and bounds (the interesection of the bounds and the mask) – the bounds of the mask itself are another matter and the Selection-work-around may have to be employed after all. 

// 2024, use it at your own risk;
var aaa = getSelectedLayers();
for (var m = 0; m < aaa.length; m++) {
activeDocument.activeLayer = aaa[m];
var theBounds = getBoundsOfActiveLayer()
if (hasLayerMask() == true) {
loadSelectionOfLayerMask();
var theSelection = activeDocument.selection.bounds;
activeDocument.selection.deselect();
alert (theBounds[0]+"\nbounds "+theBounds[3].join(", ")+"\nbounds of mask "+theSelection.join(", "));
} else {
alert (theBounds[0]+"\nbounds "+theBounds[3].join(", "));
};
};
////// bounds of active layer //////
function getBoundsOfActiveLayer () {
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") ); 
var layerDesc = executeActionGet(ref);
var theName = layerDesc.getString(stringIDToTypeID('name'));
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
var theBoundsNoMask = layerDesc.getObjectValue(stringIDToTypeID("boundsNoMask"));
var theseBoundsNoMask = [theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("left")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("top")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("right")), theBoundsNoMask.getUnitDoubleValue(stringIDToTypeID("bottom"))];
var theBounds = layerDesc.getObjectValue(stringIDToTypeID("bounds"));
var theseBounds = [theBounds.getUnitDoubleValue(stringIDToTypeID("left")), theBounds.getUnitDoubleValue(stringIDToTypeID("top")), theBounds.getUnitDoubleValue(stringIDToTypeID("right")), theBounds.getUnitDoubleValue(stringIDToTypeID("bottom"))];
return [theName, theIdentifier, theseBoundsNoMask, theseBounds]
};
////// collect bounds of selected layers //////
function getSelectedLayers () {
// set to pixels;
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.PIXELS;
// get selected layers;
var selectedLayers = new Array;
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Dcmn"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var desc = executeActionGet(ref);
if (desc.getBoolean(stringIDToTypeID("hasBackgroundLayer")) == true) {var theAdd =0}
else {var theAdd = 1};
if( desc.hasKey( stringIDToTypeID( 'targetLayers' ) ) ){
desc = desc.getList( stringIDToTypeID( 'targetLayers' ));
var c = desc.count;
var selectedLayers = new Array();
// run through selected layers;
for(var i=0;i<c;i++){
var theIndex = desc.getReference( i ).getIndex()+theAdd;
// get id for solid color layers;
try {
var ref = new ActionReference();
ref.putIndex( charIDToTypeID("Lyr "), theIndex ); 
var layerDesc = executeActionGet(ref);
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
selectedLayers.push(theIdentifier);
} catch (e) {};
};
// if only one:
}else{
var ref = new ActionReference();
ref.putEnumerated( charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var layerDesc = executeActionGet(ref);
try {
var theIdentifier = layerDesc.getInteger(stringIDToTypeID ("layerID"));
selectedLayers = [theIdentifier]
} catch (e) {};
};
//
var theResult = new Array;
for (var m = 0; m < selectedLayers.length; m++) {
selectLayerByID(selectedLayers[m], false);
theResult.push (activeDocument.activeLayer)
};
// reset;
app.preferences.rulerUnits = originalRulerUnits;
return theResult;
};
////// load layer mask //////
function loadSelectionOfLayerMask() {  
try {
var idchannel = stringIDToTypeID( "channel" );
var desc70 = new ActionDescriptor();
var ref9 = new ActionReference();
ref9.putProperty( idchannel, stringIDToTypeID( "selection" ) );
desc70.putReference( stringIDToTypeID( "null" ), ref9 );
var ref10 = new ActionReference();
ref10.putEnumerated( idchannel, idchannel, stringIDToTypeID( "mask" ) );
desc70.putReference( stringIDToTypeID( "to" ), ref10 );
executeAction( stringIDToTypeID( "set" ), desc70, DialogModes.NO );
} catch (_error) {alert (_error)}
};
// based on code by mike hale, via paul riggott;
function selectLayerByID(id,add){ 
add = undefined ? add = false:add 
var ref = new ActionReference();
ref.putIdentifier(charIDToTypeID("Lyr "), id);
var desc = new ActionDescriptor();
desc.putReference(charIDToTypeID("null"), ref );
if(add) desc.putEnumerated( stringIDToTypeID( "selectionModifier" ), stringIDToTypeID( "selectionModifierType" ), stringIDToTypeID( "addToSelection" ) ); 
desc.putBoolean( charIDToTypeID( "MkVs" ), false ); 
try{
executeAction(charIDToTypeID("slct"), desc, DialogModes.NO );
}catch(e){
alert(e.message); 
}
};
// from discussions with Mike Hale
function hasLayerMask (doc, layer) {
var ref = new ActionReference();
ref.putEnumerated(charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
var desc = executeActionGet(ref);
return desc.hasKey(charIDToTypeID("UsrM"));
};

edited

[reubenlara]

I am absolutely killing you. Now it doesn't work on layers with no mask, which i do need, and which the previous script accomplished. It says "The command Set is not currently available"

[c.pfaffenbichler]

I edited the code in the previous post, please try that. 

[reubenlara]

Excellent! It doesn't give the correct bounds for all the contents of a layer group with no mask (like the native .bounds command does), but I think I can test for that and do a native command in that case. Thank you so much for your extended help with this!

[c.pfaffenbichler]

In that case you could just change the content of the else-clause. 

What are the differences, though? 

[reubenlara]

When i apply your script to a normal layer group with no mask, the result is "bounds 0,0,1000,1000" (which is the whole document). But it should be the bounds of all the contents of the layer group, which is what the native .bounds returns?

[reubenlara]

Optimally, the result of any selection should be:

1. Layer group, no mask = bounds of everything inside the layer group
2. Single layer, no mask = bounds of the single layer
3.  Layer group, with mask = bounds of the mask selection
4. Single layer, with mask = bounds of the mask selection

[reubenlara]

But yes, I just adjusted the else statement and works perfectly as intended using a normal .bounds. Thank you so much!