PS Extendscript script to detect mask on layer and if there's a mask then select a different layer?

[xojeb]

Scripting amateur and I've tried many things, but they have not worked yet. I want the script to randomly select and make a layer or layer group active, detect if there is a layer mask and if so then select another layer until it gets to one which does not have a mask on it, then make that the active layer or layer group.

// Function to check if a layer has a mask
function hasMask(layer) {
  return layer.hasMask;
}

// Function to recursively check all child layers for a mask
function checkLayersForMask(layers) {
  for (var i = 0; i < layers.length; i++) {
    var layer = layers[i];

    // If the current layer is a group or set, recursively check its child layers for a mask
    if (layer.typename == "LayerSet" || layer.typename == "ArtLayer") {
      var childLayers = layer.layers;
      if (childLayers.length > 0) {
        var activeLayer = checkLayersForMask(childLayers);
        if (activeLayer) {
          return activeLayer;
        }
      }
    }

    // If the current layer does not have a mask, return it
    if (!hasMask(layer)) {
      return layer;
    }
  }
}

// Get the active document and its layers
var doc = app.activeDocument;
var layers = doc.layers;

// Generate a random number between 0 and the number of layers
var randomLayerIndex = Math.floor(Math.random() * layers.length);

// Make the layer at the random index the active layer
app.activeDocument.activeLayer = layers[randomLayerIndex];

// Check all the child layers for a mask
var activeLayer = checkLayersForMask(layers);

// Make the first layer without a mask active
if (activeLayer) {
  app.activeDocument.activeLayer = activeLayer;
}

[c.pfaffenbichler]

Try this function for checking for Layer Masks. 

////// has layer mask //////
function hasLayerMask () {  
    var m_Dsc01, m_Ref01;
    m_Ref01 = new ActionReference();
    m_Ref01.putEnumerated(stringIDToTypeID("layer"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
    m_Dsc01 = executeActionGet(m_Ref01);
    return m_Dsc01.hasKey(charIDToTypeID("Usrs"));
    };

[c.pfaffenbichler]

In images with many Layers using DOM-code to evaluate them is usually much slower than AM-code, so you may want to look into that. 

[jazz-y · Accepted solution]

If I understand correctly - there are many layers with masks in the document and you need to find one without a mask and make it active? Why is a random number generator used in this context?

s2t = stringIDToTypeID,
    layers = [];
(r = new ActionReference()).putProperty(s2t('property'), p = s2t('numberOfLayers'));
r.putEnumerated(s2t('document'), s2t('ordinal'), s2t('targetEnum'));
var len = executeActionGet(r).getInteger(p);
for (var i = 1; i <= len; i++) {
    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('layerKind'));
    r.putIndex(s2t('layer'), i);
    if (executeActionGet(r).getInteger(p) == 13) continue;
    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('hasUserMask'));
    r.putIndex(s2t('layer'), i);
    if (!executeActionGet(r).getBoolean(p)) {
        (r = new ActionReference()).putProperty(s2t('property'), p = s2t('layerID'));
        r.putIndex(s2t('layer'), i);
        layers.push(executeActionGet(r).getInteger(p));
    }
}
(r = new ActionReference()).putIdentifier(s2t("layer"), layers[Math.round(Math.random() * (layers.length - 1))]);
(d = new ActionDescriptor()).putReference(s2t("target"), r);
executeAction(s2t("select"), d, DialogModes.NO);

[xojeb]

Thank you, yes, I need it to find a random layer without a mask ideally—not in any particular order. 

[xojeb]

Thank you very much, this works well. You saved me a headache and time. I appreciate it!