Detect and Select Layer Mask

[Nick Combs]

I recently needed to do the following with javascript

1.  Detect if a layer has a mask applied.
2.  If the layer has a mask, select the pixels of that mask (as if using Ctrl/Cmd + LMB on the mask icon in the Layers panel).

The following code was successful, so I'm sharing it in case it becomes useful to anyone else. See the reply below.

[Nick Combs · Accepted solution]

function layerMask( lyr ) {

  if( lyr && lyr.typename && lyr.typename == 'ArtLayer' ) 
    activeDocument.activeLayer = lyr
  else 
    lyr = activeDocument.activeLayer

  var r = {}, d = {}, st = stringIDToTypeID, ch = charIDToTypeID

  r['activeLayer'] = new ActionReference()
  r['activeLayer'].putEnumerated( ch('Lyr '), ch('Ordn'), ch('Trgt') )
  d['activeLayer'] = executeActionGet( r['activeLayer'] )

  if( d['activeLayer'].getBoolean( st('hasUserMask') ) ) {

    d['maskSelection'] = new ActionDescriptor()

    r['maskSelection'] = new ActionReference()
    r['maskSelection'].putProperty( ch('Chnl'), ch('fsel') )
    d['maskSelection'].putReference( ch('null'), r['maskSelection'] )

    r['activeChannel'] = new ActionReference()
    r['activeChannel'].putEnumerated( ch('Chnl'), ch('Ordn'), ch('Msk ') )
    d['maskSelection'].putReference( ch('T   '), r['activeChannel'] )

    executeAction( ch('setd'), d['maskSelection'], DialogModes.NO )

    return activeDocument.selection
  }
  return false
}

[Stephen Marsh]

@Nick Combs - thank you for sharing!