Change "Quick Mask Options" from "Selected" to "Masked Areas" via ExtendScript

[danielsian.com]

How can I change "Quick Mask Options" from "Selected Areas" to "Masked Areas" via JS ExtendScript?

function setQuickMask(set) {
	set ? set = "set" : set = "clearEvent";
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putProperty( stringIDToTypeID('property'), stringIDToTypeID('quickMask') );
	ref.putEnumerated( stringIDToTypeID('document'), stringIDToTypeID('ordinal'), stringIDToTypeID('targetEnum') );
	desc.putReference( stringIDToTypeID('null'), ref );
	//
	// == >>>> CODE TO SET "Masked Areas" ??
	//
	executeAction( stringIDToTypeID(set), desc, DialogModes.NO );
}

[Stephen_A_Marsh · Accepted solution]

Mask (scriptinglistener code):

// =======================================================
var idset = stringIDToTypeID( "set" );
    var desc363 = new ActionDescriptor();
    var idnull = stringIDToTypeID( "null" );
        var ref87 = new ActionReference();
        var idchannel = stringIDToTypeID( "channel" );
        var idordinal = stringIDToTypeID( "ordinal" );
        var idtargetEnum = stringIDToTypeID( "targetEnum" );
        ref87.putEnumerated( idchannel, idordinal, idtargetEnum );
    desc363.putReference( idnull, ref87 );
    var idto = stringIDToTypeID( "to" );
        var desc364 = new ActionDescriptor();
        var idcolorIndicates = stringIDToTypeID( "colorIndicates" );
        var idmaskIndicator = stringIDToTypeID( "maskIndicator" );
        var idmaskedAreas = stringIDToTypeID( "maskedAreas" );
        desc364.putEnumerated( idcolorIndicates, idmaskIndicator, idmaskedAreas );
    var idchannel = stringIDToTypeID( "channel" );
    desc363.putObject( idto, idchannel, desc364 );
executeAction( idset, desc363, DialogModes.NO );

Selected (scriptinglistener code):

// =======================================================
var idset = stringIDToTypeID( "set" );
    var desc367 = new ActionDescriptor();
    var idnull = stringIDToTypeID( "null" );
        var ref89 = new ActionReference();
        var idchannel = stringIDToTypeID( "channel" );
        var idordinal = stringIDToTypeID( "ordinal" );
        var idtargetEnum = stringIDToTypeID( "targetEnum" );
        ref89.putEnumerated( idchannel, idordinal, idtargetEnum );
    desc367.putReference( idnull, ref89 );
    var idto = stringIDToTypeID( "to" );
        var desc368 = new ActionDescriptor();
        var idcolorIndicates = stringIDToTypeID( "colorIndicates" );
        var idmaskIndicator = stringIDToTypeID( "maskIndicator" );
        var idselectedAreas = stringIDToTypeID( "selectedAreas" );
        desc368.putEnumerated( idcolorIndicates, idmaskIndicator, idselectedAreas );
    var idchannel = stringIDToTypeID( "channel" );
    desc367.putObject( idto, idchannel, desc368 );
executeAction( idset, desc367, DialogModes.NO );

Here it is in a function with a parameter/variable (scriptinglistener code passed through CleanSL + a manual variable and parameter added):

setQM("selectedAreas"); // or "maskedAreas"

function setQM(qmOption) {
	var s2t = function (s) {
		return app.stringIDToTypeID(s);
	};
	var descriptor = new ActionDescriptor();
	var descriptor2 = new ActionDescriptor();
	var reference = new ActionReference();
	reference.putEnumerated( s2t( "channel" ), s2t( "ordinal" ), s2t( "targetEnum" ));
    descriptor.putReference(s2t("null"), reference);
    /* "selectedAreas" or "maskedAreas" */
	descriptor2.putEnumerated( s2t( "colorIndicates" ), s2t( "maskIndicator" ), s2t( qmOption ));
	descriptor.putObject( s2t( "to" ), s2t( "channel" ), descriptor2 );
	executeAction( s2t( "set" ), descriptor, DialogModes.NO );
}

Another option would be to simply select the Quick Mask channel and invert.

selectChannelByName("Quick Mask");
var idinvert = stringIDToTypeID( "invert" );
executeAction( idinvert, undefined, DialogModes.NO );
//app.activeDocument.activeChannels = app.activeDocument.componentChannels;

function selectChannelByName(channelName) {
    var channelByName = new Array(app.activeDocument.channels[channelName]);
    app.activeDocument.activeChannels = channelByName;
}

[danielsian.com]

@Stephen_A_Marsh  thanks for you quick response, I really appreciated that!

In my original problem, I was chasing a global change in Photoshop as a solution, and because of that, the ScriptingListener wasn't recording any commands as I was trying this change by double-clicking the toolbar button. But when you came up with your solution, I realized that ScriptingListener was able to record the command only when Quick Mask Mode was already in Edit mode. This is a temporary Quick Mask change, but it worked just fine for my problem.
So, this is my final function:

function setQuickMask( set, quickMaskOption ) {
	// set: True to turn Quick Mask on, otherwise False
	// quickMaskOption: 'maskedAreas' or 'selectedAreas'
	set ? quickMask = "set" : quickMask = "clearEvent";
	//=== turn Quick Mask Mode on
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putProperty( stringIDToTypeID('property'), stringIDToTypeID('quickMask') );
	ref.putEnumerated( stringIDToTypeID('document'), stringIDToTypeID('ordinal'), stringIDToTypeID('targetEnum') );
	desc.putReference( stringIDToTypeID('null'), ref );
	executeAction( stringIDToTypeID(quickMask), desc, DialogModes.NO );
	//=== set Quick Mask Options
	var desc = new ActionDescriptor();
	var ref = new ActionReference();
	ref.putEnumerated( stringIDToTypeID('channel'), stringIDToTypeID('ordinal'), stringIDToTypeID('targetEnum') );
	desc.putReference( stringIDToTypeID('null'), ref );
	var desc2 = new ActionDescriptor();
	desc2.putEnumerated( stringIDToTypeID('colorIndicates'), stringIDToTypeID('maskIndicator'), stringIDToTypeID(quickMaskOption) );
	desc.putObject( stringIDToTypeID('to'), stringIDToTypeID('channel'), desc2 );
	if ( !set ) return;
	executeAction( stringIDToTypeID("set"), desc, DialogModes.NO );
}

[Stephen_A_Marsh]

Slightly different code to enter/exit QM mode from an active selection:

quickMask("set"); // or "clearEvent"

function quickMask(qmOpt) {
	var s2t = function (s) {
		return app.stringIDToTypeID(s);
	};
	var descriptor = new ActionDescriptor();
	var reference = new ActionReference();
	reference.putProperty( s2t( "property" ), s2t( "quickMask" ));
	reference.putEnumerated( s2t( "document" ), s2t( "ordinal" ), s2t( "targetEnum" ));
    descriptor.putReference(s2t("null"), reference);
    // "set" to enter | "clearEvent" to exit
	executeAction( s2t( qmOpt ), descriptor, DialogModes.NO );
}

[Kukurykus]

activeDocument.quickMaskMode ^= 1