how to programmatically check if the layer or the layer mask is selected?

@Flowgun – Try this (raster mask):

/* https://community.adobe.com/t5/photoshop-ecosystem-discussions/detect-if-mask-or-layer-is-selected-with-js-script/m-p/11153109 */

// Return true for layer mask selected, or false for composite channel

layerMaskIsSelected();

function layerMaskIsSelected() {
	s2t = stringIDToTypeID;

	(r = new ActionReference()).putProperty(s2t('property'), p = s2t('hasUserMask'));
	r.putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
	if (executeActionGet(r).getBoolean(p)) {

	    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('name'));
	    (r = new ActionReference()).putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
	    layerName = executeActionGet(r).getString(p);

	    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('channelName'));
	    r.putEnumerated(s2t("channel"), s2t("ordinal"), s2t("targetEnum"));
	    channelName = executeActionGet(r).getString(p);

	    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('alphaChannelOptions'));
	    r.putEnumerated(s2t("channel"), s2t("ordinal"), s2t("targetEnum"));
	    alphaChannel = executeActionGet(r).hasKey(p);

		// return (channelName.indexOf(layerName) == 0 && !alphaChannel);
        alert(channelName.indexOf(layerName) == 0 && !alphaChannel);
	}
}

Edit: Perhaps more usable:

layerMaskIsSelected();

function layerMaskIsSelected() {
    s2t = stringIDToTypeID;

    (r = new ActionReference()).putProperty(s2t('property'), p = s2t('hasUserMask'));
    r.putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
    if (executeActionGet(r).getBoolean(p)) {

        (r = new ActionReference()).putProperty(s2t('property'), p = s2t('name'));
        (r = new ActionReference()).putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
        layerName = executeActionGet(r).getString(p);

        (r = new ActionReference()).putProperty(s2t('property'), p = s2t('channelName'));
        r.putEnumerated(s2t("channel"), s2t("ordinal"), s2t("targetEnum"));
        channelName = executeActionGet(r).getString(p);

        (r = new ActionReference()).putProperty(s2t('property'), p = s2t('alphaChannelOptions'));
        r.putEnumerated(s2t("channel"), s2t("ordinal"), s2t("targetEnum"));
        alphaChannel = executeActionGet(r).hasKey(p);

        if (channelName.indexOf(layerName) === 0 && !alphaChannel) {
            alert("Layer mask is selected.");
        } else {
            alert("Layer mask is NOT selected.");
        }
    }
}