SmartObject in Photoshop

@code_seeeeeker  – The following script is for a single active layer (I started work on it before your full requirements were known). I'll look into adapting it for all layers.

/*
Check if the vector smart object in Photoshop is in RGB or CMYK mode.jsx
Stephen Marsh
v1.0 - 11th March 2025
https://community.adobe.com/t5/photoshop-ecosystem-discussions/smartobject-in-photoshop/td-p/15195068
Info: Alerts the color mode of the active vector smart object layer
*/

#target photoshop

// Check if there is an active document
if (app.documents.length > 0) {

    try {

        // Single history stage undo
        app.activeDocument.suspendHistory("Undo Script", "main()");

        function main() {

            // Check that the SO is an Adobe Illustrator file
            var mySO = getSmartObjectName();
            if (mySO.found && /\.ai$/.test(mySO.fileRef)) {

                // Check that the active layer is an embedded SO
                var ref = new ActionReference();
                ref.putEnumerated(charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
                ref.putProperty(charIDToTypeID("Prpr"), stringIDToTypeID("smartObject"));
                ref.putEnumerated(charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
                var so = executeActionGet(ref).getObjectValue(stringIDToTypeID("smartObject"));
                if (so.getBoolean(stringIDToTypeID("linked")) === false) {

                    // Convert the vector smart object to a linked PDF file, as a means to read the color mode
                    placedLayerConvertToLinked("~/Desktop/_temp-VectorToDelete.pdf");

                    // Undo the previous action to keep the vector smart object as it was
                    executeAction(charIDToTypeID("undo"), undefined, DialogModes.NO);

                    // Set the path to the temporary vector file
                    var tempVectorFile = File("~/Desktop/_temp-VectorToDelete.pdf");

                    // Get the active document color mode
                    var doc = app.activeDocument;
                    var colorMode = doc.mode;
                    var docColorMode = "Document color mode = ";
                    switch (colorMode) {
                        case DocumentMode.GRAYSCALE:
                            docColorMode += "Grayscale";
                            break;
                        case DocumentMode.RGB:
                            docColorMode += "RGB";
                            break;
                        case DocumentMode.CMYK:
                            docColorMode += "CMYK";
                            break;
                        case DocumentMode.LAB:
                            docColorMode += "Lab";
                            break;
                    }

                    // Read the binary data of the temporary vector file and alert if the color mode is RGB or CMYK
                    if (tempVectorFile) {
                        var binaryData = readBinaryFile(tempVectorFile);
                        if (binaryData !== null) {
                            var searchString = "_ColorModel: 1"; // %AI9_ColorModel: 2 for CMYK
                            var found = searchInBinaryData(binaryData, searchString);
                            if (found) {
                                alert("Color Info:" + "\r" + docColorMode + "\r" + "Vector smart object color mode = RGB");
                            } else {
                                alert("Color Info: " + "\r" + docColorMode + "\r" + "Vector smart object color mode = CMYK");
                            }
                        }
                    } else {
                        alert("The file '_temp-VectorToDelete.pdf' wasn't found on the desktop!");
                    }

                    // Delete the temporary vector file
                    tempVectorFile.remove();

                } else {
                    alert("The layer isn't an Embedded Smart Object!");
                }

            } else {
                alert("Select an Adobe Illustrator Smart Object layer!");
            }
        }

    } catch (error) {
        //alert(error + ', Line: ' + error.line);
    }

} else {
    alert("A file must be open to run this script!");
}


///// Functions /////

function placedLayerConvertToLinked(theTempFile) {
    try {
        var s2t = function (s) {
            return app.stringIDToTypeID(s);
        };
        var descriptor = new ActionDescriptor();
        var reference = new ActionReference();
        reference.putEnumerated(s2t("layer"), s2t("ordinal"), s2t("targetEnum"));
        descriptor.putReference(s2t("null"), reference);
        descriptor.putPath(s2t("using"), new File(theTempFile));
        executeAction(s2t("placedLayerConvertToLinked"), descriptor, DialogModes.NO);
    } catch (error) {
        //alert(error + ', Line: ' + error.line);
    }
}

function readBinaryFile(tempVectorFile) {
    try {
        if (!tempVectorFile.exists) {
            alert("File not found!");
            return null;
        }
        tempVectorFile.open("r");
        tempVectorFile.encoding = "BINARY";
        var binaryData = tempVectorFile.read();
        tempVectorFile.close();
        return binaryData;
    } catch (error) {
        alert(error + ', Line: ' + error.line);
    }
}

function searchInBinaryData(binaryData, searchString) {
    try {
        var regex = new RegExp(searchString, "g");
        return regex.test(binaryData);
    } catch (error) {
        alert(error + ', Line: ' + error.line);
    }
}

function getSmartObjectName() {
    /* https://stackoverflow.com/questions/63010107/get-a-smart-objects-layers-files-directory-source-in-jsx */
    try {
        var smartObject = {
            found: false,
            fileRef: '',
            linked: false,
        };
        var ref, so;
        ref = new ActionReference();
        ref.putProperty(charIDToTypeID("Prpr"), stringIDToTypeID("smartObject"));
        ref.putEnumerated(charIDToTypeID("Lyr "), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
        so = executeActionGet(ref).getObjectValue(stringIDToTypeID("smartObject"));
        smartObject.found = true;
        smartObject.linked = so.getBoolean(stringIDToTypeID("linked"));
        smartObject.fileRef = so.getString(stringIDToTypeID("fileReference"));
        return smartObject;
    }
    catch (error) {
        //alert(error);
        return smartObject;
    }
}