The PSD format and layer groups

You are not only one who tried to solve this problem. Look for already made solution first ;-)

There could be  bit of confusion with the terminolgy here, grouped layers are not layersets or smart objects.

here is a function that will return if the layer index is a Layerset start/finish or is just a normal layer.

// returns one of: 
// 'layerSectionStart'     Start of a layer set 
// 'layerSectionEnd'       End of a layer set 
// 'layerSectionConent'  A content layer 
function getLayerLayerSectionByIndex( index ) {   
   var ref = new ActionReference(); 
   ref.putProperty( charIDToTypeID('Prpr') , stringIDToTypeID('layerSection'));
   ref.putIndex(charIDToTypeID('Lyr '), index);
   try{
   return typeIDToStringID(executeActionGet(ref).getEnumerationValue(stringIDToTypeID('layerSection')));
   }catch(e){$.writeln("error in index " + index); return;}
};

Here is another script you can run against a psd from ExtendScript Toolkit.

This will walk the PSD giving basic info including if Smart Object.

#target photoshop
if(documents.length) LayerInfo();
function LayerInfo(){ 
   var ref = new ActionReference(); 
   ref.putEnumerated( charIDToTypeID('Dcmn'), charIDToTypeID('Ordn'), charIDToTypeID('Trgt') ); 
   var count = executeActionGet(ref).getInteger(charIDToTypeID('NmbL')) +1; 
   var Layerinfo=[];
try{
    activeDocument.backgroundLayer;
var i = 0; }catch(e){ var i = 1; };
   for(i;i<count;i++){ 
        ref = new ActionReference(); 
        ref.putIndex( charIDToTypeID( 'Lyr ' ), i );
        var desc = executeActionGet(ref);
        var layerName = desc.getString(charIDToTypeID( 'Nm  ' ));
        var Id = desc.getInteger(stringIDToTypeID( 'layerID' ));
        var group = desc.getBoolean(stringIDToTypeID('group'));
        var kind = getLayerKindByIndex( i);
     if(layerName.match(/^<\/Layer group/) ) layerName = "End_Layerset";
        var layerType = typeIDToStringID(desc.getEnumerationValue( stringIDToTypeID( 'layerSection' )));
        var isLayerSet =( layerType == 'layerSectionContent') ? false:true;
        if(isLayerSet) layerName = ">--->" + layerName;
            Layerinfo.push([[layerName],[isLayerSet],[kind],,[Id],[group]]);
   }; 
Layerinfo.reverse();
for(var z in Layerinfo){
    $.writeln(Layerinfo.toString());
    }
};
function getLayerKindByIndex( index ) {
    var ref, desc, adjustmentDesc, layerSectionType;
   ref = new ActionReference();
   ref.putIndex(charIDToTypeID( "Lyr " ), index );
   desc =  executeActionGet(ref);
   if( desc.hasKey( stringIDToTypeID( 'textKey' ) ) ) return LayerKind.TEXT;
   if( desc.hasKey( stringIDToTypeID( 'smartObject' ) ) ) return LayerKind.SMARTOBJECT;// includes LayerKind.VIDEO
   if( desc.hasKey( stringIDToTypeID( 'layer3D' ) ) ) return LayerKind.LAYER3D;
    var vMask = desc.getBoolean(stringIDToTypeID('hasVectorMask' )); 
     if( desc.hasKey( stringIDToTypeID( 'adjustment' ) )  && vMask === true) return "LayerKind.SHAPE";
   if( desc.hasKey( stringIDToTypeID( 'adjustment' ) ) ){
      switch(typeIDToStringID(desc.getList (stringIDToTypeID('adjustment')).getClass (0))){
      case 'photoFilter' : return LayerKind.PHOTOFILTER;
      case 'solidColorLayer' : return LayerKind.SOLIDFILL;
      case 'gradientMapClass' : return LayerKind.GRADIENTMAP;
      case 'gradientMapLayer' : return LayerKind.GRADIENTFILL;
      case 'hueSaturation' : return LayerKind.HUESATURATION;
      case 'colorLookup' : return undefined; //this does not exist and errors with getting layer kind
      case 'colorBalance' : return LayerKind.COLORBALANCE;
      case 'patternLayer' : return LayerKind.PATTERNFILL;
      case 'invert' : return LayerKind.INVERSION;
      case 'posterization' : return LayerKind.POSTERIZE;
      case 'thresholdClassEvent' : return LayerKind.THRESHOLD;
      case 'blackAndWhite' : return LayerKind.BLACKANDWHITE;
      case 'selectiveColor' : return LayerKind.SELECTIVECOLOR;
      case 'vibrance' : return LayerKind.VIBRANCE;
      case 'brightnessEvent' : return LayerKind.BRIGHTNESSCONTRAST;
      case  'channelMixer' : return LayerKind.CHANNELMIXER;
      case 'curves' : return LayerKind.CURVES;
      case 'exposure' : return LayerKind.EXPOSURE;
      // if not one of the above adjustments return - adjustment layer type
      default : return typeIDToStringID(desc.getList (stringIDToTypeID('adjustment')).getClass (0));
      }
   }
var layerType = typeIDToStringID(desc.getEnumerationValue( stringIDToTypeID( 'layerSection' )));
return ( layerType == 'layerSectionContent') ? LayerKind.NORMAL:undefined;
}

Hope it gives a bit of insight.

P.S. I don't have CC so things may be slightly different now.