Hello @c.pfaffenbichler
that exactly what I'm look for.
But your script seam not working with smart layer. When I run it I get this error

To run the script I need to rasterise the layer and remove the mask but I get this result :

Here you can find a sample PSD file :
But your script seam not working with smart layer. When I run it I get this error
That’s why I wrote »meaningfully adapted«; obviously that Script doesn’t work for your case, it just was an example of trying to determine a rectangular shape from a Layer Mask.
Please try this:
// rotateLayerAccordingToRectangularLayerMask.jsx
// rotate according to rectangular layer mask;
// 2023, use it at your own risk;
if (app.documents.length > 0) {
if (hasLayerMask () == true) {
var myDocument = activeDocument;
loadSelectionOfLayerMask();
var theID = makeWorkPath (3);
var thisPath = collectPathInfoFromDesc2012 (myDocument, myDocument.pathItems[myDocument.pathItems.length-1]);
deletePath (theID);
// work through supathitems;
if (thisPath.length == 1) {
var thisOne = thisPath[0];
var thePoints = new Array;
var theAngles = new Array;
var theDistances = new Array;
for (var m = 0; m < thisOne.length-2; m++) {
if (m == 0) {var thePrevious = thisOne.length-3}
else {var thePrevious = m-1};
if (m == thisOne.length-3) {var theNext = 0}
else {var theNext = m+1};
theDistances.push(getDistance(thisOne[m][0], thisOne[thePrevious][0]));
theAngles.push(getAngle (thisOne[m][0], thisOne[theNext][0]));
thePoints.push(thisOne[m][0]);
};
// try to remove extra points;
var theAngleVar = 4;
var theResults = [thePoints[0]];
var theAngle = theAngles[0];
for (var n = 0; n < theAngles.length; n++) {
var nextAngle = theAngles[n];
if (absAngleDiff (theAngle, nextAngle) < theAngleVar) {}
else {
if (theDistances[n] > 5) {
theAngle = nextAngle;
theResults.push(thePoints[n])
}
};
};
// check path;
var anArray = [[]];
for (var x = 0; x < theResults.length; x++) {
anArray[0].push([theResults[x], theResults[x], theResults[x], false])
};
// check for four points;
if (anArray[0].length == 4) {
// check the four angles;
var angle1 = getAngle(anArray[0][0][0], anArray[0][1][0]);
var angle2 = getAngle(anArray[0][1][0], anArray[0][2][0]);
var angle3 = getAngle(anArray[0][2][0], anArray[0][3][0]);
var angle4 = getAngle(anArray[0][3][0], anArray[0][0][0]);
//alert (anArray[0].join("\n")+"\n\nangle1\n"+angle1+"\nangle4\n"+(360-angle4));
var angleX = 360-angle4;
// rotate;
if (absAngleDiff(angle1, angleLimit (angle3+180)) < 4 && absAngleDiff(angle1, angleLimit (angle2-90)) < 4 && absAngleDiff(angle1, angleLimit (angle4+90)) < 4) {
if (angle1 < angleX) {
var theAngle = angle1*(-1)
}
else {
var theAngle = angleX
};
scaleRotateLayer (100, theAngle, 0, 0)
}
} else {alert ("the layer mask doesn't work out")}
};
} else {alert ("no layer mask")}
};
////////////////////////////////////
////// collect path info from actiondescriptor, smooth added //////
function collectPathInfoFromDesc2012 (myDocument, thePath) {
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.POINTS;
// based of functions from xbytor’s stdlib;
var idPath = charIDToTypeID( "Path" );
var ref = new ActionReference();
for (var l = 0; l < myDocument.pathItems.length; l++) {
var thisPath = myDocument.pathItems[l];
if (thisPath == thePath && thisPath.kind == PathKind.WORKPATH) {
ref.putProperty(idPath, cTID("WrPt"));
};
if (thisPath == thePath && thisPath.kind != PathKind.WORKPATH && thisPath.kind != PathKind.VECTORMASK) {
ref.putIndex(idPath, l + 1);
};
if (thisPath == thePath && thisPath.kind == PathKind.VECTORMASK) {
var idvectorMask = stringIDToTypeID( "vectorMask" );
ref.putEnumerated( idPath, idPath, idvectorMask );
};
};
var desc = app.executeActionGet(ref);
var pname = desc.getString(cTID('PthN'));
// create new array;
var theArray = new Array;
var pathComponents = desc.getObjectValue(cTID("PthC")).getList(sTID('pathComponents'));
// for subpathitems;
for (var m = 0; m < pathComponents.count; m++) {
var listKey = pathComponents.getObjectValue(m).getList(sTID("subpathListKey"));
var operation1 = pathComponents.getObjectValue(m).getEnumerationValue(sTID("shapeOperation"));
switch (operation1) {
case 1097098272:
var operation = 1097098272 //cTID('Add ');
break;
case 1398961266:
var operation = 1398961266 //cTID('Sbtr');
break;
case 1231975538:
var operation = 1231975538 //cTID('Intr');
break;
default:
// case 1102:
var operation = sTID('xor') //ShapeOperation.SHAPEXOR;
break;
};
// for subpathitem’s count;
for (var n = 0; n < listKey.count; n++) {
theArray.push(new Array);
var points = listKey.getObjectValue(n).getList(sTID('points'));
try {var closed = listKey.getObjectValue(n).getBoolean(sTID("closedSubpath"))}
catch (e) {var closed = false};
// for subpathitem’s segment’s number of points;
for (var o = 0; o < points.count; o++) {
var anchorObj = points.getObjectValue(o).getObjectValue(sTID("anchor"));
var anchor = [anchorObj.getUnitDoubleValue(sTID('horizontal')), anchorObj.getUnitDoubleValue(sTID('vertical'))];
var thisPoint = [anchor];
try {
var left = points.getObjectValue(o).getObjectValue(cTID("Fwd "));
var leftDirection = [left.getUnitDoubleValue(sTID('horizontal')), left.getUnitDoubleValue(sTID('vertical'))];
thisPoint.push(leftDirection)
}
catch (e) {
thisPoint.push(anchor)
};
try {
var right = points.getObjectValue(o).getObjectValue(cTID("Bwd "));
var rightDirection = [right.getUnitDoubleValue(sTID('horizontal')), right.getUnitDoubleValue(sTID('vertical'))];
thisPoint.push(rightDirection)
}
catch (e) {
thisPoint.push(anchor)
};
try {
var smoothOr = points.getObjectValue(o).getBoolean(cTID("Smoo"));
thisPoint.push(smoothOr)
}
catch (e) {thisPoint.push(false)};
theArray[theArray.length - 1].push(thisPoint);
};
theArray[theArray.length - 1].push(closed);
theArray[theArray.length - 1].push(operation);
};
};
// by xbytor, thanks to him;
function cTID (s) { return cTID[s] || cTID[s] = app.charIDToTypeID(s); };
function sTID (s) { return sTID[s] || sTID[s] = app.stringIDToTypeID(s); };
// reset;
app.preferences.rulerUnits = originalRulerUnits;
return theArray;
};
////// radians //////
function radiansOf (theAngle) {
return theAngle * Math.PI / 180
};
////// get an angle, 3:00 being 0˚, 6:00 90˚, etc. //////
function getAngle (pointOne, pointTwo) {
// calculate the triangle sides;
var width = pointTwo[0] - pointOne[0];
var height = pointTwo[1] - pointOne[1];
var sideC = Math.sqrt(Math.pow(width, 2) + Math.pow(height, 2));
// calculate the angles;
if (width+width > width) {theAngle = Math.asin(height / sideC) * 360 / 2 / Math.PI}
else {theAngle = 180 - (Math.asin(height / sideC) * 360 / 2 / Math.PI)};
if (theAngle < 0) {theAngle = (360 + theAngle)};
// if (theAngle > 180) {theAngle = (360 - theAngle) * (-1)};
return theAngle
};
////// delete path by id //////
function deletePath (theID) {
// =======================================================
var desc32 = new ActionDescriptor();
var ref9 = new ActionReference();
ref9.putIndex ( stringIDToTypeID( "path" ), theID );
desc32.putReference( stringIDToTypeID( "null" ), ref9 );
executeAction( stringIDToTypeID( "delete" ), desc32, DialogModes.NO );
};
////// get a distance between two points //////
function getDistance (pointOne, pointTwo) {
// calculate the triangle sides;
var width = pointTwo[0] - pointOne[0];
var height = pointTwo[1] - pointOne[1];
var sideC = Math.sqrt(Math.pow(width, 2) + Math.pow(height, 2));
return sideC
};
////// make work path from selection //////
function makeWorkPath (theTolerance) {
// =======================================================
var idMk = charIDToTypeID( "Mk " );
var desc5 = new ActionDescriptor();
var idnull = charIDToTypeID( "null" );
var ref3 = new ActionReference();
var idPath = charIDToTypeID( "Path" );
ref3.putClass( idPath );
desc5.putReference( idnull, ref3 );
var idFrom = charIDToTypeID( "From" );
var ref4 = new ActionReference();
var idcsel = charIDToTypeID( "csel" );
var idfsel = charIDToTypeID( "fsel" );
ref4.putProperty( idcsel, idfsel );
desc5.putReference( idFrom, ref4 );
var idTlrn = charIDToTypeID( "Tlrn" );
var idPxl = charIDToTypeID( "#Pxl" );
desc5.putUnitDouble( idTlrn, idPxl, theTolerance );
executeAction( idMk, desc5, DialogModes.NO );
/// get index;
var ref = new ActionReference();
ref.putProperty(stringIDToTypeID("property"), stringIDToTypeID("itemIndex"));
ref.putEnumerated( charIDToTypeID("Path"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt") );
var pathDesc = executeActionGet(ref);
// return index;
return pathDesc.getInteger(stringIDToTypeID("itemIndex"))
};
////// load transparency //////
function loadTransparency (theInvert) {
var desc3 = new ActionDescriptor();
var ref2 = new ActionReference();
ref2.putProperty( charIDToTypeID( "Chnl" ), charIDToTypeID( "fsel" ) );
desc3.putReference( charIDToTypeID( "null" ), ref2 );
var ref3 = new ActionReference();
ref3.putEnumerated( charIDToTypeID( "Chnl" ), charIDToTypeID( "Chnl" ), charIDToTypeID( "Trsp" ) );
desc3.putReference( charIDToTypeID( "T " ), ref3 );
desc3.putBoolean(charIDToTypeID("Invr"), theInvert);
executeAction( charIDToTypeID( "setd" ), desc3, DialogModes.NO );
};
////// get absolute number of angle difference //////
function absAngleDiff (theAngle, nextAngle) {
return Math.min(Math.abs((Math.max(theAngle, nextAngle)-Math.min(theAngle, nextAngle))),
Math.abs(Math.max(theAngle, nextAngle)-(Math.min(theAngle, nextAngle)+360)))
};
//////
function angleLimit (theAngle) {
while (theAngle > 360) {theAngle = theAngle - 360};
while (theAngle < 360) {theAngle = theAngle + 360};
return theAngle
};
////// has layer mask //////
function hasLayerMask () {
var m_Dsc01, m_Ref01;
m_Ref01 = new ActionReference();
m_Ref01.putEnumerated(stringIDToTypeID("layer"), charIDToTypeID("Ordn"), charIDToTypeID("Trgt"));
m_Dsc01 = executeActionGet(m_Ref01);
return m_Dsc01.hasKey(charIDToTypeID("Usrs"));
};
////// load layer mask //////
function loadSelectionOfLayerMask() {
try {
var idchannel = stringIDToTypeID( "channel" );
var desc70 = new ActionDescriptor();
var ref9 = new ActionReference();
ref9.putProperty( idchannel, stringIDToTypeID( "selection" ) );
desc70.putReference( stringIDToTypeID( "null" ), ref9 );
var ref10 = new ActionReference();
ref10.putEnumerated( idchannel, idchannel, stringIDToTypeID( "mask" ) );
desc70.putReference( stringIDToTypeID( "to" ), ref10 );
executeAction( stringIDToTypeID( "set" ), desc70, DialogModes.NO );
} catch (_error) {alert (_error)}
};
////// scale active layer to canvas dimensions //////
function scaleRotateLayer (theScale, theAngle, offsetX, offsetY) {
// scale smart object:
var originalRulerUnits = app.preferences.rulerUnits;
app.preferences.rulerUnits = Units.PIXELS;
// transform;
var desc23 = new ActionDescriptor();
var ref2 = new ActionReference();
ref2.putEnumerated( charIDToTypeID( "Lyr " ), charIDToTypeID( "Ordn" ), charIDToTypeID( "Trgt" ) );
desc23.putReference( charIDToTypeID( "null" ), ref2 );
var idOfst = charIDToTypeID( "Ofst" );
var desc24 = new ActionDescriptor();
var idPxl = charIDToTypeID( "#Pxl" );
desc24.putUnitDouble( charIDToTypeID( "Hrzn" ), idPxl, offsetX );
desc24.putUnitDouble( charIDToTypeID( "Vrtc" ), idPxl, offsetY );
desc23.putObject( idOfst, idOfst, desc24 );
var idPrc = charIDToTypeID( "#Prc" );
desc23.putUnitDouble( charIDToTypeID( "Wdth" ), idPrc, theScale );
desc23.putUnitDouble( charIDToTypeID( "Hght" ), idPrc, theScale );
desc23.putUnitDouble( charIDToTypeID( "Angl" ), charIDToTypeID( "#Ang" ), theAngle );
desc23.putEnumerated( charIDToTypeID( "Intr" ), charIDToTypeID( "Intp" ), stringIDToTypeID( "bicubicAutomatic" ) );
desc23.putEnumerated( stringIDToTypeID( "freeTransformCenterState" ), stringIDToTypeID( "quadCenterState" ), stringIDToTypeID( "QCSAverage" ) );
// desc23.putBoolean( charIDToTypeID( "Cpy " ), true );
executeAction( charIDToTypeID( "Trnf" ), desc23, DialogModes.NO );
app.preferences.rulerUnits = originalRulerUnits;
};