Skip to main content
K
Kalow
Known Participant
March 13, 2017
Answered

Prevent drop shadow from showing on transparent pixels

  • March 13, 2017
  • 2 replies
  • 6513 views

On the attached example - how can I prevent the drop shadows from casting on transparent pixels?

Each fish is a smart object with a drop shadow effect that could quickly be replaced with a banana, bowling-pin, etc. I want the shadows on each of the smart objects behind, and the inside and flaps of the box.

Toggling the "lock transparent pixels" option in the layers panel doesn't seem to apply to effects.

I've been working on a solution for a while and can't find anything that seems right. Any help is appreciated! Thanks.

This topic has been closed for replies.
Correct answer Chuck Uebele

Now it works with Smart Objects:

#target photoshop
var doc = activeDocument;
var gp = doc.activeLayer
var bound = false
var curLayer
getLayers (gp)
function getLayers(gp1){
    for(var i=0;i<gp1.layers.length;i++){
        try{
            var check = doc.selection.bounds
            bound =true;
            }
        catch(e){bound = false}
        doc.activeLayer = curLayer = gp1.layers;
        if(curLayer.typename =='LayerSet'){getLayers (gp1.layers)}
        else{
                if(curLayer.kind == 'LayerKind.NORMAL' || curLayer.kind=='LayerKind.SMARTOBJECT'){
                    if(bound){selPx ()}
                    else{selPxFirst ()};
                    }
                else{
                    try{
                        if(bound){selShape ()}
                        else{selShapeFirst ()};
                        }
                    catch(e){}       
                    }                
            }
        }
    doc.activeLayer = gp
    }//end function
function selPxFirst(){
    var idsetd = charIDToTypeID( "setd" );
        var desc7 = new ActionDescriptor();
        var idnull = charIDToTypeID( "null" );
            var ref7 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idfsel = charIDToTypeID( "fsel" );
            ref7.putProperty( idChnl, idfsel );
        desc7.putReference( idnull, ref7 );
        var idT = charIDToTypeID( "T   " );
            var ref8 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idChnl = charIDToTypeID( "Chnl" );
            var idTrsp = charIDToTypeID( "Trsp" );
            ref8.putEnumerated( idChnl, idChnl, idTrsp );
        desc7.putReference( idT, ref8 );
    executeAction( idsetd, desc7, DialogModes.NO );
    }
function selPx(){
    var idAdd = charIDToTypeID( "Add " );
        var desc5 = new ActionDescriptor();
        var idnull = charIDToTypeID( "null" );
            var ref5 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idChnl = charIDToTypeID( "Chnl" );
            var idTrsp = charIDToTypeID( "Trsp" );
            ref5.putEnumerated( idChnl, idChnl, idTrsp );
        desc5.putReference( idnull, ref5 );
        var idT = charIDToTypeID( "T   " );
            var ref6 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idfsel = charIDToTypeID( "fsel" );
            ref6.putProperty( idChnl, idfsel );
        desc5.putReference( idT, ref6 );
    executeAction( idAdd, desc5, DialogModes.NO );
}
function selShapeFirst(){
    var idsetd = charIDToTypeID( "setd" );
        var desc2 = new ActionDescriptor();
        var idnull = charIDToTypeID( "null" );
            var ref1 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idfsel = charIDToTypeID( "fsel" );
            ref1.putProperty( idChnl, idfsel );
        desc2.putReference( idnull, ref1 );
        var idT = charIDToTypeID( "T   " );
            var ref2 = new ActionReference();
            var idPath = charIDToTypeID( "Path" );
            var idPath = charIDToTypeID( "Path" );
            var idvectorMask = stringIDToTypeID( "vectorMask" );
            ref2.putEnumerated( idPath, idPath, idvectorMask );
            var idLyr = charIDToTypeID( "Lyr " );
            var idOrdn = charIDToTypeID( "Ordn" );
            var idTrgt = charIDToTypeID( "Trgt" );
            ref2.putEnumerated( idLyr, idOrdn, idTrgt );
        desc2.putReference( idT, ref2 );
        var idVrsn = charIDToTypeID( "Vrsn" );
        desc2.putInteger( idVrsn, 1 );
        var idvectorMaskParams = stringIDToTypeID( "vectorMaskParams" );
        desc2.putBoolean( idvectorMaskParams, true );
    executeAction( idsetd, desc2, DialogModes.NO );
    }
function selShape(){
    var idAddT = charIDToTypeID( "AddT" );
        var desc13 = new ActionDescriptor();
        var idnull = charIDToTypeID( "null" );
            var ref14 = new ActionReference();
            var idChnl = charIDToTypeID( "Chnl" );
            var idfsel = charIDToTypeID( "fsel" );
            ref14.putProperty( idChnl, idfsel );
        desc13.putReference( idnull, ref14 );
        var idT = charIDToTypeID( "T   " );
            var ref15 = new ActionReference();
            var idPath = charIDToTypeID( "Path" );
            var idPath = charIDToTypeID( "Path" );
            var idvectorMask = stringIDToTypeID( "vectorMask" );
            ref15.putEnumerated( idPath, idPath, idvectorMask );
            var idLyr = charIDToTypeID( "Lyr " );
            var idOrdn = charIDToTypeID( "Ordn" );
            var idTrgt = charIDToTypeID( "Trgt" );
            ref15.putEnumerated( idLyr, idOrdn, idTrgt );
        desc13.putReference( idT, ref15 );
        var idVrsn = charIDToTypeID( "Vrsn" );
        desc13.putInteger( idVrsn, 1 );
        var idvectorMaskParams = stringIDToTypeID( "vectorMaskParams" );
        desc13.putBoolean( idvectorMaskParams, true );
    executeAction( idAddT, desc13, DialogModes.NO );
    }

2 replies

Chuck Uebele
Community Expert
Chuck UebeleCommunity Expert
Community Expert
March 14, 2017

To make life easier, I wrote a script for another thread that makes a selection of all layers in a group. You could use that to create a new mask. However, some code should be added to turn off the layer effects visibility.

forums.adobe.com/message/7753221#7753221

K
KalowAuthor
Known Participant
March 14, 2017

https://forums.adobe.com/people/Chuck+Uebele  wrote

To make life easier, I wrote a script for another thread that makes a selection of all layers in a group. You could use that to create a new mask. However, some code should be added to turn off the layer effects visibility.

forums.adobe.com/message/7753221#7753221

This is very cool. Unfortunately it doesn't seem to work with Smart Objects, only after I rasterize each layer.

No need to turn the shadow effect off though, It selects only the drawn pixels.

Chuck Uebele
Community Expert
Chuck UebeleCommunity Expert
Community Expert
March 14, 2017

Haven't tested it with SO, guess I'll have to take a look at that.

S
SJRiegel
Legend
March 13, 2017

Make a group of all your layers.
Turn off all of your shadow effects.

Use the magic wand or quick select tool to select the transparent area, then add a mask of it to your group.

When you turn the shadows back on, they should be only on your non-transparnet areas.

K
KalowAuthor
Known Participant
March 13, 2017

Thank you - I've tried that in the past and it does work, but is kind of clumsy and time consuming.

This requires turning shadow effects on-and-off for 30-40 objects sometimes (can't find a way to turn them all off with one click) and repeat each time I replace the contents of the smart object.

Any other ideas?

Trevor.Dennis
Community Expert
Trevor.DennisCommunity Expert
Community Expert
March 13, 2017

Your problem, of course, is that you need the shadows to overlap the other box contents 'inside' the box, so I can't think of any significant shortcuts to achieving this.  The first thing you need to do is check Layer mask hides effect in the layer style panel

You'll already be using this workflow I suspect, but it then boils down to:

Adding a layer mask to each box object

Ctrl clicking the relevant box layer to load it as a selection, inverting the selection, and filling each layer mask with black

That will make the part outside the box disappear, so Ctrl click each object in turn, and fill its layer mask with white.

You now have the box contents back, but the drop shadow overlap will be gone outside the box, so it is a matter of hand painting it back in the layer mask.

If you were feeling very clever, you could arrive at the required selection using calculations, but that would make my head hurt.

Here's a slightly better workflow. Before you add the layer mask, Ctrl click each box object in turn to load as a selection, and add the layer mask. That leaves you with no shadow.

Then hand paint the shadows back in.  You can help yourself by Ctrl clicking other objects to restrain where the shadow is painted in.   That's the best I can think of right now, but what often happens here is that one idea prompts other users to think of another wrinkle, so fingers crossed.

Terms & ConditionsAccessibility statement
Using the community Terms of use Privacy Cookie preferences Do not sell or share my personal information ad choicesAdChoices