Why can't I detect the target style in the group?

It is because your else is misplaced. The first entry for which the name does not match you are returning false. Use the following

var ifStyEx = styleExist("myPar", "allParagraphStyles")
//Check if the style exists
function styleExist(stn, astc) {
    var d = app.activeDocument;
    for (var i = 0; i < d[astc].length; i++) {
        if (d[astc][i].name == stn) {
            //style = d[astc][i];
            //break;
            return true;
        }
    }
    return false;
}

The problem was the else inside your loop. In the original at line 11 as soon as the first style was not myPar, so it never checked the remaining styles.

So this searches for
styles directly in the document
styles inside paragraph style groups
styles inside nested paragraph style groups

var ifStyEx = styleExist("myPar");
alert(ifStyEx);

// Check if a paragraph style exists, including styles inside nested groups.
function styleExist(styleName) {
    return styleExistsInContainer(app.activeDocument, styleName);
}

function styleExistsInContainer(container, styleName) {
    var i;
    var styles = container.paragraphStyles;

    for (i = 0; i < styles.length; i++) {
        if (styles[i].name == styleName) {
            return true;
        }
    }

    var groups = container.paragraphStyleGroups;
    for (i = 0; i < groups.length; i++) {
        if (styleExistsInContainer(groups[i], styleName)) {
            return true;
        }
    }

    return false;
}