I had reproduced problem with this code:
<script type="text/javascript" >
document.showInnerPopup = function(){
alert("Inner popup");
}
function btnOnClick(){
var popup = document.parentWindow.createPopup();
popup.document.body.innerHTML = "<div style=\"border: 1px solid red;width: 100%; height:100%\" id=\"popupDiv\" onclick=\"javascript:document.showInnerPopup(this,event)\">Click to show another popup</div>";
popup.document.showInnerPopup = function(obj, evt) {
var doc = evt.srcElement.ownerDocument;
var popup = doc.parentWindow.createPopup();
popup.document.body.innerHTML = "<div style=\"border: 1px solid green;width: 100%; height:100%\">Nested popup</div>"
popup.show(10, -20, 150, 150, doc.body);
}
popup.show(10, -20, 100, 100, document.getElementById("btn"));
document.btn = document.getElementById("btn");
document.popup = popup;
}
</script>
<input type="button" value="Show Popup" onclick="btnOnClick()" id="btn" />
This behavior is because of IE setting “Allow script-initiated windows without size or position constraints” which by-default has value “Disable” even for trusted zone