It's perfectly possible to capture all key events in documents with designMode turned on, though you have to use addEventListener on document in Firefox (and possibly others) rather than assigning your handler to document.onkeypress.
To capture the user typing the '@' character (or indeed any printable character), you should use the keypress event rather than keydown.
// Assuming you have a reference to your iframe in a variable called 'iframe':
function handleIframeKeyPress(evt) {
    evt = evt || iframe.contentWindow.event;
    var charCode = evt.keyCode || evt.which;
    var charTyped = String.fromCharCode(charCode);
    if (charTyped === "@") {
        alert("@ typed");
    }
}
var doc = iframe.contentWindow.document;
if (doc.addEventListener) {
    doc.addEventListener("keypress", handleIframeKeyPress, false);
} else if (doc.attachEvent) {
    doc.attachEvent("onkeypress", handleIframeKeyPress);
} else {
    doc.onkeypress = handleIframeKeyPress;
}