At this point, it seems only causing an alert dialog to pop up does the trick... this seems a bit too intrusive, I feel, particularly given the use you're trying to put it to. Instead of causing it to flash, though, you could modify document.title to grab the user's attention, either by prepending some signal (perhaps the string "NEW!") to the site's name, and then using an interval to constantly change it to "", which would then give a nice little "flashing" illusion.
Bare-bones example:
<html>
<head>
<title>Chat System</title>
<script>
var timer, old_t = document.title, blink_amount = 5, cur_b = 0;
function notify()
{
cur_b = 0;
timer = setInterval(function()
{
if (cur_b < blink_amount * 2)
{
cur_b++;
document.title = (document.title.indexOf('NEW! ') != -1) ? old_t : 'NEW! ' + old_t;
}
else
{
clearInterval(timer);
}
}, 600);
}
notify();
// From here, it's just a matter of calling the
// notify() function whenever you detect a new message.
</script>
</head>
<body>
</body>
</html>