Fx 2 and 3 have bug: https://bugzilla.mozilla.org/show_bug.cgi?id=463644

You check relatedTarget in try-catch block, so mouseenter event will never dispatch.

I think you should try to find first block with scrollhHeight > 0 if you can't access to event.relatedTarget.tagName.

My patch to withinElement function:

var withinElement = function(event) {

var parent = event.relatedTarget,
    fxBug = false;

try {
    // fx2 allow access to xul elements

    // fx3 throw the exception: Permission denied to get property XULElement.tagName
    if (parent.tagName.toLowerCase().indexOf('xul:') == 0) {
        fxBug = true;
    }
} catch (e) {
    fxBug = true;
}

if (fxBug) {
    parent = event.target;
    // find first element with scroll
    while (parent = parent.parentNode) {
        if (parent.scrollHeight > 0) {
            break;
        }
    }
}

while ( parent && parent != this )
	try { parent = parent.parentNode; }
	catch(e) { parent = this; }

if( parent != this ){
	// set the correct event type
	event.type = event.data;
	// handle event if we actually just moused on to a non sub-element
	jQuery.event.handle.apply( this, arguments );
}
}

Test page: http://androsov.net/bugs/invalid_relatedTarget_jQuery13.html