Trap #3 in the Host driver (assertion failed)

[Valery V. Sedletski]

eax=00000001 ebx=003617c0 ecx=00000000 edx=00000002 esi=003617bc edi=00000000
eip=f08d7280 esp=f8ecfba4 ebp=f8ecfbdc iopl=0 -- -- -- nv up ei pl nz na po nc
cs=0178 ss=1550 ds=0170 es=0170 fs=0000 gs=0000 cr2=1cd3d040 cr3=0020c000 p=00
0178:f08d7280 cc int 3
0178:f08d4a8c vboxdrv:TEXT32:_supdrvIDC + 27f4
0178:f08d8408 _supdrvQueryVTCapsInternal - 1188
0178:f08d7280 cc int 3
0178:f08d7281 90 nop
0178:f08d7282 ebce jmp f08d7252
0178:f08d7284 83ec08 sub esp,+08
0178:f08d7287 6a00 push +00
0178:f08d7289 ff75dc push dword ptr [ebp-24]
0178:f08d728c e887cd0100 call _RTR0MemObjFree (f08f4018)
0178:f08d7291 83c410 add esp,+10
0178:f08d7294 85c0 test eax,eax
0178:f08d7296 89c3 mov ebx,eax
0178:f08d7298 7853 js f08d72ed
0178:f08d729a 83ec08 sub esp,+08
0178:f08d318e fd34f7b4 00000001 00000000 003617b8 _supdrvIOCtlFast + 2606
0178:f08d46df 0000000a f9d56fd0 fd34f7b4 003617a0 _supdrvIOCtl + 8b
0178:f08cea84 0000000a f9d56fd0 fd34f7b4 003617a0 _VBoxDrvClose + 5e8
0178:f08ce041 00000139 000000c0 0000000a 003617a0 _VBoxDrvEP_GenIOCtl_Other_32 + 22
0178:016854a1 fff9de3b 00000178 3f880000 00000246 _gItemString + 1675a61
##

[Valery V. Sedletski]

The _supdrvIOCtlFast + 2606 address is in the src\VBox\HostDrivers?\Support\SUPDrv.cpp, function SUPR0PageAllocEx:

                if (paPages)
                {
                    uint32_t iPage = cPages;
                    while (iPage-- > 0)
                    {
                        paPages[iPage] = RTR0MemObjGetPagePhysAddr(Mem.MapObjR3, iPage); // NIL_RTHCPHYS
                        Assert(paPages[iPage] != NIL_RTHCPHYS); // The failed assertion
                    }
                }
                return VINF_SUCCESS;
            }

            rc2 = RTR0MemObjFree(Mem.MapObjR3, false);
            AssertRC(rc2);
        }

        rc2 = RTR0MemObjFree(Mem.MemObj, false);
        AssertRC(rc2);
    }
    return rc;
}

[Valery V. Sedletski]

The assertion was failed because of RTR0MemObjGetPagePhysAddr() function returning NIL_RTHCPHYS == (long long)(-1). Looking into this function and setting the trace printfs, we see that vbox\src\VBox\Runtime\r0drv\memobj-r0drv.cpp, RTR0MemObjGetPagePhysAddr:

    /*
     * Do the job.
     */
    return rtR0MemObjNativeGetPagePhysAddr(pMem, iPage); // vs: here
}

calls rtR0MemObjNativeGetPagePhysAddr() in vbox\src\VBox\Runtime\r0drv\os2\memobj-r0drv-os2.cpp, rtR0MemObjNativeGetPagePhysAddr() -- the OS-dependent function, which returns NIL_RTHCPHYS:

DECLHIDDEN(RTHCPHYS) rtR0MemObjNativeGetPagePhysAddr(PRTR0MEMOBJINTERNAL pMem, size_t iPage)
{
    PRTR0MEMOBJOS2 pMemOs2 = (PRTR0MEMOBJOS2)pMem;

    switch (pMemOs2->Core.enmType)
    {
        case RTR0MEMOBJTYPE_PAGE:
        case RTR0MEMOBJTYPE_LOW:
        case RTR0MEMOBJTYPE_LOCK:
        case RTR0MEMOBJTYPE_PHYS_NC:
            return pMemOs2->aPages[iPage].Addr;

        case RTR0MEMOBJTYPE_CONT:
            return pMemOs2->Core.u.Cont.Phys + (iPage << PAGE_SHIFT);

        case RTR0MEMOBJTYPE_PHYS:
            return pMemOs2->Core.u.Phys.PhysBase + (iPage << PAGE_SHIFT);

        case RTR0MEMOBJTYPE_MAPPING: // vs
            return rtR0MemObjNativeGetPagePhysAddr(pMemOs2->Core.uRel.Child.pParent, iPage);

        case RTR0MEMOBJTYPE_RES_VIRT:
        //case RTR0MEMOBJTYPE_MAPPING: // vs: == 8
        default:
            return NIL_RTHCPHYS; // vs
    }
}

-- here we added the new case RTR0MEMOBJTYPE_MAPPING, which was unimplemented (I commented out the case at the end, with default processing, and added the processing below)

        case RTR0MEMOBJTYPE_MAPPING: // vs
            return rtR0MemObjNativeGetPagePhysAddr(pMemOs2->Core.uRel.Child.pParent, iPage);

which was taken from the Linux version.

Now it doesn't trap, and several other problems appear.

[dmik]

Applied the fix in r56.