From 9c0da8f57ffe4bbc17e59bc55420f0e08168c07e Mon Sep 17 00:00:00 2001 From: vit9696 Date: Fri, 26 Jul 2019 19:43:29 +0300 Subject: [PATCH] OcMemoryLib: Add virtual memory support --- Include/IndustryStandard/VirtualMemory.h | 153 +++++++++ Include/Library/OcMemoryLib.h | 133 +++++++- Library/OcMemoryLib/OcMemoryLib.inf | 1 + Library/OcMemoryLib/VirtualMemory.c | 418 +++++++++++++++++++++++ 4 files changed, 702 insertions(+), 3 deletions(-) create mode 100755 Include/IndustryStandard/VirtualMemory.h create mode 100755 Library/OcMemoryLib/VirtualMemory.c diff --git a/Include/IndustryStandard/VirtualMemory.h b/Include/IndustryStandard/VirtualMemory.h new file mode 100755 index 00000000..a9c81c2d --- /dev/null +++ b/Include/IndustryStandard/VirtualMemory.h @@ -0,0 +1,153 @@ +/** @file + x64 Long Mode Virtual Memory Management Definitions + + References: + 1) IA-32 Intel(R) Architecture Software Developer's Manual Volume 1:Basic Architecture, Intel + 2) IA-32 Intel(R) Architecture Software Developer's Manual Volume 2:Instruction Set Reference, Intel + 3) IA-32 Intel(R) Architecture Software Developer's Manual Volume 3:System Programmer's Guide, Intel + 4) AMD64 Architecture Programmer's Manual Volume 2: System Programming + +Copyright (c) 2006 - 2018, Intel Corporation. All rights reserved.
+Copyright (c) 2017, AMD Incorporated. All rights reserved.
+Copyright (c) 2011, dmazar. All rights reserved.
+ +SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#ifndef VIRTUAL_MEMORY_H +#define VIRTUAL_MEMORY_H + +#pragma pack(push, 1) + +// +// Page-Map Level-4 Offset (PML4) and +// Page-Directory-Pointer Offset (PDPE) entries 4K & 2MB +// + +typedef union { + struct { + UINT64 Present:1; // 0 = Not present in memory, 1 = Present in memory + UINT64 ReadWrite:1; // 0 = Read-Only, 1= Read/Write + UINT64 UserSupervisor:1; // 0 = Supervisor, 1=User + UINT64 WriteThrough:1; // 0 = Write-Back caching, 1=Write-Through caching + UINT64 CacheDisabled:1; // 0 = Cached, 1=Non-Cached + UINT64 Accessed:1; // 0 = Not accessed, 1 = Accessed (set by CPU) + UINT64 Reserved:1; // Reserved + UINT64 MustBeZero:2; // Must Be Zero + UINT64 Available:3; // Available for use by system software + UINT64 PageTableBaseAddress:40; // Page Table Base Address + UINT64 AvabilableHigh:11; // Available for use by system software + UINT64 Nx:1; // No Execute bit + } Bits; + UINT64 Uint64; +} PAGE_MAP_AND_DIRECTORY_POINTER; + +// +// Page Table Entry 4KB +// +typedef union { + struct { + UINT64 Present:1; // 0 = Not present in memory, 1 = Present in memory + UINT64 ReadWrite:1; // 0 = Read-Only, 1= Read/Write + UINT64 UserSupervisor:1; // 0 = Supervisor, 1=User + UINT64 WriteThrough:1; // 0 = Write-Back caching, 1=Write-Through caching + UINT64 CacheDisabled:1; // 0 = Cached, 1=Non-Cached + UINT64 Accessed:1; // 0 = Not accessed, 1 = Accessed (set by CPU) + UINT64 Dirty:1; // 0 = Not Dirty, 1 = written by processor on access to page + UINT64 PAT:1; // + UINT64 Global:1; // 0 = Not global page, 1 = global page TLB not cleared on CR3 write + UINT64 Available:3; // Available for use by system software + UINT64 PageTableBaseAddress:40; // Page Table Base Address + UINT64 AvabilableHigh:11; // Available for use by system software + UINT64 Nx:1; // 0 = Execute Code, 1 = No Code Execution + } Bits; + UINT64 Uint64; +} PAGE_TABLE_4K_ENTRY; + +// +// Page Table Entry 2MB +// +typedef union { + struct { + UINT64 Present:1; // 0 = Not present in memory, 1 = Present in memory + UINT64 ReadWrite:1; // 0 = Read-Only, 1= Read/Write + UINT64 UserSupervisor:1; // 0 = Supervisor, 1=User + UINT64 WriteThrough:1; // 0 = Write-Back caching, 1=Write-Through caching + UINT64 CacheDisabled:1; // 0 = Cached, 1=Non-Cached + UINT64 Accessed:1; // 0 = Not accessed, 1 = Accessed (set by CPU) + UINT64 Dirty:1; // 0 = Not Dirty, 1 = written by processor on access to page + UINT64 MustBe1:1; // Must be 1 + UINT64 Global:1; // 0 = Not global page, 1 = global page TLB not cleared on CR3 write + UINT64 Available:3; // Available for use by system software + UINT64 PAT:1; // + UINT64 MustBeZero:8; // Must be zero; + UINT64 PageTableBaseAddress:31; // Page Table Base Address + UINT64 AvabilableHigh:11; // Available for use by system software + UINT64 Nx:1; // 0 = Execute Code, 1 = No Code Execution + } Bits; + UINT64 Uint64; +} PAGE_TABLE_2M_ENTRY; + +// +// Page Table Entry 1GB +// +typedef union { + struct { + UINT64 Present:1; // 0 = Not present in memory, 1 = Present in memory + UINT64 ReadWrite:1; // 0 = Read-Only, 1= Read/Write + UINT64 UserSupervisor:1; // 0 = Supervisor, 1=User + UINT64 WriteThrough:1; // 0 = Write-Back caching, 1=Write-Through caching + UINT64 CacheDisabled:1; // 0 = Cached, 1=Non-Cached + UINT64 Accessed:1; // 0 = Not accessed, 1 = Accessed (set by CPU) + UINT64 Dirty:1; // 0 = Not Dirty, 1 = written by processor on access to page + UINT64 MustBe1:1; // Must be 1 + UINT64 Global:1; // 0 = Not global page, 1 = global page TLB not cleared on CR3 write + UINT64 Available:3; // Available for use by system software + UINT64 PAT:1; // + UINT64 MustBeZero:17; // Must be zero; + UINT64 PageTableBaseAddress:22; // Page Table Base Address + UINT64 AvabilableHigh:11; // Available for use by system software + UINT64 Nx:1; // 0 = Execute Code, 1 = No Code Execution + } Bits; + UINT64 Uint64; +} PAGE_TABLE_1G_ENTRY; + +typedef union { + struct { + UINT64 PhysPgOffset:12; // 0 = Physical Page Offset + UINT64 PTOffset:9; // 0 = Page Table Offset + UINT64 PDOffset:9; // 0 = Page Directory Offset + UINT64 PDPOffset:9; // 0 = Page Directory Pointer Offset + UINT64 PML4Offset:9; // 0 = Page Map Level 4 Offset + UINT64 SignExtend:16; // 0 = Sign Extend + } Pg4K; + struct { + UINT64 PhysPgOffset:21; // 0 = Physical Page Offset + UINT64 PDOffset:9; // 0 = Page Directory Offset + UINT64 PDPOffset:9; // 0 = Page Directory Pointer Offset + UINT64 PML4Offset:9; // 0 = Page Map Level 4 Offset + UINT64 SignExtend:16; // 0 = Sign Extend + } Pg2M; + struct { + UINT64 PhysPgOffset:30; // 0 = Physical Page Offset + UINT64 PDPOffset:9; // 0 = Page Directory Pointer Offset + UINT64 PML4Offset:9; // 0 = Page Map Level 4 Offset + UINT64 SignExtend:16; // 0 = Sign Extend + } Pg1G; + UINT64 Uint64; +} VIRTUAL_ADDR; + +#define VA_FIX_SIGN_EXTEND(VA) ((VA).Pg4K.SignExtend = ((VA).Pg4K.PML4Offset & 0x100U) ? 0xFFFFU : 0U); + +#pragma pack(pop) + +#define CR3_ADDR_MASK 0x000FFFFFFFFFF000ull +#define CR3_FLAG_PWT 0x0000000000000008ull +#define CR3_FLAG_PCD 0x0000000000000010ull + +#define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull +#define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull +#define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull + +#endif // VIRTUAL_MEMORY_H diff --git a/Include/Library/OcMemoryLib.h b/Include/Library/OcMemoryLib.h index 03708f1c..f3d19902 100644 --- a/Include/Library/OcMemoryLib.h +++ b/Include/Library/OcMemoryLib.h @@ -15,6 +15,8 @@ #ifndef OC_MEMORY_LIB_H #define OC_MEMORY_LIB_H +#include + /** Lock the legacy region specified to enable modification. @@ -86,7 +88,7 @@ GetCurrentMemoryMapAlloc ( ); /** - Shrinks memory map by joining non-runtime records. + Shrink memory map by joining non-runtime records. @param[in,out] MemoryMapSize Memory map size in bytes, updated on shrink. @param[in,out] MemoryMap Memory map to shrink. @@ -115,7 +117,7 @@ BOOLEAN ); /** - Alloctes pages from the top of physical memory up to address specified in Memory. + Allocate pages from the top of physical memory up to address specified in Memory. Unlike AllocateMaxAddress, this method guarantees to choose top most address. @param[in] MemoryType Allocated memory type. @@ -126,7 +128,7 @@ BOOLEAN @retval EFI_SUCCESS on successful allocation. **/ -RETURN_STATUS +EFI_STATUS AllocatePagesFromTop ( IN EFI_MEMORY_TYPE MemoryType, IN UINTN Pages, @@ -135,4 +137,129 @@ AllocatePagesFromTop ( IN CHECK_ALLOCATION_RANGE CheckRange OPTIONAL ); +/** + Return pointer to PML4 table in PageTable and PWT and PCD flags in Flags. + + @param[out] PageTable Current page table address. + @param[out] Flags Current page table PWT and PCT flags. +**/ +VOID +GetCurrentPageTable ( + OUT PAGE_MAP_AND_DIRECTORY_POINTER **PageTable, + OUT UINTN *Flags OPTIONAL + ); + +/** + Return physical addrress for given virtual addrress. + + @param[in] PageTable Page table to use for solving. + @param[in] VirtualAddr Virtual address to look up. + @param[out] PhysicalAddr Physical address to return. + + @retval EFI_SUCCESS on successful lookup. +**/ +EFI_STATUS +GetPhysicalAddress ( + IN PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + OUT EFI_PHYSICAL_ADDRESS *PhysicalAddr + ); + +/** + Virtual memory context +**/ +typedef struct OC_VMEM_CONTEXT_ { + /// + /// Memory pool containing memory to be spread across allocations. + /// + UINT8 *MemoryPool; + /// + /// Free pages in the memory pool. + /// + UINTN FreePages; +} OC_VMEM_CONTEXT; + +/** + Reasonable default virtual memory page pool size (2 MB). +**/ +#define OC_DEFAULT_VMEM_PAGE_COUNT 0x200 + +/** + Allocate EfiBootServicesData virtual memory pool from boot services + in the end of BASE_4GB of RAM. Should be called while boot services are + still usable. + + @param[out] Context Virtual memory pool context. + @param[in] NumPages Number of pages to be allocated in the pool. + + @retval EFI_SUCCESS on successful allocation. +**/ +EFI_STATUS +VmAllocateMemoryPool ( + OUT OC_VMEM_CONTEXT *Context, + IN UINTN NumPages + ); + +/** + Allocate pages for e.g. vm page maps. + + @param[in,out] Context Virtual memory pool context. + @param[in] NumPages Number of pages to allocate. + + @retval allocated pages or NULL. +**/ +VOID * +VmAllocatePages ( + IN OUT OC_VMEM_CONTEXT *Context, + IN UINTN NumPages + ); + +/** + Map (remap) given page at physical address to given virtual address in + the specified page table. + + @param[in,out] Context Virtual memory pool context. + @param[in] PageTable Page table to update. + @param[in] VirtualAddr Virtual memory address to map at. + @param[in] PhysicalAddr Physical memory address to map from. + + @retval EFI_SUCCESS on success. +**/ +EFI_STATUS +VmMapVirtualPage ( + IN OUT OC_VMEM_CONTEXT *Context, + IN OUT PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + IN EFI_PHYSICAL_ADDRESS PhysicalAddr + ); + +/** + Map (remap) a range of 4K pages at physical address to given virtual address + in the specified page table. + + @param[in,out] Context Virtual memory pool context. + @param[in] PageTable Page table to update. + @param[in] VirtualAddr Virtual memory address to map at. + @param[in] NumPages Number of 4K pages to map. + @param[in] PhysicalAddr Physical memory address to map from. + + @retval EFI_SUCCESS on success. +**/ +EFI_STATUS +VmMapVirtualPages ( + IN OUT OC_VMEM_CONTEXT *Context, + IN OUT PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + IN UINTN NumPages, + IN EFI_PHYSICAL_ADDRESS PhysicalAddr + ); + +/** + Flushes TLB caches. +**/ +VOID +VmFlushCaches ( + VOID + ); + #endif // OC_MEMORY_LIB_H diff --git a/Library/OcMemoryLib/OcMemoryLib.inf b/Library/OcMemoryLib/OcMemoryLib.inf index 8f97a09c..efa98f5e 100755 --- a/Library/OcMemoryLib/OcMemoryLib.inf +++ b/Library/OcMemoryLib/OcMemoryLib.inf @@ -45,3 +45,4 @@ MemoryMap.c LegacyRegionLock.c LegacyRegionUnLock.c + VirtualMemory.c diff --git a/Library/OcMemoryLib/VirtualMemory.c b/Library/OcMemoryLib/VirtualMemory.c new file mode 100755 index 00000000..32abc91c --- /dev/null +++ b/Library/OcMemoryLib/VirtualMemory.c @@ -0,0 +1,418 @@ +/** @file + Copyright (C) 2011, dmazar. All rights reserved. + Copyright (C) 2019, vit9696. All rights reserved. + + All rights reserved. + + This program and the accompanying materials + are licensed and made available under the terms and conditions of the BSD License + which accompanies this distribution. The full text of the license may be found at + http://opensource.org/licenses/bsd-license.php + + THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +**/ + +#include + +#include +#include +#include +#include + +VOID +GetCurrentPageTable ( + OUT PAGE_MAP_AND_DIRECTORY_POINTER **PageTable, + OUT UINTN *Flags OPTIONAL + ) +{ + UINTN CR3; + + CR3 = AsmReadCr3 (); + + *PageTable = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN) (CR3 & CR3_ADDR_MASK); + + if (Flags != NULL) { + *Flags = CR3 & (CR3_FLAG_PWT | CR3_FLAG_PCD); + } +} + +EFI_STATUS +GetPhysicalAddress ( + IN PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + OUT EFI_PHYSICAL_ADDRESS *PhysicalAddr + ) +{ + EFI_PHYSICAL_ADDRESS Start; + VIRTUAL_ADDR VA; + VIRTUAL_ADDR VAStart; + VIRTUAL_ADDR VAEnd; + PAGE_MAP_AND_DIRECTORY_POINTER *PML4; + PAGE_MAP_AND_DIRECTORY_POINTER *PDPE; + PAGE_MAP_AND_DIRECTORY_POINTER *PDE; + PAGE_TABLE_4K_ENTRY *PTE4K; + PAGE_TABLE_2M_ENTRY *PTE2M; + PAGE_TABLE_1G_ENTRY *PTE1G; + + if (PageTable == NULL) { + GetCurrentPageTable (&PageTable, NULL); + } + + VA.Uint64 = (UINT64) VirtualAddr; + + // + // PML4 + // + PML4 = PageTable; + PML4 += VA.Pg4K.PML4Offset; + VAStart.Uint64 = 0; + VAStart.Pg4K.PML4Offset = VA.Pg4K.PML4Offset; + VA_FIX_SIGN_EXTEND (VAStart); + VAEnd.Uint64 = ~(UINT64) 0; + VAEnd.Pg4K.PML4Offset = VA.Pg4K.PML4Offset; + VA_FIX_SIGN_EXTEND (VAEnd); + + if (!PML4->Bits.Present) { + return EFI_NO_MAPPING; + } + + // + // PDPE + // + PDPE = (PAGE_MAP_AND_DIRECTORY_POINTER *) (PML4->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PDPE += VA.Pg4K.PDPOffset; + VAStart.Pg4K.PDPOffset = VA.Pg4K.PDPOffset; + VAEnd.Pg4K.PDPOffset = VA.Pg4K.PDPOffset; + + if (!PDPE->Bits.Present) { + return EFI_NO_MAPPING; + } + + if (PDPE->Bits.MustBeZero & 0x1) { + // + // 1GB PDPE + // + PTE1G = (PAGE_TABLE_1G_ENTRY *) PDPE; + Start = PTE1G->Uint64 & PAGING_1G_ADDRESS_MASK_64; + *PhysicalAddr = Start + VA.Pg1G.PhysPgOffset; + return EFI_SUCCESS; + } + + // + // PDE + // + PDE = (PAGE_MAP_AND_DIRECTORY_POINTER *)(PDPE->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PDE += VA.Pg4K.PDOffset; + VAStart.Pg4K.PDOffset = VA.Pg4K.PDOffset; + VAEnd.Pg4K.PDOffset = VA.Pg4K.PDOffset; + + if (!PDE->Bits.Present) { + return EFI_NO_MAPPING; + } + + if (PDE->Bits.MustBeZero & 0x1) { + // + // 2MB PDE + // + PTE2M = (PAGE_TABLE_2M_ENTRY *) PDE; + Start = PTE2M->Uint64 & PAGING_2M_ADDRESS_MASK_64; + *PhysicalAddr = Start + VA.Pg2M.PhysPgOffset; + return EFI_SUCCESS; + } + + // + // PTE + // + PTE4K = (PAGE_TABLE_4K_ENTRY *) (PDE->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PTE4K += VA.Pg4K.PTOffset; + VAStart.Pg4K.PTOffset = VA.Pg4K.PTOffset; + VAEnd.Pg4K.PTOffset = VA.Pg4K.PTOffset; + + if (!PTE4K->Bits.Present) { + return EFI_NO_MAPPING; + } + + Start = PTE4K->Uint64 & PAGING_4K_ADDRESS_MASK_64; + *PhysicalAddr = Start + VA.Pg4K.PhysPgOffset; + + return EFI_SUCCESS; +} + +EFI_STATUS +VmAllocateMemoryPool ( + OUT OC_VMEM_CONTEXT *Context, + IN UINTN NumPages + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS Addr; + + Addr = BASE_4GB; + Status = AllocatePagesFromTop ( + EfiBootServicesData, + NumPages, + &Addr, + NULL, + NULL + ); + + if (!EFI_ERROR (Status)) { + Context->MemoryPool = (UINT8 *) Addr; + Context->FreePages = NumPages; + } else { + DEBUG ((DEBUG_ERROR, "OCVM: memory pool allocation failure - %r\n", Status)); + } + + return Status; +} + +VOID * +VmAllocatePages ( + IN OUT OC_VMEM_CONTEXT *Context, + IN UINTN NumPages + ) +{ + VOID *AllocatedPages; + + AllocatedPages = NULL; + + if (Context->FreePages >= NumPages) { + AllocatedPages = Context->MemoryPool; + Context->MemoryPool += EFI_PAGES_TO_SIZE (NumPages); + Context->FreePages -= NumPages; + } else { + DEBUG ((DEBUG_ERROR, "OCVM: memory pool out of free pages\n")); + } + + return AllocatedPages; +} + +EFI_STATUS +VmMapVirtualPage ( + IN OUT OC_VMEM_CONTEXT *Context, + IN OUT PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + IN EFI_PHYSICAL_ADDRESS PhysicalAddr + ) +{ + EFI_PHYSICAL_ADDRESS Start; + VIRTUAL_ADDR VA; + VIRTUAL_ADDR VAStart; + VIRTUAL_ADDR VAEnd; + PAGE_MAP_AND_DIRECTORY_POINTER *PML4; + PAGE_MAP_AND_DIRECTORY_POINTER *PDPE; + PAGE_MAP_AND_DIRECTORY_POINTER *PDE; + PAGE_TABLE_4K_ENTRY *PTE4K; + PAGE_TABLE_4K_ENTRY *PTE4KTmp; + PAGE_TABLE_2M_ENTRY *PTE2M; + PAGE_TABLE_1G_ENTRY *PTE1G; + UINTN Index; + + if (PageTable == NULL) { + GetCurrentPageTable (&PageTable, NULL); + } + + VA.Uint64 = (UINT64) VirtualAddr; + + // + // PML4 + // + PML4 = PageTable; + PML4 += VA.Pg4K.PML4Offset; + + // + // There is a problem if our PML4 points to the same table as first PML4 entry + // since we may mess the mapping of first virtual region (happens in VBox and probably DUET). + // Check for this on first call and if true, just clear our PML4 - we'll rebuild at a later step. + // + if (PML4 != PageTable && PML4->Bits.Present + && PageTable->Bits.PageTableBaseAddress == PML4->Bits.PageTableBaseAddress) { + PML4->Uint64 = 0; + } + + VAStart.Uint64 = 0; + VAStart.Pg4K.PML4Offset = VA.Pg4K.PML4Offset; + VA_FIX_SIGN_EXTEND (VAStart); + VAEnd.Uint64 = ~(UINT64) 0; + VAEnd.Pg4K.PML4Offset = VA.Pg4K.PML4Offset; + VA_FIX_SIGN_EXTEND (VAEnd); + + if (!PML4->Bits.Present) { + PDPE = (PAGE_MAP_AND_DIRECTORY_POINTER *) VmAllocatePages (Context, 1); + + if (PDPE == NULL) { + return EFI_NO_MAPPING; + } + + ZeroMem (PDPE, EFI_PAGE_SIZE); + + // + // Init this whole 512 GB region with 512 1GB entry pages to map + // the first 512 GB physical space. + // + PTE1G = (PAGE_TABLE_1G_ENTRY *) PDPE; + Start = 0; + for (Index = 0; Index < 512; ++Index) { + PTE1G->Uint64 = Start & PAGING_1G_ADDRESS_MASK_64; + PTE1G->Bits.ReadWrite = 1; + PTE1G->Bits.Present = 1; + PTE1G->Bits.MustBe1 = 1; + PTE1G++; + Start += BASE_1GB; + } + + // + // Put it to PML4. + // + PML4->Uint64 = ((UINT64)PDPE) & PAGING_4K_ADDRESS_MASK_64; + PML4->Bits.ReadWrite = 1; + PML4->Bits.Present = 1; + } + + // + // PDPE + // + PDPE = (PAGE_MAP_AND_DIRECTORY_POINTER *) (PML4->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PDPE += VA.Pg4K.PDPOffset; + VAStart.Pg4K.PDPOffset = VA.Pg4K.PDPOffset; + VAEnd.Pg4K.PDPOffset = VA.Pg4K.PDPOffset; + + if (!PDPE->Bits.Present || (PDPE->Bits.MustBeZero & 0x1)) { + PDE = (PAGE_MAP_AND_DIRECTORY_POINTER *) VmAllocatePages(Context, 1); + + if (PDE == NULL) { + return EFI_NO_MAPPING; + } + + ZeroMem (PDE, EFI_PAGE_SIZE); + + if (PDPE->Bits.MustBeZero & 0x1) { + // + // This is 1 GB page. Init new PDE array to get the same + // mapping but with 2MB pages. + // + PTE2M = (PAGE_TABLE_2M_ENTRY *) PDE; + Start = PDPE->Uint64 & PAGING_1G_ADDRESS_MASK_64; + + for (Index = 0; Index < 512; ++Index) { + PTE2M->Uint64 = Start & PAGING_2M_ADDRESS_MASK_64; + PTE2M->Bits.ReadWrite = 1; + PTE2M->Bits.Present = 1; + PTE2M->Bits.MustBe1 = 1; + PTE2M++; + Start += BASE_2MB; + } + } + + // + // Put it to PDPE. + // + PDPE->Uint64 = ((UINT64) PDE) & PAGING_4K_ADDRESS_MASK_64; + PDPE->Bits.ReadWrite = 1; + PDPE->Bits.Present = 1; + } + + // + // PDE + // + PDE = (PAGE_MAP_AND_DIRECTORY_POINTER *) (PDPE->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PDE += VA.Pg4K.PDOffset; + VAStart.Pg4K.PDOffset = VA.Pg4K.PDOffset; + VAEnd.Pg4K.PDOffset = VA.Pg4K.PDOffset; + + if (!PDE->Bits.Present || (PDE->Bits.MustBeZero & 0x1)) { + PTE4K = (PAGE_TABLE_4K_ENTRY *) VmAllocatePages (Context, 1); + + if (PTE4K == NULL) { + return EFI_NO_MAPPING; + } + + ZeroMem (PTE4K, EFI_PAGE_SIZE); + + if (PDE->Bits.MustBeZero & 0x1) { + // + // This is 2 MB page. Init new PTE array to get the same + // mapping but with 4KB pages. + // + PTE4KTmp = (PAGE_TABLE_4K_ENTRY *)PTE4K; + Start = PDE->Uint64 & PAGING_2M_ADDRESS_MASK_64; + + for (Index = 0; Index < 512; ++Index) { + PTE4KTmp->Uint64 = Start & PAGING_4K_ADDRESS_MASK_64; + PTE4KTmp->Bits.ReadWrite = 1; + PTE4KTmp->Bits.Present = 1; + PTE4KTmp++; + Start += BASE_4KB; + } + } + + // + // Put it to PDE. + // + PDE->Uint64 = ((UINT64) PTE4K) & PAGING_4K_ADDRESS_MASK_64; + PDE->Bits.ReadWrite = 1; + PDE->Bits.Present = 1; + } + + // + // PTE + // + PTE4K = (PAGE_TABLE_4K_ENTRY *)(PDE->Uint64 & PAGING_4K_ADDRESS_MASK_64); + PTE4K += VA.Pg4K.PTOffset; + VAStart.Pg4K.PTOffset = VA.Pg4K.PTOffset; + VAEnd.Pg4K.PTOffset = VA.Pg4K.PTOffset; + + // + // Put it to PTE. + // + PTE4K->Uint64 = ((UINT64) PhysicalAddr) & PAGING_4K_ADDRESS_MASK_64; + PTE4K->Bits.ReadWrite = 1; + PTE4K->Bits.Present = 1; + + return EFI_SUCCESS; +} + +EFI_STATUS +VmMapVirtualPages ( + IN OUT OC_VMEM_CONTEXT *Context, + IN OUT PAGE_MAP_AND_DIRECTORY_POINTER *PageTable OPTIONAL, + IN EFI_VIRTUAL_ADDRESS VirtualAddr, + IN UINTN NumPages, + IN EFI_PHYSICAL_ADDRESS PhysicalAddr + ) +{ + EFI_STATUS Status; + + if (PageTable == NULL) { + GetCurrentPageTable (&PageTable, NULL); + } + + Status = EFI_SUCCESS; + + while (NumPages > 0 && !EFI_ERROR (Status)) { + Status = VmMapVirtualPage ( + Context, + PageTable, + VirtualAddr, + PhysicalAddr + ); + + VirtualAddr += EFI_PAGE_SIZE; + PhysicalAddr += EFI_PAGE_SIZE; + NumPages--; + } + + return Status; +} + +VOID +VmFlushCaches ( + VOID + ) +{ + // + // Simply reload CR3 register. + // + AsmWriteCr3 (AsmReadCr3 ()); +}