I hereby relicense all my work on Sortix under the ISC license as below. All Sortix contributions by other people are already under this license, are not substantial enough to be copyrightable, or have been removed. All imported code from other projects is compatible with this license. All GPL licensed code from other projects had previously been removed. Copyright 2011-2016 Jonas 'Sortie' Termansen and contributors. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
844 lines
21 KiB
C++
844 lines
21 KiB
C++
/*
|
|
* Copyright (c) 2011, 2012, 2014, 2015 Jonas 'Sortie' Termansen.
|
|
*
|
|
* Permission to use, copy, modify, and distribute this software for any
|
|
* purpose with or without fee is hereby granted, provided that the above
|
|
* copyright notice and this permission notice appear in all copies.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
*
|
|
* x86-family/memorymanagement.cpp
|
|
* Handles memory for the x86 family of architectures.
|
|
*/
|
|
|
|
#include <assert.h>
|
|
#include <errno.h>
|
|
#include <string.h>
|
|
|
|
#include <sortix/mman.h>
|
|
|
|
#include <sortix/kernel/kernel.h>
|
|
#include <sortix/kernel/kthread.h>
|
|
#include <sortix/kernel/memorymanagement.h>
|
|
#include <sortix/kernel/panic.h>
|
|
#include <sortix/kernel/pat.h>
|
|
#include <sortix/kernel/syscall.h>
|
|
|
|
#include "multiboot.h"
|
|
#include "memorymanagement.h"
|
|
#include "msr.h"
|
|
|
|
namespace Sortix {
|
|
|
|
extern size_t end;
|
|
|
|
} // namespace Sortix
|
|
|
|
namespace Sortix {
|
|
namespace Page {
|
|
|
|
void InitPushRegion(addr_t position, size_t length);
|
|
size_t pagesnotonstack = 0;
|
|
size_t stackused = 0;
|
|
size_t stackreserved = 0;
|
|
size_t stacklength = 4096 / sizeof(addr_t);
|
|
size_t totalmem = 0;
|
|
size_t page_usage_counts[PAGE_USAGE_NUM_KINDS];
|
|
kthread_mutex_t pagelock = KTHREAD_MUTEX_INITIALIZER;
|
|
|
|
} // namespace Page
|
|
} // namespace Sortix
|
|
|
|
namespace Sortix {
|
|
namespace Memory {
|
|
|
|
addr_t PAT2PMLFlags[PAT_NUM];
|
|
|
|
static bool CheckUsedRange(addr_t test,
|
|
addr_t from_unaligned,
|
|
size_t size_unaligned,
|
|
size_t* dist_ptr)
|
|
{
|
|
addr_t from = Page::AlignDown(from_unaligned);
|
|
size_unaligned += from_unaligned - from;
|
|
size_t size = Page::AlignUp(size_unaligned);
|
|
if ( from <= test && test < from + size )
|
|
return *dist_ptr = from + size - test, true;
|
|
if ( test < from && from - test < *dist_ptr )
|
|
*dist_ptr = from - test;
|
|
return false;
|
|
}
|
|
|
|
static bool CheckUsedString(addr_t test,
|
|
const char* string,
|
|
size_t* dist_ptr)
|
|
{
|
|
size_t size = strlen(string) + 1;
|
|
return CheckUsedRange(test, (addr_t) string, size, dist_ptr);
|
|
}
|
|
|
|
static bool CheckUsedRanges(multiboot_info_t* bootinfo,
|
|
addr_t test,
|
|
size_t* dist_ptr)
|
|
{
|
|
addr_t kernel_end = (addr_t) &end;
|
|
if ( CheckUsedRange(test, 0, kernel_end, dist_ptr) )
|
|
return true;
|
|
|
|
if ( CheckUsedRange(test, (addr_t) bootinfo, sizeof(*bootinfo), dist_ptr) )
|
|
return true;
|
|
|
|
const char* cmdline = (const char*) (uintptr_t) bootinfo->cmdline;
|
|
if ( CheckUsedString(test, cmdline, dist_ptr) )
|
|
return true;
|
|
|
|
size_t mods_size = bootinfo->mods_count * sizeof(struct multiboot_mod_list);
|
|
if ( CheckUsedRange(test, bootinfo->mods_addr, mods_size, dist_ptr) )
|
|
return true;
|
|
|
|
struct multiboot_mod_list* modules =
|
|
(struct multiboot_mod_list*) (uintptr_t) bootinfo->mods_addr;
|
|
for ( uint32_t i = 0; i < bootinfo->mods_count; i++ )
|
|
{
|
|
struct multiboot_mod_list* module = &modules[i];
|
|
assert(module->mod_start <= module->mod_end);
|
|
size_t mod_size = module->mod_end - module->mod_start;
|
|
if ( CheckUsedRange(test, module->mod_start, mod_size, dist_ptr) )
|
|
return true;
|
|
const char* mod_cmdline = (const char*) (uintptr_t) module->cmdline;
|
|
if ( CheckUsedString(test, mod_cmdline, dist_ptr) )
|
|
return true;
|
|
}
|
|
|
|
if ( CheckUsedRange(test, bootinfo->mmap_addr, bootinfo->mmap_length,
|
|
dist_ptr) )
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
void Init(multiboot_info_t* bootinfo)
|
|
{
|
|
if ( !(bootinfo->flags & MULTIBOOT_INFO_MEM_MAP) )
|
|
Panic("The memory map flag was't set in the multiboot structure.");
|
|
|
|
// If supported, setup the Page Attribute Table feature that allows
|
|
// us to control the memory type (caching) of memory more precisely.
|
|
if ( IsPATSupported() )
|
|
{
|
|
InitializePAT();
|
|
for ( addr_t i = 0; i < PAT_NUM; i++ )
|
|
PAT2PMLFlags[i] = EncodePATAsPMLFlag(i);
|
|
}
|
|
// Otherwise, reroute all requests to the backwards compatible scheme.
|
|
// TODO: Not all early 32-bit x86 CPUs supports these values.
|
|
else
|
|
{
|
|
PAT2PMLFlags[PAT_UC] = PML_WRTHROUGH | PML_NOCACHE;
|
|
PAT2PMLFlags[PAT_WC] = PML_WRTHROUGH | PML_NOCACHE; // Approx.
|
|
PAT2PMLFlags[2] = 0; // No such flag.
|
|
PAT2PMLFlags[3] = 0; // No such flag.
|
|
PAT2PMLFlags[PAT_WT] = PML_WRTHROUGH;
|
|
PAT2PMLFlags[PAT_WP] = PML_WRTHROUGH; // Approx.
|
|
PAT2PMLFlags[PAT_WB] = 0;
|
|
PAT2PMLFlags[PAT_UCM] = PML_NOCACHE;
|
|
}
|
|
|
|
typedef const multiboot_memory_map_t* mmap_t;
|
|
|
|
// Loop over every detected memory region.
|
|
for ( mmap_t mmap = (mmap_t) (addr_t) bootinfo->mmap_addr;
|
|
(addr_t) mmap < bootinfo->mmap_addr + bootinfo->mmap_length;
|
|
mmap = (mmap_t) ((addr_t) mmap + mmap->size + sizeof(mmap->size)) )
|
|
{
|
|
// Check that we can use this kind of RAM.
|
|
if ( mmap->type != 1 )
|
|
continue;
|
|
|
|
// Truncate the memory area if needed.
|
|
uint64_t mmap_addr = mmap->addr;
|
|
uint64_t mmap_len = mmap->len;
|
|
#if defined(__i386__)
|
|
if ( 0xFFFFFFFFULL < mmap_addr )
|
|
continue;
|
|
if ( 0xFFFFFFFFULL < mmap_addr + mmap_len )
|
|
mmap_len = 0x100000000ULL - mmap_addr;
|
|
#endif
|
|
|
|
// Properly page align the entry if needed.
|
|
// TODO: Is the bootloader required to page align this? This could be
|
|
// raw BIOS data that might not be page aligned? But that would
|
|
// be a silly computer.
|
|
addr_t base_unaligned = (addr_t) mmap_addr;
|
|
addr_t base = Page::AlignUp(base_unaligned);
|
|
if ( mmap_len < base - base_unaligned )
|
|
continue;
|
|
size_t length_unaligned = mmap_len - (base - base_unaligned);
|
|
size_t length = Page::AlignDown(length_unaligned);
|
|
if ( !length )
|
|
continue;
|
|
|
|
// Count the amount of usable RAM.
|
|
Page::totalmem += length;
|
|
|
|
// Give all the physical memory to the physical memory allocator
|
|
// but make sure not to give it things we already use.
|
|
addr_t processed = base;
|
|
while ( processed < base + length )
|
|
{
|
|
size_t distance = base + length - processed;
|
|
if ( !CheckUsedRanges(bootinfo, processed, &distance) )
|
|
Page::InitPushRegion(processed, distance);
|
|
processed += distance;
|
|
}
|
|
}
|
|
|
|
// Prepare the non-forkable kernel PMLs such that forking the kernel address
|
|
// space will always keep the kernel mapped.
|
|
for ( size_t i = ENTRIES / 2; i < ENTRIES; i++ )
|
|
{
|
|
PML* const pml = PMLS[TOPPMLLEVEL];
|
|
if ( pml->entry[i] & PML_PRESENT )
|
|
continue;
|
|
|
|
addr_t page = Page::Get(PAGE_USAGE_PAGING_OVERHEAD);
|
|
if ( !page )
|
|
Panic("Out of memory allocating boot PMLs.");
|
|
|
|
pml->entry[i] = page | PML_WRITABLE | PML_PRESENT;
|
|
|
|
// Invalidate the new PML and reset it to zeroes.
|
|
addr_t pmladdr = (addr_t) (PMLS[TOPPMLLEVEL-1] + i);
|
|
InvalidatePage(pmladdr);
|
|
memset((void*) pmladdr, 0, sizeof(PML));
|
|
}
|
|
}
|
|
|
|
void Statistics(size_t* amountused, size_t* totalmem)
|
|
{
|
|
size_t memfree = (Page::stackused - Page::stackreserved) << 12UL;
|
|
size_t memused = Page::totalmem - memfree;
|
|
if ( amountused )
|
|
*amountused = memused;
|
|
if ( totalmem )
|
|
*totalmem = Page::totalmem;
|
|
}
|
|
|
|
} // namespace Memory
|
|
} // namespace Sortix
|
|
|
|
namespace Sortix {
|
|
namespace Page {
|
|
|
|
void PageUsageRegisterUse(addr_t where, enum page_usage usage)
|
|
{
|
|
if ( PAGE_USAGE_NUM_KINDS <= usage )
|
|
return;
|
|
(void) where;
|
|
page_usage_counts[usage]++;
|
|
}
|
|
|
|
void PageUsageRegisterFree(addr_t where, enum page_usage usage)
|
|
{
|
|
if ( PAGE_USAGE_NUM_KINDS <= usage )
|
|
return;
|
|
(void) where;
|
|
assert(page_usage_counts[usage] != 0);
|
|
page_usage_counts[usage]--;
|
|
}
|
|
|
|
void ExtendStack()
|
|
{
|
|
// This call will always succeed, if it didn't, then the stack
|
|
// wouldn't be full, and thus this function won't be called.
|
|
addr_t page = GetUnlocked(PAGE_USAGE_PHYSICAL);
|
|
|
|
// This call will also succeed, since there are plenty of physical
|
|
// pages available and it might need some.
|
|
addr_t virt = (addr_t) (STACK + stacklength);
|
|
if ( !Memory::Map(page, virt, PROT_KREAD | PROT_KWRITE) )
|
|
Panic("Unable to extend page stack, which should have worked");
|
|
|
|
// TODO: This may not be needed during the boot process!
|
|
//Memory::InvalidatePage((addr_t) (STACK + stacklength));
|
|
|
|
stacklength += 4096UL / sizeof(addr_t);
|
|
}
|
|
|
|
void InitPushRegion(addr_t position, size_t length)
|
|
{
|
|
// Align our entries on page boundaries.
|
|
addr_t newposition = Page::AlignUp(position);
|
|
length = Page::AlignDown((position + length) - newposition);
|
|
position = newposition;
|
|
|
|
while ( length )
|
|
{
|
|
if ( unlikely(stackused == stacklength) )
|
|
{
|
|
if ( stackused == MAXSTACKLENGTH )
|
|
{
|
|
pagesnotonstack += length / 4096UL;
|
|
return;
|
|
}
|
|
|
|
ExtendStack();
|
|
}
|
|
|
|
addr_t* stackentry = &(STACK[stackused++]);
|
|
*stackentry = position;
|
|
|
|
length -= 4096UL;
|
|
position += 4096UL;
|
|
}
|
|
}
|
|
|
|
bool ReserveUnlocked(size_t* counter, size_t least, size_t ideal)
|
|
{
|
|
assert(least < ideal);
|
|
size_t available = stackused - stackreserved;
|
|
if ( least < available )
|
|
return errno = ENOMEM, false;
|
|
if ( available < ideal )
|
|
ideal = available;
|
|
stackreserved += ideal;
|
|
*counter += ideal;
|
|
return true;
|
|
}
|
|
|
|
bool Reserve(size_t* counter, size_t least, size_t ideal)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
return ReserveUnlocked(counter, least, ideal);
|
|
}
|
|
|
|
bool ReserveUnlocked(size_t* counter, size_t amount)
|
|
{
|
|
return ReserveUnlocked(counter, amount, amount);
|
|
}
|
|
|
|
bool Reserve(size_t* counter, size_t amount)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
return ReserveUnlocked(counter, amount);
|
|
}
|
|
|
|
addr_t GetReservedUnlocked(size_t* counter, enum page_usage usage)
|
|
{
|
|
if ( !*counter )
|
|
return 0;
|
|
assert(stackused); // After all, we did _reserve_ the memory.
|
|
addr_t result = STACK[--stackused];
|
|
assert(result == AlignDown(result));
|
|
stackreserved--;
|
|
(*counter)--;
|
|
PageUsageRegisterUse(result, usage);
|
|
return result;
|
|
}
|
|
|
|
addr_t GetReserved(size_t* counter, enum page_usage usage)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
return GetReservedUnlocked(counter, usage);
|
|
}
|
|
|
|
addr_t GetUnlocked(enum page_usage usage)
|
|
{
|
|
assert(stackreserved <= stackused);
|
|
if ( unlikely(stackreserved == stackused) )
|
|
return errno = ENOMEM, 0;
|
|
addr_t result = STACK[--stackused];
|
|
assert(result == AlignDown(result));
|
|
PageUsageRegisterUse(result, usage);
|
|
return result;
|
|
}
|
|
|
|
addr_t Get(enum page_usage usage)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
return GetUnlocked(usage);
|
|
}
|
|
|
|
// TODO: This competes with the normal allocation for precious 32-bit pages, we
|
|
// should use different pools for this, and preferably preallocate some
|
|
// 32-bit pages exclusively for driver usage. Also, get proper hardware
|
|
// without these issues.
|
|
addr_t Get32BitUnlocked(enum page_usage usage)
|
|
{
|
|
assert(stackreserved <= stackused);
|
|
if ( unlikely(stackreserved == stackused) )
|
|
return errno = ENOMEM, 0;
|
|
for ( size_t ii = stackused; 0 < ii; ii-- )
|
|
{
|
|
size_t i = ii - 1;
|
|
addr_t result = STACK[i];
|
|
assert(result == AlignDown(result));
|
|
if ( 4 < sizeof(void*) && UINT32_MAX < result )
|
|
continue;
|
|
if ( i + 1 != stackused )
|
|
{
|
|
STACK[i] = STACK[stackused - 1];
|
|
STACK[stackused - 1] = result;
|
|
}
|
|
stackused--;
|
|
PageUsageRegisterUse(result, usage);
|
|
return result;
|
|
}
|
|
return errno = ENOMEM, 0;
|
|
}
|
|
|
|
addr_t Get32Bit(enum page_usage usage)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
return Get32BitUnlocked(usage);
|
|
}
|
|
|
|
void PutUnlocked(addr_t page, enum page_usage usage)
|
|
{
|
|
assert(page == AlignDown(page));
|
|
if ( unlikely(stackused == stacklength) )
|
|
{
|
|
if ( stackused == MAXSTACKLENGTH )
|
|
{
|
|
pagesnotonstack++;
|
|
return;
|
|
}
|
|
ExtendStack();
|
|
}
|
|
STACK[stackused++] = page;
|
|
PageUsageRegisterFree(page, usage);
|
|
}
|
|
|
|
void Put(addr_t page, enum page_usage usage)
|
|
{
|
|
ScopedLock lock(&pagelock);
|
|
PutUnlocked(page, usage);
|
|
}
|
|
|
|
void Lock()
|
|
{
|
|
kthread_mutex_lock(&pagelock);
|
|
}
|
|
|
|
void Unlock()
|
|
{
|
|
kthread_mutex_unlock(&pagelock);
|
|
}
|
|
|
|
} // namespace Page
|
|
} // namespace Sortix
|
|
|
|
namespace Sortix {
|
|
namespace Memory {
|
|
|
|
addr_t ProtectionToPMLFlags(int prot)
|
|
{
|
|
addr_t result = PML_NX;
|
|
if ( prot & PROT_EXEC )
|
|
{
|
|
result |= PML_USERSPACE;
|
|
result &= ~PML_NX;
|
|
}
|
|
if ( prot & PROT_READ )
|
|
result |= PML_USERSPACE;
|
|
if ( prot & PROT_WRITE )
|
|
result |= PML_USERSPACE | PML_WRITABLE;
|
|
if ( prot & PROT_KEXEC )
|
|
result &= ~PML_NX;
|
|
if ( prot & PROT_KREAD )
|
|
result |= 0;
|
|
if ( prot & PROT_KWRITE )
|
|
result |= PML_WRITABLE;
|
|
if ( prot & PROT_FORK )
|
|
result |= PML_FORK;
|
|
return result;
|
|
}
|
|
|
|
int PMLFlagsToProtection(addr_t flags)
|
|
{
|
|
int prot = PROT_KREAD;
|
|
if ( (flags & PML_USERSPACE) && !(flags & PML_NX) )
|
|
prot |= PROT_EXEC;
|
|
if ( (flags & PML_USERSPACE) )
|
|
prot |= PROT_READ;
|
|
if ( (flags & PML_USERSPACE) && (flags & PML_WRITABLE) )
|
|
prot |= PROT_WRITE;
|
|
if ( !(flags & PML_NX) )
|
|
prot |= PROT_KEXEC;
|
|
if ( flags & PML_WRITABLE )
|
|
prot |= PROT_KWRITE;
|
|
if ( flags & PML_FORK )
|
|
prot |= PROT_FORK;
|
|
return prot;
|
|
}
|
|
|
|
int ProvidedProtection(int prot)
|
|
{
|
|
return PMLFlagsToProtection(ProtectionToPMLFlags(prot));
|
|
}
|
|
|
|
bool LookUp(addr_t mapto, addr_t* physical, int* protection)
|
|
{
|
|
// Translate the virtual address into PML indexes.
|
|
const size_t MASK = (1<<TRANSBITS)-1;
|
|
size_t pmlchildid[TOPPMLLEVEL + 1];
|
|
for ( size_t i = 1; i <= TOPPMLLEVEL; i++ )
|
|
pmlchildid[i] = mapto >> (12 + (i-1) * TRANSBITS) & MASK;
|
|
|
|
int prot = PROT_USER | PROT_KERNEL | PROT_FORK;
|
|
|
|
// For each PML level, make sure it exists.
|
|
size_t offset = 0;
|
|
for ( size_t i = TOPPMLLEVEL; i > 1; i-- )
|
|
{
|
|
size_t childid = pmlchildid[i];
|
|
PML* pml = PMLS[i] + offset;
|
|
|
|
addr_t entry = pml->entry[childid];
|
|
if ( !(entry & PML_PRESENT) )
|
|
return false;
|
|
addr_t entryflags = entry & ~PML_ADDRESS;
|
|
int entryprot = PMLFlagsToProtection(entryflags);
|
|
prot &= entryprot;
|
|
|
|
// Find the index of the next PML in the fractal mapped memory.
|
|
offset = offset * ENTRIES + childid;
|
|
}
|
|
|
|
addr_t entry = (PMLS[1] + offset)->entry[pmlchildid[1]];
|
|
if ( !(entry & PML_PRESENT) )
|
|
return false;
|
|
|
|
addr_t entryflags = entry & ~PML_ADDRESS;
|
|
int entryprot = PMLFlagsToProtection(entryflags);
|
|
prot &= entryprot;
|
|
addr_t phys = entry & PML_ADDRESS;
|
|
|
|
if ( physical )
|
|
*physical = phys;
|
|
if ( protection )
|
|
*protection = prot;
|
|
|
|
return true;
|
|
}
|
|
|
|
void InvalidatePage(addr_t /*addr*/)
|
|
{
|
|
// TODO: Actually just call the instruction.
|
|
Flush();
|
|
}
|
|
|
|
|
|
addr_t GetAddressSpace()
|
|
{
|
|
addr_t result;
|
|
asm ( "mov %%cr3, %0" : "=r"(result) );
|
|
return result;
|
|
}
|
|
|
|
addr_t SwitchAddressSpace(addr_t addrspace)
|
|
{
|
|
assert(Page::IsAligned(addrspace));
|
|
|
|
addr_t previous = GetAddressSpace();
|
|
asm volatile ( "mov %0, %%cr3" : : "r"(addrspace) );
|
|
return previous;
|
|
}
|
|
|
|
void Flush()
|
|
{
|
|
addr_t previous;
|
|
asm ( "mov %%cr3, %0" : "=r"(previous) );
|
|
asm volatile ( "mov %0, %%cr3" : : "r"(previous) );
|
|
}
|
|
|
|
bool MapRange(addr_t where, size_t bytes, int protection, enum page_usage usage)
|
|
{
|
|
for ( addr_t page = where; page < where + bytes; page += 4096UL )
|
|
{
|
|
addr_t physicalpage = Page::Get(usage);
|
|
if ( physicalpage == 0 )
|
|
{
|
|
while ( where < page )
|
|
{
|
|
page -= 4096UL;
|
|
physicalpage = Unmap(page);
|
|
Page::Put(physicalpage, usage);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Map(physicalpage, page, protection);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool UnmapRange(addr_t where, size_t bytes, enum page_usage usage)
|
|
{
|
|
for ( addr_t page = where; page < where + bytes; page += 4096UL )
|
|
{
|
|
addr_t physicalpage = Unmap(page);
|
|
Page::Put(physicalpage, usage);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool MapInternal(addr_t physical, addr_t mapto, int prot, addr_t extraflags = 0)
|
|
{
|
|
addr_t flags = ProtectionToPMLFlags(prot) | PML_PRESENT;
|
|
|
|
// Translate the virtual address into PML indexes.
|
|
const size_t MASK = (1<<TRANSBITS)-1;
|
|
size_t pmlchildid[TOPPMLLEVEL + 1];
|
|
for ( size_t i = 1; i <= TOPPMLLEVEL; i++ )
|
|
pmlchildid[i] = mapto >> (12 + (i-1) * TRANSBITS) & MASK;
|
|
|
|
// For each PML level, make sure it exists.
|
|
size_t offset = 0;
|
|
for ( size_t i = TOPPMLLEVEL; i > 1; i-- )
|
|
{
|
|
size_t childid = pmlchildid[i];
|
|
PML* pml = PMLS[i] + offset;
|
|
|
|
addr_t& entry = pml->entry[childid];
|
|
|
|
// Find the index of the next PML in the fractal mapped memory.
|
|
size_t childoffset = offset * ENTRIES + childid;
|
|
|
|
if ( !(entry & PML_PRESENT) )
|
|
{
|
|
// TODO: Possible memory leak when page allocation fails.
|
|
addr_t page = Page::Get(PAGE_USAGE_PAGING_OVERHEAD);
|
|
|
|
if ( !page )
|
|
return false;
|
|
addr_t pmlflags = PML_PRESENT | PML_WRITABLE | PML_USERSPACE
|
|
| PML_FORK;
|
|
entry = page | pmlflags;
|
|
|
|
// Invalidate the new PML and reset it to zeroes.
|
|
addr_t pmladdr = (addr_t) (PMLS[i-1] + childoffset);
|
|
InvalidatePage(pmladdr);
|
|
memset((void*) pmladdr, 0, sizeof(PML));
|
|
}
|
|
|
|
offset = childoffset;
|
|
}
|
|
|
|
// Actually map the physical page to the virtual page.
|
|
const addr_t entry = physical | flags | extraflags;
|
|
(PMLS[1] + offset)->entry[pmlchildid[1]] = entry;
|
|
return true;
|
|
}
|
|
|
|
bool Map(addr_t physical, addr_t mapto, int prot)
|
|
{
|
|
return MapInternal(physical, mapto, prot);
|
|
}
|
|
|
|
void PageProtect(addr_t mapto, int protection)
|
|
{
|
|
addr_t phys;
|
|
if ( !LookUp(mapto, &phys, NULL) )
|
|
return;
|
|
Map(phys, mapto, protection);
|
|
}
|
|
|
|
void PageProtectAdd(addr_t mapto, int protection)
|
|
{
|
|
addr_t phys;
|
|
int prot;
|
|
if ( !LookUp(mapto, &phys, &prot) )
|
|
return;
|
|
prot |= protection;
|
|
Map(phys, mapto, prot);
|
|
}
|
|
|
|
void PageProtectSub(addr_t mapto, int protection)
|
|
{
|
|
addr_t phys;
|
|
int prot;
|
|
if ( !LookUp(mapto, &phys, &prot) )
|
|
return;
|
|
prot &= ~protection;
|
|
Map(phys, mapto, prot);
|
|
}
|
|
|
|
addr_t Unmap(addr_t mapto)
|
|
{
|
|
// Translate the virtual address into PML indexes.
|
|
const size_t MASK = (1<<TRANSBITS)-1;
|
|
size_t pmlchildid[TOPPMLLEVEL + 1];
|
|
for ( size_t i = 1; i <= TOPPMLLEVEL; i++ )
|
|
{
|
|
pmlchildid[i] = mapto >> (12 + (i-1) * TRANSBITS) & MASK;
|
|
}
|
|
|
|
// For each PML level, make sure it exists.
|
|
size_t offset = 0;
|
|
for ( size_t i = TOPPMLLEVEL; i > 1; i-- )
|
|
{
|
|
size_t childid = pmlchildid[i];
|
|
PML* pml = PMLS[i] + offset;
|
|
|
|
addr_t& entry = pml->entry[childid];
|
|
|
|
if ( !(entry & PML_PRESENT) )
|
|
PanicF("Attempted to unmap virtual page 0x%jX, but the virtual"
|
|
" page was wasn't mapped. This is a bug in the code "
|
|
"code calling this function", (uintmax_t) mapto);
|
|
|
|
// Find the index of the next PML in the fractal mapped memory.
|
|
offset = offset * ENTRIES + childid;
|
|
}
|
|
|
|
addr_t& entry = (PMLS[1] + offset)->entry[pmlchildid[1]];
|
|
addr_t result = entry & PML_ADDRESS;
|
|
entry = 0;
|
|
|
|
// TODO: If all the entries in PML[N] are not-present, then who
|
|
// unmaps its entry from PML[N-1]?
|
|
|
|
return result;
|
|
}
|
|
|
|
bool MapPAT(addr_t physical, addr_t mapto, int prot, addr_t mtype)
|
|
{
|
|
addr_t extraflags = PAT2PMLFlags[mtype];
|
|
return MapInternal(physical, mapto, prot, extraflags);
|
|
}
|
|
|
|
void ForkCleanup(size_t i, size_t level)
|
|
{
|
|
PML* destpml = FORKPML + level;
|
|
if ( !i )
|
|
return;
|
|
for ( size_t n = 0; n < i-1; n++ )
|
|
{
|
|
addr_t entry = destpml->entry[i];
|
|
if ( !(entry & PML_FORK ) )
|
|
continue;
|
|
addr_t phys = entry & PML_ADDRESS;
|
|
if ( 1 < level )
|
|
{
|
|
addr_t destaddr = (addr_t) (FORKPML + level-1);
|
|
Map(phys, destaddr, PROT_KREAD | PROT_KWRITE);
|
|
InvalidatePage(destaddr);
|
|
ForkCleanup(ENTRIES+1UL, level-1);
|
|
}
|
|
enum page_usage usage = 1 < level ? PAGE_USAGE_PAGING_OVERHEAD
|
|
: PAGE_USAGE_USER_SPACE;
|
|
Page::Put(phys, usage);
|
|
}
|
|
}
|
|
|
|
// TODO: Copying every frame is endlessly useless in many uses. It'd be
|
|
// nice to upgrade this to a copy-on-write algorithm.
|
|
bool Fork(size_t level, size_t pmloffset)
|
|
{
|
|
PML* destpml = FORKPML + level;
|
|
for ( size_t i = 0; i < ENTRIES; i++ )
|
|
{
|
|
addr_t entry = (PMLS[level] + pmloffset)->entry[i];
|
|
|
|
// Link the entry if it isn't supposed to be forked.
|
|
if ( !(entry & PML_PRESENT) || !(entry & PML_FORK ) )
|
|
{
|
|
destpml->entry[i] = entry;
|
|
continue;
|
|
}
|
|
|
|
enum page_usage usage = 1 < level ? PAGE_USAGE_PAGING_OVERHEAD
|
|
: PAGE_USAGE_USER_SPACE;
|
|
addr_t phys = Page::Get(usage);
|
|
if ( unlikely(!phys) )
|
|
{
|
|
ForkCleanup(i, level);
|
|
return false;
|
|
}
|
|
|
|
addr_t flags = entry & PML_FLAGS;
|
|
destpml->entry[i] = phys | flags;
|
|
|
|
// Map the destination page.
|
|
addr_t destaddr = (addr_t) (FORKPML + level-1);
|
|
Map(phys, destaddr, PROT_KREAD | PROT_KWRITE);
|
|
InvalidatePage(destaddr);
|
|
|
|
size_t offset = pmloffset * ENTRIES + i;
|
|
|
|
if ( 1 < level )
|
|
{
|
|
if ( !Fork(level-1, offset) )
|
|
{
|
|
Page::Put(phys, usage);
|
|
ForkCleanup(i, level);
|
|
return false;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// Determine the source page's address.
|
|
const void* src = (const void*) (offset * 4096UL);
|
|
|
|
// Determine the destination page's address.
|
|
void* dest = (void*) (FORKPML + level - 1);
|
|
|
|
memcpy(dest, src, 4096UL);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Fork(addr_t dir, size_t level, size_t pmloffset)
|
|
{
|
|
PML* destpml = FORKPML + level;
|
|
|
|
// This call always succeeds.
|
|
Map(dir, (addr_t) destpml, PROT_KREAD | PROT_KWRITE);
|
|
InvalidatePage((addr_t) destpml);
|
|
|
|
return Fork(level, pmloffset);
|
|
}
|
|
|
|
// Create an exact copy of the current address space.
|
|
addr_t Fork()
|
|
{
|
|
addr_t dir = Page::Get(PAGE_USAGE_PAGING_OVERHEAD);
|
|
if ( dir == 0 )
|
|
return 0;
|
|
if ( !Fork(dir, TOPPMLLEVEL, 0) )
|
|
{
|
|
Page::Put(dir, PAGE_USAGE_PAGING_OVERHEAD);
|
|
return 0;
|
|
}
|
|
|
|
// Now, the new top pml needs to have its fractal memory fixed.
|
|
const addr_t flags = PML_PRESENT | PML_WRITABLE;
|
|
addr_t mapto;
|
|
addr_t childaddr;
|
|
|
|
(FORKPML + TOPPMLLEVEL)->entry[ENTRIES-1] = dir | flags;
|
|
childaddr = (FORKPML + TOPPMLLEVEL)->entry[ENTRIES-2] & PML_ADDRESS;
|
|
|
|
for ( size_t i = TOPPMLLEVEL-1; i > 0; i-- )
|
|
{
|
|
mapto = (addr_t) (FORKPML + i);
|
|
Map(childaddr, mapto, PROT_KREAD | PROT_KWRITE);
|
|
InvalidatePage(mapto);
|
|
(FORKPML + i)->entry[ENTRIES-1] = dir | flags;
|
|
childaddr = (FORKPML + i)->entry[ENTRIES-2] & PML_ADDRESS;
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
} // namespace Memory
|
|
} // namespace Sortix
|