From 74247eb71e35c03db6dd89c63743fb7fca86afd7 Mon Sep 17 00:00:00 2001 From: Jonas 'Sortie' Termansen Date: Wed, 23 Oct 2013 00:40:10 +0200 Subject: [PATCH] Rewrite malloc(3). --- libc/Makefile | 10 +- libc/include/malloc.h | 613 ++++++++++++++++- libc/malloc/__heap_expand_current_part.cpp | 183 ++++++ libc/malloc/__heap_lock.cpp | 47 ++ libc/malloc/__heap_unlock.cpp | 47 ++ libc/malloc/__heap_verify.cpp | 42 ++ libc/malloc/heap_get_paranoia.cpp | 45 ++ libc/malloc/heap_init.cpp | 41 ++ libc/stdlib/free.cpp | 57 ++ libc/stdlib/heap.cpp | 729 --------------------- libc/stdlib/malloc.cpp | 104 +++ libc/stdlib/realloc.cpp | 127 ++++ 12 files changed, 1312 insertions(+), 733 deletions(-) create mode 100644 libc/malloc/__heap_expand_current_part.cpp create mode 100644 libc/malloc/__heap_lock.cpp create mode 100644 libc/malloc/__heap_unlock.cpp create mode 100644 libc/malloc/__heap_verify.cpp create mode 100644 libc/malloc/heap_get_paranoia.cpp create mode 100644 libc/malloc/heap_init.cpp create mode 100644 libc/stdlib/free.cpp delete mode 100644 libc/stdlib/heap.cpp create mode 100644 libc/stdlib/malloc.cpp create mode 100644 libc/stdlib/realloc.cpp diff --git a/libc/Makefile b/libc/Makefile index 6dee5275..82b39e99 100644 --- a/libc/Makefile +++ b/libc/Makefile @@ -56,6 +56,12 @@ inttypes/strtoimax.o \ inttypes/strtoumax.o \ inttypes/wcstoimax.o \ inttypes/wcstoumax.o \ +malloc/__heap_expand_current_part.o \ +malloc/heap_get_paranoia.o \ +malloc/heap_init.o \ +malloc/__heap_lock.o \ +malloc/__heap_unlock.o \ +malloc/__heap_verify.o \ signal/sigaddset.o \ signal/sigandset.o \ signal/sigdelset.o \ @@ -162,16 +168,18 @@ stdlib/atol.o \ stdlib/bsearch.o \ stdlib/calloc.o \ stdlib/div.o \ -stdlib/heap.o \ +stdlib/free.o \ stdlib/labs.o \ stdlib/ldiv.o \ stdlib/llabs.o \ stdlib/lldiv.o \ +stdlib/malloc.o \ stdlib/mblen.o \ stdlib/mbstowcs.o \ stdlib/mbtowc.o \ stdlib/qsort.o \ stdlib/qsort_r.o \ +stdlib/realloc.o \ stdlib/strtod.o \ stdlib/strtof.o \ stdlib/strtold.o \ diff --git a/libc/include/malloc.h b/libc/include/malloc.h index 2006ead4..07603ab0 100644 --- a/libc/include/malloc.h +++ b/libc/include/malloc.h @@ -1,6 +1,6 @@ /******************************************************************************* - Copyright(C) Jonas 'Sortie' Termansen 2012, 2014. + Copyright(C) Jonas 'Sortie' Termansen 2012, 2013, 2014. This file is part of the Sortix C Library. @@ -22,13 +22,620 @@ *******************************************************************************/ -#ifndef _MALLOC_H -#define _MALLOC_H 1 +#ifndef INCLUDE_MALLOC_H +#define INCLUDE_MALLOC_H #include +#if __is_sortix_libc +#include <__/wordsize.h> +#endif + +#if __is_sortix_libc +#include +#if !defined(__cplusplus) +#include +#include +#endif +#include +#include +#endif + __BEGIN_DECLS +int heap_get_paranoia(void); +/* TODO: Operations to verify pointers and consistency check the heap. */ + +/* NOTE: The following declarations are heap internals and are *NOT* part of the + API or ABI in any way. You should in no way depend on this information, + except perhaps for debugging purposes. */ +#if __is_sortix_libc + +/* This macro governs how paranoid the malloc implementation is. It's not a + constant as it may be a runtime function call deciding the level. + Level 0 - disables all automatic consistency checks. + Level 1 - is afraid that the caller may have corrupted the chunks passed to + free and realloc and possibly their neighbors if chunk unification + is considered. + Level 2 - is afraid the heap has been damaged, so it verifies the entire heap + before all heap operatations. + Level 3 - is afraid that malloc itself is buggy and consistency checks the + entire heap both before and after all heap operations. */ +#if 1 /* Constant paranoia, better for code optimization. */ +#define PARANOIA 1 +#else /* Dynamic paranoia, better for debugging. */ +#define PARANOIA_DEFAULT 1 +#define PARANOIA heap_get_paranoia() +#endif + +/* TODO: Proper logic here. */ +#if !defined(PARANOIA_DEFAULT) +#if PARANOIA < 3 +#undef assert +#define assert(x) do { ((void) 0); } while ( 0 ) +#define HEAP_NO_ASSERT +#endif +#endif + +/* A magic value to identify a heap part edge. The value is selected such + that it is not appropriately aligned like a real heap chunk pointer, such + that there are no ambiguous cases. */ +#if __WORDSIZE == 32 +#define HEAP_PART_MAGIC 0xBEEFBEEF +#elif __WORDSIZE == 64 +#define HEAP_PART_MAGIC 0xBEEFBEEFBEEFBEEF +#else +#warning "You need to implement HEAP_CHUNK_MAGIC for your native word width" +#endif + +/* A magic value to detect wheter a chunk is used. The value is selected such + that it is not appropriately aligned like a real heap chunk pointer, such + that there are no ambiguous cases. */ +#if __WORDSIZE == 32 +#define HEAP_CHUNK_MAGIC 0xDEADDEAD +#elif __WORDSIZE == 64 +#define HEAP_CHUNK_MAGIC 0xDEADDEADDEADDEAD +#else +#warning "You need to implement HEAP_CHUNK_MAGIC for your native word width" +#endif + +/* The heap is split into a number of parts that each consists of a number of + of chunks (used and unused). The heap normally is just a single part, but if + the address space gets too fragmented, it may not be possible to extend the + existing part. In each part of the heap, there exists no neighboring chunks + that are both unused (they would be combined into a single chunk. It is + possible to fully and unambiguously traverse the entire heap by following it + as a linked list of objects (sometimes with implied relative locations). */ + +/* This structure describes the current heap allocation state. */ +struct heap_state +{ + struct heap_part* current_part; + struct heap_chunk* bin[sizeof(size_t) * 8]; + size_t bin_filled_bitmap; +}; + +/* This structure is at the very beginning of each heap part. The size variable + includes the size of the surrounding structures. The first chunk or the end + of the opart follows immediately (use the magic value to determine which). */ +struct heap_part +{ + size_t unused[3]; /* Alignment. */ + struct heap_part* part_next; + size_t part_magic; + size_t part_size; +}; + +static_assert(sizeof(struct heap_part) * 8 == 6 * __WORDSIZE, + "sizeof(struct heap_part) * 8 == 6 * __WORDSIZE"); +static_assert(alignof(struct heap_part) * 8 == __WORDSIZE, + "alignof(struct heap_part) * 8 == __WORDSIZE"); + +/* This structure immediately preceeds all heap allocations. The size variable + includes the size of the surrounding structures. */ +struct heap_chunk +{ + size_t chunk_size; + union + { + size_t chunk_magic; + struct heap_chunk* chunk_next; + }; +}; + +static_assert(sizeof(struct heap_chunk) * 8 == 2 * __WORDSIZE, + "sizeof(struct heap_chunk) * 8 == 2 * __WORDSIZE"); +static_assert(alignof(struct heap_chunk) * 8 == __WORDSIZE, + "alignof(struct heap_chunk) * 8 == __WORDSIZE"); + +/* Inbetween these structures come the heap allocation. */ + +/* This structure immediately follows all heap allocations. The size variable + includes the size of the surrounding structures. */ +struct heap_chunk_post +{ + union + { + size_t chunk_magic; + struct heap_chunk* chunk_prev; + }; + size_t chunk_size; +}; + +static_assert(sizeof(struct heap_chunk_post) * 8 == 2 * __WORDSIZE, + "sizeof(struct heap_chunk_post) * 8 == 2 * __WORDSIZE"); +static_assert(alignof(struct heap_chunk_post) * 8 == __WORDSIZE, + "alignof(struct heap_chunk_post) * 8 == __WORDSIZE"); + +/* This structure is at the very end of each heap part. The size variable + includes the size of the surrounding structures. The first chunk in the heap + part follows immediately. */ +struct heap_part_post +{ + size_t part_size; + size_t part_magic; + struct heap_part* part_prev; + size_t unused[3]; /* Alignment. */ +}; + + +static_assert(sizeof(struct heap_part_post) * 8 == 6 * __WORDSIZE, + "sizeof(struct heap_part_post) * 8 == 6 * __WORDSIZE"); +static_assert(alignof(struct heap_part_post) * 8 == __WORDSIZE, + "alignof(struct heap_part_post) * 8 == __WORDSIZE"); + +/* Global secret variables used internally by the heap. */ +extern struct heap_state __heap_state; + +/* Internal heap functions. */ +bool __heap_expand_current_part(size_t); +void __heap_lock(void); +void __heap_unlock(void); +#if !defined(HEAP_NO_ASSERT) +void __heap_verify(void); +#endif + +#if !defined(HEAP_NO_ASSERT) +/* Utility function to verify addresses are well-aligned. */ +__attribute__((unused)) static inline +bool heap_is_pointer_aligned(void* addr, size_t size, size_t alignment) +{ + /* It is assumed that alignment is a power of two. */ + if ( ((uintptr_t) addr) & (alignment - 1) ) + return false; + if ( ((uintptr_t) size) & (alignment - 1) ) + return false; + return true; +} + +#define HEAP_IS_POINTER_ALIGNED(object, size) \ + heap_is_pointer_aligned((object), (size), alignof(*(object))) + +/* Utility function to trap bad memory accesses. */ +__attribute__((unused)) static inline +void heap_test_usable_memory(void* addr, size_t size) +{ + for ( size_t i = 0; i < size; i++ ) + (void) ((volatile unsigned char*) addr)[i]; +} +#endif + +/* Utility functions for accessing the most significant bit set. */ +__attribute__((unused)) static inline +size_t heap_bsr(size_t value) +{ + /* TODO: There currently is no builtin bsr function. */ + return (sizeof(size_t) * 8 - 1) - __builtin_clzl(value); +} + +/* Utility functions for accessing the least significant bit set. */ +__attribute__((unused)) static inline +size_t heap_bsf(size_t value) +{ + return __builtin_ctzl(value); +} + +/* Utility function for getting the minimum size of entries in a bin. */ +__attribute__((unused)) static inline +size_t heap_size_of_bin(size_t bin) +{ + assert(bin < sizeof(size_t) * 8); + return (size_t) 1 << bin; +} + +/* Utility function for determining whether a size even has a bin. */ +__attribute__((unused)) static inline +bool heap_size_has_bin(size_t size) +{ + return size <= heap_size_of_bin(sizeof(size_t) * 8 - 1); +} + +/* Utility function for determining which bin a particular size belongs to. */ +__attribute__((unused)) static inline +size_t heap_chunk_size_to_bin(size_t size) +{ + assert(size != 0); + assert(heap_size_has_bin(size)); + + for ( size_t i = 8 * sizeof(size_t) - 1; true; i-- ) + if ( heap_size_of_bin(i) <= size ) + return i; + + assert(false); + __builtin_unreachable(); +} + +/* Utility function for determining the smallest bin from which a allocation can + be satisfied. This is not the same as heap_chunk_size_to_bin() as it rounds + downwards while this rounds upwards. */ +__attribute__((unused)) static inline +size_t heap_bin_for_allocation(size_t size) +{ + assert(heap_size_has_bin(size)); + + /* TODO: Use the bsr or bsf instructions here whatever it was! */ + + for ( size_t i = 0; i < 8 * sizeof(size_t); i++ ) + if ( size <= (size_t) 1 << i ) + return i; + + assert(false); + __builtin_unreachable(); +} + +/* Rounds an allocation size up to a multiple of what malloc offers. */ +__attribute__((unused)) static inline size_t heap_align(size_t value) +{ + static_assert(sizeof(struct heap_chunk) + sizeof(struct heap_chunk_post) == 4 * __WORDSIZE / 8, + "sizeof(struct heap_chunk) + sizeof(struct heap_chunk_post) == 4 * __WORDSIZE / 8"); + + size_t mask = 4 * sizeof(size_t) - 1; + return -(-value & ~mask); +} + +/* Returns the trailing structure following the given part. */ +__attribute__((unused)) static inline +struct heap_part_post* heap_part_to_post(struct heap_part* part) +{ + assert(HEAP_IS_POINTER_ALIGNED(part, part->part_size)); + + return (struct heap_part_post*) + ((uintptr_t) part + part->part_size - sizeof(struct heap_part_post)); +} + +/* Returns the part associated with the part trailing structure. */ +__attribute__((unused)) static inline +struct heap_part* heap_post_to_part(struct heap_part_post* post_part) +{ + assert(HEAP_IS_POINTER_ALIGNED(post_part, post_part->part_size)); + + return (struct heap_part*) + ((uintptr_t) post_part - post_part->part_size + sizeof(struct heap_part)); +} + +/* Returns the ending address of a heap part. */ +__attribute__((unused)) static inline +void* heap_part_end(struct heap_part* part) +{ + assert(HEAP_IS_POINTER_ALIGNED(part, part->part_size)); + + return (uint8_t*) part + part->part_size; +} + +/* Returns whether this chunk is currently in use. */ +__attribute__((unused)) static inline +bool heap_chunk_is_used(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + return chunk->chunk_magic == HEAP_CHUNK_MAGIC; +} + +/* Returns the trailing structure following the given chunk. */ +__attribute__((unused)) static inline +struct heap_chunk_post* heap_chunk_to_post(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + return (struct heap_chunk_post*) + ((uintptr_t) chunk + chunk->chunk_size - sizeof(struct heap_chunk_post)); +} + +/* Returns the chunk associated with the chunk trailing structure. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_post_to_chunk(struct heap_chunk_post* post_chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(post_chunk, post_chunk->chunk_size)); + + return (struct heap_chunk*) + ((uintptr_t) post_chunk - post_chunk->chunk_size + sizeof(struct heap_chunk)); +} + +/* Returns the data inside the chunk. */ +__attribute__((unused)) static inline +uint8_t* heap_chunk_to_data(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + return (uint8_t*) chunk + sizeof(struct heap_chunk); +} + +/* Returns the data inside the chunk. */ +__attribute__((unused)) static inline +size_t heap_chunk_data_size(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + return chunk->chunk_size - (sizeof(struct heap_chunk) + sizeof(struct heap_chunk_post)); +} + +/* Returns the chunk whose data starts at the given address. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_data_to_chunk(uint8_t* data) +{ + struct heap_chunk* chunk = + (struct heap_chunk*) (data - sizeof(struct heap_chunk)); + + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + return chunk; +} + +/* Returns the chunk to the left of this one or NULL if none. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_chunk_left(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + struct heap_chunk_post* left_post = (struct heap_chunk_post*) + ((uintptr_t) chunk - sizeof(struct heap_chunk_post)); + assert(HEAP_IS_POINTER_ALIGNED(left_post, 0)); + + if ( left_post->chunk_magic == HEAP_PART_MAGIC ) + return NULL; + return heap_post_to_chunk(left_post); +} + +/* Returns the chunk to the right of this one or NULL if none. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_chunk_right(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + struct heap_chunk* right = (struct heap_chunk*) + ((uintptr_t) chunk + chunk->chunk_size); + assert(HEAP_IS_POINTER_ALIGNED(right, 0)); + + if ( right->chunk_magic == HEAP_PART_MAGIC ) + return NULL; + return right; +} + +/* Formats an used chunk at the given location. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_chunk_format(uint8_t* addr, size_t size) +{ + assert(heap_is_pointer_aligned(addr, size, alignof(struct heap_chunk))); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(addr, size); +#endif + + struct heap_chunk* chunk = (struct heap_chunk*) addr; + chunk->chunk_size = size; + chunk->chunk_magic = HEAP_CHUNK_MAGIC; + struct heap_chunk_post* post = heap_chunk_to_post(chunk); + post->chunk_size = size; + post->chunk_magic = HEAP_CHUNK_MAGIC; + return chunk; +} + +/* Returns the first chunk in a part. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_part_first_chunk(struct heap_part* part) +{ + assert(HEAP_IS_POINTER_ALIGNED(part, part->part_size)); + + struct heap_chunk* chunk = + (struct heap_chunk*) ((uintptr_t) part + sizeof(struct heap_part)); + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + + if ( chunk->chunk_magic == HEAP_PART_MAGIC ) + return NULL; + return chunk; +} + +/* Returns the last chunk in a part. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_part_last_chunk(struct heap_part* part) +{ + assert(HEAP_IS_POINTER_ALIGNED(part, part->part_size)); + + struct heap_part_post* post = heap_part_to_post(part); + assert(HEAP_IS_POINTER_ALIGNED(post, post->part_size)); + + struct heap_chunk_post* chunk_post = + (struct heap_chunk_post*) ((uintptr_t) post - sizeof(struct heap_chunk_post)); + assert(HEAP_IS_POINTER_ALIGNED(chunk_post, chunk_post->chunk_size)); + + if ( chunk_post->chunk_magic == HEAP_PART_MAGIC ) + return NULL; + return heap_post_to_chunk(chunk_post); +} + +/* Inserts a chunk into the heap and marks it as unused. */ +__attribute__((unused)) static inline +void heap_insert_chunk(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + struct heap_chunk_post* chunk_post = heap_chunk_to_post(chunk); + assert(HEAP_IS_POINTER_ALIGNED(chunk_post, chunk_post->chunk_size)); + assert(chunk->chunk_size == chunk_post->chunk_size); + + /* Decide which bin this chunk belongs to. */ + assert(heap_size_has_bin(chunk->chunk_size)); + size_t chunk_bin = heap_chunk_size_to_bin(chunk->chunk_size); + + /* Insert the chunk into this bin, destroying its magic values as used. */ + if ( __heap_state.bin[chunk_bin] ) + heap_chunk_to_post(__heap_state.bin[chunk_bin])->chunk_prev = chunk; + chunk->chunk_next = __heap_state.bin[chunk_bin]; + chunk_post->chunk_prev = NULL; + __heap_state.bin[chunk_bin] = chunk; + __heap_state.bin_filled_bitmap |= heap_size_of_bin(chunk_bin); +} + +/* Removes a chunk from the heap and marks it as used. */ +__attribute__((unused)) static inline +void heap_remove_chunk(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + assert(chunk->chunk_magic != HEAP_CHUNK_MAGIC); + assert(chunk->chunk_magic != HEAP_PART_MAGIC); + struct heap_chunk_post* chunk_post = heap_chunk_to_post(chunk); + assert(HEAP_IS_POINTER_ALIGNED(chunk_post, chunk_post->chunk_size)); + assert(chunk->chunk_size == chunk_post->chunk_size); + assert(chunk_post->chunk_magic != HEAP_CHUNK_MAGIC); + assert(chunk_post->chunk_magic != HEAP_PART_MAGIC); + + /* Decide which bin this chunk belongs to. */ + assert(heap_size_has_bin(chunk->chunk_size)); + size_t chunk_bin = heap_chunk_size_to_bin(chunk->chunk_size); + assert(__heap_state.bin_filled_bitmap & heap_size_of_bin(chunk_bin)); + + /* Unlink the chunk from its current bin. */ + if ( chunk == __heap_state.bin[chunk_bin] ) + { + assert(!chunk_post->chunk_prev); + if ( !(__heap_state.bin[chunk_bin] = chunk->chunk_next) ) + __heap_state.bin_filled_bitmap ^= heap_size_of_bin(chunk_bin); + } + else + { + assert(chunk_post->chunk_prev); + heap_chunk_to_post(chunk)->chunk_prev->chunk_next = chunk->chunk_next; + } + if ( chunk->chunk_next ) + heap_chunk_to_post(chunk->chunk_next)->chunk_prev = chunk_post->chunk_prev; + + /* Mark the chunk as used. */ + chunk->chunk_magic = HEAP_CHUNK_MAGIC; + chunk_post->chunk_magic = HEAP_CHUNK_MAGIC; +} + +/* Decides whether the chunk can be split into two. */ +__attribute__((unused)) static inline +bool heap_can_split_chunk(struct heap_chunk* chunk, size_t needed) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); + assert(needed <= chunk->chunk_size); + + size_t minimum_chunk_size = sizeof(struct heap_chunk) + + sizeof(struct heap_chunk_post); + return minimum_chunk_size <= chunk->chunk_size - needed; +} + +/* Splits a chunk into two - assumes it can be split. */ +__attribute__((unused)) static inline +void heap_split_chunk(struct heap_chunk* chunk, size_t needed) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(chunk, chunk->chunk_size); +#endif + assert(heap_can_split_chunk(chunk, needed)); + + bool chunk_is_used = heap_chunk_is_used(chunk); + if ( !chunk_is_used ) + heap_remove_chunk(chunk); + size_t surplus_amount = chunk->chunk_size - needed; + heap_chunk_format((uint8_t*) chunk, needed); + struct heap_chunk* surplus = + heap_chunk_format((uint8_t*) chunk + needed, surplus_amount); + if ( !chunk_is_used ) + heap_insert_chunk(chunk); + heap_insert_chunk(surplus); +} + +/* Combine a chunk with its neighbors if they are all unused. */ +__attribute__((unused)) static inline +struct heap_chunk* heap_chunk_combine_neighbors(struct heap_chunk* chunk) +{ + assert(HEAP_IS_POINTER_ALIGNED(chunk, chunk->chunk_size)); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(chunk, chunk->chunk_size); +#endif + + struct heap_chunk* left_chunk = heap_chunk_left(chunk); + struct heap_chunk* right_chunk = heap_chunk_right(chunk); + if ( right_chunk ) + { +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(right_chunk, right_chunk->chunk_size); +#endif + assert(HEAP_IS_POINTER_ALIGNED(right_chunk, right_chunk->chunk_size)); + } + + /* Attempt to combine with both the left and right chunk. */ + if ( (left_chunk && !heap_chunk_is_used(left_chunk)) && + (right_chunk && !heap_chunk_is_used(right_chunk)) ) + { + assert(HEAP_IS_POINTER_ALIGNED(left_chunk, left_chunk->chunk_size)); + assert(HEAP_IS_POINTER_ALIGNED(right_chunk, right_chunk->chunk_size)); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(left_chunk, left_chunk->chunk_size); + heap_test_usable_memory(right_chunk, right_chunk->chunk_size); +#endif + + heap_remove_chunk(chunk); + heap_remove_chunk(left_chunk); + heap_remove_chunk(right_chunk); + size_t result_size = left_chunk->chunk_size + chunk->chunk_size + right_chunk->chunk_size; + struct heap_chunk* result = heap_chunk_format((uint8_t*) left_chunk, result_size); + heap_insert_chunk(result); + return result; + } + + /* Attempt to combine with the left chunk. */ + if ( left_chunk && !heap_chunk_is_used(left_chunk) ) + { + assert(HEAP_IS_POINTER_ALIGNED(left_chunk, left_chunk->chunk_size)); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(left_chunk, left_chunk->chunk_size); +#endif + + heap_remove_chunk(chunk); + heap_remove_chunk(left_chunk); + size_t result_size = left_chunk->chunk_size + chunk->chunk_size; + struct heap_chunk* result = heap_chunk_format((uint8_t*) left_chunk, result_size); + heap_insert_chunk(result); + return result; + } + + /* Attempt to combine with the right chunk. */ + if ( right_chunk && !heap_chunk_is_used(right_chunk) ) + { + assert(HEAP_IS_POINTER_ALIGNED(right_chunk, right_chunk->chunk_size)); +#if !defined(HEAP_NO_ASSERT) + heap_test_usable_memory(right_chunk, right_chunk->chunk_size); +#endif + + heap_remove_chunk(chunk); + heap_remove_chunk(right_chunk); + size_t result_size = chunk->chunk_size + right_chunk->chunk_size; + struct heap_chunk* result = heap_chunk_format((uint8_t*) chunk, result_size); + heap_insert_chunk(result); + return result; + } + + return chunk; +} + +#endif + __END_DECLS +#if __is_sortix_libc +#include +#endif + #endif diff --git a/libc/malloc/__heap_expand_current_part.cpp b/libc/malloc/__heap_expand_current_part.cpp new file mode 100644 index 00000000..8b3afb61 --- /dev/null +++ b/libc/malloc/__heap_expand_current_part.cpp @@ -0,0 +1,183 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/__heap_expand_current_part.cpp + Attemps to extend the current part of the heap or create a new part. + +*******************************************************************************/ + +#if __STDC_HOSTED__ +#include +#endif + +#include +#include +#include +#include + +#if __is_sortix_kernel +#include +#endif + +#if __is_sortix_kernel +#include +#include +#include +#endif + +extern "C" bool __heap_expand_current_part(size_t requested_expansion) +{ + // Determine the current page size. +#if __is_sortix_kernel + size_t page_size = Sortix::Page::Size(); +#else + size_t page_size = getpagesize(); +#endif + + // Determine how much memory and how many pages are required. The worst + // case is that we'll need to create a new part, so allocate enough + // memory for that case. + size_t needed_expansion = + sizeof(struct heap_part) + requested_expansion + sizeof(heap_part_post); + size_t needed_expansion_pages = + needed_expansion / page_size + (needed_expansion % page_size ? 1 : 0); + + // Allocate at least a few pages to prevent inefficient allocations. + size_t minimum_part_pages = 0; + if ( __heap_state.current_part ) + minimum_part_pages = __heap_state.current_part->part_size / page_size; + if ( 256 < minimum_part_pages ) + minimum_part_pages = 256; + if ( minimum_part_pages < 4 ) + minimum_part_pages = 4; + if ( needed_expansion_pages < minimum_part_pages ) + needed_expansion_pages = minimum_part_pages; + + // Calculate exactly how many bytes are added to the heap. + size_t num_bytes = needed_expansion_pages * page_size; + +#if __is_sortix_kernel + // Decide where we would like to add memory to the heap. + uintptr_t mapto = Sortix::GetHeapUpper(); + assert(!__heap_state.current_part || + (uintptr_t) heap_part_end(__heap_state.current_part) == mapto); + void* mapping = (void*) mapto; + + // Attempt to allocate the needed virtual address space such that we can put + // memory there to extend the heap with. + if ( !(num_bytes = Sortix::ExpandHeap(num_bytes)) ) + return false; + + // Attempt to map actual memory at our new virtual addresses. + int prot = PROT_KREAD | PROT_KWRITE; + enum Sortix::page_usage page_usage = Sortix::PAGE_USAGE_KERNEL_HEAP; + if ( !Sortix::Memory::MapRange(mapto, num_bytes, prot, page_usage) ) + return false; + Sortix::Memory::Flush(); + + bool ideal_allocation = true; +#else + // Decide where we'd like to allocation memory. Ideally, we'd like to extend + // an existing part, but if there is none, then simply somewhere near the + // start of the address space (so it can grow upwards) will do. + void* suggestion = __heap_state.current_part ? + heap_part_end(__heap_state.current_part) : + (void*) (0x1 * page_size); + + // Attempt to allocate memory for the new part. + + void* mapping = mmap(suggestion, num_bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + // We are out of memory if we can't make a suitable allocation. + if ( mapping == MAP_FAILED ) + return false; + + bool ideal_allocation = mapping == suggestion; +#endif + + // Extend the current part if we managed to do our ideal allocation. + if ( __heap_state.current_part && ideal_allocation ) + { + struct heap_part* part = __heap_state.current_part; + struct heap_part_post* old_post = heap_part_to_post(part); + struct heap_part* part_prev = old_post->part_prev; + + uint8_t* new_chunk_begin = (uint8_t*) mapping - sizeof(struct heap_part_post); + size_t new_chunk_size = num_bytes; + + // Remove the last chunk in the part from the heap if it is unused, we + // will then simply include its bytes in our new chunk below. + struct heap_chunk* last_chunk; + if ( (last_chunk = heap_part_last_chunk(part)) && + !heap_chunk_is_used(last_chunk) ) + { + heap_remove_chunk(last_chunk); + new_chunk_begin = (uint8_t*) last_chunk; + new_chunk_size += last_chunk->chunk_size; + } + + // Extend the part by moving the end part structure further ahead. + part->part_size += num_bytes; + struct heap_part_post* post = heap_part_to_post(part); + post->part_magic = HEAP_PART_MAGIC; + post->part_prev = part_prev; + post->part_size = part->part_size; + post->unused[0] = 0; + post->unused[1] = 0; + post->unused[2] = 0; + + // Format a new heap chunk that contains the new space, possibly unified + // with whatever unused space previously existed. + assert(requested_expansion <= new_chunk_size); + heap_insert_chunk(heap_chunk_format(new_chunk_begin, new_chunk_size)); + + return true; + } + + // Format a new part in our new allocation. + struct heap_part* part = (struct heap_part*) mapping; + part->unused[0] = 0; + part->unused[1] = 0; + part->unused[2] = 0; + part->part_size = num_bytes; + part->part_magic = HEAP_PART_MAGIC; + struct heap_part_post* post = heap_part_to_post(part); + post->part_magic = HEAP_PART_MAGIC; + post->part_size = part->part_size; + post->unused[0] = 0; + post->unused[1] = 0; + post->unused[2] = 0; + + // Insert our part into the part linked list. + if ( (part->part_next = __heap_state.current_part) ) + heap_part_to_post(__heap_state.current_part)->part_prev = part; + post->part_prev = NULL; + + // Insert a new chunk into the heap containing the unused space in the part. + uint8_t* new_chunk_begin = (uint8_t*) mapping + sizeof(struct heap_part); + size_t new_chunk_size = + num_bytes - (sizeof(struct heap_part) + sizeof(struct heap_part_post)); + assert(requested_expansion <= new_chunk_size); + heap_insert_chunk(heap_chunk_format(new_chunk_begin, new_chunk_size)); + + __heap_state.current_part = part; + + return true; +} diff --git a/libc/malloc/__heap_lock.cpp b/libc/malloc/__heap_lock.cpp new file mode 100644 index 00000000..a3d6467f --- /dev/null +++ b/libc/malloc/__heap_lock.cpp @@ -0,0 +1,47 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/__heap_lock.cpp + Locks the dynamic heap. + +*******************************************************************************/ + +#include +#if __STDC_HOSTED__ +#include +#endif + +#if __is_sortix_kernel +#include +#endif + +#if __STDC_HOSTED__ +extern "C" pthread_mutex_t __heap_mutex; +#elif __is_sortix_kernel +extern "C" Sortix::kthread_mutex_t __heap_mutex; +#endif + +extern "C" void __heap_lock(void) +{ +#if __STDC_HOSTED__ + pthread_mutex_lock(&__heap_mutex); +#elif __is_sortix_kernel + Sortix::kthread_mutex_lock(&__heap_mutex); +#endif +} diff --git a/libc/malloc/__heap_unlock.cpp b/libc/malloc/__heap_unlock.cpp new file mode 100644 index 00000000..21cb226d --- /dev/null +++ b/libc/malloc/__heap_unlock.cpp @@ -0,0 +1,47 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/__heap_unlock.cpp + Unlocks the dynamic heap. + +*******************************************************************************/ + +#include +#if __STDC_HOSTED__ +#include +#endif + +#if __is_sortix_kernel +#include +#endif + +#if __STDC_HOSTED__ +extern "C" pthread_mutex_t __heap_mutex; +#elif __is_sortix_kernel +extern "C" Sortix::kthread_mutex_t __heap_mutex; +#endif + +extern "C" void __heap_unlock(void) +{ +#if __STDC_HOSTED__ + pthread_mutex_unlock(&__heap_mutex); +#elif __is_sortix_kernel + Sortix::kthread_mutex_unlock(&__heap_mutex); +#endif +} diff --git a/libc/malloc/__heap_verify.cpp b/libc/malloc/__heap_verify.cpp new file mode 100644 index 00000000..484df01d --- /dev/null +++ b/libc/malloc/__heap_verify.cpp @@ -0,0 +1,42 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/__heap_verify.cpp + Perform a heap consistency check. + +*******************************************************************************/ + +#include +#include +#include + +extern "C" void __heap_verify() +{ + for ( size_t i = 0; i < sizeof(size_t) * 8 - 1; i++ ) + { + if ( __heap_state.bin_filled_bitmap & heap_size_of_bin(i) ) + { + assert(__heap_state.bin[i] != NULL); + } + else + { + assert(__heap_state.bin[i] == NULL); + } + } +} diff --git a/libc/malloc/heap_get_paranoia.cpp b/libc/malloc/heap_get_paranoia.cpp new file mode 100644 index 00000000..c033e6d2 --- /dev/null +++ b/libc/malloc/heap_get_paranoia.cpp @@ -0,0 +1,45 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/heap_get_paranoia.cpp + Returns how paranoid the heap implementation should be. + +*******************************************************************************/ + +#include +#include + +extern "C" int heap_get_paranoia(void) +{ +#if defined(PARANOIA_DEFAULT) && !__STDC_HOSTED__ + return PARANOIA_DEFAULT; +#elif defined(PARANOIA_DEFAULT) && __STDC_HOSTED__ + static int cached_paranoia = -1; + if ( cached_paranoia < 0 ) + { + if ( const char* paranoia_str = getenv("LIBC_MALLOC_PARANOIA") ) + cached_paranoia = atoi(paranoia_str); + else + cached_paranoia = PARANOIA_DEFAULT; + } + return cached_paranoia; +#else + return PARANOIA; +#endif +} diff --git a/libc/malloc/heap_init.cpp b/libc/malloc/heap_init.cpp new file mode 100644 index 00000000..97b8fc91 --- /dev/null +++ b/libc/malloc/heap_init.cpp @@ -0,0 +1,41 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + malloc/heap_init.cpp + Initializes the dynamic heap. + +*******************************************************************************/ + +#include +#if __STDC_HOSTED__ +#include +#endif +#include + +#if __is_sortix_kernel +#include +#endif + +extern "C" { struct heap_state __heap_state; } + +#if __STDC_HOSTED__ +extern "C" { pthread_mutex_t __heap_mutex = PTHREAD_MUTEX_INITIALIZER; } +#elif __is_sortix_kernel +extern "C" { Sortix::kthread_mutex_t __heap_mutex = Sortix::KTHREAD_MUTEX_INITIALIZER; } +#endif diff --git a/libc/stdlib/free.cpp b/libc/stdlib/free.cpp new file mode 100644 index 00000000..f1ecbdac --- /dev/null +++ b/libc/stdlib/free.cpp @@ -0,0 +1,57 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2011, 2012, 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + stdlib/free.cpp + Returns a chunk of memory to the dynamic memory heap. + +*******************************************************************************/ + +#include +#include + +#if __is_sortix_kernel +#include +#endif + +#if defined(HEAP_NO_ASSERT) +#define __heap_verify() ((void) 0) +#undef assert +#define assert(x) do { ((void) 0); } while ( 0 ) +#endif + +extern "C" void free(void* addr) +{ + if ( !addr ) + return; + + __heap_lock(); + __heap_verify(); + + // Retrieve the chunk that contains this allocation. + struct heap_chunk* chunk = heap_data_to_chunk((uint8_t*) addr); + + // Return the chunk to the heap. + heap_insert_chunk(chunk); + + // Combine the chunk with its left and right neighbors if they are unused. + heap_chunk_combine_neighbors(chunk); + + __heap_verify(); + __heap_unlock(); +} diff --git a/libc/stdlib/heap.cpp b/libc/stdlib/heap.cpp deleted file mode 100644 index 38c4ed24..00000000 --- a/libc/stdlib/heap.cpp +++ /dev/null @@ -1,729 +0,0 @@ -/******************************************************************************* - - Copyright(C) Jonas 'Sortie' Termansen 2011, 2012, 2013, 2014. - - This file is part of the Sortix C Library. - - The Sortix C Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - The Sortix C Library is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public - License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with the Sortix C Library. If not, see . - - stdlib/heap.cpp - Functions that allocate/free memory from a dynamic memory heap. - -*******************************************************************************/ - -#include - -#if __STDC_HOSTED__ -#include -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include - -#define PARANOIA 1 - -#if defined(__is_sortix_kernel) -#include -#include -#include -#include -#include -#include -#endif - -// -// This first section is just magic compiler/platform stuff, you should -// skip ahead to the actual algorithm. -// - -#if defined(__x86_64__) -const size_t MAGIC = 0xDEADDEADDEADDEADUL; -const size_t ALIGNMENT = 16UL; -#else -const size_t MAGIC = 0xDEADDEADUL; -const size_t ALIGNMENT = 8UL; -#endif - -const size_t PAGESIZE = 4UL * 1024UL; // 4 KiB -const size_t NUMBINS = 8UL * sizeof(size_t); - -static uintptr_t wilderness; - -#if defined(__is_sortix_kernel) -static uintptr_t GetHeapStart() -{ - return Sortix::GetHeapLower(); -} - -static void FreeMemory(uintptr_t where, size_t bytes) -{ - assert(Sortix::Page::IsAligned(where + bytes)); - - while ( bytes ) - { - addr_t page = Sortix::Memory::Unmap(where); - Sortix::Page::Put(page, Sortix::PAGE_USAGE_KERNEL_HEAP); - - bytes -= PAGESIZE; - where += PAGESIZE; - } -} - -static bool AllocateMemory(uintptr_t where, size_t bytes) -{ - assert(Sortix::Page::IsAligned(where + bytes)); - - uintptr_t pos = where; - - while ( bytes ) - { - addr_t page = Sortix::Page::Get(Sortix::PAGE_USAGE_KERNEL_HEAP); - if ( !page ) - { - FreeMemory(where, pos-where); - return false; - } - - if ( !Sortix::Memory::Map(page, pos, PROT_KREAD | PROT_KWRITE) ) - { - Sortix::Page::Put(page, Sortix::PAGE_USAGE_KERNEL_HEAP); - FreeMemory(where, pos-where); - return false; - } - - bytes -= PAGESIZE; - pos += PAGESIZE; - } - - return true; -} - -static bool ExtendHeap(size_t bytesneeded) -{ - size_t got_bytes = Sortix::ExpandHeap(bytesneeded); - if ( !got_bytes ) - return false; - assert(bytesneeded <= got_bytes); - - if ( !AllocateMemory(wilderness, got_bytes) ) - { - Sortix::ShrinkHeap(got_bytes); - return false; - } - - return true; -} -#else -static uintptr_t GetHeapStart() -{ - uintptr_t base = (uintptr_t) sbrk(0); - uintptr_t unaligned = base % ALIGNMENT; - if ( unaligned ) - { - sbrk(ALIGNMENT-unaligned); - } - uintptr_t result = (uintptr_t) sbrk(0); - return result; -} - -static bool ExtendHeap(size_t bytesneeded) -{ - void* newheapend = sbrk(bytesneeded); - return newheapend != (void*) -1UL; -} -#endif - -// TODO: BitScanForward and BitScanReverse are x86 instructions, but -// directly using them messes with the optimizer. Once possible, use -// the inline assembly instead of the C-version of the functions. - -// Returns the index of the most significant set bit. -inline size_t BSR(size_t Value) -{ -#if 1 - assert(Value > 0); - for ( size_t I = 8*sizeof(size_t); I > 0; I-- ) - { - if ( Value & ( 1UL << (I-1) ) ) { return I-1; } - } - return 0; -#else - size_t Result; - asm("bsr %0, %1" : "=r"(Result) : "r"(Value)); - return Result; -#endif -} - -// Returns the index of the least significant set bit. -inline size_t BSF(size_t Value) -{ -#if 1 - assert(Value > 0); - for ( size_t I = 0; I < 8*sizeof(size_t); I++ ) - { - if ( Value & ( 1UL << I ) ) { return I; } - } - return 0; -#else - size_t Result; - asm("bsf %0, %1" : "=r"(Result) : "r"(Value)); - return Result; -#endif -} - -// -// Now for some helper functions and structures. -// - -struct Chunk; -struct Trailer; - -#if __STDC_HOSTED__ -pthread_mutex_t heaplock = PTHREAD_MUTEX_INITIALIZER; -#elif defined(__is_sortix_kernel) -Sortix::kthread_mutex_t heaplock = Sortix::KTHREAD_MUTEX_INITIALIZER; -#endif - -// The location where the heap originally grows from. -uintptr_t heapstart; - -// If heap grows down: Location of the first mapped page. -// If heap grows up: Location of the first not-mapped page. -#if 0 /* forward declared abvce */ -static uintptr_t wilderness; -#endif - -// How many bytes remain in the wilderness. -size_t wildernesssize = 0; - -// How many bytes are the heap allow to grow to (including wilderness). -size_t heapmaxsize = SIZE_MAX; - -// How many bytes are currently used for chunks in the heap, which -// excludes the wilderness. -size_t heapsize = 0; - -// How many bytes of actual storage the heap contains. -size_t heapstorage = 0; - -// bins[N] contain a linked list of chunks that are at least 2^(N+1) -// bytes, but less than 2^(N+2) bytes. By selecting the proper bin in -// constant time, we can allocate chunks in constant time. -Chunk* bins[NUMBINS]; - -// Bit N is set if bin[N] contains a chunk. -size_t bincontainschunks = 0; - -static bool IsGoodHeapPointer(void* ptr, size_t size) -{ - uintptr_t ptrlower = (uintptr_t) ptr; - uintptr_t ptrupper = ptrlower + size; - uintptr_t heaplower = heapstart; - uintptr_t heapupper = wilderness; - return heaplower <= ptrlower && ptrupper <= heapupper; -} - -// A preamble to every chunk providing meta-information. -struct Chunk -{ -public: - size_t size; // Includes size of Chunk and Trailer - union - { - size_t magic; - Chunk* nextunused; - }; - -public: - bool IsUsed() { return magic == MAGIC; } - Trailer* GetTrailer(); - Chunk* LeftNeighbor(); - Chunk* RightNeighbor(); - bool IsSane(); - -}; - -// A trailer to every chunk providing meta-information. -struct Trailer -{ -public: - union - { - size_t magic; - Chunk* prevunused; - }; - size_t size; // Includes size of Chunk and Trailer - -public: - bool IsUsed() { return magic == MAGIC; } - Chunk* GetChunk(); - -}; - -const size_t OVERHEAD = sizeof(Chunk) + sizeof(Trailer); - -// This is how a real chunk actually looks: -//struct RealChunk -//{ -// Chunk header; -// byte data[...]; -// Trailer footer; -// }; - -Trailer* Chunk::GetTrailer() -{ - return (Trailer*) (((uintptr_t) this) + size - sizeof(Trailer)); -} - -Chunk* Chunk::LeftNeighbor() -{ - Trailer* trailer = (Trailer*) (((uintptr_t) this) - sizeof(Trailer)); - return trailer->GetChunk(); -} - -Chunk* Chunk::RightNeighbor() -{ - return (Chunk*) (((uintptr_t) this) + size); -} - -Chunk* Trailer::GetChunk() -{ - return (Chunk*) (((uintptr_t) this) + sizeof(Trailer) - size); -} - -bool Chunk::IsSane() -{ - if ( !IsGoodHeapPointer(this, sizeof(*this)) ) - return false; - if ( !size ) { return false; } - size_t binindex = BSR(size); - Trailer* trailer = GetTrailer(); - if ( !IsGoodHeapPointer(trailer, sizeof(*trailer)) ) - return false; - if ( trailer->size != size ) { return false; } - if ( IsUsed() ) - { - if ( bins[binindex] == this ) { return false; } - if ( magic != MAGIC || trailer->magic != magic ) { return false; } - } - if ( !IsUsed() ) - { - if ( ((uintptr_t) nextunused) & (ALIGNMENT-1UL) ) { return false; } - if ( ((uintptr_t) trailer->prevunused) & (ALIGNMENT-1UL) ) { return false; } - if ( nextunused && !IsGoodHeapPointer(nextunused->GetTrailer(), - sizeof(Trailer)) ) - return false; - if ( nextunused && nextunused->GetTrailer()->prevunused != this ) { return false; } - - if ( trailer->prevunused ) - { - if ( !IsGoodHeapPointer(trailer->prevunused, - sizeof(*trailer->prevunused)) ) - return false; - if ( bins[binindex] == this ) { return false; } - if ( trailer->prevunused->nextunused != this ) { return false; } - } - if ( !trailer->prevunused ) - { - if ( bins[binindex] != this ) { return false; } - if ( !(bincontainschunks & (1UL << binindex)) ) { return false; } - } - } - return true; -} - -static void InsertChunk(Chunk* chunk) -{ - // Insert the chunk into the right bin. - size_t binindex = BSR(chunk->size); - chunk->GetTrailer()->prevunused = NULL; - chunk->nextunused = bins[binindex]; - if ( chunk->nextunused ) - { - assert(chunk->nextunused->IsSane()); - chunk->nextunused->GetTrailer()->prevunused = chunk; - } - bins[binindex] = chunk; - bincontainschunks |= (1UL << binindex); - assert(chunk->IsSane()); -} - -__attribute__((unused)) -static bool ValidateHeap() -{ - bool foundbin[NUMBINS]; - for ( size_t i = 0; i < NUMBINS; i++ ) { foundbin[i] = false; } - - Chunk* chunk = (Chunk*) heapstart; - while ( (uintptr_t) chunk < wilderness - wildernesssize ) - { - size_t timesfound = 0; - for ( size_t i = 0; i < NUMBINS; i++ ) - { - if ( chunk == bins[i] ) { foundbin[i] = true; timesfound++; } - } - if ( 1 < timesfound ) { return false; } - - if ( !chunk->IsSane() ) { return false; } - chunk = chunk->RightNeighbor(); - } - - for ( size_t i = 0; i < NUMBINS; i++ ) - { - if ( !bins[i] ) - { - if ( foundbin[i] ) { return false; } - continue; - } - if ( !foundbin[i] ) { return false; } - if ( !bins[i]->IsSane() ) { return false; } - } - - return true; -} - -// Attempts to expand the wilderness such that it contains at least -// bytesneeded bytes. This is done by mapping new pages onto into the -// virtual address-space. -static bool ExpandWilderness(size_t bytesneeded) -{ - if ( bytesneeded <= wildernesssize ) { return true; } - - // Delayed initialization of the heap. - if ( heapstart == 0 && wilderness == 0 && !heapsize ) - heapstart = wilderness = GetHeapStart(); - - bytesneeded -= wildernesssize; - - // Align the increase on page boundaries. - const size_t PAGEMASK = ~(PAGESIZE - 1UL); - bytesneeded = ( bytesneeded + PAGESIZE - 1UL ) & PAGEMASK; - - assert(bytesneeded >= PAGESIZE); - - // TODO: Overflow MAY happen here! - if ( heapmaxsize <= heapsize + wildernesssize + bytesneeded ) - return errno = ENOMEM, true; - - uintptr_t newwilderness = wilderness + bytesneeded; - - // Attempt to map pages so our wilderness grows. - if ( !ExtendHeap(bytesneeded) ) - return false; - - wildernesssize += bytesneeded; - wilderness = newwilderness; - - return true; -} - -extern "C" void* malloc(size_t size) -{ - #if __STDC_HOSTED__ - pthread_mutex_lock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_lock(&heaplock); - #endif - - #if 2 <= PARANOIA - assert(ValidateHeap()); - #endif - - // The size field keeps both the allocation and meta information. - size += OVERHEAD; - - // Round up to nearest alignment. - size = (size + ALIGNMENT - 1UL) & (~(ALIGNMENT-1UL)); - - // Find the index of the smallest usable bin. - size_t minbinindex = BSR(size-1UL)+1UL; - - // Make a bitmask that filter away all bins that are too small. - size_t minbinmask = ~((1UL << minbinindex) - 1UL); - - // Figure out which bins are usable for our chunk. - size_t availablebins = bincontainschunks & minbinmask; - - if ( availablebins ) - { - // Find the smallest available bin. - size_t binindex = BSF(availablebins); - - Chunk* chunk = bins[binindex]; - assert(chunk->IsSane()); - bins[binindex] = chunk->nextunused; - - size_t binsize = 1UL << binindex; - - // Mark the bin as empty if we emptied it. - if ( !bins[binindex] ) - { - bincontainschunks ^= binsize; - } - else - { - Trailer* trailer = bins[binindex]->GetTrailer(); - trailer->prevunused = NULL; - } - - assert(!bins[binindex] || bins[binindex]->IsSane()); - - // If we don't use the entire chunk. - size_t original_chunk_size = chunk->size; - if ( OVERHEAD <= original_chunk_size - size ) - { - size_t left = original_chunk_size - size; - chunk->size -= left; - chunk->GetTrailer()->size = chunk->size; - - Chunk* leftchunk = chunk->RightNeighbor(); - leftchunk->size = left; - Trailer* lefttrailer = leftchunk->GetTrailer(); - lefttrailer->size = left; - - InsertChunk(leftchunk); - } - - chunk->magic = MAGIC; - chunk->GetTrailer()->magic = MAGIC; - - heapstorage += chunk->size; - - #if 3 <= PARANOIA - assert(ValidateHeap()); - #endif - - uintptr_t result = ((uintptr_t) chunk) + sizeof(Chunk); - - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif - - return (void*) result; - } - - // If no bins are available, try to allocate from the wilderness. - - // Check if the wilderness can meet our requirements. - if ( wildernesssize < size && !ExpandWilderness(size) ) - { - errno = ENOMEM; - - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif - - return NULL; - } - - // Carve a new chunk out of the wilderness and initialize it. - Chunk* chunk = (Chunk*) (wilderness - wildernesssize); - assert(size <= wildernesssize); - wildernesssize -= size; - heapsize += size; - assert(IsGoodHeapPointer(chunk, sizeof(*chunk))); - chunk->size = size; - Trailer* trailer = chunk->GetTrailer(); - assert(IsGoodHeapPointer(trailer, sizeof(*trailer))); - trailer->size = size; - chunk->magic = MAGIC; - trailer->magic = MAGIC; - - heapstorage += chunk->size; - - #if 3 <= PARANOIA - assert(ValidateHeap()); - #endif - - uintptr_t result = ((uintptr_t) chunk) + sizeof(Chunk); - - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif - - return (void*) result; -} - -static bool IsLeftmostChunk(Chunk* chunk) -{ - return heapstart <= (uintptr_t) chunk; -} - -static bool IsRightmostChunk(Chunk* chunk) -{ - return heapstart + heapsize <= (uintptr_t) chunk + chunk->size; -} - -// Removes a chunk from its bin. -static void UnlinkChunk(Chunk* chunk) -{ - assert(chunk->IsSane()); - Trailer* trailer = chunk->GetTrailer(); - if ( trailer->prevunused ) - { - assert(trailer->prevunused->IsSane()); - trailer->prevunused->nextunused = chunk->nextunused; - if ( chunk->nextunused ) - { - assert(chunk->nextunused->IsSane()); - chunk->nextunused->GetTrailer()->prevunused = trailer->prevunused; - } - } - else - { - if ( chunk->nextunused ) - { - assert(chunk->nextunused->IsSane()); - chunk->nextunused->GetTrailer()->prevunused = NULL; - } - size_t binindex = BSR(chunk->size); - assert(bins[binindex] == chunk); - bins[binindex] = chunk->nextunused; - if ( !bins[binindex] ) { bincontainschunks ^= 1UL << binindex; } - else { assert(bins[binindex]->IsSane()); } - } -} - -// Transforms a chunk and its neighbors into a single chunk if possible. -static void UnifyNeighbors(Chunk** chunk) -{ - if ( !IsLeftmostChunk(*chunk) ) - { - Chunk* neighbor = (*chunk)->LeftNeighbor(); - if ( !neighbor->IsUsed() ) - { - size_t size = neighbor->size; - size_t chunksize = (*chunk)->size; - UnlinkChunk(neighbor); - *chunk = neighbor; - (*chunk)->size = size + chunksize; - (*chunk)->GetTrailer()->size = (*chunk)->size; - } - } - - if ( !IsRightmostChunk(*chunk) ) - { - Chunk* neighbor = (*chunk)->RightNeighbor(); - if ( !neighbor->IsUsed() ) - { - UnlinkChunk(neighbor); - (*chunk)->size += neighbor->size; - (*chunk)->GetTrailer()->size = (*chunk)->size; - } - } -} - -extern "C" void free(void* addr) -{ - #if __STDC_HOSTED__ - pthread_mutex_lock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_lock(&heaplock); - #endif - - #if 2 <= PARANOIA - assert(ValidateHeap()); - #endif - - if ( !addr) - { - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif - - return; - } - - Chunk* chunk = (Chunk*) ((uintptr_t) addr - sizeof(Chunk)); -#if __STDC_HOSTED__ - if ( !IsGoodHeapPointer(addr, 1) || - !IsGoodHeapPointer(chunk, sizeof(*chunk)) ) - { - error(0, 0, "attempted to free(3) non-heap pointer: %p", addr); - abort(); - } - if ( !chunk->IsUsed() ) - { - error(0, 0, "attempted to free(3) area that doesn't appear to be " - "allocated: %p + 0x%zx", chunk, chunk->size); - abort(); - } -#endif - assert(chunk->IsUsed()); - assert(chunk->IsSane()); - - heapstorage -= chunk->size; - - UnifyNeighbors(&chunk); - - bool nexttowilderness = IsRightmostChunk(chunk); - - // If possible, let the wilderness regain the memory. - if ( nexttowilderness ) - { - heapsize -= chunk->size; - wildernesssize += chunk->size; - - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif - - return; - } - - InsertChunk(chunk); - - #if 3 <= PARANOIA - assert(ValidateHeap()); - #endif - - #if __STDC_HOSTED__ - pthread_mutex_unlock(&heaplock); - #elif defined(__is_sortix_kernel) - Sortix::kthread_mutex_unlock(&heaplock); - #endif -} - -// TODO: Implement this function properly. -extern "C" void* realloc(void* ptr, size_t size) -{ - if ( !ptr ) { return malloc(size); } - Chunk* chunk = (Chunk*) ((uintptr_t) ptr - sizeof(Chunk)); - assert(chunk->IsUsed()); - assert(chunk->IsSane()); - size_t allocsize = chunk->size - OVERHEAD; - if ( size < allocsize ) { return ptr; } - void* newptr = malloc(size); - if ( !newptr ) { return NULL; } - memcpy(newptr, ptr, allocsize); - free(ptr); - return newptr; -} diff --git a/libc/stdlib/malloc.cpp b/libc/stdlib/malloc.cpp new file mode 100644 index 00000000..91d04aa0 --- /dev/null +++ b/libc/stdlib/malloc.cpp @@ -0,0 +1,104 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2011, 2012, 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + stdlib/malloc.cpp + Allocates a chunk of memory from the dynamic memory heap. + +*******************************************************************************/ + +#include +#include +#include +#include +#include + +#if __is_sortix_kernel +#include +#endif + +#if defined(HEAP_NO_ASSERT) +#define __heap_verify() ((void) 0) +#undef assert +#define assert(x) do { ((void) 0); } while ( 0 ) +#endif + +extern "C" void* malloc(size_t original_size) +{ + if ( !heap_size_has_bin(original_size) ) + return errno = ENOMEM, (void*) NULL; + + // Decide how big an allocation we would like to make. + size_t chunk_outer_size = sizeof(struct heap_chunk) + + sizeof(struct heap_chunk_post); + size_t chunk_inner_size = heap_align(original_size); + size_t chunk_size = chunk_outer_size + chunk_inner_size; + + if ( !heap_size_has_bin(chunk_size) ) + return errno = ENOMEM, (void*) NULL; + + // Decide which bins are large enough for our allocation. + size_t smallest_desirable_bin = heap_bin_for_allocation(chunk_size); + size_t smallest_desirable_bin_size = heap_size_of_bin(smallest_desirable_bin); + size_t desirable_bins = ~0UL << smallest_desirable_bin; + + __heap_lock(); + __heap_verify(); + + // Determine whether there are any bins that we can use. + size_t usable_bins = desirable_bins & __heap_state.bin_filled_bitmap; + + // If there are no usable bins, attempt to expand the current part of the + // heap or create a new part. + if ( !usable_bins && __heap_expand_current_part(smallest_desirable_bin_size) ) + usable_bins = desirable_bins & __heap_state.bin_filled_bitmap; + + // If we failed to expand the current part or make a new one - then we are + // officially out of memory until someone deallocates something. + if ( !usable_bins ) + { + __heap_verify(); + __heap_unlock(); + return (void*) NULL; + } + + // Pick the smallest of the usable bins. + size_t bin_index = heap_bsf(usable_bins); + + // Pick the first element of this bins linked list. This is our allocation. + struct heap_chunk* result_chunk = __heap_state.bin[bin_index]; + assert(result_chunk); + assert(HEAP_IS_POINTER_ALIGNED(result_chunk, result_chunk->chunk_size)); + + assert(chunk_size <= result_chunk->chunk_size); + + // Mark our chosen chunk as used and remove it from its bin. + heap_remove_chunk(result_chunk); + + // If our chunk is larger than what we really needed and it is possible to + // split the chunk into two, then we should split off a part of it and + // return it to the heap for further allocation. + if ( heap_can_split_chunk(result_chunk, chunk_size) ) + heap_split_chunk(result_chunk, chunk_size); + + __heap_verify(); + __heap_unlock(); + + // Return the inner data associated with the chunk to the caller. + return heap_chunk_to_data(result_chunk); +} diff --git a/libc/stdlib/realloc.cpp b/libc/stdlib/realloc.cpp new file mode 100644 index 00000000..5d657896 --- /dev/null +++ b/libc/stdlib/realloc.cpp @@ -0,0 +1,127 @@ +/******************************************************************************* + + Copyright(C) Jonas 'Sortie' Termansen 2011, 2012, 2013, 2014. + + This file is part of the Sortix C Library. + + The Sortix C Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + The Sortix C Library is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with the Sortix C Library. If not, see . + + stdlib/realloc.cpp + Reallocates a chunk of memory from the dynamic memory heap. + +*******************************************************************************/ + +#include +#include +#include +#include +#include + +#if __is_sortix_kernel +#include +#endif + +#if defined(HEAP_NO_ASSERT) +#define __heap_verify() ((void) 0) +#undef assert +#define assert(x) do { ((void) 0); } while ( 0 ) +#endif + +extern "C" void* realloc(void* ptr, size_t requested_size) +{ + if ( !ptr ) + return malloc(requested_size); + + if ( !heap_size_has_bin(requested_size) ) + return errno = ENOMEM, (void*) NULL; + + // Decide how big an allocation we would like to make. + size_t requested_chunk_outer_size = sizeof(struct heap_chunk) + + sizeof(struct heap_chunk_post); + size_t requested_chunk_inner_size = heap_align(requested_size); + size_t requested_chunk_size = requested_chunk_outer_size + + requested_chunk_inner_size; + + if ( !heap_size_has_bin(requested_chunk_size) ) + return errno = ENOMEM, (void*) NULL; + + __heap_lock(); + __heap_verify(); + + // Retrieve the chunk that contains this allocation. + struct heap_chunk* chunk = heap_data_to_chunk((uint8_t*) ptr); + + assert(chunk->chunk_magic == HEAP_CHUNK_MAGIC); + assert(heap_chunk_to_post(chunk)->chunk_magic == HEAP_CHUNK_MAGIC); + assert(heap_chunk_to_post(chunk)->chunk_size == chunk->chunk_size); + + // Do nothing if the chunk already has the ideal size. + if ( chunk->chunk_size == requested_chunk_size ) + { + __heap_verify(); + __heap_unlock(); + return heap_chunk_to_data(chunk); + } + + // If the ideal size is smaller than the current, attempt the shrink the + // allocation if a new chunk can be created. + if ( requested_chunk_size < chunk->chunk_size ) + { + assert(requested_chunk_size <= chunk->chunk_size); + if ( heap_can_split_chunk(chunk, requested_chunk_size) ) + heap_split_chunk(chunk, requested_chunk_size); + __heap_verify(); + __heap_unlock(); + return heap_chunk_to_data(chunk); + } + + // TODO: What if the right neighbor is the part edge? + + // If we need to expand the chunk, attempt to combine it with its right + // neighbor if it is large enough. + struct heap_chunk* right; + if ( (right = heap_chunk_right(chunk)) && + !heap_chunk_is_used(right) && + requested_chunk_size <= chunk->chunk_size + right->chunk_size ) + { + heap_remove_chunk(right); + heap_chunk_format((uint8_t*) chunk, chunk->chunk_size + right->chunk_size); + assert(requested_chunk_size <= chunk->chunk_size); + if ( heap_can_split_chunk(chunk, requested_chunk_size) ) + heap_split_chunk(chunk, requested_chunk_size); + __heap_verify(); + __heap_unlock(); + return heap_chunk_to_data(chunk); + } + + // It appears that we cannot retain the orignal allocation location and we + // will have to relocate the allocation elsewhere to expand it. + + size_t orignal_ptr_size = heap_chunk_data_size(chunk); + + __heap_verify(); + __heap_unlock(); + + assert(orignal_ptr_size < requested_size); + + void* result = malloc(requested_size); + if ( !result ) + return (void*) NULL; + + memcpy(result, ptr, orignal_ptr_size); + + free(ptr); + + return result; +}