diff --git a/include/slab.h b/include/slab.h index 594a47f..f44f941 100644 --- a/include/slab.h +++ b/include/slab.h @@ -10,7 +10,10 @@ /// Per-cache descriptor: one cache per object kind /// Usually this is allocated statically somewhere. struct slab_cache { - size_t obj_size; // bytes per object + size_t obj_size; + size_t obj_align; + size_t obj_align_shift; + size_t num_slots_per_slab; struct slab *empty_slabs; // slabs with all slots free struct slab *partial_slabs; // slabs with some free slots struct slab *full_slabs; // slabs with no free slots @@ -39,12 +42,21 @@ struct free_stack { /// Initializes the cache, without allocating any memory. /// `obj_size` must be > 0. /// constructor and destructor may be `NULL`, in which case allocated objects are not initialized. +// if not specified, default alignment is 8 (suitable for most data structures). + void slab_cache_init( struct slab_cache *cache, size_t obj_size, void (*constructor)(void *), void (*destructor)(void *)); +void slab_cache_init_with_align( + struct slab_cache *cache, + size_t obj_size, + size_t obj_align, + void (*constructor)(void*), + void (*destructor)(void*)); + /// Allocates one element from the cache. /// Returns `NULL` on memory exhaustion. void *slab_cache_alloc(struct slab_cache *cache); diff --git a/include/tlsf.h b/include/tlsf.h index f0f64b3..1d778f0 100644 --- a/include/tlsf.h +++ b/include/tlsf.h @@ -20,12 +20,6 @@ #define LV2_BITMAP_SET(bitmap, second) BIT_SET(bitmap, second) #define LV2_BITMAP_CLR(bitmap, second) BIT_CLR(bitmap, second) -#define IS_ALIGNED_16(sz) (((uint64_t)(sz) & (uint64_t)15) == 0) -#define ROUND_UP(n, shift) ((((uint64_t)n) + ((1ull << shift) - 1)) & ~(uint64_t)((1ull << shift) - 1)) -#define ROUND_UP_16(n) ROUND_UP(n, 4) - -#define IS_POWER_OF_2(n) (__builtin_popcountl(n) == 1) - #define INFO_BIT_FREE 0 #define INFO_BIT_IS_LAST_PHYSICAL 1 #define INFO_SIZE_MASK (~3ull) @@ -37,13 +31,6 @@ #define INFO_UNPACK_IS_LAST_PHYSICAL(info) BIT_GET(info, INFO_BIT_IS_LAST_PHYSICAL) #define INFO_UNPACK_IS_FREE(info) BIT_GET(info, INFO_BIT_FREE) -// Highest bit set (0-indexed). -// - 4 (100) has highest bit 2. -// - 10 (1010) has highest bit 3. -// The following holds: (1 << HIGHEST_BIT_SET(sz)) <= sz -// Don't use for `sz == 0`. -#define HIGHEST_BIT_SET(sz) (63 - __builtin_clzl(sz)) - struct tlsf_pool { size_t sli_shift; // this stuff is mostly for debug diff --git a/include/util.h b/include/util.h index 36e8ea6..50f729a 100644 --- a/include/util.h +++ b/include/util.h @@ -33,4 +33,26 @@ #define BIT_SET(bitmap, bit) ((bitmap) |= ((__typeof__(bitmap))1 << (bit))) #define BIT_CLR(bitmap, bit) ((bitmap) &= ~((__typeof__(bitmap))1 << (bit))) +#define IS_ALIGNED_16(sz) (((uint64_t)(sz) & (uint64_t)15) == 0) +#define ROUND_UP(n, shift) ((((uint64_t)n) + ((1ull << shift) - 1)) & ~(uint64_t)((1ull << shift) - 1)) +#define ROUND_UP_8(n) ROUND_UP(n, 3) +#define ROUND_UP_16(n) ROUND_UP(n, 4) + +// TODO gcc says popcount cannot be found. Use manual version for now. +#ifdef POPCOUNT_USE_BUILTIN +#define IS_POWER_OF_2(n) (__builtin_popcountl(n) == 1) +#else +#define IS_POWER_OF_2(n) ((n ^ (n - 1)) > (n - 1)) +#endif + + +// Highest bit set (0-indexed). +// - 4 (100) has highest bit 2. +// - 10 (1010) has highest bit 3. +// The following holds: (1 << HIGHEST_BIT_SET(sz)) <= sz +// Don't use for `sz == 0`. +#define HIGHEST_BIT_SET(sz) (63 - __builtin_clzl((uint64_t)sz)) + +#define LOG2(n) HIGHEST_BIT_SET(n) + #endif diff --git a/src/slab.c b/src/slab.c index a0f901d..a44c076 100644 --- a/src/slab.c +++ b/src/slab.c @@ -3,32 +3,51 @@ #include "ram.h" #include "std.h" #include "bootparam.h" - -#define EFF_SPACE_IN_SLAB (PAGE_SIZE - sizeof(struct slab)) -#define SLOT_SIZE(obj_size) ((obj_size) + sizeof(struct free_stack)) -#define SLOTS_PER_SLAB(obj_size) (EFF_SPACE_IN_SLAB / SLOT_SIZE(obj_size)) - -#define ROUND_UP_8(sz) (((sz) + 7) & ~(size_t)0x7) +#include "util.h" #define SLAB_INIT_MAGIC 0x736c6162696e6974 +#define EFF_SPACE_IN_SLAB(obj_align_shift) (PG_SIZE - ROUND_UP(sizeof(struct slab), obj_align_shift)) +#define SLOT_SIZE(obj_size, obj_align_shift) ROUND_UP(ROUND_UP_8(obj_size) + sizeof(struct free_stack), obj_align_shift) +#define NUM_SLOTS_PER_SLAB(obj_size, obj_align_shift) EFF_SPACE_IN_SLAB(obj_align_shift) / SLOT_SIZE(obj_size, obj_align_shift) + +void slab_cache_init_with_align( + struct slab_cache *cache, + size_t obj_size, + size_t obj_align, + void (*constructor)(void*), + void (*destructor)(void*)) +{ + ASSERT(cache != NULL); + ASSERT(obj_size > 0); + ASSERT(obj_align > 0); + ASSERT(IS_POWER_OF_2(obj_align)); + + cache->obj_size = obj_size, + cache->obj_align = obj_align; + size_t obj_align_shift = LOG2(obj_align); + cache->obj_align_shift = obj_align_shift; + cache->num_slots_per_slab = NUM_SLOTS_PER_SLAB(obj_size, obj_align_shift); + // NOTE: we allow one slot per slab, even though it makes some things more complicated + if (cache->num_slots_per_slab == 0) { + PANIC("slab: object too big."); + } + + cache->empty_slabs = NULL; + cache->partial_slabs = NULL; + cache->full_slabs = NULL; + cache->constructor = constructor; + cache->destructor = destructor; + cache->magic = SLAB_INIT_MAGIC; +} + void slab_cache_init( struct slab_cache *cache, size_t obj_size, void (*constructor)(void *), void (*destructor)(void *)) { - ASSERT(cache != NULL); - ASSERT(obj_size > 0); - ASSERT(SLOT_SIZE(obj_size) <= EFF_SPACE_IN_SLAB); - - cache->obj_size = ROUND_UP_8(obj_size); - cache->empty_slabs = NULL; - cache->partial_slabs = NULL; - cache->full_slabs = NULL; - cache->constructor = constructor; - cache->destructor = destructor; - cache->magic = SLAB_INIT_MAGIC; + slab_cache_init_with_align(cache, obj_size, 8, constructor, destructor); } #define SLAB_MAGIC 0x736c61626d616765 @@ -42,29 +61,35 @@ bool slab_cache_new_slab(struct slab_cache *cache) { if (!ram_alloc_frame(&ppn, RAM_PAGE_NORMAL)) { return false; } - void *raw = ppn_to_pointer(ppn); + uint8_t *raw = ppn_to_pointer(ppn); + uint8_t * const page_start = raw; + uint8_t * const page_end = page_start + PG_SIZE; - struct slab *as_slab = raw; - as_slab->free_count = SLOTS_PER_SLAB(cache->obj_size); + struct slab *as_slab = (struct slab *)raw; // build the free-slot stack in the bytes after the header - char *slots_base = (char *)raw + sizeof(struct slab); - ASSERT((intptr_t)slots_base % 8 == 0); - + raw += sizeof(struct slab); + size_t obj_size_align8 = ROUND_UP_8(cache->obj_size); struct free_stack *head = NULL; - for (size_t i = 0; i < as_slab->free_count; i++) { - // the stack structure lives after the corresponding object - void *obj = slots_base + i * SLOT_SIZE(cache->obj_size); + for (size_t i = 0; i < cache->num_slots_per_slab; i++) { + raw = (uint8_t*)ROUND_UP(raw, cache->obj_align_shift); + uint8_t *end_of_obj = raw + obj_size_align8 + sizeof(struct free_stack); + ASSERT(end_of_obj < page_end); + if (cache->constructor) { - cache->constructor(obj); + cache->constructor(raw); } - struct free_stack *node = (struct free_stack*)((char*)obj + cache->obj_size); - node->magic = SLOT_MAGIC_FREE; - node->next = head; - head = node; + raw += obj_size_align8; + struct free_stack *stack = (struct free_stack *)raw; + stack->magic = SLOT_MAGIC_FREE; + stack->next = head; + head = stack; + raw += sizeof(struct free_stack); } + as_slab->free_stack = head; + as_slab->free_count = cache->num_slots_per_slab; as_slab->cache = cache; as_slab->magic = SLAB_MAGIC; @@ -127,7 +152,7 @@ void *slab_cache_alloc(struct slab_cache *cache) { // move previously empty slab to partial or full list cache->empty_slabs = slab->next; - if (SLOTS_PER_SLAB(cache->obj_size) > 1) { + if (cache->num_slots_per_slab > 1) { slab->next = cache->partial_slabs; cache->partial_slabs = slab; } else { @@ -167,7 +192,7 @@ static void slab_cache_remove_slab_from_list( static void slab_obj_free(struct slab *slab, void *obj) { ASSERT(slab != NULL); ASSERT(obj != NULL); - ASSERT(slab->free_count < SLOTS_PER_SLAB(slab->cache->obj_size)); + ASSERT(slab->free_count < slab->cache->num_slots_per_slab); ASSERT(PAGE_BASE(obj) == (void*)slab); struct free_stack *node = (struct free_stack *)((char*)obj + slab->cache->obj_size); @@ -189,15 +214,21 @@ static void slab_cache_return_slab_to_kernel( { ASSERT(cache != NULL); ASSERT(slab != NULL); - ASSERT(slab->free_count == SLOTS_PER_SLAB(cache->obj_size)); - char *slots_base = (char *)slab + sizeof(struct slab); - ASSERT((intptr_t)slots_base % 8 == 0); + ASSERT(slab->free_count == cache->num_slots_per_slab); + uint8_t *raw = (uint8_t*)slab; + uint8_t * const page_start = raw; + uint8_t * const page_end = page_start + PG_SIZE; + raw += sizeof(struct slab); + size_t obj_size_align8 = ROUND_UP_8(cache->obj_size); - for (size_t i = 0; i < slab->free_count; i++) { - void *obj = slots_base + i * SLOT_SIZE(cache->obj_size); + for (size_t i = 0; i < cache->num_slots_per_slab; i++) { + raw = (uint8_t*)ROUND_UP(raw, cache->obj_align_shift); + uint8_t *end_of_obj = raw + obj_size_align8 + sizeof(struct free_stack); + ASSERT(end_of_obj < page_end); if (cache->destructor) { - cache->destructor(obj); + cache->destructor(raw); } + raw = end_of_obj; } ram_free(ppn_from_aligned_pa(pa_from_pointer(slab))); @@ -219,7 +250,7 @@ void slab_cache_free(void *obj) { // slab is either full or partial // afterwards it's either partial or empty - if (slab->free_count == SLOTS_PER_SLAB(slab->cache->obj_size)) { + if (slab->free_count == slab->cache->num_slots_per_slab) { // empty now if (was_partial_before) { slab_cache_remove_slab_from_list(slab->cache, slab, &slab->cache->partial_slabs);