slab: custom alignment possible

include/slab.h

@@ -10,7 +10,10 @@
 /// Per-cache descriptor: one cache per object kind
 /// Usually this is allocated statically somewhere.
 struct slab_cache {
-    size_t obj_size;            // bytes per object
+    size_t obj_size;
+    size_t obj_align;
+    size_t obj_align_shift;
+    size_t num_slots_per_slab;
     struct slab *empty_slabs;   // slabs with all slots free
     struct slab *partial_slabs; // slabs with some free slots
     struct slab *full_slabs;    // slabs with no free slots
@@ -39,12 +42,21 @@ struct free_stack {
 /// Initializes the cache, without allocating any memory.
 /// `obj_size` must be > 0.
 /// constructor and destructor may be `NULL`, in which case allocated objects are not initialized.
+// if not specified, default alignment is 8 (suitable for most data structures).
+
 void slab_cache_init(
     struct slab_cache *cache,
     size_t obj_size,
     void (*constructor)(void *),
     void (*destructor)(void *));
 
+void slab_cache_init_with_align(
+    struct slab_cache *cache,
+    size_t obj_size,
+    size_t obj_align,
+    void (*constructor)(void*),
+    void (*destructor)(void*));
+
 /// Allocates one element from the cache.
 /// Returns `NULL` on memory exhaustion.
 void *slab_cache_alloc(struct slab_cache *cache);

include/tlsf.h

@@ -20,12 +20,6 @@
 #define LV2_BITMAP_SET(bitmap, second) BIT_SET(bitmap, second)
 #define LV2_BITMAP_CLR(bitmap, second) BIT_CLR(bitmap, second)
 
-#define IS_ALIGNED_16(sz) (((uint64_t)(sz) & (uint64_t)15) == 0)
-#define ROUND_UP(n, shift) ((((uint64_t)n) + ((1ull << shift) - 1)) & ~(uint64_t)((1ull << shift) - 1))
-#define ROUND_UP_16(n) ROUND_UP(n, 4)
-
-#define IS_POWER_OF_2(n) (__builtin_popcountl(n) == 1)
-
 #define INFO_BIT_FREE 0
 #define INFO_BIT_IS_LAST_PHYSICAL 1
 #define INFO_SIZE_MASK (~3ull)
@@ -37,13 +31,6 @@
 #define INFO_UNPACK_IS_LAST_PHYSICAL(info) BIT_GET(info, INFO_BIT_IS_LAST_PHYSICAL)
 #define INFO_UNPACK_IS_FREE(info) BIT_GET(info, INFO_BIT_FREE)
 
-// Highest bit set (0-indexed).
-// - 4 (100) has highest bit 2.
-// - 10 (1010) has highest bit 3.
-// The following holds: (1 << HIGHEST_BIT_SET(sz)) <= sz
-// Don't use for `sz == 0`.
-#define HIGHEST_BIT_SET(sz) (63 - __builtin_clzl(sz))
-
 struct tlsf_pool {
     size_t sli_shift;
     // this stuff is mostly for debug

include/util.h

@@ -33,4 +33,26 @@
 #define BIT_SET(bitmap, bit) ((bitmap) |= ((__typeof__(bitmap))1 << (bit)))
 #define BIT_CLR(bitmap, bit) ((bitmap) &= ~((__typeof__(bitmap))1 << (bit)))
 
+#define IS_ALIGNED_16(sz) (((uint64_t)(sz) & (uint64_t)15) == 0)
+#define ROUND_UP(n, shift) ((((uint64_t)n) + ((1ull << shift) - 1)) & ~(uint64_t)((1ull << shift) - 1))
+#define ROUND_UP_8(n) ROUND_UP(n, 3)
+#define ROUND_UP_16(n) ROUND_UP(n, 4)
+
+// TODO gcc says popcount cannot be found. Use manual version for now.
+#ifdef POPCOUNT_USE_BUILTIN
+#define IS_POWER_OF_2(n) (__builtin_popcountl(n) == 1)
+#else
+#define IS_POWER_OF_2(n) ((n ^ (n - 1)) > (n - 1))
+#endif
+
+
+// Highest bit set (0-indexed).
+// - 4 (100) has highest bit 2.
+// - 10 (1010) has highest bit 3.
+// The following holds: (1 << HIGHEST_BIT_SET(sz)) <= sz
+// Don't use for `sz == 0`.
+#define HIGHEST_BIT_SET(sz) (63 - __builtin_clzl((uint64_t)sz))
+
+#define LOG2(n) HIGHEST_BIT_SET(n)
+
 #endif

src/slab.c

@@ -3,32 +3,51 @@
 #include "ram.h"
 #include "std.h"
 #include "bootparam.h"
-
-#define EFF_SPACE_IN_SLAB (PAGE_SIZE - sizeof(struct slab))
-#define SLOT_SIZE(obj_size) ((obj_size) + sizeof(struct free_stack))
-#define SLOTS_PER_SLAB(obj_size) (EFF_SPACE_IN_SLAB / SLOT_SIZE(obj_size))
-
-#define ROUND_UP_8(sz) (((sz) + 7) & ~(size_t)0x7)
+#include "util.h"
 
 #define SLAB_INIT_MAGIC 0x736c6162696e6974
 
+#define EFF_SPACE_IN_SLAB(obj_align_shift) (PG_SIZE - ROUND_UP(sizeof(struct slab), obj_align_shift))
+#define SLOT_SIZE(obj_size, obj_align_shift) ROUND_UP(ROUND_UP_8(obj_size) + sizeof(struct free_stack), obj_align_shift)
+#define NUM_SLOTS_PER_SLAB(obj_size, obj_align_shift) EFF_SPACE_IN_SLAB(obj_align_shift) / SLOT_SIZE(obj_size, obj_align_shift)
+
+void slab_cache_init_with_align(
+    struct slab_cache *cache,
+    size_t obj_size,
+    size_t obj_align,
+    void (*constructor)(void*),
+    void (*destructor)(void*))
+{
+    ASSERT(cache != NULL);
+    ASSERT(obj_size > 0);
+    ASSERT(obj_align > 0);
+    ASSERT(IS_POWER_OF_2(obj_align));
+
+    cache->obj_size = obj_size,
+    cache->obj_align = obj_align;
+    size_t obj_align_shift = LOG2(obj_align);
+    cache->obj_align_shift = obj_align_shift;
+    cache->num_slots_per_slab = NUM_SLOTS_PER_SLAB(obj_size, obj_align_shift);
+    // NOTE: we allow one slot per slab, even though it makes some things more complicated
+    if (cache->num_slots_per_slab == 0) {
+        PANIC("slab: object too big.");
+    }
+
+    cache->empty_slabs = NULL;
+    cache->partial_slabs = NULL;
+    cache->full_slabs = NULL;
+    cache->constructor = constructor;
+    cache->destructor = destructor;
+    cache->magic = SLAB_INIT_MAGIC;
+}
+
 void slab_cache_init(
     struct slab_cache *cache,
     size_t obj_size,
     void (*constructor)(void *),
     void (*destructor)(void *))
 {
-    ASSERT(cache != NULL);
-    ASSERT(obj_size > 0);
-    ASSERT(SLOT_SIZE(obj_size) <= EFF_SPACE_IN_SLAB);
-
-    cache->obj_size = ROUND_UP_8(obj_size);
-    cache->empty_slabs = NULL;
-    cache->partial_slabs = NULL;
-    cache->full_slabs = NULL;
-    cache->constructor = constructor;
-    cache->destructor = destructor;
-    cache->magic = SLAB_INIT_MAGIC;
+    slab_cache_init_with_align(cache, obj_size, 8, constructor, destructor);
 }
 
 #define SLAB_MAGIC 0x736c61626d616765
@@ -42,29 +61,35 @@ bool slab_cache_new_slab(struct slab_cache *cache) {
     if (!ram_alloc_frame(&ppn, RAM_PAGE_NORMAL)) {
         return false;
     }
-    void *raw = ppn_to_pointer(ppn);
+    uint8_t *raw = ppn_to_pointer(ppn);
+    uint8_t * const page_start = raw;
+    uint8_t * const page_end = page_start + PG_SIZE;
 
-    struct slab *as_slab = raw;
-    as_slab->free_count = SLOTS_PER_SLAB(cache->obj_size);
+    struct slab *as_slab = (struct slab *)raw;
 
     // build the free-slot stack in the bytes after the header
-    char *slots_base = (char *)raw + sizeof(struct slab);
-    ASSERT((intptr_t)slots_base % 8 == 0);
-
+    raw += sizeof(struct slab);
+    size_t obj_size_align8 = ROUND_UP_8(cache->obj_size);
     struct free_stack *head = NULL;
 
-    for (size_t i = 0; i < as_slab->free_count; i++) {
-        // the stack structure lives after the corresponding object
-        void *obj = slots_base + i * SLOT_SIZE(cache->obj_size);
+    for (size_t i = 0; i < cache->num_slots_per_slab; i++) {
+        raw = (uint8_t*)ROUND_UP(raw, cache->obj_align_shift);
+        uint8_t *end_of_obj = raw + obj_size_align8 + sizeof(struct free_stack);
+        ASSERT(end_of_obj < page_end);
+
         if (cache->constructor) {
-            cache->constructor(obj);
+            cache->constructor(raw);
         }
-        struct free_stack *node = (struct free_stack*)((char*)obj + cache->obj_size);
-        node->magic = SLOT_MAGIC_FREE;
-        node->next = head;
-        head = node;
+        raw += obj_size_align8;
+        struct free_stack *stack = (struct free_stack *)raw;
+        stack->magic = SLOT_MAGIC_FREE;
+        stack->next = head;
+        head = stack;
+        raw += sizeof(struct free_stack);
     }
+
     as_slab->free_stack = head;
+    as_slab->free_count = cache->num_slots_per_slab;
 
     as_slab->cache = cache;
     as_slab->magic = SLAB_MAGIC;
@@ -127,7 +152,7 @@ void *slab_cache_alloc(struct slab_cache *cache) {
 
     // move previously empty slab to partial or full list
     cache->empty_slabs = slab->next;
-    if (SLOTS_PER_SLAB(cache->obj_size) > 1) {
+    if (cache->num_slots_per_slab > 1) {
         slab->next = cache->partial_slabs;
         cache->partial_slabs = slab;
     } else {
@@ -167,7 +192,7 @@ static void slab_cache_remove_slab_from_list(
 static void slab_obj_free(struct slab *slab, void *obj) {
     ASSERT(slab != NULL);
     ASSERT(obj != NULL);
-    ASSERT(slab->free_count < SLOTS_PER_SLAB(slab->cache->obj_size));
+    ASSERT(slab->free_count < slab->cache->num_slots_per_slab);
     ASSERT(PAGE_BASE(obj) == (void*)slab);
 
     struct free_stack *node = (struct free_stack *)((char*)obj + slab->cache->obj_size);
@@ -189,15 +214,21 @@ static void slab_cache_return_slab_to_kernel(
 {
     ASSERT(cache != NULL);
     ASSERT(slab != NULL);
-    ASSERT(slab->free_count == SLOTS_PER_SLAB(cache->obj_size));
-    char *slots_base = (char *)slab + sizeof(struct slab);
-    ASSERT((intptr_t)slots_base % 8 == 0);
+    ASSERT(slab->free_count == cache->num_slots_per_slab);
+    uint8_t *raw = (uint8_t*)slab;
+    uint8_t * const page_start = raw;
+    uint8_t * const page_end = page_start + PG_SIZE;
+    raw += sizeof(struct slab);
+    size_t obj_size_align8 = ROUND_UP_8(cache->obj_size);
 
-    for (size_t i = 0; i < slab->free_count; i++) {
-        void *obj = slots_base + i * SLOT_SIZE(cache->obj_size);
+    for (size_t i = 0; i < cache->num_slots_per_slab; i++) {
+        raw = (uint8_t*)ROUND_UP(raw, cache->obj_align_shift);
+        uint8_t *end_of_obj = raw + obj_size_align8 + sizeof(struct free_stack);
+        ASSERT(end_of_obj < page_end);
         if (cache->destructor) {
-            cache->destructor(obj);
+            cache->destructor(raw);
         }
+        raw = end_of_obj;
     }
 
     ram_free(ppn_from_aligned_pa(pa_from_pointer(slab)));
@@ -219,7 +250,7 @@ void slab_cache_free(void *obj) {
     // slab is either full or partial
     // afterwards it's either partial or empty
 
-    if (slab->free_count == SLOTS_PER_SLAB(slab->cache->obj_size)) {
+    if (slab->free_count == slab->cache->num_slots_per_slab) {
         // empty now
         if (was_partial_before) {
             slab_cache_remove_slab_from_list(slab->cache, slab, &slab->cache->partial_slabs);