Meta alloc tweaks & cleanup.

1 files changed, 33 insertions, 29 deletions

diff --git a/src/zisp/gc/meta_alloc.zig b/src/zisp/gc/meta_alloc.zig
index f46e83c..56f9c88 100644
--- a/src/zisp/gc/meta_alloc.zig
+++ b/src/zisp/gc/meta_alloc.zig
@@ -71,7 +71,7 @@ const Unit = u64;
 const UnitIdx = u32;
 
 /// Length of each slab, as in Unit count.
-const SLAB_LEN: u32 = 1 << 31;
+const SLAB_LEN: UnitIdx = 1 << 31;
 
 /// A slab of memory.
 const Slab = [SLAB_LEN]Unit;
@@ -83,7 +83,8 @@ const SlotPtr = [*]Unit;
 const SIZES: [16]usize = .{
     // Important: Ensure the smallest size class is inherently immune to false
     // sharing, and no smaller than: 64 + @sizeOf(UnitIdx) * ( FSC_MAX / 2 ).
-    // That's 128 bytes assuming FSC_MAX = 32 (see below).
+    // That's exactly 128 bytes assuming FSC_MAX = 32 (see below).
+    128,
     256,
     512,
     1024,
@@ -99,7 +100,6 @@ const SIZES: [16]usize = .{
     1024 * 1024,
     1024 * 1024 * 2,
     1024 * 1024 * 4,
-    1024 * 1024 * 8,
 };
 
 /// Gets the index 0 to 15 of a size class, from the size value, returning a
@@ -112,13 +112,13 @@ inline fn getSizeClassIndex(size: usize) u8 {
     return @ctz(size >> @ctz(SIZES[0]));
 }
 
-/// Pointer to start of all 16 slabs as a contiguous vmem block.
+/// Pointer to start of all slabs as a contiguous array.
 var slabs: *[SIZES.len]Slab = undefined;
 
 /// Free-list head pointer with ABA counter.
 const FlHead = packed struct(u64) {
     // Putting aba first leads to slightly better codegen.
-    aba: u32,
+    aba: UnitIdx,
     idx: UnitIdx,
 };
 
@@ -138,23 +138,23 @@ var slab_infos: [SIZES.len]SlabInfo = @splat(.{});
 
 /// Maximum number of slots reserved by a thread when bumping a slab watermark.
 /// Halved after every two size classes, so make sure it's >= 128.
-const RESERVE_MAX: u32 = 512;
+const RESERVE_MAX: UnitIdx = 512; // Results in 4 for the two largest classes.
 
 /// Maximum number of free slot cache entries before half of them are flushed.
 /// Halved after every four size classes, so make sure it's >= 8.
-const FSC_MAX: u32 = 32;
+const FSC_MAX: UnitIdx = 32; // Results in 4 for the two largest classes.
 
 /// Given a size class index 0 to 15, returns the number of slots that a thread
 /// should reserve every time it needs to bump the global watermark.  Starts at
 /// RESERVE_MAX and is halved after every two size classes.
-inline fn reserveCountForSizeClassIndex(sci: u8) u32 {
+inline fn reserveCountForSizeClassIndex(sci: u8) UnitIdx {
     return RESERVE_MAX >> @intCast(sci / 2);
 }
 
 /// Given a size class index 0 to 15, returns the maximum number of entries in
 /// the free slot cache of a thread before half of them are flushed.  Starts at
 /// FSC_MAX and is halved after every four size classes.
-inline fn maxFscCountForSizeClassIndex(sci: u8) u32 {
+inline fn maxFscCountForSizeClassIndex(sci: u8) UnitIdx {
     return FSC_MAX >> @intCast(sci / 4);
 }
 
@@ -164,7 +164,7 @@ const TlSlabInfo = struct {
     /// possible SIMD related benefits.
     fsc: [FSC_MAX]UnitIdx align(std.atomic.cache_line) = @splat(0),
     /// Current count of entries in free slot cache.
-    fsc_count: u32 = 0,
+    fsc_count: UnitIdx = 0,
     /// Start point of memory reserved for this thread.
     wm_lo: UnitIdx = 0,
     /// End point of memory reserved for this thread.
@@ -188,7 +188,7 @@ fn mmap(size: usize) []align(4096) u8 {
 
 /// Must call this once to initialize the slabs.
 export fn init() void {
-    slabs = @ptrCast(mmap(SIZES.len * SLAB_LEN * 8));
+    slabs = @ptrCast(mmap(SIZES.len * SLAB_LEN * @sizeOf(Unit)));
 }
 
 /// Allocate a slot of the given size, which must be a power of two and greater
@@ -198,8 +198,7 @@ export fn alloc(size: usize) [*]u8 {
     const sci = getSizeClassIndex(size);
     if (sci < SIZES.len) {
         @branchHint(.likely);
-        const slot_ptr = alloc_size_class(sci, size);
-        return @ptrCast(slot_ptr);
+        return @ptrCast(alloc_size_class(sci, size));
     } else {
         return mmap(size).ptr;
     }
@@ -229,13 +228,13 @@ fn alloc_size_class(sci: u8, size: usize) SlotPtr {
         return slab[idx..].ptr;
     }
 
-    // The size in 8-byte slab/watermark units.
-    const wm_units: UnitIdx = @intCast(size / 8);
+    // The length of a slot in Units.
+    const slot_len: UnitIdx = @intCast(size / @sizeOf(Unit));
 
     // Use part of the reserved memory for this thread.
     if (tl.wm_lo < tl.wm_hi) {
         const idx = tl.wm_lo;
-        tl.wm_lo += wm_units;
+        tl.wm_lo += slot_len;
         return slab[idx..].ptr;
     }
 
@@ -244,7 +243,7 @@ fn alloc_size_class(sci: u8, size: usize) SlotPtr {
     if (alloc_from_fl(slab, info, tl)) |p| return p;
 
     const res_n = reserveCountForSizeClassIndex(sci);
-    return alloc_fresh(size, slab, info, tl, wm_units, res_n);
+    return alloc_fresh(size, slab, info, tl, slot_len, res_n);
 }
 
 fn alloc_from_fl(
@@ -292,24 +291,24 @@ fn alloc_fresh(
     slab: *Slab,
     info: *SlabInfo,
     tl: *TlSlabInfo,
-    wm_units: UnitIdx,
-    res_n: u32,
+    slot_len: UnitIdx,
+    res_n: UnitIdx,
 ) SlotPtr {
     // Have to use fresh memory from the top of the slab; bump it by a chunk,
     // atomically, to reserve memory for this thread.
-    const bump = res_n * wm_units;
+    const bump = res_n * slot_len;
     const old_wm = @atomicRmw(UnitIdx, &info.watermark, .Add, bump, .monotonic);
     const new_wm = old_wm + bump;
 
     // Check only if the old WM was already overflown; chunk count is such that
     // it's always safe to bump by a chunk if the old WM was still valid, since
-    // wm_units, res_n, and slab length are all powers of two.
+    // slot_len, res_n, and slab length are all powers of two.
     if (old_wm >= SLAB_LEN) {
         std.debug.panic("Exhausted slab for size class: {}", .{size});
     }
 
     tl.wm_hi = new_wm;
-    tl.wm_lo = old_wm + wm_units;
+    tl.wm_lo = old_wm + slot_len;
     return slab[old_wm..].ptr;
 }
 
@@ -338,7 +337,7 @@ fn free_into_fl(
     tl: *TlSlabInfo,
     ptr: SlotPtr,
     idx: UnitIdx,
-    n: u32,
+    n: UnitIdx,
 ) void {
     // Note: n = 0 is valid and must work.
     std.debug.assert(n <= FSC_MAX / 2);
@@ -349,7 +348,7 @@ fn free_into_fl(
     tl.fsc_count = split;
 
     // No need for atomic stores; we own this memory exclusively right now,
-    // unless another thread calls free() on the same slot which is a bug.
+    // unless another thread calls free() on the same slot, which is a bug.
     arr[1] = n;
 
     // Safe to copy more than needed.
@@ -378,10 +377,10 @@ fn free_into_fl(
 /// this function before terminating, or else memory is leaked.
 export fn flush_thread_reserves() void {
     var sci: u8 = 0;
-    var units: UnitIdx = SIZES[0] / 8;
+    var slot_len: UnitIdx = SIZES[0] / @sizeOf(Unit);
     while (sci < SIZES.len) : ({
         sci += 1;
-        units *= 2;
+        slot_len *= 2;
     }) {
         const info = &slab_infos[sci];
         const tl = &tl_slab_infos[sci];
@@ -398,7 +397,7 @@ export fn flush_thread_reserves() void {
         // Now turn slots between wm_lo and wm_hi into free-list entries; most
         // easily done by writing them into tl.fsc and reusing the function to
         // flush the fsc entries every time chunk_max + 1 is hit.
-        while (tl.wm_lo < tl.wm_hi) : (tl.wm_lo += units) {
+        while (tl.wm_lo < tl.wm_hi) : (tl.wm_lo += slot_len) {
             if (tl.fsc_count > chunk_max) {
                 flush_fsc(info, tl, slab, chunk_max);
             }
@@ -411,9 +410,14 @@ export fn flush_thread_reserves() void {
     }
 }
 
-fn flush_fsc(info: *SlabInfo, tl: *TlSlabInfo, slab: *Slab, extras: u32) void {
+fn flush_fsc(
+    info: *SlabInfo,
+    tl: *TlSlabInfo,
+    slab: *Slab,
+    n_more: UnitIdx,
+) void {
     tl.fsc_count -= 1;
     const idx = tl.fsc[tl.fsc_count];
     const ptr: SlotPtr = slab[idx..].ptr;
-    free_into_fl(info, tl, ptr, idx, extras);
+    free_into_fl(info, tl, ptr, idx, n_more);
 }