Meta alloc can now flush thread reserves.

1 files changed, 66 insertions, 12 deletions

diff --git a/src/zisp/gc/meta_alloc.zig b/src/zisp/gc/meta_alloc.zig
index 8937a81..3138891 100644
--- a/src/zisp/gc/meta_alloc.zig
+++ b/src/zisp/gc/meta_alloc.zig
@@ -265,9 +265,9 @@ fn alloc_from_fl(
     // Now detached from global; no need for atomics anymore.
     tl.fc_count = arr[1];
 
-    // We can safely copy more than needed since we set fc_count; this way it
-    // should compile neatly into SIMD instructions.
-    for (0..RESERVE_MAX / 2) |i| tl.fc[i] = arr[8 + i];
+    // We can safely copy more than needed since we set fc_count.
+    const half_max = RESERVE_MAX / 2;
+    @memcpy(tl.fc[0..half_max], arr[8 .. 8 + half_max]);
 
     return ptr;
 }
@@ -315,7 +315,7 @@ fn free_size_class(sci: u8, ptr: [*]u64) void {
 
     // Local cache full; need to flush some to global free-list.
     const info = &slab_infos[sci];
-    free_into_fl(info, tl, ptr, idx, res_n);
+    free_into_fl(info, tl, ptr, idx, res_n / 2);
 }
 
 fn free_into_fl(
@@ -323,20 +323,23 @@ fn free_into_fl(
     tl: *TlSlabInfo,
     ptr: [*]u64,
     idx: u32,
-    res_n: u32,
+    n: u32,
 ) void {
+    // Note: n = 0 is valid and must work.
+    std.debug.assert(n <= RESERVE_MAX / 2);
+
     const arr: [*]u32 = @ptrCast(ptr);
-    const half = res_n / 2;
+    const split = tl.fc_count - n;
 
-    // Intuitively, this belongs to the end, but it doesn't matter; the rest of
-    // this function is not allowed to fail anyway.
-    tl.fc_count = half;
+    tl.fc_count = split;
 
     // No need for atomic stores; we own this memory exclusively right now,
     // unless another thread calls free() on the same slot which is a bug.
-    arr[1] = half;
-    // Safe to copy more than needed; should compile into SIMD.
-    for (0..RESERVE_MAX / 2) |i| arr[8 + i] = tl.fc[half + i];
+    arr[1] = n;
+
+    // Safe to copy more than needed.
+    const max = RESERVE_MAX / 2;
+    @memcpy(arr[8 .. 8 + max], tl.fc[split .. split + max]);
 
     // Now atomically announce the new free-list head to global visibility.
     var fl_head = @atomicLoad(FlHead, &info.free_list, .acquire);
@@ -353,3 +356,54 @@ fn free_into_fl(
         ) orelse break;
     }
 }
+
+/// For every size class: Flushes this thread's free slot cache into the global
+/// free-list, and splits remaining reserved memory into slots which are then
+/// also pushed into the global free-list.  It's crucial for threads to call
+/// this function before terminating, or else memory is leaked.
+export fn flush_thread_reserves() void {
+    var sci: u8 = 0;
+    var wm_units: u32 = SIZES[0] / 8;
+    while (sci < SIZES.len) : ({
+        sci += 1;
+        wm_units *= 2;
+    }) {
+        const info = &slab_infos[sci];
+        const tl = &tl_slab_infos[sci];
+        const slab: [*]u64 = &slabs[sci];
+
+        flush_thread_fc(info, tl, slab);
+
+        // Now turn slots between wm_lo and wm_hi into free-list entries; most
+        // easily done by writing them into tl.fc and reusing the function to
+        // flush the fc entries.
+        var i: u32 = 0;
+        var wm = tl.wm_lo;
+        while (wm < tl.wm_hi) : ({
+            i += 1;
+            wm += wm_units;
+        }) {
+            tl.fc[i] = wm;
+        }
+        tl.fc_count = i;
+        tl.wm_lo = wm;
+        flush_thread_fc(info, tl, slab);
+    }
+}
+
+fn flush_thread_fc(info: *SlabInfo, tl: *TlSlabInfo, slab: [*]u64) void {
+    // Do in two steps if there's too many for a single free-list node.
+    if (tl.fc_count > RESERVE_MAX / 2) {
+        tl.fc_count -= 1;
+        const idx = tl.fc[tl.fc_count];
+        const ptr: [*]u64 = slab[idx..];
+        free_into_fl(info, tl, ptr, idx, RESERVE_MAX / 2);
+    }
+    if (tl.fc_count != 0) {
+        tl.fc_count -= 1;
+        const idx = tl.fc[tl.fc_count];
+        const ptr: [*]u64 = slab[idx..];
+        // Remaining fl_count may be 0; that's fine.
+        free_into_fl(info, tl, ptr, idx, tl.fc_count);
+    }
+}