/*
 * pool.h -- Fixed-size shared-memory object pool for Linux
 *
 * Bitmap-based slot allocation with CAS (lock-free).
 * Futex-based blocking when pool is exhausted.
 * PID-based stale slot recovery.
 *
 * Variants:
 *   Raw -- opaque byte slots of arbitrary elem_size
 *   I64 -- int64_t slots with atomic CAS/add
 *   F64 -- double slots
 *   I32 -- int32_t slots with atomic CAS/add
 *   Str -- fixed-length string slots (4-byte length prefix + data)
 */

#ifndef POOL_H
#define POOL_H

#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <limits.h>
#include <signal.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/syscall.h>
#include <linux/futex.h>
#include <sys/eventfd.h>

/* ================================================================
 * Constants
 * ================================================================ */

#define POOL_MAGIC        0x504F4C31U  /* "POL1" */
#define POOL_VERSION      1
#define POOL_ERR_BUFLEN   256

#define POOL_VAR_RAW  0
#define POOL_VAR_I64  1
#define POOL_VAR_F64  2
#define POOL_VAR_I32  3
#define POOL_VAR_STR  4

#define POOL_ALIGN8(x) (((x) + 7) & ~(uint64_t)7)

/* ================================================================
 * Header (128 bytes = 2 cache lines)
 * ================================================================ */

typedef struct {
    /* ---- Cache line 0 (0-63): immutable after create ---- */
    uint32_t magic;          /* 0 */
    uint32_t version;        /* 4 */
    uint32_t elem_size;      /* 8 */
    uint32_t variant_id;     /* 12 */
    uint64_t capacity;       /* 16: number of slots */
    uint64_t total_size;     /* 24: total mmap size */
    uint64_t data_off;       /* 32: offset to slot data */
    uint64_t bitmap_off;     /* 40: offset to allocation bitmap */
    uint64_t owners_off;     /* 48: offset to per-slot owner PIDs */
    uint8_t  _pad0[8];       /* 56-63 */

    /* ---- Cache line 1 (64-127): mutable state ---- */
    uint32_t used;           /* 64: allocated count (futex word) */
    uint32_t waiters;        /* 68: blocked on alloc */
    uint8_t  _pad1[8];       /* 72-79 */
    uint64_t stat_allocs;    /* 80 */
    uint64_t stat_frees;     /* 88 */
    uint64_t stat_waits;     /* 96 */
    uint64_t stat_timeouts;  /* 104 */
    uint64_t stat_recoveries;/* 112 */
    uint8_t  _pad2[8];       /* 120-127 */
} PoolHeader;

#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
_Static_assert(sizeof(PoolHeader) == 128, "PoolHeader must be 128 bytes");
#endif

/* ================================================================
 * Process-local handle
 * ================================================================ */

typedef struct {
    PoolHeader *hdr;
    uint64_t   *bitmap;
    uint32_t   *owners;
    uint8_t    *data;
    size_t      mmap_size;
    uint32_t    bitmap_words;
    uint64_t    capacity;    /* cached-at-attach geometry (peer may corrupt hdr->capacity) */
    uint32_t    elem_size;   /* cached-at-attach geometry (peer may corrupt hdr->elem_size) */
    char       *path;
    int         notify_fd;
    int         backing_fd;
    uint32_t    scan_hint;
} PoolHandle;

/* ================================================================
 * Utility
 * ================================================================ */

/* A zombie (dead but not yet reaped) still answers kill(pid,0) as alive, so a
 * process that crashed while holding the lock and lingers unreaped would never
 * be recovered.  Treat /proc/<pid>/stat state 'Z' as dead.  Linux-only (as is
 * this module); if /proc is unreadable we fall back to "alive" (safe: we never
 * force-recover a possibly-live holder). */
static inline int pool_pid_is_zombie(uint32_t pid) {
    char path[32], buf[256];
    snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
    int fd = open(path, O_RDONLY | O_CLOEXEC);
    if (fd < 0) return 0;
    ssize_t n = read(fd, buf, sizeof(buf) - 1);
    close(fd);
    if (n <= 0) return 0;
    buf[n] = '\0';
    /* "pid (comm) state ..."; comm may contain ')', so scan to the last one. */
    char *rp = strrchr(buf, ')');
    if (!rp || rp + 2 >= buf + n) return 0;   /* need ") X" within the bytes read */
    return rp[1] == ' ' && rp[2] == 'Z';
}
static inline int pool_pid_alive(uint32_t pid) {
    if (pid == 0) return 1; /* no owner recorded, assume alive */
    if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */
    return !pool_pid_is_zombie(pid); /* kill() also succeeds for a zombie -> treat as dead */
}

static inline void pool_make_deadline(double timeout, struct timespec *deadline) {
    clock_gettime(CLOCK_MONOTONIC, deadline);
    if (!(timeout < 1e9)) timeout = 1e9; /* clamp Inf/NaN/huge: avoid UB (time_t) cast -> instant spurious timeout */
    deadline->tv_sec += (time_t)timeout;
    deadline->tv_nsec += (long)((timeout - (double)(time_t)timeout) * 1e9);
    if (deadline->tv_nsec >= 1000000000L) {
        deadline->tv_sec++;
        deadline->tv_nsec -= 1000000000L;
    }
}

static inline int pool_remaining_time(const struct timespec *deadline,
                                       struct timespec *remaining) {
    struct timespec now;
    clock_gettime(CLOCK_MONOTONIC, &now);
    remaining->tv_sec = deadline->tv_sec - now.tv_sec;
    remaining->tv_nsec = deadline->tv_nsec - now.tv_nsec;
    if (remaining->tv_nsec < 0) {
        remaining->tv_sec--;
        remaining->tv_nsec += 1000000000L;
    }
    return remaining->tv_sec >= 0;
}

/* ================================================================
 * Slot access
 * ================================================================ */

static inline uint8_t *pool_slot_ptr(PoolHandle *h, uint64_t slot) {
    return h->data + slot * h->elem_size;
}

static inline int pool_is_allocated(PoolHandle *h, uint64_t slot) {
    uint32_t widx = (uint32_t)(slot / 64);
    int bit = (int)(slot % 64);
    uint64_t word = __atomic_load_n(&h->bitmap[widx], __ATOMIC_RELAXED);
    return (word >> bit) & 1;
}

/* ================================================================
 * Allocation (lock-free bitmap scan + CAS)
 * ================================================================ */

static inline int64_t pool_try_alloc(PoolHandle *h) {
    uint32_t nwords = h->bitmap_words;
    uint64_t cap = h->capacity;
    uint32_t start = h->scan_hint;
    uint32_t mypid = (uint32_t)getpid();

    for (uint32_t i = 0; i < nwords; i++) {
        uint32_t widx = (start + i) % nwords;
        uint64_t word = __atomic_load_n(&h->bitmap[widx], __ATOMIC_RELAXED);

        while (word != ~(uint64_t)0) {
            int bit = __builtin_ctzll(~word);
            uint64_t slot = (uint64_t)widx * 64 + bit;
            if (slot >= cap) break;

            uint64_t new_word = word | ((uint64_t)1 << bit);
            if (__atomic_compare_exchange_n(&h->bitmap[widx], &word, new_word,
                    1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
                __atomic_store_n(&h->owners[slot], mypid, __ATOMIC_RELAXED);
                memset(pool_slot_ptr(h, slot), 0, h->elem_size);
                __atomic_add_fetch(&h->hdr->used, 1, __ATOMIC_RELEASE);
                __atomic_add_fetch(&h->hdr->stat_allocs, 1, __ATOMIC_RELAXED);
                /* Advance hint past full word to reduce next scan */
                h->scan_hint = (new_word == ~(uint64_t)0 && nwords > 1)
                    ? (widx + 1) % nwords : widx;
                return (int64_t)slot;
            }
            /* CAS failed -- word now holds current value, retry */
        }
    }
    return -1;
}

/* Pin the per-handle allocation scan to start at the given slot's word. Pass 0 for
 * deterministic, sequential (low-to-high) allocation order, overriding the
 * getpid()-derived spread start. The spread only reduces CAS contention when many
 * processes allocate concurrently; a single allocator wants reproducible order, and
 * the bitmap stays the source of truth either way. An out-of-range slot wraps. */
static inline void pool_scan_from(PoolHandle *h, uint64_t slot) {
    h->scan_hint = (uint32_t)((slot / 64) % h->bitmap_words);
}

/* Blocking alloc. timeout<0 = infinite, 0 = non-blocking, >0 = seconds. */
static inline int64_t pool_alloc(PoolHandle *h, double timeout) {
    int64_t slot = pool_try_alloc(h);
    if (slot >= 0) return slot;
    if (timeout == 0) return -1;

    PoolHeader *hdr = h->hdr;
    struct timespec deadline, remaining;
    int has_deadline = (timeout > 0);
    if (has_deadline) pool_make_deadline(timeout, &deadline);

    __atomic_add_fetch(&hdr->stat_waits, 1, __ATOMIC_RELAXED);

    for (;;) {
        uint32_t cur_used = __atomic_load_n(&hdr->used, __ATOMIC_RELAXED);
        if ((uint64_t)cur_used < hdr->capacity) {
            slot = pool_try_alloc(h);
            if (slot >= 0) return slot;
        }

        __atomic_add_fetch(&hdr->waiters, 1, __ATOMIC_RELEASE);
        /* StoreLoad barrier: we must publish waiters++ before re-reading
         * `used`, so a concurrent releaser either sees our waiters++ (and
         * wakes us) or we see its used-- here (and don't sleep). Release/
         * acquire does not order a store before a later load to a different
         * address; without this fence both sides can miss each other on a
         * weakly-ordered CPU -> lost wakeup -> hang. */
        __atomic_thread_fence(__ATOMIC_SEQ_CST);

        cur_used = __atomic_load_n(&hdr->used, __ATOMIC_ACQUIRE);
        if ((uint64_t)cur_used >= hdr->capacity) {
            struct timespec *pts = NULL;
            if (has_deadline) {
                if (!pool_remaining_time(&deadline, &remaining)) {
                    __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);
                    __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
                    return -1;
                }
                pts = &remaining;
            }
            syscall(SYS_futex, &hdr->used, FUTEX_WAIT, cur_used, pts, NULL, 0);
        }

        __atomic_sub_fetch(&hdr->waiters, 1, __ATOMIC_RELAXED);

        slot = pool_try_alloc(h);
        if (slot >= 0) return slot;

        if (has_deadline) {
            if (!pool_remaining_time(&deadline, &remaining)) {
                __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
                return -1;
            }
        }
    }
}

/* ================================================================
 * Free
 * ================================================================ */

static inline int pool_free_slot(PoolHandle *h, uint64_t slot) {
    PoolHeader *hdr = h->hdr;
    if (slot >= h->capacity) return 0;

    uint32_t widx = (uint32_t)(slot / 64);
    int bit = (int)(slot % 64);
    uint64_t mask = (uint64_t)1 << bit;

    for (;;) {
        uint64_t word = __atomic_load_n(&h->bitmap[widx], __ATOMIC_RELAXED);
        if (!(word & mask)) return 0;

        uint64_t new_word = word & ~mask;
        /* Zero owner BEFORE releasing the bit: once the bit is clear a peer's
         * try_alloc may set owners[slot]=its pid, and doing owner=0 after the
         * release CAS would race and clobber that pid to 0 (recover_stale then
         * skips the slot -> a later crash leaks it).  recover_stale is
         * owner-first for the same reason. */
        __atomic_store_n(&h->owners[slot], 0, __ATOMIC_RELAXED);
        if (__atomic_compare_exchange_n(&h->bitmap[widx], &word, new_word,
                1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
            __atomic_sub_fetch(&hdr->used, 1, __ATOMIC_RELEASE);
            __atomic_add_fetch(&hdr->stat_frees, 1, __ATOMIC_RELAXED);
            /* StoreLoad barrier: publish used-- before reading waiters, so we
             * observe a waiter that registered just before our free (else the
             * wake is lost and it sleeps until timeout). Pairs with the fence
             * in the alloc-wait path. */
            __atomic_thread_fence(__ATOMIC_SEQ_CST);
            if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
                syscall(SYS_futex, &hdr->used, FUTEX_WAKE, 1, NULL, NULL, 0);
            return 1;
        }
    }
}

/* ================================================================
 * Batch free -- single used decrement + single futex wake
 * ================================================================ */

static inline uint32_t pool_free_n(PoolHandle *h, uint64_t *slots, uint32_t count) {
    uint32_t freed = 0;
    PoolHeader *hdr = h->hdr;

    for (uint32_t i = 0; i < count; i++) {
        uint64_t slot = slots[i];
        if (slot >= h->capacity) continue;

        uint32_t widx = (uint32_t)(slot / 64);
        int bit = (int)(slot % 64);
        uint64_t mask = (uint64_t)1 << bit;

        for (;;) {
            uint64_t word = __atomic_load_n(&h->bitmap[widx], __ATOMIC_RELAXED);
            if (!(word & mask)) break;
            uint64_t new_word = word & ~mask;
            __atomic_store_n(&h->owners[slot], 0, __ATOMIC_RELAXED);  /* owner-first (see pool_free_slot) */
            if (__atomic_compare_exchange_n(&h->bitmap[widx], &word, new_word,
                    1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
                freed++;
                break;
            }
        }
    }

    if (freed > 0) {
        __atomic_sub_fetch(&hdr->used, freed, __ATOMIC_RELEASE);
        __atomic_add_fetch(&hdr->stat_frees, freed, __ATOMIC_RELAXED);
        /* StoreLoad barrier: see pool_free_slot. */
        __atomic_thread_fence(__ATOMIC_SEQ_CST);
        if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0) {
            int wake = freed < (uint32_t)INT_MAX ? (int)freed : INT_MAX;
            syscall(SYS_futex, &hdr->used, FUTEX_WAKE, wake, NULL, NULL, 0);
        }
    }
    return freed;
}

/* ================================================================
 * Batch alloc -- all-or-nothing, shared deadline
 * ================================================================ */

static inline int pool_alloc_n(PoolHandle *h, uint64_t *out, uint32_t count,
                                double timeout) {
    if (count == 0) return 1;

    if (timeout == 0) {
        for (uint32_t i = 0; i < count; i++) {
            int64_t slot = pool_try_alloc(h);
            if (slot < 0) {
                if (i > 0) pool_free_n(h, out, i);
                return 0;
            }
            out[i] = (uint64_t)slot;
        }
        return 1;
    }

    /* count > capacity can never be satisfied: fail fast rather than grab the
     * whole pool and block forever. */
    if ((uint64_t)count > h->hdr->capacity) return 0;

    PoolHeader *hdr = h->hdr;
    struct timespec deadline, remaining;
    int has_deadline = (timeout > 0);
    if (has_deadline) pool_make_deadline(timeout, &deadline);

    /* All-or-nothing with retry, releasing any partial claim BEFORE waiting:
     * holding it across the wait let two over-demanding alloc_n callers deadlock. */
    uint32_t backoff = 0;
    for (;;) {
        uint32_t got = 0;
        for (; got < count; got++) {
            int64_t s = pool_try_alloc(h);
            if (s < 0) break;
            out[got] = (uint64_t)s;
        }
        if (got == count) return 1;
        if (got > 0) pool_free_n(h, out, got);   /* don't hold across the wait */

        if (has_deadline && !pool_remaining_time(&deadline, &remaining)) {
            __atomic_add_fetch(&hdr->stat_timeouts, 1, __ATOMIC_RELAXED);
            return 0;
        }
        /* Jittered (pid-seeded) backoff desyncs competing alloc_n callers,
         * breaking the symmetric grab/release livelock. */
        {
            long us = (long)(((getpid() & 0x7u) + backoff) % 40u) * 25 + 25;  /* 25..1000 us */
            if (backoff < 40) backoff++;
            struct timespec bt = { 0, us * 1000L };
            nanosleep(&bt, NULL);
        }
    }
}

/* ================================================================
 * Stale recovery -- CAS owner to narrow race window
 * ================================================================ */

static inline uint32_t pool_recover_stale(PoolHandle *h) {
    uint32_t recovered = 0;
    uint64_t cap = h->capacity;

    for (uint64_t slot = 0; slot < cap; slot++) {
        if (!pool_is_allocated(h, slot)) continue;
        uint32_t owner = __atomic_load_n(&h->owners[slot], __ATOMIC_ACQUIRE);
        if (owner == 0 || pool_pid_alive(owner)) continue;

        /* CAS owner from dead PID to 0 -- if it fails, slot was
         * re-allocated or already recovered by another process */
        if (!__atomic_compare_exchange_n(&h->owners[slot], &owner, 0,
                0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED))
            continue;

        /* We now own the right to free this slot's bitmap bit.
         *
         * Race window: between our owner-CAS and the bitmap-CAS below,
         * the bitmap word can transition via concurrent allocators (or,
         * with API misuse, a free+alloc cycle on the same bit). The bit
         * may be reset by an allocator before our CAS reaches it, or our
         * CAS may clear a bit a fresh allocator has just claimed.
         *
         * Mitigation: pre-CAS owner check (narrows window) + post-CAS
         * recovery accounting (always decrement used, since our CAS
         * succeeded against an "expected = bit set" state -- that bit is
         * gone from popcount). If post-CAS observes an owner already
         * stored, an allocator's CAS landed inside our window; we restore
         * the bit so their slot stays claimed. Their own used++ pairs
         * with our used-- to keep the counter in sync with popcount. */
        uint32_t widx = (uint32_t)(slot / 64);
        int bit = (int)(slot % 64);
        uint64_t mask = (uint64_t)1 << bit;

        for (;;) {
            uint64_t word = __atomic_load_n(&h->bitmap[widx], __ATOMIC_RELAXED);
            if (!(word & mask)) break;

            /* Pre-CAS: if a new allocator already populated owner, abort */
            if (__atomic_load_n(&h->owners[slot], __ATOMIC_ACQUIRE) != 0)
                break;

            uint64_t new_word = word & ~mask;
            if (__atomic_compare_exchange_n(&h->bitmap[widx], &word, new_word,
                    1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
                /* Post-CAS: if owner is now non-zero, a live allocator's
                 * CAS landed inside our window. Restore the bit so their
                 * claim stays valid; their used++ already happened. */
                if (__atomic_load_n(&h->owners[slot], __ATOMIC_ACQUIRE) != 0)
                    __atomic_or_fetch(&h->bitmap[widx], mask, __ATOMIC_RELEASE);
                /* Account for the stale slot's bit we just cleared. Use a
                 * saturating decrement: if a prior allocator was killed
                 * between owner-store and used++, used may already reflect
                 * fewer "real" allocations than popcount suggests, and a
                 * plain sub would underflow on subsequent recoveries. */
                uint32_t cur = __atomic_load_n(&h->hdr->used, __ATOMIC_RELAXED);
                while (cur > 0 && !__atomic_compare_exchange_n(&h->hdr->used,
                            &cur, cur - 1, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED))
                    ; /* CAS failed: cur reloaded with current value; retry */
                __atomic_add_fetch(&h->hdr->stat_frees, 1, __ATOMIC_RELAXED);
                /* StoreLoad barrier: see pool_free_slot. */
                __atomic_thread_fence(__ATOMIC_SEQ_CST);
                if (__atomic_load_n(&h->hdr->waiters, __ATOMIC_RELAXED) > 0)
                    syscall(SYS_futex, &h->hdr->used, FUTEX_WAKE, 1, NULL, NULL, 0);
                recovered++;
                break;
            }
        }
    }

    if (recovered > 0)
        __atomic_add_fetch(&h->hdr->stat_recoveries, recovered, __ATOMIC_RELAXED);

    return recovered;
}

/* ================================================================
 * Layout calculation
 * ================================================================ */

static inline void pool_calc_layout(uint64_t capacity, uint32_t elem_size,
                                     uint64_t *bitmap_off, uint64_t *owners_off,
                                     uint64_t *data_off, uint64_t *total_size) {
    uint64_t bwords = (capacity + 63) / 64;
    uint64_t bitmap_sz = bwords * 8;
    uint64_t owners_sz = POOL_ALIGN8(capacity * 4);
    uint64_t data_sz   = (uint64_t)capacity * elem_size;

    *bitmap_off   = sizeof(PoolHeader);
    *owners_off   = *bitmap_off + bitmap_sz;
    *data_off     = *owners_off + owners_sz;
    *total_size   = *data_off + data_sz;
}

/* ================================================================
 * Header initialization (shared by pool_create and pool_create_memfd)
 * ================================================================ */

static inline void pool_init_header(void *base, uint64_t total,
                                     uint32_t elem_size, uint32_t variant_id,
                                     uint64_t capacity, uint64_t bm_off,
                                     uint64_t own_off, uint64_t dat_off) {
    PoolHeader *hdr = (PoolHeader *)base;
    memset(base, 0, (size_t)total);
    hdr->version    = POOL_VERSION;
    hdr->elem_size  = elem_size;
    hdr->variant_id = variant_id;
    hdr->capacity   = capacity;
    hdr->total_size = total;
    hdr->data_off   = dat_off;
    hdr->bitmap_off = bm_off;
    hdr->owners_off = own_off;
    __atomic_thread_fence(__ATOMIC_SEQ_CST);
    /* Publish magic LAST, as a release store: it is the commit point, so a
       creator killed before it leaves magic==0 and never a file mistaken for
       a valid one.  A kill during the field stores leaves one to remove by
       hand. */
    __atomic_store_n(&hdr->magic, POOL_MAGIC, __ATOMIC_RELEASE);

}

/* ================================================================
 * Create / Open / Close
 * ================================================================ */

#define POOL_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, POOL_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while(0)

/* Max capacity to prevent bitmap_words (uint32_t) truncation */
#define POOL_MAX_CAPACITY ((uint64_t)UINT32_MAX * 64)

/* Fixed element width for the typed variants; 0 for RAW/STR (caller-chosen).
 * A crafted file could pair a typed variant_id with a wrong elem_size and drive
 * the typed accessors past a slot or into a misaligned atomic; pin it here. */
static inline uint32_t pool_variant_elem_size(uint32_t variant_id) {
    switch (variant_id) {
        case POOL_VAR_I64: case POOL_VAR_F64: return 8;
        case POOL_VAR_I32:                    return 4;
        default:                              return 0;
    }
}

/* Validate header: magic, version, variant, sizes, AND layout offsets. */
static inline int pool_validate_header(const PoolHeader *hdr, uint64_t file_size,
                                        uint32_t expected_variant) {
    if (hdr->magic != POOL_MAGIC) return 0;
    if (hdr->version != POOL_VERSION) return 0;
    if (hdr->variant_id != expected_variant) return 0;
    { uint32_t w = pool_variant_elem_size(hdr->variant_id);
      if (w && hdr->elem_size != w) return 0; }   /* typed variant: elem_size must match its fixed width */
    if (hdr->capacity == 0 || hdr->capacity > POOL_MAX_CAPACITY) return 0;
    if (hdr->elem_size == 0) return 0;
    /* Full-layout overflow check, identical to the one the create paths apply:
     * total = header + bitmap + owners + data. Counting only sizeof(PoolHeader)
     * here was weaker than at creation, so a crafted header could wrap
     * total_size mod 2^64, satisfy the equality checks below, and leave the
     * derived slot pointers far outside the mapping.
     * capacity <= POOL_MAX_CAPACITY (checked above) bounds the bitmap/owners
     * products, so computing the overhead cannot itself overflow. */
    {   uint64_t overhead = sizeof(PoolHeader)
                          + ((hdr->capacity + 63) / 64) * 8
                          + POOL_ALIGN8(hdr->capacity * 4);
        if (hdr->capacity > (UINT64_MAX - overhead) / hdr->elem_size) return 0;
    }
    if (hdr->total_size != file_size) return 0;

    uint64_t bm_off, own_off, dat_off, total;
    pool_calc_layout(hdr->capacity, hdr->elem_size, &bm_off, &own_off, &dat_off, &total);

    if (hdr->bitmap_off != bm_off) return 0;
    if (hdr->owners_off != own_off) return 0;
    if (hdr->data_off   != dat_off) return 0;
    if (hdr->total_size != total)   return 0;
    return 1;
}

static inline PoolHandle *pool_setup_handle(void *base, size_t map_size,
                                             const char *path, int backing_fd) {
    PoolHeader *hdr = (PoolHeader *)base;
    PoolHandle *h = (PoolHandle *)calloc(1, sizeof(PoolHandle));
    if (!h) { munmap(base, map_size); if (backing_fd >= 0) close(backing_fd); return NULL; }

    h->hdr         = hdr;
    h->bitmap      = (uint64_t *)((uint8_t *)base + hdr->bitmap_off);
    h->owners      = (uint32_t *)((uint8_t *)base + hdr->owners_off);
    h->data        = (uint8_t *)base + hdr->data_off;
    h->mmap_size   = map_size;
    h->bitmap_words = (uint32_t)((hdr->capacity + 63) / 64);
    h->capacity    = hdr->capacity;   /* validated at attach; use cached copy for all indexing/bounds */
    h->elem_size   = hdr->elem_size;  /* validated at attach; use cached copy for all slot addressing */
    h->path        = path ? strdup(path) : NULL;
    h->notify_fd   = -1;
    h->backing_fd  = backing_fd;
    h->scan_hint   = (uint32_t)getpid() % h->bitmap_words;

    return h;
}

/* Securely obtain a fd: create exclusively (O_CREAT|O_EXCL|O_NOFOLLOW at mode,
 * default 0600), or attach an existing file (O_RDWR|O_NOFOLLOW, no O_CREAT). */
static int pool_secure_open(const char *path, mode_t mode, char *errbuf) {
    for (int attempt = 0; attempt < 100; attempt++) {
        int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, mode);
        if (fd >= 0) { (void)fchmod(fd, mode); return fd; }   /* exact mode: umask narrowed the O_EXCL create */
        if (errno != EEXIST) { POOL_ERR("create %s: %s", path, strerror(errno)); return -1; }
        fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
        if (fd >= 0) return fd;
        if (errno == ENOENT) continue;   /* creator unlinked between our two opens; retry */
        POOL_ERR("open %s: %s", path, strerror(errno));  /* ELOOP => symlink rejected */
        return -1;
    }
    POOL_ERR("open %s: create/attach kept racing", path);
    return -1;
}

/* True iff the whole mapped region is zero -- what an abandoned mid-init
   creator leaves.  Lets recovery re-init only a provably-empty file, never
   one that merely starts with a zero word.  Cold path, so a byte scan is
   fine. */
static inline int pool_region_is_zero(const void *p, size_t n) {
    const unsigned char *b = (const unsigned char *)p;
    for (size_t i = 0; i < n; i++) if (b[i]) return 0;
    return 1;
}

static PoolHandle *pool_create(const char *path, uint64_t capacity,
                                uint32_t elem_size, uint32_t variant_id,
                                mode_t mode, char *errbuf) {
    if (errbuf) errbuf[0] = '\0';

    if (capacity == 0) { POOL_ERR("capacity must be > 0"); return NULL; }
    if (elem_size == 0) { POOL_ERR("elem_size must be > 0"); return NULL; }
    if (capacity > POOL_MAX_CAPACITY) { POOL_ERR("capacity too large"); return NULL; }
    {   /* full-layout overflow check: total = header + bitmap + owners + data.
           capacity <= POOL_MAX_CAPACITY bounds the bitmap/owners products. */
        uint64_t overhead = sizeof(PoolHeader) + ((capacity + 63) / 64) * 8 + POOL_ALIGN8(capacity * 4);
        if (capacity > (UINT64_MAX - overhead) / elem_size) {
            POOL_ERR("capacity * elem_size overflow"); return NULL;
        }
    }

    uint64_t bm_off, own_off, dat_off, total;
    pool_calc_layout(capacity, elem_size, &bm_off, &own_off, &dat_off, &total);

    int fd = -1;
    int anonymous = (path == NULL);
    size_t map_size;
    void *base;

    if (anonymous) {
        map_size = (size_t)total;
        base = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
                     MAP_SHARED | MAP_ANONYMOUS, -1, 0);
        if (base == MAP_FAILED) {
            POOL_ERR("mmap(anonymous): %s", strerror(errno));
            return NULL;
        }
    } else {
        fd = pool_secure_open(path, mode, errbuf);
        if (fd < 0) return NULL;

        if (flock(fd, LOCK_EX) < 0) {
            POOL_ERR("flock(%s): %s", path, strerror(errno));
            close(fd); return NULL;
        }

        struct stat st;
        if (fstat(fd, &st) < 0) {
            POOL_ERR("fstat(%s): %s", path, strerror(errno));
            flock(fd, LOCK_UN); close(fd); return NULL;
        }

        int is_new = (st.st_size == 0);

        if (!is_new && (uint64_t)st.st_size < sizeof(PoolHeader)) {
            POOL_ERR("%s: file too small (%lld)", path, (long long)st.st_size);
            flock(fd, LOCK_UN); close(fd); return NULL;
        }

        if (is_new && (st.st_uid != geteuid() || fchmod(fd, mode) < 0)) {
            POOL_ERR("%s: refusing to initialize file not owned by us", path);
            flock(fd, LOCK_UN); close(fd); return NULL;
        }
        if (is_new) {
            if (ftruncate(fd, (off_t)total) < 0) {
                POOL_ERR("ftruncate(%s): %s", path, strerror(errno));
                flock(fd, LOCK_UN); close(fd); return NULL;
            }
        }

        map_size = is_new ? (size_t)total : (size_t)st.st_size;
        base = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
        if (base == MAP_FAILED) {
            POOL_ERR("mmap(%s): %s", path, strerror(errno));
            flock(fd, LOCK_UN); close(fd); return NULL;
        }

        if (!is_new) {
            if (!pool_validate_header((PoolHeader *)base, (uint64_t)st.st_size, variant_id)) {
                /* Recover an abandoned mid-init file: a creator killed
                 * between the ftruncate and the header init leaves a
                 * full-size, all-zero (magic==0) file that would brick every
                 * future open of this path.  Re-initialize ONLY when it is
                 * exactly our size, still uninitialized, and owned by us;
                 * anything else still errors. */
                if (((PoolHeader *)base)->magic == 0 && (uint64_t)st.st_size == total
                    && st.st_uid == geteuid() && pool_region_is_zero(base, map_size)) {
                    if (fchmod(fd, mode) < 0) {
                        POOL_ERR("%s: fchmod: %s", path, strerror(errno));
                        munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
                    }
                    pool_init_header(base, total, elem_size, variant_id, capacity,
                                     bm_off, own_off, dat_off);
                    flock(fd, LOCK_UN); close(fd);
                    return pool_setup_handle(base, map_size, path, -1);
                }
                if (((PoolHeader *)base)->magic == 0 && (uint64_t)st.st_size == total
                    && st.st_uid == geteuid())
                    POOL_ERR("%s: incomplete pool file left by an interrupted create; remove it and retry", path);
                else
                    POOL_ERR("%s: invalid or incompatible pool file", path);
                munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
            }
            /* Attaching an existing file must match the requested geometry, else
             * a caller that grew capacity/elem_size in code but kept the old file
             * would silently get the old (smaller) pool (shmget-style EINVAL). */
            PoolHeader *ehdr = (PoolHeader *)base;
            if (ehdr->capacity != capacity || ehdr->elem_size != elem_size) {
                POOL_ERR("%s: geometry mismatch (file capacity=%llu elem_size=%u; requested capacity=%llu elem_size=%u)",
                         path, (unsigned long long)ehdr->capacity, ehdr->elem_size,
                         (unsigned long long)capacity, elem_size);
                munmap(base, map_size); flock(fd, LOCK_UN); close(fd); return NULL;
            }
            flock(fd, LOCK_UN);
            close(fd);
            return pool_setup_handle(base, map_size, path, -1);
        }
    }

    /* Initialize header -- flock still held for file-backed new files */
    pool_init_header(base, total, elem_size, variant_id, capacity,
                     bm_off, own_off, dat_off);

    if (fd >= 0) {
        flock(fd, LOCK_UN);
        close(fd);
    }

    return pool_setup_handle(base, map_size, path, -1);
}

static PoolHandle *pool_create_memfd(const char *name, uint64_t capacity,
                                      uint32_t elem_size, uint32_t variant_id,
                                      char *errbuf) {
    if (errbuf) errbuf[0] = '\0';

    if (capacity == 0) { POOL_ERR("capacity must be > 0"); return NULL; }
    if (elem_size == 0) { POOL_ERR("elem_size must be > 0"); return NULL; }
    if (capacity > POOL_MAX_CAPACITY) { POOL_ERR("capacity too large"); return NULL; }
    {   /* full-layout overflow check: total = header + bitmap + owners + data.
           capacity <= POOL_MAX_CAPACITY bounds the bitmap/owners products. */
        uint64_t overhead = sizeof(PoolHeader) + ((capacity + 63) / 64) * 8 + POOL_ALIGN8(capacity * 4);
        if (capacity > (UINT64_MAX - overhead) / elem_size) {
            POOL_ERR("capacity * elem_size overflow"); return NULL;
        }
    }

    uint64_t bm_off, own_off, dat_off, total;
    pool_calc_layout(capacity, elem_size, &bm_off, &own_off, &dat_off, &total);

    int fd = memfd_create(name ? name : "pool", MFD_CLOEXEC | MFD_ALLOW_SEALING);
    if (fd < 0) { POOL_ERR("memfd_create: %s", strerror(errno)); return NULL; }

    if (ftruncate(fd, (off_t)total) < 0) {
        POOL_ERR("ftruncate(memfd): %s", strerror(errno));
        close(fd); return NULL;
    }

    /* Seal against shrink/grow to block ftruncate-based SIGBUS attacks via
     * SCM_RIGHTS-shared fds. Peers can still write; only size is immutable. */
    (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);

    void *base = mmap(NULL, (size_t)total, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (base == MAP_FAILED) {
        POOL_ERR("mmap(memfd): %s", strerror(errno));
        close(fd); return NULL;
    }

    pool_init_header(base, total, elem_size, variant_id, capacity,
                     bm_off, own_off, dat_off);

    return pool_setup_handle(base, (size_t)total, NULL, fd);
}

static PoolHandle *pool_open_fd(int fd, uint32_t variant_id, char *errbuf) {
    if (errbuf) errbuf[0] = '\0';

    struct stat st;
    if (fstat(fd, &st) < 0) {
        POOL_ERR("fstat(fd=%d): %s", fd, strerror(errno));
        return NULL;
    }

    if ((uint64_t)st.st_size < sizeof(PoolHeader)) {
        POOL_ERR("fd %d: too small (%lld)", fd, (long long)st.st_size);
        return NULL;
    }

    size_t map_size = (size_t)st.st_size;
    void *base = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (base == MAP_FAILED) {
        POOL_ERR("mmap(fd=%d): %s", fd, strerror(errno));
        return NULL;
    }

    if (!pool_validate_header((PoolHeader *)base, (uint64_t)st.st_size, variant_id)) {
        POOL_ERR("fd %d: invalid or incompatible pool", fd);
        munmap(base, map_size);
        return NULL;
    }

    int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
    if (myfd < 0) {
        POOL_ERR("fcntl(F_DUPFD_CLOEXEC): %s", strerror(errno));
        munmap(base, map_size);
        return NULL;
    }

    return pool_setup_handle(base, map_size, NULL, myfd);
}

static void pool_destroy(PoolHandle *h) {
    if (!h) return;
    if (h->notify_fd >= 0) close(h->notify_fd);
    if (h->backing_fd >= 0) close(h->backing_fd);
    if (h->hdr) munmap(h->hdr, h->mmap_size);
    free(h->path);
    free(h);
}

/* ================================================================
 * Eventfd integration
 * ================================================================ */

static int pool_create_eventfd(PoolHandle *h) {
    if (h->notify_fd >= 0) return h->notify_fd;
    int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
    if (efd < 0) return -1;
    h->notify_fd = efd;
    return efd;
}

static int pool_notify(PoolHandle *h) {
    if (h->notify_fd < 0) return 0;
    uint64_t val = 1;
    return write(h->notify_fd, &val, sizeof(val)) == sizeof(val);
}

static int64_t pool_eventfd_consume(PoolHandle *h) {
    if (h->notify_fd < 0) return -1;
    uint64_t val = 0;
    if (read(h->notify_fd, &val, sizeof(val)) != sizeof(val)) return -1;
    return (int64_t)val;
}

static int pool_msync(PoolHandle *h) {
    return msync(h->hdr, h->mmap_size, MS_SYNC);
}

/* ================================================================
 * Typed accessors -- integers (atomic)
 * ================================================================ */

static inline int64_t pool_get_i64(PoolHandle *h, uint64_t slot) {
    return __atomic_load_n((int64_t *)pool_slot_ptr(h, slot), __ATOMIC_RELAXED);
}

static inline void pool_set_i64(PoolHandle *h, uint64_t slot, int64_t val) {
    __atomic_store_n((int64_t *)pool_slot_ptr(h, slot), val, __ATOMIC_RELAXED);
}

static inline int pool_cas_i64(PoolHandle *h, uint64_t slot,
                                int64_t expected, int64_t desired) {
    return __atomic_compare_exchange_n((int64_t *)pool_slot_ptr(h, slot),
            &expected, desired, 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
}

static inline int64_t pool_cmpxchg_i64(PoolHandle *h, uint64_t slot,
                                        int64_t expected, int64_t desired) {
    __atomic_compare_exchange_n((int64_t *)pool_slot_ptr(h, slot),
            &expected, desired, 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
    return expected;
}

static inline int64_t pool_xchg_i64(PoolHandle *h, uint64_t slot, int64_t val) {
    return __atomic_exchange_n((int64_t *)pool_slot_ptr(h, slot), val, __ATOMIC_ACQ_REL);
}

static inline int64_t pool_add_i64(PoolHandle *h, uint64_t slot, int64_t delta) {
    return __atomic_add_fetch((int64_t *)pool_slot_ptr(h, slot), delta, __ATOMIC_ACQ_REL);
}

static inline int32_t pool_get_i32(PoolHandle *h, uint64_t slot) {
    return __atomic_load_n((int32_t *)pool_slot_ptr(h, slot), __ATOMIC_RELAXED);
}

static inline void pool_set_i32(PoolHandle *h, uint64_t slot, int32_t val) {
    __atomic_store_n((int32_t *)pool_slot_ptr(h, slot), val, __ATOMIC_RELAXED);
}

static inline int pool_cas_i32(PoolHandle *h, uint64_t slot,
                                int32_t expected, int32_t desired) {
    return __atomic_compare_exchange_n((int32_t *)pool_slot_ptr(h, slot),
            &expected, desired, 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
}

static inline int32_t pool_cmpxchg_i32(PoolHandle *h, uint64_t slot,
                                        int32_t expected, int32_t desired) {
    __atomic_compare_exchange_n((int32_t *)pool_slot_ptr(h, slot),
            &expected, desired, 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
    return expected;
}

static inline int32_t pool_xchg_i32(PoolHandle *h, uint64_t slot, int32_t val) {
    return __atomic_exchange_n((int32_t *)pool_slot_ptr(h, slot), val, __ATOMIC_ACQ_REL);
}

static inline int32_t pool_add_i32(PoolHandle *h, uint64_t slot, int32_t delta) {
    return __atomic_add_fetch((int32_t *)pool_slot_ptr(h, slot), delta, __ATOMIC_ACQ_REL);
}

/* ================================================================
 * Typed accessors -- float (non-atomic)
 * ================================================================ */

static inline double pool_get_f64(PoolHandle *h, uint64_t slot) {
    double v;
    memcpy(&v, pool_slot_ptr(h, slot), sizeof(double));
    return v;
}

static inline void pool_set_f64(PoolHandle *h, uint64_t slot, double val) {
    memcpy(pool_slot_ptr(h, slot), &val, sizeof(double));
}

/* ================================================================
 * Typed accessors -- string (4-byte length prefix + data)
 * ================================================================ */

static inline uint32_t pool_get_str_len(PoolHandle *h, uint64_t slot) {
    /* elem_size is attacker-controlled (a local peer can craft the backing file);
     * a Str slot must be large enough to hold the 4-byte length prefix. If it is
     * not, the file is corrupt: skip the prefix read and return 0 rather than let
     * max_len underflow and defeat the clamp below (which would otherwise return a
     * ~4 GiB length -> OOB read). Legit Str pools have elem_size >= 5, so this is a
     * never-taken branch for valid data. */
    uint32_t elem = h->elem_size;
    if (elem <= sizeof(uint32_t)) return 0;
    uint32_t len;
    memcpy(&len, pool_slot_ptr(h, slot), sizeof(uint32_t));
    uint32_t max_len = elem - (uint32_t)sizeof(uint32_t);
    if (len > max_len) len = max_len;
    return len;
}

static inline const char *pool_get_str_ptr(PoolHandle *h, uint64_t slot) {
    return (const char *)(pool_slot_ptr(h, slot) + sizeof(uint32_t));
}

static inline void pool_set_str(PoolHandle *h, uint64_t slot,
                                 const char *str, uint32_t len) {
    /* See pool_get_str_len: reject slots too small to hold the length prefix so
     * the max_len computation cannot underflow (which would defeat the clamp and
     * let the data memcpy run off the end of the data region -> OOB write). */
    uint32_t elem = h->elem_size;
    if (elem <= sizeof(uint32_t)) return;
    uint32_t max_len = elem - (uint32_t)sizeof(uint32_t);
    if (len > max_len) len = max_len;
    memcpy(pool_slot_ptr(h, slot), &len, sizeof(uint32_t));
    memcpy(pool_slot_ptr(h, slot) + sizeof(uint32_t), str, len);
}

/* ================================================================
 * Reset -- free all slots (NOT concurrency-safe, caller must
 * ensure no other process is accessing the pool)
 * ================================================================ */

static inline void pool_reset(PoolHandle *h) {
    PoolHeader *hdr = h->hdr;
    memset(h->bitmap, 0, (size_t)h->bitmap_words * 8);
    memset(h->owners, 0, (size_t)h->capacity * 4);
    __atomic_store_n(&hdr->used, 0, __ATOMIC_RELEASE);
    /* StoreLoad barrier: see pool_free_slot (reset is documented single-writer,
     * but keep the handshake correct if a waiter is nonetheless parked). */
    __atomic_thread_fence(__ATOMIC_SEQ_CST);
    if (__atomic_load_n(&hdr->waiters, __ATOMIC_RELAXED) > 0)
        syscall(SYS_futex, &hdr->used, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
}

#endif /* POOL_H */
