/*
 * shm_generic.h -- Macro-template for shared-memory hash maps.
 *
 * Before including, define:
 *   SHM_PREFIX         -- function prefix (e.g., shm_ii)
 *   SHM_NODE_TYPE      -- node struct name
 *   SHM_VARIANT_ID     -- unique integer for header validation
 *
 * Key type (choose one):
 *   SHM_KEY_IS_INT + SHM_KEY_INT_TYPE  -- integer key
 *   (leave undefined for string keys via arena)
 *
 * Value type (choose one):
 *   SHM_VAL_IS_STR                      -- string value via arena
 *   SHM_VAL_INT_TYPE                    -- integer value
 *
 * Optional:
 *   SHM_HAS_COUNTERS  -- generate incr/decr/incr_by, max/min, and integer cas (integer values only)
 */

/* ---- Part 1: shared definitions (included once) ---- */

#ifndef SHM_DEFS_H
#define SHM_DEFS_H

#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <time.h>
#include <sys/mman.h>
#include <sys/stat.h>

#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#error "shm_generic.h: inline string packing requires little-endian architecture"
#endif
#include <sys/file.h>
#include <pthread.h>  /* pthread_atfork -- available in libc on modern glibc; no -lpthread needed */
#include <sys/syscall.h>
#include <limits.h>
#include <signal.h>
#include <errno.h>
#include <linux/futex.h>

#ifdef __SSE2__
#include <emmintrin.h>
#endif

#define XXH_INLINE_ALL
#include "xxhash.h"

/* ---- Constants ---- */

#define SHM_MAGIC       0x53484D31U  /* "SHM1" */
#define SHM_VERSION     10U  /* 10: added the occupancy bitmap region (layout change) */
/* Ceiling on new_sharded's shard count: the power-of-two round-up must not
 * overflow. */
#define SHM_MAX_SHARDS  4096U

#ifndef SHM_READER_SLOTS
#define SHM_READER_SLOTS 1024  /* max concurrent reader processes for dead-process recovery */
#endif
/* One bit per reader slot, set on claim and cleared on clean release, so a
 * writer's drain scan visits only occupied slots. */
#define SHM_OCC_WORDS   (((SHM_READER_SLOTS) + 63) / 64)   /* 16 for 1024 slots */
#define SHM_OCC_BYTES   ((uint64_t)SHM_OCC_WORDS * 8)      /* 128 bytes */
#define SHM_INITIAL_CAP 16
#define SHM_MAX_STR_LEN 0x3FFFFFFFU  /* ~1GB, bit 30 reserved for inline flag */
#define SHM_LRU_NONE    UINT32_MAX

/* UINT32_MAX = use default TTL; 0 = no TTL; other = per-key TTL */
#define SHM_TTL_USE_DEFAULT UINT32_MAX

/* Which half of the 64-bit hash picks the shard.  A set keeps the scheme it was
 * written under: re-routing would move keys between shard files. */
#define SHM_ROUTING_LEGACY 0  /* low half, the half the in-shard probe uses */
#define SHM_ROUTING_SPLIT  1  /* high half, which the slot index ignores */

#define SHM_IS_EXPIRED(h, i, now) \
    ((h)->expires_at && (h)->expires_at[(i)] && \
     (now) >= (h)->expires_at[(i)])

/* Fast monotonic seconds -- avoids time() syscall overhead.
 * CLOCK_MONOTONIC_COARSE is always vDSO on Linux (~2ns). */
static inline uint32_t shm_now(void) {
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
    return (uint32_t)ts.tv_sec;
}

static inline uint32_t shm_expiry_ts(uint32_t ttl) {
    uint64_t sum = (uint64_t)shm_now() + ttl;
    return (sum > UINT32_MAX) ? UINT32_MAX : (uint32_t)sum;
}

#define SHM_EMPTY     0
#define SHM_TOMBSTONE 1
#define SHM_TAG_MIN   2   /* state values 2-255 = LIVE with hash tag */
#define SHM_IS_LIVE(st) ((st) >= SHM_TAG_MIN)
#define SHM_MAKE_TAG(hash) ((uint8_t)(((hash) >> 24) % 254 + SHM_TAG_MIN))
/* Invariant: TOMBSTONE < TAG_MIN so tag-based probe filtering works.
 * Compile-time check via negative-size array trick: */
typedef char shm_tag_invariant_check[(SHM_TOMBSTONE < SHM_TAG_MIN) ? 1 : -1];

/* Find the next live slot at or after *pos; 1 and *pos updated, or 0. */
static inline int shm_find_next_live(const uint8_t *states, uint32_t cap, uint32_t *pos) {
    uint32_t i = *pos;
#ifdef __SSE2__
    /* Live is neither empty(0) nor tombstone(1), by unsigned saturation:
     * sub_sat(byte, 1) > 0 iff byte >= 2 = SHM_TAG_MIN */
    __m128i ones = _mm_set1_epi8(1);
    uint32_t align_end = (i + 15) & ~(uint32_t)15;
    if (align_end > cap) align_end = cap;
    for (; i < align_end; i++) {
        if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; }
    }
    /* SIMD scan: subs_u8(chunk, 1) is nonzero for bytes >= 2 */
    for (; i + 16 <= cap; i += 16) {
        __m128i chunk = _mm_loadu_si128((const __m128i *)(states + i));
        __m128i sub = _mm_subs_epu8(chunk, ones);  /* unsigned saturating sub */
        int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(sub, _mm_setzero_si128()));
        mask = ~mask & 0xFFFF;  /* invert: bits set where sub != 0 (live) */
        if (mask) { *pos = i + __builtin_ctz(mask); return 1; }
    }
#endif
    for (; i < cap; i++) {
        if (states[i] >= SHM_TAG_MIN) { *pos = i; return 1; }
    }
    return 0;
}

/* Scan 16 state bytes for tag hits and empty slots, returned as bitmasks.
 * Contiguous memory only: the caller handles table wrap. */
#ifdef __SSE2__
static inline void shm_probe_group(const uint8_t *states, uint32_t pos,
                                    uint8_t tag, uint16_t *match_mask,
                                    uint16_t *empty_mask) {
    __m128i group = _mm_loadu_si128((const __m128i *)(states + pos));
    __m128i tag_v = _mm_set1_epi8((char)tag);
    __m128i zero_v = _mm_setzero_si128();
    *match_mask = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(group, tag_v));
    *empty_mask = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(group, zero_v));
}
#endif

#define SHM_ARENA_NUM_CLASSES 16  /* 2^4..2^19 = 16..524288 */
#define SHM_EVICT_SEARCH 32       /* oldest entries searched for a block of the request's class */
#define SHM_ARENA_MIN_ALLOC   16

/* ---- UTF-8 and inline-string flag packing ----
 * key_len / val_len (uint32_t): bit 31 = UTF-8, bit 30 = inline, bits 0-29 =
 * length.  Inline: _off plus bits 0-23 of _len hold up to 7 bytes of data,
 * bits 24-26 its length.  Arena: _off is the offset, bits 0-29 the length. */

#define SHM_UTF8_FLAG    ((uint32_t)0x80000000U)
#define SHM_INLINE_FLAG  ((uint32_t)0x40000000U)
#define SHM_LEN_MASK     ((uint32_t)0x3FFFFFFFU)
#define SHM_INLINE_MAX   7  /* max bytes that fit inline */

#define SHM_PACK_LEN(len, utf8)   ((uint32_t)(len) | ((utf8) ? SHM_UTF8_FLAG : 0))
#define SHM_UNPACK_LEN(packed)    ((uint32_t)((packed) & SHM_LEN_MASK))
#define SHM_UNPACK_UTF8(packed)   (((packed) & SHM_UTF8_FLAG) != 0)
#define SHM_IS_INLINE(packed)     (((packed) & SHM_INLINE_FLAG) != 0)

#define SHM_STR_LEN(packed) \
    (SHM_IS_INLINE(packed) ? shm_inline_len(packed) : SHM_UNPACK_LEN(packed))

static inline void shm_inline_pack(uint32_t *off, uint32_t *len_field,
                                    const char *str, uint32_t slen, bool utf8) {
    uint32_t lf = SHM_INLINE_FLAG | ((uint32_t)slen << 24);
    if (utf8) lf |= SHM_UTF8_FLAG;
    uint32_t o = 0;
    memcpy(&o, str, slen > 4 ? 4 : slen);
    if (slen > 4) {
        uint32_t rest = 0;
        memcpy(&rest, str + 4, slen - 4);
        lf |= rest;
    }
    /* Publish through the inline-empty state: shm_str_free is a no-op on an
     * inline field, so a writer killed between these stores leaves no (off,len)
     * the free list would misclassify.  Atomic, or the dead interim store is
     * optimised away. */
    __atomic_store_n(len_field, SHM_INLINE_FLAG, __ATOMIC_RELEASE);
    __atomic_store_n(off, o, __ATOMIC_RELEASE);
    __atomic_store_n(len_field, lf, __ATOMIC_RELEASE);
}

static inline uint32_t shm_inline_len(uint32_t len_field) {
    return (len_field >> 24) & 0x7;
}

static inline const char *shm_inline_read(uint32_t off, uint32_t len_field,
                                           char *buf) {
    uint32_t slen = shm_inline_len(len_field);
    memcpy(buf, &off, slen > 4 ? 4 : slen);
    if (slen > 4) {
        uint32_t rest = len_field & 0x00FFFFFFU;
        memcpy(buf + 4, &rest, slen - 4);
    }
    return buf;
}

/* Get string pointer + length, handling both inline and arena modes.
 * For inline, copies to buf and returns buf. For arena, returns arena pointer directly. */
static inline const char *shm_str_ptr(uint32_t off, uint32_t len_field,
                                       const char *arena, uint64_t arena_cap,
                                       char *inline_buf,
                                       uint32_t *out_len) {
    if (SHM_IS_INLINE(len_field)) {
        *out_len = shm_inline_len(len_field);
        return shm_inline_read(off, len_field, inline_buf);
    }
    uint32_t len = SHM_UNPACK_LEN(len_field);
    if ((uint64_t)off + len > arena_cap) {
        /* off/len come from a peer-writable node: an out-of-bounds pointer
         * would reach newSVpvn (CWE-125).  Deliver an empty string. */
        *out_len = 0;
        return inline_buf;
    }
    *out_len = len;
    return arena + off;
}

/* ---- Shared memory header (256 bytes, 4 cache lines, in mmap) ---- */

#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
#define SHM_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)
#else
#define SHM_STATIC_ASSERT(cond, msg)
#endif

typedef struct {
    /* ---- Cache line 0 (0-63): immutable after create ---- */
    uint32_t magic;           /* 0 */
    uint32_t version;         /* 4 */
    uint32_t variant_id;      /* 8 */
    uint32_t node_size;       /* 12 */
    uint32_t max_table_cap;   /* 16 */
    uint32_t table_cap;       /* 20: changes on resize only */
    uint32_t max_size;        /* 24: LRU capacity, 0 = disabled */
    uint32_t default_ttl;     /* 28: TTL seconds, 0 = disabled */
    uint64_t total_size;      /* 32 */
    uint64_t nodes_off;       /* 40 */
    uint64_t states_off;      /* 48 */
    uint64_t arena_off;       /* 56 */

    /* ---- Cache line 1 (64-127): seqlock + read-path data ---- */
    uint32_t seq;             /* 64: seqlock counter, odd = writer active */
    uint32_t drain_seq;       /* 68: futex bumped by a reader releasing under a draining writer
                                 (wakes it). Was rwlock_writers_waiting; same offset/size. */
    uint64_t arena_cap;       /* 72: immutable, read by seqlock string path */
    uint64_t reader_slots_off;/* 80: offset of reader-PID slot table for dead-reader recovery */
    uint32_t slotless_rdepth; /* 88: read-locks held by readers with no reader-slot (documented
                                 residual). Was slotless_readers; same offset/size. */
    uint32_t arena_large_free;/* 92: head of the >2^19 large-block free list (was reserved; 0=empty) */
    uint8_t  sealed;          /* 96: 0 = mutable, 1 = frozen (read-only; lock-free reads).
                                 Carved from the reserved pad -- struct size is unchanged
                                 (see the _Static_assert below) so there is NO on-disk
                                 version bump; a pre-freeze file has this byte 0 (zeroed pad). */
    uint8_t  routing;         /* 97: SHM_ROUTING_*; carved from the pad like `sealed`,
                                 so a file written before 0.20 reads 0 (legacy). */
    uint8_t  shard_log2;      /* 98: log2(shards)+1 for a set written by 0.20+; 0 =
                                 a single map, a set predating the stamp, a shard
                                 the creator had not stamped when it died, or one
                                 created by an open that was then refused */
    uint8_t  _reserved1a;     /* 99 */
    uint32_t pop_cursor;      /* 100: next slot a non-LRU pop or drain examines (wraps);
                                 carved from the pad, so a file written before it reads 0 */
    uint32_t shift_cursor;    /* 104: the slot above the next one a non-LRU shift examines,
                                 0 meaning the top of the table */
    uint8_t  _reserved1[20];  /* 108-127 */

    /* ---- Cache line 2 (128-191): rwlock + write-hot fields ---- */
    uint32_t wlock;           /* 128: WRITER word ONLY: 0 (free) or 0x80000000|pid.  NOT a reader count. */
    uint32_t rwait;           /* 132: parked-waiter hint (readers+writers blocked on wlock); over-count-safe */
    uint32_t size;            /* 136 */
    uint32_t tombstones;      /* 140 */
    uint32_t lru_head;        /* 144: MRU slot index */
    uint32_t lru_tail;        /* 148: LRU slot index */
    uint32_t flush_cursor;    /* 152: partial flush_expired scan cursor */
    uint32_t table_gen;       /* 156: incremented on every resize */
    uint64_t arena_bump;      /* 160 */
    uint64_t stat_evictions;  /* 168: cumulative LRU eviction count */
    uint64_t stat_expired;    /* 176: cumulative TTL expiration count */
    uint32_t stat_recoveries; /* 184: cumulative stale lock recovery count */
    uint32_t lru_skip;        /* 188: promotion skip mask (power-of-2 minus 1, 0=strict LRU) */

    /* ---- Cache line 3 (192-255): arena free lists ---- */
    uint32_t arena_free[SHM_ARENA_NUM_CLASSES]; /* 192-255 */
} ShmHeader;

SHM_STATIC_ASSERT(sizeof(ShmHeader) == 256, "ShmHeader must be exactly 256 bytes (4 cache lines)");
/* shm_create_sharded claims the shard count by CASing shard_log2 across
 * processes: libatomic's fallback lock table is per process, so both creators
 * would believe they won. */
SHM_STATIC_ASSERT(__atomic_always_lock_free(sizeof(((ShmHeader *)0)->shard_log2), 0),
                  "cross-process CAS on shard_log2 needs a lock-free byte atomic");

/* Per-process slot for dead-process recovery.  A reader's entire contribution
 * to the lock is `rdepth` in its own slot, so a draining writer neutralises a
 * dead reader by clearing the slot's pid.  _rsv1/_rsv2 preserve the 16-byte
 * slot size of the released builds. */
typedef struct {
    uint32_t pid;      /* 0 = unclaimed */
    uint32_t rdepth;   /* read-locks THIS process currently holds (recursion-safe) */
    uint32_t _rsv1;    /* reserved (was waiters_parked); unused, kept for layout size */
    uint32_t _rsv2;    /* reserved (was writers_parked); unused, kept for layout size */
} ShmReaderSlot;

/* ---- Process-local handle ---- */

typedef struct ShmHandle_s {
    ShmHeader *hdr;
    void      *nodes;
    uint8_t   *states;
    char      *arena;
    uint32_t  *lru_prev;    /* NULL if LRU disabled */
    uint32_t  *lru_next;    /* NULL if LRU disabled */
    uint8_t   *lru_accessed; /* NULL if LRU disabled -- clock second-chance bit */
    uint32_t  *expires_at;  /* NULL if TTL disabled */
    ShmReaderSlot *reader_slots; /* SHM_READER_SLOTS entries */
    uint64_t  *occ;          /* SHM_OCC_WORDS-word slot-occupancy bitmap (trusted layout offset) */
    uint32_t   my_slot_idx;  /* UINT32_MAX if all slots taken (no recovery for this handle) */
    uint32_t   cached_pid;   /* getpid() cached at last slot claim */
    uint32_t   cached_fork_gen; /* shm_fork_gen value at last slot claim -- mismatch triggers reclaim */
    uint32_t slotless_held; /* rwlock read-locks held with no reader-slot */
    uint32_t lock_depth;    /* locks this process holds via RDLOCK_GUARD/WRSEQ_GUARD */
    uint8_t  pending_close; /* DESTROY arrived while lock_depth > 0; free at depth 0 */
    int      readonly;      /* 1 = frozen O_RDONLY/PROT_READ view: reads lock-free, mutation croaks.
                               A read-only handle NEVER writes the mapping (no rdepth, no clock bit). */
    size_t     mmap_size;
    uint32_t   max_mask;    /* max_table_cap - 1, for seqlock bounds clamping */
    uint32_t   iter_pos;
    char      *copy_buf;
    uint32_t   copy_buf_size;
    uint32_t   iterating;   /* active iterator count (each + cursors) */
    uint32_t   iter_gen;    /* table_gen snapshot for each() */
    uint8_t    iter_active; /* 1 = built-in each is in progress */
    uint8_t    deferred;    /* shrink/compact deferred while iterating */
    char      *path;        /* backing file path (strdup'd) */
    int        backing_fd;  /* memfd fd to close on destroy, -1 otherwise */
    /* Sharding: if shard_handles != NULL, this is a sharded map dispatcher */
    struct ShmHandle_s **shard_handles; /* NULL for single map */
    uint32_t   num_shards;
    uint32_t   shard_mask;     /* num_shards - 1 (power of 2) */
    uint8_t    route_high;     /* shard on the high hash half (see SHM_ROUTING_*) */
    uint32_t   shard_iter;     /* current shard for each()/cursor iteration */
    uint32_t   shard_rr;       /* round-robin start for pop/shift/drain: separate
                                  from shard_iter so draining a sharded map during
                                  an each() cannot skip its unvisited shards */
} ShmHandle;

/* ---- Cursor (independent iterator) ---- */

typedef struct {
    ShmHandle *handle;       /* for single maps, direct handle; for sharded, the dispatcher */
    ShmHandle *current;      /* current shard handle (== handle for single maps) */
    SV        *owner;        /* ref to the map's referent SV; keeps the mmap/handle alive while the cursor lives */
    uint32_t   iter_pos;
    uint32_t   gen;          /* table_gen snapshot -- reset on mismatch */
    uint32_t   shard_idx;    /* current shard index (0 for single maps) */
    uint32_t   shard_count;  /* total shards (1 for single maps) */
    char      *copy_buf;
    uint32_t   copy_buf_size;
} ShmCursor;

/* Grow a copy buffer to hold `needed` bytes; returns 0 on OOM */
static inline int shm_grow_buf(char **buf, uint32_t *cap, uint32_t needed) {
    if (needed == 0) needed = 1;
    if (needed <= *cap) return 1;
    uint32_t ns = *cap ? *cap : 64;
    while (ns < needed) {
        uint32_t next = ns * 2;
        if (next <= ns) { ns = needed; break; } /* overflow guard */
        ns = next;
    }
    char *nb = (char *)realloc(*buf, ns);
    if (!nb) return 0;
    *buf = nb;
    *cap = ns;
    return 1;
}

static inline int shm_ensure_copy_buf(ShmHandle *h, uint32_t needed) {
    return shm_grow_buf(&h->copy_buf, &h->copy_buf_size, needed);
}

static inline int shm_cursor_ensure_copy_buf(ShmCursor *c, uint32_t needed) {
    return shm_grow_buf(&c->copy_buf, &c->copy_buf_size, needed);
}

/* ---- Hash functions (xxHash, XXH3) ---- */

static inline uint64_t shm_hash_int64(int64_t key) {
    return XXH3_64bits(&key, sizeof(key));
}

static inline uint64_t shm_hash_string(const char *data, uint32_t len) {
    return XXH3_64bits(data, (size_t)len);
}

/* ---- Futex-based read-write lock ---- */

#define SHM_RWLOCK_SPIN_LIMIT 32
#define SHM_LOCK_TIMEOUT_SEC  2  /* FUTEX_WAIT timeout for stale lock detection */

static inline void shm_rwlock_spin_pause(void) {
#if defined(__x86_64__) || defined(__i386__)
    __asm__ volatile("pause" ::: "memory");
#elif defined(__aarch64__)
    __asm__ volatile("yield" ::: "memory");
#else
    __asm__ volatile("" ::: "memory");
#endif
}

/* Writer word encoding: WRITER_BIT|pid when write-locked, 0 when free. */
#define SHM_RWLOCK_WRITER_BIT 0x80000000U
#define SHM_RWLOCK_PID_MASK   0x7FFFFFFFU
#define SHM_RWLOCK_WR(pid)    (SHM_RWLOCK_WRITER_BIT | ((uint32_t)(pid) & SHM_RWLOCK_PID_MASK))

/* A zombie still answers kill(pid,0) as alive, so a crashed lock holder that
 * lingers unreaped would never be recovered: treat /proc/<pid>/stat state 'Z'
 * as dead.  An unreadable /proc falls back to alive, never force-recovering a
 * possibly-live holder. */
static inline int shm_pid_is_zombie(uint32_t pid) {
    char path[32], buf[256];
    snprintf(path, sizeof(path), "/proc/%u/stat", (unsigned)pid);
    int fd = open(path, O_RDONLY | O_CLOEXEC);
    if (fd < 0) return 0;
    ssize_t n = read(fd, buf, sizeof(buf) - 1);
    close(fd);
    if (n <= 0) return 0;
    buf[n] = '\0';
    /* "pid (comm) state ..."; comm may contain ')', so scan to the last one. */
    char *rp = strrchr(buf, ')');
    if (!rp || rp + 2 >= buf + n) return 0;   /* need ") X" within the bytes read */
    return rp[1] == ' ' && rp[2] == 'Z';
}
/* 1 if alive or unknown, 0 if definitely dead.  A recycled pid reports alive
 * and its slot is not reclaimed until that process exits. */
static inline int shm_pid_alive(uint32_t pid) {
    if (pid == 0) return 0; /* no owner recorded: not a live holder.  kill(0,0)
                               signals our own process group, so it could never
                               answer for one; the lock paths would spin. */
    if (kill((pid_t)pid, 0) == -1 && errno == ESRCH) return 0; /* definitely dead */
    return !shm_pid_is_zombie(pid); /* kill() also succeeds for a zombie -> treat as dead */
}

static void shm_lru_rebuild_if_corrupt(ShmHandle *h);
static void shm_recount_counters(ShmHandle *h);

/* Force-recover a stale write lock left by a dead process: CAS to our own pid,
 * so a later recoverer can re-recover if we crash mid-repair. */
static inline void shm_recover_stale_lock(ShmHandle *h, uint32_t observed_wlock) {
    ShmHeader *hdr = h->hdr;
    uint32_t mypid = SHM_RWLOCK_WR((uint32_t)getpid());
    if (!__atomic_compare_exchange_n(&hdr->wlock, &observed_wlock,
            mypid, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
        return;
    /* Clear the dead writer's odd seq first so spinning readers proceed: the
     * repair below touches only the LRU arrays and counters, which no lock-free
     * reader reads. */
    uint32_t seq = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
    if (seq & 1)
        __atomic_store_n(&hdr->seq, seq + 1, __ATOMIC_RELEASE);
    shm_lru_rebuild_if_corrupt(h);
    shm_recount_counters(h);
    __atomic_add_fetch(&hdr->stat_recoveries, 1, __ATOMIC_RELAXED);
    /* Release the lock */
    __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
    if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
        syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
}

static const struct timespec shm_lock_timeout = { SHM_LOCK_TIMEOUT_SEC, 0 };

/* Bumped in the pthread_atfork child callback so a handle detects a fork on
 * its next lock call without a getpid() on the hot path. */
static uint32_t shm_fork_gen = 1;
static pthread_once_t shm_atfork_once = PTHREAD_ONCE_INIT;
static void shm_on_fork_child(void) {
    __atomic_add_fetch(&shm_fork_gen, 1, __ATOMIC_RELAXED);
}
static void shm_atfork_init(void) {
    pthread_atfork(NULL, NULL, shm_on_fork_child);
}

/* Set a slot's bit on claim, clear it on a clean release: a crash leaves the
 * bit set and a later writer scan or re-claim reclaims it, as it does the pid.
 * SEQ_CST: the bit-set is ordered before the slot's rdepth can go non-zero, so
 * a writer's bitmap scan never misses a committed reader. */
static inline void shm_occ_set(ShmHandle *h, uint32_t s) {
    __atomic_fetch_or(&h->occ[s >> 6], (uint64_t)1 << (s & 63), __ATOMIC_SEQ_CST);
}
static inline void shm_occ_clear(ShmHandle *h, uint32_t s) {
    __atomic_fetch_and(&h->occ[s >> 6], ~((uint64_t)1 << (s & 63)), __ATOMIC_SEQ_CST);
}

/* Ensure this process owns a reader slot, so a forked child takes its own
 * rather than sharing the parent's.  The hot path is one relaxed load. */
static inline void shm_claim_reader_slot(ShmHandle *h) {
    uint32_t cur_gen = __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED);
    if (__builtin_expect(cur_gen == h->cached_fork_gen && h->my_slot_idx != UINT32_MAX, 1))
        return;
    /* Cold path -- register the atfork hook once per process, then claim. */
    pthread_once(&shm_atfork_once, shm_atfork_init);
    /* Re-read after pthread_once: shm_on_fork_child may have bumped it. */
    cur_gen = __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED);
    uint32_t now_pid = (uint32_t)getpid();
    h->cached_pid = now_pid;
    if (cur_gen != h->cached_fork_gen) h->slotless_held = 0;  /* fork: child holds none of the parent's slotless read locks */
    h->cached_fork_gen = cur_gen;
    h->my_slot_idx = UINT32_MAX;
    uint32_t start = now_pid % SHM_READER_SLOTS;
    /* Pass 1: take a free slot. */
    for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) {
        uint32_t s = (start + i) % SHM_READER_SLOTS;
        uint32_t expected = 0;
        if (__atomic_compare_exchange_n(&h->reader_slots[s].pid,
                &expected, now_pid, 0,
                __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
            /* Fresh owner holds no read locks yet; clear any stale rdepth left by
             * a dead predecessor (its contribution is dropped as we take over). */
            __atomic_store_n(&h->reader_slots[s].rdepth, 0, __ATOMIC_RELAXED);
            shm_occ_set(h, s);   /* mark occupied BEFORE any rdlock can bump rdepth */
            h->my_slot_idx = s;
            return;
        }
    }
    /* Pass 2: no free slot, so reclaim one whose owner is dead.  Safe even at
     * rdepth>0: a writer scan ignores rdepth when pid==0. */
    for (uint32_t i = 0; i < SHM_READER_SLOTS; i++) {
        uint32_t dpid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
        if (dpid == 0 || dpid == now_pid || shm_pid_alive(dpid)) continue;
        uint32_t expected = dpid;
        if (__atomic_compare_exchange_n(&h->reader_slots[i].pid, &expected, now_pid, 0,
                __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
            __atomic_store_n(&h->reader_slots[i].rdepth, 0, __ATOMIC_RELAXED);
            shm_occ_set(h, i);
            h->my_slot_idx = i;
            return;
        }
    }
    /* Table full: my_slot_idx stays UINT32_MAX and the handle runs slotless.
     * The lock still works; this reader's death is not recoverable. */
}

/* After a futex-wait timeout, force-recover the lock if a dead writer holds it.
 * Dead readers need nothing here: the draining writer clears them in its scan. */
static inline void shm_recover_after_timeout(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    uint32_t val = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
    if (val >= SHM_RWLOCK_WRITER_BIT) {
        uint32_t pid = val & SHM_RWLOCK_PID_MASK;
        if (!shm_pid_alive(pid))
            shm_recover_stale_lock(h, val);
    }
}

/* Parked-waiter hint, so wrunlock knows whether a FUTEX_WAKE is worth a
 * syscall.  A waiter SIGKILLed while parked over-counts, costing at most a
 * spurious wake; it can never under-count, so no wakeup is lost. */
static inline void shm_park(ShmHandle *h) {
    __atomic_add_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
}
static inline void shm_unpark(ShmHandle *h) {
    __atomic_sub_fetch(&h->hdr->rwait, 1, __ATOMIC_RELAXED);
}

/* Publish or retract this reader's presence.  inc() is SEQ_CST so the wlock
 * re-check after it in rdlock forms a Dekker handshake with the writer's
 * wlock-store and rdepth-scan.  dec() peels slotless first, so a slot claimed
 * mid-hold cannot misattribute the decrement. */
static inline void shm_rdepth_inc(ShmHandle *h) {
    if (h->my_slot_idx != UINT32_MAX) {
        __atomic_add_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_SEQ_CST);
    } else {
        __atomic_add_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_SEQ_CST);
        h->slotless_held++;
    }
}
static inline void shm_rdepth_dec(ShmHandle *h) {
    if (h->slotless_held > 0) {
        h->slotless_held--;
        __atomic_sub_fetch(&h->hdr->slotless_rdepth, 1, __ATOMIC_RELEASE);
    } else if (h->my_slot_idx != UINT32_MAX) {
        __atomic_sub_fetch(&h->reader_slots[h->my_slot_idx].rdepth, 1, __ATOMIC_RELEASE);
    }
}

/* Wake a writer draining readers, so a released read lock lets it re-scan
 * instead of waiting out its timeout. */
static inline void shm_reader_wake_drain(ShmHandle *h) {
    if (__atomic_load_n(&h->hdr->wlock, __ATOMIC_ACQUIRE) != 0) {
        __atomic_add_fetch(&h->hdr->drain_seq, 1, __ATOMIC_RELEASE);
        syscall(SYS_futex, &h->hdr->drain_seq, FUTEX_WAKE, 1, NULL, NULL, 0);
    }
}

static inline void shm_rwlock_rdlock(ShmHandle *h) {
    /* Frozen view: no writer can exist, and the PROT_READ mapping would fault
     * on the rdepth publish.  Skip the lock entirely. */
    if (h->readonly) return;
    shm_claim_reader_slot(h);
    ShmHeader *hdr = h->hdr;
    for (int spin = 0; ; spin++) {
        uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_ACQUIRE);
        if (cur == 0) {
            /* Publish rdepth, then re-check wlock: against the writer's CAS and
             * rdepth scan, the SEQ_CST total order forbids both sides missing
             * each other. */
            shm_rdepth_inc(h);
            if (__atomic_load_n(&hdr->wlock, __ATOMIC_SEQ_CST) == 0)
                return;                       /* no writer after our publish -> we hold the read lock */
            /* A writer appeared during our publish -- yield to it (write-preferring). */
            shm_rdepth_dec(h);
            shm_reader_wake_drain(h);          /* let the draining writer see rdepth drop */
            spin = 0;
            continue;
        }
        /* wlock != 0: a writer holds or is acquiring.  Recover if it is dead. */
        if (cur >= SHM_RWLOCK_WRITER_BIT &&
            !shm_pid_alive(cur & SHM_RWLOCK_PID_MASK)) {
            shm_recover_stale_lock(h, cur);
            spin = 0;
            continue;
        }
        if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) {
            shm_rwlock_spin_pause();
            continue;
        }
        shm_park(h);
        cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
        if (cur != 0) {
            long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
                              &shm_lock_timeout, NULL, 0);
            if (rc == -1 && errno == ETIMEDOUT) {
                shm_unpark(h);
                shm_recover_after_timeout(h);
                spin = 0;
                continue;
            }
        }
        shm_unpark(h);
        spin = 0;
    }
}

static inline void shm_rwlock_rdunlock(ShmHandle *h) {
    if (h->readonly) return;           /* frozen view took no lock -- see shm_rwlock_rdlock */
    shm_rdepth_dec(h);                 /* RELEASE: drop our entire contribution */
    shm_reader_wake_drain(h);          /* if a writer is draining, wake it to re-scan */
}

static inline void shm_rwlock_wrlock(ShmHandle *h) {
    /* Keep this pure C with no Perl API: xt/slotless_reader_recovery.t compiles
     * it standalone with plain cc, so a Perl_croak here breaks that harness.
     * A frozen handle never arrives: every mutator croaks before locking. */
    shm_claim_reader_slot(h);  /* refresh cached_pid across fork */
    ShmHeader *hdr = h->hdr;
    /* Encode PID in the wlock word itself (0x80000000 | pid) to eliminate any
     * crash window between acquiring the lock and storing the owner. */
    uint32_t mypid = SHM_RWLOCK_WR(h->cached_pid);
    /* Phase 1: acquire the writer word (mutual exclusion among writers). */
    for (int spin = 0; ; spin++) {
        uint32_t expected = 0;
        if (__atomic_compare_exchange_n(&hdr->wlock, &expected, mypid,
                0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
            break;
        /* Contended: expected now holds the current wlock value. */
        if (expected >= SHM_RWLOCK_WRITER_BIT &&
            !shm_pid_alive(expected & SHM_RWLOCK_PID_MASK)) {
            shm_recover_stale_lock(h, expected);
            spin = 0;
            continue;
        }
        if (__builtin_expect(spin < SHM_RWLOCK_SPIN_LIMIT, 1)) {
            shm_rwlock_spin_pause();
            continue;
        }
        shm_park(h);
        uint32_t cur = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
        if (cur != 0) {
            long rc = syscall(SYS_futex, &hdr->wlock, FUTEX_WAIT, cur,
                              &shm_lock_timeout, NULL, 0);
            if (rc == -1 && errno == ETIMEDOUT) {
                shm_unpark(h);
                shm_recover_after_timeout(h);
                spin = 0;
                continue;
            }
        }
        shm_unpark(h);
        spin = 0;
    }
    /* Phase 2: we own wlock, so no new reader joins.  Drain those already
     * holding; the CAS above and the loads below are the writer side of the
     * Dekker handshake. */
    for (;;) {
        uint32_t v = __atomic_load_n(&hdr->drain_seq, __ATOMIC_RELAXED);  /* snapshot BEFORE scan */
        int busy = 0;
        /* Occupied slots only: a committed reader's bit is set in its claim,
         * before its rdepth++, so this SEQ_CST scan skips no held slot. */
        for (uint32_t w = 0; w < SHM_OCC_WORDS; w++) {
            uint64_t word = __atomic_load_n(&h->occ[w], __ATOMIC_SEQ_CST);
            while (word) {
                uint32_t i = (w << 6) + (uint32_t)__builtin_ctzll(word);
                word &= word - 1;                          /* consume this bit (local copy) */
                uint32_t rd = __atomic_load_n(&h->reader_slots[i].rdepth, __ATOMIC_SEQ_CST);
                if (rd == 0) continue;                      /* occupied but not read-locking now */
                uint32_t pid = __atomic_load_n(&h->reader_slots[i].pid, __ATOMIC_ACQUIRE);
                if (pid == 0) continue;                     /* stale rdepth on a freed slot */
                if (!shm_pid_alive(pid)) {
                    /* Dead reader: drop its pid so the slot stops counting.  The
                     * occ bit stays set, so as not to race a claimant. */
                    uint32_t ep = pid;
                    __atomic_compare_exchange_n(&h->reader_slots[i].pid, &ep, 0,
                            0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
                    continue;
                }
                busy = 1;                                   /* live reader still holding */
            }
        }
        /* A live slotless reader keeps us waiting; a crashed slotless reader that
         * cannot be attributed to a pid is the documented slotless limitation. */
        if (__atomic_load_n(&hdr->slotless_rdepth, __ATOMIC_SEQ_CST) != 0)
            busy = 1;
        if (!busy)
            return;                                    /* exclusive: wlock held + every rdepth 0 */
        /* Wait for a reader to release (drain_seq bump) or time out to re-scan
         * (which reclaims any newly-dead slotted reader). */
        syscall(SYS_futex, &hdr->drain_seq, FUTEX_WAIT, v, &shm_lock_timeout, NULL, 0);
    }
}

static inline void shm_rwlock_wrunlock(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    __atomic_store_n(&hdr->wlock, 0, __ATOMIC_RELEASE);
    if (__atomic_load_n(&hdr->rwait, __ATOMIC_RELAXED) > 0)
        syscall(SYS_futex, &hdr->wlock, FUTEX_WAKE, INT_MAX, NULL, NULL, 0);
}

/* ---- Seqlock (lock-free readers) ---- */

static inline uint32_t shm_seqlock_read_begin(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    int spin = 0;
    for (;;) {
        uint32_t s = __atomic_load_n(&hdr->seq, __ATOMIC_ACQUIRE);
        if (__builtin_expect((s & 1) == 0, 1)) return s;
        if (__builtin_expect(spin < 100000, 1)) {
            shm_rwlock_spin_pause();
            spin++;
            continue;
        }
        /* Prolonged odd seq -- check for dead writer */
        uint32_t val = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
        if (val >= SHM_RWLOCK_WRITER_BIT) {
            uint32_t pid = val & SHM_RWLOCK_PID_MASK;
            if (pid == 0 || !shm_pid_alive(pid)) {
                /* A PROT_READ view cannot CAS without faulting, and nobody else
                 * will finish this section: return stale rather than spin. */
                if (h->readonly) return s;
                shm_recover_stale_lock(h, val);
                spin = 0;
                continue;
            }
        } else if (h->readonly) {
            return s;   /* odd seq with no holder: nothing will ever republish */
        }
        struct timespec ts = {0, 1000000}; /* 1ms */
        nanosleep(&ts, NULL);
        spin = 0;
    }
}

static inline int shm_seqlock_read_retry(uint32_t *seq, uint32_t start) {
    __atomic_thread_fence(__ATOMIC_ACQUIRE);  /* ensure data loads complete before retry check */
    return __atomic_load_n(seq, __ATOMIC_RELAXED) != start;
}

static inline void shm_seqlock_write_begin(uint32_t *seq) {
    __atomic_add_fetch(seq, 1, __ATOMIC_RELEASE);  /* seq becomes odd */
    /* StoreStore: the odd seq must be visible before the writes that follow, or
     * an ARM64 reader loads an even seq over half-written data and passes
     * read_retry. */
    __atomic_thread_fence(__ATOMIC_RELEASE);
}

static inline void shm_seqlock_write_end(uint32_t *seq) {
    __atomic_add_fetch(seq, 1, __ATOMIC_RELEASE);  /* seq becomes even */
}

/* ---- Arena allocator ---- */

static inline uint32_t shm_next_pow2(uint32_t v);

static inline uint32_t shm_arena_round_up(uint32_t len) {
    if (len < SHM_ARENA_MIN_ALLOC) return SHM_ARENA_MIN_ALLOC;
    return shm_next_pow2(len);
}

static inline int shm_arena_class_index(uint32_t alloc_size) {
    if (alloc_size <= SHM_ARENA_MIN_ALLOC) return 0;
    if (alloc_size > (SHM_ARENA_MIN_ALLOC << (SHM_ARENA_NUM_CLASSES - 1))) return -1;
    return 32 - __builtin_clz(alloc_size - 1) - 4;  /* log2(alloc_size) - 4 */
}

static inline uint32_t shm_arena_alloc(ShmHeader *hdr, char *arena, uint32_t len) {
    uint32_t asize = shm_arena_round_up(len);
    int cls = shm_arena_class_index(asize);

    if (cls >= 0 && hdr->arena_free[cls] != 0) {
        uint32_t head = hdr->arena_free[cls];
        /* Free-list heads are peer-writable: a wild head sends the pop read and
         * the caller's store out of bounds.  Treat as an empty class. */
        if ((uint64_t)head + asize <= hdr->arena_cap) {
            uint32_t next;
            memcpy(&next, arena + head, sizeof(uint32_t));
            hdr->arena_free[cls] = next;
            return head;
        }
    }
    if (cls < 0) {
        /* Large request: first-fit over the large free list before bumping. */
        uint32_t prev = 0, cur = hdr->arena_large_free;
        while (cur != 0) {
            uint32_t next, blk;
            /* cur is peer-writable: abandon the walk rather than dereference a
             * wild offset.  cur+asize also covers the [next][size] read. */
            if ((uint64_t)cur + asize > hdr->arena_cap) break;
            memcpy(&next, arena + cur, sizeof(uint32_t));
            memcpy(&blk, arena + cur + sizeof(uint32_t), sizeof(uint32_t));
            /* Exact class only: a larger block handed to a smaller request is
             * refiled at the smaller size on free, losing the surplus. */
            if (blk == asize) {
                if (prev == 0) hdr->arena_large_free = next;
                else memcpy(arena + prev, &next, sizeof(uint32_t));
                return cur;
            }
            prev = cur; cur = next;
        }
    }

    uint64_t off = hdr->arena_bump;
    if (off + asize > hdr->arena_cap || off + asize > (uint64_t)UINT32_MAX)
        return 0;
    hdr->arena_bump = off + asize;
    return (uint32_t)off;
}

static inline void shm_arena_free_block(ShmHeader *hdr, char *arena,
                                         uint32_t off, uint32_t len) {
    uint32_t asize = shm_arena_round_up(len);
    int cls = shm_arena_class_index(asize);
    if (off == 0) return;
    /* off comes from a peer-writable node field: a wild value would send the
     * free-list link write below out of bounds.  Leak the block instead. */
    if ((uint64_t)off + asize > hdr->arena_cap) return;
    if (cls < 0) {
        /* Large block: one first-fit list keyed by size, so churn recycles
         * instead of leaking to bump-only. */
        uint32_t old_head = hdr->arena_large_free;
        memcpy(arena + off, &old_head, sizeof(uint32_t));                  /* next */
        memcpy(arena + off + sizeof(uint32_t), &asize, sizeof(uint32_t));  /* size */
        hdr->arena_large_free = off;
        return;
    }
    uint32_t old_head = hdr->arena_free[cls];
    memcpy(arena + off, &old_head, sizeof(uint32_t));
    hdr->arena_free[cls] = off;
}

/* Store a string: inline if <= 7 bytes, arena otherwise. Returns 1 on success, 0 on arena OOM. */
static inline int shm_str_store(ShmHeader *hdr, char *arena,
                                 uint32_t *off, uint32_t *len_field,
                                 const char *str, uint32_t slen, bool utf8) {
    if (slen <= SHM_INLINE_MAX) {
        shm_inline_pack(off, len_field, str, slen, utf8);
        return 1;
    }
    uint32_t aoff = shm_arena_alloc(hdr, arena, slen);
    if (aoff == 0) return 0;
    memcpy(arena + aoff, str, slen);
    /* Interim inline-empty state -- see shm_inline_pack. */
    __atomic_store_n(len_field, SHM_INLINE_FLAG, __ATOMIC_RELEASE);
    __atomic_store_n(off, aoff, __ATOMIC_RELEASE);
    __atomic_store_n(len_field, SHM_PACK_LEN(slen, utf8), __ATOMIC_RELEASE);
    return 1;
}

/* Free a string's arena block (no-op for inline strings) */
static inline void shm_str_free(ShmHeader *hdr, char *arena,
                                 uint32_t off, uint32_t len_field) {
    if (!SHM_IS_INLINE(len_field))
        shm_arena_free_block(hdr, arena, off, SHM_UNPACK_LEN(len_field));
}

static inline void shm_str_copy(char *dst, uint32_t off, uint32_t len_field,
                                 const char *arena, uint32_t arena_cap, uint32_t len) {
    if (SHM_IS_INLINE(len_field)) {
        shm_inline_read(off, len_field, dst);
    } else if ((uint64_t)off + len <= arena_cap) {
        memcpy(dst, arena + off, len);
    } else {
        /* A poisoned record delivers zeros rather than reading out of bounds
         * (CWE-125).  Every reader that takes an arena pointer instead of
         * copying bounds off+len against arena_cap itself: grep arena_cap. */
        memset(dst, 0, len);
    }
}

/* ---- Utility ---- */

static inline uint32_t shm_next_pow2(uint32_t v) {
    if (v < 2) return 2;
    if (v > 0x80000000U) return 0;  /* overflow: no valid power of 2 */
    v--;
    v |= v >> 1; v |= v >> 2; v |= v >> 4; v |= v >> 8; v |= v >> 16;
    return v + 1;
}

/* Largest power-of-2 table capacity for max_entries.  The 2^31 cap comes
 * before next_pow2: otherwise a huge max_entries overflows the uint32 cast or
 * makes next_pow2 return 0, silently yielding a tiny table. */
static inline uint32_t shm_max_tcap_from_entries(uint32_t max_entries) {
    uint64_t want = (uint64_t)max_entries * 4 / 3 + 1;
    uint32_t cap = (want > 0x80000000ULL) ? 0x80000000U : shm_next_pow2((uint32_t)want);
    return cap < SHM_INITIAL_CAP ? SHM_INITIAL_CAP : cap;
}

/* Over the 75% design load: live entries plus tombstones. */
static inline int shm_over_load(uint32_t size, uint32_t tomb, uint32_t cap) {
    return (uint64_t)(size + tomb) * 4 > (uint64_t)cap * 3;
}

/* Enough tombstones to be worth a rehash even under the load factor. */
static inline int shm_needs_compaction(uint32_t size, uint32_t tomb, uint32_t cap) {
    return tomb > size || tomb > cap / 4;
}

/* Under the 25% shrink threshold, and not already at the floor. */
static inline int shm_under_load(uint32_t size, uint32_t cap) {
    return cap > SHM_INITIAL_CAP && (uint64_t)size * 4 < cap;
}

/* The capacity a table holding `size` entries shrinks to -- all the way, in one
 * call, because the batch removers reach it once per batch, not per entry. */
static inline uint32_t shm_shrink_target(uint32_t size, uint32_t cap) {
    while (shm_under_load(size, cap)) cap /= 2;
    if (cap < SHM_INITIAL_CAP) cap = SHM_INITIAL_CAP;
    return cap;
}

/* lru_skip percentage to the promotion mask: 50 -> 1 (every 2nd), 90 -> 15,
 * 95 -> 31.  Outside 1..99 skipping is off. */
static inline uint32_t shm_lru_skip_to_mask(uint32_t lru_skip) {
    if (lru_skip == 0 || lru_skip >= 100) return 0;
    uint32_t interval = 100 / (100 - lru_skip);
    uint32_t p = 1;
    while (p < interval) p <<= 1;
    return p - 1;
}

/* ---- LRU helpers ---- */

static inline void shm_lru_unlink(ShmHandle *h, uint32_t idx) {
    uint32_t *prev = h->lru_prev;
    uint32_t *next = h->lru_next;
    ShmHeader *hdr = h->hdr;
    uint32_t p = prev[idx], n = next[idx];
    if (p == SHM_LRU_NONE && n == SHM_LRU_NONE && hdr->lru_head != idx) return;
    if (p != SHM_LRU_NONE) next[p] = n;
    else hdr->lru_head = n;
    if (n != SHM_LRU_NONE) prev[n] = p;
    else hdr->lru_tail = p;
    prev[idx] = next[idx] = SHM_LRU_NONE;
}

static inline void shm_lru_push_front(ShmHandle *h, uint32_t idx) {
    uint32_t *prev = h->lru_prev;
    uint32_t *next = h->lru_next;
    ShmHeader *hdr = h->hdr;
    if (h->lru_accessed) __atomic_store_n(&h->lru_accessed[idx], 0, __ATOMIC_RELAXED);  /* clear stale clock bit */
    prev[idx] = SHM_LRU_NONE;
    next[idx] = hdr->lru_head;
    if (hdr->lru_head != SHM_LRU_NONE) prev[hdr->lru_head] = idx;
    else hdr->lru_tail = idx;
    hdr->lru_head = idx;
}

static inline void shm_lru_promote(ShmHandle *h, uint32_t idx) {
    ShmHeader *hdr = h->hdr;
    if (hdr->lru_head == idx) return;
    /* Promote every (mask+1)th access; the tail is never skipped, or eviction
     * would take a hot entry. */
    if (hdr->lru_skip > 0 && idx != hdr->lru_tail) {
        static __thread uint32_t promote_ctr = 0;
        if ((++promote_ctr & hdr->lru_skip) != 0)
            return;
    }
    shm_lru_unlink(h, idx);
    shm_lru_push_front(h, idx);
}

/* Recount size/tombstones from states[]: a writer killed between publishing
 * states[idx] and bumping the counters leaves size behind the live count, and
 * resize() then truncates its save loop and drops live entries. */
static void shm_recount_counters(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    uint32_t cap = hdr->table_cap, live = 0, tomb = 0;
    for (uint32_t i = 0; i < cap; i++) {
        uint8_t st = h->states[i];
        if (st == SHM_TOMBSTONE) tomb++;
        else if (SHM_IS_LIVE(st)) live++;
    }
    /* Bracket the stores only: an attach validates size and tombstones
     * together, but holding the seqlock across the scan above would stall
     * readers for O(table_cap). */
    shm_seqlock_write_begin(&hdr->seq);
    /* Publish the decreasing counter first: an increase can transiently exceed
     * table_cap, and a crash there is unrepairable.  Atomic, or gcc merges the
     * arms into one unconditional sequence. */
    if (tomb <= hdr->tombstones) {
        __atomic_store_n(&hdr->tombstones, tomb, __ATOMIC_RELEASE);
        __atomic_store_n(&hdr->size, live, __ATOMIC_RELEASE);
    } else {
        __atomic_store_n(&hdr->size, live, __ATOMIC_RELEASE);
        __atomic_store_n(&hdr->tombstones, tomb, __ATOMIC_RELEASE);
    }
    shm_seqlock_write_end(&hdr->seq);
}

/* Validate the LRU list and rebuild it from states[] if inconsistent: a writer
 * killed mid-unlink/push_front/promote leaves a one-way break that would
 * infinite-loop the next eviction.  The rebuild is in slot order, losing
 * recency but not correctness. */
static void shm_lru_rebuild_if_corrupt(ShmHandle *h) {
    if (!h->lru_prev) return;  /* LRU disabled */
    ShmHeader *hdr = h->hdr;
    uint32_t cap = hdr->table_cap;
    uint32_t head = hdr->lru_head;
    uint32_t tail = hdr->lru_tail;
    int corrupt = 0;
    uint32_t chain_len = 0;

    if ((head != SHM_LRU_NONE && head >= cap) ||
        (tail != SHM_LRU_NONE && tail >= cap)) {
        corrupt = 1;
    } else {
        uint32_t prev_idx = SHM_LRU_NONE;
        uint32_t idx = head;
        while (idx != SHM_LRU_NONE) {
            if (idx >= cap || !SHM_IS_LIVE(h->states[idx]) ||
                h->lru_prev[idx] != prev_idx) { corrupt = 1; break; }
            prev_idx = idx;
            idx = h->lru_next[idx];
            if (++chain_len > cap) { corrupt = 1; break; }  /* cycle */
        }
        if (!corrupt && prev_idx != tail) corrupt = 1;
    }
    /* Orphan check against the counted live entries, not hdr->size: a writer
     * killed between states[i]=LIVE, size++ and push_front can leave size
     * itself behind, which chain_len vs hdr->size would miss. */
    uint32_t live_count = 0;
    if (!corrupt) {
        for (uint32_t i = 0; i < cap; i++)
            if (SHM_IS_LIVE(h->states[i])) live_count++;
        if (chain_len != live_count) corrupt = 1;
    }
    if (!corrupt) return;

    memset(h->lru_prev, 0xFF, (size_t)cap * sizeof(uint32_t));
    memset(h->lru_next, 0xFF, (size_t)cap * sizeof(uint32_t));
    if (h->lru_accessed) memset(h->lru_accessed, 0, cap);
    uint32_t prev = SHM_LRU_NONE;
    uint32_t new_head = SHM_LRU_NONE;
    for (uint32_t i = 0; i < cap; i++) {
        if (!SHM_IS_LIVE(h->states[i])) continue;
        h->lru_prev[i] = prev;
        if (prev != SHM_LRU_NONE) h->lru_next[prev] = i;
        else new_head = i;
        prev = i;
    }
    hdr->lru_head = new_head;
    hdr->lru_tail = prev;
    /* Counters are reconciled by shm_recount_counters, and the recovery event
     * is counted once by shm_recover_stale_lock -- not here. */
}

/* ---- Create / Open / Close ---- */

#define SHM_ERR_BUFLEN 256

/* Sizes and offsets of the variable-length regions after the header. */
typedef struct {
    uint64_t nodes_off, states_off;
    uint64_t lru_prev_off, lru_next_off, lru_accessed_off;
    uint64_t expires_off;
    uint64_t reader_slots_off;
    uint64_t occ_off;
    uint64_t arena_off, arena_cap;
    uint64_t total_size;
    uint64_t end_off;  /* end of LRU/TTL region -- caller verifies file is at least this large */
} ShmLayout;

/* 4096-byte floor so the arena is always functional, UINT32_MAX ceiling
 * because arena offsets are uint32. */
static inline uint64_t shm_clamp_arena_cap(uint64_t want) {
    if (want < 4096) return 4096;
    if (want > UINT32_MAX) return UINT32_MAX;
    return want;
}

/* Compute the region offsets after the header.  arena_cap is
 * arena_cap_override when nonzero, else max(max_entries*128, 4096), both
 * clamped.  Create maps lo->total_size bytes; reopen checks the file against
 * lo->end_off and takes reader_slots_off and total_size from the header. */
static inline void shm_compute_layout(ShmLayout *lo, uint32_t max_tcap,
                                       uint32_t node_size, int has_lru,
                                       int has_ttl, int has_arena,
                                       uint32_t max_entries,
                                       uint64_t arena_cap_override) {
    lo->nodes_off  = sizeof(ShmHeader);
    lo->states_off = lo->nodes_off + (uint64_t)max_tcap * node_size;
    uint64_t off = lo->states_off + max_tcap;
    lo->lru_prev_off = lo->lru_next_off = lo->lru_accessed_off = 0;
    lo->expires_off = 0;
    if (has_lru) {
        off = (off + 3) & ~(uint64_t)3;
        lo->lru_prev_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
        lo->lru_next_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
        lo->lru_accessed_off = off; off += max_tcap;  /* uint8_t per slot */
    }
    if (has_ttl) {
        off = (off + 3) & ~(uint64_t)3;
        lo->expires_off = off; off += (uint64_t)max_tcap * sizeof(uint32_t);
    }
    lo->end_off = off;
    off = (off + 7) & ~(uint64_t)7;
    lo->reader_slots_off = off;
    off += (uint64_t)SHM_READER_SLOTS * sizeof(ShmReaderSlot);
    lo->occ_off = off;                 /* occupancy bitmap right after reader_slots */
    off += SHM_OCC_BYTES;
    lo->arena_off = lo->arena_cap = 0;
    if (has_arena) {
        lo->arena_off = (off + 7) & ~(uint64_t)7;
        uint64_t want = arena_cap_override ? arena_cap_override
                                           : (uint64_t)max_entries * 128;
        lo->arena_cap = shm_clamp_arena_cap(want);
        lo->total_size = lo->arena_off + lo->arena_cap;
    } else {
        lo->total_size = off;
    }
}

/* Initialize a freshly-mmap'd header and zero out the optional regions.
 * Used by both shm_create_map and shm_create_memfd. */
static inline void shm_init_header(ShmHeader *hdr, void *base,
                                    const ShmLayout *lo, uint32_t max_tcap,
                                    uint32_t node_size, uint32_t variant_id,
                                    int has_arena, int has_lru, int has_ttl,
                                    uint32_t max_size, uint32_t default_ttl,
                                    uint32_t lru_skip) {
    memset(hdr, 0, sizeof(ShmHeader));
    hdr->version       = SHM_VERSION;
    hdr->variant_id    = variant_id;
    hdr->node_size     = node_size;
    hdr->max_table_cap = max_tcap;
    hdr->table_cap     = SHM_INITIAL_CAP;
    hdr->total_size    = lo->total_size;
    hdr->nodes_off     = lo->nodes_off;
    hdr->states_off    = lo->states_off;
    hdr->arena_off     = has_arena ? lo->arena_off : 0;
    hdr->arena_cap     = lo->arena_cap;
    hdr->reader_slots_off = lo->reader_slots_off;
    hdr->arena_bump    = SHM_ARENA_MIN_ALLOC;  /* reserve offset 0 */
    hdr->routing       = SHM_ROUTING_SPLIT;
    hdr->max_size      = max_size;
    hdr->default_ttl   = default_ttl;
    hdr->lru_skip      = shm_lru_skip_to_mask(lru_skip);
    hdr->lru_head      = SHM_LRU_NONE;
    hdr->lru_tail      = SHM_LRU_NONE;
    if (has_lru) {
        memset((char *)base + lo->lru_prev_off, 0xFF, max_tcap * sizeof(uint32_t));
        memset((char *)base + lo->lru_next_off, 0xFF, max_tcap * sizeof(uint32_t));
        memset((char *)base + lo->lru_accessed_off, 0, max_tcap);
    }
    if (has_ttl)
        memset((char *)base + lo->expires_off, 0, max_tcap * sizeof(uint32_t));
    memset((char *)base + lo->reader_slots_off, 0,
           SHM_READER_SLOTS * sizeof(ShmReaderSlot));
    /* Zero the occupancy bitmap explicitly: create does not memset the whole
     * mapping, so do not rely on OS zero-fill for this region. */
    memset((char *)base + lo->occ_off, 0, SHM_OCC_BYTES);
    /* Publish magic LAST, as a release store: it is the commit point, so a
       creator killed before it leaves magic==0 and never a file mistaken for
       a valid one.  A kill during the field stores leaves one to remove by
       hand. */
    __atomic_store_n(&hdr->magic, SHM_MAGIC, __ATOMIC_RELEASE);
    __atomic_thread_fence(__ATOMIC_SEQ_CST);
}

/* Validate a header read from disk or fd.  The caller must already have
 * checked the file against sizeof(ShmHeader) and hdr->total_size. */
static inline int shm_validate_header(const ShmHeader *hdr,
                                       uint32_t variant_id, uint32_t node_size) {
    return (hdr->magic == SHM_MAGIC &&
            hdr->version == SHM_VERSION &&
            hdr->variant_id == variant_id &&
            hdr->node_size == node_size &&
            hdr->nodes_off >= sizeof(ShmHeader) &&
            hdr->states_off > hdr->nodes_off &&
            hdr->states_off < hdr->total_size &&
            (!hdr->arena_off || (hdr->arena_off < hdr->total_size &&
                                 hdr->arena_off + hdr->arena_cap <= hdr->total_size &&
                                 hdr->arena_bump <= hdr->arena_cap &&
                                 hdr->arena_bump >= SHM_ARENA_MIN_ALLOC)) &&
            hdr->max_table_cap > 0 &&
            (hdr->max_table_cap & (hdr->max_table_cap - 1)) == 0 &&
            hdr->table_cap > 0 &&
            (hdr->table_cap & (hdr->table_cap - 1)) == 0 &&
            hdr->table_cap <= hdr->max_table_cap &&
            hdr->states_off + hdr->max_table_cap <= hdr->total_size &&
            hdr->nodes_off + (uint64_t)hdr->max_table_cap * hdr->node_size <= hdr->states_off &&
            hdr->size <= hdr->table_cap &&
            hdr->tombstones <= hdr->table_cap - hdr->size &&
            (!hdr->max_size ||
             ((hdr->lru_head == SHM_LRU_NONE || hdr->lru_head < hdr->max_table_cap) &&
              (hdr->lru_tail == SHM_LRU_NONE || hdr->lru_tail < hdr->max_table_cap))));
}

/* Attach-time validation of a live mapping.  size + tombstones <= table_cap can
 * read torn on a healthy saturated map, so reject only when the seqlock shows
 * no writer was active across the check. */
static inline int shm_validate_header_live(const ShmHeader *hdr,
                                            uint32_t variant_id, uint32_t node_size) {
    for (int attempt = 0; attempt < 4096; attempt++) {
        uint32_t s1 = __atomic_load_n(&hdr->seq, __ATOMIC_ACQUIRE);
        int ok = shm_validate_header(hdr, variant_id, node_size);
        __atomic_thread_fence(__ATOMIC_ACQUIRE);
        uint32_t s2 = __atomic_load_n(&hdr->seq, __ATOMIC_RELAXED);
        if (ok) return 1;
        /* Only a live holder can still republish, and a repair scan runs with
         * seq even, so seq stability alone proves nothing. */
        uint32_t w = __atomic_load_n(&hdr->wlock, __ATOMIC_RELAXED);
        /* Our own pid in the lock word is a dead writer whose pid we recycled:
         * we cannot be mid-write and attaching at once. */
        uint32_t wpid = w & SHM_RWLOCK_PID_MASK;
        int live_holder = (w >= SHM_RWLOCK_WRITER_BIT) && wpid != 0 &&
                          wpid != (uint32_t)getpid() &&
                          shm_pid_alive(wpid);
        if (!live_holder) {
            /* Nothing moved: an even seq is a stable violation, refuse now; an
             * odd one is a dead writer's leftover, so one more read decides.  A
             * moved seq proves the read was torn, not corrupt: retry rather
             * than decide on a single re-read. */
            if (s1 == s2)
                return (s1 & 1) ? shm_validate_header(hdr, variant_id, node_size) : 0;
        }
        if (attempt >= 8) { struct timespec ts = { 0, 500000 }; nanosleep(&ts, NULL); }
    }
    return shm_validate_header(hdr, variant_id, node_size);
}

/* Format a header-validation error, naming the first failing field.  `prefix`
 * is the path, or the literal "fd" on the fd-reopen path. */
static inline void shm_format_header_error(char *errbuf, const char *prefix,
                                            const ShmHeader *hdr,
                                            uint32_t variant_id) {
    if (!errbuf) return;
    if (hdr->magic != SHM_MAGIC)
        snprintf(errbuf, SHM_ERR_BUFLEN, "%s: bad magic (not a HashMap::Shared file)", prefix);
    else if (hdr->version != SHM_VERSION)
        snprintf(errbuf, SHM_ERR_BUFLEN, "%s: version mismatch (file=%u, expected=%u)",
                 prefix, hdr->version, SHM_VERSION);
    else if (hdr->variant_id != variant_id)
        snprintf(errbuf, SHM_ERR_BUFLEN, "%s: variant mismatch (file=%u, expected=%u)",
                 prefix, hdr->variant_id, variant_id);
    else
        snprintf(errbuf, SHM_ERR_BUFLEN, "%s: corrupt header", prefix);
}

/* Recompute layout from a validated header and verify that the LRU/TTL
 * region, the reader_slots table and the occupancy bitmap after it all fit
 * inside `mapped_size`.  Returns 1 on success, with lo holding the locally
 * computed layout (the stored reader_slots_off is only sanity-checked); 0 on
 * failure (errbuf populated). */
static inline int shm_validate_layout_regions(ShmLayout *lo, const ShmHeader *hdr,
                                                int has_arena, uint64_t mapped_size,
                                                char *errbuf, const char *prefix) {
    int has_lru = (hdr->max_size > 0);
    int has_ttl = (hdr->default_ttl > 0);
    shm_compute_layout(lo, hdr->max_table_cap, hdr->node_size,
                       has_lru, has_ttl, has_arena, 0, 0);
    if (lo->end_off > mapped_size) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                             "%s: file too small for LRU/TTL arrays", prefix);
        return 0;
    }
    /* Locate reader_slots from the locally computed layout, never the
     * peer-writable hdr->reader_slots_off, which is only sanity-gated here: a
     * peer corrupting it must not misdirect our pointer. */
    uint64_t rs_off = hdr->reader_slots_off;
    if (!rs_off || rs_off < lo->end_off ||
        lo->reader_slots_off + SHM_READER_SLOTS * sizeof(ShmReaderSlot) > mapped_size ||
        lo->occ_off + SHM_OCC_BYTES > mapped_size) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                             "%s: reader_slots region missing or out of bounds", prefix);
        return 0;
    }
    return 1;
}

/* Build the process-local handle over an already-mmap'd base: the create,
 * memfd and fd-reopen paths differ only in how they acquire `base`. */
static ShmHandle *shm_alloc_handle(void *base, uint64_t total_size,
                                    int has_arena, int has_lru, int has_ttl,
                                    const ShmLayout *lo,
                                    const char *path, int backing_fd, char *errbuf) {
    ShmHeader *hdr = (ShmHeader *)base;
    ShmHandle *h = (ShmHandle *)calloc(1, sizeof(ShmHandle));
    if (!h) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory");
        munmap(base, (size_t)total_size);
        if (backing_fd >= 0) close(backing_fd);
        return NULL;
    }
    h->hdr       = hdr;
    h->nodes     = (char *)hdr + hdr->nodes_off;
    h->states    = (uint8_t *)((char *)hdr + hdr->states_off);
    h->arena     = has_arena ? (char *)hdr + lo->arena_off : NULL;   /* trusted layout offset, like reader_slots */
    h->lru_prev  = has_lru ? (uint32_t *)((char *)hdr + lo->lru_prev_off) : NULL;
    h->lru_next  = has_lru ? (uint32_t *)((char *)hdr + lo->lru_next_off) : NULL;
    h->lru_accessed = has_lru ? (uint8_t *)((char *)hdr + lo->lru_accessed_off) : NULL;
    h->expires_at = has_ttl ? (uint32_t *)((char *)hdr + lo->expires_off) : NULL;
    h->reader_slots = (ShmReaderSlot *)((char *)hdr + lo->reader_slots_off);
    h->occ          = (uint64_t *)((char *)hdr + lo->occ_off);   /* trusted layout offset */
    /* Slot claimed lazily on first lock -- see shm_claim_reader_slot. */
    h->my_slot_idx = UINT32_MAX;
    h->cached_pid = 0;
    h->mmap_size = (size_t)total_size;
    h->max_mask  = hdr->max_table_cap - 1;
    h->iter_pos  = 0;
    h->backing_fd = backing_fd;
    /* Lookups are random-access: skip read-ahead.  Best-effort. */
    (void)madvise(base, (size_t)total_size, MADV_RANDOM);
    if (path) {
        h->path = strdup(path);
        if (!h->path) {
            if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory");
            munmap(base, (size_t)total_size);
            if (backing_fd >= 0) close(backing_fd);
            free(h);
            return NULL;
        }
    }
    /* Pre-size copy_buf for string variants to avoid realloc on first access */
    if (has_arena) {
        h->copy_buf = (char *)malloc(256);
        if (h->copy_buf) h->copy_buf_size = 256;
    }
    return h;
}

/* Create exclusively (O_CREAT|O_EXCL|O_NOFOLLOW) or attach an existing file
 * (O_RDWR|O_NOFOLLOW), blocking a symlink swap or a pre-seeded backing file.
 * Cross-user sharing is opt-in through a wider file_mode. */
static int shm_secure_open(const char *path, mode_t file_mode, char *errbuf) {
    for (int attempt = 0; attempt < 100; attempt++) {
        int fd = open(path, O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, file_mode);
        if (fd >= 0) { (void)fchmod(fd, file_mode); return fd; }   /* exact mode: umask narrowed the O_EXCL create */
        if (errno != EEXIST) {
            if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "create(%s): %s", path, strerror(errno));
            return -1;
        }
        fd = open(path, O_RDWR|O_NOFOLLOW|O_CLOEXEC);
        if (fd >= 0) return fd;
        if (errno == ENOENT) continue;   /* creator unlinked between our two opens; retry */
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, strerror(errno));
        return -1;
    }
    if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): create/attach kept racing", path);
    return -1;
}

/* True iff the whole mapped region is zero -- what an abandoned mid-init
   creator leaves.  Lets recovery re-init only a provably-empty file, never
   one that merely starts with a zero word.  Cold path, so a byte scan is
   fine. */
static inline int shm_region_is_zero(const void *p, size_t n) {
    const unsigned char *b = (const unsigned char *)p;
    for (size_t i = 0; i < n; i++) if (b[i]) return 0;
    return 1;
}

static ShmHandle *shm_create_map(const char *path, uint32_t max_entries,
                                  uint32_t node_size, uint32_t variant_id,
                                  int has_arena, uint32_t max_size,
                                  uint32_t default_ttl, uint32_t lru_skip,
                                  uint64_t arena_cap_override, mode_t file_mode, char *errbuf) {
    if (errbuf) errbuf[0] = '\0';
    uint32_t max_tcap = shm_max_tcap_from_entries(max_entries);

    int has_lru = (max_size > 0);
    int has_ttl = (default_ttl > 0);

    ShmLayout lo;
    shm_compute_layout(&lo, max_tcap, node_size, has_lru, has_ttl, has_arena, max_entries, arena_cap_override);

    #define SHM_ERR(fmt, ...) do { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, fmt, ##__VA_ARGS__); } while(0)

    int anonymous = (path == NULL);
    int fd = -1;
    int is_new;
    struct stat st = { 0 };
    void *base;

    if (anonymous) {
        base = mmap(NULL, lo.total_size, PROT_READ | PROT_WRITE,
                     MAP_SHARED | MAP_ANONYMOUS, -1, 0);
        if (base == MAP_FAILED) { SHM_ERR("mmap(anon): %s", strerror(errno)); return NULL; }
        is_new = 1;
    } else {
        fd = shm_secure_open(path, file_mode, errbuf);
        if (fd < 0) return NULL;

        while (flock(fd, LOCK_EX) < 0) {
            if (errno == EINTR) continue;  /* retry: a signal interrupted the blocking lock */
            SHM_ERR("flock(%s): %s", path, strerror(errno)); close(fd); return NULL;
        }

        if (fstat(fd, &st) < 0) { SHM_ERR("fstat(%s): %s", path, strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; }

        is_new = (st.st_size == 0);

        if (!is_new && (uint64_t)st.st_size < sizeof(ShmHeader)) {
            SHM_ERR("%s: file too small (%lld bytes, need %zu)", path,
                    (long long)st.st_size, sizeof(ShmHeader));
            flock(fd, LOCK_UN); close(fd); return NULL;
        }

        if (is_new) {
            if (st.st_uid != geteuid()) {
                SHM_ERR("%s: refusing to initialize file not owned by us", path);
                flock(fd, LOCK_UN); close(fd); return NULL;
            }
            if (fchmod(fd, file_mode) < 0) {
                SHM_ERR("fchmod(%s): %s", path, strerror(errno));
                flock(fd, LOCK_UN); close(fd); return NULL;
            }
            if (ftruncate(fd, (off_t)lo.total_size) < 0) {
                SHM_ERR("ftruncate(%s, %llu): %s", path, (unsigned long long)lo.total_size, strerror(errno));
                flock(fd, LOCK_UN); close(fd); return NULL;
            }
        }

        base = mmap(NULL, is_new ? lo.total_size : (size_t)st.st_size,
                     PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
        if (base == MAP_FAILED) { SHM_ERR("mmap(%s): %s", path, strerror(errno)); flock(fd, LOCK_UN); close(fd); return NULL; }
    }

    ShmHeader *hdr = (ShmHeader *)base;
    uint64_t mapped_size = is_new ? lo.total_size : (uint64_t)st.st_size;

    if (is_new) {
        shm_init_header(hdr, base, &lo, max_tcap, node_size, variant_id,
                        has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);
    } else {
        int ok = (hdr->total_size == (uint64_t)st.st_size &&
                  shm_validate_header_live(hdr, variant_id, node_size));
        if (ok) {
            has_lru = (hdr->max_size > 0);
            has_ttl = (hdr->default_ttl > 0);
            ok = shm_validate_layout_regions(&lo, hdr, has_arena, mapped_size, errbuf, path);
        } else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size
                   && st.st_uid == geteuid()
                   && shm_region_is_zero(base, (size_t)st.st_size)) {
            /* A creator killed between the ftruncate and the header init
             * leaves a full-size all-zero file that would brick every future
             * open.  Re-initialize only when it is exactly our size, still
             * uninitialized, and owned by us. */
            if (fchmod(fd, file_mode) < 0) {
                SHM_ERR("%s: fchmod: %s", path, strerror(errno));
                munmap(base, (size_t)st.st_size); flock(fd, LOCK_UN); close(fd); return NULL;
            }
            shm_init_header(hdr, base, &lo, max_tcap, node_size, variant_id,
                            has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);
            ok = 1;   /* recovered -> valid; fall through to shm_alloc_handle */
        } else if (hdr->magic == 0 && (uint64_t)st.st_size == lo.total_size
                   && st.st_uid == geteuid()) {
            /* Our own full-size file, uninitialized but not all-zero: a creator
             * died between the field stores and the magic commit, so it holds
             * no data and is safe to remove. */
            SHM_ERR("%s: incomplete map file left by an interrupted create; remove it and retry", path);
        } else {
            shm_format_header_error(errbuf, path, hdr, variant_id);
        }
        if (!ok) {
            munmap(base, (size_t)st.st_size);
            flock(fd, LOCK_UN); close(fd);
            return NULL;
        }
        if (hdr->sealed) {   /* a frozen file is immutable: refuse a read-write attach */
            SHM_ERR("%s is frozen (read-only); open it with new_readonly", path);
            munmap(base, (size_t)st.st_size);
            flock(fd, LOCK_UN); close(fd);
            return NULL;
        }
    }

    if (fd >= 0) { flock(fd, LOCK_UN); close(fd); }
    #undef SHM_ERR

    return shm_alloc_handle(base, mapped_size, has_arena, has_lru, has_ttl,
                             &lo, path, -1, errbuf);
}

/* ---- memfd-backed map (fd-shareable, no filesystem presence) ---- */
static ShmHandle *shm_create_memfd(const char *name, uint32_t max_entries,
                                    uint32_t node_size, uint32_t variant_id,
                                    int has_arena, uint32_t max_size,
                                    uint32_t default_ttl, uint32_t lru_skip,
                                    uint64_t arena_cap_override, char *errbuf) {
    if (errbuf) errbuf[0] = '\0';

    uint32_t max_tcap = shm_max_tcap_from_entries(max_entries);
    int has_lru = (max_size > 0);
    int has_ttl = (default_ttl > 0);

    ShmLayout lo;
    shm_compute_layout(&lo, max_tcap, node_size, has_lru, has_ttl, has_arena, max_entries, arena_cap_override);

    int fd = memfd_create(name ? name : "hashmap", MFD_CLOEXEC | MFD_ALLOW_SEALING);
    if (fd < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "memfd_create: %s", strerror(errno));
        return NULL;
    }
    if (ftruncate(fd, (off_t)lo.total_size) < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "ftruncate: %s", strerror(errno));
        close(fd); return NULL;
    }
    (void)fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW);
    void *base = mmap(NULL, (size_t)lo.total_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (base == MAP_FAILED) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno));
        close(fd); return NULL;
    }

    shm_init_header((ShmHeader *)base, base, &lo, max_tcap, node_size, variant_id,
                    has_arena, has_lru, has_ttl, max_size, default_ttl, lru_skip);

    return shm_alloc_handle(base, lo.total_size, has_arena, has_lru, has_ttl,
                             &lo, NULL, fd, errbuf);
}

/* ---- Re-open a memfd (or any existing SHM-formatted fd) ---- */
static ShmHandle *shm_open_fd_map(int fd, uint32_t variant_id, uint32_t node_size,
                                   char *errbuf) {
    if (errbuf) errbuf[0] = '\0';
    struct stat st;
    if (fstat(fd, &st) < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat: %s", strerror(errno));
        return NULL;
    }
    if ((uint64_t)st.st_size < sizeof(ShmHeader)) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fd: file too small for header");
        return NULL;
    }
    size_t ms = (size_t)st.st_size;
    void *base = mmap(NULL, ms, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (base == MAP_FAILED) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap: %s", strerror(errno));
        return NULL;
    }

    ShmHeader *hdr = (ShmHeader *)base;
    if (hdr->total_size != (uint64_t)st.st_size ||
        !shm_validate_header_live(hdr, variant_id, node_size)) {
        shm_format_header_error(errbuf, "fd", hdr, variant_id);
        munmap(base, ms);
        return NULL;
    }
    if (hdr->sealed) {   /* frozen: refuse a read-write attach (new_readonly needs a path; a memfd has none) */
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                             "this map is frozen (read-only); open a file with new_readonly; "
                             "a frozen memfd cannot be reopened");
        munmap(base, ms);
        return NULL;
    }

    int has_arena = (hdr->arena_off != 0);
    int has_lru   = (hdr->max_size > 0);
    int has_ttl   = (hdr->default_ttl > 0);
    ShmLayout lo;
    if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, "fd")) {
        munmap(base, ms);
        return NULL;
    }

    int myfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
    if (myfd < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fcntl: %s", strerror(errno));
        munmap(base, ms);
        return NULL;
    }
    return shm_alloc_handle(base, hdr->total_size, has_arena, has_lru, has_ttl,
                             &lo, NULL, myfd, errbuf);
}

/* Open a sealed map read-only: O_RDONLY + PROT_READ, no lock ever, so it works
 * from a read-only filesystem and any number of processes can share it.  A
 * non-frozen file is refused, its lock-free readers would race a live writer.
 * Single-file only. */
static ShmHandle *shm_open_readonly_map(const char *path, uint32_t variant_id,
                                        uint32_t node_size, char *errbuf) {
    if (errbuf) errbuf[0] = '\0';
    int fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
    if (fd < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "open(%s): %s", path, strerror(errno));
        return NULL;
    }
    struct stat st;
    if (fstat(fd, &st) < 0) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "fstat(%s): %s", path, strerror(errno));
        close(fd); return NULL;
    }
    if ((uint64_t)st.st_size < sizeof(ShmHeader)) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "%s: file too small for header", path);
        close(fd); return NULL;
    }
    size_t ms = (size_t)st.st_size;
    void *base = mmap(NULL, ms, PROT_READ, MAP_SHARED, fd, 0);
    int mmap_errno = errno;
    close(fd);   /* the mapping keeps the file; a read-only view needs no fd (no msync/ftruncate) */
    if (base == MAP_FAILED) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "mmap(%s): %s", path, strerror(mmap_errno));
        return NULL;
    }
    ShmHeader *hdr = (ShmHeader *)base;
    if (hdr->total_size != (uint64_t)st.st_size ||
        !shm_validate_header_live(hdr, variant_id, node_size)) {
        shm_format_header_error(errbuf, path, hdr, variant_id);
        munmap(base, ms);
        return NULL;
    }
    if (!hdr->sealed) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                             "%s is not frozen: call ->freeze on the producer before opening read-only", path);
        munmap(base, ms);
        return NULL;
    }

    /* A write in flight on a sealed file can never be repaired, since a
     * read-write attach refuses one.  Refuse either way, but say which: freeze
     * seals while still holding the lock, so a live holder will finish. */
    if ((hdr->seq & 1) || hdr->wlock != 0) {
        uint32_t holder = hdr->wlock & SHM_RWLOCK_PID_MASK;
        if (errbuf) {
            if (holder && shm_pid_alive(holder))
                snprintf(errbuf, SHM_ERR_BUFLEN,
                    "%s: a write is still in flight on this frozen file (pid %u); retry",
                    path, holder);
            else
                snprintf(errbuf, SHM_ERR_BUFLEN,
                    "%s: frozen file was left mid-update by a crashed writer; recreate it", path);
        }
        munmap(base, ms);
        return NULL;
    }
    int has_arena = (hdr->arena_off != 0);
    int has_lru   = (hdr->max_size > 0);
    int has_ttl   = (hdr->default_ttl > 0);
    ShmLayout lo;
    if (!shm_validate_layout_regions(&lo, hdr, has_arena, hdr->total_size, errbuf, path)) {
        munmap(base, ms);
        return NULL;
    }
    ShmHandle *h = shm_alloc_handle(base, hdr->total_size, has_arena, has_lru, has_ttl,
                                    &lo, path, -1, errbuf);   /* munmaps + frees on OOM */
    if (!h) return NULL;
    h->readonly = 1;   /* reads lock-free; every mutator now croaks */
    return h;
}

static inline int shm_msync(ShmHandle *h) {
    if (!h) return 0;
    if (h->readonly) return 0;   /* PROT_READ view has nothing to flush -- sync() is a silent no-op */
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++) {
            int rc;
            do { rc = msync(h->shard_handles[i]->hdr,
                            h->shard_handles[i]->mmap_size, MS_SYNC); }
            while (rc != 0 && errno == EINTR);  /* retry on signal interruption */
            if (rc != 0) return rc;
        }
        return 0;
    }
    if (!h->hdr) return 0;
    int rc;
    do { rc = msync(h->hdr, h->mmap_size, MS_SYNC); }
    while (rc != 0 && errno == EINTR);
    return rc;
}

/* Seal a map permanently immutable, under the write lock so no mutation is in
 * flight; every shard of a sharded map is sealed.  Returns 0, or non-zero with
 * errno on a failed msync.  The handle is still writable here: shm_mark_readonly
 * runs only after this returns. */
static int shm_freeze(ShmHandle *h) {
    if (!h) return 0;
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++) {
            int rc = shm_freeze(h->shard_handles[i]);
            if (rc != 0) return rc;
        }
        return 0;
    }
    shm_rwlock_wrlock(h);
    __atomic_store_n(&h->hdr->sealed, 1, __ATOMIC_RELEASE);
    shm_rwlock_wrunlock(h);
    if (h->path || h->backing_fd >= 0) return shm_msync(h);  /* durability for file/memfd-backed */
    return 0;   /* anonymous: the seal lives in shared memory (visible to forks); nothing to flush */
}

/* Mark this handle (and, for a sharded map, every shard handle) read-only after
 * a successful freeze: subsequent mutators croak and reads go lock-free. */
static void shm_mark_readonly(ShmHandle *h) {
    if (!h) return;
    h->readonly = 1;
    if (h->shard_handles)
        for (uint32_t i = 0; i < h->num_shards; i++)
            if (h->shard_handles[i]) h->shard_handles[i]->readonly = 1;
}

/* Is the underlying FILE sealed?  h->readonly is process-local and misses a
 * handle opened read-write before the freeze, which would then keep writing to
 * an immutable map, so consult the shared header.  Sharded: freeze seals shard
 * 0 first and stops at the first failure, so shard 0 answers for the set. */
static inline int shm_is_sealed(const ShmHandle *h) {
    if (!h) return 0;
    if (h->shard_handles) {
        const ShmHandle *s = h->num_shards ? h->shard_handles[0] : NULL;
        return s && s->hdr && __atomic_load_n(&s->hdr->sealed, __ATOMIC_RELAXED);
    }
    return h->hdr && __atomic_load_n(&h->hdr->sealed, __ATOMIC_RELAXED);
}

static void shm_close_map_now(ShmHandle *h);

/* Destroying a handle this process holds a lock on would leave the save-stack
 * cleanup dereferencing freed memory on unwind, and argument magic can call
 * DESTROY from inside a guarded region: defer the free to the last unlock.
 * lock_depth is process-local and needs no atomics, since CLONE_SKIP forbids
 * sharing a handle between threads. */
static void shm_close_map(ShmHandle *h) {
    if (!h) return;
    if (h->lock_depth) { h->pending_close = 1; return; }
    shm_close_map_now(h);
}

static void shm_close_map_now(ShmHandle *h) {
    if (!h) return;
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++)
            shm_close_map(h->shard_handles[i]);
        free(h->shard_handles);
        free(h->path);
        free(h);
        return;
    }
    /* Release our slot only if we still own it and no fork has intervened: a
     * child that inherited the handle without locking must not clear the
     * parent's slot through the inherited cached_pid. */
    if (h->reader_slots && h->my_slot_idx != UINT32_MAX && h->cached_pid &&
        h->cached_fork_gen == __atomic_load_n(&shm_fork_gen, __ATOMIC_RELAXED) &&
        __atomic_load_n(&h->reader_slots[h->my_slot_idx].rdepth, __ATOMIC_ACQUIRE) == 0) {
        /* rdepth==0: a still-held read lock's slot must survive for recovery */
        /* Clear the occ bit before freeing the slot: we still own the pid, so
         * no claimant can take it mid-clear. */
        shm_occ_clear(h, h->my_slot_idx);
        uint32_t expected = h->cached_pid;
        /* CAS pid to 0 and leave rdepth alone: a new claimant between the CAS
         * and a store would have its increments clobbered.  The next claim
         * zeroes it. */
        __atomic_compare_exchange_n(&h->reader_slots[h->my_slot_idx].pid,
                &expected, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
    }
    if (h->hdr) munmap(h->hdr, h->mmap_size);
    if (h->backing_fd >= 0) close(h->backing_fd);
    free(h->copy_buf);
    free(h->path);
    free(h);
}

/* Unwind a partially built sharded dispatcher: `created` shards are open. */
static ShmHandle *shm_sharded_fail(ShmHandle *h, uint32_t created) {
    for (uint32_t j = 0; j < created; j++)
        shm_close_map(h->shard_handles[j]);
    free(h->shard_handles);
    free(h->path);
    free(h);
    return NULL;
}

/* Create a sharded map: N independent maps behind one handle */
static ShmHandle *shm_create_sharded(const char *path_prefix, uint32_t num_shards,
                                      uint32_t max_entries, uint32_t node_size,
                                      uint32_t variant_id, int has_arena,
                                      uint32_t max_size, uint32_t default_ttl,
                                      uint32_t lru_skip, uint64_t arena_cap_override,
                                      mode_t file_mode, char *errbuf) {
    if (!path_prefix) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "new_sharded requires a path_prefix");
        return NULL;
    }
    /* Round up to a power of two.  Bound first: above 2^31 the shift overflows
     * to 0 and the loop never terminates. */
    if (num_shards == 0) num_shards = 1;
    if (num_shards > SHM_MAX_SHARDS) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                             "num_shards %u exceeds the maximum of %u",
                             num_shards, (unsigned)SHM_MAX_SHARDS);
        return NULL;
    }
    uint32_t ns = 1;
    uint8_t want_log2 = 1;
    while (ns < num_shards) { ns <<= 1; want_log2++; }
    num_shards = ns;

    ShmHandle *h = (ShmHandle *)calloc(1, sizeof(ShmHandle));
    if (!h) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return NULL; }

    h->shard_handles = (ShmHandle **)calloc(num_shards, sizeof(ShmHandle *));
    if (!h->shard_handles) { if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "calloc: out of memory"); return shm_sharded_fail(h, 0); }

    h->num_shards = num_shards;
    h->shard_mask = num_shards - 1;
    h->backing_fd = -1;   /* dispatcher owns no fd; calloc left it 0 (=stdin) */
    h->path = strdup(path_prefix);
    if (!h->path) {
        if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "strdup: out of memory");
        return shm_sharded_fail(h, 0);
    }

    for (uint32_t i = 0; i < num_shards; i++) {
        char shard_path[4096];
        int sn = snprintf(shard_path, sizeof(shard_path), "%s.%u", path_prefix, i);
        if (sn < 0 || sn >= (int)sizeof(shard_path)) {
            if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN, "shard path too long");
            return shm_sharded_fail(h, i);
        }
        h->shard_handles[i] = shm_create_map(shard_path, max_entries, node_size,
                                               variant_id, has_arena, max_size,
                                               default_ttl, lru_skip, arena_cap_override, file_mode, errbuf);
        if (!h->shard_handles[i]) return shm_sharded_fail(h, i);
        /* A recorded count must match whichever shard carries it: opening a set
         * with the wrong one hides every key that routes elsewhere.  A shard
         * carrying 0 was never stamped and agrees with any count. */
        const ShmHeader *sh = h->shard_handles[i]->hdr;
        if (sh->routing != SHM_ROUTING_LEGACY && sh->shard_log2 &&
            sh->shard_log2 != want_log2) {
            unsigned recorded = 1u << ((sh->shard_log2 - 1) & 31);
            if (errbuf && i == 0)
                snprintf(errbuf, SHM_ERR_BUFLEN,
                    "%s was created with %u shards, not %u",
                    path_prefix, recorded, num_shards);
            else if (errbuf)
                snprintf(errbuf, SHM_ERR_BUFLEN,
                    "%s.%u records %u shards, not %u",
                    path_prefix, i, recorded, num_shards);
            return shm_sharded_fail(h, i + 1);
        }
        if (i > 0) {
            /* Every shard must carry the same configuration, or a stray file
             * from an earlier run is adopted into the set.  Shard 0 is the
             * reference, so a stray there is reported against the first shard
             * that differs from it.  The count is checked per shard above. */
            const ShmHeader *a = h->shard_handles[0]->hdr;
            const ShmHeader *b = sh;
            const char *what =
                b->max_table_cap != a->max_table_cap ? "max_entries" :
                b->max_size      != a->max_size      ? "max_size"    :
                b->default_ttl   != a->default_ttl   ? "ttl"         :
                b->lru_skip      != a->lru_skip      ? "lru_skip"    :
                b->arena_cap     != a->arena_cap     ? "arena_cap"   :
                b->routing       != a->routing       ? "routing"     : NULL;
            if (what) {
                if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                    "%s.%u disagrees with %s.0 on %s; the shards were not created "
                    "together", path_prefix, i, path_prefix, what);
                return shm_sharded_fail(h, i + 1);
            }
        }
    }
    /* Stamp only once every shard has agreed, or a refused open cements this
     * caller's count into a shard the next opener would trust.  Shard 0 is the
     * serialisation point: a second creator with a different count loses the
     * CAS and is refused, its shard files left behind. */
    ShmHeader *s0 = h->shard_handles[0]->hdr;
    if (s0->routing != SHM_ROUTING_LEGACY) {
        uint8_t seen = 0;
        if (!__atomic_compare_exchange_n(&s0->shard_log2, &seen, want_log2, 0,
                                         __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE) &&
            seen != want_log2) {
            if (errbuf) snprintf(errbuf, SHM_ERR_BUFLEN,
                "%s was created with %u shards, not %u",
                path_prefix, 1u << ((seen - 1) & 31), num_shards);
            return shm_sharded_fail(h, num_shards);
        }
        for (uint32_t i = 1; i < num_shards; i++) {
            ShmHeader *s = h->shard_handles[i]->hdr;
            if (s->shard_log2 == 0) s->shard_log2 = want_log2;
        }
    }

    h->route_high = (s0->routing != SHM_ROUTING_LEGACY);
    return h;
}

/* Unlink the backing file. Returns 1 on success, 0 on failure. */
static int shm_unlink_path(const char *path) {
    return (unlink(path) == 0) ? 1 : 0;
}

static int shm_unlink_sharded(ShmHandle *h) {
    if (!h->shard_handles) {
        if (!h->path) return 0;
        return shm_unlink_path(h->path);
    }
    int ok = 1;
    for (uint32_t i = 0; i < h->num_shards; i++) {
        if (h->shard_handles[i] && h->shard_handles[i]->path) {
            if (unlink(h->shard_handles[i]->path) != 0) ok = 0;
        }
    }
    return ok;
}

static inline ShmCursor *shm_cursor_create(ShmHandle *h) {
    ShmCursor *c = (ShmCursor *)calloc(1, sizeof(ShmCursor));
    if (!c) return NULL;
    c->handle = h;
    if (h->shard_handles) {
        c->shard_count = h->num_shards;
        c->shard_idx = 0;
        c->current = h->shard_handles[0];
    } else {
        c->shard_count = 1;
        c->shard_idx = 0;
        c->current = h;
    }
    c->gen = c->current->hdr->table_gen;
    c->current->iterating++;
    return c;
}

static inline uint32_t shm_shard_index(const ShmHandle *h, uint64_t hash64) {
    uint32_t bits = h->route_high ? (uint32_t)(hash64 >> 32) : (uint32_t)hash64;
    return bits & h->shard_mask;
}

static inline void shm_cursor_destroy(ShmCursor *c) {
    if (!c) return;
    ShmHandle *cur = c->current;
    if (cur && cur->iterating > 0)
        cur->iterating--;
    free(c->copy_buf);
    free(c);
}

#endif /* SHM_DEFS_H */


/* ---- Part 2: template (included once per variant) ---- */

#define SHM_PASTE2(a, b) a##_##b
#define SHM_PASTE(a, b)  SHM_PASTE2(a, b)
#define SHM_FN(name)     SHM_PASTE(SHM_PREFIX, name)

/* ---- Node struct ---- */

typedef struct {
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key;
#else
    uint32_t key_off;
    uint32_t key_len; /* high bit = UTF-8 */
#endif
#ifdef SHM_VAL_IS_STR
    uint32_t val_off;
    uint32_t val_len; /* high bit = UTF-8 */
#else
    SHM_VAL_INT_TYPE value;
#endif
} SHM_NODE_TYPE;

/* ---- Shard dispatch for entry-point functions ---- */
#undef SHM_SHARD_DISPATCH
#ifdef SHM_KEY_IS_INT
  #define SHM_SHARD_DISPATCH(h, key_arg) \
      if ((h)->shard_handles) { \
          h = (h)->shard_handles[shm_shard_index((h), SHM_HASH_KEY64(key_arg))]; \
      }
#else
  #define SHM_SHARD_DISPATCH(h, key_str_arg, key_len_arg) \
      if ((h)->shard_handles) { \
          h = (h)->shard_handles[shm_shard_index((h), SHM_HASH_KEY_STR64(key_str_arg, key_len_arg))]; \
      }
#endif

/* ---- Key hashing ---- */

#ifdef SHM_KEY_IS_INT
  #define SHM_HASH_KEY(k) ((uint32_t)(shm_hash_int64((int64_t)(k))))
  #define SHM_HASH_KEY64(k) (shm_hash_int64((int64_t)(k)))
  #define SHM_KEY_EQ(node_ptr, k) ((node_ptr)->key == (k))
#else
  #define SHM_HASH_KEY_STR(str, len) ((uint32_t)shm_hash_string((str), (len)))
  #define SHM_HASH_KEY_STR64(str, len) (shm_hash_string((str), (len)))
  /* Keys compare by bytes only: the stored UTF8 flag is metadata for
   * restoration on retrieval, and letting it affect equality would make an
   * ASCII key miss its own value after a utf8::upgrade. */
  static inline int SHM_PASTE(SHM_PREFIX, _key_eq_str)(
      const SHM_NODE_TYPE *np, const char *arena, uint64_t arena_cap,
      const char *str, uint32_t len, bool utf8) {
      (void)utf8;
      uint32_t kl_packed = np->key_len;
      if (SHM_IS_INLINE(kl_packed)) {
          if (shm_inline_len(kl_packed) != len) return 0;
          char buf[SHM_INLINE_MAX];
          shm_inline_read(np->key_off, kl_packed, buf);
          return memcmp(buf, str, len) == 0;
      }
      if (SHM_UNPACK_LEN(kl_packed) != len) return 0;
      /* A corrupt key_off must never reach memcmp (CWE-125); treat it as a miss
       * so the probe continues.  get() and exists() bound their own inlined
       * compare but end the probe there, so a poisoned slot hides later keys
       * from them. */
      if ((uint64_t)np->key_off + len > arena_cap) return 0;
      return memcmp(arena + np->key_off, str, len) == 0;
  }
  #define SHM_KEY_EQ_STR(node_ptr, arena, arena_cap, str, len, utf8) \
      SHM_PASTE(SHM_PREFIX, _key_eq_str)((node_ptr), (arena), (arena_cap), \
                                         (str), (len), (utf8))
#endif

/* ---- Create / Open ---- */

/* This variant uses the arena (string keys and/or string values). */
#if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
  #define SHM_HAS_ARENA 1
#else
  #define SHM_HAS_ARENA 0
#endif

static ShmHandle *SHM_FN(create)(const char *path, uint32_t max_entries,
                                  uint32_t max_size, uint32_t default_ttl,
                                  uint32_t lru_skip, uint64_t arena_cap_override,
                                  mode_t file_mode, char *errbuf) {
    return shm_create_map(path, max_entries,
                           (uint32_t)sizeof(SHM_NODE_TYPE),
                           SHM_VARIANT_ID, SHM_HAS_ARENA,
                           max_size, default_ttl, lru_skip, arena_cap_override, file_mode, errbuf);
}

static ShmHandle *SHM_FN(create_sharded)(const char *path_prefix,
                                          uint32_t num_shards,
                                          uint32_t max_entries,
                                          uint32_t max_size, uint32_t default_ttl,
                                          uint32_t lru_skip, uint64_t arena_cap_override,
                                          mode_t file_mode, char *errbuf) {
    return shm_create_sharded(path_prefix, num_shards, max_entries,
                               (uint32_t)sizeof(SHM_NODE_TYPE),
                               SHM_VARIANT_ID, SHM_HAS_ARENA,
                               max_size, default_ttl, lru_skip, arena_cap_override, file_mode, errbuf);
}

static ShmHandle *SHM_FN(create_memfd)(const char *name, uint32_t max_entries,
                                        uint32_t max_size, uint32_t default_ttl,
                                        uint32_t lru_skip, uint64_t arena_cap_override,
                                        char *errbuf) {
    return shm_create_memfd(name, max_entries,
                             (uint32_t)sizeof(SHM_NODE_TYPE),
                             SHM_VARIANT_ID, SHM_HAS_ARENA,
                             max_size, default_ttl, lru_skip, arena_cap_override, errbuf);
}

static ShmHandle *SHM_FN(open_fd)(int fd, char *errbuf) {
    return shm_open_fd_map(fd, SHM_VARIANT_ID,
                            (uint32_t)sizeof(SHM_NODE_TYPE), errbuf);
}

/* Open a frozen file read-only for THIS variant (validates variant id + node
 * size, like open_fd).  See shm_open_readonly_map. */
static ShmHandle *SHM_FN(open_readonly)(const char *path, char *errbuf) {
    return shm_open_readonly_map(path, SHM_VARIANT_ID,
                                  (uint32_t)sizeof(SHM_NODE_TYPE), errbuf);
}

/* ---- Rehash helper (used during resize) -- returns new index ---- */

static uint32_t SHM_FN(rehash_insert_raw)(ShmHandle *h, SHM_NODE_TYPE *node) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t mask = hdr->table_cap - 1;

#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(node->key);
#else
    char ibuf[SHM_INLINE_MAX];
    uint32_t klen;
    const char *kptr = shm_str_ptr(node->key_off, node->key_len, h->arena, h->hdr->arena_cap, ibuf, &klen);
    uint32_t hash = shm_hash_string(kptr, klen);
#endif

    uint32_t pos = hash & mask;
    while (SHM_IS_LIVE(states[pos]))
        pos = (pos + 1) & mask;

    nodes[pos] = *node;
    states[pos] = SHM_MAKE_TAG(hash);
    return pos;
}

/* ---- Tombstone at index (helper for eviction/expiry) ---- */

static void SHM_FN(tombstone_at)(ShmHandle *h, uint32_t idx) {
    ShmHeader *hdr = h->hdr;
#if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
#endif
    /* Retire the slot before releasing its arena blocks: a writer killed
     * between the two would leave a live node pointing at a block the free
     * list has clobbered with its next pointer. */
    h->states[idx] = SHM_TOMBSTONE;
    hdr->size--;
    hdr->tombstones++;
#ifndef SHM_KEY_IS_INT
    shm_str_free(hdr, h->arena, nodes[idx].key_off, nodes[idx].key_len);
#endif
#ifdef SHM_VAL_IS_STR
    shm_str_free(hdr, h->arena, nodes[idx].val_off, nodes[idx].val_len);
#endif
}

/* Remove a caller-located entry: detach from LRU, clear TTL, tombstone.
 * Distinct from expire_at, which bills stat_expired, and from lru_evict_one,
 * which picks its own victim. */
static inline void SHM_FN(remove_at)(ShmHandle *h, uint32_t idx) {
    if (h->lru_prev) shm_lru_unlink(h, idx);
    if (h->expires_at) h->expires_at[idx] = 0;
    SHM_FN(tombstone_at)(h, idx);
}

/* ---- LRU eviction ---- */

/* Retire `idx` under capacity pressure.  A victim already past its TTL is
 * billed to expiration, so stat_evictions reflects true pressure. */
static void SHM_FN(lru_evict_at)(ShmHandle *h, uint32_t idx) {
    int was_expired = SHM_IS_EXPIRED(h, idx, shm_now());
    SHM_FN(remove_at)(h, idx);
    __atomic_add_fetch(was_expired ? &h->hdr->stat_expired
                                   : &h->hdr->stat_evictions, 1, __ATOMIC_RELAXED);
}

/* Second-chance clock from the tail: an accessed entry is promoted instead
 * and the first unaccessed one is evicted.  `keep` is never the victim: it is
 * the entry an overwrite is replacing (SHM_LRU_NONE for an insert). */
static void SHM_FN(lru_evict_one_keep)(ShmHandle *h, uint32_t keep) {
    ShmHeader *hdr = h->hdr;
    uint32_t victim = hdr->lru_tail;
    while (victim != SHM_LRU_NONE) {
        if (victim == keep) {
            victim = h->lru_prev[victim];
            continue;
        }
        if (h->lru_accessed && __atomic_load_n(&h->lru_accessed[victim], __ATOMIC_RELAXED)) {
            __atomic_store_n(&h->lru_accessed[victim], 0, __ATOMIC_RELAXED);
            uint32_t prev = h->lru_prev[victim];
            shm_lru_unlink(h, victim);
            shm_lru_push_front(h, victim);
            victim = prev;
            if (victim == SHM_LRU_NONE) victim = hdr->lru_tail;
            continue;
        }
        break;
    }
    if (victim == SHM_LRU_NONE) return;
    SHM_FN(lru_evict_at)(h, victim);
}

static void SHM_FN(lru_evict_one)(ShmHandle *h) {
    SHM_FN(lru_evict_one_keep)(h, SHM_LRU_NONE);
}

/* ---- TTL expiration ---- */

static void SHM_FN(expire_at)(ShmHandle *h, uint32_t idx) {
    SHM_FN(remove_at)(h, idx);
    __atomic_add_fetch(&h->hdr->stat_expired, 1, __ATOMIC_RELAXED);
}

#if SHM_HAS_ARENA
/* Whether a stored string's block is exactly `asize`: classes are never split
 * or merged, so only the request's own class can serve it. */
static inline int SHM_FN(block_is)(uint32_t len_field, uint32_t asize) {
    return !SHM_IS_INLINE(len_field) && shm_arena_round_up(SHM_UNPACK_LEN(len_field)) == asize;
}

/* Make room for `slen`: evict the oldest of the SHM_EVICT_SEARCH oldest
 * entries holding a block of the request's class, else the clock victim, whose
 * block may be the wrong size.  Returns 1 if an entry was evicted. */
static int SHM_FN(lru_evict_for)(ShmHandle *h, uint32_t slen, uint32_t keep) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint32_t asize = shm_arena_round_up(slen);
    uint32_t pos = hdr->lru_tail;
    for (uint32_t n = 0; n < SHM_EVICT_SEARCH && pos != SHM_LRU_NONE; n++) {
        if (pos != keep && (
#ifndef SHM_KEY_IS_INT
                SHM_FN(block_is)(nodes[pos].key_len, asize) ||
#endif
#ifdef SHM_VAL_IS_STR
                SHM_FN(block_is)(nodes[pos].val_len, asize) ||
#endif
                0)) {
            SHM_FN(lru_evict_at)(h, pos);
            return 1;
        }
        pos = h->lru_prev[pos];
    }
    uint32_t before = hdr->size;
    SHM_FN(lru_evict_one_keep)(h, keep);
    return hdr->size < before;
}

/* Store a string, evicting once when the arena is exhausted on an LRU map:
 * otherwise the map would refuse every insert and keep its oldest entries.
 * `keep` is the entry an overwrite is replacing, never the victim. */
static int SHM_FN(store_or_evict)(ShmHandle *h, uint32_t *off, uint32_t *len,
                                  const char *str, uint32_t slen, bool utf8, uint32_t keep) {
    if (shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8)) return 1;
    if (!h->lru_prev) return 0;
    /* Offset 0 is reserved, so a block of exactly arena_cap never fits either. */
    if (slen > SHM_INLINE_MAX &&
        (uint64_t)SHM_ARENA_MIN_ALLOC + shm_arena_round_up(slen) > h->hdr->arena_cap)
        return 0;
    /* One eviction only: if the store still fails, the freed block was the
     * wrong size and further evictions are equally blind. */
    if (!SHM_FN(lru_evict_for)(h, slen, keep)) return 0;
    return shm_str_store(h->hdr, h->arena, off, len, str, slen, utf8);
}
#endif

/* Called only when the insert probe found no slot, so every slot is live:
 * flush every expired entry at once, then return the first freed slot on this
 * key's own probe path, or UINT32_MAX when nothing had expired. */
static uint32_t SHM_FN(reclaim_expired_slot)(ShmHandle *h, uint32_t pos, uint32_t mask) {
    if (!h->expires_at) return UINT32_MAX;
    uint32_t now = shm_now();
    uint32_t freed = 0;
    for (uint32_t i = 0; i <= mask; i++) {
        if (SHM_IS_LIVE(h->states[i]) && SHM_IS_EXPIRED(h, i, now)) {
            SHM_FN(expire_at)(h, i);
            freed++;
        }
    }
    if (!freed) return UINT32_MAX;
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        if (h->states[idx] == SHM_TOMBSTONE) return idx;
    }
    return UINT32_MAX;
}

/* ---- Resize (elastic grow/shrink) ---- */

static int SHM_FN(resize)(ShmHandle *h, uint32_t new_cap) {
    ShmHeader *hdr = h->hdr;
    uint32_t old_cap = hdr->table_cap;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;

    uint32_t live = hdr->size;
    SHM_NODE_TYPE *saved = NULL;
    uint32_t *saved_indices = NULL;
    uint32_t *old_to_new = NULL;
    uint32_t *saved_exp = NULL;

    /* Save LRU order (tail-to-head) */
    uint32_t *lru_order = NULL;
    uint32_t lru_count = 0;
    int need_mapping = (h->lru_prev || h->expires_at);

    if (live > 0) {
        saved = (SHM_NODE_TYPE *)malloc((size_t)live * sizeof(SHM_NODE_TYPE));
        if (!saved) return 0;

        if (need_mapping) {
            saved_indices = (uint32_t *)malloc((size_t)live * sizeof(uint32_t));
            old_to_new = (uint32_t *)malloc((size_t)old_cap * sizeof(uint32_t));
            if (!saved_indices || !old_to_new) {
                free(saved); free(saved_indices); free(old_to_new);
                return 0;
            }
            memset(old_to_new, 0xFF, old_cap * sizeof(uint32_t));
        }

        if (h->lru_prev) {
            lru_order = (uint32_t *)malloc((size_t)live * sizeof(uint32_t));
            if (!lru_order) {
                free(saved); free(saved_indices); free(old_to_new);
                return 0;
            }
            uint32_t idx = hdr->lru_tail;
            while (idx != SHM_LRU_NONE && lru_count < live) {
                lru_order[lru_count++] = idx;
                idx = h->lru_prev[idx];
            }
        }

        if (h->expires_at) {
            saved_exp = (uint32_t *)malloc(old_cap * sizeof(uint32_t));
            if (!saved_exp) {
                free(saved); free(saved_indices); free(old_to_new); free(lru_order);
                return 0;
            }
            memcpy(saved_exp, h->expires_at, old_cap * sizeof(uint32_t));
        }

        uint32_t j = 0;
        for (uint32_t i = 0; i < old_cap && j < live; i++) {
            if (SHM_IS_LIVE(states[i])) {
                saved[j] = nodes[i];
                if (saved_indices) saved_indices[j] = i;
                j++;
            }
        }
        live = j;
    }

    memset(states, SHM_EMPTY, new_cap);
    /* tombstones before capacity: the other order leaves
     * size + tombstones > table_cap on a shrink, and a crash there is
     * unrepairable.  Atomic, or gcc reorders them. */
    __atomic_store_n(&hdr->tombstones, 0, __ATOMIC_RELEASE);
    __atomic_store_n(&hdr->table_cap, new_cap, __ATOMIC_RELEASE);

    if (h->lru_prev) {
        memset(h->lru_prev, 0xFF, new_cap * sizeof(uint32_t));
        memset(h->lru_next, 0xFF, new_cap * sizeof(uint32_t));
        if (h->lru_accessed) memset(h->lru_accessed, 0, new_cap);
        hdr->lru_head = SHM_LRU_NONE;
        hdr->lru_tail = SHM_LRU_NONE;
    }
    if (h->expires_at) {
        memset(h->expires_at, 0, new_cap * sizeof(uint32_t));
    }

    for (uint32_t k = 0; k < live; k++) {
        uint32_t new_idx = SHM_FN(rehash_insert_raw)(h, &saved[k]);
        if (old_to_new) old_to_new[saved_indices[k]] = new_idx;
    }

    /* Rebuild LRU chain in original order (lru_order[0]=tail/LRU, last=head/MRU).
     * Push front from LRU to MRU so the last push (MRU) ends up at head. */
    if (h->lru_prev && lru_order) {
        for (uint32_t i = 0; i < lru_count; i++) {
            uint32_t new_idx = old_to_new[lru_order[i]];
            if (new_idx != SHM_LRU_NONE)
                shm_lru_push_front(h, new_idx);
        }
    }

    if (h->expires_at && saved_exp) {
        for (uint32_t k = 0; k < live; k++) {
            uint32_t new_idx = old_to_new[saved_indices[k]];
            if (new_idx != SHM_LRU_NONE)
                h->expires_at[new_idx] = saved_exp[saved_indices[k]];
        }
    }

    if (new_cap < old_cap) {
        size_t node_shrink = (size_t)(old_cap - new_cap) * sizeof(SHM_NODE_TYPE);
        madvise((char *)nodes + (size_t)new_cap * sizeof(SHM_NODE_TYPE),
                node_shrink, MADV_DONTNEED);
        madvise(states + new_cap, old_cap - new_cap, MADV_DONTNEED);
    }

    hdr->table_gen++;

    free(saved);
    free(saved_indices);
    free(old_to_new);
    free(lru_order);
    free(saved_exp);
    return 1;
}

static inline void SHM_FN(maybe_grow)(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    uint32_t size = hdr->size, tomb = hdr->tombstones, cap = hdr->table_cap;
    if (__builtin_expect(shm_over_load(size, tomb, cap), 0)) {
        /* Grow even while iterating, or an abandoned each wedges the table at
         * capacity: both iterators auto-reset on the table_gen bump.
         * Compaction and shrink stay deferred. */
        if (cap < hdr->max_table_cap)  /* cap is pow2; cap*2 can't overflow here */
            SHM_FN(resize)(h, cap * 2);
        else if (shm_needs_compaction(size, tomb, cap))
            SHM_FN(resize)(h, cap);  /* at max capacity: compact tombstones in place */
    } else if (__builtin_expect(shm_needs_compaction(size, tomb, cap), 0)) {
        if (h->iterating > 0) { h->deferred = 1; return; }
        SHM_FN(resize)(h, cap);
    }
}

static inline void SHM_FN(maybe_shrink)(ShmHandle *h) {
    ShmHeader *hdr = h->hdr;
    if (__builtin_expect(shm_under_load(hdr->size, hdr->table_cap), 0)) {
        if (h->iterating > 0) { h->deferred = 1; return; }
        SHM_FN(resize)(h, shm_shrink_target(hdr->size, hdr->table_cap));
    }
}

static inline void SHM_FN(flush_deferred)(ShmHandle *h) {
    if (!h || !h->deferred || h->iterating > 0) return;
    h->deferred = 0;
    /* A resize deferred while an iterator was live outlives freeze(), and the
     * paths that reach here are not mutators and carry no frozen guard: an
     * each() ending, iter_reset, and a cursor's next, reset, seek and
     * teardown. */
    if (h->readonly || shm_is_sealed(h)) return;
    ShmHeader *hdr = h->hdr;
    shm_rwlock_wrlock(h);
    /* Re-test under the lock: freeze() seals under this same lock, so a call
     * parked here would resize a map sealed meanwhile. */
    if (shm_is_sealed(h)) { shm_rwlock_wrunlock(h); return; }
    shm_seqlock_write_begin(&hdr->seq);
    uint32_t size = hdr->size, tomb = hdr->tombstones, cap = hdr->table_cap;
    if (shm_over_load(size, tomb, cap)) {
        if (cap < hdr->max_table_cap)  /* cap is pow2; cap*2 can't overflow here */
            SHM_FN(resize)(h, cap * 2);
        else if (shm_needs_compaction(size, tomb, cap))
            SHM_FN(resize)(h, cap);  /* at max capacity: compact tombstones in place */
    } else if (shm_under_load(size, cap)) {
        SHM_FN(resize)(h, shm_shrink_target(size, cap));
    } else if (shm_needs_compaction(size, tomb, cap)) {
        SHM_FN(resize)(h, cap);
    }
    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
}

/* ---- Put ---- */

/* put_inner: probe+insert/update WITHOUT locking. Caller holds wrlock+seqlock.
 * Returns 1 on success, 0 on failure (arena full / table full). */
static int SHM_FN(put_inner)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint32_t insert_pos = UINT32_MAX;

    uint32_t exp_ts = 0;
    if (h->expires_at) {
        uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec;
        if (ttl > 0)
            exp_ts = shm_expiry_ts(ttl);
    }

    uint8_t tag = SHM_MAKE_TAG(hash);
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);

        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
#ifdef SHM_VAL_IS_STR
            {
                uint32_t old_off = nodes[idx].val_off;
                uint32_t old_lf = nodes[idx].val_len;
                if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx))
                    return 0;
                shm_str_free(hdr, h->arena, old_off, old_lf);
            }
#else
            __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
#endif
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at) {
                if (ttl_sec != SHM_TTL_USE_DEFAULT || h->expires_at[idx] != 0)
                    h->expires_at[idx] = exp_ts;
            }
            return 1;
        }
    }

    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX)
        return 0;

    /* LRU eviction only when actually inserting */
    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);

#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE))
        return 0;
#endif

#ifdef SHM_VAL_IS_STR
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
#ifndef SHM_KEY_IS_INT
        shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
#endif
        return 0;
    }
#else
    __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
#endif

    states[insert_pos] = SHM_MAKE_TAG(hash);
    /* tombstones-- before size++: the other order briefly makes
     * size + tombstones == table_cap + 1, and a concurrent open in that window
     * refuses a healthy saturated map as corrupt. */
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at) h->expires_at[insert_pos] = exp_ts;

    return 1;
}

static int SHM_FN(put_impl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&h->hdr->seq);
    int rc = SHM_FN(put_inner)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        ttl_sec);
    shm_seqlock_write_end(&h->hdr->seq);
    shm_rwlock_wrunlock(h);
    return rc;
}

static inline int SHM_FN(put)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8
#else
    SHM_VAL_INT_TYPE value
#endif
) {
    return SHM_FN(put_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        SHM_TTL_USE_DEFAULT);
}

static inline int SHM_FN(put_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
    if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
    return SHM_FN(put_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        ttl_sec);
}

/* ---- Get (seqlock -- lock-free read path) ---- */

static int SHM_FN(get)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_str, uint32_t *out_len, bool *out_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    /* Lock-free for every map: LRU sets the clock bit rather than taking the
     * write lock, and the TTL check runs under seqlock retry. */
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t max_mask = h->max_mask;
#if !defined(SHM_KEY_IS_INT) || defined(SHM_VAL_IS_STR)
    uint64_t arena_cap = hdr->arena_cap;  /* immutable after create */
#endif
#ifndef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#else
    uint32_t hash = SHM_HASH_KEY(key);
#endif

    for (;;) {
        uint32_t seq = shm_seqlock_read_begin(h);

        uint32_t mask = (hdr->table_cap - 1) & max_mask;
        uint32_t pos = hash & mask;
        int found = 0;
        uint32_t local_idx = 0;

#ifdef SHM_VAL_IS_STR
        uint32_t local_vl = 0, local_voff = 0, local_vlen_packed = 0;
#else
        SHM_VAL_INT_TYPE local_value = 0;
#endif

        uint8_t tag = SHM_MAKE_TAG(hash);
        uint32_t probe_start = 0;  /* scalar loop starts here */

#ifdef __SSE2__
        /* Bound by the clamped mask+1, never a live re-read of table_cap: a
         * peer corrupting it past max_table_cap must not let this group load
         * run off states[]. */
        if (pos + 16 <= mask + 1) {
            uint16_t mmask, emask;
            shm_probe_group(states, pos, tag, &mmask, &emask);
            /* Only consider matches before first empty */
            uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF;
            uint16_t relevant = mmask & cutoff;
            while (relevant) {
                int bit = __builtin_ctz(relevant);
                uint32_t idx = pos + bit;
                __builtin_prefetch(&nodes[idx], 0, 1);
#ifdef SHM_KEY_IS_INT
                if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
                {
                    uint32_t kl_packed = nodes[idx].key_len;
                    uint32_t kl = SHM_STR_LEN(kl_packed);
                    (void)key_utf8;  /* flag is metadata for retrieval, not part of key identity */
                    if (kl != key_len) goto simd_next;
                    if (SHM_IS_INLINE(kl_packed)) {
                        char ibuf[SHM_INLINE_MAX];
                        shm_inline_read(nodes[idx].key_off, kl_packed, ibuf);
                        if (memcmp(ibuf, key_str, kl) != 0) goto simd_next;
                    } else {
                        uint32_t koff = nodes[idx].key_off;
                        if ((uint64_t)koff + kl > arena_cap) goto simd_done;
                        if (memcmp(h->arena + koff, key_str, kl) != 0) goto simd_next;
                    }
                }
                {
#endif
#ifdef SHM_VAL_IS_STR
                    local_vlen_packed = nodes[idx].val_len;
                    local_vl = SHM_STR_LEN(local_vlen_packed);
                    local_voff = nodes[idx].val_off;
#else
                    local_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
                    local_idx = idx;
                    found = 1;
                    goto simd_done;
                }
#ifndef SHM_KEY_IS_INT
            simd_next:  /* goto target only exists on the string-key compare path */
#endif
                relevant &= relevant - 1;
            }
            if (emask) goto simd_done;  /* hit empty -- key absent */
            probe_start = 16;           /* all 16 occupied, continue scalar from pos+16 */
        }
simd_done:
#endif
        if (!found) {
            for (uint32_t i = probe_start; i <= mask; i++) {
                uint32_t idx = (pos + i) & mask;
                uint8_t st = states[idx];
                __builtin_prefetch(&nodes[idx], 0, 1);
                __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);

                if (st == SHM_EMPTY) break;
                if (st != tag) continue;

#ifdef SHM_KEY_IS_INT
                if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
                {
                    uint32_t kl_packed = nodes[idx].key_len;
                    uint32_t koff = nodes[idx].key_off;
                    uint32_t kl = SHM_STR_LEN(kl_packed);
                    if (kl != key_len) continue;
                    /* key_utf8 is metadata for retrieval, not part of identity */
                    if (SHM_IS_INLINE(kl_packed)) {
                        char ibuf[SHM_INLINE_MAX];
                        shm_inline_read(koff, kl_packed, ibuf);
                        if (memcmp(ibuf, key_str, kl) != 0) continue;
                    } else {
                        if ((uint64_t)koff + kl > arena_cap) break;
                        if (memcmp(h->arena + koff, key_str, kl) != 0) continue;
                    }
                }
                {
#endif
#ifdef SHM_VAL_IS_STR
                    local_vlen_packed = nodes[idx].val_len;
                    local_vl = SHM_STR_LEN(local_vlen_packed);
                    local_voff = nodes[idx].val_off;
#else
                    local_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
                    local_idx = idx;
                    found = 1;
                    break;
                }
            }
        }

        if (found) {
            /* TTL check under seqlock (torn expiry caught by retry) */
            if (h->expires_at) {
                uint32_t exp = h->expires_at[local_idx];
                if (exp != 0 && shm_now() >= exp) {
                    found = 0;  /* expired -- treat as absent */
                }
            }
        }

        if (found) {
#ifdef SHM_VAL_IS_STR
            if (SHM_IS_INLINE(local_vlen_packed)) {
                if (local_vl > h->copy_buf_size || !h->copy_buf) {
                    if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) return 0;
                    continue;
                }
                shm_inline_read(local_voff, local_vlen_packed, h->copy_buf);
            } else {
                /* Arena value -- bounds check before copy */
                if ((uint64_t)local_voff + local_vl > arena_cap) {
                    /* Retry only if the sequence moved, which means a torn
                     * record the next pass reads consistently.  An unchanged
                     * sequence is stable corruption, and looping would spin
                     * forever: deliver zeros, as shm_str_copy does. */
                    if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
                    if (local_vl > h->copy_buf_size || !h->copy_buf) {
                        if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) return 0;
                        continue;
                    }
                    memset(h->copy_buf, 0, local_vl);
                } else {
                    if (local_vl > h->copy_buf_size || !h->copy_buf) {
                        if (!shm_ensure_copy_buf(h, local_vl > 0 ? local_vl : 1)) return 0;
                        continue;
                    }
                    memcpy(h->copy_buf, h->arena + local_voff, local_vl);
                }
            }
#endif
            if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;

            /* A frozen view maps PROT_READ: skip the clock-bit store, the only
             * write on this read path, so a lookup cannot fault. */
            if (h->lru_accessed && !h->readonly)
                __atomic_store_n(&h->lru_accessed[local_idx], 1, __ATOMIC_RELAXED);
#ifdef SHM_VAL_IS_STR
            *out_str = h->copy_buf;
            *out_len = local_vl;
            *out_utf8 = SHM_UNPACK_UTF8(local_vlen_packed);
#else
            *out_value = local_value;
#endif
            return 1;
        }

        if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
        return 0;
    }
}

/* ---- Exists (with TTL check under rdlock) ---- */

static int SHM_FN(exists_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = shm_now();

    shm_rwlock_rdlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;  /* tombstone or tag mismatch */
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            int found = !SHM_IS_EXPIRED(h, idx, now);
            shm_rwlock_rdunlock(h);
            return found;
        }
    }

    shm_rwlock_rdunlock(h);
    return 0;
}

/* ---- Exists (seqlock -- lock-free read path) ---- */

static int SHM_FN(exists)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    /* TTL active: use rdlock path for expiry check */
    if (h->expires_at) {
#ifdef SHM_KEY_IS_INT
        return SHM_FN(exists_ttl)(h, key);
#else
        return SHM_FN(exists_ttl)(h, key_str, key_len, key_utf8);
#endif
    }

    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t max_mask = h->max_mask;
#ifndef SHM_KEY_IS_INT
    uint64_t arena_cap = hdr->arena_cap;  /* immutable after create */
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#else
    uint32_t hash = SHM_HASH_KEY(key);
#endif

    for (;;) {
        uint32_t seq = shm_seqlock_read_begin(h);

        uint32_t mask = (hdr->table_cap - 1) & max_mask;
        uint32_t pos = hash & mask;
        int found = 0;

        uint8_t tag = SHM_MAKE_TAG(hash);
        for (uint32_t i = 0; i <= mask; i++) {
            uint32_t idx = (pos + i) & mask;
            uint8_t st = states[idx];
            __builtin_prefetch(&nodes[idx], 0, 1);
            __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
            if (st == SHM_EMPTY) break;
            if (st != tag) continue;  /* tombstone or tag mismatch */
#ifdef SHM_KEY_IS_INT
            if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
            /* Bounded compare, matching the lock-free get() path: a poisoned
             * key_off must never reach memcmp (CWE-125). */
            {
                uint32_t kl_packed = nodes[idx].key_len;
                uint32_t koff = nodes[idx].key_off;
                uint32_t kl = SHM_STR_LEN(kl_packed);
                if (kl != key_len) continue;
                if (SHM_IS_INLINE(kl_packed)) {
                    char ibuf[SHM_INLINE_MAX];
                    shm_inline_read(koff, kl_packed, ibuf);
                    if (memcmp(ibuf, key_str, kl) != 0) continue;
                } else {
                    if ((uint64_t)koff + kl > arena_cap) break;
                    if (memcmp(h->arena + koff, key_str, kl) != 0) continue;
                }
            }
            {
#endif
                found = 1;
                break;
            }
        }

        if (shm_seqlock_read_retry(&hdr->seq, seq)) continue;
        return found;
    }
}

/* ---- Remove ---- */

/* Lockless remove body -- caller must hold writer lock + seqlock and have
 * already shard-dispatched. Used by remove() and remove_multi. */
static int SHM_FN(remove_inner)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t now = h->expires_at ? shm_now() : 0;
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) return 0;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            /* Expired is absent, as every other operation reports it. */
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                return 0;
            }
            SHM_FN(remove_at)(h, idx);
            return 1;
        }
    }
    return 0;
}

static int SHM_FN(remove)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);
    int rc = SHM_FN(remove_inner)(h,
#ifdef SHM_KEY_IS_INT
        key
#else
        key_str, key_len, key_utf8
#endif
    );
    if (rc) SHM_FN(maybe_shrink)(h);
    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return rc;
}

/* ---- Add (insert only if key absent, returns 1=inserted, 0=already exists or full) ---- */

static int SHM_FN(add_impl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);
    uint32_t insert_pos = UINT32_MAX;

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            /* Live key blocks add; expired entries are reclaimed */
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                if (insert_pos == UINT32_MAX) insert_pos = idx;
                break;
            }
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
    }

    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }

    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#endif
#ifdef SHM_VAL_IS_STR
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
#ifndef SHM_KEY_IS_INT
        shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
#endif
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#else
    __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
#endif
    states[insert_pos] = SHM_MAKE_TAG(hash);
    /* tombstones-- before size++: the other order briefly makes
     * size + tombstones == table_cap + 1, and a concurrent open in that window
     * refuses a healthy saturated map as corrupt. */
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at) {
        uint32_t ttl = (ttl_sec == SHM_TTL_USE_DEFAULT) ? hdr->default_ttl : ttl_sec;
        h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0;
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 1;
}

static inline int SHM_FN(add)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8
#else
    SHM_VAL_INT_TYPE value
#endif
) {
    return SHM_FN(add_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        SHM_TTL_USE_DEFAULT);
}

static inline int SHM_FN(add_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
    if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
    return SHM_FN(add_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        ttl_sec);
}

/* ---- Update (overwrite only if key exists, returns 1=updated, 0=not found) ---- */

static int SHM_FN(update_impl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
#ifdef SHM_VAL_IS_STR
            {
                uint32_t old_off = nodes[idx].val_off;
                uint32_t old_lf = nodes[idx].val_len;
                if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) {
                    shm_seqlock_write_end(&hdr->seq);
                    shm_rwlock_wrunlock(h);
                    return 0;
                }
                shm_str_free(hdr, h->arena, old_off, old_lf);
            }
#else
            __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
#endif
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at) {
                if (ttl_sec == SHM_TTL_USE_DEFAULT) {
                    if (hdr->default_ttl > 0 && h->expires_at[idx] != 0)
                        h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
                } else {
                    h->expires_at[idx] = ttl_sec > 0 ? shm_expiry_ts(ttl_sec) : 0;
                }
            }

            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 0;
}

static inline int SHM_FN(update)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8
#else
    SHM_VAL_INT_TYPE value
#endif
) {
    return SHM_FN(update_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        SHM_TTL_USE_DEFAULT);
}

static inline int SHM_FN(update_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
#else
    SHM_VAL_INT_TYPE value,
#endif
    uint32_t ttl_sec
) {
    if (ttl_sec >= SHM_TTL_USE_DEFAULT - 1) ttl_sec = SHM_TTL_USE_DEFAULT - 2;
    return SHM_FN(update_impl)(h,
#ifdef SHM_KEY_IS_INT
        key,
#else
        key_str, key_len, key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
        val_str, val_len, val_utf8,
#else
        value,
#endif
        ttl_sec);
}

/* ---- Swap (put + return old value; returns 1=swapped existing, 2=inserted new, 0=full) ---- */

static int SHM_FN(swap)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *val_str, uint32_t val_len, bool val_utf8,
    const char **out_str, uint32_t *out_len, bool *out_utf8
#else
    SHM_VAL_INT_TYPE value,
    SHM_VAL_INT_TYPE *out_value
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);
    uint32_t insert_pos = UINT32_MAX;

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                if (insert_pos == UINT32_MAX) insert_pos = idx;
                break;
            }
#ifdef SHM_VAL_IS_STR
            {
                uint32_t old_vl = SHM_STR_LEN(nodes[idx].val_len);
                if (!shm_ensure_copy_buf(h, old_vl)) {
                    shm_seqlock_write_end(&hdr->seq);
                    shm_rwlock_wrunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, old_vl);
                *out_str = h->copy_buf;
                *out_len = old_vl;
                *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);

                {
                    uint32_t old_off = nodes[idx].val_off;
                    uint32_t old_lf = nodes[idx].val_len;
                    if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len, val_str, val_len, val_utf8, idx)) {
                        shm_seqlock_write_end(&hdr->seq);
                        shm_rwlock_wrunlock(h);
                        return 0;
                    }
                    shm_str_free(hdr, h->arena, old_off, old_lf);
                }
            }
#else
            *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
            __atomic_store_n(&nodes[idx].value, value, __ATOMIC_RELAXED);
#endif
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
                h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);

            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1; /* swapped existing */
        }
    }

    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }

    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);
#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#endif
#ifdef SHM_VAL_IS_STR
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, val_str, val_len, val_utf8, SHM_LRU_NONE)) {
#ifndef SHM_KEY_IS_INT
        shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
#endif
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#else
    __atomic_store_n(&nodes[insert_pos].value, value, __ATOMIC_RELAXED);
#endif
    states[insert_pos] = SHM_MAKE_TAG(hash);
    /* tombstones-- before size++: the other order briefly makes
     * size + tombstones == table_cap + 1, and a concurrent open in that window
     * refuses a healthy saturated map as corrupt. */
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at) {
        uint32_t ttl = hdr->default_ttl;
        h->expires_at[insert_pos] = ttl > 0 ? shm_expiry_ts(ttl) : 0;
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 2; /* inserted new (no old value) */
}

/* ---- Take (remove and return value) ---- */

static int SHM_FN(take)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_str, uint32_t *out_len, bool *out_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;  /* tombstone or tag mismatch */

#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            /* check TTL */
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }

            /* copy value out before tombstoning */
#ifdef SHM_VAL_IS_STR
            {
                uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
                if (!shm_ensure_copy_buf(h, vl)) {
                    shm_seqlock_write_end(&hdr->seq);
                    shm_rwlock_wrunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
                *out_str = h->copy_buf;
                *out_len = vl;
                *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
            }
#else
            *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
            SHM_FN(remove_at)(h, idx);

            SHM_FN(maybe_shrink)(h);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 0;
}

/* ---- Compare-and-take (atomic remove if the value matches) ----
 * 1 if matched and removed, the old value copied out; 0 if the key is missing
 * or expired or the value differs.  String values compare by bytes only. */
static int SHM_FN(cas_take)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *expected_str, uint32_t expected_len,
    const char **out_str, uint32_t *out_len, bool *out_utf8
#else
    SHM_VAL_INT_TYPE expected, SHM_VAL_INT_TYPE *out_value
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
#ifdef SHM_VAL_IS_STR
            char ibuf[SHM_INLINE_MAX];
            uint32_t cur_len;
            const char *cur_str = shm_str_ptr(nodes[idx].val_off, nodes[idx].val_len,
                                              h->arena, h->hdr->arena_cap, ibuf, &cur_len);
            if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) {
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            if (!shm_ensure_copy_buf(h, cur_len)) {
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            memcpy(h->copy_buf, cur_str, cur_len);
            *out_str = h->copy_buf;
            *out_len = cur_len;
            *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
#else
            if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) != expected) {
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
            SHM_FN(remove_at)(h, idx);
            SHM_FN(maybe_shrink)(h);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 0;
}

/* ---- Pop (remove and return one entry: LRU tail if LRU, else first found) ---- */

static int SHM_FN(pop)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE *out_key,
#else
    const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++) {
            uint32_t si = (h->shard_rr + i) % h->num_shards;
            int rc = SHM_FN(pop)(h->shard_handles[si],
#ifdef SHM_KEY_IS_INT
                out_key,
#else
                out_key_str, out_key_len, out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
                out_val_str, out_val_len, out_val_utf8
#else
                out_value
#endif
            );
            if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; }
        }
        return 0;
    }
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    /* Find victim: LRU tail if available, else linear scan */
    uint32_t idx = UINT32_MAX;
    if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) {
        uint32_t pos = hdr->lru_tail;
        while (pos != SHM_LRU_NONE) {
            if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
            uint32_t prev = h->lru_prev[pos];
            SHM_FN(expire_at)(h, pos);
            pos = prev;
        }
    } else {
        /* No LRU: sweep forward from where the last pop or drain stopped,
         * expiring stale entries along the way */
        uint32_t cap = hdr->table_cap;
        uint32_t start = hdr->pop_cursor < cap ? hdr->pop_cursor : 0;
        for (uint32_t n = 0; n < cap; n++) {
            uint32_t i = (start + n) % cap;
            if (!SHM_IS_LIVE(states[i])) continue;
            if (SHM_IS_EXPIRED(h, i, now)) {
                SHM_FN(expire_at)(h, i);
                continue;
            }
            idx = i;
            hdr->pop_cursor = (i + 1) % cap;
            break;
        }
    }

    if (idx == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }

#ifdef SHM_KEY_IS_INT
    *out_key = nodes[idx].key;
#else
    {
        uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
        if (!shm_ensure_copy_buf(h, kl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        shm_str_copy(h->copy_buf, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
        *out_key_str = h->copy_buf;
        *out_key_len = kl;
        *out_key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
    }
#endif
#ifdef SHM_VAL_IS_STR
    {
        uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
#ifndef SHM_KEY_IS_INT
        uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
        /* key is in copy_buf[0..kl), put value after it.
         * Reassign out_key_str in case realloc moved the buffer. */
        if (!shm_ensure_copy_buf(h, kl + vl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        *out_key_str = h->copy_buf;
        shm_str_copy(h->copy_buf + kl, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
        *out_val_str = h->copy_buf + kl;
#else
        if (!shm_ensure_copy_buf(h, vl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
        *out_val_str = h->copy_buf;
#endif
        *out_val_len = vl;
        *out_val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
    }
#else
    *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif

    SHM_FN(remove_at)(h, idx);
    SHM_FN(maybe_shrink)(h);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 1;
}

/* ---- Shift (remove and return one entry: LRU head if LRU, else last found) ---- */

static int SHM_FN(shift)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE *out_key,
#else
    const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++) {
            uint32_t si = (h->shard_rr + i) % h->num_shards;
            int rc = SHM_FN(shift)(h->shard_handles[si],
#ifdef SHM_KEY_IS_INT
                out_key,
#else
                out_key_str, out_key_len, out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
                out_val_str, out_val_len, out_val_utf8
#else
                out_value
#endif
            );
            if (rc) { h->shard_rr = (si + 1) % h->num_shards; return 1; }
        }
        return 0;
    }
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    /* Find victim: LRU head if available, else scan backward */
    uint32_t idx = UINT32_MAX;
    if (h->lru_prev && hdr->lru_head != SHM_LRU_NONE) {
        uint32_t pos = hdr->lru_head;
        while (pos != SHM_LRU_NONE) {
            if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
            uint32_t nxt = h->lru_next[pos];
            SHM_FN(expire_at)(h, pos);
            pos = nxt;
        }
    } else {
        /* No LRU: sweep backward from where the last shift stopped, expiring
         * stale entries along the way */
        uint32_t cap = hdr->table_cap;
        uint32_t start = (hdr->shift_cursor == 0 || hdr->shift_cursor > cap) ? cap : hdr->shift_cursor;
        for (uint32_t n = 0; n < cap; n++) {
            uint32_t i = (start + cap - 1 - n) % cap;
            if (!SHM_IS_LIVE(states[i])) continue;
            if (SHM_IS_EXPIRED(h, i, now)) {
                SHM_FN(expire_at)(h, i);
                continue;
            }
            idx = i;
            hdr->shift_cursor = i;
            break;
        }
    }

    if (idx == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }

#ifdef SHM_KEY_IS_INT
    *out_key = nodes[idx].key;
#else
    {
        uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
        if (!shm_ensure_copy_buf(h, kl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        shm_str_copy(h->copy_buf, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
        *out_key_str = h->copy_buf;
        *out_key_len = kl;
        *out_key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
    }
#endif
#ifdef SHM_VAL_IS_STR
    {
        uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
#ifndef SHM_KEY_IS_INT
        uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
        /* Reassign out_key_str in case realloc moved the buffer. */
        if (!shm_ensure_copy_buf(h, kl + vl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        *out_key_str = h->copy_buf;
        shm_str_copy(h->copy_buf + kl, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
        *out_val_str = h->copy_buf + kl;
#else
        if (!shm_ensure_copy_buf(h, vl + 1)) {
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
        shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
        *out_val_str = h->copy_buf;
#endif
        *out_val_len = vl;
        *out_val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
    }
#else
    *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif

    SHM_FN(remove_at)(h, idx);
    SHM_FN(maybe_shrink)(h);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 1;
}

/* ---- Drain (pop up to N entries, returns count) ---- */

typedef struct {
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key;
#else
    uint32_t key_off;  /* offset into drain_buf */
    uint32_t key_len;
    bool     key_utf8;
#endif
#ifdef SHM_VAL_IS_STR
    uint32_t val_off;  /* offset into drain_buf */
    uint32_t val_len;
    bool     val_utf8;
#else
    SHM_VAL_INT_TYPE value;
#endif
} SHM_PASTE(SHM_PREFIX, drain_entry);

static uint32_t SHM_FN(drain_inner)(ShmHandle *h, uint32_t limit,
    SHM_PASTE(SHM_PREFIX, drain_entry) *out, char **buf, uint32_t *buf_cap,
    uint32_t *buf_used_p) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;
    uint32_t count = 0;
    uint32_t buf_used = *buf_used_p;
    (void)buf; (void)buf_cap;  /* used only by the string key/value copy paths below */

    if (limit == 0) return 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    /* non-LRU: sweep forward from where the last pop or drain stopped, at
     * most once around the table */
    uint32_t cap = hdr->table_cap;
    uint32_t cur = hdr->pop_cursor < cap ? hdr->pop_cursor : 0;
    uint32_t scanned = 0;
    while (count < limit) {
        uint32_t idx = UINT32_MAX;

        if (h->lru_prev && hdr->lru_tail != SHM_LRU_NONE) {
            uint32_t pos = hdr->lru_tail;
            while (pos != SHM_LRU_NONE) {
                if (!SHM_IS_EXPIRED(h, pos, now)) { idx = pos; break; }
                uint32_t prev = h->lru_prev[pos];
                SHM_FN(expire_at)(h, pos);
                pos = prev;
            }
        } else {
            while (scanned < cap) {
                uint32_t i = cur;
                cur = (cur + 1) % cap;
                scanned++;
                if (!SHM_IS_LIVE(states[i])) continue;
                if (SHM_IS_EXPIRED(h, i, now)) {
                    SHM_FN(expire_at)(h, i);
                    continue;
                }
                idx = i;
                break;
            }
        }

        if (idx == UINT32_MAX) break;

        /* Copy key */
#ifdef SHM_KEY_IS_INT
        out[count].key = nodes[idx].key;
#else
        {
            uint32_t kl = SHM_STR_LEN(nodes[idx].key_len);
            if ((uint64_t)buf_used + kl > UINT32_MAX) break;  /* drain buffer would exceed 4GB */
            if (!shm_grow_buf(buf, buf_cap, buf_used + kl)) break;
            shm_str_copy(*buf + buf_used, nodes[idx].key_off, nodes[idx].key_len, h->arena, h->hdr->arena_cap, kl);
            out[count].key_off = buf_used;
            out[count].key_len = kl;
            out[count].key_utf8 = SHM_UNPACK_UTF8(nodes[idx].key_len);
            buf_used += kl;
        }
#endif
        /* Copy value */
#ifdef SHM_VAL_IS_STR
        {
            uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
            if ((uint64_t)buf_used + vl > UINT32_MAX) break;  /* drain buffer would exceed 4GB */
            if (!shm_grow_buf(buf, buf_cap, buf_used + vl)) break;
            shm_str_copy(*buf + buf_used, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
            out[count].val_off = buf_used;
            out[count].val_len = vl;
            out[count].val_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
            buf_used += vl;
        }
#else
        out[count].value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif

        SHM_FN(remove_at)(h, idx);
        count++;
    }

    if (!h->lru_prev) hdr->pop_cursor = cur;
    if (count > 0) SHM_FN(maybe_shrink)(h);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    *buf_used_p = buf_used;
    return count;
}

static uint32_t SHM_FN(drain)(ShmHandle *h, uint32_t limit,
    SHM_PASTE(SHM_PREFIX, drain_entry) *out, char **buf, uint32_t *buf_cap) {
    uint32_t buf_used = 0;
    if (h->shard_handles) {
        uint32_t total = 0;
        for (uint32_t i = 0; i < h->num_shards && total < limit; i++) {
            uint32_t si = (h->shard_rr + i) % h->num_shards;
            uint32_t got = SHM_FN(drain_inner)(h->shard_handles[si], limit - total,
                                                out + total, buf, buf_cap, &buf_used);
            total += got;
        }
        if (total > 0) h->shard_rr = (h->shard_rr + 1) % h->num_shards;
        return total;
    }
    return SHM_FN(drain_inner)(h, limit, out, buf, buf_cap, &buf_used);
}

/* ---- Counter operations, atomic max/min, and integer-value cas ---- */

#ifdef SHM_HAS_COUNTERS

static inline int SHM_FN(find_slot)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    uint32_t *out_idx) {
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);
    uint32_t probe_start = 0;

#ifdef __SSE2__
    if (pos + 16 <= hdr->table_cap) {
        uint16_t mmask, emask;
        shm_probe_group(states, pos, tag, &mmask, &emask);
        uint16_t cutoff = emask ? (uint16_t)((1U << __builtin_ctz(emask)) - 1) : 0xFFFF;
        uint16_t relevant = mmask & cutoff;
        while (relevant) {
            int bit = __builtin_ctz(relevant);
            uint32_t idx = pos + bit;
#ifdef SHM_KEY_IS_INT
            if (SHM_KEY_EQ(&nodes[idx], key)) { *out_idx = idx; return 1; }
#else
            if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) { *out_idx = idx; return 1; }
#endif
            relevant &= relevant - 1;
        }
        if (emask) return 0;
        probe_start = 16;
    }
#endif

    for (uint32_t i = probe_start; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) return 0;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            *out_idx = idx;
            return 1;
        }
    }
    return 0;
}

static SHM_VAL_INT_TYPE SHM_FN(incr_by)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    SHM_VAL_INT_TYPE delta, int *ok) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint32_t now = h->expires_at ? shm_now() : 0;

    /* Fast path, no LRU or TTL: only the `value` word changes and a concurrent
     * get() reads it with one atomic load, so per-location coherence hands it
     * old-or-new.  The seqlock guards multi-field snapshots against structural
     * writers; this path touches neither key nor state. */
    if (!h->lru_prev && !h->expires_at) {
        shm_rwlock_rdlock(h);
        uint32_t idx;
#ifdef SHM_KEY_IS_INT
        if (SHM_FN(find_slot)(h, key, &idx)) {
#else
        if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) {
#endif
            SHM_VAL_INT_TYPE result =
                __atomic_add_fetch(&nodes[idx].value, delta, __ATOMIC_ACQ_REL);
            shm_rwlock_rdunlock(h);
            *ok = 1;
            return result;
        }
        shm_rwlock_rdunlock(h);
    }

    /* slow path: find-or-insert under write lock */
    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint32_t insert_pos = UINT32_MAX;

    uint8_t tag = SHM_MAKE_TAG(hash);
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t slot = (pos + i) & mask;
        uint8_t st = h->states[slot];
        __builtin_prefetch(&nodes[slot], 0, 1);
        __builtin_prefetch(&nodes[(slot + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = slot;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = slot;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[slot], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            /* TTL check */
            if (SHM_IS_EXPIRED(h, slot, now)) {
                SHM_FN(expire_at)(h, slot);
                /* treat as not found -- will insert below */
                if (insert_pos == UINT32_MAX) insert_pos = slot;
                break;
            }

            __atomic_add_fetch(&nodes[slot].value, delta, __ATOMIC_RELAXED);
            SHM_VAL_INT_TYPE result = __atomic_load_n(&nodes[slot].value, __ATOMIC_RELAXED);
            if (h->lru_prev) shm_lru_promote(h, slot);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0)
                h->expires_at[slot] = shm_expiry_ts(hdr->default_ttl);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            *ok = 1;
            return result;
        }
    }

    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        *ok = 0;
        return 0;
    }

    /* LRU eviction only when actually inserting */
    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (h->states[insert_pos] == SHM_TOMBSTONE);
#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        *ok = 0;
        return 0;
    }
#endif
    __atomic_store_n(&nodes[insert_pos].value, delta, __ATOMIC_RELAXED);
    h->states[insert_pos] = SHM_MAKE_TAG(hash);
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at && hdr->default_ttl > 0)
        h->expires_at[insert_pos] = shm_expiry_ts(hdr->default_ttl);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    *ok = 1;
    return delta;
}

/* ---- Atomic max / min (monotonic raise/lower; insert-if-absent) ----
 * want_max stores max(current, desired), else min; returns the stored value
 * and inserts `desired` for an absent key.  One lock acquisition covers the
 * compare and the set, leaving no gap for a concurrent counter operation. */
static SHM_VAL_INT_TYPE SHM_FN(set_minmax)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    SHM_VAL_INT_TYPE desired, int want_max, int *ok) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint32_t now = h->expires_at ? shm_now() : 0;

    /* fast path: existing key under read lock + atomic compare-and-set loop
       (no LRU/TTL). The common "already past the bound" case returns read-only,
       writing nothing. No seqlock bump needed (see incr_by): only the single
       `value` word changes, read atomically by get(); key/state are untouched. */
    if (!h->lru_prev && !h->expires_at) {
        shm_rwlock_rdlock(h);
        uint32_t idx;
#ifdef SHM_KEY_IS_INT
        if (SHM_FN(find_slot)(h, key, &idx)) {
#else
        if (SHM_FN(find_slot)(h, key_str, key_len, key_utf8, &idx)) {
#endif
            SHM_VAL_INT_TYPE cur = __atomic_load_n(&nodes[idx].value, __ATOMIC_ACQUIRE);
            while (want_max ? (desired > cur) : (desired < cur)) {
                /* on failure __atomic_compare_exchange_n reloads cur, so a
                   concurrent raise/lower is observed and the loop re-checks */
                if (__atomic_compare_exchange_n(&nodes[idx].value, &cur, desired,
                        0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) {
                    cur = desired;
                    break;
                }
            }
            shm_rwlock_rdunlock(h);
            *ok = 1;
            return cur;
        }
        shm_rwlock_rdunlock(h);
    }

    /* slow path: find-or-insert under write lock (handles LRU/TTL like incr_by) */
    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint32_t insert_pos = UINT32_MAX;

    uint8_t tag = SHM_MAKE_TAG(hash);
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t slot = (pos + i) & mask;
        uint8_t st = h->states[slot];
        __builtin_prefetch(&nodes[slot], 0, 1);
        __builtin_prefetch(&nodes[(slot + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = slot;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = slot;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[slot], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[slot], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, slot, now)) {
                SHM_FN(expire_at)(h, slot);
                /* treat as not found -- will insert below */
                if (insert_pos == UINT32_MAX) insert_pos = slot;
                break;
            }

            SHM_VAL_INT_TYPE cur = __atomic_load_n(&nodes[slot].value, __ATOMIC_RELAXED);
            SHM_VAL_INT_TYPE result =
                want_max ? (desired > cur ? desired : cur)
                         : (desired < cur ? desired : cur);
            __atomic_store_n(&nodes[slot].value, result, __ATOMIC_RELAXED);
            if (h->lru_prev) shm_lru_promote(h, slot);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[slot] != 0)
                h->expires_at[slot] = shm_expiry_ts(hdr->default_ttl);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            *ok = 1;
            return result;
        }
    }

    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        *ok = 0;
        return 0;
    }

    /* LRU eviction only when actually inserting */
    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (h->states[insert_pos] == SHM_TOMBSTONE);
#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        *ok = 0;
        return 0;
    }
#endif
    __atomic_store_n(&nodes[insert_pos].value, desired, __ATOMIC_RELAXED);
    h->states[insert_pos] = SHM_MAKE_TAG(hash);
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at && hdr->default_ttl > 0)
        h->expires_at[insert_pos] = shm_expiry_ts(hdr->default_ttl);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    *ok = 1;
    return desired;
}

/* ---- Compare-and-swap (atomic, integer-value variants) ---- */

static int SHM_FN(cas)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    SHM_VAL_INT_TYPE expected, SHM_VAL_INT_TYPE desired
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            if (__atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED) == expected) {
                __atomic_store_n(&nodes[idx].value, desired, __ATOMIC_RELAXED);
                if (h->lru_prev) shm_lru_promote(h, idx);
                if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
                    h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 1;
            }
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 0;
        }
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 0;
}

#endif /* SHM_HAS_COUNTERS */

/* ---- Compare-and-swap (atomic, string-value variants) ---- */

#ifdef SHM_VAL_IS_STR
static int SHM_FN(cas)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    const char *expected_str, uint32_t expected_len,
    const char *desired_str, uint32_t desired_len, bool desired_utf8
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            char ibuf[SHM_INLINE_MAX];
            uint32_t cur_len;
            const char *cur_str = shm_str_ptr(nodes[idx].val_off, nodes[idx].val_len,
                                              h->arena, h->hdr->arena_cap, ibuf, &cur_len);
            if (cur_len != expected_len || memcmp(cur_str, expected_str, cur_len) != 0) {
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            uint32_t old_off = nodes[idx].val_off;
            uint32_t old_lf = nodes[idx].val_len;
            if (!SHM_FN(store_or_evict)(h, &nodes[idx].val_off, &nodes[idx].val_len,
                               desired_str, desired_len, desired_utf8, idx)) {
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            shm_str_free(hdr, h->arena, old_off, old_lf);
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
                h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 0;
}
#endif /* SHM_VAL_IS_STR */

/* ---- Size ---- */

static inline uint32_t SHM_FN(size)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(size)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    return __atomic_load_n(&h->hdr->size, __ATOMIC_ACQUIRE);
}

/* ---- Max entries ---- */

static inline uint32_t SHM_FN(max_entries)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(max_entries)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    /* cast first: max_table_cap can be 2^31, where *3 would overflow uint32 */
    return (uint32_t)((uint64_t)h->hdr->max_table_cap * 3 / 4);
}

/* ---- Accessors ---- */

static inline uint32_t SHM_FN(max_size)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(max_size)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    return h->hdr->max_size;
}

static inline uint32_t SHM_FN(ttl)(ShmHandle *h) {
    if (h->shard_handles) return SHM_FN(ttl)(h->shard_handles[0]);
    return h->hdr->default_ttl;
}

/* ---- Get value with TTL remaining (atomic snapshot) ----
 * 1 if the key is live, 0 if missing or expired.  *out_ttl_remaining is -1
 * without TTL, 0 for a permanent entry, else the seconds left; NULL if only
 * the value is wanted. */
static int SHM_FN(get_with_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_str, uint32_t *out_len, bool *out_utf8,
#else
    SHM_VAL_INT_TYPE *out_value,
#endif
    int64_t *out_ttl_remaining
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now_ts = h->expires_at ? shm_now() : 0;

    shm_rwlock_rdlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now_ts)) {
                shm_rwlock_rdunlock(h);
                return 0;
            }
#ifdef SHM_VAL_IS_STR
            {
                uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
                if (!shm_ensure_copy_buf(h, vl)) {
                    shm_rwlock_rdunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
                *out_str = h->copy_buf;
                *out_len = vl;
                *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
            }
#else
            *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
            /* count as an access for LRU second-chance, like get() does.
             * Read-only view maps PROT_READ -- skip the clock-bit store (no fault). */
            if (h->lru_accessed && !h->readonly)
                __atomic_store_n(&h->lru_accessed[idx], 1, __ATOMIC_RELAXED);
            if (out_ttl_remaining) {
                if (!h->expires_at) *out_ttl_remaining = -1;
                else if (h->expires_at[idx] == 0) *out_ttl_remaining = 0;
                else *out_ttl_remaining = (int64_t)(h->expires_at[idx] - now_ts);
            }
            shm_rwlock_rdunlock(h);
            return 1;
        }
    }

    shm_rwlock_rdunlock(h);
    return 0;
}

/* ---- TTL remaining for a key (-1 = not found/expired, 0 = permanent) ---- */

static int64_t SHM_FN(ttl_remaining)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    if (!h->expires_at) return -1;  /* TTL not enabled */

    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;

    shm_rwlock_rdlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;  /* tombstone or tag mismatch */
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            uint32_t exp = h->expires_at[idx];
            if (exp == 0) {
                shm_rwlock_rdunlock(h);
                return 0;  /* permanent entry */
            }
            uint32_t now = shm_now();
            if (now >= exp) {
                shm_rwlock_rdunlock(h);
                return -1;  /* expired */
            }
            int64_t remaining = (int64_t)(exp - now);
            shm_rwlock_rdunlock(h);
            return remaining;
        }
    }

    shm_rwlock_rdunlock(h);
    return -1;  /* not found */
}

/* ---- Persist (remove TTL from a key, making it permanent) ---- */
/* Seqlock required: get() reads expires_at[] under seqlock. */

static int SHM_FN(persist)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    if (!h->expires_at) return 0;

    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = shm_now();

    shm_rwlock_wrlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                shm_seqlock_write_begin(&hdr->seq);
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            shm_seqlock_write_begin(&hdr->seq);
            h->expires_at[idx] = 0;  /* permanent */
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_rwlock_wrunlock(h);
    return 0;
}

/* ---- Set TTL (change TTL without changing value) ---- */

static int SHM_FN(set_ttl)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
    uint32_t ttl_sec
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    if (!h->expires_at) return 0;

    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = shm_now();

    shm_rwlock_wrlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                shm_seqlock_write_begin(&hdr->seq);
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            shm_seqlock_write_begin(&hdr->seq);
            if (ttl_sec == 0)
                h->expires_at[idx] = 0;  /* permanent */
            else
                h->expires_at[idx] = shm_expiry_ts(ttl_sec);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_rwlock_wrunlock(h);
    return 0;
}

/* ---- Stats ---- */

static inline uint32_t SHM_FN(capacity)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(capacity)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    return h->hdr->table_cap;
}

static inline uint32_t SHM_FN(tombstones)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(tombstones)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    return h->hdr->tombstones;
}

static inline size_t SHM_FN(mmap_size)(ShmHandle *h) {
    if (h->shard_handles) {
        size_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(mmap_size)(h->shard_handles[i]);
        return total;
    }
    return h->mmap_size;
}

/* ---- Flush expired entries (partial / gradual) ---- */

/* Scan up to `limit` slots from the shared flush_cursor, returning the number
 * expired and setting *done_out when the cursor completes a cycle.  The write
 * lock is held for those slots only, not the whole table. */
static uint32_t SHM_FN(flush_expired_partial)(ShmHandle *h, uint32_t limit, int *done_out) {
    if (h->shard_handles) {
        uint32_t total = 0;
        int all_done = 1;
        for (uint32_t i = 0; i < h->num_shards; i++) {
            int done = 0;
            total += SHM_FN(flush_expired_partial)(h->shard_handles[i], limit, &done);
            if (!done) all_done = 0;
        }
        if (done_out) *done_out = all_done;
        return total;
    }
    if (!h->expires_at) {
        if (done_out) *done_out = 1;  /* trivially complete */
        return 0;
    }
    if (done_out) *done_out = 0;

    ShmHeader *hdr = h->hdr;
    uint8_t *states = h->states;
    uint32_t now = shm_now();
    uint32_t flushed = 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    uint32_t cap = hdr->table_cap;
    uint32_t start = hdr->flush_cursor;
    if (start >= cap) start = 0;       /* clamp after shrink */
    if (limit == 0) limit = 1;         /* scan at least one slot */
    if (limit > cap) limit = cap;      /* can't scan more than exists */

    for (uint32_t n = 0; n < limit; n++) {
        uint32_t i = (start + n) % cap;
        if (SHM_IS_LIVE(states[i]) && h->expires_at[i] != 0 && now >= h->expires_at[i]) {
            SHM_FN(expire_at)(h, i);
            flushed++;
        }
    }

    uint32_t next = (start + limit) % cap;
    /* Full cycle complete when this chunk crossed the end of the table */
    int done = (limit >= cap || (uint64_t)start + limit >= cap);
    hdr->flush_cursor = done ? 0 : next;

    if (done_out) *done_out = done;

    /* Only shrink/compact when a full cycle is done, otherwise the rehash
     * can move entries to slots the cursor already passed. */
    if (done && flushed > 0)
        SHM_FN(maybe_shrink)(h);

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return flushed;
}

/* Convenience: full scan in one call (original behavior). */
static uint32_t SHM_FN(flush_expired)(ShmHandle *h) {
    if (h->shard_handles) {
        uint32_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(flush_expired)(h->shard_handles[i]);
        return total;
    }
    if (!h->expires_at) return 0;
    int done;
    return SHM_FN(flush_expired_partial)(h, UINT32_MAX, &done);
}

/* ---- Touch (refresh TTL / promote LRU without changing value) ---- */

static int SHM_FN(touch)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    if (!h->lru_prev && !h->expires_at) return 0;  /* nothing to do */

    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);

    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;  /* tombstone or tag mismatch */
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                shm_seqlock_write_begin(&hdr->seq);
                SHM_FN(expire_at)(h, idx);
                SHM_FN(maybe_shrink)(h);
                shm_seqlock_write_end(&hdr->seq);
                shm_rwlock_wrunlock(h);
                return 0;
            }
            shm_seqlock_write_begin(&hdr->seq);
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0) {
                h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
            }
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    shm_rwlock_wrunlock(h);
    return 0;
}

/* ---- Reserve (pre-grow table to target capacity) ---- */

static int SHM_FN(reserve)(ShmHandle *h, uint32_t target) {
    if (h->shard_handles) {
        int ok = 1;
        for (uint32_t i = 0; i < h->num_shards; i++)
            ok &= SHM_FN(reserve)(h->shard_handles[i], target);
        return ok;
    }
    ShmHeader *hdr = h->hdr;
    /* The ceiling is what max_entries() reports, the 75% design load, so
     * reserve() accepts exactly the values that accessor names. */
    if (target > (uint32_t)((uint64_t)hdr->max_table_cap * 3 / 4)) return 0;
    /* compute min capacity for target entries at <75% load */
    uint32_t needed = shm_next_pow2(target + target / 3 + 1);
    /* The top of the range rounds one power of two above max_table_cap; the
     * target still fits, so grow to the maximum rather than refuse it. */
    if (needed == 0 || needed > hdr->max_table_cap) needed = hdr->max_table_cap;
    if (needed < SHM_INITIAL_CAP) needed = SHM_INITIAL_CAP;
    if (needed <= hdr->table_cap) return 1;  /* already big enough */

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);
    int ok = 1;
    if (needed > hdr->table_cap)
        ok = SHM_FN(resize)(h, needed);
    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return ok;
}

/* ---- Cache stats accessors ---- */

static inline uint64_t SHM_FN(stat_evictions)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(stat_evictions)(h->shard_handles[i]);
        return (uint64_t)total;
    }
    return __atomic_load_n(&h->hdr->stat_evictions, __ATOMIC_RELAXED);
}

static inline uint64_t SHM_FN(stat_expired)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(stat_expired)(h->shard_handles[i]);
        return (uint64_t)total;
    }
    return __atomic_load_n(&h->hdr->stat_expired, __ATOMIC_RELAXED);
}

static inline uint32_t SHM_FN(stat_recoveries)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(stat_recoveries)(h->shard_handles[i]);
        return (uint32_t)(total > UINT32_MAX ? UINT32_MAX : total);
    }
    return __atomic_load_n(&h->hdr->stat_recoveries, __ATOMIC_RELAXED);
}

static inline uint64_t SHM_FN(arena_used)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(arena_used)(h->shard_handles[i]);
        return (uint64_t)total;
    }
    return h->arena ? __atomic_load_n(&h->hdr->arena_bump, __ATOMIC_RELAXED) : 0;
}

static inline uint64_t SHM_FN(arena_cap)(ShmHandle *h) {
    if (h->shard_handles) {
        uint64_t total = 0;
        for (uint32_t i = 0; i < h->num_shards; i++)
            total += SHM_FN(arena_cap)(h->shard_handles[i]);
        return (uint64_t)total;
    }
    return h->arena ? h->hdr->arena_cap : 0;
}

/* ---- Clear ---- */

static void SHM_FN(clear)(ShmHandle *h) {
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++)
            SHM_FN(clear)(h->shard_handles[i]);
        /* Clearing invalidates this handle's iteration, or a later each()
         * resumes at the shard it had reached and skips the ones below.
         * shard_iter is per handle: another process's pass is still cut short. */
        h->shard_iter = 0;
        return;
    }
    ShmHeader *hdr = h->hdr;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    memset(h->states, SHM_EMPTY, hdr->table_cap);
    hdr->size = 0;
    hdr->tombstones = 0;

    /* shrink back to initial capacity */
    if (hdr->table_cap > SHM_INITIAL_CAP) {
        uint32_t old_cap = hdr->table_cap;
        hdr->table_cap = SHM_INITIAL_CAP;
        size_t node_shrink = (size_t)(old_cap - SHM_INITIAL_CAP) * sizeof(SHM_NODE_TYPE);
        madvise((char *)h->nodes + (size_t)SHM_INITIAL_CAP * sizeof(SHM_NODE_TYPE),
                node_shrink, MADV_DONTNEED);
        madvise(h->states + SHM_INITIAL_CAP, old_cap - SHM_INITIAL_CAP, MADV_DONTNEED);
    }

    /* reset arena */
    if (h->arena) {
        hdr->arena_bump = SHM_ARENA_MIN_ALLOC;
        memset(hdr->arena_free, 0, sizeof(hdr->arena_free));
        hdr->arena_large_free = 0;
        madvise(h->arena, (size_t)hdr->arena_cap, MADV_DONTNEED);
    }

    /* reset LRU (full max_table_cap range) */
    if (h->lru_prev) {
        memset(h->lru_prev, 0xFF, hdr->max_table_cap * sizeof(uint32_t));
        memset(h->lru_next, 0xFF, hdr->max_table_cap * sizeof(uint32_t));
        if (h->lru_accessed) memset(h->lru_accessed, 0, hdr->max_table_cap);
        hdr->lru_head = SHM_LRU_NONE;
        hdr->lru_tail = SHM_LRU_NONE;
    }

    /* reset TTL (full max_table_cap range) */
    if (h->expires_at) {
        memset(h->expires_at, 0, hdr->max_table_cap * sizeof(uint32_t));
        hdr->flush_cursor = 0;
    }
    hdr->pop_cursor = 0;
    hdr->shift_cursor = 0;

    hdr->table_gen++;

    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);

    h->iter_pos = 0;
    if (h->iter_active) {
        h->iter_active = 0;
        if (h->iterating > 0) h->iterating--;
    }
    h->deferred = 0;
}

/* ---- Iterator reset ---- */

static inline void SHM_FN(iter_reset)(ShmHandle *h) {
    if (h->shard_handles) {
        for (uint32_t i = 0; i < h->num_shards; i++) {
            SHM_FN(iter_reset)(h->shard_handles[i]);
            /* the caller flushes the handle it was given, and the dispatcher's
             * own deferred flag is never set -- the work is on the shards */
            SHM_FN(flush_deferred)(h->shard_handles[i]);
        }
        h->shard_iter = 0;
        return;
    }
    if (h->iter_active) {
        h->iter_active = 0;
        if (h->iterating > 0) h->iterating--;
    }
    h->iter_pos = 0;
}

/* ---- Get or set (atomic: lookup + insert under single wrlock) ---- */

static int SHM_FN(get_or_set)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key,
#else
    const char *key_str, uint32_t key_len, bool key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char *def_str, uint32_t def_len, bool def_utf8,
    const char **out_str, uint32_t *out_len, bool *out_utf8
#else
    SHM_VAL_INT_TYPE def_value,
    SHM_VAL_INT_TYPE *out_value
#endif
) {
#ifdef SHM_KEY_IS_INT
    SHM_SHARD_DISPATCH(h, key);
#else
    SHM_SHARD_DISPATCH(h, key_str, key_len);
#endif
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_wrlock(h);
    shm_seqlock_write_begin(&hdr->seq);

    SHM_FN(maybe_grow)(h);
    uint32_t mask = hdr->table_cap - 1;
#ifdef SHM_KEY_IS_INT
    uint32_t hash = SHM_HASH_KEY(key);
#else
    uint32_t hash = SHM_HASH_KEY_STR(key_str, key_len);
#endif
    uint32_t pos = hash & mask;
    uint32_t insert_pos = UINT32_MAX;

    uint8_t tag = SHM_MAKE_TAG(hash);
    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);

        if (st == SHM_EMPTY) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            break;
        }
        if (st == SHM_TOMBSTONE) {
            if (insert_pos == UINT32_MAX) insert_pos = idx;
            continue;
        }
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            /* TTL check */
            if (SHM_IS_EXPIRED(h, idx, now)) {
                SHM_FN(expire_at)(h, idx);
                if (insert_pos == UINT32_MAX) insert_pos = idx;
                break;
            }

#ifdef SHM_VAL_IS_STR
            {
                uint32_t vl = SHM_STR_LEN(nodes[idx].val_len);
                if (!shm_ensure_copy_buf(h, vl)) {
                    shm_seqlock_write_end(&hdr->seq);
                    shm_rwlock_wrunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[idx].val_off, nodes[idx].val_len, h->arena, h->hdr->arena_cap, vl);
                *out_str = h->copy_buf;
                *out_len = vl;
                *out_utf8 = SHM_UNPACK_UTF8(nodes[idx].val_len);
            }
#else
            *out_value = __atomic_load_n(&nodes[idx].value, __ATOMIC_RELAXED);
#endif
            if (h->lru_prev) shm_lru_promote(h, idx);
            if (h->expires_at && hdr->default_ttl > 0 && h->expires_at[idx] != 0)
                h->expires_at[idx] = shm_expiry_ts(hdr->default_ttl);
            shm_seqlock_write_end(&hdr->seq);
            shm_rwlock_wrunlock(h);
            return 1;
        }
    }

    /* not found -- insert default value */
    if (insert_pos == UINT32_MAX &&
        (insert_pos = SHM_FN(reclaim_expired_slot)(h, pos, mask)) == UINT32_MAX) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }

    /* LRU eviction only when actually inserting a new entry */
    if (hdr->max_size > 0 && hdr->size >= hdr->max_size)
        SHM_FN(lru_evict_one)(h);

    int was_tombstone = (states[insert_pos] == SHM_TOMBSTONE);

#ifdef SHM_KEY_IS_INT
    nodes[insert_pos].key = key;
#else
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].key_off, &nodes[insert_pos].key_len, key_str, key_len, key_utf8, SHM_LRU_NONE)) {
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#endif

#ifdef SHM_VAL_IS_STR
    if (!shm_ensure_copy_buf(h, def_len > 0 ? def_len : 1)) {
#ifndef SHM_KEY_IS_INT
        shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
#endif
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
    if (!SHM_FN(store_or_evict)(h, &nodes[insert_pos].val_off, &nodes[insert_pos].val_len, def_str, def_len, def_utf8, SHM_LRU_NONE)) {
#ifndef SHM_KEY_IS_INT
        shm_str_free(hdr, h->arena, nodes[insert_pos].key_off, nodes[insert_pos].key_len);
#endif
        shm_seqlock_write_end(&hdr->seq);
        shm_rwlock_wrunlock(h);
        return 0;
    }
#else
    __atomic_store_n(&nodes[insert_pos].value, def_value, __ATOMIC_RELAXED);
#endif

    states[insert_pos] = SHM_MAKE_TAG(hash);
    /* tombstones-- before size++: the other order briefly makes
     * size + tombstones == table_cap + 1, and a concurrent open in that window
     * refuses a healthy saturated map as corrupt. */
    if (was_tombstone) hdr->tombstones--;
    hdr->size++;

    if (h->lru_prev) shm_lru_push_front(h, insert_pos);
    if (h->expires_at && hdr->default_ttl > 0)
        h->expires_at[insert_pos] = shm_expiry_ts(hdr->default_ttl);

#ifdef SHM_VAL_IS_STR
    memcpy(h->copy_buf, def_str, def_len);
    *out_str = h->copy_buf;
    *out_len = def_len;
    *out_utf8 = def_utf8;
#else
    *out_value = def_value;
#endif
    shm_seqlock_write_end(&hdr->seq);
    shm_rwlock_wrunlock(h);
    return 2; /* 2 = inserted new */
}

/* ---- Each (iterator) ---- */

static int SHM_FN(each)(ShmHandle *h,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE *out_key,
#else
    const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
    /* Sharded: chain each() across shards */
    if (h->shard_handles) {
        while (h->shard_iter < h->num_shards) {
            int rc = SHM_FN(each)(h->shard_handles[h->shard_iter],
#ifdef SHM_KEY_IS_INT
                out_key,
#else
                out_key_str, out_key_len, out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
                out_val_str, out_val_len, out_val_utf8
#else
                out_value
#endif
            );
            if (rc) return 1;
            SHM_FN(flush_deferred)(h->shard_handles[h->shard_iter]);
            h->shard_iter++;
        }
        h->shard_iter = 0;
        return 0;
    }
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_rdlock(h);

    if (!h->iter_active) {
        h->iter_active = 1;
        h->iter_gen = hdr->table_gen;
        h->iterating++;
    }

    /* Auto-reset on cross-process resize */
    if (h->iter_gen != hdr->table_gen) {
        h->iter_pos = 0;
        h->iter_gen = hdr->table_gen;
    }

    while (shm_find_next_live(states, hdr->table_cap, &h->iter_pos)) {
        uint32_t pos = h->iter_pos++;
        {
            /* skip expired entries */
            if (SHM_IS_EXPIRED(h, pos, now))
                continue;

#ifdef SHM_KEY_IS_INT
            *out_key = nodes[pos].key;
#else
            {
                uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
#ifdef SHM_VAL_IS_STR
                uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
                uint64_t total64 = (uint64_t)kl + vl;
                if (total64 > UINT32_MAX) {
                    h->iter_pos = 0;
                    h->iter_active = 0;
                    if (h->iterating > 0) h->iterating--;
                    shm_rwlock_rdunlock(h);
                    return 0;
                }
                uint32_t total = (uint32_t)total64;
#else
                uint32_t total = kl;
#endif
                if (!shm_ensure_copy_buf(h, total)) {
                    h->iter_pos = 0;
                    h->iter_active = 0;
                    if (h->iterating > 0) h->iterating--;
                    shm_rwlock_rdunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[pos].key_off, nodes[pos].key_len, h->arena, h->hdr->arena_cap, kl);
                *out_key_str = h->copy_buf;
                *out_key_len = kl;
                *out_key_utf8 = SHM_UNPACK_UTF8(nodes[pos].key_len);
            }
#endif
#ifdef SHM_VAL_IS_STR
            {
                uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
#ifndef SHM_KEY_IS_INT
                uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
                shm_str_copy(h->copy_buf + kl, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
                *out_val_str = h->copy_buf + kl;
#else
                if (!shm_ensure_copy_buf(h, vl)) {
                    h->iter_pos = 0;
                    h->iter_active = 0;
                    if (h->iterating > 0) h->iterating--;
                    shm_rwlock_rdunlock(h);
                    return 0;
                }
                shm_str_copy(h->copy_buf, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
                *out_val_str = h->copy_buf;
#endif
                *out_val_len = vl;
                *out_val_utf8 = SHM_UNPACK_UTF8(nodes[pos].val_len);
            }
#else
            *out_value = __atomic_load_n(&nodes[pos].value, __ATOMIC_RELAXED);
#endif
            shm_rwlock_rdunlock(h);
            return 1;
        }
    }

    h->iter_pos = 0;
    h->iter_active = 0;
    if (h->iterating > 0) h->iterating--;
    shm_rwlock_rdunlock(h);
    return 0;  /* caller should call flush_deferred */
}

/* ---- Cursor iteration ---- */

static int SHM_FN(cursor_next)(ShmCursor *c,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE *out_key,
#else
    const char **out_key_str, uint32_t *out_key_len, bool *out_key_utf8,
#endif
#ifdef SHM_VAL_IS_STR
    const char **out_val_str, uint32_t *out_val_len, bool *out_val_utf8
#else
    SHM_VAL_INT_TYPE *out_value
#endif
) {
    /* Chain across shards: when current shard exhausted, move to next */
    while (c->shard_idx < c->shard_count) {
        ShmHandle *h = c->current;
        ShmHeader *hdr = h->hdr;
        SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
        uint8_t *states = h->states;
        uint32_t now = h->expires_at ? shm_now() : 0;

        shm_rwlock_rdlock(h);

        /* Auto-reset on cross-process resize */
        if (c->gen != hdr->table_gen) {
            c->iter_pos = 0;
            c->gen = hdr->table_gen;
        }

        while (shm_find_next_live(states, hdr->table_cap, &c->iter_pos)) {
            uint32_t pos = c->iter_pos++;
            {
                if (SHM_IS_EXPIRED(h, pos, now))
                    continue;

#ifdef SHM_KEY_IS_INT
                *out_key = nodes[pos].key;
#else
                {
                    uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
#ifdef SHM_VAL_IS_STR
                    uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
                    uint64_t total64 = (uint64_t)kl + vl;
                    if (total64 > UINT32_MAX) {
                        shm_rwlock_rdunlock(h);
                        return 0;
                    }
                    uint32_t total = (uint32_t)total64;
#else
                    uint32_t total = kl;
#endif
                    if (!shm_cursor_ensure_copy_buf(c, total)) {
                        shm_rwlock_rdunlock(h);
                        return 0;
                    }
                    shm_str_copy(c->copy_buf, nodes[pos].key_off, nodes[pos].key_len, h->arena, h->hdr->arena_cap, kl);
                    *out_key_str = c->copy_buf;
                    *out_key_len = kl;
                    *out_key_utf8 = SHM_UNPACK_UTF8(nodes[pos].key_len);
                }
#endif
#ifdef SHM_VAL_IS_STR
                {
                    uint32_t vl = SHM_STR_LEN(nodes[pos].val_len);
#ifndef SHM_KEY_IS_INT
                    uint32_t kl = SHM_STR_LEN(nodes[pos].key_len);
                    shm_str_copy(c->copy_buf + kl, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
                    *out_val_str = c->copy_buf + kl;
#else
                    if (!shm_cursor_ensure_copy_buf(c, vl)) {
                        shm_rwlock_rdunlock(h);
                        return 0;
                    }
                    shm_str_copy(c->copy_buf, nodes[pos].val_off, nodes[pos].val_len, h->arena, h->hdr->arena_cap, vl);
                    *out_val_str = c->copy_buf;
#endif
                    *out_val_len = vl;
                    *out_val_utf8 = SHM_UNPACK_UTF8(nodes[pos].val_len);
                }
#else
                *out_value = __atomic_load_n(&nodes[pos].value, __ATOMIC_RELAXED);
#endif
                shm_rwlock_rdunlock(h);
                return 1;
            }
        }

        shm_rwlock_rdunlock(h);

        /* Current shard exhausted -- flush deferred work and advance */
        if (h->iterating > 0) h->iterating--;
        SHM_FN(flush_deferred)(h);
        c->shard_idx++;
        if (c->shard_idx < c->shard_count) {
            ShmHandle *parent = c->handle;
            c->current = parent->shard_handles[c->shard_idx];
            c->current->iterating++;
            c->iter_pos = 0;
            c->gen = c->current->hdr->table_gen;
        } else {
            /* Clear c->current, or cursor_destroy, reset and seek
             * double-decrement the last shard's `iterating` counter. */
            c->current = NULL;
        }
    }

    return 0;
}

static inline void SHM_FN(cursor_reset)(ShmCursor *c) {
    /* Decrement current shard's iterating counter and flush deferred */
    if (c->current && c->current->iterating > 0)
        c->current->iterating--;
    SHM_FN(flush_deferred)(c->current);
    /* Reset to first shard */
    c->shard_idx = 0;
    if (c->handle->shard_handles) {
        c->current = c->handle->shard_handles[0];
    } else {
        c->current = c->handle;
    }
    c->current->iterating++;
    c->iter_pos = 0;
    c->gen = c->current->hdr->table_gen;
}

/* Position cursor at the slot of a specific key. Returns 1 if found, 0 if not.
   Next cursor_next call will return this key's entry, then continue forward. */
static int SHM_FN(cursor_seek)(ShmCursor *c,
#ifdef SHM_KEY_IS_INT
    SHM_KEY_INT_TYPE key
#else
    const char *key_str, uint32_t key_len, bool key_utf8
#endif
) {
    /* For sharded maps, route to the correct shard based on key hash. */
#ifdef SHM_KEY_IS_INT
    uint64_t hash64 = SHM_HASH_KEY64(key);
#else
    uint64_t hash64 = SHM_HASH_KEY_STR64(key_str, key_len);
#endif
    uint32_t hash = (uint32_t)hash64;
    ShmHandle *parent = c->handle;
    ShmHandle *target;
    uint32_t target_shard = 0;
    if (parent->shard_handles) {
        target_shard = shm_shard_index(parent, hash64);
        target = parent->shard_handles[target_shard];
    } else {
        target = parent;
    }

    /* Probe before moving the cursor: a seek that finds nothing must leave the
     * iteration where it was. */
    ShmHandle *h = target;
    ShmHeader *hdr = h->hdr;
    SHM_NODE_TYPE *nodes = (SHM_NODE_TYPE *)h->nodes;
    uint8_t *states = h->states;
    uint32_t now = h->expires_at ? shm_now() : 0;

    shm_rwlock_rdlock(h);

    uint32_t mask = hdr->table_cap - 1;
    uint32_t pos = hash & mask;
    uint8_t tag = SHM_MAKE_TAG(hash);

    for (uint32_t i = 0; i <= mask; i++) {
        uint32_t idx = (pos + i) & mask;
        uint8_t st = states[idx];
        __builtin_prefetch(&nodes[idx], 0, 1);
        __builtin_prefetch(&nodes[(idx + 1) & mask], 0, 1);
        if (st == SHM_EMPTY) break;
        if (st != tag) continue;
#ifdef SHM_KEY_IS_INT
        if (SHM_KEY_EQ(&nodes[idx], key)) {
#else
        if (SHM_KEY_EQ_STR(&nodes[idx], h->arena, h->hdr->arena_cap, key_str, key_len, key_utf8)) {
#endif
            if (SHM_IS_EXPIRED(h, idx, now)) {
                shm_rwlock_rdunlock(h);
                return 0;
            }
            uint32_t found_gen = hdr->table_gen;
            shm_rwlock_rdunlock(h);
            /* Committed: the key is there, so move the cursor.  gen is the one
             * read under the lock, so a resize in this window leaves a stale
             * gen and cursor_next resets rather than trusting iter_pos. */
            if (target != c->current) {
                if (c->current && c->current->iterating > 0)
                    c->current->iterating--;
                SHM_FN(flush_deferred)(c->current);
                c->current = target;
                c->current->iterating++;
                c->shard_idx = target_shard;
            }
            c->iter_pos = idx;
            c->gen = found_gen;
            return 1;
        }
    }

    shm_rwlock_rdunlock(h);
    return 0;
}

/* ---- Undefine template macros ---- */

#undef SHM_PASTE2
#undef SHM_PASTE
#undef SHM_FN
#undef SHM_NODE_TYPE
#undef SHM_PREFIX
#undef SHM_VARIANT_ID
#undef SHM_HAS_ARENA

#ifdef SHM_KEY_IS_INT
  #undef SHM_KEY_IS_INT
  #undef SHM_KEY_INT_TYPE
  #undef SHM_HASH_KEY
  #undef SHM_HASH_KEY64
  #undef SHM_KEY_EQ
#else
  #undef SHM_HASH_KEY_STR
  #undef SHM_HASH_KEY_STR64
  #undef SHM_KEY_EQ_STR
#endif

#ifdef SHM_VAL_IS_STR
  #undef SHM_VAL_IS_STR
#else
  #undef SHM_VAL_INT_TYPE
#endif

#ifdef SHM_HAS_COUNTERS
  #undef SHM_HAS_COUNTERS
#endif
