mizu_ext.h

Go to the documentation of this file.

 
#ifndef MIZU_EXT_H
#define MIZU_EXT_H
 
#include <math.h>
#include <string.h>
 
#include "mizu.h"
 
#ifdef __cplusplus
extern "C" {
#endif
 
#if defined(__GNUC__) || defined(__clang__)
#  define MIZU_EXT_INLINE static inline __attribute__((unused))
#else
#  define MIZU_EXT_INLINE static inline
#endif
 
#ifdef __cplusplus
#  define MIZU_EXT_LOAD_ACQ(p) \
     std::atomic_load_explicit((p), std::memory_order_acquire)
#else
#  define MIZU_EXT_LOAD_ACQ(p) \
     atomic_load_explicit((p), memory_order_acquire)
#endif
 
// Binding-tier constants ---------------------------------------------------------
 
#ifdef _WIN32
#  define MIZU_PREFIX_LITERAL "Local\\mizu_"
#else
#  define MIZU_PREFIX_LITERAL "/mizu_"
#endif
 
#define MIZU_ALIGN64(x) (((x) + 63) & ~(size_t) 63)
 
#define MIZU_ZC_FLOOR     ((size_t) 32768)
#define MIZU_ZC_FLOOR_RAW ((size_t) (256 << 10))
 
#define MIZU_OPEN_CACHE_MAX  16
 
#define MIZU_CODEC_MAGIC 0x52u   
#define MIZU_PYMIZU_CODEC_MAGIC 0x50u   
#define MIZU_INTEROP_MAGIC 0x49u   
// Language and capability registries ----------------------------------------------
 
#define MIZU_LANG_NONE   0u
#define MIZU_LANG_BYTES  1u   
#define MIZU_LANG_R      2u
#define MIZU_LANG_PYTHON 3u
 
#define MIZU_CAP_MIZS  (1u << 0)   
#define MIZU_CAP_ATTRS (1u << 1)   
#define MIZU_CAP_MIZL  (1u << 2)   
#define MIZU_CAP_TASKREF (1u << 3) 
#define MIZU_CAP_MIZL_REF (1u << 4) 
#define MIZU_CAP_MIZL_TSN (1u << 5) 
#define MIZU_CAP_MIZL_U64V (1u << 6) 
#define MIZU_IDENT(lang, caps) \
  ((uint64_t) (uint8_t) (lang) | ((uint64_t) (uint32_t) (caps) << 32))
 
// Handle views -------------------------------------------------------------------
 
typedef struct mizu_handle_s mizu_handle;
typedef struct mizu_result_sink_s mizu_result_sink;
 
enum { MIZU_HTYPE_CHANNEL = 1, MIZU_HTYPE_POOL = 2 };
 
MIZU_API int mizu_handle_kind(const mizu_handle *);
MIZU_API int mizu_handle_churn(const mizu_handle *);
MIZU_API int64_t mizu_handle_keep_out(const mizu_handle *);
MIZU_API void mizu_handle_spill_info(const mizu_handle *,
                                   uint32_t *fl_entries,
                                   uint32_t *ledger_entries);
 
// The binding seam ---------------------------------------------------------------
 
typedef int (*mizu_stage_fn)(void *obj, mizu_slot_hdr *hdr,
                            uint8_t *payload, uint32_t inline_max,
                            mizu_handle *, void *ctx);
 
#define MIZU_READ_CONSUME 1u
 
typedef struct mizu_read_ctx_s {
  uint32_t size;        
  int32_t outcome;      
  int32_t gone;         
  int32_t died_slot;    
  int64_t died_pid;     
  mizu_handle *handle;   
  void   *binding_ctx;  
  uint32_t flags;       
  void   *reserved[4];  
} mizu_read_ctx;
 
typedef void *(*mizu_read_fn)(const mizu_slot_hdr *, const uint8_t *payload,
                             size_t limit, mizu_read_ctx *);
 
typedef int (*mizu_exec_fn)(const mizu_slot_hdr *hdr, const uint8_t *payload,
                           size_t limit, mizu_result_sink *, int catching,
                           mizu_read_ctx *ctx);
typedef int (*mizu_check_fn)(void *ctx);
typedef void (*mizu_park_fn)(void *ctx, int entering);
typedef void (*mizu_sweep_fn)(void *ctx);
typedef void (*mizu_drop_fn)(void *ctx, void *pin);
 
typedef struct mizu_binding_s {
  uint32_t     size;    
  mizu_stage_fn stage;
  mizu_read_fn  read;
  mizu_exec_fn  exec;    
  mizu_check_fn check;   
  mizu_park_fn  park;    
  mizu_sweep_fn sweep;   
  mizu_drop_fn  drop;    
  void        *ctx;     
  uint64_t     ident;   
} mizu_binding;
 
MIZU_API void mizu_binding_init(mizu_binding *);
 
MIZU_API uint64_t mizu_channel_peer_ident(const mizu_channel *);
MIZU_API uint64_t mizu_pool_worker_ident(const mizu_pool *);
 
MIZU_API void *mizu_stage_arena_alloc(mizu_handle *, size_t n, uint64_t *off);
MIZU_API mizu_status mizu_stage_spill_get(mizu_handle *, size_t n,
                                       mizu_shm **out);
MIZU_API void mizu_stage_retain(mizu_handle *, mizu_shm *region);
MIZU_API void mizu_stage_retain_zc(mizu_handle *, mizu_shm *region);
MIZU_API void mizu_stage_pin(mizu_handle *, void *pin);
MIZU_API void mizu_stage_reap(mizu_handle *);
MIZU_API void mizu_stage_rollback(mizu_handle *);
 
MIZU_API void *mizu_stage_raw_spill(mizu_handle *, uint64_t n, int wire_type,
                                  mizu_slot_hdr *, uint8_t *payload,
                                  uint32_t inline_max);
#ifdef MIZU_EXT_NO_INLINES
MIZU_API void *mizu_stage_raw(mizu_handle *, uint64_t n, int wire_type,
                            mizu_slot_hdr *, uint8_t *payload,
                            uint32_t inline_max);
#else
MIZU_EXT_INLINE void *mizu_stage_raw(mizu_handle *h, uint64_t n, int wire_type,
                                   mizu_slot_hdr *hdr, uint8_t *payload,
                                   uint32_t inline_max) {
  if (n <= (uint64_t) inline_max) {
    hdr->kind = MIZU_KIND_RAWVEC;
    hdr->len = (uint32_t) n;
    hdr->aux = (uint64_t) (uint32_t) wire_type;
    return payload;
  }
  return mizu_stage_raw_spill(h, n, wire_type, hdr, payload, inline_max);
}
#endif
 
MIZU_API mizu_shm *mizu_read_region(mizu_read_ctx *, const uint8_t *name,
                                 uint32_t len);
 
MIZU_API int mizu_result_publish(mizu_result_sink *, void *value);
MIZU_API int mizu_result_publish_err(mizu_result_sink *, void *flattened,
                                   uint32_t inline_n);
MIZU_API void mizu_result_publish_lost(mizu_result_sink *);
 
// The built-in bytes binding -----------------------------------------------------
 
typedef struct mizu_bytes_s {
  void *data;
  size_t len;
} mizu_bytes;
 
MIZU_API void mizu_binding_bytes(mizu_binding *);
MIZU_API void mizu_bytes_free(mizu_bytes *);
 
// The region handle struct ---------------------------------------------------------
 
struct mizu_shm_s {
  void *addr;
  size_t size;
  char name[MIZU_NAME_MAX];
  uint8_t name_len;
  unsigned int pid;            
#ifdef _WIN32
  void *handle;
#endif
};
 
// Parker (wait_*.c) ---------------------------------------------------------------
 
typedef struct mizu_parker_s {
  MIZU_ATOMIC(uint32_t) *epoch;   
#ifdef _WIN32
  void *event;                   
#endif
} mizu_parker;
 
typedef enum mizu_park_result_e { MIZU_PARK_WOKEN = 0, MIZU_PARK_TIMEOUT,
                                 MIZU_PARK_INTR } mizu_park_result;
 
MIZU_API int mizu_parker_attach(mizu_parker *pk, MIZU_ATOMIC(uint32_t) *epoch,
                              const char *region_name, int entity,
                              int create);
MIZU_API void mizu_parker_detach(mizu_parker *pk);
 
MIZU_API int mizu_park(mizu_parker *pk, uint32_t snapshot, long timeout_ms);
MIZU_API void mizu_unpark(mizu_parker *pk);
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API uint32_t mizu_parker_snapshot(const mizu_parker *pk);
#else
MIZU_EXT_INLINE uint32_t mizu_parker_snapshot(const mizu_parker *pk) {
  return MIZU_EXT_LOAD_ACQ(pk->epoch);
}
#endif
 
// Death watch (wait_linux.c / wait_macos.c / wait_win32.c) ------------------------
 
typedef struct mizu_death_watch_s mizu_death_watch;
 
MIZU_API mizu_death_watch *mizu_death_watch_start(long pid,
                                               MIZU_ATOMIC(int) *flag,
                                               const mizu_parker *pk);
MIZU_API void mizu_death_watch_stop(mizu_death_watch *w);
 
MIZU_API void mizu_death_listener_teardown(void);
 
// Liveness lock (liveness.c) ---------------------------------------------------
 
typedef enum mizu_live_probe_e { MIZU_LIVE_ACQUIRED = 0,
                                MIZU_LIVE_HELD = 1 } mizu_live_probe;
 
MIZU_API const char *mizu_live_dir(void);
 
MIZU_API int mizu_live_open(const char *path, intptr_t *out);
MIZU_API int mizu_live_try(intptr_t h);
MIZU_API void mizu_live_close(intptr_t h);
 
// Preamble (preamble.c) ----------------------------------------------------------
 
MIZU_API void mizu_preamble_write(void *region, const mizu_preamble *p);
MIZU_API const char *mizu_preamble_validate(const void *region,
                                          size_t region_size,
                                          mizu_preamble *out);
 
// Map support (pool.c; a binding's map rides these) ---------------------------------
 
typedef struct mizu_pool_sig_s {
  MIZU_ATOMIC(uint32_t) *help_wanted;
  MIZU_ATOMIC(uint32_t) *shutdown;
  MIZU_ATOMIC(int)      *owner_dead;
} mizu_pool_sig;
 
MIZU_API mizu_pool_sig *mizu_pool_signals(mizu_pool *);
MIZU_API int mizu_pool_help_once(mizu_pool *);
MIZU_API int mizu_pool_deque_pull(mizu_pool *, uint32_t n);
MIZU_API int mizu_pool_map_caps(mizu_pool *, uint32_t *free_rs,
                              uint32_t *inj_cap, uint32_t *inline_entry);
MIZU_API mizu_status mizu_pool_submit_flags(mizu_pool *, void *task_obj,
                                         uint16_t flags, mizu_task *out,
                                         double timeout_ms);
 
// Map morsel protocol (morsel.c) -------------------------------------------------
 
#define MIZU_MORSEL_CANCEL_OFF ((uint64_t) 0)
#define MIZU_MORSEL_GEN_OFF    ((uint64_t) 4)
#define MIZU_MORSEL_CURSOR_OFF ((uint64_t) 64)
#define MIZU_MORSEL_CLAIM_OFF  ((uint64_t) 128)
#define MIZU_MORSEL_GEN_MASK   ((uint32_t) 0x3FFFFFFF)
 
enum { MIZU_MORSEL_IDLE = 0, MIZU_MORSEL_RUNNING, MIZU_MORSEL_ABANDONED };
enum { MIZU_MORSEL_X_DESC = 0, MIZU_MORSEL_X_RAW };
 
#define MIZU_MORSEL_T_TARGET  200e-6
#define MIZU_MORSEL_BATCH_CAP 64
 
#define MIZU_MORSEL_MAGIC 0x4D495A4Du
 
typedef struct mizu_morsel_hdr_s {
  uint32_t magic;          
  uint32_t version;        
  uint32_t flags;          
  uint32_t x_kind;         
  uint32_t x_type;         
  uint32_t out_type;       
  uint32_t out_elt;        
  uint32_t claim_n;        
  uint64_t n;              
  uint64_t desc_off, desc_len;
  uint64_t x_off, x_len;
  uint64_t out_off;
  uint64_t out_m;          
  uint64_t morsel_size;    
  uint64_t n_morsels;      
  uint64_t state_off;      
  uint8_t  pad[16];
} mizu_morsel_hdr;
MIZU_STATIC_ASSERT(sizeof(mizu_morsel_hdr) == 128,
                  "mizu_morsel_hdr is the wire format");
 
MIZU_API uint64_t mizu_morsel_layout(mizu_morsel_hdr *h,
                                   uint64_t n, uint64_t morsel_size,
                                   uint64_t desc_len, uint32_t x_type,
                                   uint64_t x_len, uint32_t out_type,
                                   uint64_t out_m, uint32_t claim_n);
MIZU_API const char *mizu_morsel_hdr_check(const void *base, size_t size,
                                         mizu_morsel_hdr *out);
 
typedef struct mizu_morsel_sizer_s {
  int32_t  run_r;          
  uint32_t run_gen;
  uint64_t k;              
  uint64_t k_last;         
  double   t_last;         
  double   cost;           
  int      skip;           
} mizu_morsel_sizer;
MIZU_API void mizu_morsel_sizer_init(mizu_morsel_sizer *);
 
MIZU_API int mizu_morsel_next(void *base, const mizu_morsel_hdr *h,
                            mizu_morsel_sizer *sz, uint32_t r, uint32_t gen,
                            const mizu_pool_sig *sig, uint64_t pin_k,
                            double now, uint64_t *m_out, uint64_t *k_out,
                            int *help_out);
 
MIZU_API uint32_t mizu_morsel_reset(void *base, const mizu_morsel_hdr *h);
 
MIZU_API int mizu_morsel_abandon(void *base, const mizu_morsel_hdr *h,
                               uint32_t r, uint32_t gen);
 
MIZU_API void mizu_morsel_cancel_set(void *base, const mizu_morsel_hdr *h);
MIZU_API int mizu_morsel_cancel_get(const void *base, const mizu_morsel_hdr *h);
MIZU_API uint32_t mizu_morsel_generation(const void *base,
                                       const mizu_morsel_hdr *h);
MIZU_API uint64_t mizu_morsel_cursor(const void *base,
                                   const mizu_morsel_hdr *h);
MIZU_API uint32_t mizu_morsel_claim(const void *base, const mizu_morsel_hdr *h,
                                  uint32_t r);
 
MIZU_API void mizu_morsel_span_of(const mizu_morsel_hdr *h, uint64_t m,
                                uint64_t k, uint64_t *lo, uint64_t *hi);
 
typedef struct mizu_morsel_span_s { uint64_t lo, hi; } mizu_morsel_span;
MIZU_API size_t mizu_morsel_lost(mizu_morsel_span *spans, size_t n,
                               uint64_t bound, mizu_morsel_span *out);
 
// The result sink and the unwind path (pool.c) ----------------------------------
 
struct mizu_result_sink_s {
  mizu_pool     *p;
  mizu_rs_hdr   *rs;
  unsigned char *payload;   
  uint32_t      rs_index;
  uint32_t      inline_max; 
  uint16_t      sub_slot;   
  uint64_t      seq;        
  uint64_t      task_id;
};
 
MIZU_API void mizu_pool_eval_mark(mizu_pool *, int in_flight);
MIZU_API int mizu_pool_unwind_sink(mizu_pool *, mizu_result_sink *out);
 
// Zero-copy refcount words ---------------------------------------------------------
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API MIZU_ATOMIC(uint32_t) *mizu_zc_rc(void *base);
MIZU_API MIZU_ATOMIC(uint32_t) *mizu_zc_flags_(void *base);
#else
MIZU_EXT_INLINE MIZU_ATOMIC(uint32_t) *mizu_zc_rc(void *base) {
  return (MIZU_ATOMIC(uint32_t) *) ((unsigned char *) base +
                                   MIZU_ZC_REFCOUNT_OFF);
}
MIZU_EXT_INLINE MIZU_ATOMIC(uint32_t) *mizu_zc_flags_(void *base) {
  return (MIZU_ATOMIC(uint32_t) *) ((unsigned char *) base +
                                   MIZU_ZC_FLAGS_OFF);
}
#endif
 
// Wire-format helpers --------------------------------------------------------------
 
typedef struct mizu_mizs_geom_s {
  int64_t validity;
  int64_t offsets;
  int64_t encoding;
  int64_t data;
} mizu_mizs_geom;
 
typedef struct mizu_mizl_entry_s {
  int64_t data_offset;   
  int64_t data_size;
  int32_t sexptype;
  int32_t attrs_size;
  int64_t length;
  int64_t valid[2];      
} mizu_mizl_entry;
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API mizu_mizs_geom mizu_mizs_geometry(int64_t n);
#else
MIZU_EXT_INLINE mizu_mizs_geom mizu_mizs_geometry(int64_t n) {
  mizu_mizs_geom g;
  g.validity = 0;
  g.offsets  = (int64_t) MIZU_ALIGN64((uint64_t) (n + 7) / 8);
  g.encoding = g.offsets + (int64_t) MIZU_ALIGN64(8 * ((uint64_t) n + 1));
  g.data     = g.encoding + (int64_t) MIZU_ALIGN64((uint64_t) n);
  return g;
}
#endif
 
MIZU_EXT_INLINE int mizu_ext_flags_known(const void *base) {
  uint32_t flags;
  memcpy(&flags, (const unsigned char *) base + MIZU_HDR_FLAGS_OFF, 4);
  return (flags & ~MIZU_HDR_FLAG_S4) == 0;
}
 
MIZU_EXT_INLINE int mizu_ext_valid_ok(int64_t off, int64_t count,
                                     uint64_t n, size_t size) {
  if (off == 0) return count == 0 || count == -1;
  if (count < 0 || (uint64_t) count > n || (off & 63) != 0) return 0;
  const uint64_t bytes = (n + 7) / 8;
  return (uint64_t) off <= (uint64_t) size &&
         bytes <= (uint64_t) size - (uint64_t) off;
}
 
MIZU_EXT_INLINE int mizu_ext_mizl_tag_ok(int32_t sexptype) {
  switch (sexptype & ~(int32_t) MIZU_MIZL_S4) {
  case 0:
  case MIZU_TYPE_LGL:
  case MIZU_TYPE_INT:
  case MIZU_TYPE_REAL:
  case MIZU_TYPE_CPLX:
  case MIZU_TYPE_STR:
  case MIZU_TYPE_VEC:
  case MIZU_TYPE_RAW:
  case MIZU_TYPE_INT64:
  case 33:                        /* remote leaf */
    return 1;
  default:
    return 0;
  }
}
 
MIZU_EXT_INLINE int mizu_ext_na_real_bits(uint64_t bits) {
  return ((bits >> 52) & 0x7FF) == 0x7FF && (uint32_t) bits == 0x7A2u;
}
MIZU_EXT_INLINE int mizu_ext_na_at(int type, const void *src, uint64_t i) {
  const unsigned char *p = (const unsigned char *) src;
  uint64_t bits, re, im;
  int32_t v32;
  int64_t v64;
  switch (type) {
  case MIZU_TYPE_LGL:
  case MIZU_TYPE_INT:
    memcpy(&v32, p + 4 * i, 4);
    return v32 == MIZU_NA_INT32;
  case MIZU_TYPE_REAL:
    memcpy(&bits, p + 8 * i, 8);
    return mizu_ext_na_real_bits(bits);
  case MIZU_TYPE_CPLX:
    memcpy(&re, p + 16 * i, 8);
    memcpy(&im, p + 16 * i + 8, 8);
    return mizu_ext_na_real_bits(re) || mizu_ext_na_real_bits(im);
  case MIZU_TYPE_INT64:
    memcpy(&v64, p + 8 * i, 8);
    return v64 == MIZU_NA_INT64;
  default:
    return 0;
  }
}
 
MIZU_EXT_INLINE void mizu_ext_na_store(int type, void *dst, uint64_t i) {
  unsigned char *p = (unsigned char *) dst;
  const int32_t na32 = MIZU_NA_INT32;
  const int64_t na64 = MIZU_NA_INT64;
  const uint64_t nabits = MIZU_NA_REAL_BITS;
  switch (type) {
  case MIZU_TYPE_LGL:
  case MIZU_TYPE_INT:
    memcpy(p + 4 * i, &na32, 4);
    break;
  case MIZU_TYPE_REAL:
    memcpy(p + 8 * i, &nabits, 8);
    break;
  case MIZU_TYPE_CPLX:
    memcpy(p + 16 * i, &nabits, 8);
    memcpy(p + 16 * i + 8, &nabits, 8);
    break;
  case MIZU_TYPE_INT64:
    memcpy(p + 8 * i, &na64, 8);
    break;
  default:
    break;
  }
}
 
MIZU_EXT_INLINE int mizu_ext_mizl_ent(const void *base, size_t size,
                                     int64_t i, mizu_mizl_entry *e) {
  const unsigned char *dir = (const unsigned char *) base +
    MIZU_HEADER_SIZE + 32 * (size_t) i;
  memcpy(&e->data_offset, dir, 8);
  memcpy(&e->data_size, dir + 8, 8);
  memcpy(&e->sexptype, dir + 16, 4);
  memcpy(&e->attrs_size, dir + 20, 4);
  memcpy(&e->length, dir + 24, 8);
  if (e->data_offset < 0 || (e->data_offset & 63) != 0 ||
      e->data_size < 0 ||
      (uint64_t) e->data_offset > (uint64_t) size ||
      (uint64_t) e->data_size >
        (uint64_t) size - (uint64_t) e->data_offset ||
      e->attrs_size < 0)
    return -1;
  if (!mizu_ext_mizl_tag_ok(e->sexptype)) return -1;
  const int32_t tag = e->sexptype & ~(int32_t) MIZU_MIZL_S4;
  if (tag == 33) {              /* remote leaf: the span is the identifier */
    if ((e->sexptype & (int32_t) MIZU_MIZL_S4) != 0 || e->length < 0 ||
        e->data_size < 1 || e->data_size > 255)
      return -1;
    return 0;
  }
  if ((int64_t) e->attrs_size > e->data_size) return -1;
  const int64_t body = e->data_size - (int64_t) e->attrs_size;
  const size_t elt = mizu_type_elt_size(tag);
  if (elt != 0) {
    if (e->length < 0 || e->length > body / (int64_t) elt) return -1;
  } else if (tag == MIZU_TYPE_STR) {
    if (e->length < 0 || e->length > ((int64_t) 1 << 50) ||
        mizu_mizs_geometry(e->length).data > body)
      return -1;
  } else if (tag == MIZU_TYPE_VEC) {
    if (e->length < 0) return -1;
  }
  return 0;
}
 
MIZU_EXT_INLINE int mizu_ext_mizh_check_impl(const void *base, size_t size,
                                            int *wire_type, int64_t *n_elems,
                                            int64_t valid[2]) {
  if (size < MIZU_HEADER_SIZE) return -1;
  uint32_t magic;
  int32_t t32;
  int64_t len, attrs, voff, vcount;
  memcpy(&magic, base, 4);
  if (magic != MIZU_MAGIC_VEC) return -1;
  memcpy(&t32, (const unsigned char *) base + 4, 4);
  memcpy(&len, (const unsigned char *) base + 8, 8);
  memcpy(&attrs, (const unsigned char *) base + 16, 8);
  memcpy(&voff, (const unsigned char *) base + MIZU_HDR_VALID_OFF, 8);
  memcpy(&vcount, (const unsigned char *) base + MIZU_HDR_VALID_COUNT, 8);
  const size_t elt = mizu_type_elt_size(t32);
  if (elt == 0 || len < 0 || attrs < 0 ||
      len > ((int64_t) size - (int64_t) MIZU_HEADER_SIZE) / (int64_t) elt ||
      attrs > (int64_t) size - (int64_t) MIZU_HEADER_SIZE -
              len * (int64_t) elt)
    return -1;
  if (!mizu_ext_flags_known(base) ||
      !mizu_ext_valid_ok(voff, vcount, (uint64_t) len, size))
    return -1;
  *wire_type = t32;
  *n_elems = len;
  valid[0] = voff;
  valid[1] = vcount;
  return 0;
}
 
MIZU_EXT_INLINE int mizu_ext_mizs_check_impl(const void *base, size_t size,
                                            int64_t *n, int64_t *str_size,
                                            int64_t *attrs_size) {
  if (size < MIZU_HEADER_SIZE) return -1;
  uint32_t magic;
  int32_t attrs;
  int64_t count, block;
  memcpy(&magic, base, 4);
  if (magic != MIZU_MAGIC_STR) return -1;
  memcpy(&attrs, (const unsigned char *) base + 4, 4);
  memcpy(&count, (const unsigned char *) base + 8, 8);
  memcpy(&block, (const unsigned char *) base + 16, 8);
  if (attrs < 0 || count < 0 || count > ((int64_t) 1 << 50) || block < 0)
    return -1;
  if (!mizu_ext_flags_known(base)) return -1;
  const int64_t fixed = mizu_mizs_geometry(count).data;
  if (fixed > block ||
      block > (int64_t) size - (int64_t) MIZU_HEADER_SIZE ||
      (int64_t) attrs >
        (int64_t) size - (int64_t) MIZU_HEADER_SIZE - block)
    return -1;
  *n = count;
  *str_size = block;
  *attrs_size = attrs;
  return 0;
}
 
MIZU_EXT_INLINE int mizu_ext_mizl_check_impl(const void *base, size_t size,
                                            int64_t *n, int64_t *attrs_off,
                                            int64_t *attrs_size,
                                            int64_t valid[2]) {
  if (size < MIZU_HEADER_SIZE) return -1;
  uint32_t magic;
  int32_t n32;
  int64_t aoff, asz, voff, vcount;
  memcpy(&magic, base, 4);
  if (magic != MIZU_MAGIC_LIST) return -1;
  memcpy(&n32, (const unsigned char *) base + 4, 4);
  memcpy(&aoff, (const unsigned char *) base + 8, 8);
  memcpy(&asz, (const unsigned char *) base + 16, 8);
  memcpy(&voff, (const unsigned char *) base + MIZU_HDR_VALID_OFF, 8);
  memcpy(&vcount, (const unsigned char *) base + MIZU_HDR_VALID_COUNT, 8);
  if (n32 < 0 ||
      (uint64_t) (uint32_t) n32 >
        ((uint64_t) size - (uint64_t) MIZU_HEADER_SIZE) / 32)
    return -1;
  if (aoff < 0 || asz < 0 ||
      (uint64_t) aoff > (uint64_t) size ||
      (uint64_t) asz > (uint64_t) size - (uint64_t) aoff)
    return -1;
  if (!mizu_ext_flags_known(base)) return -1;
  uint64_t sum = 0;
  for (int64_t i = 0; i < (int64_t) n32; i++) {
    mizu_mizl_entry e;
    if (mizu_ext_mizl_ent(base, size, i, &e) != 0) return -1;
    const int32_t tag = e.sexptype & ~(int32_t) MIZU_MIZL_S4;
    /* vcount is an int64, so the bound needs the sum only up to
       INT64_MAX: saturate. Overlapping crafted leaves in a huge region
       can otherwise wrap the accumulation and over-reject. */
    if (mizu_type_elt_size(tag) != 0 && sum < (uint64_t) INT64_MAX) {
      const uint64_t len = (uint64_t) e.length;
      sum = len > (uint64_t) INT64_MAX - sum ? (uint64_t) INT64_MAX
                                             : sum + len;
    }
  }
  if (voff == 0) {
    if (vcount != 0 && vcount != -1) return -1;
  } else {
    const uint64_t bytes = (uint64_t) (uint32_t) n32 * 16;
    if (vcount < 0 || (uint64_t) vcount > sum || (voff & 63) != 0 ||
        (uint64_t) voff > (uint64_t) size ||
        bytes > (uint64_t) size - (uint64_t) voff)
      return -1;
  }
  *n = n32;
  *attrs_off = aoff;
  *attrs_size = asz;
  valid[0] = voff;
  valid[1] = vcount;
  return 0;
}
 
MIZU_EXT_INLINE int mizu_ext_mizl_elem_impl(const void *base, size_t size,
                                           int64_t i,
                                           mizu_mizl_entry *elem) {
  if (size < MIZU_HEADER_SIZE) return -1;
  uint32_t magic;
  int32_t n32;
  int64_t voff, vcount;
  memcpy(&magic, base, 4);
  if (magic != MIZU_MAGIC_LIST) return -1;
  memcpy(&n32, (const unsigned char *) base + 4, 4);
  if (n32 < 0 || i < 0 || i >= (int64_t) n32 ||
      (uint64_t) (uint32_t) n32 >
        ((uint64_t) size - (uint64_t) MIZU_HEADER_SIZE) / 32)
    return -1;
  if (!mizu_ext_flags_known(base)) return -1;
  if (mizu_ext_mizl_ent(base, size, i, elem) != 0) return -1;
  memcpy(&voff, (const unsigned char *) base + MIZU_HDR_VALID_OFF, 8);
  memcpy(&vcount, (const unsigned char *) base + MIZU_HDR_VALID_COUNT, 8);
  if (voff == 0) {
    if (vcount != 0 && vcount != -1) return -1;
    elem->valid[0] = 0;
    elem->valid[1] = vcount;
    return 0;
  }
  if ((voff & 63) != 0) return -1;
  const uint64_t bytes = (uint64_t) (uint32_t) n32 * 16;
  if ((uint64_t) voff > (uint64_t) size ||
      bytes > (uint64_t) size - (uint64_t) voff)
    return -1;
  const unsigned char *tab = (const unsigned char *) base + voff;
  int64_t loff, lcount;
  memcpy(&loff, tab + 16 * (size_t) i, 8);
  memcpy(&lcount, tab + 16 * (size_t) i + 8, 8);
  /* a remote leaf (tag 33) carries a {0,0} / {0,-1} claim alone — a
     bitmap offset is region-local and cannot describe a remote column.
     STR / VEC / serialized leaves carry no table bitmap either: the
     string block, the nested header, and the blob each speak for
     themselves, and the writers stamp those rows {0,0} */
  const int32_t ltag = elem->sexptype & ~(int32_t) MIZU_MIZL_S4;
  if ((ltag == 33 || mizu_type_elt_size(ltag) == 0) && loff != 0)
    return -1;
  if (!mizu_ext_valid_ok(loff, lcount, (uint64_t) elem->length, size))
    return -1;
  elem->valid[0] = loff;
  elem->valid[1] = lcount;
  return 0;
}
 
MIZU_EXT_INLINE uint64_t mizu_ext_na_build_impl(int type, uint8_t *bitmap,
                                               const void *src, uint64_t n,
                                               uint64_t bit_off) {
  uint64_t nulls = 0;
  for (uint64_t i = 0; i < n; i++) {
    const uint64_t bit = bit_off + i;
    if (mizu_ext_na_at(type, src, i)) {
      bitmap[bit >> 3] &= (uint8_t) ~(1u << (bit & 7));
      nulls++;
    } else {
      bitmap[bit >> 3] |= (uint8_t) (1u << (bit & 7));
    }
  }
  return nulls;
}
 
MIZU_EXT_INLINE uint64_t mizu_ext_na_apply_impl(int type, void *dst,
                                               const void *src,
                                               const uint8_t *bitmap,
                                               uint64_t n) {
  const size_t elt = mizu_type_elt_size(type);
  if (elt == 0) return 0;
  if (dst != src) memcpy(dst, src, (size_t) n * elt);
  uint64_t nulls = 0;
  for (uint64_t i = 0; i < n; i++) {
    if (((bitmap[i >> 3] >> (i & 7)) & 1) == 0) {
      mizu_ext_na_store(type, dst, i);
      nulls++;
    }
  }
  return nulls;
}
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API double mizu_timeout_ms(double seconds);
MIZU_API void mizu_store_na_real(void *dst);
MIZU_API uint64_t mizu_aux_rawspill_pool(int type, uint32_t name_len);
MIZU_API uint64_t mizu_aux_shm_vec(int type, uint64_t total);
MIZU_API int mizu_aux_type(uint64_t aux);
MIZU_API uint64_t mizu_aux_hi(uint64_t aux);
MIZU_API void mizu_mizh_write(void *base, int wire_type, int64_t n_elems);
MIZU_API int mizu_mizh_check(const void *base, size_t size,
                           int *wire_type, int64_t *n_elems, int64_t valid[2]);
MIZU_API void mizu_mizh_validity_set(void *base, int64_t off,
                                   int64_t count);
MIZU_API int mizu_mizs_check(const void *base, size_t size, int64_t *n,
                           int64_t *str_size, int64_t *attrs_size);
MIZU_API int mizu_mizl_check(const void *base, size_t size, int64_t *n,
                           int64_t *attrs_off, int64_t *attrs_size,
                           int64_t valid[2]);
MIZU_API int mizu_mizl_elem(const void *base, size_t size, int64_t i,
                           mizu_mizl_entry *elem);
MIZU_API uint64_t mizu_na_build(int type, uint8_t *bitmap, const void *src,
                              uint64_t n, uint64_t bit_off);
MIZU_API uint64_t mizu_na_apply(int type, void *dst, const void *src,
                              const uint8_t *bitmap, uint64_t n);
#else
MIZU_EXT_INLINE double mizu_timeout_ms(double seconds) {
  if (!isfinite(seconds)) return -1;
  if (seconds <= 0) return 0;
  const double ms = seconds * 1000;
  return isfinite(ms) ? ms : -1;
}
MIZU_EXT_INLINE void mizu_store_na_real(void *dst) {
  const uint64_t bits = MIZU_NA_REAL_BITS;
  memcpy(dst, &bits, 8);
}
MIZU_EXT_INLINE uint64_t mizu_aux_rawspill_pool(int type, uint32_t name_len) {
  return (uint64_t) (uint32_t) type | ((uint64_t) name_len << 8);
}
MIZU_EXT_INLINE uint64_t mizu_aux_shm_vec(int type, uint64_t total) {
  return (uint64_t) (uint32_t) type | (total << 8);
}
MIZU_EXT_INLINE int mizu_aux_type(uint64_t aux) {
  return (int) (aux & 0xff);
}
MIZU_EXT_INLINE uint64_t mizu_aux_hi(uint64_t aux) {
  return aux >> 8;
}
MIZU_EXT_INLINE void mizu_mizh_write(void *base, int wire_type,
                                   int64_t n_elems) {
  const uint32_t magic = MIZU_MAGIC_VEC;
  const int32_t t32 = wire_type;
  const int64_t zero64 = 0;
  memcpy(base, &magic, 4);
  memcpy((unsigned char *) base + 4, &t32, 4);
  memcpy((unsigned char *) base + 8, &n_elems, 8);
  memcpy((unsigned char *) base + 16, &zero64, 8);
  memset((unsigned char *) base + 24, 0, MIZU_HEADER_SIZE - 24);
}
MIZU_EXT_INLINE int mizu_mizh_check(const void *base, size_t size,
                                  int *wire_type, int64_t *n_elems,
                                  int64_t valid[2]) {
  return mizu_ext_mizh_check_impl(base, size, wire_type, n_elems, valid);
}
MIZU_EXT_INLINE void mizu_mizh_validity_set(void *base, int64_t off,
                                          int64_t count) {
  memcpy((unsigned char *) base + MIZU_HDR_VALID_OFF, &off, 8);
  memcpy((unsigned char *) base + MIZU_HDR_VALID_COUNT, &count, 8);
}
MIZU_EXT_INLINE int mizu_mizs_check(const void *base, size_t size,
                                  int64_t *n, int64_t *str_size,
                                  int64_t *attrs_size) {
  return mizu_ext_mizs_check_impl(base, size, n, str_size, attrs_size);
}
MIZU_EXT_INLINE int mizu_mizl_check(const void *base, size_t size,
                                  int64_t *n, int64_t *attrs_off,
                                  int64_t *attrs_size, int64_t valid[2]) {
  return mizu_ext_mizl_check_impl(base, size, n, attrs_off, attrs_size,
                                 valid);
}
MIZU_EXT_INLINE int mizu_mizl_elem(const void *base, size_t size, int64_t i,
                                 mizu_mizl_entry *elem) {
  return mizu_ext_mizl_elem_impl(base, size, i, elem);
}
MIZU_EXT_INLINE uint64_t mizu_na_build(int type, uint8_t *bitmap,
                                     const void *src, uint64_t n,
                                     uint64_t bit_off) {
  return mizu_ext_na_build_impl(type, bitmap, src, n, bit_off);
}
MIZU_EXT_INLINE uint64_t mizu_na_apply(int type, void *dst, const void *src,
                                     const uint8_t *bitmap, uint64_t n) {
  return mizu_ext_na_apply_impl(type, dst, src, bitmap, n);
}
#endif
 
// The interchange stream ('I') ---------------------------------------------------
 
#define MIZU_IX_VERSION 0x01u
#define MIZU_IX_DEPTH_MAX 64
 
enum {
  MIZU_IX_TAG_NIL   = 0x00,
  MIZU_IX_TAG_LGL1  = 0x01,
  MIZU_IX_TAG_INT   = 0x02,
  MIZU_IX_TAG_REAL  = 0x03,
  MIZU_IX_TAG_STR   = 0x04,
  MIZU_IX_TAG_BYTES = 0x05,
  MIZU_IX_TAG_LGLV  = 0x06,
  MIZU_IX_TAG_INTV  = 0x07,
  MIZU_IX_TAG_REALV = 0x08,
  MIZU_IX_TAG_CPLXV = 0x09,
  MIZU_IX_TAG_RAWV  = 0x0a,
  MIZU_IX_TAG_STRV  = 0x0b,
  MIZU_IX_TAG_LIST  = 0x0c,
  MIZU_IX_TAG_DICT  = 0x0d,
  MIZU_IX_TAG_I64V  = 0x0e,
  MIZU_IX_TAG_ATTR  = 0x0f,
  MIZU_IX_TAG_CPLX  = 0x10,
  MIZU_IX_TAG_ERR   = 0x11,
  MIZU_IX_TAG_TASK  = 0x12,
  MIZU_IX_TAG_REF   = 0x13
};
 
enum {
  MIZU_IX_NIL = 0,
  MIZU_IX_LGL,          
  MIZU_IX_INT,          
  MIZU_IX_REAL,         
  MIZU_IX_CPLX,         
  MIZU_IX_STR1,         
  MIZU_IX_STR,          
  MIZU_IX_BYTES,        
  MIZU_IX_VEC,          
  MIZU_IX_STRV,         
  MIZU_IX_LIST,         
  MIZU_IX_DICT,         
  MIZU_IX_ATTR,         
  MIZU_IX_ERR,          
  MIZU_IX_TASK,         
  MIZU_IX_REF           
};
 
typedef struct mizu_ix_item_s {
  uint32_t kind;        
  uint32_t type;        
  int na;               
  int key;              
  const unsigned char *ptr;  
  uint64_t len;         
  uint64_t count;       
  uint64_t u64[2];      
  uint32_t target;      
  uint32_t task_kind;   
  uint32_t err_flags;   
  uint64_t err_index;   
  struct { const unsigned char *ptr; uint64_t len; } err_str[3];
} mizu_ix_item;
 
typedef struct mizu_ix_s {
  const unsigned char *p;
  const unsigned char *end;
  uint32_t depth;
  int done;             
  int err;              
  struct {
    uint32_t kind;      
    uint64_t remaining; 
  } stack[MIZU_IX_DEPTH_MAX];
} mizu_ix;
 
MIZU_API mizu_status mizu_ix_open(mizu_ix *cur, const void *buf, size_t len);
MIZU_API mizu_status mizu_ix_next(mizu_ix *cur, mizu_ix_item *item);
MIZU_API mizu_status mizu_ix_end(mizu_ix *cur);
 
MIZU_API mizu_status mizu_ixt_open(mizu_ix *cur, const void *buf,
                                   size_t len, mizu_ix_item *item,
                                   int max_kind);
MIZU_API mizu_status mizu_ixt_want_code(mizu_ix *cur, mizu_ix_item *item);
MIZU_API mizu_status mizu_ixt_want_list(mizu_ix *cur, mizu_ix_item *item);
MIZU_API mizu_status mizu_ixt_want_dict(mizu_ix *cur, mizu_ix_item *item);
 
#define MIZU_IX_ERR_OVERHEAD 25u
#define MIZU_IX_ERR_TYPE_SHARE 128u
 
#define MIZU_IX_ATTR_NAMES    "names"
#define MIZU_IX_ATTR_LEVELS   "levels"
#define MIZU_IX_ATTR_DIM      "dim"
#define MIZU_IX_ATTR_CLASS    "class"
#define MIZU_IX_ATTR_UNITS    "units"
#define MIZU_IX_ATTR_ROWNAMES "row.names"
#define MIZU_IX_ATTR_TZONE    "tzone"
#define MIZU_IX_ATTR_TSCALE   "tscale"
#define MIZU_IX_ATTR_U64      "u64"
 
#define MIZU_IX_CLASS_FACTOR    "factor"
#define MIZU_IX_CLASS_DATE      "Date"
#define MIZU_IX_CLASS_POSIXCT   "POSIXct"
#define MIZU_IX_CLASS_POSIXT    "POSIXt"
#define MIZU_IX_CLASS_DIFFTIME  "difftime"
#define MIZU_IX_CLASS_DATAFRAME "data.frame"
 
#define MIZU_IX_UNIT_SECS  "secs"
#define MIZU_IX_UNIT_MINS  "mins"
#define MIZU_IX_UNIT_HOURS "hours"
#define MIZU_IX_UNIT_DAYS  "days"
#define MIZU_IX_UNIT_WEEKS "weeks"
 
MIZU_EXT_INLINE int mizu_ext_ix_utf8_valid_impl(const void *s, size_t n) {
  const unsigned char *b = (const unsigned char *) s;
  size_t i = 0;
  while (i < n) {
    const unsigned char c = b[i];
    if (c < 0x80) {
      i++;
    } else if (c < 0xC2 || c >= 0xF5) {   /* stray cont., overlong, or
                                             past U+10FFFF */
      return 0;
    } else if (c < 0xE0) {             /* 2 bytes */
      if (i + 2 > n || (b[i + 1] & 0xC0) != 0x80) return 0;
      i += 2;
    } else if (c < 0xF0) {             /* 3 bytes */
      if (i + 3 > n || (b[i + 1] & 0xC0) != 0x80 ||
          (b[i + 2] & 0xC0) != 0x80) return 0;
      if (c == 0xE0 && b[i + 1] < 0xA0) return 0;   /* overlong */
      if (c == 0xED && b[i + 1] >= 0xA0) return 0;  /* surrogate */
      i += 3;
    } else {                           /* 4 bytes: 0xF0..0xF4 */
      if (i + 4 > n || (b[i + 1] & 0xC0) != 0x80 ||
          (b[i + 2] & 0xC0) != 0x80 || (b[i + 3] & 0xC0) != 0x80)
        return 0;
      if (c == 0xF0 && b[i + 1] < 0x90) return 0;   /* overlong */
      if (c == 0xF4 && b[i + 1] > 0x8F) return 0;   /* > U+10FFFF */
      i += 4;
    }
  }
  return 1;
}
 
MIZU_EXT_INLINE int mizu_ext_ix_tag_of_impl(int wire_type) {
  switch (wire_type) {
  case MIZU_TYPE_LGL:   return MIZU_IX_TAG_LGLV;
  case MIZU_TYPE_INT:   return MIZU_IX_TAG_INTV;
  case MIZU_TYPE_REAL:  return MIZU_IX_TAG_REALV;
  case MIZU_TYPE_CPLX:  return MIZU_IX_TAG_CPLXV;
  case MIZU_TYPE_RAW:   return MIZU_IX_TAG_RAWV;
  case MIZU_TYPE_INT64: return MIZU_IX_TAG_I64V;
  default:              return MIZU_IX_TAG_NIL;
  }
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_header_impl(unsigned char *dst) {
  if (dst != NULL) {
    dst[0] = (unsigned char) MIZU_INTEROP_MAGIC;
    dst[1] = (unsigned char) MIZU_IX_VERSION;
  }
  return 2;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_tag8_impl(unsigned char *dst,
                                                uint32_t tag, uint64_t v) {
  if (dst != NULL) {
    dst[0] = (unsigned char) tag;
    memcpy(dst + 1, &v, 8);
  }
  return 9;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_nil_impl(unsigned char *dst) {
  if (dst != NULL) dst[0] = MIZU_IX_TAG_NIL;
  return 1;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_lgl_impl(unsigned char *dst,
                                               int value) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_LGL1;
    dst[1] = (unsigned char) value;
  }
  return 2;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_int_impl(unsigned char *dst,
                                               int64_t value) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_INT;
    memcpy(dst + 1, &value, 8);
  }
  return 9;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_real_impl(unsigned char *dst,
                                                double value) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_REAL;
    memcpy(dst + 1, &value, 8);
  }
  return 9;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_cplx_impl(unsigned char *dst,
                                                double re, double im) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_CPLX;
    memcpy(dst + 1, &re, 8);
    memcpy(dst + 9, &im, 8);
  }
  return 17;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_str_impl(unsigned char *dst,
                                               const void *s, int32_t len) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_STR;
    memcpy(dst + 1, &len, 4);
    if (len > 0) memcpy(dst + 5, s, (size_t) len);
  }
  return (size_t) 5 + (len > 0 ? (size_t) len : 0);
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_bytes_impl(unsigned char *dst,
                                                 const void *data,
                                                 uint64_t count) {
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_BYTES;
    memcpy(dst + 1, &count, 8);
    if (count != 0) memcpy(dst + 9, data, (size_t) count);
  }
  return (size_t) 9 + (size_t) count;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_vec_impl(unsigned char *dst,
                                               int wire_type,
                                               const void *data,
                                               uint64_t count) {
  const int tag = mizu_ext_ix_tag_of_impl(wire_type);
  if (tag == MIZU_IX_TAG_NIL) return 0;
  const size_t elt = mizu_type_elt_size(wire_type);
  if (dst != NULL) {
    dst[0] = (unsigned char) tag;
    memcpy(dst + 1, &count, 8);
    if (count != 0) memcpy(dst + 9, data, (size_t) count * elt);
  }
  return (size_t) 9 + (size_t) count * elt;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_strv_begin_impl(unsigned char *dst,
                                                      uint64_t count) {
  return mizu_ext_ix_put_tag8_impl(dst, MIZU_IX_TAG_STRV, count);
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_strelt_impl(unsigned char *dst,
                                                  const void *s,
                                                  int32_t len) {
  if (dst != NULL) {
    memcpy(dst, &len, 4);
    if (len > 0) memcpy(dst + 4, s, (size_t) len);
  }
  return (size_t) 4 + (len > 0 ? (size_t) len : 0);
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_list_begin_impl(unsigned char *dst,
                                                      uint64_t count) {
  return mizu_ext_ix_put_tag8_impl(dst, MIZU_IX_TAG_LIST, count);
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_dict_begin_impl(unsigned char *dst,
                                                      uint64_t count) {
  return mizu_ext_ix_put_tag8_impl(dst, MIZU_IX_TAG_DICT, count);
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_key_impl(unsigned char *dst,
                                               const void *s, uint32_t len) {
  if (dst != NULL) {
    memcpy(dst, &len, 4);
    if (len != 0) memcpy(dst + 4, s, (size_t) len);
  }
  return (size_t) 4 + (size_t) len;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_attr_impl(unsigned char *dst) {
  if (dst != NULL) dst[0] = MIZU_IX_TAG_ATTR;
  return 1;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_err_impl(unsigned char *dst,
                                               int has_index, uint64_t index,
                                               const void *type,
                                               uint32_t type_len,
                                               const void *message,
                                               uint32_t message_len,
                                               const void *detail,
                                               uint32_t detail_len) {
  if (dst != NULL) {
    const uint16_t flags = (uint16_t) (has_index != 0);
    dst[0] = MIZU_IX_TAG_ERR;
    memcpy(dst + 1, &flags, 2);
    size_t off = 3;
    if (has_index) {
      memcpy(dst + off, &index, 8);
      off += 8;
    }
    const void *strs[3] = { type, message, detail };
    const uint32_t lens[3] = { type_len, message_len, detail_len };
    for (int i = 0; i < 3; i++) {
      memcpy(dst + off, &lens[i], 4);
      off += 4;
      if (lens[i] != 0) {
        memcpy(dst + off, strs[i], (size_t) lens[i]);
        off += (size_t) lens[i];
      }
    }
    return off;
  }
  return (size_t) 3 + (has_index ? 8 : 0) +
         (size_t) 12 + (size_t) type_len + (size_t) message_len +
         (size_t) detail_len;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_task_impl(unsigned char *dst,
                                                int target, int kind,
                                                uint64_t ident) {
  if (dst != NULL) {
    const uint16_t zero = 0;
    dst[0] = MIZU_IX_TAG_TASK;
    dst[1] = (unsigned char) target;
    dst[2] = (unsigned char) kind;
    memcpy(dst + 3, &zero, 2);
    memcpy(dst + 5, &ident, 8);
  }
  return 13;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_put_ref_impl(unsigned char *dst,
                                                const void *name,
                                                uint32_t name_len) {
  if (name_len == 0 || name_len > 255) return 0;
  if (dst != NULL) {
    dst[0] = MIZU_IX_TAG_REF;
    dst[1] = (unsigned char) name_len;
    memcpy(dst + 2, name, (size_t) name_len);
  }
  return (size_t) 2 + name_len;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_err_floor_impl(const void *s, size_t n,
                                                  size_t share) {
  if (n <= share) return n;
  const unsigned char *b = (const unsigned char *) s;
  size_t len = share;
  while (len > 0 && (b[len] & 0xC0) == 0x80) len--;
  return len;
}
 
MIZU_EXT_INLINE size_t mizu_ext_ix_write_err_impl(
    unsigned char *dst, uint32_t inline_max,
    const void *type, size_t type_n,
    const void *msg, size_t msg_n,
    const void *detail, size_t detail_n,
    int has_index, uint64_t index) {
  const size_t budget = inline_max;
  const size_t avail =
    budget > MIZU_IX_ERR_OVERHEAD ? budget - MIZU_IX_ERR_OVERHEAD : 0;
  size_t cap =
    avail < MIZU_IX_ERR_TYPE_SHARE ? avail : MIZU_IX_ERR_TYPE_SHARE;
  const size_t tn = mizu_ext_ix_err_floor_impl(type, type_n, cap);
  cap = budget / 2;
  if (cap > avail - tn) cap = avail - tn;
  const size_t mn = mizu_ext_ix_err_floor_impl(msg, msg_n, cap);
  const size_t dn =
    mizu_ext_ix_err_floor_impl(detail, detail_n, avail - tn - mn);
  size_t n = mizu_ext_ix_put_header_impl(dst);
  n += mizu_ext_ix_put_err_impl(dst != NULL ? dst + n : NULL, has_index,
                                index, type, (uint32_t) tn, msg,
                                (uint32_t) mn, detail, (uint32_t) dn);
  return n;
}
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API int mizu_ix_utf8_valid(const void *s, size_t n);
MIZU_API int mizu_ix_tag_of(int wire_type);
MIZU_API size_t mizu_ix_put_header(unsigned char *dst);
MIZU_API size_t mizu_ix_put_nil(unsigned char *dst);
MIZU_API size_t mizu_ix_put_lgl(unsigned char *dst, int value);
MIZU_API size_t mizu_ix_put_int(unsigned char *dst, int64_t value);
MIZU_API size_t mizu_ix_put_real(unsigned char *dst, double value);
MIZU_API size_t mizu_ix_put_cplx(unsigned char *dst, double re, double im);
MIZU_API size_t mizu_ix_put_str(unsigned char *dst, const void *s,
                              int32_t len);
MIZU_API size_t mizu_ix_put_bytes(unsigned char *dst, const void *data,
                                uint64_t count);
MIZU_API size_t mizu_ix_put_vec(unsigned char *dst, int wire_type,
                              const void *data, uint64_t count);
MIZU_API size_t mizu_ix_put_strv_begin(unsigned char *dst, uint64_t count);
MIZU_API size_t mizu_ix_put_strelt(unsigned char *dst, const void *s,
                                 int32_t len);
MIZU_API size_t mizu_ix_put_list_begin(unsigned char *dst, uint64_t count);
MIZU_API size_t mizu_ix_put_dict_begin(unsigned char *dst, uint64_t count);
MIZU_API size_t mizu_ix_put_key(unsigned char *dst, const void *s,
                              uint32_t len);
MIZU_API size_t mizu_ix_put_attr(unsigned char *dst);
MIZU_API size_t mizu_ix_put_err(unsigned char *dst, int has_index,
                              uint64_t index,
                              const void *type, uint32_t type_len,
                              const void *message, uint32_t message_len,
                              const void *detail, uint32_t detail_len);
MIZU_API size_t mizu_ix_put_task(unsigned char *dst, int target, int kind,
                               uint64_t ident);
MIZU_API size_t mizu_ix_put_ref(unsigned char *dst, const void *name,
                              uint32_t name_len);
MIZU_API size_t mizu_ix_write_err(unsigned char *dst, uint32_t inline_max,
                                  const void *type, size_t type_n,
                                  const void *msg, size_t msg_n,
                                  const void *detail, size_t detail_n,
                                  int has_index, uint64_t index);
#else
MIZU_EXT_INLINE int mizu_ix_utf8_valid(const void *s, size_t n) {
  return mizu_ext_ix_utf8_valid_impl(s, n);
}
MIZU_EXT_INLINE int mizu_ix_tag_of(int wire_type) {
  return mizu_ext_ix_tag_of_impl(wire_type);
}
MIZU_EXT_INLINE size_t mizu_ix_put_header(unsigned char *dst) {
  return mizu_ext_ix_put_header_impl(dst);
}
MIZU_EXT_INLINE size_t mizu_ix_put_nil(unsigned char *dst) {
  return mizu_ext_ix_put_nil_impl(dst);
}
MIZU_EXT_INLINE size_t mizu_ix_put_lgl(unsigned char *dst, int value) {
  return mizu_ext_ix_put_lgl_impl(dst, value);
}
MIZU_EXT_INLINE size_t mizu_ix_put_int(unsigned char *dst, int64_t value) {
  return mizu_ext_ix_put_int_impl(dst, value);
}
MIZU_EXT_INLINE size_t mizu_ix_put_real(unsigned char *dst, double value) {
  return mizu_ext_ix_put_real_impl(dst, value);
}
MIZU_EXT_INLINE size_t mizu_ix_put_cplx(unsigned char *dst, double re,
                                      double im) {
  return mizu_ext_ix_put_cplx_impl(dst, re, im);
}
MIZU_EXT_INLINE size_t mizu_ix_put_str(unsigned char *dst, const void *s,
                                     int32_t len) {
  return mizu_ext_ix_put_str_impl(dst, s, len);
}
MIZU_EXT_INLINE size_t mizu_ix_put_bytes(unsigned char *dst,
                                       const void *data, uint64_t count) {
  return mizu_ext_ix_put_bytes_impl(dst, data, count);
}
MIZU_EXT_INLINE size_t mizu_ix_put_vec(unsigned char *dst, int wire_type,
                                     const void *data, uint64_t count) {
  return mizu_ext_ix_put_vec_impl(dst, wire_type, data, count);
}
MIZU_EXT_INLINE size_t mizu_ix_put_strv_begin(unsigned char *dst,
                                            uint64_t count) {
  return mizu_ext_ix_put_strv_begin_impl(dst, count);
}
MIZU_EXT_INLINE size_t mizu_ix_put_strelt(unsigned char *dst, const void *s,
                                        int32_t len) {
  return mizu_ext_ix_put_strelt_impl(dst, s, len);
}
MIZU_EXT_INLINE size_t mizu_ix_put_list_begin(unsigned char *dst,
                                            uint64_t count) {
  return mizu_ext_ix_put_list_begin_impl(dst, count);
}
MIZU_EXT_INLINE size_t mizu_ix_put_dict_begin(unsigned char *dst,
                                            uint64_t count) {
  return mizu_ext_ix_put_dict_begin_impl(dst, count);
}
MIZU_EXT_INLINE size_t mizu_ix_put_key(unsigned char *dst, const void *s,
                                     uint32_t len) {
  return mizu_ext_ix_put_key_impl(dst, s, len);
}
MIZU_EXT_INLINE size_t mizu_ix_put_attr(unsigned char *dst) {
  return mizu_ext_ix_put_attr_impl(dst);
}
MIZU_EXT_INLINE size_t mizu_ix_put_err(unsigned char *dst, int has_index,
                                     uint64_t index,
                                     const void *type, uint32_t type_len,
                                     const void *message,
                                     uint32_t message_len,
                                     const void *detail,
                                     uint32_t detail_len) {
  return mizu_ext_ix_put_err_impl(dst, has_index, index, type, type_len,
                                 message, message_len, detail, detail_len);
}
MIZU_EXT_INLINE size_t mizu_ix_put_task(unsigned char *dst, int target,
                                      int kind, uint64_t ident) {
  return mizu_ext_ix_put_task_impl(dst, target, kind, ident);
}
MIZU_EXT_INLINE size_t mizu_ix_put_ref(unsigned char *dst, const void *name,
                                     uint32_t name_len) {
  return mizu_ext_ix_put_ref_impl(dst, name, name_len);
}
MIZU_EXT_INLINE size_t mizu_ix_write_err(unsigned char *dst,
                                         uint32_t inline_max,
                                         const void *type, size_t type_n,
                                         const void *msg, size_t msg_n,
                                         const void *detail, size_t detail_n,
                                         int has_index, uint64_t index) {
  return mizu_ext_ix_write_err_impl(dst, inline_max, type, type_n, msg,
                                    msg_n, detail, detail_n, has_index,
                                    index);
}
#endif
 
// The Arrow frame writer (arrow.c) -----------------------------------------------
 
#ifndef ARROW_C_DATA_INTERFACE
#define ARROW_C_DATA_INTERFACE
typedef struct ArrowSchema {
  const char *format;
  const char *name;
  const char *metadata;
  int64_t flags;
  int64_t n_children;
  struct ArrowSchema **children;
  struct ArrowSchema *dictionary;
  void (*release)(struct ArrowSchema *);
  void *private_data;
} ArrowSchema;
 
typedef struct ArrowArray {
  int64_t length;
  int64_t null_count;
  int64_t offset;
  int64_t n_buffers;
  int64_t n_children;
  const void **buffers;
  struct ArrowArray **children;
  struct ArrowArray *dictionary;
  void (*release)(struct ArrowArray *);
  void *private_data;
} ArrowArray;
#endif
 
#ifndef ARROW_C_STREAM_INTERFACE
#define ARROW_C_STREAM_INTERFACE
typedef struct ArrowArrayStream {
  int (*get_schema)(struct ArrowArrayStream *, struct ArrowSchema *out);
  int (*get_next)(struct ArrowArrayStream *, struct ArrowArray *out);
  const char *(*get_last_error)(struct ArrowArrayStream *);
  void (*release)(struct ArrowArrayStream *);
  void *private_data;
} ArrowArrayStream;
#endif
 
/* The element helpers: pure accessors over the ABI structs, dual-form like
   the 'I' cursor's — the frame writer's own loops and a binding's 'I'
   emitters share the one definition. */
MIZU_EXT_INLINE int mizu_ext_arrow_valid_impl(const ArrowArray *arr,
                                              int64_t k) {
  if (arr->null_count == 0 || arr->buffers[0] == NULL) return 1;
  int64_t i = arr->offset + k;
  const uint8_t *bm = (const uint8_t *) arr->buffers[0];
  return (bm[i >> 3] >> (i & 7)) & 1;
}
 
MIZU_EXT_INLINE int mizu_ext_arrow_str_at_impl(const ArrowArray *arr,
                                               int64_t k, int form,
                                               const uint8_t **ptr,
                                               int32_t *len) {
  if (!mizu_ext_arrow_valid_impl(arr, k)) return 0;
  int64_t i = arr->offset + k;
  if (form == 1) {
    const uint8_t *offs = (const uint8_t *) arr->buffers[1];
    int32_t o0, o1;
    memcpy(&o0, offs + 4 * i, 4);
    memcpy(&o1, offs + 4 * (i + 1), 4);
    if (o0 < 0 || o1 < o0) return -1;
    *ptr = (const uint8_t *) arr->buffers[2] + o0;
    *len = o1 - o0;
  } else if (form == 2) {
    const uint8_t *offs = (const uint8_t *) arr->buffers[1];
    int64_t o0, o1;
    memcpy(&o0, offs + 8 * i, 8);
    memcpy(&o1, offs + 8 * (i + 1), 8);
    if (o0 < 0 || o1 < o0 || o1 - o0 > INT32_MAX)
      return -1;
    *ptr = (const uint8_t *) arr->buffers[2] + o0;
    *len = (int32_t) (o1 - o0);
  } else {
    const uint8_t *vw = (const uint8_t *) arr->buffers[1] + i * 16;
    int32_t l;
    memcpy(&l, vw, 4);
    if (l < 0) return -1;
    if (l <= 12) {
      *ptr = vw + 4;
    } else {
      int32_t bi, off;
      memcpy(&bi, vw + 8, 4);
      memcpy(&off, vw + 12, 4);
      /* The variadic data buffers are slots [2, n_buffers): the C Data
         spec's trailing slot is the int64 sizes array, but a producer
         that skips it (a nanoarrow nested or standalone export) puts a
         data buffer there. The reference tells the layouts apart: a
         spec-layout view never names the trailing slot, so one that
         does proves the no-sizes layout, its span the producer's word;
         any other span the sizes array bounds by the declared size. */
      if (bi < 0 || off < 0 || 2 + (int64_t) bi >= arr->n_buffers)
        return -1;
      if (2 + bi < arr->n_buffers - 1) {
        const uint8_t *sizes =
          (const uint8_t *) arr->buffers[arr->n_buffers - 1];
        if (sizes != NULL) {
          int64_t sz;
          memcpy(&sz, sizes + 8 * (ptrdiff_t) bi, 8);
          if ((int64_t) off + l > sz) return -1;
        }
      }
      *ptr = (const uint8_t *) arr->buffers[2 + bi] + off;
    }
    *len = l;
  }
  return 1;
}
 
#ifdef MIZU_EXT_NO_INLINES
MIZU_API int mizu_arrow_valid(const ArrowArray *, int64_t k);
MIZU_API int mizu_arrow_str_at(const ArrowArray *, int64_t k, int form,
                               const uint8_t **ptr, int32_t *len);
#else
MIZU_EXT_INLINE int mizu_arrow_valid(const ArrowArray *arr, int64_t k) {
  return mizu_ext_arrow_valid_impl(arr, k);
}
MIZU_EXT_INLINE int mizu_arrow_str_at(const ArrowArray *arr, int64_t k,
                                      int form, const uint8_t **ptr,
                                      int32_t *len) {
  return mizu_ext_arrow_str_at_impl(arr, k, form, ptr, len);
}
#endif
 
#define MIZU_ARROW_FLAG_DICTIONARY_ORDERED 1
#define MIZU_ARROW_FLAG_NULLABLE 2
 
enum {
  MIZU_CVT_COPY = 0,  
  MIZU_CVT_I8_INT,    
  MIZU_CVT_I16_INT,
  MIZU_CVT_U16_INT,   
  MIZU_CVT_U32_REAL,  
  MIZU_CVT_U64_REAL,
  MIZU_CVT_F16_REAL,  
  MIZU_CVT_F32_REAL,  
  MIZU_CVT_BOOL8_LGL, 
  MIZU_CVT_BIT_LGL,   
  MIZU_CVT_C64_CPLX   
};
 
typedef struct mizu_cvt_row_s {
  int wire;         
  int cvt;          
  uint8_t w_in;     
  uint8_t w_out;
} mizu_cvt_row;
 
typedef struct mizu_arrow_warn_s {
  uint64_t n_range;   
  uint64_t n_intmin;  
  uint64_t n_int64;   
} mizu_arrow_warn;
 
MIZU_API const mizu_cvt_row *mizu_cvt_for_arrow(const char *fmt);
 
MIZU_API void mizu_cvt_convert(uint8_t *dst, const uint8_t *src,
                               const uint8_t *valid, uint64_t off, size_t n,
                               const mizu_cvt_row *row,
                               mizu_arrow_warn *warn);
 
typedef struct mizu_arrow mizu_arrow;
 
enum {
  MIZU_ARROW_CVT = 0,  
  MIZU_ARROW_STR,      
  MIZU_ARROW_DICT,     
  MIZU_ARROW_DATE,     
  MIZU_ARROW_TS,       
  MIZU_ARROW_TD        
};
 
typedef struct mizu_arrow_col_s {
  int kind;                  
  const mizu_cvt_row *row;   
  int str_form;              
  int u64_verbatim;          
} mizu_arrow_col;
 
MIZU_API mizu_status mizu_arrow_open(mizu_arrow **out,
                                     const ArrowSchema *schema,
                                     const ArrowArray *const *batches,
                                     size_t n_batches, uint32_t peer_caps);
 
MIZU_API void mizu_arrow_col_info(const mizu_arrow *, int64_t col,
                                  mizu_arrow_col *out);
 
MIZU_API size_t mizu_arrow_col_body(const mizu_arrow *, int64_t col,
                                    uint8_t *dst, mizu_arrow_warn *warn);
 
MIZU_API size_t mizu_arrow_col_blob(const mizu_arrow *, int64_t col,
                                    uint8_t *dst);
 
MIZU_API size_t mizu_arrow_write_ix_col(const mizu_arrow *, int64_t col,
                                        uint8_t *dst, mizu_arrow_warn *warn);
 
MIZU_API size_t mizu_arrow_write_ix(const mizu_arrow *, uint8_t *dst,
                                    mizu_arrow_warn *warn);
 
MIZU_API int mizu_arrow_remote(mizu_arrow *, int64_t col,
                               const char *ident, size_t ident_len,
                               void *ref_base, size_t ref_size,
                               int64_t ref_leaf);
 
MIZU_API void mizu_arrow_root_blob(mizu_arrow *, const void *blob,
                                   size_t len);
 
MIZU_API void mizu_arrow_tz(mizu_arrow *, int64_t col, const char *tz);
 
MIZU_API size_t mizu_arrow_size(mizu_arrow *);
 
MIZU_API mizu_status mizu_arrow_stage(mizu_handle *, mizu_arrow *,
                                      mizu_slot_hdr *hdr,
                                      unsigned char *payload,
                                      mizu_arrow_warn *warn);
MIZU_API void mizu_arrow_close(mizu_arrow *);
 
// Utilities ----------------------------------------------------------------------
 
MIZU_API double mizu_now(void);
MIZU_API long mizu_self_pid(void);
 
MIZU_API void mizu_err_describe(mizu_errcat, const char **summary,
                              const char **hint);
 
MIZU_API void mizu_rng_jump(int *seed);
 
MIZU_API void mizu_tune(void);
 
#ifdef __cplusplus
}
#endif
 
#endif