diff --git a/src/io/csv.c b/src/io/csv.c index 25a92c1c7..5f766db72 100644 --- a/src/io/csv.c +++ b/src/io/csv.c @@ -1037,6 +1037,7 @@ static void csv_parse_fn(void* arg, uint32_t worker_id, case CSV_TYPE_I16: ((int16_t*)ctx->col_data[c])[row] = NULL_I16; break; case CSV_TYPE_I32: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; case CSV_TYPE_I64: ((int64_t*)ctx->col_data[c])[row] = NULL_I64; break; + case CSV_TYPE_F32: ((float*)ctx->col_data[c])[row] = NULL_F32; break; case CSV_TYPE_F64: ((double*)ctx->col_data[c])[row] = NULL_F64; break; case CSV_TYPE_DATE: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; case CSV_TYPE_TIME: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; @@ -1114,6 +1115,13 @@ static void csv_parse_fn(void* arg, uint32_t worker_id, if (is_null) my_had_null[c] = true; break; } + case CSV_TYPE_F32: { + bool is_null; + double v = fast_f64(fld, flen, &is_null); + ((float*)ctx->col_data[c])[row] = is_null ? NULL_F32 : (float)v; + if (is_null) my_had_null[c] = true; + break; + } case CSV_TYPE_DATE: { bool is_null; int32_t v = fast_date(fld, flen, &is_null); @@ -1206,6 +1214,7 @@ static void csv_parse_serial(const char* buf, size_t buf_size, case CSV_TYPE_I16: ((int16_t*)col_data[c])[row] = NULL_I16; break; case CSV_TYPE_I32: ((int32_t*)col_data[c])[row] = NULL_I32; break; case CSV_TYPE_I64: ((int64_t*)col_data[c])[row] = NULL_I64; break; + case CSV_TYPE_F32: ((float*)col_data[c])[row] = NULL_F32; break; case CSV_TYPE_F64: ((double*)col_data[c])[row] = NULL_F64; break; case CSV_TYPE_DATE: ((int32_t*)col_data[c])[row] = NULL_I32; break; case CSV_TYPE_TIME: ((int32_t*)col_data[c])[row] = NULL_I32; break; @@ -1281,6 +1290,13 @@ static void csv_parse_serial(const char* buf, size_t buf_size, if (is_null) col_had_null[c] = true; break; } + case CSV_TYPE_F32: { + bool is_null; + double v = fast_f64(fld, flen, &is_null); + ((float*)col_data[c])[row] = is_null ? NULL_F32 : (float)v; + if (is_null) col_had_null[c] = true; + break; + } case CSV_TYPE_DATE: { bool is_null; int32_t v = fast_date(fld, flen, &is_null); @@ -1629,6 +1645,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, case RAY_I16: parse_types[c] = CSV_TYPE_I16; break; case RAY_I32: parse_types[c] = CSV_TYPE_I32; break; case RAY_I64: parse_types[c] = CSV_TYPE_I64; break; + case RAY_F32: parse_types[c] = CSV_TYPE_F32; break; case RAY_F64: parse_types[c] = CSV_TYPE_F64; break; case RAY_DATE: parse_types[c] = CSV_TYPE_DATE; break; case RAY_TIME: parse_types[c] = CSV_TYPE_TIME; break; @@ -2037,6 +2054,7 @@ ray_t* ray_read_csv_named_opts(const char* path, char delimiter, bool header, case RAY_I16: parse_types[c] = CSV_TYPE_I16; break; case RAY_I32: parse_types[c] = CSV_TYPE_I32; break; case RAY_I64: parse_types[c] = CSV_TYPE_I64; break; + case RAY_F32: parse_types[c] = CSV_TYPE_F32; break; case RAY_F64: parse_types[c] = CSV_TYPE_F64; break; case RAY_DATE: parse_types[c] = CSV_TYPE_DATE; break; case RAY_TIME: parse_types[c] = CSV_TYPE_TIME; break; @@ -3128,6 +3146,9 @@ static void csv_write_cell(csv_writer_t* w, const csv_col_info_t* ci, int64_t r) case RAY_F64: csv_write_f64(w, ((const double*)d)[dr]); break; + case RAY_F32: + csv_write_f64(w, (double)((const float*)d)[dr]); + break; case RAY_DATE: csv_write_date(w, ((const int32_t*)d)[dr]); break; diff --git a/src/io/csv.h b/src/io/csv.h index f77b2af79..6b6b4c270 100644 --- a/src/io/csv.h +++ b/src/io/csv.h @@ -47,6 +47,7 @@ typedef enum { CSV_TYPE_U8, CSV_TYPE_I16, CSV_TYPE_I32, + CSV_TYPE_F32, /* Marker for "pick the narrowest integer width automatically" — used only * during schema processing, never returned by the resolver. */ CSV_TYPE_AUTO diff --git a/src/lang/compile.c b/src/lang/compile.c index 909e87e11..7475c5053 100644 --- a/src/lang/compile.c +++ b/src/lang/compile.c @@ -192,7 +192,7 @@ static void patch_jump(compiler_t *c, int32_t pos) { } /* Cached sym IDs for special forms */ -static _Thread_local int64_t sf_set = -1, sf_let = -1, sf_if = -1, sf_do = -1, sf_fn = -1, sf_self = -1, sf_try = -1, sf_return = -1; +static _Thread_local int64_t sf_set = -1, sf_let = -1, sf_if = -1, sf_do = -1, sf_fn = -1, sf_self = -1, sf_try = -1, sf_return = -1, sf_null = -1; static _Thread_local int64_t sf_eval = -1, sf_resolve = -1; static void init_sf_syms(void) { @@ -205,6 +205,7 @@ static void init_sf_syms(void) { sf_self = ray_sym_intern("self", 4); sf_try = ray_sym_intern("try", 3); sf_return = ray_sym_intern("return", 6); + sf_null = ray_sym_intern("null", 4); sf_eval = ray_sym_intern("eval", 4); sf_resolve = ray_sym_intern("resolve", 7); } @@ -522,6 +523,14 @@ static void compile_expr(compiler_t *c, ray_t *ast) { if (ray_is_atom(ast)) { if (ast->type == -RAY_SYM && !(ast->attrs & ATTR_QUOTED)) { + /* The tree walker treats bare `null` as the singleton value, not + * an environment lookup. Compiled lambdas must do the same; + * otherwise any branch returning null fails with a name error. */ + init_sf_syms(); + if (ast->i64 == sf_null) { + emit_const(c, add_constant(c, RAY_NULL_OBJ)); + return; + } int32_t slot = find_local(c, ast->i64); if (slot >= 0) { emit(c, OP_LOADENV); @@ -639,6 +648,6 @@ ray_span_t ray_bc_dbg_get(ray_t* dbg, int32_t ip) { } void ray_compile_reset(void) { - sf_set = sf_let = sf_if = sf_do = sf_fn = sf_self = sf_try = sf_return = -1; + sf_set = sf_let = sf_if = sf_do = sf_fn = sf_self = sf_try = sf_return = sf_null = -1; sf_eval = sf_resolve = -1; } diff --git a/src/lang/internal.h b/src/lang/internal.h index e45a04cf6..6c030de3f 100644 --- a/src/lang/internal.h +++ b/src/lang/internal.h @@ -138,6 +138,10 @@ static inline int is_float_op(ray_t* a, ray_t* b) { a->type == -RAY_F32 || b->type == -RAY_F32; } +static inline int both_f32_atoms(ray_t* a, ray_t* b) { + return a->type == -RAY_F32 && b->type == -RAY_F32; +} + /* ══════════════════════════════════════════ * Null/type helpers * ══════════════════════════════════════════ */ @@ -148,6 +152,8 @@ static inline int is_float_op(ray_t* a, ray_t* b) { static inline ray_t* null_for_promoted(ray_t* a, ray_t* b) { if (a->type == -RAY_F64 || b->type == -RAY_F64) return ray_typed_null(-RAY_F64); + if (a->type == -RAY_F32 || b->type == -RAY_F32) + return ray_typed_null(both_f32_atoms(a, b) ? -RAY_F32 : -RAY_F64); if (a->type == -RAY_I64 || b->type == -RAY_I64) return ray_typed_null(-RAY_I64); if (a->type == -RAY_I32 || b->type == -RAY_I32) diff --git a/src/ops/agg.c b/src/ops/agg.c index 0c1d5f7ea..6c42d299e 100644 --- a/src/ops/agg.c +++ b/src/ops/agg.c @@ -93,7 +93,7 @@ static void nth_element_dbl(double* a, int64_t lo, int64_t hi, int64_t k) { static int agg_parted_numeric_base(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64 || + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64 || t == RAY_DATE || t == RAY_TIME || t == RAY_TIMESTAMP; } @@ -130,15 +130,17 @@ static ray_t* agg_parted_sum(ray_t* x) { if (!agg_parted_numeric_base(base) || base == RAY_DATE) return ray_error("type", "sum expects a numeric or time-duration parted column, got %s", ray_type_name(base)); ray_t** segs = (ray_t**)ray_data(x); - if (base == RAY_F64) { + if (base == RAY_F64 || base == RAY_F32) { double sum = 0.0; for (int64_t s = 0; s < x->len; s++) { ray_t* seg = segs[s]; if (!seg) continue; - double* d = (double*)ray_data(seg); int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; for (int64_t i = 0; i < seg->len; i++) - if (!has_nulls || !ray_vec_is_null(seg, i)) sum += d[i]; + if (!has_nulls || !ray_vec_is_null(seg, i)) { + if (base == RAY_F64) sum += ((double*)ray_data(seg))[i]; + else sum += (double)((float*)ray_data(seg))[i]; + } } return make_f64(sum); } @@ -165,11 +167,12 @@ static ray_t* agg_parted_avg(ray_t* x) { ray_t* seg = segs[s]; if (!seg) continue; int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; - if (base == RAY_F64) { - double* d = (double*)ray_data(seg); + if (base == RAY_F64 || base == RAY_F32) { for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - sum += d[i]; cnt++; + if (base == RAY_F64) sum += ((double*)ray_data(seg))[i]; + else sum += (double)((float*)ray_data(seg))[i]; + cnt++; } } else { for (int64_t i = 0; i < seg->len; i++) { @@ -189,16 +192,16 @@ static ray_t* agg_parted_prod(ray_t* x) { return ray_error("type", "prod expects a numeric parted column, got %s", ray_type_name(base)); } ray_t** segs = (ray_t**)ray_data(x); - if (base == RAY_F64) { + if (base == RAY_F64 || base == RAY_F32) { double prod = 1.0; for (int64_t s = 0; s < x->len; s++) { ray_t* seg = segs[s]; if (!seg) continue; - double* d = (double*)ray_data(seg); int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - prod *= d[i]; + if (base == RAY_F64) prod *= ((double*)ray_data(seg))[i]; + else prod *= (double)((float*)ray_data(seg))[i]; } } return make_f64(prod); @@ -228,11 +231,12 @@ static ray_t* agg_parted_minmax(ray_t* x, int want_max) { ray_t* seg = segs[s]; if (!seg) continue; int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; - if (base == RAY_F64) { - double* d = (double*)ray_data(seg); + if (base == RAY_F64 || base == RAY_F32) { for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - double v = d[i]; + double v = base == RAY_F64 + ? ((double*)ray_data(seg))[i] + : (double)((float*)ray_data(seg))[i]; if (!found || (want_max ? v > best_f : v < best_f)) { best_f = v; found = 1; } @@ -249,18 +253,20 @@ static ray_t* agg_parted_minmax(ray_t* x, int want_max) { } if (!found) return ray_typed_null(-base); if (base == RAY_F64) return make_f64(best_f); + if (base == RAY_F32) return ray_f32((float)best_f); return agg_atom_i64_for_type(base, best_i); } static int agg_numeric_flat_type(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } static int agg_read_vec_f64(ray_t* v, int64_t i, double* out) { void* d = ray_data(v); switch (v->type) { case RAY_F64: *out = ((double*)d)[i]; return 1; + case RAY_F32: *out = (double)((float*)d)[i]; return 1; case RAY_I64: *out = (double)((int64_t*)d)[i]; return 1; case RAY_I32: *out = (double)((int32_t*)d)[i]; return 1; case RAY_I16: *out = (double)((int16_t*)d)[i]; return 1; @@ -362,7 +368,9 @@ static ray_t* agg_pair_vec(ray_t* x, ray_t* y, uint16_t op) { double xv = 0.0, yv = 0.0; agg_read_vec_f64(x, i, &xv); agg_read_vec_f64(y, i, &yv); - sx += xv; sy += yv; sxx += xv * xv; syy += yv * yv; sxy += xv * yv; cnt++; + sx += xv; sy += yv; sxx += xv * xv; syy += yv * yv; + sxy += xv * yv; + cnt++; } if (op == OP_WSUM) return ray_f64(ray_f64_fin(sxy)); @@ -449,10 +457,10 @@ ray_t* ray_sum_fn(ray_t* x) { for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "sum: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); if (RAY_ATOM_IS_NULL(elems[i])) { - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; continue; } - if (elems[i]->type == -RAY_F64) { has_float = 1; fsum += elems[i]->f64; } + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) { has_float = 1; fsum += elems[i]->f64; } else if (elems[i]->type == -RAY_I64) { isum += elems[i]->i64; fsum += (double)elems[i]->i64; } else { int64_t v = (int64_t)as_f64(elems[i]); isum += v; fsum += (double)v; } } @@ -479,10 +487,10 @@ ray_t* ray_prod_fn(ray_t* x) { if (!is_numeric(elems[i])) return ray_error("type", "prod: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); if (RAY_ATOM_IS_NULL(elems[i])) { - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; continue; } - if (elems[i]->type == -RAY_F64) { + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) { has_float = 1; fprod *= elems[i]->f64; } else { @@ -613,7 +621,7 @@ ray_t* ray_min_fn(ray_t* x) { double fmin = 0; int64_t imin = 0; for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "min: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; if (RAY_ATOM_IS_NULL(elems[i])) continue; double v = as_f64(elems[i]); if (!found || v < fmin) { fmin = v; imin = elems[i]->type == -RAY_I64 ? elems[i]->i64 : 0; found = 1; } @@ -667,7 +675,7 @@ ray_t* ray_max_fn(ray_t* x) { double fmax = 0; int64_t imax = 0; for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "max: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; if (RAY_ATOM_IS_NULL(elems[i])) continue; double v = as_f64(elems[i]); if (!found || v > fmax) { fmax = v; imax = elems[i]->type == -RAY_I64 ? elems[i]->i64 : 0; found = 1; } @@ -770,6 +778,9 @@ static ray_t* vec_to_f64_scratch(ray_t* x, double** out_vals) { } else if (x->type == RAY_F64) { double* d = (double*)ray_data(x); for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = d[i]; } + } else if (x->type == RAY_F32) { + float* d = (float*)ray_data(x); + for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = (double)d[i]; } } else if (x->type == RAY_I32) { int32_t* d = (int32_t*)ray_data(x); for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = (double)d[i]; } diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 34b4bf447..c4c4091d7 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -85,7 +85,8 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { if (!in || in->opcode != OP_SCAN) return false; ray_op_ext_t* ie = find_ext(g, in->id); ray_t* ic = ie ? ray_table_get_col(tbl, ie->sym) : NULL; - if (!ic || !agg_resolve(ext->agg_ops[a], ic->type)) return false; + const agg_vtable_t* vt = ic ? agg_resolve(ext->agg_ops[a], ic->type) : NULL; + if (!vt || vt->kind != ACC_STREAMING) return false; continue; /* admitted */ } if (ext->agg_ins2 && ext->agg_ins2[a] != RAY_OP_NONE) { @@ -110,7 +111,8 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { ray_op_ext_t* ie = find_ext(g, in->id); ray_t* ic = (ie) ? ray_table_get_col(tbl, ie->sym) : NULL; if (!ic) return false; - if (!agg_resolve(ext->agg_ops[a], ic->type)) return false; + const agg_vtable_t* vt = agg_resolve(ext->agg_ops[a], ic->type); + if (!vt || vt->kind != ACC_STREAMING) return false; } return true; } @@ -119,11 +121,10 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * Decides whether the key tuple packs into a bounded direct-index slot space * (gid = sum_k (key_k - min_k)*strides[k]) so grouping can skip hashing. * Eligible iff: 1..16 keys, every key is an integer/temporal/SYM type with no - * nulls, and the product of per-key ranges stays at or below DENSE_MAX_SLOTS - * (computed overflow-safely). Performs one min/max prescan per key column. + * nulls, and the product of per-key ranges is no larger than the input row + * count (computed overflow-safely). Performs one min/max prescan per key column. * Aggregates may be ACC_STREAMING or ACC_BUFFERED (median/top-k): the dense - * serial driver (agg_run_one) and the dense parallel path both carry the - * per-group destroy lifecycle, so buffered state is handled. */ + * serial driver carries the per-group destroy lifecycle for buffered state. */ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, const agg_vtable_t** vts, uint32_t n_aggs, int64_t nrows, dense_plan_t* out) { @@ -149,15 +150,21 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, if (kc->attrs & RAY_ATTR_HAS_NULLS) return false; if (nrows <= 0) return false; /* empty → no min/max, maxtype == RAY_SYM) { switch (kc->attrs & RAY_SYM_W_MASK) { - case RAY_SYM_W8: out->mins[k] = 0; out->ranges[k] = 256; continue; - case RAY_SYM_W16: out->mins[k] = 0; out->ranges[k] = 65536; continue; + case RAY_SYM_W8: + if (nrows >= (int64_t)UINT8_MAX + 1) { + out->mins[k] = 0; out->ranges[k] = (int64_t)UINT8_MAX + 1; continue; + } + break; + case RAY_SYM_W16: + if (nrows >= (int64_t)UINT16_MAX + 1) { + out->mins[k] = 0; out->ranges[k] = (int64_t)UINT16_MAX + 1; continue; + } + break; default: break; /* W32/W64: try domain bounds, else prescan */ } /* W32/W64 SYM codes are positions in [0, domain_count). When the @@ -167,7 +174,7 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, * domains still scan: their actual used range may be far tighter * than the full count, and the array would otherwise blow the cap. */ int64_t dc = ray_sym_domain_count(ray_sym_vec_domain(kc)); - if (dc > 0 && dc <= 65536 && dc <= nrows) { + if (dc > 0 && dc <= nrows) { out->mins[k] = 0; out->ranges[k] = dc; continue; } } @@ -210,17 +217,17 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, out->ranges[k] = mx - mn + 1; } - /* Composite packing; bail if a range is non-positive or the product would - * exceed the cap. Overflow-safe: check before multiplying. */ + /* Composite packing; keep dense state O(input). */ int64_t total = 1; + int64_t dense_limit = nrows < (int64_t)UINT32_MAX + ? nrows : (int64_t)UINT32_MAX; for (uint32_t k = 0; k < n_keys; k++) { int64_t rng = out->ranges[k]; if (rng <= 0) return false; - if (total > DENSE_MAX_SLOTS / rng) return false; /* would exceed cap */ + if (total > dense_limit / rng) return false; out->strides[k] = total; total *= rng; } - if (total > DENSE_MAX_SLOTS) return false; out->total_slots = total; out->ok = true; @@ -690,14 +697,15 @@ static bool agg_dense_plan_sel(ray_t** key_cols, uint32_t n_keys, int64_t n_sel, scratch_free(pre_hdr); int64_t total = 1; + int64_t dense_limit = n_sel < (int64_t)UINT32_MAX + ? n_sel : (int64_t)UINT32_MAX; for (uint32_t k = 0; k < n_keys; k++) { int64_t rng = out->ranges[k]; if (rng <= 0) return false; - if (total > DENSE_MAX_SLOTS / rng) return false; + if (total > dense_limit / rng) return false; out->strides[k] = total; total *= rng; } - if (total > DENSE_MAX_SLOTS) return false; out->total_slots = total; out->ok = true; return true; @@ -806,12 +814,9 @@ typedef struct { agg_local_t* locals; /* [nw] */ } agg_par_ctx_t; -/* (first_row, group idx) pair. The radix path packs (part << 32 | local gid) - * into .idx so finalize can recover both the partition and the in-partition - * state offset for each emitted group; .fr carries the representative row used - * to gather the key columns. Group-by output order is UNSPECIFIED — groups are - * emitted in build order, so there is no sort. */ -typedef struct { int64_t fr; int64_t idx; } agg_fr_pair_t; +/* Radix output order. idx packs (part << 32 | local gid) so finalize can + * recover both the partition and the in-partition state offset. */ +typedef struct { int64_t idx; } agg_radix_order_t; /* Phase A: per-worker local group + accumulate over chunk [start,end). */ static void agg_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { @@ -1408,19 +1413,16 @@ static ray_t* exec_group_v2_parallel_dense( } /* ══════════════════════════════════════════════════════════════════════ - * Parallel SMALL-HASH group-by (high-reduction sparse int/SYM keys). + * Parallel SMALL-HASH fallback for sparse int/SYM keys. * - * Targets the shape the dense path rejects (key range exceeds the dense cap) - * but that still has FEW distinct groups — e.g. 64 groups whose values span - * 0..6.3e9. Routing such a shape to RADIX scatters all N rows into 256 - * partitions for a handful of groups (the ~4.7× "scatter tax"); routing it to - * the generic hash path sizes every per-worker structure to N (htable, states, - * first_row). This path is the missing O(groups)-sized strategy: per-worker - * structures sized to the estimated CARDINALITY, growable on a wrong-low guess. + * This is the allocation-failure fallback for the deterministic radix route. + * Its per-worker structures start at one group and grow from observed + * cardinality, so a sparse, high-reduction input can still complete without a + * sampled cardinality estimate or a cache/RAM-derived routing crossover. * * Streaming aggs only (selector gates ACC_BUFFERED → radix). Per-worker hash: - * open-addressing, next_pow2(4*estimate) with a small floor, GROWABLE (rehash - * on load-factor exceed); states + first_row grow with the hash, never with N. + * open-addressing and growable on load-factor exceed; states + first_row grow + * with the hash, never with N. * Phase A interleaves probe + accumulate in fixed CHUNK_ROWS chunks so the gid * buffer is chunk-sized, not nrows-sized. Phase B merges per-worker hashes into * a global hash (≤ groups×nworkers entries — cheap). Phase C emits in build @@ -1428,92 +1430,6 @@ static ray_t* exec_group_v2_parallel_dense( * row per group, finalize each group's streaming state. * ══════════════════════════════════════════════════════════════════════ */ -/* Cardinality estimate via a bounded sample of the first rows. Inserts key - * tuples into a small open-addressing hash (reusing agg_tuple_hash/agg_tuple_eq - * for byte-consistency with the real grouping) and EARLY-EXITS the moment the - * distinct count exceeds t_route → returns t_route+1 to signal "high card" so - * high-card shapes (q10/S2) pay only the bounded sample, never a full pass. - * On no early-exit, returns the exact distinct count of the sample (capped at - * t_route). sample_rows is clamped to nrows. n_keys is uint32_t (unbounded: - * carries the untruncated ext->n_keys). */ -static int64_t agg_estimate_card(ray_t** key_cols, const void** key_data, - uint32_t n_keys, int64_t nrows, - int64_t sample_rows, int64_t t_route) { - int64_t ns = sample_rows < nrows ? sample_rows : nrows; - if (ns <= 0) return 0; - /* Hash sized to comfortably hold t_route distinct keys at <0.5 load. */ - int64_t htcap = 16; - while (htcap < (t_route + 1) * 4) htcap <<= 1; - uint64_t htmask = (uint64_t)htcap - 1; - int32_t* ht = ray_alloc_raw((size_t)htcap * sizeof(int32_t)); - int64_t* rep = ray_alloc_raw((size_t)(t_route + 2) * sizeof(int64_t)); /* gid -> sample row */ - if (!ht || !rep) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; /* assume high */ } - for (int64_t i = 0; i < htcap; i++) ht[i] = -1; - int64_t distinct = 0; - for (int64_t r = 0; r < ns; r++) { - uint64_t h = agg_tuple_hash(key_cols, key_data, n_keys, r); - uint64_t slot = h & htmask; - for (;;) { - int32_t gp = ht[slot]; - if (gp < 0) { - ht[slot] = (int32_t)distinct; - rep[distinct] = r; - distinct++; - if (distinct > t_route) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - break; - } - if (agg_tuple_eq(key_cols, key_data, n_keys, r, rep[gp])) break; - slot = (slot + 1) & htmask; - } - } - ray_free_raw(ht); ray_free_raw(rep); - return distinct; -} - -/* Sel-mode cardinality estimate: samples the first `sample_rows` SELECTED rows - * (decoded to ORIGINAL indices via the cursor) so the routing reflects the - * cardinality of the rows that actually survive the WHERE — not the full table. - * Same early-exit-above-t_route contract as agg_estimate_card. n_keys is - * uint32_t (unbounded), same contract as agg_estimate_card above. */ -static int64_t agg_estimate_card_sel(ray_t** key_cols, const void** key_data, - uint32_t n_keys, ray_t* sel, - const int64_t* sel_prefix, int64_t n_sel, - int64_t sample_rows, int64_t t_route) { - int64_t ns = sample_rows < n_sel ? sample_rows : n_sel; - if (ns <= 0) return 0; - int64_t htcap = 16; - while (htcap < (t_route + 1) * 4) htcap <<= 1; - uint64_t htmask = (uint64_t)htcap - 1; - int32_t* ht = ray_alloc_raw((size_t)htcap * sizeof(int32_t)); - int64_t* rep = ray_alloc_raw((size_t)(t_route + 2) * sizeof(int64_t)); - if (!ht || !rep) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - for (int64_t i = 0; i < htcap; i++) ht[i] = -1; - int64_t distinct = 0, seen = 0; - int64_t rows[AGG_SEL_CHUNK]; - agg_sel_cursor_t cur; - agg_sel_cursor_init(&cur, sel, sel_prefix, 0, ns); - int64_t cn; - while (seen < ns && (cn = agg_sel_cursor_next(&cur, rows)) > 0) { - for (int64_t i = 0; i < cn && seen < ns; i++, seen++) { - int64_t r = rows[i]; - uint64_t h = agg_tuple_hash(key_cols, key_data, n_keys, r); - uint64_t slot = h & htmask; - for (;;) { - int32_t gp = ht[slot]; - if (gp < 0) { - ht[slot] = (int32_t)distinct; rep[distinct] = r; distinct++; - if (distinct > t_route) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - break; - } - if (agg_tuple_eq(key_cols, key_data, n_keys, r, rep[gp])) break; - slot = (slot + 1) & htmask; - } - } - } - ray_free_raw(ht); ray_free_raw(rep); - return distinct; -} - /* Per-worker (and global merge) GROWABLE small-hash group table: open-addressing * hash on the tuple-hash → local gid; AoS per-group state blocks of `block` * bytes. Sized to cardinality (not N) and grown by rehash on load-factor. */ @@ -1530,7 +1446,7 @@ typedef struct { static int agg_sh_init(agg_sh_t* sh, int64_t cap, size_t block) { if (cap < 1) cap = 1; - int64_t htcap = 16; + int64_t htcap = 2; while (htcap < cap * 2) htcap <<= 1; /* <0.5 load at full cap */ sh->ng = 0; sh->cap = cap; sh->block = block; sh->oom = 0; sh->htcap = htcap; sh->htmask = (uint64_t)htcap - 1; @@ -1583,15 +1499,6 @@ static int agg_sh_grow_ht(agg_sh_t* sh, ray_t** key_cols, const void** key_data, } #define AGG_SH_CHUNK 4096 - -/* Adaptive promotion: the cardinality estimate is sample-based and can - * underestimate on clustered/ordered data, misrouting a genuinely high-card - * group to this small-hash path where the per-worker hash thrashes cache. - * Guard against it WITHOUT trusting the estimate: when any worker's live group - * count crosses this threshold the per-worker hash no longer fits cache, so we - * abort and re-run on the radix path (data-driven, no sampling bias). Set well - * above the smallhash/radix crossover so medium-card groups are never promoted. */ -#define AGG_SH_PROMOTE (1 << 16) static ray_t* exec_group_v2_parallel_radix( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, @@ -1608,9 +1515,7 @@ typedef struct { uint32_t n_aggs; const void** val_data; const int8_t* val_types; const bool* val_hasnull; const uint8_t* val_esz; const void** val2_data; const int8_t* val2_types; const bool* val2_hasnull; const uint8_t* val2_esz; - int64_t est_card; /* sizing hint for per-worker hashes */ agg_sh_t* locals; /* [nw] */ - _Atomic(int) overflow; /* set when any worker crosses AGG_SH_PROMOTE */ /* Sel-mode (pushed WHERE filter): chunk-iterate selected rows of [0,n_sel). */ ray_t* sel; const int64_t* sel_prefix; @@ -1670,10 +1575,6 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, start, end); int64_t cn; while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) { - /* Adaptive promotion trip-wire: too many distinct keys for cache, or - * another worker already tripped → abort to the radix path. */ - if (sh->ng > AGG_SH_PROMOTE) atomic_store_explicit(&c->overflow, 1, memory_order_relaxed); - if (atomic_load_explicit(&c->overflow, memory_order_relaxed)) { agg_sel_scratch_free(&sc); return; } for (int64_t i = 0; i < cn; i++) { int32_t g = agg_sh_find_or_insert(sh, c->key_cols, c->key_data, c->n_keys, c->vts, c->off, c->n_aggs, rows[i]); @@ -1689,11 +1590,6 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en } for (int64_t cs = start; cs < end; cs += AGG_SH_CHUNK) { - /* Adaptive promotion trip-wire (see AGG_SH_PROMOTE): per-worker hash no - * longer fits cache, or another worker already tripped → abort so the - * caller re-runs on radix. */ - if (sh->ng > AGG_SH_PROMOTE) atomic_store_explicit(&c->overflow, 1, memory_order_relaxed); - if (atomic_load_explicit(&c->overflow, memory_order_relaxed)) return; int64_t ce = cs + AGG_SH_CHUNK; if (ce > end) ce = end; int64_t n = ce - cs; for (int64_t r = cs; r < ce; r++) { @@ -1723,14 +1619,13 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en } } -/* Parallel small-hash path. Precondition: int/SYM keys, all aggs ACC_STREAMING, - * estimated cardinality LOW (caller-gated). est_card sizes the per-worker - * hashes; they grow if the estimate proves too low. */ +/* Parallel small-hash fallback. Precondition: int/SYM keys and streaming + * accumulators. Hashes start at one group and grow from observed cardinality. */ static ray_t* exec_group_v2_parallel_smallhash( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, - int64_t est_card, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { + ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { ray_op_ext_t* ext = find_ext(g, op->id); uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; ray_pool_t* pool = ray_pool_get(); @@ -1745,18 +1640,11 @@ static ray_t* exec_group_v2_parallel_smallhash( const bool* val2_hasnull = d.val2_hasnull; const uint8_t* val2_esz = d.val2_esz; const int64_t* agg_syms = d.agg_syms; - /* Per-worker hash sized to ~4× the estimated cardinality (floor 256) — the - * cap, not N. Grows by rehash if a worker sees more distinct keys than the - * sample predicted (rare-key tail). Cap is bounded by nrows for safety. */ - int64_t cap0 = est_card * 4; - if (cap0 < 256) cap0 = 256; - if (nrows > 0 && cap0 > nrows) cap0 = nrows; - agg_sh_t* locals = ray_calloc_raw((size_t)((size_t)nw) * (sizeof(agg_sh_t))); if (!locals) { agg_desc_free(&d); return ray_error("oom", NULL); } int alloc_oom = 0; for (uint32_t w = 0; w < nw; w++) - if (agg_sh_init(&locals[w], cap0, block) != 0) { alloc_oom = 1; break; } + if (agg_sh_init(&locals[w], 1, block) != 0) { alloc_oom = 1; break; } if (alloc_oom) { for (uint32_t w = 0; w < nw; w++) agg_sh_destroy(&locals[w]); ray_free_raw(locals); @@ -1771,7 +1659,7 @@ static ray_t* exec_group_v2_parallel_smallhash( .val_hasnull = val_hasnull, .val_esz = val_esz, .val2_data = val2_data, .val2_types = val2_types, .val2_hasnull = val2_hasnull, .val2_esz = val2_esz, - .est_card = est_card, .locals = locals, .overflow = 0, + .locals = locals, .sel = sel, .sel_prefix = sel_prefix, .vd = { .n_aggs = n_aggs, .vts = vts, .off = off, .block = block, .val_data = val_data, .val_types = val_types, .val_hasnull = val_hasnull, .val_esz = val_esz, @@ -1779,17 +1667,6 @@ static ray_t* exec_group_v2_parallel_smallhash( }; ray_pool_dispatch(pool, agg_sh_phaseA_fn, &ctx, sel ? n_sel : nrows); - /* Adaptive promotion: a worker found the live group count too high for the - * per-worker hash (estimate underestimated). Discard the partial small-hash - * state and re-run on the radix path — data-driven, no sampling bias. */ - if (atomic_load_explicit(&ctx.overflow, memory_order_relaxed)) { - for (uint32_t w = 0; w < nw; w++) agg_sh_destroy(&locals[w]); - ray_free_raw(locals); - agg_desc_free(&d); - return exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, - vts, off, block, sel, sel_prefix, n_sel); - } - for (uint32_t w = 0; w < nw; w++) if (locals[w].oom) { for (uint32_t i = 0; i < nw; i++) agg_sh_destroy(&locals[i]); @@ -1874,14 +1751,15 @@ static ray_t* exec_group_v2_parallel_smallhash( /* ══════════════════════════════════════════════════════════════════════ * Parallel RADIX group-by (high-card int/SYM keys, ACC_STREAMING aggs only). * - * Partition rows by key-hash into AGG_RADIX_P disjoint partitions: every row + * Partition rows by key-hash into a worker-derived power-of-two partition set: + * every row * whose tuple-hash maps to partition p lands in p across ALL workers (same key * → same hash → same partition). Partitions therefore hold DISJOINT key sets, * so each can be grouped+accumulated fully in parallel with NO cross-partition * merge — eliminating the serial Phase-B bottleneck of the hash path. * * Phase 1 (parallel over rows): scatter each row's INDEX into a per-(worker, - * partition) growable int64 buffer keyed by RADIX_PART(hash). + * partition) growable int64 buffer keyed by the hash partition. * Phase 2 (parallel over partitions): gather all workers' index buffers for * partition p; build a small open-addressing hash over those rows * (keys disjoint from other partitions); assign partition-local gids @@ -1895,7 +1773,15 @@ static ray_t* exec_group_v2_parallel_smallhash( * Streaming aggs only (selector guarantees it) → no buffered destroy lifecycle. * ══════════════════════════════════════════════════════════════════════ */ -#define AGG_RADIX_P 256 +static uint32_t agg_radix_part_count(uint32_t nworkers, int64_t nrows) { + uint32_t n = 1; + uint64_t rows = nrows > 0 ? (uint64_t)nrows : 1; + while ((n < nworkers || + (uint64_t)n < rows / n + (rows % n != 0)) && + n <= UINT32_MAX / 2) + n <<= 1; + return n; +} /* Growable contiguous PAYLOAD buffer (per worker, per partition). Phase 1 * scatters one fixed-size record per row instead of a bare row index: the @@ -1910,7 +1796,8 @@ typedef struct { char* buf; uint32_t n, cap; } agg_pay_buf_t; /* n = #records * /* Reserve room for one more record of `rec` bytes; returns dest ptr or NULL. */ static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec) { if (b->n == b->cap) { - uint32_t nc = b->cap ? b->cap * 2 : 64; + if (b->cap > UINT32_MAX / 2) return NULL; + uint32_t nc = b->cap ? b->cap * 2 : 1; char* nb = ray_realloc_raw(b->buf, (size_t)nc * rec); if (!nb) return NULL; b->buf = nb; b->cap = nc; @@ -1921,7 +1808,7 @@ static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec) { /* Per-partition result slot (filled by Phase 2). */ typedef struct { char* states; /* [ng * block] AoS group state (every group init'd) */ - int64_t* first_row; /* [ng] MIN row idx per partition-local group */ + int64_t* first_row; /* [ng] MIN logical input position per local group */ int64_t* keys; /* [ng * n_keys] packed keys (build order) for the * representative record of each group — Phase 3 emits * the key columns by un-packing these SEQUENTIALLY @@ -1955,10 +1842,7 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, /* Build a result key column of src_col's type for the radix path by un-packing * the per-group packed key value (column key_idx of each group's representative - * record) SEQUENTIALLY from the per-partition `keys` buffers, in emit order - * (partition-major, then partition-local gid). This replaces the scattered - * gather of the original key column at first_row[gi] — for huge ngroups the - * sequential read is far more cache-friendly. + * record) from the per-partition `keys` buffers in stable first-seen order. * * agg_read_key_i64 widened each key into int64 (sign-extend for signed types, * zero-extend for U8/BOOL/SYM intern ids); write_col_i64 is its exact inverse, @@ -1970,7 +1854,7 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, * agg_v2_can_handle) and key_idx indexes the packed-key stride below. */ static ray_t* agg_unpack_key_col(ray_t* src_col, uint32_t key_idx, uint32_t n_keys, const agg_radix_part_t* parts, uint32_t nparts, - int64_t n) { + const agg_radix_order_t* pairs, int64_t n) { ray_t* out = col_vec_new(src_col, n); if (!out || RAY_IS_ERR(out)) return out; if (out->type == RAY_SYM) @@ -1978,12 +1862,12 @@ static ray_t* agg_unpack_key_col(ray_t* src_col, uint32_t key_idx, uint32_t n_ke out->len = n; int8_t type = src_col->type; uint8_t attrs = src_col->attrs; void* dst = ray_data(out); - int64_t i = 0; - for (uint32_t p = 0; p < nparts; p++) { - const int64_t* keys = parts[p].keys; - int64_t png = parts[p].ng; - for (int64_t gg = 0; gg < png; gg++, i++) - write_col_i64(dst, i, keys[(size_t)gg * n_keys + key_idx], type, attrs); + (void)nparts; + for (int64_t i = 0; i < n; i++) { + uint32_t p = (uint32_t)(pairs[i].idx >> 32); + uint32_t gg = (uint32_t)pairs[i].idx; + write_col_i64(dst, i, + parts[p].keys[(size_t)gg * n_keys + key_idx], type, attrs); } if (src_col->attrs & RAY_ATTR_HAS_NULLS) out->attrs |= RAY_ATTR_HAS_NULLS; return out; @@ -2000,26 +1884,25 @@ typedef struct { const void** val_data; const int8_t* val_types; const bool* val_hasnull; const uint8_t* val_esz; const void** val2_data; const int8_t* val2_types; const bool* val2_hasnull; const uint8_t* val2_esz; uint32_t nw; - agg_pay_buf_t* bufs; /* [nw * AGG_RADIX_P] payload records */ - agg_radix_part_t* parts; /* [AGG_RADIX_P] */ + uint32_t n_parts; + agg_pay_buf_t* bufs; /* [nw * n_parts] payload records */ + agg_radix_part_t* parts; /* [n_parts] */ int phase1_oom; /* set by any Phase-1 worker on push failure */ /* Per-row payload record layout (bytes), computed once by the caller: * [0 .. n_keys*8) packed keys (each widened via agg_read_key_i64) * [val_off[a] ..] agg a's input value at native esz (if val_data[a]) * [val2_off[a] ..] agg a's 2nd input (pearson) at native esz - * [row_off ..] int64 source row index (for first_row) */ + * [row_off ..] int64 logical input position */ size_t rec; /* total record size, 8-aligned */ const size_t* val_off; /* [n_aggs], carved by the caller */ const size_t* val2_off; /* [n_aggs], carved by the caller */ size_t row_off; - /* When false, NO admitted aggregate consults the original row (first_row is - * dead): the record omits its 8-byte row index and Phase-2/3 skip it. Set - * once from the agg kinds; row-dependent aggs (FIRST/LAST) flip it true. */ + /* Stable group ordering records one logical input position per payload. */ bool needs_row; /* Sel-mode (pushed WHERE filter): when sel != NULL, scatter the SELECTED * rows of [0,n_sel) (decoded to ORIGINAL row indices) instead of [start,end). - * The packed record's row_off ALWAYS stores the ORIGINAL decoded row, so - * first_row / key-unpack stay correct in original-row space. */ + * The packed record stores the selected-row ordinal, while column reads use + * the decoded original row. */ ray_t* sel; const int64_t* sel_prefix; /* Per-WORKER key-staging row: [nw * n_keys] int64. The scatter reads all @@ -2033,7 +1916,8 @@ typedef struct { /* Scatter one ORIGINAL row r's packed payload record into the worker's per- * partition buffer. Returns 0 or -1 on push OOM (sets phase1_oom). */ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, - int64_t* kv, int64_t r) { + int64_t* kv, int64_t r, + int64_t input_order) { uint32_t n_keys = c->n_keys, n_aggs = c->n_aggs; /* kv: this worker's key-staging slice (c->kv_scratch + wid*n_keys), sized * for any key count — no fixed [16] cap. */ @@ -2043,7 +1927,7 @@ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, kv[k] = v; h ^= (uint64_t)v; h *= 1099511628211ULL; } - uint32_t p = (uint32_t)(h & (AGG_RADIX_P - 1)); + uint32_t p = (uint32_t)(h & (c->n_parts - 1)); char* rec = agg_pay_reserve(&my[p], c->rec); if (!rec) { c->phase1_oom = 1; return -1; } int64_t* kdst = (int64_t*)rec; @@ -2058,7 +1942,8 @@ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, memcpy(rec + c->val2_off[a], (const char*)c->val2_data[a] + (size_t)r * ez2, ez2); } } - if (c->needs_row) *(int64_t*)(rec + c->row_off) = r; /* loop-invariant branch */ + if (c->needs_row) + *(int64_t*)(rec + c->row_off) = input_order; return 0; } @@ -2077,21 +1962,23 @@ RAY_INLINE void agg_radix_scatter_range(agg_radix_ctx_t* c, agg_pay_buf_t* my, int64_t rows[AGG_SEL_CHUNK]; agg_sel_cursor_t cur; agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, start, end); + int64_t input_order = start; int64_t cn; while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) for (int64_t i = 0; i < cn; i++) - if (agg_radix_scatter_one(c, my, kv, rows[i]) != 0) return; + if (agg_radix_scatter_one(c, my, kv, rows[i], + input_order++) != 0) return; return; } for (int64_t r = start; r < end; r++) - if (agg_radix_scatter_one(c, my, kv, r) != 0) return; + if (agg_radix_scatter_one(c, my, kv, r, r) != 0) return; } /* Phase 1: scatter one packed payload record per row into per-(worker, - * partition) contiguous buffers, keyed by RADIX_PART(tuple hash). */ + * partition) contiguous buffers, keyed by the tuple-hash partition. */ static void agg_radix_scatter_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { agg_radix_ctx_t* c = (agg_radix_ctx_t*)vctx; - agg_pay_buf_t* my = &c->bufs[(size_t)wid * AGG_RADIX_P]; + agg_pay_buf_t* my = &c->bufs[(size_t)wid * c->n_parts]; /* Key-staging row for this worker. The common bounded case (n_keys <= 16) * stages through a STACK array: a compiler-opaque heap slice defeated the * scalarization/register-promotion of the per-row kv[k]=v stage (flat @@ -2127,7 +2014,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t /* Total rows hashing to p across all workers. */ int64_t total = 0; for (uint32_t w = 0; w < c->nw; w++) - total += c->bufs[(size_t)w * AGG_RADIX_P + p].n; + total += c->bufs[(size_t)w * c->n_parts + p].n; if (total == 0) continue; /* Open-addressing hash sized to next_pow2(2*total). */ @@ -2179,7 +2066,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t int64_t ng = 0, ri = 0; for (uint32_t w = 0; w < c->nw; w++) { - agg_pay_buf_t* b = &c->bufs[(size_t)w * AGG_RADIX_P + p]; + agg_pay_buf_t* b = &c->bufs[(size_t)w * c->n_parts + p]; const char* rec = b->buf; for (uint32_t i = 0; i < b->n; i++, rec += c->rec) { const int64_t* keys = (const int64_t*)rec; @@ -2300,7 +2187,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t * SERIALLY by the caller (q10 and other scalar shapes get the full win). */ typedef struct { agg_radix_part_t* parts; - const agg_fr_pair_t* pairs; /* [ng], sorted by first_row */ + const agg_radix_order_t* pairs; /* [ng], stable first-seen order */ const agg_vtable_t** vts; const size_t* off; size_t block; @@ -2365,19 +2252,9 @@ static ray_t* exec_group_v2_parallel_radix( const bool* val2_hasnull = d.val2_hasnull; const uint8_t* val2_esz = d.val2_esz; const int64_t* agg_syms = d.agg_syms; - /* Row-dependent aggregates (FIRST/LAST) consult the representative original - * row; every other admitted agg (COUNT/SUM/AVG/MIN/MAX/VAR/STDDEV/PEARSON/ - * MEDIAN/TOP/BOT) accumulates from gathered values and never touches it, and - * the key columns are un-packed from the packed keys — so first_row/row_idx - * are dead weight. Drop them from the record layout when none is needed. - * NOTE: today FIRST/LAST never reach this kernel (agg_resolve() returns NULL - * for them, so the v2 admission gate rejects them), so needs_row is always - * false here — the scan below is a forward-guard, NOT a working restore path: - * admitting a row-dependent agg would also require finalize wiring to read - * the row, not just this layout flag. */ - bool needs_row = false; - for (uint32_t a = 0; a < n_aggs; a++) - if (ext->agg_ops[a] == OP_FIRST || ext->agg_ops[a] == OP_LAST) { needs_row = true; break; } + /* Every group records its earliest source row so emitted order does not + * depend on the number of radix partitions used by this execution. */ + bool needs_row = true; /* Per-row payload record layout: [packed keys][agg values][row_idx?]. * val_off/val2_off are per-dispatch (filled once here, read by workers). @@ -2398,13 +2275,15 @@ static ray_t* exec_group_v2_parallel_radix( size_t row_off = rec_cur; if (needs_row) rec_cur += 8; size_t rec = (rec_cur + 7u) & ~(size_t)7u; /* 8-align records */ - size_t nbuf = (size_t)nw * AGG_RADIX_P; + uint32_t n_parts = agg_radix_part_count(nw, sel ? n_sel : nrows); + size_t nbuf = (size_t)nw * n_parts; agg_pay_buf_t* bufs = ray_calloc_raw((size_t)(nbuf) * (sizeof(agg_pay_buf_t))); - agg_radix_part_t* parts = ray_calloc_raw((size_t)(AGG_RADIX_P) * (sizeof(agg_radix_part_t))); + agg_radix_part_t* parts = ray_calloc_raw((size_t)n_parts * sizeof(agg_radix_part_t)); if (!bufs || !parts) { ray_free_raw(bufs); ray_free_raw(parts); scratch_free(off_hdr); agg_desc_free(&d); - return ray_error("oom", NULL); + return exec_group_v2_parallel_smallhash(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); } agg_radix_ctx_t ctx = { @@ -2412,7 +2291,8 @@ static ray_t* exec_group_v2_parallel_radix( .vts = vts, .off = off, .block = block, .n_aggs = n_aggs, .val_data = val_data, .val_types = val_types, .val_hasnull = val_hasnull, .val_esz = val_esz, .val2_data = val2_data, .val2_types = val2_types, .val2_hasnull = val2_hasnull, .val2_esz = val2_esz, - .nw = nw, .bufs = bufs, .parts = parts, .phase1_oom = 0, + .nw = nw, .n_parts = n_parts, + .bufs = bufs, .parts = parts, .phase1_oom = 0, .rec = rec, .row_off = row_off, .needs_row = needs_row, .sel = sel, .sel_prefix = sel_prefix, .val_off = val_off, .val2_off = val2_off, @@ -2426,55 +2306,71 @@ static ray_t* exec_group_v2_parallel_radix( /* Phase 2: per-partition group+accumulate. */ int oom = ctx.phase1_oom; if (!oom) - ray_pool_dispatch_n(pool, agg_radix_group_fn, &ctx, AGG_RADIX_P); + ray_pool_dispatch_n(pool, agg_radix_group_fn, &ctx, n_parts); if (!oom) - for (uint32_t p = 0; p < AGG_RADIX_P; p++) + for (uint32_t p = 0; p < n_parts; p++) if (parts[p].oom) { oom = 1; break; } /* Phases 1+2 done (both dispatches joined); val_off/val2_off no longer read. */ scratch_free(off_hdr); if (oom) { - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); - return ray_error("oom", NULL); + return exec_group_v2_parallel_smallhash(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); } - /* Phase 3: emit in natural partition/group order. ── - * Group-by output order is UNSPECIFIED, so we emit groups in partition order - * then partition-local build order (no global sort by first_row). The - * (part<<32 | gid) idx encoding locates each group's state during finalize; - * key columns are un-packed sequentially from the contiguous packed-key - * buffers (agg_unpack_key_col), not gathered from scattered first_row[]. */ + /* Phase 3: restore stable first-seen order across partitions. */ int64_t ng = 0; - for (uint32_t p = 0; p < AGG_RADIX_P; p++) ng += parts[p].ng; - - /* (first_row, part, local gid) collected globally in build order. idx packs - * (part << 32 | local gid) so finalize recovers both the partition and the - * in-partition state offset. */ - agg_fr_pair_t* pairs = ray_alloc_raw((size_t)(ng > 0 ? ng : 1) * sizeof(agg_fr_pair_t)); + for (uint32_t p = 0; p < n_parts; p++) ng += parts[p].ng; + + /* Place each group directly at its first logical input position, then + * compact those occupied positions in one linear pass. This restores + * first-seen order without a comparison sort or a routing threshold. */ + int64_t input_count = sel ? n_sel : nrows; + agg_radix_order_t* pairs = ray_alloc_raw( + (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); if (!pairs) { - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); return ray_error("oom", NULL); } - { int64_t i = 0; - for (uint32_t p = 0; p < AGG_RADIX_P; p++) - for (int64_t gg = 0; gg < parts[p].ng; gg++) { - pairs[i].fr = parts[p].first_row ? parts[p].first_row[gg] : 0; /* unused by emit */ - pairs[i].idx = ((int64_t)p << 32) | (uint32_t)gg; /* part | gid */ - i++; - } + for (int64_t i = 0; i < input_count; i++) pairs[i].idx = -1; + bool order_ok = true; + for (uint32_t p = 0; p < n_parts && order_ok; p++) { + for (int64_t gg = 0; gg < parts[p].ng; gg++) { + int64_t first = parts[p].first_row[gg]; + if (first < 0 || first >= input_count || pairs[first].idx != -1) { + order_ok = false; + break; + } + pairs[first].idx = ((int64_t)p << 32) | (uint32_t)gg; + } + } + int64_t ordered = 0; + if (order_ok) { + for (int64_t i = 0; i < input_count; i++) + if (pairs[i].idx != -1) pairs[ordered++].idx = pairs[i].idx; + order_ok = ordered == ng; + } + if (!order_ok) { + ray_free_raw(pairs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); ray_free_raw(parts); + agg_desc_free(&d); + return ray_error("group", "failed to order radix groups"); } ray_t* result = ray_table_new(n_keys + n_aggs); if (!result || RAY_IS_ERR(result)) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2486,10 +2382,11 @@ static ray_t* exec_group_v2_parallel_radix( * original columns at first_row[]. Byte-identical to agg_gather_key_col * (see agg_unpack_key_col), incl. SYM payload + domain. */ for (uint32_t k = 0; k < n_keys; k++) { - ray_t* kc = agg_unpack_key_col(key_cols[k], k, n_keys, parts, AGG_RADIX_P, ng); + ray_t* kc = agg_unpack_key_col(key_cols[k], k, n_keys, parts, n_parts, + pairs, ng); if (!kc || RAY_IS_ERR(kc)) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2507,7 +2404,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_t** outs = (ray_t**)scratch_calloc(&outs_hdr, (size_t)n_aggs * (sizeof(ray_t*) + sizeof(int64_t))); if (!outs) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2522,7 +2419,7 @@ static ray_t* exec_group_v2_parallel_radix( if (!out || RAY_IS_ERR(out)) { for (uint32_t b = 0; b < a; b++) ray_release(outs[b]); ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); scratch_free(outs_hdr); agg_desc_free(&d); @@ -2594,7 +2491,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_free_raw(pairs); /* Finalize is done reading every partition's buffered group state → destroy * exactly once before freeing the partition slabs. */ - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); scratch_free(outs_hdr); agg_desc_free(&d); @@ -2680,28 +2577,28 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * count when a filter is active, else the full table. */ int64_t eff_n = sel ? n_sel : nrows; + /* Buffered accumulators retain every contributing value per group. Running + * them through the parallel radix strategy overlaps those buffers with the + * full scatter payload and can exhaust the heap on large inputs. Keep + * buffered shapes on the serial v2 driver; streaming shapes retain all + * parallel strategies below. */ + bool all_streaming = true; + for (uint32_t a = 0; a < ext->n_aggs && all_streaming; a++) + if (vts[a]->kind != ACC_STREAMING) all_streaming = false; + ray_pool_t* pool = ray_pool_get(); - if (pool && eff_n >= RAY_PARALLEL_THRESHOLD) { - /* Per-worker memory budget gate: dense parallel allocates a full - * total_slots*(block + first_row) slab per worker. Low-card (the perf - * target) has tiny total_slots → always within budget → dense parallel. - * Mid-card-over-budget dense plans fall back to hash parallel (correct). */ - /* Compute per_worker_bytes ONLY when the dense plan is valid: a bailed - * plan (dp.ok == false) leaves dp.total_slots unset, and the multiply - * would overflow on garbage (UBSan signed-overflow). Short-circuit on - * dp.ok first. */ - bool dense_par_ok = false; - if (dp.ok) { - int64_t per_worker_bytes = dp.total_slots * (int64_t)(block + 8 /*first_row*/); - dense_par_ok = per_worker_bytes <= (8LL << 20); /* 8 MB/worker cap */ - } + if (pool && eff_n >= RAY_PARALLEL_THRESHOLD && all_streaming) { + /* A parallel dense plan owns one slot slab per worker. Admit it only + * when the aggregate slot count across all workers remains O(input); + * this is data-derived and independent of cache or RAM size. */ + uint32_t dense_workers = ray_pool_total_workers(pool); + bool dense_par_ok = dp.ok && dense_workers > 0 + && dp.total_slots <= eff_n / (int64_t)dense_workers; /* RADIX eligibility: every key an int/SYM type with no nulls (same * type-set check as agg_dense_plan). Radix takes the high-card - * remainder of int-key queries (dense handles the low-card head), - * for BOTH streaming and buffered (median/top) aggs — radix runs the - * full init/update_batch/finalize/destroy lifecycle. Hash handles - * F64 / STR keys. */ + * remainder of streaming int-key queries (dense handles the low-card + * head). Hash handles F64 / STR keys. */ bool keys_intsym = true; for (uint32_t k = 0; k < ext->n_keys && keys_intsym; k++) { ray_t* kc = key_cols[k]; @@ -2714,67 +2611,17 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (kc->attrs & RAY_ATTR_HAS_NULLS) keys_intsym = false; } - /* All aggregates streaming? (Buffered median/top-k keep radix — its - * per-group destroy lifecycle handles them; the small-hash path is - * streaming-only.) */ - bool all_streaming = true; - for (uint32_t a = 0; a < ext->n_aggs && all_streaming; a++) - if (vts[a]->kind != ACC_STREAMING) all_streaming = false; - if (dense_par_ok) { ray_t* r = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, sel, sel_prefix, n_sel); agg_vo_free(&vo); scratch_free(kc_hdr); return r; } if (keys_intsym) { - /* Dense-FAIL int/SYM streaming shapes: probe cardinality on a bounded - * sample. LOW card → small-hash (O(groups) working set, no scatter - * tax); HIGH card → radix. The estimate early-exits above T_route so - * high-card shapes pay only the bounded sample. In sel mode the - * sample is taken over the SELECTED rows (the cardinality that - * actually reaches grouping), so a high-selectivity filter over a - * high-card column still routes to small-hash when the surviving - * rows are few-and-low-card. */ - if (all_streaming) { - /* Exact-size carve (unbounded keys) for the estimate's key-data - * pointers; freed at the smallhash return and on fall-through. */ - ray_t* kd_hdr; - const void** key_data = (const void**)scratch_alloc(&kd_hdr, (size_t)ext->n_keys * sizeof(void*)); - if (!key_data) { agg_vo_free(&vo); scratch_free(kc_hdr); return ray_error("oom", NULL); } - for (uint32_t k = 0; k < ext->n_keys; k++) key_data[k] = ray_data(key_cols[k]); - /* T_ROUTE: the group cardinality where the small-hash - * per-worker working set stops fitting cache and radix's - * partitioned scatter wins. Small-hash keeps one hash - * table of `groups` entries per worker (≈ groups × - * (key + state) bytes) and probes it per input row; while - * that table stays resident in fast cache the probes are - * near-free and small-hash beats radix's extra - * partition/scatter pass. Once `groups` grows past cache - * residency the probes start missing to DRAM and per-row - * cost climbs steeply, whereas radix is cardinality-flat - * (it partitions keys into cache-sized buckets up front). - * Measured 1-key 10M-row sweep on this class of machine: - * the streaming-SUM crossover lands right at ~16384 groups - * (smallhash ≈ radix there; smallhash ~2x faster below, - * ~2x slower above), which is ~0.4 MB of per-worker table — - * the L2-residency edge. Lighter aggs (COUNT) cross a bit - * lower; since radix degrades gracefully past the crossover - * while small-hash blows up, we sit at the high end of that - * band (16384) so a misroute costs little. */ - const int64_t T_ROUTE = 16384; - int64_t est = sel - ? agg_estimate_card_sel(key_cols, key_data, ext->n_keys, sel, sel_prefix, n_sel, 65536, T_ROUTE) - : agg_estimate_card(key_cols, key_data, ext->n_keys, nrows, 65536, T_ROUTE); - if (est <= T_ROUTE) { - ray_t* r = exec_group_v2_parallel_smallhash(g, op, tbl, nrows, - key_cols, key_syms, vts, off, block, est, sel, sel_prefix, n_sel); - scratch_free(kd_hdr); agg_vo_free(&vo); scratch_free(kc_hdr); return r; - } - scratch_free(kd_hdr); /* fall through to radix */ - } - { ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, vts, off, block, - sel, sel_prefix, n_sel); - agg_vo_free(&vo); scratch_free(kc_hdr); return r; } + /* Sparse integer/SYM ranges use radix deterministically. No sampled + * cardinality or cache-size crossover is baked into routing. */ + ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); + agg_vo_free(&vo); scratch_free(kc_hdr); return r; } /* Hash fallback (F64 / STR keys): not a chunked strategy — compact. */ if (sel) AGG_RUN_COMPACT_FALLBACK(); diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 1868a0bf9..2d66c877c 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -61,10 +61,9 @@ ray_t* agg_run_one(const agg_vtable_t* vt, ray_t* val_col, const uint32_t* gids, int64_t nrows, int64_t ngroups, int64_t kparam); -/* ── Dense grouping eligibility selector (low-cardinality int/SYM keys) ── - * Mirrors the old direct-array group path's cap. When dense applies, a group - * id is the packed key offset (O(1) direct index) rather than a hash slot. */ -#define DENSE_MAX_SLOTS 262144 /* mirror the old DA path's cap */ +/* ── Dense grouping eligibility selector (compact-range int/SYM keys) ── + * When dense applies, a group id is the packed key offset (O(1) direct index) + * rather than a hash slot. */ typedef struct { bool ok; @@ -77,8 +76,8 @@ typedef struct { /* Decide if dense grouping applies to (key_cols, aggs). Eligible iff: * - every key type in {I64,I32,I16,U8,BOOL,DATE,TIME,TIMESTAMP,SYM} and NOT HAS_NULLS - * - every agg vtable is ACC_STREAMING (no buffered median/top) - * - product of per-key ranges <= DENSE_MAX_SLOTS (no overflow) + * - product of per-key ranges is no larger than the contributing row count + * (so dense state is O(input), never controlled by a machine-size budget) * Does one min/max prescan over the key columns. Sets out->ok accordingly. * n_keys is uint32_t (untruncated ext->n_keys): dense direct-index routing * self-limits to <=16 keys here (wider shapes are rejected to v2's unbounded diff --git a/src/ops/arith.c b/src/ops/arith.c index 03cd50bc2..6c1e7c17d 100644 --- a/src/ops/arith.c +++ b/src/ops/arith.c @@ -32,7 +32,7 @@ typedef ray_op_t* (*arith_unary_ctor)(ray_graph_t*, ray_op_t*); static bool arith_numeric_vec_type(int8_t t) { if (RAY_IS_PARTED(t)) t = (int8_t)RAY_PARTED_BASETYPE(t); return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } static ray_t* arith_unary_dag_vec(ray_t* x, arith_unary_ctor ctor) { @@ -74,7 +74,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { /* Vector fast path — only when at least one operand is a typed vector */ /* Temporal + integer arithmetic (only int types, not float) */ - if (is_temporal(a) && is_numeric(b) && b->type != -RAY_F64) { + if (is_temporal(a) && is_numeric(b) && + b->type != -RAY_F64 && b->type != -RAY_F32) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(a->type); @@ -83,7 +84,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { if (a->type == -RAY_TIME) return ray_time(a->i64 + v); if (a->type == -RAY_TIMESTAMP) return ray_timestamp(a->i64 + v); } - if (is_numeric(a) && a->type != -RAY_F64 && is_temporal(b)) { + if (is_numeric(a) && a->type != -RAY_F64 && + a->type != -RAY_F32 && is_temporal(b)) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(b->type); @@ -93,7 +95,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { if (b->type == -RAY_TIMESTAMP) return ray_timestamp(b->i64 + v); } /* Reject float + temporal */ - if ((a->type == -RAY_F64 && is_temporal(b)) || (is_temporal(a) && b->type == -RAY_F64)) + if (((a->type == -RAY_F64 || a->type == -RAY_F32) && is_temporal(b)) || + (is_temporal(a) && (b->type == -RAY_F64 || b->type == -RAY_F32))) return ray_error("type", "add: temporal arithmetic requires integer offsets, got %s and %s", ray_type_name(a->type), ray_type_name(b->type)); /* Reject null_numeric + temporal (for null floats etc) */ @@ -132,6 +135,7 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { ray_type_name(a->type), ray_type_name(b->type)); /* Null propagation */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return null_for_promoted(a, b); + if (both_f32_atoms(a, b)) return ray_f32((float)(as_f64(a) + as_f64(b))); if (is_float_op(a, b)) return make_f64(as_f64(a) + as_f64(b)); int8_t rt = promote_int_type(a, b); return make_typed_int(rt, as_i64(a) + as_i64(b)); @@ -204,6 +208,7 @@ ray_t* ray_sub_fn(ray_t* a, ray_t* b) { if (is_float_op(a, b)) { double r = as_f64(a) - as_f64(b); if (r == 0.0) r = 0.0; /* normalize -0.0 to +0.0 */ + if (both_f32_atoms(a, b)) return ray_f32((float)r); return make_f64(r); } int8_t rt = promote_int_type_right(a, b); @@ -233,6 +238,7 @@ ray_t* ray_mul_fn(ray_t* a, ray_t* b) { ray_type_name(a->type), ray_type_name(b->type)); /* Null propagation */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return null_for_promoted(a, b); + if (both_f32_atoms(a, b)) return ray_f32((float)(as_f64(a) * as_f64(b))); if (is_float_op(a, b)) return make_f64(as_f64(a) * as_f64(b)); int8_t rt = promote_int_type(a, b); return make_typed_int(rt, as_i64(a) * as_i64(b)); @@ -275,8 +281,8 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(a->type); int64_t bv; - if (b->type == -RAY_F64) { - double bvf = b->f64; + if (b->type == -RAY_F64 || b->type == -RAY_F32) { + double bvf = as_f64(b); if (bvf == 0.0) return ray_typed_null(a->type); bv = (int64_t)bvf; @@ -310,21 +316,21 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { /* Null propagation and division by zero: null type follows RIGHT operand */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) { - int8_t rt = (b->type == -RAY_F64 || a->type == -RAY_F64) ? -RAY_F64 : b->type; - return ray_typed_null(rt); + return null_for_promoted(a, b); } /* Float modulo: result = a - b * floor(a/b), type follows RIGHT or f64 */ if (is_float_op(a, b)) { double av = as_f64(a), bv = as_f64(b); if (bv == 0.0) { - int8_t rt = (b->type == -RAY_F64 || a->type == -RAY_F64) ? -RAY_F64 : b->type; - return ray_typed_null(rt); + return null_for_promoted(a, b); } double result = av - bv * floor(av / bv); /* Snap tiny residual to 0 */ if (fabs(result) < 1e-12 || fabs(result - fabs(bv)) < 1e-12) result = bv > 0 ? 0.0 : -0.0; - if (b->type == -RAY_F64 || a->type == -RAY_F64) return make_f64(result); + if (both_f32_atoms(a, b)) return ray_f32((float)result); + if (b->type == -RAY_F64 || a->type == -RAY_F64 || + b->type == -RAY_F32 || a->type == -RAY_F32) return make_f64(result); if (b->type == -RAY_I32) return make_i32((int32_t)(int64_t)result); if (b->type == -RAY_I16) return make_i16((int16_t)(int64_t)result); return make_i64((int64_t)result); @@ -348,6 +354,7 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { ray_t* ray_neg_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(-x->f64); + if (x->type == -RAY_F32) return ray_f32((float)-x->f64); /* INT_MIN is the lone overflow case for signed negation: -INT_MIN * doesn't fit in the same width. By convention, surface this * as a typed null of the same width — preserving type, avoiding UB, @@ -370,8 +377,12 @@ ray_t* ray_neg_fn(ray_t* x) { /* round: round to nearest integer (ties go away from zero), returns f64 */ ray_t* ray_round_fn(ray_t* x) { - if (RAY_ATOM_IS_NULL(x)) return ray_typed_null(-RAY_F64); + if (RAY_ATOM_IS_NULL(x)) { + if (x->type == -RAY_F32) return ray_typed_null(-RAY_F32); + return ray_typed_null(-RAY_F64); + } if (x->type == -RAY_F64) return make_f64(round(x->f64)); + if (x->type == -RAY_F32) return ray_f32(roundf((float)x->f64)); if (is_numeric(x)) return make_f64(round(as_f64(x))); return ray_error("type", "round: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -380,6 +391,7 @@ ray_t* ray_round_fn(ray_t* x) { ray_t* ray_floor_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(floor(x->f64)); + if (x->type == -RAY_F32) return ray_f32(floorf((float)x->f64)); if (is_numeric(x)) { ray_retain(x); return x; } return ray_error("type", "floor: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -388,6 +400,7 @@ ray_t* ray_floor_fn(ray_t* x) { ray_t* ray_ceil_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(ceil(x->f64)); + if (x->type == -RAY_F32) return ray_f32(ceilf((float)x->f64)); if (is_numeric(x)) { ray_retain(x); return x; } return ray_error("type", "ceil: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -399,6 +412,7 @@ ray_t* ray_ceil_fn(ray_t* x) { ray_t* ray_abs_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(fabs(x->f64)); + if (x->type == -RAY_F32) return ray_f32(fabsf((float)x->f64)); if (x->type == -RAY_I64) { if (RAY_UNLIKELY(x->i64 == INT64_MIN)) return ray_typed_null(-RAY_I64); return make_i64(x->i64 < 0 ? -x->i64 : x->i64); diff --git a/src/ops/expr.c b/src/ops/expr.c index b99e9bb6a..332747f6d 100644 --- a/src/ops/expr.c +++ b/src/ops/expr.c @@ -37,6 +37,11 @@ static bool atom_to_numeric(ray_t* atom, double* out_f, int64_t* out_i, bool* ou *out_i = (int64_t)atom->f64; *out_is_f64 = true; return true; + case -RAY_F32: + *out_f = (double)(float)atom->f64; + *out_i = (int64_t)(float)atom->f64; + *out_is_f64 = true; + return true; case -RAY_I64: case -RAY_SYM: case -RAY_DATE: @@ -673,7 +678,8 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].is_parted = true; out->regs[r].col_obj = NULL; /* parted: per-segment, no zone-skip */ out->regs[r].parted_col = col; - out->regs[r].type = (base == RAY_F64) ? RAY_F64 : RAY_I64; + out->regs[r].type = (base == RAY_F64 || base == RAY_F32) + ? RAY_F64 : RAY_I64; out->regs[r].nullable = col_nulls; out->has_parted = true; } else { @@ -683,7 +689,8 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].is_parted = false; out->regs[r].col_obj = col; out->regs[r].parted_col = NULL; - out->regs[r].type = (col->type == RAY_F64) ? RAY_F64 : RAY_I64; + out->regs[r].type = (col->type == RAY_F64 || col->type == RAY_F32) + ? RAY_F64 : RAY_I64; out->regs[r].nullable = col_nulls; } } else if (node->opcode == OP_CONST) { @@ -712,6 +719,12 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].const_f64 = cf; out->regs[r].const_i64 = ci; } else if (expr_is_elementwise(node->opcode)) { + /* The fused VM has F64/I64 lanes only. F32 inputs can be read + * exactly through the F64 lane, but a node whose result is F32 + * must round and store at F32 width. Keep every such node on + * the typed executor until the VM has a real F32 lane. */ + if (node->out_type == RAY_F32) + EXPR_BAIL(EXPR_BAIL_OTHER); if (!op_child(g, node, 0)) EXPR_BAIL(EXPR_BAIL_OTHER); uint8_t s1 = node_reg[node->in_id[0]]; if (s1 == 0xFF) EXPR_BAIL(EXPR_BAIL_OTHER); @@ -912,6 +925,14 @@ static void expr_load_f64(double* dst, const void* data, int8_t col_type, /* NaN already canonical in place — direct copy */ memcpy(dst, (const double*)data + start, (size_t)n * 8); break; + case RAY_F32: { + const float* s = (const float*)data + start; + if (has_nulls) + for (int64_t j = 0; j < n; j++) + dst[j] = (s[j] != s[j]) ? NULL_F64 : (double)s[j]; + else + for (int64_t j = 0; j < n; j++) dst[j] = (double)s[j]; + } break; case RAY_I64: case RAY_TIMESTAMP: { const int64_t* s = (const int64_t*)data + start; if (has_nulls) @@ -2193,7 +2214,7 @@ static int8_t ew_value_base_type(ray_t* x) { static bool ew_pow_type_admitted(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } /* Resolve data pointer for a vector, accounting for slices. @@ -2333,7 +2354,7 @@ static bool ew_unary_math_type_admitted(uint16_t opcode, int8_t t) { if (opcode != OP_SIGNUM && !expr_is_f64_unary_math(opcode)) return true; if (RAY_IS_PARTED(t)) t = (int8_t)RAY_PARTED_BASETYPE(t); return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } ray_t* exec_elementwise_unary(ray_graph_t* g, ray_op_t* op, ray_t* input) { @@ -2360,7 +2381,90 @@ ray_t* exec_elementwise_unary(ray_graph_t* g, ray_op_t* op, ray_t* input) { int64_t out_off = 0; uint16_t opc = op->opcode; - if (in_type == RAY_F64 && out_type == RAY_F64) { + if (opc == OP_CAST && out_type == RAY_F32) { + /* The query compiler admits `(as 'F32 expr)` for every numeric input. + * Keep that surface complete in the typed fallback. Null propagation + * below rewrites source sentinels to NULL_F32 after conversion. */ + if (in_type != RAY_F64 && in_type != RAY_F32 && + in_type != RAY_I64 && in_type != RAY_TIMESTAMP && + in_type != RAY_I32 && in_type != RAY_DATE && in_type != RAY_TIME && + in_type != RAY_I16 && in_type != RAY_U8 && in_type != RAY_BOOL) { + ray_release(result); + return ray_error("type", "cast: cannot convert %s to F32", + ray_type_name(in_type)); + } + while (ray_morsel_next(&m)) { + int64_t n = m.morsel_len; + float* dst = (float*)ray_data(result) + out_off; + if (in_type == RAY_F64) { + const double* src = (const double*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_F32) { + memcpy(dst, m.morsel_ptr, (size_t)n * sizeof(float)); + } else if (in_type == RAY_I64 || in_type == RAY_TIMESTAMP) { + const int64_t* src = (const int64_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_I32 || in_type == RAY_DATE || in_type == RAY_TIME) { + const int32_t* src = (const int32_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_I16) { + const int16_t* src = (const int16_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else { + const uint8_t* src = (const uint8_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } + out_off += n; + } + } else if (in_type == RAY_F32 && out_type == RAY_F32) { + while (ray_morsel_next(&m)) { + int64_t n = m.morsel_len; + float* src = (float*)m.morsel_ptr; + float* dst = (float*)ray_data(result) + out_off; + switch (opc) { + case OP_NEG: for (int64_t i=0;i 0.0f) - (src[i] < 0.0f); + else + for (int64_t i=0;itype, lhs->attrs); if (lhs->type == RAY_F64) lp_f64 = (double*)lbase; + else if (lhs->type == RAY_F32) lp_f32 = (float*)lbase; else if (lhs->type == RAY_I64 || lhs->type == RAY_TIMESTAMP) lp_i64 = (int64_t*)lbase; else if (RAY_IS_SYM(lhs->type)) { uint8_t w = lhs->attrs & RAY_SYM_W_MASK; @@ -2960,6 +3067,7 @@ static void binary_range(ray_op_t* op, int8_t out_type, void* r_data = resolve_vec_data(rhs, &r_off); void* rbase = (char*)r_data + r_off * ray_sym_elem_size(rhs->type, rhs->attrs); if (rhs->type == RAY_F64) rp_f64 = (double*)rbase; + else if (rhs->type == RAY_F32) rp_f32 = (float*)rbase; else if (rhs->type == RAY_I64 || rhs->type == RAY_TIMESTAMP) rp_i64 = (int64_t*)rbase; else if (RAY_IS_SYM(rhs->type)) { uint8_t w = rhs->attrs & RAY_SYM_W_MASK; @@ -2993,12 +3101,16 @@ static void binary_range(ray_op_t* op, int8_t out_type, /* Resolve the lhs/rhs reader to a single decision made once before the loop. * Each of these yields a double for the given index. */ -#define LV_READ(i) (lp_f64 ? lp_f64[i] : lp_i64 ? (double)lp_i64[i] : lp_i32 ? (double)lp_i32[i] : lp_u32 ? (double)lp_u32[i] : lp_i16 ? (double)lp_i16[i] : lp_bool ? (double)lp_bool[i] : (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64)) ? l_f64 : (double)l_i64) -#define RV_READ(i) (rp_f64 ? rp_f64[i] : rp_i64 ? (double)rp_i64[i] : rp_i32 ? (double)rp_i32[i] : rp_u32 ? (double)rp_u32[i] : rp_i16 ? (double)rp_i16[i] : rp_bool ? (double)rp_bool[i] : (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64)) ? r_f64 : (double)r_i64) +#define LV_READ(i) (lp_f64 ? lp_f64[i] : lp_f32 ? (double)lp_f32[i] : lp_i64 ? (double)lp_i64[i] : lp_i32 ? (double)lp_i32[i] : lp_u32 ? (double)lp_u32[i] : lp_i16 ? (double)lp_i16[i] : lp_bool ? (double)lp_bool[i] : (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64 || lhs->type == -RAY_F32 || lhs->type == RAY_F32)) ? l_f64 : (double)l_i64) +#define RV_READ(i) (rp_f64 ? rp_f64[i] : rp_f32 ? (double)rp_f32[i] : rp_i64 ? (double)rp_i64[i] : rp_i32 ? (double)rp_i32[i] : rp_u32 ? (double)rp_u32[i] : rp_i16 ? (double)rp_i16[i] : rp_bool ? (double)rp_bool[i] : (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || rhs->type == -RAY_F32 || rhs->type == RAY_F32)) ? r_f64 : (double)r_i64) /* Compute once: is lhs/rhs integer-family (not float)? Used by BOOL path. */ - int l_is_int = !(lp_f64 || (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64))); - int r_is_int = !(rp_f64 || (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64))); + int l_is_int = !(lp_f64 || lp_f32 || (l_scalar && + (lhs->type == -RAY_F64 || lhs->type == RAY_F64 || + lhs->type == -RAY_F32 || lhs->type == RAY_F32))); + int r_is_int = !(rp_f64 || rp_f32 || (r_scalar && + (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || + rhs->type == -RAY_F32 || rhs->type == RAY_F32))); int src_is_i64_all = l_is_int && r_is_int; /* Hoist out_type outside the loop. Each branch is a tight per-element kernel. */ @@ -3035,6 +3147,25 @@ static void binary_range(ray_op_t* op, int8_t out_type, if (any_nan) __atomic_fetch_or(&result->attrs, (uint8_t)RAY_ATTR_HAS_NULLS, __ATOMIC_RELAXED); } + } else if (out_type == RAY_F32) { + float* odst = (float*)dst; + switch (op->opcode) { + case OP_ADD: for (int64_t i=0;i0)!=(rv>0)))v+=rv;v=isfinite(v)?v:NULL_F32;}else v=NULL_F32;odst[i]=v; } break; + case OP_MIN2: for (int64_t i=0;irv?lv:rv; } break; + default: for (int64_t i=0;iopcode == OP_ADD || op->opcode == OP_SUB || + op->opcode == OP_MUL || op->opcode == OP_MOD) { + int any_nan = 0; + for (int64_t i = 0; i < n; i++) any_nan |= (odst[i] != odst[i]); + if (any_nan) + __atomic_fetch_or(&result->attrs, (uint8_t)RAY_ATTR_HAS_NULLS, + __ATOMIC_RELAXED); + } } else if (out_type == RAY_I64 || out_type == RAY_TIMESTAMP) { int64_t* odst = (int64_t*)dst; switch (op->opcode) { @@ -3296,7 +3427,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* if (str_resolved && lhs->type == -RAY_STR) l_i64_val = resolved_sym_id; else if (ray_is_atom(lhs)) { - if (lhs->type == -RAY_F64) l_f64_val = lhs->f64; + if (lhs->type == -RAY_F64 || lhs->type == -RAY_F32) + l_f64_val = lhs->f64; else if (lhs->type == -RAY_I32 || lhs->type == -RAY_DATE || lhs->type == -RAY_TIME) l_i64_val = (int64_t)lhs->i32; else if (lhs->type == -RAY_I16) l_i64_val = (int64_t)lhs->i16; @@ -3307,6 +3439,7 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* int64_t elem = 0; void* data = resolve_vec_data(lhs, &elem); if (t == RAY_F64) l_f64_val = ((double*)data)[elem]; + else if (t == RAY_F32) l_f64_val = (double)((float*)data)[elem]; else l_i64_val = read_col_i64(data, elem, t, lhs->attrs); } } @@ -3314,7 +3447,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* if (str_resolved && rhs->type == -RAY_STR) r_i64_val = resolved_sym_id; else if (ray_is_atom(rhs)) { - if (rhs->type == -RAY_F64) r_f64_val = rhs->f64; + if (rhs->type == -RAY_F64 || rhs->type == -RAY_F32) + r_f64_val = rhs->f64; else if (rhs->type == -RAY_I32 || rhs->type == -RAY_DATE || rhs->type == -RAY_TIME) r_i64_val = (int64_t)rhs->i32; else if (rhs->type == -RAY_I16) r_i64_val = (int64_t)rhs->i16; @@ -3325,6 +3459,7 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* int64_t elem = 0; void* data = resolve_vec_data(rhs, &elem); if (t == RAY_F64) r_f64_val = ((double*)data)[elem]; + else if (t == RAY_F32) r_f64_val = (double)((float*)data)[elem]; else r_i64_val = read_col_i64(data, elem, t, rhs->attrs); } } @@ -3424,7 +3559,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* } else { /* Scalar divisor: check for zero using the correct type */ bool is_zero = false; - if (rhs->type == -RAY_F64 || rhs->type == RAY_F64) + if (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || + rhs->type == -RAY_F32 || rhs->type == RAY_F32) is_zero = (r_f64_val == 0.0); else is_zero = (r_i64_val == 0); diff --git a/src/ops/graph.c b/src/ops/graph.c index 1e14f3cd9..9775a6920 100644 --- a/src/ops/graph.c +++ b/src/ops/graph.c @@ -403,11 +403,15 @@ static ray_op_t* make_binary(ray_graph_t* g, uint16_t opcode, ray_op_t* a, ray_o return n; } -/* Type promotion: BOOL < U8 < I16 < I32 < I64 < F64. +/* Type promotion: BOOL < U8 < I16 < I32 < I64 < F32 < F64. + * Two F32 operands retain F32 arithmetic and storage; mixing F32 with another + * numeric width promotes to F64 so no integer precision is silently narrowed. * RAY_STR is its own type class — not promotable to numeric types. */ static int8_t promote(int8_t a, int8_t b) { if (a == RAY_STR || b == RAY_STR) return RAY_STR; if (a == RAY_F64 || b == RAY_F64) return RAY_F64; + if (a == RAY_F32 || b == RAY_F32) + return (a == RAY_F32 && b == RAY_F32) ? RAY_F32 : RAY_F64; if (a == RAY_I64 || b == RAY_I64 || a == RAY_SYM || b == RAY_SYM || a == RAY_TIMESTAMP || b == RAY_TIMESTAMP) return RAY_I64; if (a == RAY_I32 || b == RAY_I32 || @@ -624,16 +628,16 @@ ray_op_t* ray_concat(ray_graph_t* g, ray_op_t** args, int n) { * Reduction ops * -------------------------------------------------------------------------- */ -ray_op_t* ray_sum(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_SUM, a, a->out_type == RAY_F64 ? RAY_F64 : RAY_I64); } -ray_op_t* ray_prod(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_PROD, a, a->out_type == RAY_F64 ? RAY_F64 : RAY_I64); } +ray_op_t* ray_sum(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_SUM, a, (a->out_type == RAY_F64 || a->out_type == RAY_F32) ? RAY_F64 : RAY_I64); } +ray_op_t* ray_prod(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_PROD, a, (a->out_type == RAY_F64 || a->out_type == RAY_F32) ? RAY_F64 : RAY_I64); } ray_op_t* ray_all(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ALL, a, RAY_BOOL); } ray_op_t* ray_any(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ANY, a, RAY_BOOL); } -ray_op_t* ray_min_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MIN, a, a->out_type); } -ray_op_t* ray_max_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MAX, a, a->out_type); } +ray_op_t* ray_min_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MIN, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } +ray_op_t* ray_max_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MAX, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } ray_op_t* ray_count(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_COUNT, a, RAY_I64); } ray_op_t* ray_avg(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_AVG, a, RAY_F64); } -ray_op_t* ray_first(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_FIRST, a, a->out_type); } -ray_op_t* ray_last(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_LAST, a, a->out_type); } +ray_op_t* ray_first(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_FIRST, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } +ray_op_t* ray_last(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_LAST, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } ray_op_t* ray_count_distinct(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_COUNT_DISTINCT, a, RAY_I64); } ray_op_t* ray_distinct_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_DISTINCT, a, a->out_type); } ray_op_t* ray_asc_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ASC, a, a->out_type); } diff --git a/src/ops/group.c b/src/ops/group.c index 5c8bb310e..dc6dff256 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -35,6 +35,18 @@ #include +/* Group accumulators use F64 lanes for both floating storage widths. Keep the + * source read width explicit so F32 payload bits are never mistaken for an + * integer or loaded through a double pointer. */ +static inline bool group_fp_type(int8_t t) { + return t == RAY_F32 || t == RAY_F64; +} + +static inline double group_fp_at(const void* data, int8_t t, int64_t row) { + return t == RAY_F64 ? ((const double*)data)[row] + : (double)((const float*)data)[row]; +} + /* Monotonic count of exec_group_per_partition entries (the streaming parted * GROUP kernel). Bumped ONCE at the kernel entry — O(1) per query, never * per-row, so it does not violate the "instrumentation never costs O(data)" @@ -227,13 +239,13 @@ static ray_t* agg_wide_reduce(ray_t* input, uint16_t op, } while (0) /* Float reduction loop — see REDUCE_LOOP_I for HAS_NULLS/HAS_IDX semantics. - * F64 null = NaN (NULL_F64); detect via v != v (only NaN fails self-equality). */ -#define REDUCE_LOOP_F(base, start, end, acc, HAS_NULLS, HAS_IDX, idx) \ + * F32/F64 null = NaN; both accumulate in the shared F64 reduction lanes. */ +#define REDUCE_LOOP_F(T, base, start, end, acc, HAS_NULLS, HAS_IDX, idx) \ do { \ - const double* d = (const double*)(base); \ + const T* d = (const T*)(base); \ for (int64_t i = start; i < end; i++) { \ int64_t row = (HAS_IDX) ? (idx)[i] : i; \ - double v = d[row]; \ + double v = (double)d[row]; \ if ((HAS_NULLS) && v != v) { (acc)->null_count++; continue; } \ (acc)->sum_f += v; (acc)->sum_sq_f += v * v; (acc)->prod_f *= v; \ if (v == 0.0) (acc)->zero_count++; \ @@ -259,40 +271,18 @@ static ray_t* agg_wide_reduce(ray_t* input, uint16_t op, REDUCE_LOOP_I(T, NULL_SENT, base, start, end, acc, 1, 1, idx); \ } while (0) -#define DISPATCH_F(base, start, end, acc, has_nulls, idx) \ +#define DISPATCH_F(T, base, start, end, acc, has_nulls, idx) \ do { \ if (!(has_nulls) && !(idx)) \ - REDUCE_LOOP_F(base, start, end, acc, 0, 0, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 0, 0, idx); \ else if (!(has_nulls)) \ - REDUCE_LOOP_F(base, start, end, acc, 0, 1, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 0, 1, idx); \ else if (!(idx)) \ - REDUCE_LOOP_F(base, start, end, acc, 1, 0, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 1, 0, idx); \ else \ - REDUCE_LOOP_F(base, start, end, acc, 1, 1, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 1, 1, idx); \ } while (0) -/* Pin the keyless reduction kernel's hot-loop/jump alignment. This kernel's - * tight inner reduction loops are alignment-fragile on 32-byte-fetch cores (DSB - * / JCC-erratum class): as the branch as a whole grows, .o files linked - * before group.o shift reduce_range's ABSOLUTE address (it is early in the TU, - * so the shift is cumulative binary-layout, not a group.c edit), which moved - * its hot I64 loop off its 32-byte boundary and dropped IPC 1.56→1.01 on q03/ - * q02 (+48% cycles, byte-identical instructions) — RCA task-9. Entry alignment - * is NOT the lever (the loop's offset from entry is fixed, so its absolute - * alignment tracks the entry's low bits). `align-loops=32` re-pins every loop - * head AND `align-jumps=32` re-pins the branch targets inside the unrolled - * reduction (the min/max-update return path) — both are needed: loops alone - * only recovered ~⅔ of the gap (IPC 1.37); adding jump alignment restores full - * parity. Now the hot loop's internal alignment is invariant to reduce_range's - * absolute address, so future binary-layout churn can no longer reshuffle it - * onto a bad boundary. aligned(64) keeps the entry cacheline-stable too. - * Per-function attributes only — no global codegen flag, no -m target hack. - * GCC-only: clang has no `optimize` function attribute and -Werror promotes - * the unknown-attribute warning to an error; the pin is performance-only, - * so clang builds simply go without it (the pre-pin status quo). */ -#if defined(__GNUC__) && !defined(__clang__) -__attribute__((aligned(64), optimize("align-loops=32","align-jumps=32"))) -#endif static void reduce_range(ray_t* input, int64_t start, int64_t end, reduce_acc_t* acc, bool has_nulls, const int64_t* idx) { @@ -324,8 +314,10 @@ static void reduce_range(ray_t* input, int64_t start, int64_t end, DISPATCH_I(int32_t, NULL_I32, base, start, end, acc, has_nulls, idx); break; case RAY_I64: case RAY_TIMESTAMP: DISPATCH_I(int64_t, NULL_I64, base, start, end, acc, has_nulls, idx); break; + case RAY_F32: + DISPATCH_F(float, base, start, end, acc, has_nulls, idx); break; case RAY_F64: - DISPATCH_F(base, start, end, acc, has_nulls, idx); break; + DISPATCH_F(double, base, start, end, acc, has_nulls, idx); break; case RAY_SYM: { /* Adaptive-width SYM columns — read_col_i64 produces the i64 * sym id; id 0 is the canonical null sym (interned empty string @@ -405,7 +397,7 @@ static void par_reduce_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t static void reduce_merge(reduce_acc_t* dst, const reduce_acc_t* src, int8_t in_type, struct ray_sym_domain_s* sym_dom) { - if (in_type == RAY_F64) { + if (in_type == RAY_F64 || in_type == RAY_F32) { dst->sum_f += src->sum_f; dst->sum_sq_f += src->sum_sq_f; dst->prod_f *= src->prod_f; @@ -2230,6 +2222,7 @@ typedef struct { const int64_t* offsets; const int64_t* grp_cnt; ray_t* out_list; + _Atomic(int)* oom; } topk_par_ctx_t; /* Read src element as f64 (for the F64 path). Matches med_read_as_f64 @@ -2311,6 +2304,7 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, int64_t K = c->k; uint8_t desc = c->desc; for (int64_t gi = start; gi < end; gi++) { + if (atomic_load_explicit(c->oom, memory_order_relaxed)) break; ray_t* cell = ray_list_get(c->out_list, gi); if (!cell) continue; int64_t cnt = c->grp_cnt[gi]; @@ -2364,11 +2358,26 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, } cell->len = kept; } else { - /* Integer source: stage heap in stack buffer (K <= 1024 → - * 8KB), then narrow back to cell esz on write. */ + /* Integer sources need an I64 working heap before narrowing back + * to the result cell's physical width. Its size follows this + * group's actual min(K, count); no fixed K limit is involved. */ void* dst = ray_data(cell); uint8_t esz = ray_sym_elem_size(t, cell->attrs); - int64_t heap[1024]; + int64_t heap_nmax = cnt < K ? cnt : K; + ray_t* heap_hdr = NULL; + int64_t* heap = NULL; + if (heap_nmax > 0) { + if ((uint64_t)heap_nmax > SIZE_MAX / sizeof(int64_t)) { + atomic_store_explicit(c->oom, 1, memory_order_relaxed); + break; + } + heap = (int64_t*)scratch_alloc(&heap_hdr, + (size_t)heap_nmax * sizeof(int64_t)); + if (!heap) { + atomic_store_explicit(c->oom, 1, memory_order_relaxed); + break; + } + } int64_t kept = 0; int64_t init_end = 0; for (int64_t i = 0; i < cnt && kept < K; i++) { @@ -2399,6 +2408,7 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, for (int64_t i = 0; i < kept; i++) topk_write_i64(dst, i, heap[i], esz); cell->len = kept; + scratch_free(heap_hdr); } } } @@ -2411,7 +2421,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, const int64_t* grp_cnt, int64_t n_groups) { if (!src || RAY_IS_ERR(src) || n_groups < 0) return NULL; - if (k < 1 || k > 1024) return NULL; + if (k < 1) return NULL; int8_t t = src->type; if (t != RAY_F64 && t != RAY_I64 && t != RAY_I32 && t != RAY_I16 && t != RAY_U8 && t != RAY_BOOL && t != RAY_DATE && t != RAY_TIME && @@ -2445,6 +2455,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, out = new_out; } + _Atomic(int) oom = 0; topk_par_ctx_t ctx = { .base = ray_data(src), .src_type = t, @@ -2455,6 +2466,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, .offsets = offsets, .grp_cnt = grp_cnt, .out_list = out, + .oom = &oom, }; ray_pool_t* pool = ray_pool_get(); @@ -2470,6 +2482,11 @@ ray_t* ray_topk_per_group_buf(ray_t* src, topk_per_group_fn(&ctx, 0, 0, n_groups); } + if (atomic_load_explicit(&oom, memory_order_relaxed)) { + ray_release(out); + return ray_error("oom", NULL); + } + return out; } @@ -2543,6 +2560,7 @@ static ray_t* reduction_extreme_result(ray_op_t* op, int8_t in_type, bool found, /* Single-null float model: min/max of finite inputs is finite, but guard * against an ±Inf init sentinel surfacing as a value. */ if (out_type == RAY_F64) return ray_f64(ray_f64_fin(fval)); + if (out_type == RAY_F32) return ray_f32((float)fval); return reduction_i64_result(ival, out_type, out_type == RAY_SYM ? src : NULL); } @@ -2639,7 +2657,7 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { * full reduction pass. Non-numeric types (STR, GUID) fall through * to the serial reduction path below. */ if ((op->opcode == OP_FIRST || op->opcode == OP_LAST) && - (in_type == RAY_I64 || in_type == RAY_F64 || in_type == RAY_I32 || + (in_type == RAY_I64 || in_type == RAY_F64 || in_type == RAY_F32 || in_type == RAY_I32 || in_type == RAY_I16 || in_type == RAY_BOOL || in_type == RAY_U8 || in_type == RAY_TIMESTAMP || in_type == RAY_DATE || in_type == RAY_TIME || in_type == RAY_SYM)) { @@ -2660,6 +2678,7 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { return ray_typed_null(-in_type); void* base = ray_data(input); if (in_type == RAY_F64) return ray_f64(((const double*)base)[row]); + if (in_type == RAY_F32) return ray_f64((double)((const float*)base)[row]); return reduction_i64_result(read_col_i64(base, row, in_type, input->attrs), in_type, in_type == RAY_SYM ? input : NULL); } @@ -2687,14 +2706,14 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* first = accs[first worker with data], last = accs[last worker with data] */ for (uint32_t i = 0; i < nw; i++) { if (accs[i].has_first) { - if (in_type == RAY_F64) merged.first_f = accs[i].first_f; + if (in_type == RAY_F64 || in_type == RAY_F32) merged.first_f = accs[i].first_f; else merged.first_i = accs[i].first_i; break; } } for (int32_t i = (int32_t)nw - 1; i >= 0; i--) { if (accs[i].has_first) { - if (in_type == RAY_F64) merged.last_f = accs[i].last_f; + if (in_type == RAY_F64 || in_type == RAY_F32) merged.last_f = accs[i].last_f; else merged.last_i = accs[i].last_i; break; } @@ -2702,8 +2721,8 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { ray_t* result; switch (op->opcode) { - case OP_SUM: result = in_type == RAY_F64 ? ray_f64(ray_f64_fin(merged.sum_f)) : (in_type == RAY_TIME ? ray_time(merged.sum_i) : ray_i64(merged.sum_i)); break; - case OP_PROD: result = in_type == RAY_F64 ? ray_f64(ray_f64_fin(merged.prod_f)) : ray_i64(merged.prod_i); break; + case OP_SUM: result = (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(merged.sum_f)) : (in_type == RAY_TIME ? ray_time(merged.sum_i) : ray_i64(merged.sum_i)); break; + case OP_PROD: result = (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(merged.prod_f)) : ray_i64(merged.prod_i); break; case OP_ALL: result = ray_bool(merged.zero_count == 0); break; case OP_ANY: result = ray_bool(merged.cnt > merged.zero_count); break; case OP_MIN: result = reduction_extreme_result(op, in_type, merged.cnt > 0, merged.min_f, merged.min_i, input); break; @@ -2711,15 +2730,15 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* COUNT returns total length including nulls — matches ray_count_fn's * "count all elements" semantics, not SQL's COUNT(col) non-null count. */ case OP_COUNT: result = ray_i64(scan_n); break; - case OP_AVG: result = merged.cnt > 0 ? ray_f64(ray_f64_fin(in_type == RAY_F64 ? merged.sum_f / merged.cnt : merged.sum_d / merged.cnt)) : ray_typed_null(-RAY_F64); break; - case OP_FIRST: result = merged.has_first ? (in_type == RAY_F64 ? ray_f64(merged.first_f) : reduction_i64_result(merged.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); break; - case OP_LAST: result = merged.has_first ? (in_type == RAY_F64 ? ray_f64(merged.last_f) : reduction_i64_result(merged.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); break; + case OP_AVG: result = merged.cnt > 0 ? ray_f64(ray_f64_fin((in_type == RAY_F64 || in_type == RAY_F32) ? merged.sum_f / merged.cnt : merged.sum_d / merged.cnt)) : ray_typed_null(-RAY_F64); break; + case OP_FIRST: result = merged.has_first ? (group_fp_type(in_type) ? ray_f64(merged.first_f) : reduction_i64_result(merged.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); break; + case OP_LAST: result = merged.has_first ? (group_fp_type(in_type) ? ray_f64(merged.last_f) : reduction_i64_result(merged.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); break; case OP_VAR: case OP_VAR_POP: case OP_STDDEV: case OP_STDDEV_POP: { bool insufficient = (op->opcode == OP_VAR || op->opcode == OP_STDDEV) ? merged.cnt <= 1 : merged.cnt <= 0; if (insufficient) { result = ray_typed_null(-RAY_F64); break; } double mean, var_pop; - if (in_type == RAY_F64) { mean = merged.sum_f / merged.cnt; var_pop = merged.sum_sq_f / merged.cnt - mean * mean; } + if (in_type == RAY_F64 || in_type == RAY_F32) { mean = merged.sum_f / merged.cnt; var_pop = merged.sum_sq_f / merged.cnt - mean * mean; } else { mean = merged.sum_d / merged.cnt; var_pop = (double)merged.sum_sq_i / merged.cnt - mean * mean; } if (var_pop < 0) var_pop = 0; double val; @@ -2743,8 +2762,8 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { if (sel_idx_block) ray_release(sel_idx_block); switch (op->opcode) { - case OP_SUM: return in_type == RAY_F64 ? ray_f64(ray_f64_fin(acc.sum_f)) : (in_type == RAY_TIME ? ray_time(acc.sum_i) : ray_i64(acc.sum_i)); - case OP_PROD: return in_type == RAY_F64 ? ray_f64(ray_f64_fin(acc.prod_f)) : ray_i64(acc.prod_i); + case OP_SUM: return (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(acc.sum_f)) : (in_type == RAY_TIME ? ray_time(acc.sum_i) : ray_i64(acc.sum_i)); + case OP_PROD: return (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(acc.prod_f)) : ray_i64(acc.prod_i); case OP_ALL: return ray_bool(acc.zero_count == 0); case OP_ANY: return ray_bool(acc.cnt > acc.zero_count); case OP_MIN: return reduction_extreme_result(op, in_type, acc.cnt > 0, acc.min_f, acc.min_i, input); @@ -2752,15 +2771,15 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* COUNT returns total length including nulls — matches ray_count_fn's * "count all elements" semantics, not SQL's COUNT(col) non-null count. */ case OP_COUNT: return ray_i64(scan_n); - case OP_AVG: return acc.cnt > 0 ? ray_f64(ray_f64_fin(in_type == RAY_F64 ? acc.sum_f / acc.cnt : acc.sum_d / acc.cnt)) : ray_typed_null(-RAY_F64); - case OP_FIRST: return acc.has_first ? (in_type == RAY_F64 ? ray_f64(acc.first_f) : reduction_i64_result(acc.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); - case OP_LAST: return acc.has_first ? (in_type == RAY_F64 ? ray_f64(acc.last_f) : reduction_i64_result(acc.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); + case OP_AVG: return acc.cnt > 0 ? ray_f64(ray_f64_fin((in_type == RAY_F64 || in_type == RAY_F32) ? acc.sum_f / acc.cnt : acc.sum_d / acc.cnt)) : ray_typed_null(-RAY_F64); + case OP_FIRST: return acc.has_first ? (group_fp_type(in_type) ? ray_f64(acc.first_f) : reduction_i64_result(acc.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); + case OP_LAST: return acc.has_first ? (group_fp_type(in_type) ? ray_f64(acc.last_f) : reduction_i64_result(acc.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); case OP_VAR: case OP_VAR_POP: case OP_STDDEV: case OP_STDDEV_POP: { bool insufficient = (op->opcode == OP_VAR || op->opcode == OP_STDDEV) ? acc.cnt <= 1 : acc.cnt <= 0; if (insufficient) return ray_typed_null(-RAY_F64); double mean, var_pop; - if (in_type == RAY_F64) { mean = acc.sum_f / acc.cnt; var_pop = acc.sum_sq_f / acc.cnt - mean * mean; } + if (in_type == RAY_F64 || in_type == RAY_F32) { mean = acc.sum_f / acc.cnt; var_pop = acc.sum_sq_f / acc.cnt - mean * mean; } else { mean = acc.sum_d / acc.cnt; var_pop = (double)acc.sum_sq_i / acc.cnt - mean * mean; } if (var_pop < 0) var_pop = 0; double val; @@ -2879,7 +2898,8 @@ void ght_layout_copy(ght_layout_t* dst, const ght_layout_t* src) { } bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, - ray_t** agg_vecs, uint8_t need_flags, + ray_t** agg_vecs, ray_t** agg_vecs2, + uint8_t need_flags, const uint16_t* agg_ops, const int8_t* key_types) { memset(out, 0, sizeof(*out)); @@ -2945,7 +2965,10 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->agg_val_slot[a] = -1; } else if (agg_vecs[a]) { out->agg_val_slot[a] = (int8_t)nv; - if (agg_vecs[a]->type == RAY_F64) + bool pair = agg_ops && agg_is_binary_agg(agg_ops[a]); + bool pair_fp = pair && agg_vecs2 && agg_vecs2[a] && + group_fp_type(agg_vecs2[a]->type); + if (group_fp_type(agg_vecs[a]->type) || pair_fp) af |= GHT_AF_F64; if (agg_vecs[a]->type == RAY_SYM) { af |= GHT_AF_SYM; @@ -3029,10 +3052,9 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->key_region = (uint16_t)koff; } uint16_t key_region = out->key_region; - /* Entry layout: hash | keys | null_mask | agg_vals | [entry_row?] - * Tail entry_row slot is appended only when any agg is FIRST/LAST, - * carrying the source-row index needed to merge correctly under - * work-stealing dispatch (see radix_phase1_fn / accum_from_entry). + /* Entry layout: hash | keys | null_mask | agg_vals | entry_row + * The tail source-row slot provides stable first-seen group ordering + * across partitions as well as FIRST/LAST merge correctness. * * The agg region is a representational budget, exactly like the key * region above: agg_val_slot is int8 (so ≤ INT8_MAX value slots), and @@ -3042,8 +3064,7 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, * silent wrap here scatters aggs to wrong offsets (the key region was * already budget-guarded; the agg region was not). */ bool has_first_last = (agg_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; - uint32_t entry_tail = has_first_last ? 8u : 0u; - uint32_t entry_stride = 8u + key_region + (uint32_t)nv * 8 + entry_tail; + uint32_t entry_stride = 8u + key_region + (uint32_t)nv * 8 + 8u; uint32_t off = 8u + key_region; uint32_t block = (uint32_t)nv * 8; @@ -3066,6 +3087,7 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->off_sumsq_y = (uint16_t)off; off += block; out->off_sumxy = (uint16_t)off; off += block; } + out->off_group_first = (uint16_t)off; off += 8u; /* Per-slot non-null count block — only when a nullable agg is present. * Null-free shapes leave off_nn == 0 and finalize on the group row count, * so their row_stride and layout are byte-identical to before. */ @@ -3142,7 +3164,7 @@ static bool group_ht_init_sized(group_ht_t* ht, uint32_t cap, } bool group_ht_init(group_ht_t* ht, uint32_t cap, const ght_layout_t* ly) { - return group_ht_init_sized(ht, cap, ly, 256); + return group_ht_init_sized(ht, cap, ly, 1); } /* Populate key_data[k] for wide-key resolution. Called by the HT path @@ -3434,7 +3456,7 @@ static void accum_from_entry_nullable(char* row, const char* entry, /* Initialize accumulators for a new group from entry's inline agg values. * Each unified block has n_agg_vals slots of 8 bytes, typed by agg_is_f64. */ static inline void init_accum_from_entry(char* row, const char* entry, - const ght_layout_t* ly) { + const ght_layout_t* ly) { if (ly->any_agg_null) { init_accum_from_entry_nullable(row, entry, ly); return; } /* key_region-based (mirrors init_accum_from_entry_nullable): the old * `8 + (n_keys+1)*8` assumed exactly one trailing null-mask word. For @@ -3459,7 +3481,8 @@ static inline void init_accum_from_entry(char* row, const char* entry, uint16_t na = ly->n_aggs; uint8_t nf = ly->need_flags; bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; - /* Entry tail slot carries the source-row index when has_fl. */ + /* The always-present entry tail carries the source row; FIRST/LAST + * consumes it only for those aggregate shapes. */ int64_t entry_row = 0; if (has_fl) memcpy(&entry_row, entry + ly->entry_stride - 8, 8); @@ -3539,7 +3562,7 @@ static inline void init_accum_from_entry(char* row, const char* entry, /* Accumulate into existing group from entry's inline agg values */ static inline void accum_from_entry(char* row, const char* entry, - const ght_layout_t* ly) { + const ght_layout_t* ly) { if (ly->any_agg_null) { accum_from_entry_nullable(row, entry, ly); return; } const char* agg_data = entry + 8 + (size_t)ly->key_region; uint16_t na = ly->n_aggs; @@ -3844,6 +3867,8 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, uint64_t hash = *(const uint64_t*)entry; const char* ekeys = entry + 8; uint8_t salt = HT_SALT(hash); + int64_t entry_row; + memcpy(&entry_row, entry + ly->entry_stride - 8, 8); uint32_t slot = (uint32_t)(hash & mask); uint16_t key_bytes = ly->key_region; /* keys + null mask (incl. inline STR descriptors) */ @@ -3870,6 +3895,11 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, memcpy(row + 8, ekeys, key_bytes); if (!accum_skip) init_accum_from_entry(row, entry, ly); + else if (ly->row_stride > 8 + key_bytes) + memset(row + 8 + key_bytes, 0, + ly->row_stride - (8 + key_bytes)); + /* Accumulator initialization clears the row-state region. */ + memcpy(row + ly->off_group_first, &entry_row, 8); ht->slots[slot] = HT_PACK(salt, gid); if (ht->grp_count * 2 > ht->ht_cap) { group_ht_rehash(ht, key_types); @@ -3883,6 +3913,10 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, if (group_keys_equal((const int64_t*)(row + 8), (const int64_t*)ekeys, ly, ht->key_data, ht->key_pool)) { (*(int64_t*)row)++; /* count++ */ + int64_t first; + memcpy(&first, row + ly->off_group_first, 8); + if (entry_row < first) + memcpy(row + ly->off_group_first, &entry_row, 8); if (!accum_skip) accum_from_entry(row, entry, ly); return mask; @@ -3937,10 +3971,9 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, const uint8_t* const kflags = ly->key_flags; const uint8_t* const aflags = ly->agg_flags; uint8_t wide_any = ly->any_wide_key; - bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; uint32_t mask = ht->ht_cap - 1; /* Stack buffer for one entry: hash + (nk+1) key slots + nv agg_vals - * + optional 8-byte source-row tail (FIRST/LAST). The ≤8-key/≤8-agg + * + 8-byte source-row tail. The ≤8-key/≤8-agg * layout fits 8 + 9*8 + 8*8 + 8 = 152 bytes (byte-identical to the legacy * fixed buffer); a wider layout spills to one per-call heap block sized * to entry_stride (carved once here, never per row — unbounded-slots @@ -4026,16 +4059,24 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, if (!ac) continue; if (agg_strlen && agg_strlen[a]) ev[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; /* Binary aggregator: pack y after x in the same entry. */ if ((af & GHT_AF_BINARY) && agg_vecs2 && agg_vecs2[a]) { ray_t* ay = agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; @@ -4044,8 +4085,7 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, /* Tail slot: source row index for FIRST/LAST tie-breaking. Same * layout as the radix path's entries so accum_from_entry can read * it from the same offset. */ - if (has_fl) - memcpy(ebuf + ly->entry_stride - 8, &row, 8); + memcpy(ebuf + ly->entry_stride - 8, &row, 8); mask = group_probe_entry(ht, ebuf, key_types, mask); } @@ -4063,10 +4103,27 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, * Pass 3: Build result columns from inline group rows. * ============================================================================ */ -#define RADIX_BITS 8 -#define RADIX_P (1u << RADIX_BITS) /* 256 partitions */ -#define RADIX_MASK (RADIX_P - 1) -#define RADIX_PART(h) (((uint32_t)((h) >> 16)) & RADIX_MASK) +/* Balance O(P) partition metadata and O(N/P) rows per partition. Both inputs + * come from the execution; there is no fixed fanout or machine-size cap. */ +static uint32_t group_radix_part_count(uint32_t n_workers, int64_t n_rows) { + uint32_t n_parts = 1; + uint64_t rows = n_rows > 0 ? (uint64_t)n_rows : 1; + while ((n_parts < n_workers || + (uint64_t)n_parts < rows / n_parts + (rows % n_parts != 0)) && + n_parts <= UINT32_MAX / 2) + n_parts <<= 1; + return n_parts; +} + +static inline uint32_t group_radix_part(uint64_t hash, uint32_t n_parts) { + return ((uint32_t)(hash >> 16)) & (n_parts - 1); +} + +typedef struct { + int64_t value; + uint32_t part; + uint32_t gid; +} group_topn_item_t; /* Selection-aware group iteration gate. When a WHERE leaves fewer than * nrows >> SEL_MATCH_GATE_SHIFT survivors, the high-card group build iterates @@ -4091,11 +4148,11 @@ typedef struct { static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, uint64_t hash, const int64_t* key_region_buf, const int64_t* agg_vals, uint16_t n_agg_vals, - bool has_first_last, int64_t row, - uint16_t key_region) { + int64_t row, uint16_t key_region) { if (__builtin_expect(buf->count >= buf->cap, 0)) { uint32_t old_cap = buf->cap; - uint32_t new_cap = old_cap * 2; + if (old_cap > UINT32_MAX / 2) { buf->oom = true; return; } + uint32_t new_cap = old_cap ? old_cap * 2 : 1; char* new_data = (char*)scratch_realloc( &buf->_hdr, (size_t)old_cap * entry_stride, (size_t)new_cap * entry_stride); @@ -4108,9 +4165,7 @@ static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, memcpy(dst + 8, key_region_buf, key_region); if (n_agg_vals) memcpy(dst + 8 + key_region, agg_vals, (size_t)n_agg_vals * 8); - /* Tail slot: source row index for FIRST/LAST tie-breaking. */ - if (has_first_last) - memcpy(dst + entry_stride - 8, &row, 8); + memcpy(dst + entry_stride - 8, &row, 8); buf->count++; } @@ -4129,7 +4184,8 @@ typedef struct { ray_t** agg_vecs2; uint8_t* agg_strlen; uint32_t n_workers; - radix_buf_t* bufs; /* [n_workers * RADIX_P] */ + uint32_t n_parts; + radix_buf_t* bufs; /* [n_workers * n_parts] */ ght_layout_t layout; ray_t* rowsel; /* When non-NULL, workers iterate match_idx[start..end) and @@ -4140,7 +4196,7 @@ typedef struct { static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { radix_phase1_ctx_t* c = (radix_phase1_ctx_t*)ctx; const ght_layout_t* ly = &c->layout; - radix_buf_t* my_bufs = &c->bufs[(size_t)worker_id * RADIX_P]; + radix_buf_t* my_bufs = &c->bufs[(size_t)worker_id * c->n_parts]; uint16_t nk = ly->n_keys; uint16_t na = ly->n_aggs; uint16_t nv = ly->n_agg_vals; @@ -4149,7 +4205,6 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ uint8_t wide_any = ly->any_wide_key; uint8_t inline_str = ly->any_inline_str; uint16_t estride = ly->entry_stride; - bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; const int64_t* match_idx = c->match_idx; /* Per-worker key/agg staging. The ≤8-key/≤8-agg layout stages through @@ -4229,8 +4284,12 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ if (!ac) continue; if (c->agg_strlen && c->agg_strlen[a]) agg_vals[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&agg_vals[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } else agg_vals[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; @@ -4241,18 +4300,22 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * support F64 inputs cleanly — i64 path is a perf followup). */ if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { ray_t* ay = c->agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&agg_vals[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } else agg_vals[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; } } - uint32_t part = RADIX_PART(h); + uint32_t part = group_radix_part(h, c->n_parts); radix_buf_push(&my_bufs[part], estride, h, inline_str ? (const int64_t*)keybuf : keys, - agg_vals, nv, has_fl, row, ly->key_region); + agg_vals, nv, row, ly->key_region); } scratch_free(stage_hdr); /* NULL (inline staging) → no-op */ } @@ -4302,6 +4365,7 @@ typedef struct { typedef struct { group_ht_t* part_hts; uint32_t* part_offsets; + const uint32_t* group_out; /* flat partition gid -> first-seen output row */ char** key_dsts; int8_t* key_types; uint8_t* key_attrs; @@ -4345,7 +4409,7 @@ static void radix_phase3_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * (→ use cnt) for null-free layouts (byte-identical to before). */ const int64_t* nnbase = ly->off_nn ? (const int64_t*)(const void*)(row + ly->off_nn) : NULL; - uint32_t di = off + gi; + uint32_t di = c->group_out ? c->group_out[off + gi] : off + gi; /* Scatter keys to result columns */ for (uint32_t k = 0; k < nk; k++) { @@ -4539,6 +4603,7 @@ typedef struct { int8_t* key_types; uint32_t n_keys; /* widened from uint8_t (unbounded-ready) */ uint32_t n_workers; + uint32_t n_parts; radix_buf_t* bufs; group_ht_t* part_hts; ght_layout_t layout; @@ -4556,21 +4621,10 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ for (int64_t p = start; p < end; p++) { uint32_t total = 0; for (uint32_t w = 0; w < c->n_workers; w++) - total += c->bufs[(size_t)w * RADIX_P + p].count; + total += c->bufs[(size_t)w * c->n_parts + p].count; if (total == 0) continue; - uint32_t part_ht_cap = 256; - { - uint64_t target = (uint64_t)total * 2; - if (target < 256) target = 256; - while (part_ht_cap < target) part_ht_cap *= 2; - } - /* Pre-size group store to avoid grows. Use next_pow2(total) as upper - * bound on groups. Over-allocation is bounded: worst case total >> groups, - * but total * row_stride is already committed via HT capacity anyway. */ - uint32_t init_grp = 256; - while (init_grp < total && init_grp < 65536) init_grp *= 2; - if (!group_ht_init_sized(&c->part_hts[p], part_ht_cap, &c->layout, init_grp)) + if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) continue; /* Wide keys need source-column resolution during probe/rehash. */ if (c->layout.any_wide_key && c->key_data) { @@ -4579,7 +4633,7 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ } for (uint32_t w = 0; w < c->n_workers; w++) { - radix_buf_t* buf = &c->bufs[(size_t)w * RADIX_P + p]; + radix_buf_t* buf = &c->bufs[(size_t)w * c->n_parts + p]; if (buf->count == 0) continue; group_rows_indirect(&c->part_hts[p], c->key_types, buf->data, buf->count, estride); @@ -4597,13 +4651,9 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * knows how to combine counts; SUM/AVG/MIN/MAX would need their own * state-merge logic (next increment). * - * Per-(worker, partition) HT for a 10M-row count-by-UserID: ~3M distinct - * keys ÷ 256 parts ÷ 8 workers ≈ 1.5K groups → cap ~4K slots → ~64 KB - * row store, L1/L2-resident. Worker w processes its row range; per row - * it hashes keys, computes partition = RADIX_PART(h), probes its local - * HT_p. Phase2 dispatches partitions across workers; each merges the n - * worker HTs for one partition into a final partition HT in part_hts[p]. - * Phase3 (radix_phase3_fn) emits from part_hts[] exactly as before. + * Worker w processes its row ranges; per row it hashes keys, computes the + * execution-derived partition, and probes its local HT_p. Phase2 dispatches + * partitions across workers and merges the worker HTs for each partition. * ============================================================================ */ /* Merge one source group row into the target HT. Hash is recomputed from @@ -4653,6 +4703,11 @@ static inline uint32_t group_merge_row(group_ht_t* ht, if (group_keys_equal((const int64_t*)(row + 8), skeys, ly, ht->key_data, ht->key_pool)) { *(int64_t*)row += src_count; + int64_t dst_first, src_first; + memcpy(&dst_first, row + ly->off_group_first, 8); + memcpy(&src_first, src_row + ly->off_group_first, 8); + if (src_first < dst_first) + memcpy(row + ly->off_group_first, &src_first, 8); if (need_sum) { for (uint32_t a = 0; a < na; a++) { int8_t s = vslot[a]; @@ -4687,7 +4742,8 @@ typedef struct { uint8_t* agg_strlen; uint8_t nullable_mask; uint32_t n_workers; - group_ht_t* wpart_hts; /* [n_workers * RADIX_P] */ + uint32_t n_parts; + group_ht_t* wpart_hts; /* [n_workers * n_parts] */ ght_layout_t layout; ray_t* rowsel; const int64_t* match_idx; @@ -4707,22 +4763,13 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, uint8_t nullable = c->nullable_mask; const int64_t* match_idx = c->match_idx; - group_ht_t* my_hts = &c->wpart_hts[(size_t)worker_id * RADIX_P]; - /* Lazily init this worker's 256 partition HTs. */ - for (uint32_t p = 0; p < RADIX_P; p++) { - if (!my_hts[p].slots) { - if (!group_ht_init_sized(&my_hts[p], 256, ly, 128)) { - atomic_store_explicit(&c->oom, 1, memory_order_relaxed); - return; - } - if (wide_any && c->key_data) { - group_ht_set_key_data(&my_hts[p], c->key_data); - group_ht_set_key_pool(&my_hts[p], c->key_vecs); - } - } - } - uint32_t masks[RADIX_P]; - for (uint32_t p = 0; p < RADIX_P; p++) masks[p] = my_hts[p].ht_cap - 1; + group_ht_t* my_hts = &c->wpart_hts[(size_t)worker_id * c->n_parts]; + uint32_t masks[c->n_parts]; + /* A worker may be called for several morsels. Its partition HTs persist + * across calls, so restore each live probe mask instead of treating every + * invocation as first use. */ + for (uint32_t p = 0; p < c->n_parts; p++) + masks[p] = my_hts[p].slots ? my_hts[p].ht_cap - 1 : 0; /* Stack-resident transient entry + per-key str-pool table, same layout as * group_rows_range. ≤8 keys/aggs stay on the stack (byte-identical); @@ -4797,22 +4844,45 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, if (!ac) continue; if (c->agg_strlen && c->agg_strlen[a]) ev[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { ray_t* ay = c->agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; } } } - uint32_t p = RADIX_PART(h); + memcpy(ebuf + ly->entry_stride - 8, &row, 8); + uint32_t p = group_radix_part(h, c->n_parts); + if (!my_hts[p].slots) { + /* Demand-driven initialization: the first observed row creates + * the smallest valid table and normal growth follows actual + * cardinality. Empty worker/partition pairs allocate nothing. */ + if (!group_ht_init_sized(&my_hts[p], 2, ly, 1)) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + break; + } + if (wide_any && c->key_data) { + group_ht_set_key_data(&my_hts[p], c->key_data); + group_ht_set_key_pool(&my_hts[p], c->key_vecs); + } + masks[p] = my_hts[p].ht_cap - 1; + } uint32_t new_mask = group_probe_entry(&my_hts[p], ebuf, c->key_types, masks[p]); if (my_hts[p].oom) { @@ -4825,10 +4895,11 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, } typedef struct { - group_ht_t* wpart_hts; /* [n_workers * RADIX_P] — input */ - group_ht_t* part_hts; /* [RADIX_P] — output */ + group_ht_t* wpart_hts; /* [n_workers * n_parts] — input */ + group_ht_t* part_hts; /* [n_parts] — output */ int8_t* key_types; uint32_t n_workers; + uint32_t n_parts; ght_layout_t layout; void** key_data; const void** key_pool; /* [n_keys] str-pool base per wide STR key */ @@ -4847,17 +4918,9 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, * fold those, so the final grp_count is ≤ this sum). */ uint32_t total_grps = 0; for (uint32_t w = 0; w < c->n_workers; w++) - total_grps += c->wpart_hts[(size_t)w * RADIX_P + p].grp_count; + total_grps += c->wpart_hts[(size_t)w * c->n_parts + p].grp_count; if (total_grps == 0) continue; - uint32_t ht_cap = 256; - { - uint64_t target = (uint64_t)total_grps * 2; - if (target < 256) target = 256; - while (ht_cap < target) ht_cap *= 2; - } - uint32_t init_grp = 256; - while (init_grp < total_grps && init_grp < 65536) init_grp *= 2; - if (!group_ht_init_sized(&c->part_hts[p], ht_cap, &c->layout, init_grp)) { + if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) { atomic_store_explicit(&c->oom, 1, memory_order_relaxed); return; } @@ -4867,7 +4930,7 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, } uint32_t mask = c->part_hts[p].ht_cap - 1; for (uint32_t w = 0; w < c->n_workers; w++) { - group_ht_t* src = &c->wpart_hts[(size_t)w * RADIX_P + p]; + group_ht_t* src = &c->wpart_hts[(size_t)w * c->n_parts + p]; if (src->grp_count == 0) continue; const char* rows = src->rows; for (uint32_t gi = 0; gi < src->grp_count; gi++) { @@ -4897,10 +4960,8 @@ typedef struct { uint32_t n_workers; const int64_t* match_idx; /* NULL = no selection */ ray_t* rowsel; - /* DA-path early-out: once any worker observes a key span wider than - * span_budget the direct-array path is provably infeasible (its slot - * count would exceed DA_MAX_COMPOSITE_SLOTS), so the whole scan can - * stop instead of reading the rest of a 10M-row column for nothing. */ + /* DA-path early-out: once any worker observes a key span wider than the + * contributing row count, direct-array state would be superlinear. */ int64_t span_budget; _Atomic(int)* abort_flag; } minmax_ctx_t; @@ -5032,7 +5093,7 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* * product accumulates F64; the linear i64 plan accumulates i64 * even when its compiled expression promotes to F64. */ bool is_f64 = agg_col - ? (agg_col->type == RAY_F64) + ? group_fp_type(agg_col->type) : (prod && prod[a].enabled); int8_t out_type; switch (agg_op) { @@ -5063,7 +5124,8 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col = ray_vec_new(out_type, (int64_t)grp_count); if (!new_col || RAY_IS_ERR(new_col)) continue; @@ -5517,8 +5579,7 @@ static bool sparse_i64_touch(sparse_i64_ht_t* ht, int64_t key, uint8_t n_aggs, return true; } -/* Composite GID from multi-key. Arithmetic overflow is prevented in practice - * by the DA budget check (DA_PER_WORKER_MAX) which limits total_slots to 262K. */ +/* Composite GID from multi-key. Planning bounds total_slots to INT32_MAX. */ static inline int32_t da_composite_gid(da_ctx_t* c, int64_t r) { int32_t gid = 0; for (uint32_t k = 0; k < c->n_keys; k++) { @@ -5546,8 +5607,8 @@ DEFINE_DA_COMPOSITE_GID_TYPED(i64, int64_t) static inline void da_read_val(const void* ptr, int8_t type, uint8_t attrs, int64_t r, double* out_f64, int64_t* out_i64) { - if (type == RAY_F64) { - *out_f64 = ((const double*)ptr)[r]; + if (group_fp_type(type)) { + *out_f64 = group_fp_at(ptr, type, r); *out_i64 = (int64_t)*out_f64; } else { *out_i64 = read_col_i64(ptr, r, type, attrs); @@ -5873,7 +5934,7 @@ static inline void scalar_accum_row(scalar_ctx_t* c, da_accum_t* acc, int64_t r) } } uint16_t op = c->agg_ops[a]; - bool is_f = (c->agg_types[a] == RAY_F64); + bool is_f = group_fp_type(c->agg_types[a]); /* NULL_I* sentinel = null. */ bool int_null = !is_f && c->agg_int_null_has[a] && iv == c->agg_int_null_sentinel[a]; @@ -5995,7 +6056,7 @@ static inline void da_accum_row(da_ctx_t* c, da_accum_t* acc, int32_t gid, int64 if (nn) nn[idx]++; } else if (f64m & ((uint64_t)1 << a)) { /* NaN payload = null, skip from sum. */ - double v = ((const double*)c->agg_ptrs[a])[r]; + double v = group_fp_at(c->agg_ptrs[a], c->agg_types[a], r); if (RAY_LIKELY(v == v)) { acc->sum[idx].f += v; if (nn) nn[idx]++; } } else { /* NULL_I* sentinel = null, skip from sum. Only paid when @@ -6050,7 +6111,7 @@ static inline void da_accum_row(da_ctx_t* c, da_accum_t* acc, int32_t gid, int64 da_read_val(c->agg_ptrs[a], c->agg_types[a], attrs, r, &fv, &iv); } uint16_t op = c->agg_ops[a]; - bool is_f = (c->agg_types[a] == RAY_F64); + bool is_f = group_fp_type(c->agg_types[a]); /* NULL_I* sentinel = null. Bit set in agg_int_null_mask AND * value equal to per-agg sentinel means this row is null for * an integer aggregation column. */ @@ -6286,12 +6347,12 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); } - } else if (agg_types[a] == RAY_F64) + } else if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; @@ -6300,7 +6361,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (c->need_flags & DA_NEED_MIN) { for (uint32_t a = 0; a < n_aggs; a++) { size_t idx = base + a; - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[idx].f < merged->min_val[idx].f) merged->min_val[idx].f = wa->min_val[idx].f; } else if (agg_types[a] == RAY_SYM) { @@ -6318,7 +6379,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (c->need_flags & DA_NEED_MAX) { for (uint32_t a = 0; a < n_aggs; a++) { size_t idx = base + a; - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[idx].f > merged->max_val[idx].f) merged->max_val[idx].f = wa->max_val[idx].f; } else if (agg_types[a] == RAY_SYM) { @@ -6349,7 +6410,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { * part_hts[] + part_offsets[], we still need to materialize per-group * value slices to feed the holistic quickselect kernel. This pass: * - * 1. Re-probe each source row against part_hts[RADIX_PART(h)] to + * 1. Re-probe each source row against its hash partition to * recover its global gid (parallel, lookup-only — no inserts). * Writes row_gid[r] = part_offsets[p] + local_gid. * 2. Build idx_buf + offsets via the idxbuf hist/scat pattern over @@ -6402,31 +6463,18 @@ typedef struct { const void** key_pool; /* [n_keys] str-pool base per wide STR key */ group_ht_t* part_hts; const uint32_t* part_offsets; + const uint32_t* group_out; + uint32_t n_parts; int64_t* row_gid; /* output [nrows] */ const int64_t* match_idx; ray_t* rowsel; /* non-NULL when selection carried as rowsel */ } reprobe_ctx_t; -static void reprobe_rows_fn(void* vctx, uint32_t worker_id, - int64_t start, int64_t end) { - (void)worker_id; - reprobe_ctx_t* c = (reprobe_ctx_t*)vctx; +/* Resolve one source row to the flat partition-local group id. The caller + * owns reusable key staging so a scan performs no per-row allocation. */ +static inline int64_t reprobe_flat_gid(const reprobe_ctx_t* c, int64_t row, + int64_t* ek_buf, char* keybuf) { uint32_t nk = c->n_keys; - /* Key lookup staging: ≤8 keys stay on the stack (ek_buf: 8 keys + 1 null - * word; keybuf: inline-STR region), wider layouts carve one per-worker - * heap block ONCE (never per row — unbounded-slots cut 4). */ - int64_t ek_buf_stk[9]; - char keybuf_stk[136]; - int64_t* ek_buf = ek_buf_stk; - char* keybuf = keybuf_stk; - ray_t* rp_stage_hdr = NULL; - if ((size_t)c->layout->key_region > sizeof(ek_buf_stk)) { - size_t kb = c->layout->key_region; - char* blk = (char*)scratch_alloc(&rp_stage_hdr, kb + kb); - if (!blk) return; /* OOM on a pathologically wide layout: drop worker */ - keybuf = blk; - ek_buf = (int64_t*)(blk + kb); - } int8_t* key_types = c->key_types; void** key_data = c->key_data; uint8_t* key_attrs = c->key_attrs; @@ -6434,26 +6482,15 @@ static void reprobe_rows_fn(void* vctx, uint32_t worker_id, uint8_t nullable = c->nullable_mask; const uint8_t* const kflags = c->layout->key_flags; uint8_t wide_any = c->layout->any_wide_key; - const int64_t* match_idx = c->match_idx; - ray_t* rowsel = c->rowsel; - for (int64_t i = start; i < end; i++) { - if (((i - start) & 65535) == 0 && ray_interrupted()) break; - int64_t row = match_idx ? match_idx[i] : i; - /* Honor a rowsel-carried selection: filtered-out rows must not map to - * any group, else the holistic (median/top) idx_buf would include - * them. -1 is the same "no group" sentinel the HT-miss case uses. */ - if (!match_idx && rowsel && !group_rowsel_pass(rowsel, row)) { - c->row_gid[row] = -1; - continue; - } - uint64_t h = 0; - const int64_t* lookup_keys; - if (c->layout->any_inline_str) { - h = inline_build_keys(c->layout, key_types, key_data, key_attrs, - c->key_pool, key_vecs, nullable, row, keybuf); - lookup_keys = (const int64_t*)keybuf; - } else { - int64_t* nullw = ek_buf + nk; /* null-mask words at key_off[nk]==nk*8 */ + uint64_t h = 0; + const int64_t* lookup_keys; + + if (c->layout->any_inline_str) { + h = inline_build_keys(c->layout, key_types, key_data, key_attrs, + c->key_pool, key_vecs, nullable, row, keybuf); + lookup_keys = (const int64_t*)keybuf; + } else { + int64_t* nullw = ek_buf + nk; uint32_t null_words = c->layout->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; for (uint32_t k = 0; k < nk; k++) { @@ -6482,15 +6519,53 @@ static void reprobe_rows_fn(void* vctx, uint32_t worker_id, } h = ght_hash_null_words(h, nullw, null_words); lookup_keys = ek_buf; - } + } - uint32_t part = RADIX_PART(h); - uint32_t local = group_ht_lookup_gid(&c->part_hts[part], h, - lookup_keys, key_types); - if (local == UINT32_MAX || local >= c->part_hts[part].grp_count) { + uint32_t part = group_radix_part(h, c->n_parts); + uint32_t local = group_ht_lookup_gid(&c->part_hts[part], h, + lookup_keys, key_types); + if (local == UINT32_MAX || local >= c->part_hts[part].grp_count) + return -1; + return (int64_t)c->part_offsets[part] + local; +} + +static void reprobe_rows_fn(void* vctx, uint32_t worker_id, + int64_t start, int64_t end) { + (void)worker_id; + reprobe_ctx_t* c = (reprobe_ctx_t*)vctx; + /* Key lookup staging: ≤8 keys stay on the stack (ek_buf: 8 keys + 1 null + * word; keybuf: inline-STR region), wider layouts carve one per-worker + * heap block ONCE (never per row — unbounded-slots cut 4). */ + int64_t ek_buf_stk[9]; + char keybuf_stk[136]; + int64_t* ek_buf = ek_buf_stk; + char* keybuf = keybuf_stk; + ray_t* rp_stage_hdr = NULL; + if ((size_t)c->layout->key_region > sizeof(ek_buf_stk)) { + size_t kb = c->layout->key_region; + char* blk = (char*)scratch_alloc(&rp_stage_hdr, kb + kb); + if (!blk) return; /* OOM on a pathologically wide layout: drop worker */ + keybuf = blk; + ek_buf = (int64_t*)(blk + kb); + } + const int64_t* match_idx = c->match_idx; + ray_t* rowsel = c->rowsel; + for (int64_t i = start; i < end; i++) { + if (((i - start) & 65535) == 0 && ray_interrupted()) break; + int64_t row = match_idx ? match_idx[i] : i; + /* Honor a rowsel-carried selection: filtered-out rows must not map to + * any group, else the holistic (median/top) idx_buf would include + * them. -1 is the same "no group" sentinel the HT-miss case uses. */ + if (!match_idx && rowsel && !group_rowsel_pass(rowsel, row)) { + c->row_gid[row] = -1; + continue; + } + int64_t flat = reprobe_flat_gid(c, row, ek_buf, keybuf); + if (flat < 0) { c->row_gid[row] = -1; } else { - c->row_gid[row] = (int64_t)c->part_offsets[part] + (int64_t)local; + c->row_gid[row] = c->group_out ? (int64_t)c->group_out[flat] + : flat; } } scratch_free(rp_stage_hdr); /* NULL (inline staging) → no-op */ @@ -7490,15 +7565,17 @@ static inline double sg_prod_range(const agg_prod_t* p, int64_t r0, int64_t n, } static inline double sg_num_at(ray_t* v, int64_t row) { - if (v->type == RAY_F64) - return ((const double*)ray_data(v))[row]; + if (group_fp_type(v->type)) + return group_fp_at(ray_data(v), v->type, row); return (double)read_col_i64(ray_data(v), row, v->type, v->attrs); } -static inline void sg_pair_accum(ray_t* x, ray_t* y, const int64_t* rows, +static inline void sg_pair_accum(ray_t* x, ray_t* y, uint16_t op, + const int64_t* rows, int64_t n, bool contig, int64_t r0, double* sx, double* sy, double* sxx, double* syy, double* sxy) { + (void)op; double ax = 0.0, ay = 0.0, axx = 0.0, ayy = 0.0, axy = 0.0; if (contig) { for (int64_t j = 0; j < n; j++) { @@ -7537,7 +7614,7 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { uint16_t op = c->agg_ops[a]; if (agg_is_binary_agg(op)) { double sx = 0.0, sy = 0.0, sxx = 0.0, syy = 0.0, sxy = 0.0; - sg_pair_accum(c->agg_vecs[a], c->agg_vecs2[a], rows, n, + sg_pair_accum(c->agg_vecs[a], c->agg_vecs2[a], op, rows, n, contig, r0, &sx, &sy, &sxx, &syy, &sxy); c->partials[idx].f = sx; if (c->partial_sumsq) c->partial_sumsq[idx] = sxx; @@ -7568,19 +7645,17 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { * directly whichever order they appear in). */ } else if (op == OP_COUNT) { /* counts[gi] above suffices. */ - } else if (c->agg_vecs[a]->type == RAY_F64) { + } else if (group_fp_type(c->agg_vecs[a]->type)) { /* NaN payload = null, skip from sum (mirror all_sum). */ - const double* restrict d = - (const double*)ray_data(c->agg_vecs[a]); double acc = 0.0; double ssq = 0.0; bool need_sq = c->partial_sumsq && (op == OP_STDDEV || op == OP_STDDEV_POP || op == OP_VAR || op == OP_VAR_POP); if (contig) { - const double* restrict dr = d + r0; for (int64_t j = 0; j < n; j++) { - double v = dr[j]; + double v = group_fp_at(ray_data(c->agg_vecs[a]), + c->agg_vecs[a]->type, r0 + j); if (RAY_LIKELY(v == v)) { acc += v; if (need_sq) ssq += v * v; @@ -7588,7 +7663,8 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { } } else { for (int64_t j = 0; j < n; j++) { - double v = d[rows[j]]; + double v = group_fp_at(ray_data(c->agg_vecs[a]), + c->agg_vecs[a]->type, rows[j]); if (RAY_LIKELY(v == v)) { acc += v; if (need_sq) ssq += v * v; @@ -7733,8 +7809,8 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!agg_type_admitted(aop, col->type)) return false; switch (col->type) { case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: - case RAY_TIME: case RAY_F64: break; - default: return false; /* F32 / exotic → generic path */ + case RAY_TIME: case RAY_F32: case RAY_F64: break; + default: return false; } agg_vecs[a] = col; if (pair) { @@ -7749,7 +7825,7 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!agg_type_admitted(aop, col2->type)) return false; switch (col2->type) { case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: - case RAY_F64: break; + case RAY_F32: case RAY_F64: break; default: return false; } agg_vecs2[a] = col2; @@ -7919,7 +7995,7 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, size_t si = (size_t)ti * n_aggs + a; bool pair = agg_is_binary_agg(ext->agg_ops[a]); if (pair || prod[a].enabled || ext->agg_ops[a] == OP_COUNT || - (agg_vecs[a] && agg_vecs[a]->type == RAY_F64)) + (agg_vecs[a] && group_fp_type(agg_vecs[a]->type))) sums[di].f += partials[si].f; else sums[di].i = (int64_t)((uint64_t)sums[di].i + @@ -8240,7 +8316,7 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { sums[a].i += group_strlen_at_cached( \ agg_vecs[a], dyn_row, strlen_sym_strings, strlen_sym_count); \ else if (agg_f64_mask & ((uint64_t)1 << a)) \ - sums[a].f += ((const double*)agg_ptrs[a])[dyn_row]; \ + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], dyn_row); \ else \ sums[a].i += read_col_i64(agg_ptrs[a], dyn_row, agg_types[a], 0); \ } \ @@ -8373,7 +8449,7 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { sums[a].i += group_strlen_at_cached( \ agg_vecs[a], dyn_row, strlen_sym_strings, strlen_sym_count); \ else if (agg_f64_mask & ((uint64_t)1 << a)) \ - sums[a].f += ((const double*)agg_ptrs[a])[dyn_row]; \ + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], dyn_row); \ else \ sums[a].i += read_col_i64(agg_ptrs[a], dyn_row, agg_types[a], 0);\ } \ @@ -8440,26 +8516,6 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { return NULL; } -/* Pin the DA-prescan dense-group finalize alignment. This function owns the - * dict/dense-array (DA) group path whose finalize loop (the total_groups / - * keep_min / range_count compaction region below, ~line 8395-8475, family - * da_count_emit_keep_min_u32) is the dominant symbol for high-cardinality - * group-by + count + desc-sort + take shapes. The unbounded-slots work - * added lines elsewhere in group.c - * (the legacy ght_layout_t hash path — unrelated, byte-identical to baseline - * at the hot lines), shifting this loop's absolute address/alignment and - * dropping IPC 2.07→1.96 (+7.2% cycles, +1.6% instructions — a placement - * artifact, not new work; RCA in .superpowers/sdd/task-2-bench-checkpoint.md). - * Same remedy as reduce_range's cut-3 pin: aligned(64) stabilizes the entry - * cacheline; align-loops=32 re-pins every loop head and align-jumps=32 the - * unrolled branch targets, making the finalize loop's internal alignment - * invariant to future whole-TU layout churn. Per-function attributes only — - * no global codegen flag. GCC-only: clang lacks the `optimize` attribute and - * -Werror would promote the unknown-attribute warning; the pin is - * performance-only, so clang builds go without it. */ -#if defined(__GNUC__) && !defined(__clang__) -__attribute__((aligned(64), optimize("align-loops=32","align-jumps=32"))) -#endif static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) { if (!tbl || RAY_IS_ERR(tbl)) return tbl; @@ -9002,7 +9058,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, t == RAY_DATE || t == RAY_TIME || t == RAY_TIMESTAMP); sc_int_null_has[a] = is_sentinel_typed && (agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS); if ((agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS) && - (agg_vecs[a]->type == RAY_F64 || is_sentinel_typed)) + (group_fp_type(agg_vecs[a]->type) || is_sentinel_typed)) sc_any_nullable = true; } else { agg_ptrs[a] = NULL; @@ -9186,7 +9242,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, n_aggs * sizeof(da_val_t)); if (!sc_acc[w].min_val) { alloc_ok = false; break; } for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) sc_acc[w].min_val[a].f = DBL_MAX; + if (group_fp_type(agg_types[a])) sc_acc[w].min_val[a].f = DBL_MAX; else sc_acc[w].min_val[a].i = INT64_MAX; } } @@ -9195,7 +9251,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, n_aggs * sizeof(da_val_t)); if (!sc_acc[w].max_val) { alloc_ok = false; break; } for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) sc_acc[w].max_val[a].f = -DBL_MAX; + if (group_fp_type(agg_types[a])) sc_acc[w].max_val[a].f = -DBL_MAX; else sc_acc[w].max_val[a].i = INT64_MIN; } } @@ -9252,7 +9308,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, scalar_fn_t sc_fn = scalar_accum_fn; bool agg0_has_nulls = n_aggs > 0 && (sc_int_null_has[0] || - (agg_vecs[0] && agg_vecs[0]->type == RAY_F64 && + (agg_vecs[0] && group_fp_type(agg_vecs[0]->type) && (agg_vecs[0]->attrs & RAY_ATTR_HAS_NULLS))); if (n_aggs == 1 && !match_idx && !rowsel && agg_ptrs[0] != NULL && !agg0_has_nulls) { uint16_t op0 = ext->agg_ops[0]; @@ -9294,13 +9350,13 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (wnn > 0) { if (mnn == 0) m->sum[a] = wa->sum[a]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) m->sum[a].f *= wa->sum[a].f; else m->sum[a].i = (int64_t)((uint64_t)m->sum[a].i * (uint64_t)wa->sum[a].i); } } else { - if (agg_types[a] == RAY_F64) + if (group_fp_type(agg_types[a])) m->sum[a].f += wa->sum[a].f; else m->sum[a].i += wa->sum[a].i; @@ -9313,7 +9369,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, } if (need_flags & DA_NEED_MIN) { for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[a].f < m->min_val[a].f) m->min_val[a].f = wa->min_val[a].f; } else if (agg_types[a] == RAY_SYM) { @@ -9330,7 +9386,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, } if (need_flags & DA_NEED_MAX) { for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[a].f > m->max_val[a].f) m->max_val[a].f = wa->max_val[a].f; } else if (agg_types[a] == RAY_SYM) { @@ -9479,22 +9535,14 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, da_path:; /* ---- Direct-array fast path for low-cardinality integer keys ---- */ - /* Supports multi-key via composite index: product of ranges <= MAX */ - #define DA_MAX_COMPOSITE_SLOTS 262144 /* 256K slots max */ - #define DA_MEM_BUDGET (256ULL << 20) /* 256 MB total across all workers */ - #define DA_PER_WORKER_MAX (6ULL << 20) /* 6 MB per-worker max */ + /* Supports multi-key via a composite index when its state remains O(input). */ { /* n_aggs <= 64 is not a slot cap — it is the width of the da_ctx_t * agg_f64_mask/agg_int_null_mask bitmasks (uint64_t, one bit per * agg). Wider agg lists route to the hash (HT) path instead, which * carries per-agg arrays rather than a fixed-width bitmask. */ - bool da_eligible = (nrows > 0 && n_keys > 0 && n_keys <= 8 && + bool da_eligible = (n_scan > 0 && n_keys > 0 && n_keys <= 8 && n_aggs <= 64); - if (da_eligible && rowsel && n_keys == 1) { - ray_rowsel_t* sm = ray_rowsel_meta(rowsel); - if (sm && sm->total_pass * 4 < nrows) - da_eligible = false; - } /* Binary aggregators (OP_PEARSON_CORR) are not wired into the * dense-array accumulator's per-worker da_accum_t struct — force * the HT path which has the row-layout offsets allocated. @@ -9537,7 +9585,8 @@ da_path:; * a known distinct-count (dense codes 0..n_distinct-1 over the full * column), so over the filtered survivors n_distinct is an upper bound * on its slot span. If the product of such KNOWN bounds already - * exceeds the DA budget, the composite is infeasible — reject now and + * exceeds the contributing row count, dense state would be superlinear + * in the input — reject now and * skip the min/max prescan (a full gather-scan of the survivors) * entirely. Conservative in the CORRECTNESS direction: keys without a * known count contribute factor 1, and rejecting only ever SKIPS the @@ -9550,12 +9599,12 @@ da_path:; if (key_scan_sym[k] < 0) continue; int64_t card = ray_grp_card_lookup(key_scan_sym[k]); if (card <= 1) continue; - if ((uint64_t)card > (uint64_t)DA_MAX_COMPOSITE_SLOTS / known_ub) { + if ((uint64_t)card > (uint64_t)n_scan / known_ub) { da_eligible = false; break; } known_ub *= (uint64_t)card; - if (known_ub > DA_MAX_COMPOSITE_SLOTS) { da_eligible = false; break; } + if (known_ub > (uint64_t)n_scan) { da_eligible = false; break; } } } @@ -9589,7 +9638,7 @@ da_path:; .n_workers = mm_n, .match_idx = match_idx, .rowsel = rowsel, - .span_budget = DA_MAX_COMPOSITE_SLOTS, + .span_budget = n_scan, .abort_flag = &mm_abort, }; if (mm_n > 1) { @@ -9613,37 +9662,17 @@ da_path:; if (span > (uint64_t)INT64_MAX) { da_fits = false; break; } da_key_range[k] = (int64_t)span; if (da_key_range[k] <= 0) { da_fits = false; break; } - total_slots *= (uint64_t)da_key_range[k]; - if (total_slots > DA_MAX_COMPOSITE_SLOTS) da_fits = false; + uint64_t range = (uint64_t)da_key_range[k]; + uint64_t slot_limit = (uint64_t)n_scan < INT32_MAX + ? (uint64_t)n_scan : INT32_MAX; + if (total_slots > slot_limit / range) { + da_fits = false; + break; + } + total_slots *= range; } } - if (da_fits) { - /* Compute which accumulator arrays we actually need */ - uint8_t need_flags = DA_NEED_COUNT; /* always need count */ - for (uint32_t a = 0; a < n_aggs; a++) { - uint16_t aop = ext->agg_ops[a]; - if (aop == OP_SUM || aop == OP_PROD || aop == OP_AVG || aop == OP_ALL || aop == OP_ANY || aop == OP_FIRST || aop == OP_LAST) need_flags |= DA_NEED_SUM; - else if (aop == OP_STDDEV || aop == OP_STDDEV_POP || aop == OP_VAR || aop == OP_VAR_POP) - { need_flags |= DA_NEED_SUM; need_flags |= DA_NEED_SUMSQ; } - else if (aop == OP_MIN) need_flags |= DA_NEED_MIN; - else if (aop == OP_MAX) need_flags |= DA_NEED_MAX; - } - - /* Compute per-worker memory budget. Actual allocation is 1 union - * array per type, but MIN/MAX use conditional random writes that - * perform worse than radix-partitioned HT at high group counts. - * Weight MIN/MAX at 2x to keep those queries on the HT path. */ - uint32_t arrays_per_agg = 0; - if (need_flags & DA_NEED_SUM) arrays_per_agg += 1; - if (need_flags & DA_NEED_MIN) arrays_per_agg += 2; /* 2x: DA MIN slow at high cardinality */ - if (need_flags & DA_NEED_MAX) arrays_per_agg += 2; /* 2x: DA MAX slow at high cardinality */ - if (need_flags & DA_NEED_SUMSQ) arrays_per_agg += 1; - uint64_t per_worker = total_slots * (arrays_per_agg * n_aggs + 1u) * 8u; - if (per_worker > DA_PER_WORKER_MAX) - da_fits = false; - } - if (da_fits) { /* Recompute need_flags (da_fits may have changed scope) */ uint8_t need_flags = DA_NEED_COUNT; @@ -9709,7 +9738,7 @@ da_path:; if (agg_vecs[a]) { agg_ptrs[a] = ray_data(agg_vecs[a]); agg_types[a] = agg_vecs[a]->type; - if (agg_vecs[a]->type == RAY_F64) + if (group_fp_type(agg_vecs[a]->type)) agg_f64_mask |= ((uint64_t)1 << a); da_int_null_sentinel[a] = agg_int_null_sentinel_for(agg_vecs[a]->type); /* Only set the int-null mask bit for storage types whose @@ -9723,7 +9752,7 @@ da_path:; if (is_sentinel_typed && (agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS)) da_int_null_mask |= ((uint64_t)1 << a); if ((agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS) && - (agg_vecs[a]->type == RAY_F64 || is_sentinel_typed)) + (group_fp_type(agg_vecs[a]->type) || is_sentinel_typed)) da_any_nullable = true; } else { agg_ptrs[a] = NULL; @@ -9736,19 +9765,22 @@ da_path:; uint32_t da_n_workers = (da_pool && nrows >= RAY_PARALLEL_THRESHOLD) ? ray_pool_total_workers(da_pool) : 1; - /* Check memory budget — need one accumulator set per worker. - * Weight MIN/MAX at 2x in budget (same as eligibility check) to - * keep MIN/MAX-heavy queries on the faster radix-HT path. */ + /* Bound aggregate state structurally: across workers, direct-array + * cells must remain O(contributing rows). */ uint32_t arrays_per_agg = 0; if (need_flags & DA_NEED_SUM) arrays_per_agg += 1; - if (need_flags & DA_NEED_MIN) arrays_per_agg += 2; - if (need_flags & DA_NEED_MAX) arrays_per_agg += 2; + if (need_flags & DA_NEED_MIN) arrays_per_agg += 1; + if (need_flags & DA_NEED_MAX) arrays_per_agg += 1; if (need_flags & DA_NEED_SUMSQ) arrays_per_agg += 1; /* Nullable aggs add a per-(group, agg) non-null count array. * ~8 bytes per (group, agg). */ if (da_any_nullable) arrays_per_agg += 1; - uint64_t per_worker_bytes = (uint64_t)n_slots * (arrays_per_agg * n_aggs + 1u) * 8u; - if ((uint64_t)da_n_workers * per_worker_bytes > DA_MEM_BUDGET) + uint64_t cells_per_worker = (uint64_t)n_slots + * ((uint64_t)arrays_per_agg * n_aggs + 1u); + uint64_t max_workers = cells_per_worker + ? (uint64_t)n_scan / cells_per_worker : 1; + if (max_workers < 1) max_workers = 1; + if ((uint64_t)da_n_workers > max_workers) da_n_workers = 1; ray_t* accums_hdr; @@ -9774,7 +9806,7 @@ da_path:; if (!accums[w].min_val) { alloc_ok = false; break; } for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) accums[w].min_val[i].f = DBL_MAX; + if (group_fp_type(agg_types[a])) accums[w].min_val[i].f = DBL_MAX; else accums[w].min_val[i].i = INT64_MAX; } } @@ -9784,7 +9816,7 @@ da_path:; if (!accums[w].max_val) { alloc_ok = false; break; } for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) accums[w].max_val[i].f = -DBL_MAX; + if (group_fp_type(agg_types[a])) accums[w].max_val[i].f = -DBL_MAX; else accums[w].max_val[i].i = INT64_MIN; } } @@ -9902,7 +9934,7 @@ da_path:; size_t idx = base + a; uint16_t aop = ext->agg_ops[a]; if (aop == OP_SUM || aop == OP_AVG || aop == OP_ALL || aop == OP_ANY || aop == OP_STDDEV || aop == OP_STDDEV_POP || aop == OP_VAR || aop == OP_VAR_POP) { - if (agg_types[a] == RAY_F64) merged->sum[idx].f += wa->sum[idx].f; + if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; } else if (aop == OP_PROD) { /* Use per-(group, agg) non-null counts when @@ -9913,7 +9945,7 @@ da_path:; if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); @@ -9931,7 +9963,7 @@ da_path:; if (need_flags & DA_NEED_MIN) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[i].f < merged->min_val[i].f) merged->min_val[i].f = wa->min_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -9951,7 +9983,7 @@ da_path:; if (need_flags & DA_NEED_MAX) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[i].f > merged->max_val[i].f) merged->max_val[i].f = wa->max_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10013,12 +10045,12 @@ da_path:; if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); } - } else if (agg_types[a] == RAY_F64) + } else if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; @@ -10028,7 +10060,7 @@ da_path:; if (need_flags & DA_NEED_MIN) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[i].f < merged->min_val[i].f) merged->min_val[i].f = wa->min_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10048,7 +10080,7 @@ da_path:; if (need_flags & DA_NEED_MAX) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[i].f > merged->max_val[i].f) merged->max_val[i].f = wa->max_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10245,7 +10277,7 @@ da_path:; if (agg_vecs[a]) { agg_ptrs[a] = ray_data(agg_vecs[a]); agg_types[a] = agg_vecs[a]->type; - if (agg_vecs[a]->type == RAY_F64) + if (group_fp_type(agg_vecs[a]->type)) agg_f64_mask |= ((uint64_t)1 << a); } else { agg_ptrs[a] = NULL; @@ -10347,7 +10379,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); @@ -10455,7 +10487,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); @@ -10544,7 +10576,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); } @@ -10685,7 +10717,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); } @@ -10763,7 +10795,8 @@ ht_path:; * stride budgets or the int8 value-slot count — none of which a ≤8 shape * can hit. */ ght_layout_t ght_layout; - if (!ght_compute_layout(&ght_layout, n_keys, n_aggs, agg_vecs, ght_need, + if (!ght_compute_layout(&ght_layout, n_keys, n_aggs, agg_vecs, agg_vecs2, + ght_need, ext->agg_ops, key_types)) { for (uint32_t a = 0; a < n_aggs; a++) { if (agg_owned[a] && agg_vecs[a]) ray_release(agg_vecs[a]); @@ -10774,17 +10807,13 @@ ht_path:; return ray_error("limit", "group: layout stride/slot budget exceeded (or OOM)"); } - /* Right-sized hash table: start small, rehash on load > 0.5 */ - uint32_t ht_cap = 256; - { - uint64_t target = (uint64_t)nrows < 65536 ? (uint64_t)nrows : 65536; - if (target < 256) target = 256; - while (ht_cap < target) ht_cap *= 2; - } + /* Sequential fallback grows strictly from observed groups. */ + uint32_t ht_cap = 2; /* Parallel path: radix-partitioned group-by */ ray_pool_t* pool = ray_pool_get(); uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; + uint32_t radix_n_parts = group_radix_part_count(n_total, n_scan); group_ht_t single_ht; group_ht_t top_ht; @@ -10796,6 +10825,10 @@ ht_path:; radix_buf_t* radix_bufs = NULL; ray_t* part_hts_hdr = NULL; group_ht_t* part_hts = NULL; + ray_t* part_offsets_hdr = NULL; + uint32_t* part_offsets = NULL; + ray_t* group_out_hdr = NULL; + uint32_t* group_out = NULL; /* Top-N-by-count (`select … by … desc:c take:N`) is served by the * parallel radix_v2 path below: phase1/phase2 build per-partition HTs @@ -10805,7 +10838,8 @@ ht_path:; * across all input sizes (measured), so there is no separate * size-gated shortcut here. */ - if (pool && nrows >= RAY_PARALLEL_THRESHOLD && n_total > 1) { + if (pool && n_total > 1 && + (nrows >= RAY_PARALLEL_THRESHOLD || use_topn_filter)) { /* Filtered (rowsel/match_idx) group-bys are allowed on the parallel * paths: both radix_v2_phase1_fn and radix_phase1_fn honour c->rowsel * (skip non-passing rows during scatter), so partitioned data holds @@ -10855,13 +10889,13 @@ ht_path:; if (v2_ok && !(ght_layout.agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST | GHT_AF_HOLISTIC | GHT_AF_BINARY))) { ray_t* wpart_hdr = NULL; - size_t v2_n_w = (size_t)n_total * RADIX_P; + size_t v2_n_w = (size_t)n_total * radix_n_parts; group_ht_t* wpart_hts = (group_ht_t*)scratch_calloc( &wpart_hdr, v2_n_w * sizeof(group_ht_t)); ray_t* v2_part_hdr = NULL; group_ht_t* v2_part_hts = wpart_hts ? (group_ht_t*)scratch_calloc(&v2_part_hdr, - RADIX_P * sizeof(group_ht_t)) + (size_t)radix_n_parts * sizeof(group_ht_t)) : NULL; if (!wpart_hts || !v2_part_hts) { if (wpart_hts) scratch_free(wpart_hdr); @@ -10904,6 +10938,7 @@ ht_path:; .agg_strlen = agg_strlen, .nullable_mask = v2_nullable, .n_workers = n_total, + .n_parts = radix_n_parts, .wpart_hts = wpart_hts, .rowsel = rowsel, .match_idx = sel_match, @@ -10929,6 +10964,7 @@ ht_path:; .part_hts = v2_part_hts, .key_types = key_types, .n_workers = n_total, + .n_parts = radix_n_parts, .key_data = key_data, .oom = 0, }; @@ -10952,7 +10988,8 @@ ht_path:; } derive_key_pool(&ght_layout, key_vecs, v2p2.key_pool); } - ray_pool_dispatch_n(pool, radix_v2_phase2_fn, &v2p2, RADIX_P); + ray_pool_dispatch_n(pool, radix_v2_phase2_fn, &v2p2, + radix_n_parts); scratch_free(v2p2_kp_hdr); CHECK_CANCEL_GOTO(pool, cleanup); /* Worker HTs are no longer needed once the merge is done. */ @@ -10960,7 +10997,7 @@ ht_path:; group_ht_free(&wpart_hts[i]); scratch_free(wpart_hdr); if (atomic_load_explicit(&v2p2.oom, memory_order_relaxed)) { - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < radix_n_parts; p++) group_ht_free(&v2_part_hts[p]); scratch_free(v2_part_hdr); goto v2_done; @@ -10971,31 +11008,12 @@ ht_path:; goto v2_emit; } v2_done:; - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * radix_n_parts; radix_bufs = (radix_buf_t*)scratch_calloc(&radix_bufs_hdr, n_bufs * sizeof(radix_buf_t)); if (!radix_bufs) goto sequential_fallback; - /* Pre-size each buffer: 1.5x expected, capped so total ≤ 2 GB. - * Buffers grow on demand via radix_buf_push doubling. */ - uint32_t buf_init = (uint32_t)((uint64_t)nrows / (RADIX_P * n_total)); - if (buf_init < 64) buf_init = 64; - buf_init = buf_init + buf_init / 2; /* 1.5x headroom */ - uint16_t estride = ght_layout.entry_stride; - { - /* Cap: total pre-alloc ≤ 2 GB */ - size_t total_pre = (size_t)n_bufs * buf_init * estride; - if (total_pre > (size_t)2 << 30) { - buf_init = (uint32_t)(((size_t)2 << 30) / ((size_t)n_bufs * estride)); - if (buf_init < 64) buf_init = 64; - } - } - for (size_t i = 0; i < n_bufs; i++) { - radix_bufs[i].data = (char*)scratch_alloc( - &radix_bufs[i]._hdr, (size_t)buf_init * estride); - radix_bufs[i].count = 0; - radix_bufs[i].cap = buf_init; - } + /* Buffers start empty and grow from observed partition demand. */ /* Any key column that may hold nulls — lets phase1 skip the per-key * null check on the common (no-nulls) fast path. 0/1 summary so key @@ -11015,6 +11033,7 @@ v2_done:; .agg_vecs2 = agg_vecs2, .agg_strlen = agg_strlen, .n_workers = n_total, + .n_parts = radix_n_parts, .bufs = radix_bufs, .rowsel = rowsel, .match_idx = match_idx, @@ -11049,7 +11068,7 @@ v2_done:; /* Pass 2: parallel per-partition aggregation (no column access) */ part_hts = (group_ht_t*)scratch_calloc(&part_hts_hdr, - RADIX_P * sizeof(group_ht_t)); + (size_t)radix_n_parts * sizeof(group_ht_t)); if (!part_hts) { for (size_t i = 0; i < n_bufs; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); @@ -11061,6 +11080,7 @@ v2_done:; .key_types = key_types, .n_keys = n_keys, .n_workers = n_total, + .n_parts = radix_n_parts, .bufs = radix_bufs, .part_hts = part_hts, .key_data = key_data, @@ -11076,12 +11096,12 @@ v2_done:; if (!p2ctx.key_pool) { result = ray_error("oom", NULL); goto cleanup; } derive_key_pool(&ght_layout, key_vecs, p2ctx.key_pool); } - ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, radix_n_parts); scratch_free(p2_kp_hdr); CHECK_CANCEL_GOTO(pool, cleanup); if (radix_bufs) { - size_t n_bufs_free = (size_t)n_total * RADIX_P; + size_t n_bufs_free = (size_t)n_total * radix_n_parts; for (size_t i = 0; i < n_bufs_free; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); @@ -11099,7 +11119,7 @@ v2_emit:; * partition's row array to contain only globally-surviving rows. * Phase3 below then emits N rows total instead of total_grps — * the major win for high-cardinality keys like UserID/URL where - * total_grps is in the millions but N is ≤ 1024. + * total_grps is in the millions but N is much smaller. * * Implementation notes: * - The bounded heap orders by count (the agg at COUNT slot, the @@ -11113,8 +11133,8 @@ v2_emit:; * min_count_exclusive-only filters fall through unchanged. */ if (use_topn_filter) { int64_t k_take = emit_filter.top_count_take; - uint32_t total_pre = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + uint64_t total_pre = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) total_pre += part_hts[p].grp_count; /* Resolve the in-row offset of the order-by agg's value. For * COUNT it's the leading int64 at offset 0; for SUM/MIN/MAX @@ -11158,15 +11178,16 @@ v2_emit:; * isn't set (old single-bit filter shape). Producer code in * query.c sets it explicitly. */ if (order_op == OP_COUNT && !emit_filter.desc) desc_dir = 1; - if ((int64_t)total_pre > k_take && k_take > 0 && k_take <= 1024) { - /* Stack heap: (val, part, gid) triples. k_take ≤ 1024 - * caps the footprint at 1024 * 16 B = 16 KiB. The heap - * invariant flips by direction: min-heap for desc (we - * evict the smallest to keep the largest N), max-heap - * for asc (evict the largest to keep the smallest N). */ - int64_t hval[1024]; - uint32_t hpart[1024]; - uint32_t hgid[1024]; + if (total_pre > (uint64_t)k_take && k_take > 0 && + (uint64_t)k_take <= SIZE_MAX / sizeof(group_topn_item_t)) { + /* The heap is sized by the requested result cardinality. + * There is no machine-tuned maximum: allocation failure only + * skips this optional compaction, leaving the normal final + * sort/take path to produce the same answer. */ + ray_t* heap_hdr = NULL; + group_topn_item_t* heap = (group_topn_item_t*)scratch_alloc( + &heap_hdr, (size_t)k_take * sizeof(group_topn_item_t)); + if (!heap) goto topn_compact_skip; int64_t hn = 0; /* For top-N largest (desc=1): min-heap. Root is smallest; * incoming v replaces root iff v > root. Heap invariant: @@ -11182,7 +11203,7 @@ v2_emit:; (desc_dir ? ((parent) > (child)) : ((parent) < (child))) #define TOPN_SHOULD_REPLACE(new_v, root_v) \ (desc_dir ? ((new_v) > (root_v)) : ((new_v) < (root_v))) - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < radix_n_parts; p++) { group_ht_t* ph = &part_hts[p]; uint16_t rs = ph->layout.row_stride; uint32_t gc = ph->grp_count; @@ -11192,31 +11213,32 @@ v2_emit:; (row + order_off); if (hn < k_take) { int64_t j = hn++; - hval[j] = v; hpart[j] = p; hgid[j] = gi; + heap[j] = (group_topn_item_t){v, p, gi}; /* Sift up: bubble new entry toward root while * parent violates invariant. */ while (j > 0) { int64_t pr = (j - 1) >> 1; - if (!TOPN_NEEDS_SWAP(hval[pr], hval[j])) break; - int64_t tc = hval[pr]; hval[pr] = hval[j]; hval[j] = tc; - uint32_t tp = hpart[pr]; hpart[pr] = hpart[j]; hpart[j] = tp; - uint32_t tg = hgid[pr]; hgid[pr] = hgid[j]; hgid[j] = tg; + if (!TOPN_NEEDS_SWAP(heap[pr].value, + heap[j].value)) break; + group_topn_item_t tmp = heap[pr]; + heap[pr] = heap[j]; heap[j] = tmp; j = pr; } - } else if (TOPN_SHOULD_REPLACE(v, hval[0])) { - hval[0] = v; hpart[0] = p; hgid[0] = gi; + } else if (TOPN_SHOULD_REPLACE(v, heap[0].value)) { + heap[0] = (group_topn_item_t){v, p, gi}; int64_t j = 0; /* Sift down: find the child that should be * promoted (the one most violating the * invariant) and swap. */ for (;;) { int64_t l = j * 2 + 1, r = l + 1, m = j; - if (l < hn && TOPN_NEEDS_SWAP(hval[m], hval[l])) m = l; - if (r < hn && TOPN_NEEDS_SWAP(hval[m], hval[r])) m = r; + if (l < hn && TOPN_NEEDS_SWAP(heap[m].value, + heap[l].value)) m = l; + if (r < hn && TOPN_NEEDS_SWAP(heap[m].value, + heap[r].value)) m = r; if (m == j) break; - int64_t tc = hval[m]; hval[m] = hval[j]; hval[j] = tc; - uint32_t tp = hpart[m]; hpart[m] = hpart[j]; hpart[j] = tp; - uint32_t tg = hgid[m]; hgid[m] = hgid[j]; hgid[j] = tg; + group_topn_item_t tmp = heap[m]; + heap[m] = heap[j]; heap[j] = tmp; j = m; } } @@ -11225,56 +11247,124 @@ v2_emit:; #undef TOPN_NEEDS_SWAP #undef TOPN_SHOULD_REPLACE if (hn > 0) { - /* Build per-partition keep lists (sorted asc by gid so - * the in-place compact below is a single forward sweep). */ - uint16_t keep_n[RADIX_P]; - for (uint32_t p = 0; p < RADIX_P; p++) keep_n[p] = 0; - /* Cap per-partition kept count at hn (≤ k_take ≤ 1024). */ - uint32_t kgid[RADIX_P][1024]; - for (int64_t i = 0; i < hn; i++) { - uint32_t p = hpart[i]; - uint16_t kn = keep_n[p]; - /* Insertion-sort into kgid[p][] keeping asc order. */ - uint16_t j = kn; - while (j > 0 && kgid[p][j - 1] > hgid[i]) { - kgid[p][j] = kgid[p][j - 1]; - j--; + uint64_t all_groups = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) + all_groups += part_hts[p].grp_count; + ray_t* keep_hdr = NULL; + ray_t* off_hdr = NULL; + uint8_t* keep = all_groups <= SIZE_MAX + ? (uint8_t*)scratch_calloc(&keep_hdr, + (size_t)all_groups) + : NULL; + uint32_t* keep_off = + (size_t)radix_n_parts + 1 <= SIZE_MAX / sizeof(uint32_t) + ? (uint32_t*)scratch_alloc(&off_hdr, + ((size_t)radix_n_parts + 1) * sizeof(uint32_t)) + : NULL; + if (keep && keep_off) { + keep_off[0] = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) + keep_off[p + 1] = keep_off[p] + + part_hts[p].grp_count; + for (int64_t i = 0; i < hn; i++) + keep[keep_off[heap[i].part] + heap[i].gid] = 1; + + /* In-place compact each partition in original gid + * order, preserving deterministic tie behaviour. */ + bool rebuilt_slots = false; + for (uint32_t p = 0; p < radix_n_parts; p++) { + group_ht_t* ph = &part_hts[p]; + uint16_t rs = ph->layout.row_stride; + uint32_t old_n = ph->grp_count; + uint32_t kn = 0; + for (uint32_t gi = 0; gi < old_n; gi++) { + if (!keep[keep_off[p] + gi]) continue; + if (gi != kn) + memmove(ph->rows + (size_t)kn * rs, + ph->rows + (size_t)gi * rs, rs); + kn++; + } + if (kn == ph->grp_count) continue; + rebuilt_slots = true; + ph->grp_count = kn; } - kgid[p][j] = hgid[i]; - keep_n[p] = (uint16_t)(kn + 1); - } - /* In-place compact each partition. */ - bool rebuilt_slots = false; - for (uint32_t p = 0; p < RADIX_P; p++) { - group_ht_t* ph = &part_hts[p]; - uint16_t rs = ph->layout.row_stride; - uint16_t kn = keep_n[p]; - if (kn == ph->grp_count) continue; /* all kept */ - rebuilt_slots = true; - if (kn == 0) { ph->grp_count = 0; continue; } - for (uint16_t i = 0; i < kn; i++) { - uint32_t src = kgid[p][i]; - if (src == (uint32_t)i) continue; - memmove(ph->rows + (size_t)i * rs, - ph->rows + (size_t)src * rs, rs); + if (rebuilt_slots) { + for (uint32_t p = 0; p < radix_n_parts; p++) + group_ht_rebuild_slots(&part_hts[p], key_types); } - ph->grp_count = kn; - } - if (rebuilt_slots) { - for (uint32_t p = 0; p < RADIX_P; p++) - group_ht_rebuild_slots(&part_hts[p], key_types); } + scratch_free(off_hdr); + scratch_free(keep_hdr); } + scratch_free(heap_hdr); } topn_compact_skip:; } /* Prefix offsets */ - uint32_t part_offsets[RADIX_P + 1]; + part_offsets = (uint32_t*)scratch_alloc(&part_offsets_hdr, + ((size_t)radix_n_parts + 1) * sizeof(uint32_t)); + if (!part_offsets) { result = ray_error("oom", NULL); goto cleanup; } part_offsets[0] = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < radix_n_parts; p++) part_offsets[p + 1] = part_offsets[p] + part_hts[p].grp_count; - uint32_t total_grps = part_offsets[RADIX_P]; + uint32_t total_grps = part_offsets[radix_n_parts]; + + /* Radix partitioning is an execution detail, not an ordering rule. + * Index each group by its earliest source row, then scan that row + * domain once to assign stable output ids. This is a single linear + * algorithm: no comparison sort, crossover, or fixed fanout. */ + if (total_grps > 0) { + group_out = (uint32_t*)scratch_alloc(&group_out_hdr, + (size_t)total_grps * sizeof(uint32_t)); + if (!group_out) { + result = ray_error("oom", NULL); + goto cleanup; + } + uint64_t row_count = nrows > 0 ? (uint64_t)nrows : 0; + if (row_count > SIZE_MAX / sizeof(uint32_t)) { + result = ray_error("limit", "group row index is too large"); + goto cleanup; + } + ray_t* first_hdr = NULL; + uint32_t* first_to_flat = (uint32_t*)scratch_alloc( + &first_hdr, (size_t)row_count * sizeof(uint32_t)); + if (!first_to_flat) { + result = ray_error("oom", NULL); + goto cleanup; + } + memset(first_to_flat, 0xff, + (size_t)row_count * sizeof(uint32_t)); + bool order_ok = true; + for (uint32_t p = 0; p < radix_n_parts && order_ok; p++) { + group_ht_t* ph = &part_hts[p]; + for (uint32_t gi = 0; gi < ph->grp_count; gi++) { + uint32_t flat = part_offsets[p] + gi; + const char* group_row = ph->rows + + (size_t)gi * ph->layout.row_stride; + int64_t first; + memcpy(&first, + group_row + ph->layout.off_group_first, 8); + if (first < 0 || first >= nrows || + first_to_flat[first] != UINT32_MAX) { + order_ok = false; + break; + } + first_to_flat[first] = flat; + } + } + uint32_t out = 0; + if (order_ok) + for (int64_t row = 0; row < nrows; row++) { + uint32_t flat = first_to_flat[row]; + if (flat != UINT32_MAX) group_out[flat] = out++; + } + scratch_free(first_hdr); + if (!order_ok || out != total_grps) { + result = ray_error("group", "failed to order all groups"); + goto cleanup; + } + } /* Build result directly from partition HTs */ int64_t total_cols = n_keys + n_aggs; @@ -11333,7 +11423,7 @@ v2_emit:; for (uint32_t a = 0; a < n_aggs; a++) { uint16_t agg_op = ext->agg_ops[a]; ray_t* agg_col = agg_vecs[a]; - bool is_f64 = agg_col && agg_col->type == RAY_F64; + bool is_f64 = (ght_layout.agg_flags[a] & GHT_AF_F64) != 0; int8_t out_type; switch (agg_op) { case OP_AVG: @@ -11359,7 +11449,8 @@ v2_emit:; case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col = ray_vec_new(out_type, (int64_t)total_grps); if (!new_col || RAY_IS_ERR(new_col)) { @@ -11389,6 +11480,7 @@ v2_emit:; radix_phase3_ctx_t p3ctx = { .part_hts = part_hts, .part_offsets = part_offsets, + .group_out = group_out, .key_dsts = key_dsts, .key_types = key_out_types, .key_attrs = key_attrs, @@ -11399,7 +11491,8 @@ v2_emit:; .n_aggs = n_aggs, .key_src_data = key_data, }; - ray_pool_dispatch_n(pool, radix_phase3_fn, &p3ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase3_fn, &p3ctx, + radix_n_parts); } /* Post-radix holistic fill: OP_MEDIAN slots need a per-group @@ -11431,6 +11524,8 @@ v2_emit:; .layout = &ght_layout, .part_hts = part_hts, .part_offsets = part_offsets, + .group_out = group_out, + .n_parts = radix_n_parts, .row_gid = row_gid, .match_idx = match_idx, .rowsel = rowsel, @@ -11952,7 +12047,7 @@ sequential_fallback:; for (uint32_t a = 0; a < n_aggs; a++) { uint16_t agg_op = ext->agg_ops[a]; ray_t* agg_col = agg_vecs[a]; - bool is_f64 = agg_col && agg_col->type == RAY_F64; + bool is_f64 = (ly->agg_flags[a] & GHT_AF_F64) != 0; int8_t out_type; switch (agg_op) { case OP_AVG: @@ -11984,7 +12079,8 @@ sequential_fallback:; case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col; /* Drive off the layout bitmask, not the op literal: wide-element @@ -12204,16 +12300,18 @@ sequential_fallback:; group_ht_free(&top_ht); } if (radix_bufs) { - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * radix_n_parts; for (size_t i = 0; i < n_bufs; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); } if (part_hts) { - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < radix_n_parts; p++) { if (part_hts[p].rows) group_ht_free(&part_hts[p]); } scratch_free(part_hts_hdr); } + scratch_free(part_offsets_hdr); + scratch_free(group_out_hdr); /* Master layout owns any spill block; every by-value copy borrowed it. * cleanup: is reached only after ght_layout is initialised (all gotos to * it are below the ght_compute_layout call). NULL-safe / no-op inline. */ @@ -13144,7 +13242,7 @@ exec_group_per_partition(ray_graph_t* g, ray_t* parted_tbl, ray_op_ext_t* ext, * the result in per-partition HTs with prefix offsets so the caller * can iterate grouped rows without knowing about the radix internals. * Falls back to a single sequential HT for tiny inputs or when no - * pool is available — the caller iterates n_parts ∈ {1, RADIX_P}. + * pool is available — the caller iterates the returned n_parts value. * ══════════════════════════════════════════════════════════════════════ */ static void pivot_ingest_sequential(pivot_ingest_t* out, const ght_layout_t* ly, @@ -13172,9 +13270,15 @@ bool pivot_ingest_run(pivot_ingest_t* out, memset(out, 0, sizeof(*out)); out->row_stride = ly->row_stride; - /* Allocate a small offsets buffer up front (RADIX_P+1 is the max). */ + ray_pool_t* pool = ray_pool_get(); + uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; + bool parallel_ok = (pool && n_scan >= RAY_PARALLEL_THRESHOLD && n_total > 1); + uint32_t n_parts = parallel_ok + ? group_radix_part_count(n_total, n_scan) : 1; + + /* Prefix offsets are sized by the execution-derived partition count. */ out->part_offsets = (uint32_t*)scratch_alloc(&out->_offsets_hdr, - (size_t)(RADIX_P + 1) * sizeof(uint32_t)); + ((size_t)n_parts + 1) * sizeof(uint32_t)); if (!out->part_offsets) return false; /* uint32_t, not uint8_t: ly->n_keys is uint16_t (unbounded-ready, Task @@ -13187,10 +13291,6 @@ bool pivot_ingest_run(pivot_ingest_t* out, * index key is wide (RAY_GUID/RAY_STR) and n_keys > 255. */ uint32_t n_keys = ly->n_keys; - ray_pool_t* pool = ray_pool_get(); - uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; - bool parallel_ok = (pool && n_scan >= RAY_PARALLEL_THRESHOLD && n_total > 1); - if (!parallel_ok) { /* Sequential single-HT path — allocate the HT in its own scratch * block and wire part_hts/n_parts immediately so every failure @@ -13200,10 +13300,7 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (!seq) return false; out->part_hts = seq; out->n_parts = 1; - uint32_t seq_cap = 1024; - uint64_t target = (uint64_t)n_scan * 2; - while ((uint64_t)seq_cap < target && seq_cap < (1u << 24)) seq_cap <<= 1; - if (!group_ht_init(seq, seq_cap, ly)) return false; + if (!group_ht_init(seq, 2, ly)) return false; pivot_ingest_sequential(out, ly, key_data, key_types, key_attrs, key_vecs, agg_vecs, n_scan, seq); /* Surface grow-path OOM from group_probe_entry so callers don't @@ -13213,30 +13310,14 @@ bool pivot_ingest_run(pivot_ingest_t* out, } /* ═════ Parallel radix path ═════ */ - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * n_parts; out->_n_bufs = n_bufs; radix_buf_t* radix_bufs = (radix_buf_t*)scratch_calloc(&out->_radix_bufs_hdr, n_bufs * sizeof(radix_buf_t)); if (!radix_bufs) return false; out->_radix_bufs = radix_bufs; - uint32_t buf_init = (uint32_t)((uint64_t)n_scan / (RADIX_P * n_total)); - if (buf_init < 64) buf_init = 64; - buf_init = buf_init + buf_init / 2; - uint16_t estride = ly->entry_stride; - { - size_t total_pre = (size_t)n_bufs * buf_init * estride; - if (total_pre > (size_t)2 << 30) { - buf_init = (uint32_t)(((size_t)2 << 30) / ((size_t)n_bufs * estride)); - if (buf_init < 64) buf_init = 64; - } - } - for (size_t i = 0; i < n_bufs; i++) { - radix_bufs[i].data = (char*)scratch_alloc(&radix_bufs[i]._hdr, - (size_t)buf_init * estride); - radix_bufs[i].count = 0; - radix_bufs[i].cap = buf_init; - } + /* Buffers start empty and grow from observed partition demand. */ uint8_t p1_nullable = 0; /* 0/1: any key may be null (per-key re-checked in phase1) */ for (uint32_t k = 0; k < n_keys; k++) @@ -13251,6 +13332,7 @@ bool pivot_ingest_run(pivot_ingest_t* out, .agg_vecs = agg_vecs, .agg_vecs2 = NULL, /* this scratch path doesn't use binary aggs */ .n_workers = n_total, + .n_parts = n_parts, .bufs = radix_bufs, .match_idx = NULL, }; @@ -13273,18 +13355,19 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (radix_bufs[i].oom) return false; group_ht_t* part_hts = (group_ht_t*)scratch_calloc(&out->_part_hts_hdr, - RADIX_P * sizeof(group_ht_t)); + (size_t)n_parts * sizeof(group_ht_t)); if (!part_hts) return false; /* Wire out->part_hts/n_parts at alloc time (like the sequential path) so * every failure below funnels through pivot_ingest_free — the calloc-zeroed * HTs are skipped by its rows||slots guard until phase2 fills them. */ out->part_hts = part_hts; - out->n_parts = RADIX_P; + out->n_parts = n_parts; radix_phase2_ctx_t p2ctx = { .key_types = key_types, .n_keys = n_keys, .n_workers = n_total, + .n_parts = n_parts, .bufs = radix_bufs, .part_hts = part_hts, .key_data = key_data, @@ -13298,11 +13381,11 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (!p2ctx.key_pool) return false; derive_key_pool(ly, key_vecs, p2ctx.key_pool); } - ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, n_parts); scratch_free(p2_kp_hdr); if (ray_interrupted()) return true; - /* Sync point — partitions materialized; show RADIX_P/RADIX_P. */ - ray_progress_update(NULL, "per-partition aggregate", RADIX_P, RADIX_P); + /* Sync point — every execution-derived partition is materialized. */ + ray_progress_update(NULL, "per-partition aggregate", n_parts, n_parts); /* OOM detection for the parallel path. Two distinct failure modes * must be caught here so callers never see a silently-truncated @@ -13314,19 +13397,19 @@ bool pivot_ingest_run(pivot_ingest_t* out, * (b) grow-path OOM — group_probe_entry sets part_hts[p].oom * on scratch_realloc failure and returns without inserting * the key, silently truncating later groups. */ - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < n_parts; p++) { if (part_hts[p].oom) return false; if (part_hts[p].rows) continue; uint32_t pcount = 0; for (uint32_t w = 0; w < n_total; w++) - pcount += radix_bufs[(size_t)w * RADIX_P + p].count; + pcount += radix_bufs[(size_t)w * n_parts + p].count; if (pcount) return false; } out->part_offsets[0] = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < n_parts; p++) out->part_offsets[p + 1] = out->part_offsets[p] + part_hts[p].grp_count; - out->total_grps = out->part_offsets[RADIX_P]; + out->total_grps = out->part_offsets[n_parts]; return true; } diff --git a/src/ops/internal.h b/src/ops/internal.h index ac2493ab3..3c6034bc4 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -804,7 +804,7 @@ static inline uint8_t radix_key_bytes(int8_t type) { switch (type) { case RAY_BOOL: case RAY_U8: return 1; case RAY_I16: return 2; - case RAY_I32: case RAY_DATE: case RAY_TIME: return 4; + case RAY_I32: case RAY_F32: case RAY_DATE: case RAY_TIME: return 4; default: return 8; /* I64, F64, TIMESTAMP, SYM */ } } @@ -1116,6 +1116,10 @@ typedef struct { uint16_t off_sum_y; uint16_t off_sumsq_y; uint16_t off_sumxy; + /* Earliest contributing source row for this group. Every packed entry + * carries its source row in the tail slot; partition merges retain the + * minimum so output order is independent of radix partition count. */ + uint16_t off_group_first; /* Per-slot non-null count block (n_agg_vals int64 slots), allocated only * when any_agg_null (any agg input column HAS_NULLS). Zero otherwise — * finalize then divides by the group row count exactly as before, so @@ -1247,7 +1251,8 @@ static inline int64_t agg_int_null_sentinel_for(int8_t t) { } bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, - ray_t** agg_vecs, uint8_t need_flags, + ray_t** agg_vecs, ray_t** agg_vecs2, + uint8_t need_flags, const uint16_t* agg_ops, const int8_t* key_types); /* By-value copy that fixes the base pointers. Dispatches on STORAGE, not @@ -1320,7 +1325,7 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, typedef struct { group_ht_t* part_hts; /* n_parts entries */ uint32_t* part_offsets; /* n_parts+1 entries (prefix sums of grp_counts) */ - uint32_t n_parts; /* 1 when sequential, RADIX_P when parallel */ + uint32_t n_parts; /* 1 sequential; execution-derived when parallel */ uint32_t total_grps; uint16_t row_stride; diff --git a/src/ops/opt.c b/src/ops/opt.c index 369356c28..208416c9c 100644 --- a/src/ops/opt.c +++ b/src/ops/opt.c @@ -79,6 +79,8 @@ bool ray_opt_no_group_pushdown = false; static int8_t promote_type(int8_t a, int8_t b) { if (a == RAY_STR || b == RAY_STR) return RAY_STR; if (a == RAY_F64 || b == RAY_F64) return RAY_F64; + if (a == RAY_F32 || b == RAY_F32) + return (a == RAY_F32 && b == RAY_F32) ? RAY_F32 : RAY_F64; /* Treat SYM/TIMESTAMP/DATE/TIME as integer-class types */ if (a == RAY_I64 || b == RAY_I64 || a == RAY_SYM || b == RAY_SYM || a == RAY_TIMESTAMP || b == RAY_TIMESTAMP) return RAY_I64; @@ -293,6 +295,11 @@ static bool atom_to_numeric(ray_t* v, double* out_f, int64_t* out_i, bool* is_f6 *out_i = (int64_t)v->f64; *is_f64 = true; return true; + case -RAY_F32: + *out_f = (double)(float)v->f64; + *out_i = (int64_t)(float)v->f64; + *is_f64 = true; + return true; case -RAY_I64: case -RAY_SYM: case -RAY_DATE: @@ -327,7 +334,8 @@ static bool pow_atom_type_admitted(ray_t* v) { if (!v || !ray_is_atom(v)) return false; return v->type == -RAY_BOOL || v->type == -RAY_U8 || v->type == -RAY_I16 || v->type == -RAY_I32 || - v->type == -RAY_I64 || v->type == -RAY_F64; + v->type == -RAY_I64 || v->type == -RAY_F32 || + v->type == -RAY_F64; } static bool replace_with_const(ray_graph_t* g, ray_op_t* node, ray_t* literal) { @@ -367,12 +375,15 @@ static bool fold_unary_const(ray_graph_t* g, ray_op_t* node) { ray_t* folded = NULL; switch (node->opcode) { case OP_NEG: - if (is_f64) folded = ray_f64(-vf); + if (node->out_type == RAY_F32) folded = ray_f32((float)-vf); + else if (is_f64) folded = ray_f64(-vf); else if (vi == INT64_MIN) return false; /* -INT64_MIN overflows */ else folded = ray_i64(-vi); break; case OP_ABS: - if (is_f64) + if (node->out_type == RAY_F32) + folded = ray_f32(fabsf((float)vf)); + else if (is_f64) folded = ray_f64(fabs(vf)); else if (vi == INT64_MIN) return false; /* -INT64_MIN overflows */ else folded = ray_i64(vi < 0 ? -vi : vi); @@ -418,10 +429,12 @@ static bool fold_unary_const(ray_graph_t* g, ray_op_t* node) { break; } case OP_CEIL: - folded = is_f64 ? ray_f64(ceil(vf)) : ray_i64(vi); + folded = node->out_type == RAY_F32 ? ray_f32(ceilf((float)vf)) + : is_f64 ? ray_f64(ceil(vf)) : ray_i64(vi); break; case OP_FLOOR: - folded = is_f64 ? ray_f64(floor(vf)) : ray_i64(vi); + folded = node->out_type == RAY_F32 ? ray_f32(floorf((float)vf)) + : is_f64 ? ray_f64(floor(vf)) : ray_i64(vi); break; default: return false; @@ -453,6 +466,28 @@ static bool fold_binary_const(ray_graph_t* g, ray_op_t* node) { ray_t* folded = NULL; switch (node->out_type) { + case RAY_F32: { + float lv = (float)(l_is_f64 ? lf : (double)li); + float rv = (float)(r_is_f64 ? rf : (double)ri); + float r = 0.0f; + switch (node->opcode) { + case OP_ADD: r = lv + rv; break; + case OP_SUB: r = lv - rv; break; + case OP_MUL: r = lv * rv; break; + case OP_MOD: + if (rv == 0.0f) r = NULL_F32; + else { + r = fmodf(lv, rv); + if (r != 0.0f && ((r > 0.0f) != (rv > 0.0f))) r += rv; + } + break; + case OP_MIN2: r = lv < rv ? lv : rv; break; + case OP_MAX2: r = lv > rv ? lv : rv; break; + default: return false; + } + folded = ray_f32(isfinite(r) ? r : NULL_F32); + break; + } case RAY_F64: { double lv = l_is_f64 ? lf : (double)li; double rv = r_is_f64 ? rf : (double)ri; diff --git a/src/ops/pivot.c b/src/ops/pivot.c index 7ef5a0e06..4b8f14457 100644 --- a/src/ops/pivot.c +++ b/src/ops/pivot.c @@ -937,7 +937,8 @@ ray_t* exec_pivot(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * point; every exit below (early return AND pivot_cleanup) must call * it exactly once so a spilled layout's heap block is never leaked. */ ght_layout_t ly; - if (!ght_compute_layout(&ly, n_keys, 1, agg_vecs, need_flags, agg_ops, key_types)) { + if (!ght_compute_layout(&ly, n_keys, 1, agg_vecs, NULL, + need_flags, agg_ops, key_types)) { scratch_free(key_hdr); return ray_error("limit", "pivot: key stride budget exceeded"); } diff --git a/src/ops/rowsel.c b/src/ops/rowsel.c index 321e8d709..0dc883587 100644 --- a/src/ops/rowsel.c +++ b/src/ops/rowsel.c @@ -681,9 +681,8 @@ ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b) { else if (bv < av) bp++; else { oi[cum++] = av; ap++; bp++; } } + of[s] = count == 0 ? RAY_SEL_NONE : RAY_SEL_MIX; } - if (of[s] != RAY_SEL_ALL) - of[s] = cum == oo[s] ? RAY_SEL_NONE : RAY_SEL_MIX; } } oo[ns] = cum; diff --git a/src/ops/sort.c b/src/ops/sort.c index 09a0738ca..b50abf921 100644 --- a/src/ops/sort.c +++ b/src/ops/sort.c @@ -55,6 +55,11 @@ int sort_cmp(const sort_cmp_ctx_t* ctx, int64_t a, int64_t b) { double vb = ((double*)ray_data(col))[b]; if (va < vb) cmp = -1; else if (va > vb) cmp = 1; + } else if (col->type == RAY_F32) { + float va = ((float*)ray_data(col))[a]; + float vb = ((float*)ray_data(col))[b]; + if (va < vb) cmp = -1; + else if (va > vb) cmp = 1; } else if (col->type == RAY_I64 || col->type == RAY_TIMESTAMP) { int64_t va = ((int64_t*)ray_data(col))[a]; int64_t vb = ((int64_t*)ray_data(col))[b]; @@ -997,6 +1002,26 @@ void radix_encode_fn(void* arg, uint32_t wid, int64_t start, int64_t end) { } break; } + case RAY_F32: { + const float* d = (const float*)c->data; + bool nf = c->nulls_first; + bool desc = c->desc; + uint32_t nan_e = (nf ^ desc) ? 0U : UINT32_MAX; + for (int64_t i = start; i < end; i++) { + uint32_t bits; + memcpy(&bits, &d[i], 4); + uint32_t e; + if ((bits & 0x7F800000U) == 0x7F800000U && + (bits & 0x007FFFFFU)) { + e = nan_e; + } else { + uint32_t mask = -(bits >> 31) | ((uint32_t)1 << 31); + e = bits ^ mask; + } + c->keys[i] = desc ? (uint64_t)(~e) : (uint64_t)e; + } + break; + } case RAY_I32: case RAY_DATE: case RAY_TIME: { const int32_t* d = (const int32_t*)c->data; bool has_nulls = c->col && (c->col->attrs & RAY_ATTR_HAS_NULLS); @@ -2215,6 +2240,15 @@ static void radix_decode_into(void* dst, int8_t type, const uint64_t* sorted_key uint64_t bits = k ^ mask; memcpy(&d[i], &bits, 8); } + } else if (type == RAY_F32) { + float* d = (float*)dst; + for (int64_t i = 0; i < n; i++) { + uint32_t k = (uint32_t)sorted_keys[i]; + if (desc) k = ~k; + uint32_t mask = (k >> 31) ? ((uint32_t)1 << 31) : ~(uint32_t)0; + uint32_t bits = k ^ mask; + memcpy(&d[i], &bits, 4); + } } else if (type == RAY_I32 || type == RAY_DATE || type == RAY_TIME) { int32_t* d = (int32_t*)dst; if (desc) @@ -2308,7 +2342,9 @@ static ray_t* sort_indices_ex(ray_t** cols, uint8_t* descs, uint8_t* nulls_first if (!cols[k]) { can_radix = false; break; } int8_t t = cols[k]->type; if (t == RAY_STR || t == RAY_GUID) { has_wide_key = true; continue; } - if (t != RAY_I64 && t != RAY_F64 && t != RAY_I32 && t != RAY_I16 && + if (t == RAY_F32 && n_cols != 1) { can_radix = false; break; } + if (t != RAY_I64 && t != RAY_F64 && t != RAY_F32 && + t != RAY_I32 && t != RAY_I16 && t != RAY_BOOL && t != RAY_U8 && t != RAY_SYM && t != RAY_DATE && t != RAY_TIME && t != RAY_TIMESTAMP) { can_radix = false; break; @@ -3113,7 +3149,8 @@ static ray_t* topk_indices_single(ray_t* col, uint8_t desc, uint8_t nf, int8_t type = col->type; /* Whitelist of types where radix_encode_fn produces an order-preserving * uint64 — exactly the cases topk can handle without a comparator. */ - bool ok = (type == RAY_I64 || type == RAY_TIMESTAMP || type == RAY_F64 || + bool ok = (type == RAY_I64 || type == RAY_TIMESTAMP || + type == RAY_F64 || type == RAY_F32 || type == RAY_I32 || type == RAY_DATE || type == RAY_TIME || type == RAY_SYM || type == RAY_STR || type == RAY_GUID || type == RAY_I16 || type == RAY_BOOL || type == RAY_U8); diff --git a/src/ops/window.c b/src/ops/window.c index df4612af4..78f154d35 100644 --- a/src/ops/window.c +++ b/src/ops/window.c @@ -45,6 +45,12 @@ static inline bool win_keys_differ(ray_t* const* vecs, uint32_t n_keys, if (a != b) return true; break; } + case RAY_F32: { + float a = ((const float*)ray_data(col))[ra]; + float b = ((const float*)ray_data(col))[rb]; + if (a != b) return true; + break; + } case RAY_I32: case RAY_DATE: case RAY_TIME: if (((const int32_t*)ray_data(col))[ra] != ((const int32_t*)ray_data(col))[rb]) return true; @@ -89,6 +95,7 @@ static inline bool win_keys_differ(ray_t* const* vecs, uint32_t n_keys, static inline double win_read_f64(ray_t* col, int64_t row) { switch (col->type) { case RAY_F64: return ((const double*)ray_data(col))[row]; + case RAY_F32: return (double)((const float*)ray_data(col))[row]; case RAY_I64: case RAY_TIMESTAMP: return (double)((const int64_t*)ray_data(col))[row]; case RAY_I32: case RAY_DATE: case RAY_TIME: @@ -110,6 +117,7 @@ static inline int64_t win_read_i64(ray_t* col, int64_t row) { case RAY_SYM: return sym_cell_runtime_id(col, row); case RAY_F64: return (int64_t)((const double*)ray_data(col))[row]; + case RAY_F32: return (int64_t)((const float*)ray_data(col))[row]; case RAY_I16: return (int64_t)((const int16_t*)ray_data(col))[row]; case RAY_BOOL: case RAY_U8: return (int64_t)((const uint8_t*)ray_data(col))[row]; default: return 0; @@ -1346,7 +1354,7 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { kind == RAY_WIN_MAX || kind == RAY_WIN_LAG || kind == RAY_WIN_LEAD || kind == RAY_WIN_FIRST_VALUE || kind == RAY_WIN_LAST_VALUE || kind == RAY_WIN_NTH_VALUE) { - out_f64 = fvec && fvec->type == RAY_F64; + out_f64 = fvec && (fvec->type == RAY_F64 || fvec->type == RAY_F32); } is_f64[f] = out_f64; diff --git a/src/store/col.c b/src/store/col.c index 7e8b9ca08..b6af7d00c 100644 --- a/src/store/col.c +++ b/src/store/col.c @@ -146,7 +146,7 @@ static size_t col_str_pool_payload_len(const ray_t* vec); static bool is_serializable_type(int8_t t) { switch (t) { case RAY_BOOL: case RAY_U8: case RAY_I16: - case RAY_I32: case RAY_I64: case RAY_F64: + case RAY_I32: case RAY_I64: case RAY_F32: case RAY_F64: case RAY_DATE: case RAY_TIME: case RAY_TIMESTAMP: case RAY_GUID: case RAY_SYM: case RAY_STR: return true; diff --git a/test/rfl/lang/null_keyword.rfl b/test/rfl/lang/null_keyword.rfl index 73ff8305f..10e03d5b1 100644 --- a/test/rfl/lang/null_keyword.rfl +++ b/test/rfl/lang/null_keyword.rfl @@ -13,6 +13,10 @@ ;; null is falsy in a condition → else branch, no crash (if null 1 2) -- 2 + +;; Compiled lambdas use the same null-keyword semantics as the tree walker. +(nil? ((fn [] null))) -- true +(nil? ((fn [x] (if x 1 null)) false)) -- true (if 1 10 20) -- 10 ;; producer parity: split-of-empty, ser/de round-trip, resolve-of-unbound diff --git a/test/rfl/query/query_select_fastpath_coverage.rfl b/test/rfl/query/query_select_fastpath_coverage.rfl index d635bc9ca..a281eaa11 100644 --- a/test/rfl/query/query_select_fastpath_coverage.rfl +++ b/test/rfl/query/query_select_fastpath_coverage.rfl @@ -316,6 +316,15 @@ (set Tint5 (table [v] (list [1 2 3 4 5]))) (at (at (select {f: (as 'F64 v) from: Tint5}) 'f) 0) -- 1.0 +;; Explicit F64 -> F32 projection cast uses the typed executor fallback. +;; It must round the expression result, not return an uninitialised vector. +(set Tf32proj (select {f: (as 'F32 (/ (+ v 0.3) 2.0)) from: Tint5})) +(type (at Tf32proj 'f)) -- 'F32 +(at (at Tf32proj 'f) 0) -- 0.65e +(at (at (select {f: (as 'F32 v) from: Tint5}) 'f) 4) -- 5.0e +(set Tf32i32 (table [v] (list (as 'I32 [7 8])))) +(at (at (select {f: (as 'F32 v) from: Tf32i32}) 'f) 1) -- 8.0e + ;; as cast I64→I32 in DAG (at (at (select {i: (as 'I32 v) from: Tint5}) 'i) 0) -- 1i diff --git a/test/rfl/query/window.rfl b/test/rfl/query/window.rfl index 1cf618ef4..ba65e8ffc 100644 --- a/test/rfl/query/window.rfl +++ b/test/rfl/query/window.rfl @@ -39,6 +39,32 @@ (at MW 'mx) -- [10 20 100 30 200 40] (at MW 'n) -- [1 2 1 3 2 3] +;; ── F32 inputs promote numeric window results to F64 ────────────────────── +;; F32 used to fall through the I64 readers and silently broadcast zeros. +(.sys.exec "printf 'k,o,v\\na,1,1.5\\na,2,2.5\\nb,1,10.0\\na,3,3.5\\nb,2,20.0\\n' > /tmp/rfl_window_f32.csv") +(set F (.csv.read [SYMBOL I64 F32] "/tmp/rfl_window_f32.csv")) +(set FW (window {from: F part: [k] funcs: {sm: (sum v) av: (avg v) mn: (min v) mx: (max v)}})) +(type (at FW 'sm)) -- 'F64 +(type (at FW 'av)) -- 'F64 +(type (at FW 'mn)) -- 'F64 +(type (at FW 'mx)) -- 'F64 +(at FW 'sm) -- [7.5 7.5 30.0 7.5 30.0] +(at FW 'av) -- [2.5 2.5 15.0 2.5 15.0] +(at FW 'mn) -- [1.5 1.5 10.0 1.5 10.0] +(at FW 'mx) -- [3.5 3.5 20.0 3.5 20.0] + +;; The same readers are used by running/trailing frames and value functions. +(set FT (window {from: F part: [k] order: [o] frame: 2 funcs: {sm: (sum v) av: (avg v) mn: (min v) mx: (max v)}})) +(at FT 'sm) -- [1.5 4.0 10.0 6.0 30.0] +(at FT 'av) -- [1.5 2.0 10.0 3.0 15.0] +(at FT 'mn) -- [1.5 1.5 10.0 2.5 10.0] +(at FT 'mx) -- [1.5 2.5 10.0 3.5 20.0] +(set FV (window {from: F part: [k] order: [o] funcs: {f: (first v) l: (last v)}})) +(type (at FV 'f)) -- 'F64 +(at FV 'f) -- [1.5 1.5 10.0 1.5 10.0] +(at FV 'l) -- [3.5 3.5 20.0 3.5 20.0] +(.sys.exec "rm -f /tmp/rfl_window_f32.csv") + ;; ── fby shape: rows equal to their group's min (keeps ties) ───────────────── ;; min over k: a→2 (only row v=2), b→4 (rows v=4,4 both) → 3 rows total (count (select {from: (window {from: T part: [k] funcs: {mn: (min v)}}) where: (== v mn)})) -- 3 diff --git a/test/rfl/sort/sort_coverage2.rfl b/test/rfl/sort/sort_coverage2.rfl index beca3bbf1..ffe0fb8b8 100644 --- a/test/rfl/sort/sort_coverage2.rfl +++ b/test/rfl/sort/sort_coverage2.rfl @@ -250,6 +250,18 @@ (at (at (xasc Txasc2 'k) 'k) 0) -- 1 (at (at (xdesc Txasc2 'k) 'k) 0) -- 20000000 +;; F32 single-key sort: query projections can explicitly narrow an F64 +;; expression to F32. Cover both the order-preserving F32 radix encoding and +;; stable equal-key order on a table large enough to take the radix path. +(set Tf32base (table [i v] (list (til 4097) (take [9.0 1.0 5.0 1.0 7.0] 4097)))) +(set Tf32sort (select {i: i v: (as 'F32 v) from: Tf32base})) +(type (at Tf32sort 'v)) -- 'F32 +(at (at (xasc Tf32sort 'v) 'v) 0) -- 1.0e +(at (at (xdesc Tf32sort 'v) 'v) 0) -- 9.0e +(take (at (xdesc Tf32sort 'v) 'i) 3) -- [0 5 10] +;; <=64 rows uses the stable comparison-sort path and its F32 comparator. +(take (at (xasc (take Tf32sort 5) 'v) 'i) 2) -- [1 3] + ;; ──────────────────────────────────────────────────────────────────── ;; 11. Multi-column composite sort with small n (< SMALL_POOL_THRESHOLD=8192) ;; but > RADIX_SORT_THRESHOLD=4096 — hits the else-branch diff --git a/test/rfl/system/csv_explicit_numeric_types.rfl b/test/rfl/system/csv_explicit_numeric_types.rfl new file mode 100644 index 000000000..d053c5209 --- /dev/null +++ b/test/rfl/system/csv_explicit_numeric_types.rfl @@ -0,0 +1,48 @@ +;; Explicit numeric schemas must survive the streaming .csv.splayed writer +;; and the mmap-backed .db.splayed.get reload without widening. + +(.sys.exec "rm -rf rf_test_explicit_numeric rf_test_explicit_numeric.csv") -- 0 +(.sys.exec "printf 'f32a,f32b,i32,i16\n1.25,0.25,100000,32000\n-2.5,1.5,-7,-300\n3.75,-0.25,42,9\n16777216,1,100001,9\n' > rf_test_explicit_numeric.csv") -- 0 + +(set T (.csv.splayed [F32 F32 I32 I16] "rf_test_explicit_numeric.csv" "rf_test_explicit_numeric/")) +(count T) -- 4 +(type (at T 'f32a)) -- 'F32 +(type (at T 'f32b)) -- 'F32 +(type (at T 'i32)) -- 'I32 +(type (at T 'i16)) -- 'I16 +(at (at T 'f32a) 0) -- 1.25 +(at (at T 'f32a) 1) -- -2.5 +(at (at (select {x: (- f32a f32b) from: T}) 'x) 0) -- 1.0 +(at (at (select {x: (- f32a f32b) from: T}) 'x) 2) -- 4.0 +(type (at (select {x: (+ f32a f32b) from: T}) 'x)) -- 'F32 +;; 2^24 + 1 rounds back to 2^24 at F32 precision. +(at (at (select {x: (+ f32a f32b) from: T}) 'x) 3) -- 16777216.0 +(type (at (at T 'f32a) 0)) -- 'f32 +(type (at (at T 'f32b) 0)) -- 'f32 +(type (+ (at (at T 'f32a) 0) (at (at T 'f32b) 0))) -- 'f32 +(+ (at (at T 'f32a) 0) (at (at T 'f32b) 0)) -- 1.5 +(type (at (select {x: (neg f32a) from: T}) 'x)) -- 'F32 +(at (at (select {x: (neg f32a) from: T}) 'x) 1) -- 2.5 +(type (at (select {x: (+ f32a i32) from: T}) 'x)) -- 'F64 +(at (at (select {x: (+ f32a i32) from: T}) 'x) 0) -- 100001.25 +(type (wavg (at T 'i32) (at T 'f32b))) -- 'f64 +(set G (select {mx: (max f32a) wa: (wavg i32 f32b) by: {k: i16} from: T})) +(count G) -- 3 +(type (at G 'mx)) -- 'F64 +(type (at G 'wa)) -- 'F64 +(set G1 (select {mx: (max f32a) wa: (wavg i32 f32b) from: T where: (== i16 32000)})) +(at (at G1 'mx) 0) -- 1.25 +(at (at G1 'wa) 0) -- 0.25 +(at (at T 'i32) 0) -- 100000 +(at (at T 'i16) 1) -- -300 + +(set R (.db.splayed.get "rf_test_explicit_numeric/")) +(type (at R 'f32a)) -- 'F32 +(type (at R 'f32b)) -- 'F32 +(type (at R 'i32)) -- 'I32 +(type (at R 'i16)) -- 'I16 +(at (at R 'f32a) 2) -- 3.75 +(at (at R 'i32) 1) -- -7 +(at (at R 'i16) 2) -- 9 + +(.sys.exec "rm -rf rf_test_explicit_numeric rf_test_explicit_numeric.csv") -- 0 diff --git a/test/test_agg_engine.c b/test/test_agg_engine.c index 9c4c9eff0..c66147cbd 100644 --- a/test/test_agg_engine.c +++ b/test/test_agg_engine.c @@ -208,7 +208,7 @@ static test_result_t test_dense_plan_single_i64(void) { PASS(); } -/* (b) two I64 keys, ranges 100 and 50 → dense, slots 5000, strides {1,100}. */ +/* (b) two I64 keys, ranges 10 and 10 → dense, slots equal input rows. */ static test_result_t test_dense_plan_two_keys(void) { ray_heap_init(); (void)ray_sym_init(); @@ -217,7 +217,7 @@ static test_result_t test_dense_plan_two_keys(void) { ray_t* k2 = ray_vec_new(RAY_I64, 100); k2->len = 100; int64_t* d1 = (int64_t*)ray_data(k1); int64_t* d2 = (int64_t*)ray_data(k2); - for (int64_t i = 0; i < 100; i++) { d1[i] = i; d2[i] = i % 50; } /* ranges 100, 50 */ + for (int64_t i = 0; i < 100; i++) { d1[i] = i % 10; d2[i] = i / 10; } const agg_vtable_t* vt = dense_sum_i64_vt(); TEST_ASSERT_NOT_NULL(vt); @@ -229,11 +229,11 @@ static test_result_t test_dense_plan_two_keys(void) { TEST_ASSERT_TRUE(ok); TEST_ASSERT_TRUE(pl.ok); - TEST_ASSERT_EQ_I(pl.total_slots, 5000); - TEST_ASSERT_EQ_I(pl.ranges[0], 100); - TEST_ASSERT_EQ_I(pl.ranges[1], 50); + TEST_ASSERT_EQ_I(pl.total_slots, 100); + TEST_ASSERT_EQ_I(pl.ranges[0], 10); + TEST_ASSERT_EQ_I(pl.ranges[1], 10); TEST_ASSERT_EQ_I(pl.strides[0], 1); - TEST_ASSERT_EQ_I(pl.strides[1], 100); + TEST_ASSERT_EQ_I(pl.strides[1], 10); ray_release(k1); ray_release(k2); @@ -242,14 +242,14 @@ static test_result_t test_dense_plan_two_keys(void) { PASS(); } -/* (c) single I64 key with huge range (0 and 10,000,000) → not dense (over cap). */ +/* (c) A two-row input cannot justify a 10,000,001-slot dense state. */ static test_result_t test_dense_plan_huge_range(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* k = ray_vec_new(RAY_I64, 2); k->len = 2; int64_t* kd = (int64_t*)ray_data(k); - kd[0] = 0; kd[1] = 10000000; /* range > DENSE_MAX_SLOTS */ + kd[0] = 0; kd[1] = 10000000; /* dense state would exceed input size */ const agg_vtable_t* vt = dense_sum_i64_vt(); TEST_ASSERT_NOT_NULL(vt); @@ -862,14 +862,13 @@ static ray_t* diff_make_2i64_hc(int64_t n, int64_t nk) { /* ── RADIX-FORCING builders (Phase 1c radix) ───────────────────────────── * The radix path is selected for high-card STREAMING int/SYM-key queries whose - * DENSE plan is rejected (total_slots > DENSE_MAX_SLOTS=262144, or per-worker - * slab > 8MB). These builders give a WIDE key value range so the dense plan - * fails and the selector routes to exec_group_v2_parallel_radix; the resulting - * table is still compared byte-for-byte (as a key-sorted multiset) vs the old - * engine. At N=70000 (≥ RAY_PARALLEL_THRESHOLD) the parallel dispatch runs. */ - -/* Single I64 key "k" = (i % 50000) * 7 (≈50000 groups, range ≈350000 > - * DENSE_MAX_SLOTS → dense plan rejected → radix). I64 value "v" = i. */ + * dense representation would require more slots than the contributing input + * has rows. These builders give a wide key range so the structural dense-plan + * rule selects radix; results are compared byte-for-byte (as a key-sorted + * multiset) with the old engine. */ + +/* Single I64 key "k" = (i % 50000) * 7: its range needs more dense slots than + * the input has rows, so it follows the radix representation. */ static ray_t* diff_make_i64_radix(int64_t n, int64_t nk) { (void)nk; ray_t* kvec = ray_vec_new(RAY_I64, n); kvec->len = n; @@ -888,8 +887,8 @@ static ray_t* diff_make_i64_radix(int64_t n, int64_t nk) { /* Two I64 keys with WIDE composite range → dense plan rejected → radix. * k1 = (i%4000)*5 (range ≈20000), k2 = ((i/4000)%4000)*5 → product of ranges - * ≫ DENSE_MAX_SLOTS. ≈ up to 16M composite slots but only ~17500 live tuples - * at N=70000. I64 value "v" = i. */ + * greatly exceeds the input row count. There are up to 16M composite slots + * but only about 17500 live tuples at N=70000. */ static ray_t* diff_make_2i64_radix(int64_t n, int64_t nk) { (void)nk; ray_t* k1 = ray_vec_new(RAY_I64, n); k1->len = n; @@ -939,7 +938,7 @@ static ray_t* diff_make_pearson_radix(int64_t n, int64_t nk) { double* xd = (double*)ray_data(xvec); double* yd = (double*)ray_data(yvec); for (int64_t i = 0; i < n; i++) { - kd[i] = (i % 20000) * 17; /* range ≈340000 > DENSE_MAX_SLOTS */ + kd[i] = (i % 20000) * 17; /* dense range exceeds input rows */ xd[i] = (double)((i % 89) * 1.0); yd[i] = (double)(((i * 7) % 83) * 1.0); } @@ -953,7 +952,7 @@ static ray_t* diff_make_pearson_radix(int64_t n, int64_t nk) { /* Q10-shape 6-key tuple (high card → radix): six int/SYM keys k1..k6 + I64 "v". * Keys vary at independent rates so the 6-tuple space is broad (≈ thousands of * live tuples at N=70000) and the composite range is astronomically larger than - * DENSE_MAX_SLOTS → dense rejected → radix exercises the 6-key tuple hash/eq. */ + * the input row count, so radix exercises the 6-key tuple hash/eq. */ static ray_t* diff_make_6keys_radix(int64_t n, int64_t nk) { (void)nk; ray_t* k1 = ray_vec_new(RAY_I64, n); k1->len = n; @@ -1427,6 +1426,12 @@ static ray_op_t* gb_top100(ray_graph_t* g) { int64_t kk[] = { 100 }; ray_op_t* keys[] = { k }; return ray_group_build(g, keys, 1, ops, ins, NULL, kk, 1); } +static ray_op_t* gb_top2048(ray_graph_t* g) { + ray_op_t* k = ray_scan(g, "k"); ray_op_t* v = ray_scan(g, "v"); + uint16_t ops[] = { OP_TOP_N }; ray_op_t* ins[] = { v }; + int64_t kk[] = { 2048 }; ray_op_t* keys[] = { k }; + return ray_group_build(g, keys, 1, ops, ins, NULL, kk, 1); +} /* heterogeneous: sum + top2 (LIST col) + count, single I64 key. */ static ray_op_t* gb_sum_top2_count(ray_graph_t* g) { ray_op_t* k = ray_scan(g, "k"); ray_op_t* v = ray_scan(g, "v"); @@ -1603,6 +1608,17 @@ static test_result_t test_diff_group_top100(void) { return r; } +/* K above the former fixed stack ceiling. One 3000-row group ensures both + * engines materialize and compare a full 2048-element typed result cell. */ +static test_result_t test_diff_group_top2048(void) { + ray_heap_init(); (void)ray_sym_init(); + ray_t* tbl = diff_make_topk_i64_distinctv(3000, 1); + test_result_t r = diff_group(tbl, gb_top2048, 1); + ray_release(tbl); + ray_sym_destroy(); ray_heap_destroy(); + return r; +} + /* ══════════════════════════════════════════════════════════════════════ * WHERE-FILTERED differentials (architectural enabler: g->selection routes * through exec_group_v2's compact-table prologue). Each runs OP_GROUP with a @@ -1679,10 +1695,10 @@ DIFF_SHAPE(test_diff_group_2k_avg, diff_make_2i64_f64, gb_2k_avg, 2) DIFF_SHAPE(test_diff_group_2k_keynull, diff_make_2i64_keynull, gb_2k_sum, 2) /* ══════════════════════════════════════════════════════════════════════ - * HIGH-CARDINALITY differentials (Phase 1c). N=70000 (≥ RAY_PARALLEL_THRESHOLD - * 65536) with the v2 flag ON forces the parallel two-phase path; MANY groups - * mean each per-worker local table holds many groups and Phase B merges real - * work. Compared as MULTISET (key-sorted) vs the OLD engine. + * HIGH-CARDINALITY differentials (Phase 1c). These inputs are large enough to + * exercise the engine's parallel two-phase path; many groups make every local + * table and the merge do real work. Compared as a key-sorted multiset with the + * old engine. * ══════════════════════════════════════════════════════════════════════ */ #define HC_N 70000 @@ -1777,8 +1793,8 @@ static test_result_t test_diff_group_radix_2k_sum(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* big = diff_make_2i64_radix(HC_N, 0); int64_t ng = v2_group_count(big, gb_2k_sum); - /* All (i%4000, i/4000) pairs distinct at N=70000 → 70000 groups; the wide - * composite range (≈1.7M slots > DENSE_MAX_SLOTS) is what forces radix. */ + /* All (i%4000, i/4000) pairs are distinct at N=70000; their composite + * dense range exceeds the input size, which structurally selects radix. */ TEST_ASSERT_FMT(ng == 70000, "expected 70000 groups, got %lld", (long long)ng); test_result_t r = diff_group(big, gb_2k_sum, 2); ray_release(big); @@ -1811,14 +1827,12 @@ static test_result_t test_diff_group_radix_6k(void) { } /* ══════════════════════════════════════════════════════════════════════ - * SMALL-HASH differential: HIGH-REDUCTION sparse-low-card shape. + * SPARSE LOW-CARDINALITY differential. * * Few groups (64) whose key VALUES are wide/sparse: k=(i%64)*100000001 spans - * 0..~6.3e9, so the dense plan is rejected (range ≫ DENSE_MAX_SLOTS), yet the - * cardinality is tiny. N=70000 ≥ RAY_PARALLEL_THRESHOLD so it runs parallel, - * and the streaming int-key + LOW-estimate routing sends it to the O(groups) - * small-hash path. sum/count/min/max must match the OLD engine exactly - * (key-sorted multiset). This correctness-gates the new routing. + * 0..~6.3e9, so a dense representation would be superlinear in the input even + * though cardinality is tiny. The deterministic sparse-key route must match + * the old engine exactly for sum/count/min/max (key-sorted multiset). * ══════════════════════════════════════════════════════════════════════ */ static ray_t* diff_make_i64_sparse_lowcard(int64_t n, int64_t nk) { (void)nk; @@ -1836,7 +1850,7 @@ static ray_t* diff_make_i64_sparse_lowcard(int64_t n, int64_t nk) { return tbl; } -/* 64 groups, sparse keys, SUM+COUNT+MIN+MAX → small-hash path. */ +/* 64 groups, sparse keys, SUM+COUNT+MIN+MAX. */ static test_result_t test_diff_group_smallhash_sparse_lowcard(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* big = diff_make_i64_sparse_lowcard(HC_N, 0); @@ -2045,6 +2059,7 @@ const test_entry_t agg_engine_entries[] = { { "diff_group_2k_top2", test_diff_group_2k_top2, NULL, NULL }, { "diff_group_mixed_top2", test_diff_group_mixed_top2, NULL, NULL }, { "diff_group_top100", test_diff_group_top100, NULL, NULL }, + { "diff_group_top2048", test_diff_group_top2048, NULL, NULL }, { "diff_group_2k_sum", test_diff_group_2k_sum, NULL, NULL }, { "diff_group_2k_four", test_diff_group_2k_four, NULL, NULL }, { "diff_group_3k_count", test_diff_group_3k_count, NULL, NULL }, diff --git a/test/test_csv.c b/test/test_csv.c index cc9031882..9d0ac1f4e 100644 --- a/test/test_csv.c +++ b/test/test_csv.c @@ -111,6 +111,41 @@ static test_result_t test_csv_roundtrip_f64(void) { PASS(); } +static test_result_t test_csv_roundtrip_f32(void) { + ray_heap_init(); + (void)ray_sym_init(); + + float vals[] = {41.87f, -0.125f, 0.0f}; + ray_t* vec = ray_vec_from_raw(RAY_F32, vals, 3); + int64_t name = ray_sym_intern("price", 5); + ray_t* tbl = ray_table_new(1); + tbl = ray_table_add_col(tbl, name, vec); + ray_release(vec); + + ray_err_t err = ray_write_csv(tbl, TMP_CSV); + TEST_ASSERT_EQ_I(err, RAY_OK); + + int8_t schema[] = {RAY_F32}; + ray_t* loaded = ray_read_csv_opts(TMP_CSV, ',', true, schema, 1); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + TEST_ASSERT_EQ_I(ray_table_nrows(loaded), 3); + + ray_t* col = ray_table_get_col_idx(loaded, 0); + TEST_ASSERT_NOT_NULL(col); + TEST_ASSERT_EQ_I(col->type, RAY_F32); + float* loaded_data = (float*)ray_data(col); + TEST_ASSERT_EQ_F(loaded_data[0], vals[0], 1e-6); + TEST_ASSERT_EQ_F(loaded_data[1], vals[1], 1e-6); + TEST_ASSERT_EQ_F(loaded_data[2], vals[2], 1e-6); + + ray_release(loaded); + ray_release(tbl); + unlink(TMP_CSV); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + static test_result_t test_csv_multi_column(void) { ray_heap_init(); (void)ray_sym_init(); @@ -1513,6 +1548,7 @@ const test_entry_t csv_entries[] = { { "csv/roundtrip_i64", test_csv_roundtrip_i64, NULL, NULL }, { "csv/roundtrip_guid", test_csv_guid_roundtrip, NULL, NULL }, { "csv/roundtrip_f64", test_csv_roundtrip_f64, NULL, NULL }, + { "csv/roundtrip_f32", test_csv_roundtrip_f32, NULL, NULL }, { "csv/multi_column", test_csv_multi_column, NULL, NULL }, { "csv/empty_table", test_csv_empty_table, NULL, NULL }, { "csv/null_i64", test_csv_null_i64, NULL, NULL }, diff --git a/test/test_group_extra.c b/test/test_group_extra.c index 4b957c064..3b3154a27 100644 --- a/test/test_group_extra.c +++ b/test/test_group_extra.c @@ -2319,7 +2319,7 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { } ght_layout_t master; - TEST_ASSERT_TRUE(ght_compute_layout(&master, NK, NA, agg_vecs, + TEST_ASSERT_TRUE(ght_compute_layout(&master, NK, NA, agg_vecs, NULL, GHT_NEED_SUM, agg_ops, key_types)); /* > GHT_INLINE on both axes: this must be a real, owned spill block. */ TEST_ASSERT_NOT_NULL(master.spill_hdr); @@ -2375,7 +2375,7 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { * must have its copy RE-POINTED at the destination's own inline arrays, * never left aliasing the source's — the mirror of the spill leg above. */ ght_layout_t inl; - TEST_ASSERT_TRUE(ght_compute_layout(&inl, 2, 2, agg_vecs, + TEST_ASSERT_TRUE(ght_compute_layout(&inl, 2, 2, agg_vecs, NULL, GHT_NEED_SUM, agg_ops, key_types)); TEST_ASSERT_NULL(inl.spill_hdr); TEST_ASSERT_TRUE(inl.agg_val_slot == inl.agg_val_slot_in); diff --git a/test/test_rowsel.c b/test/test_rowsel.c index 1f83c2569..8a28a0a3e 100644 --- a/test/test_rowsel.c +++ b/test/test_rowsel.c @@ -422,6 +422,59 @@ static test_result_t test_rowsel_refine_null_existing(void) { PASS(); } +/* MIX∩MIX must initialize its output flag even when the recycled rowsel + * payload previously held ALL. The allocator zeroes object headers, not + * payload bytes, so reading the output flag before assigning it can turn a + * partial intersection into an entire morsel. */ +static test_result_t test_rowsel_intersect_mixed_reused_block(void) { + ray_heap_init(); + enum { N = RAY_MORSEL_ELEMS }; + uint8_t a_bits[N]; + uint8_t b_bits[N]; + memset(a_bits, 0, sizeof(a_bits)); + memset(b_bits, 0, sizeof(b_bits)); + + const int64_t a_start = N / 4; + const int64_t b_end = (N * 5) / 8; + for (int64_t i = a_start; i < N; i++) a_bits[i] = 1; + for (int64_t i = 0; i < b_end; i++) b_bits[i] = 1; + + ray_t* a_pred = make_pred(a_bits, N); + ray_t* b_pred = make_pred(b_bits, N); + TEST_ASSERT_NOT_NULL(a_pred); + TEST_ASSERT_NOT_NULL(b_pred); + ray_t* a = ray_rowsel_from_pred(a_pred); + ray_t* b = ray_rowsel_from_pred(b_pred); + TEST_ASSERT_NOT_NULL(a); + TEST_ASSERT_NOT_NULL(b); + TEST_ASSERT_EQ_I(ray_rowsel_flags(a)[0], RAY_SEL_MIX); + TEST_ASSERT_EQ_I(ray_rowsel_flags(b)[0], RAY_SEL_MIX); + + const int64_t expected = b_end - a_start; + ray_t* poison = ray_rowsel_new(N, expected, expected); + TEST_ASSERT_NOT_NULL(poison); + ray_rowsel_flags(poison)[0] = RAY_SEL_ALL; + ray_rowsel_release(poison); + + ray_t* out = ray_rowsel_intersect(a, b); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQ_I(ray_rowsel_meta(out)->total_pass, expected); + TEST_ASSERT_EQ_I(ray_rowsel_flags(out)[0], RAY_SEL_MIX); + int64_t rows[N]; + int64_t count = reconstruct(out, rows); + TEST_ASSERT_EQ_I(count, expected); + TEST_ASSERT_EQ_I(rows[0], a_start); + TEST_ASSERT_EQ_I(rows[count - 1], b_end - 1); + + ray_rowsel_release(out); + ray_rowsel_release(a); + ray_rowsel_release(b); + ray_release(a_pred); + ray_release(b_pred); + ray_heap_destroy(); + PASS(); +} + /* Streaming emitter must produce a block byte-identical to the * whole-vec ray_rowsel_from_pred over the same bools. Pattern spans * >1 morsel with a NONE seg, an ALL seg, and MIX segs. */ @@ -486,8 +539,8 @@ const test_entry_t rowsel_entries[] = { { "rowsel/to_indices", test_rowsel_to_indices, NULL, NULL }, { "rowsel/refine", test_rowsel_refine, NULL, NULL }, { "rowsel/refine_null_existing", test_rowsel_refine_null_existing, NULL, NULL }, + { "rowsel/intersect_mixed_reused_block", test_rowsel_intersect_mixed_reused_block, NULL, NULL }, { "rowsel/emit_equiv", test_rowsel_emit_equiv, NULL, NULL }, { NULL, NULL, NULL, NULL }, }; - diff --git a/test/test_store.c b/test/test_store.c index 9888e96c5..4edb0210b 100644 --- a/test/test_store.c +++ b/test/test_store.c @@ -141,6 +141,43 @@ static test_result_t test_col_mmap_f64(void) { PASS(); } +/* ---- test_col_mmap_f32 ------------------------------------------------- */ + +static test_result_t test_col_mmap_f32(void) { + float raw[] = {1.25f, -2.5f, 3.75f, 0.0f}; + ray_t* vec = ray_vec_from_raw(RAY_F32, raw, 4); + TEST_ASSERT_NOT_NULL(vec); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + + ray_err_t err = ray_col_save(vec, TMP_COL_PATH); + TEST_ASSERT_EQ_I(err, RAY_OK); + + ray_t* loaded = ray_col_load(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(loaded); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + TEST_ASSERT_EQ_I(loaded->type, RAY_F32); + TEST_ASSERT_EQ_I(loaded->len, 4); + float* loaded_data = (float*)ray_data(loaded); + for (int i = 0; i < 4; i++) + TEST_ASSERT(loaded_data[i] == raw[i], "f32 load value mismatch"); + ray_release(loaded); + + ray_t* mapped = ray_col_mmap(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(mapped); + TEST_ASSERT_FALSE(RAY_IS_ERR(mapped)); + TEST_ASSERT_EQ_U(mapped->mmod, 1); + TEST_ASSERT_EQ_I(mapped->type, RAY_F32); + TEST_ASSERT_EQ_I(mapped->len, 4); + float* mapped_data = (float*)ray_data(mapped); + for (int i = 0; i < 4; i++) + TEST_ASSERT(mapped_data[i] == raw[i], "f32 mmap value mismatch"); + + ray_release(mapped); + ray_release(vec); + unlink(TMP_COL_PATH); + PASS(); +} + /* ---- test_col_mmap_cow ------------------------------------------------- */ static test_result_t test_col_mmap_cow(void) { @@ -4909,6 +4946,7 @@ static test_result_t test_col_save_nyi_type(void) { const test_entry_t store_entries[] = { { "store/col_mmap_i64", test_col_mmap_i64, store_setup, store_teardown }, { "store/col_mmap_f64", test_col_mmap_f64, store_setup, store_teardown }, + { "store/col_mmap_f32", test_col_mmap_f32, store_setup, store_teardown }, { "store/col_mmap_cow", test_col_mmap_cow, store_setup, store_teardown }, { "store/col_mmap_refcount", test_col_mmap_refcount, store_setup, store_teardown }, { "store/col_mmap_corrupt", test_col_mmap_corrupt, store_setup, store_teardown },