From 2281d8101da510b5a8387f9a2e6a4af433e51e09 Mon Sep 17 00:00:00 2001 From: Anton Date: Thu, 9 Jul 2026 22:58:46 +0200 Subject: [PATCH 01/10] perf(aggr): fuse indexed statistical groups --- src/ops/group.c | 270 ++++++++++++++++++++++++--- src/ops/query.c | 37 ++-- test/rfl/integration/slice_group.rfl | 18 ++ 3 files changed, 278 insertions(+), 47 deletions(-) diff --git a/src/ops/group.c b/src/ops/group.c index 7098b02bb..42a83a593 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -5020,7 +5020,10 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* const agg_affine_t* affine, const agg_prod_t* prod, const double* sumsq_f64, - const int64_t* nn_counts) { + const int64_t* nn_counts, + const double* sum_y_f64, + const double* sumsq_y_f64, + const double* sumxy_f64) { for (uint32_t a = 0; a < n_aggs; a++) { uint16_t agg_op = ext->agg_ops[a]; ray_t* agg_col = agg_vecs[a]; @@ -5036,6 +5039,9 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* case OP_AVG: case OP_STDDEV: case OP_STDDEV_POP: case OP_VAR: case OP_VAR_POP: + case OP_PEARSON_CORR: + case OP_COV: case OP_SCOV: + case OP_WSUM: case OP_WAVG: out_type = RAY_F64; break; case OP_ALL: case OP_ANY: @@ -5117,6 +5123,40 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* else v = sqrt(var_pop * cnt / (cnt - 1)); break; } + case OP_PEARSON_CORR: + case OP_COV: + case OP_SCOV: + case OP_WSUM: + case OP_WAVG: { + int64_t cnt = nn; + if ((agg_op == OP_PEARSON_CORR || agg_op == OP_SCOV) && cnt < 2) { + v = NULL_F64; ray_vec_set_null(new_col, gi, true); break; + } + if ((agg_op == OP_COV || agg_op == OP_WAVG) && cnt < 1) { + v = NULL_F64; ray_vec_set_null(new_col, gi, true); break; + } + double sx = sum_f64 ? sum_f64[idx] : 0.0; + double sy = sum_y_f64 ? sum_y_f64[idx] : 0.0; + double sxx = sumsq_f64 ? sumsq_f64[idx] : 0.0; + double syy = sumsq_y_f64 ? sumsq_y_f64[idx] : 0.0; + double sxy = sumxy_f64 ? sumxy_f64[idx] : 0.0; + if (agg_op == OP_WSUM) { v = sxy; break; } + if (agg_op == OP_WAVG) { + if (sx == 0.0) { v = NULL_F64; ray_vec_set_null(new_col, gi, true); break; } + v = sxy / sx; break; + } + double dn = (double)cnt; + if (agg_op == OP_COV) { v = (sxy - sx * sy / dn) / dn; break; } + if (agg_op == OP_SCOV) { v = (sxy - sx * sy / dn) / (dn - 1.0); break; } + double num = dn * sxy - sx * sy; + double dx = dn * sxx - sx * sx; + double dy = dn * syy - sy * sy; + if (dx <= 0.0 || dy <= 0.0) { + v = NULL_F64; ray_vec_set_null(new_col, gi, true); break; + } + v = num / sqrt(dx * dy); + break; + } default: v = 0.0; break; } /* Single-null float model: canonicalize a non-finite computed @@ -7336,10 +7376,10 @@ ray_t* exec_group(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) * filter scan, the survivor gather and group discovery all vanish; per * group the accumulation applies the same all_sum recurrence da_accum_row * applies row-by-row (same read helpers, same in-order accumulate), so - * results match the generic path. Ineligible shapes — non-SUM/AVG/COUNT - * aggs, non-scan non-prod-fusable inputs, HAS_NULLS or F32/exotic agg - * columns, binary/holistic aggs, HEAD limits, emit filters — return NULL - * and the caller folds the slices into the equivalent selection. */ + * results match the generic path. Ineligible shapes — unsupported aggs, + * non-scan non-prod-fusable inputs, HAS_NULLS or F32/exotic agg columns, + * limits, emit filters — return NULL and the caller folds the slices into + * the equivalent selection. */ /* Row-chunk task: slice gi, rows [lo, hi) within that slice. Slices are * chunked so one dominant key (a most-frequent sym can carry ~95% of the @@ -7354,10 +7394,15 @@ typedef struct { const ray_idx_slice_t* slices; const sg_task_t* tasks; ray_t* const* agg_vecs; + ray_t* const* agg_vecs2; const agg_prod_t* prod; const uint16_t* agg_ops; uint8_t n_aggs; da_val_t* partials; /* [n_tasks * n_aggs], zeroed */ + double* partial_sumsq; + double* partial_sum_y; + double* partial_sumsq_y; + double* partial_sumxy; /* Shared-stream fusion: pair_sum[a] = sibling SUM agg slot whose bare * scan is prod[a]'s int side (-1 = none); fused_by[b] = the prod slot * that computes SUM b (-1 = b runs its own loop). */ @@ -7443,6 +7488,37 @@ static inline double sg_prod_range(const agg_prod_t* p, int64_t r0, int64_t n, return (a0 + a1) + (a2 + a3); } +static inline double sg_num_at(ray_t* v, int64_t row) { + if (v->type == RAY_F64) + return ((const double*)ray_data(v))[row]; + return (double)read_col_i64(ray_data(v), row, v->type, v->attrs); +} + +static inline void sg_pair_accum(ray_t* x, ray_t* y, const int64_t* rows, + int64_t n, bool contig, int64_t r0, + double* sx, double* sy, double* sxx, + double* syy, double* sxy) { + double ax = 0.0, ay = 0.0, axx = 0.0, ayy = 0.0, axy = 0.0; + if (contig) { + for (int64_t j = 0; j < n; j++) { + int64_t r = r0 + j; + double xv = sg_num_at(x, r); + double yv = sg_num_at(y, r); + ax += xv; ay += yv; + axx += xv * xv; ayy += yv * yv; axy += xv * yv; + } + } else { + for (int64_t j = 0; j < n; j++) { + int64_t r = rows[j]; + double xv = sg_num_at(x, r); + double yv = sg_num_at(y, r); + ax += xv; ay += yv; + axx += xv * xv; ayy += yv * yv; axy += xv * yv; + } + } + *sx = ax; *sy = ay; *sxx = axx; *syy = ayy; *sxy = axy; +} + static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { (void)wid; sg_ctx_t* c = (sg_ctx_t*)raw; @@ -7457,7 +7533,17 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { int64_t r0 = (n > 0) ? rows[0] : 0; for (uint32_t a = 0; a < c->n_aggs; a++) { size_t idx = (size_t)ti * c->n_aggs + a; - if (c->prod[a].enabled) { + uint16_t op = c->agg_ops[a]; + if (agg_is_binary_agg(op)) { + double sx = 0.0, sy = 0.0, sxx = 0.0, syy = 0.0, sxy = 0.0; + sg_pair_accum(c->agg_vecs[a], c->agg_vecs2[a], rows, n, + contig, r0, &sx, &sy, &sxx, &syy, &sxy); + c->partials[idx].f = sx; + if (c->partial_sumsq) c->partial_sumsq[idx] = sxx; + if (c->partial_sum_y) c->partial_sum_y[idx] = sy; + if (c->partial_sumsq_y) c->partial_sumsq_y[idx] = syy; + if (c->partial_sumxy) c->partial_sumxy[idx] = sxy; + } else if (c->prod[a].enabled) { /* Fused product: gated null-free, every row counts. */ double acc = 0.0; if (contig) { @@ -7479,26 +7565,37 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { * iterate ascending and pairing enforces prod < sum? no — * pairing is order-free: the prod branch writes this slot * directly whichever order they appear in). */ - } else if (c->agg_ops[a] == OP_COUNT) { + } else if (op == OP_COUNT) { /* counts[gi] above suffices. */ } else if (c->agg_vecs[a]->type == RAY_F64) { /* NaN payload = null, skip from sum (mirror all_sum). */ const double* restrict d = (const double*)ray_data(c->agg_vecs[a]); double acc = 0.0; + double ssq = 0.0; + bool need_sq = c->partial_sumsq && + (op == OP_STDDEV || op == OP_STDDEV_POP || + op == OP_VAR || op == OP_VAR_POP); if (contig) { const double* restrict dr = d + r0; for (int64_t j = 0; j < n; j++) { double v = dr[j]; - if (RAY_LIKELY(v == v)) acc += v; + if (RAY_LIKELY(v == v)) { + acc += v; + if (need_sq) ssq += v * v; + } } } else { for (int64_t j = 0; j < n; j++) { double v = d[rows[j]]; - if (RAY_LIKELY(v == v)) acc += v; + if (RAY_LIKELY(v == v)) { + acc += v; + if (need_sq) ssq += v * v; + } } } c->partials[idx].f = acc; + if (need_sq) c->partial_sumsq[idx] = ssq; } else { /* Integer family, null-free by admission (unsigned wrap * add mirrors da_accum_row). */ @@ -7507,17 +7604,33 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { int8_t t = av->type; uint8_t at = av->attrs; uint64_t acc = 0; + double ssq = 0.0; + bool need_sq = c->partial_sumsq && + (op == OP_STDDEV || op == OP_STDDEV_POP || + op == OP_VAR || op == OP_VAR_POP); if (contig && (t == RAY_I64 || t == RAY_TIME)) { const int64_t* restrict x = (const int64_t*)p + r0; - for (int64_t j = 0; j < n; j++) acc += (uint64_t)x[j]; + for (int64_t j = 0; j < n; j++) { + int64_t v = x[j]; + acc += (uint64_t)v; + if (need_sq) { double d = (double)v; ssq += d * d; } + } } else if (contig && t == RAY_I32) { const int32_t* restrict x = (const int32_t*)p + r0; - for (int64_t j = 0; j < n; j++) acc += (uint64_t)(int64_t)x[j]; + for (int64_t j = 0; j < n; j++) { + int64_t v = (int64_t)x[j]; + acc += (uint64_t)v; + if (need_sq) { double d = (double)v; ssq += d * d; } + } } else { - for (int64_t j = 0; j < n; j++) - acc += (uint64_t)read_col_i64(p, rows[j], t, at); + for (int64_t j = 0; j < n; j++) { + int64_t v = read_col_i64(p, rows[j], t, at); + acc += (uint64_t)v; + if (need_sq) { double d = (double)v; ssq += d * d; } + } } c->partials[idx].i = (int64_t)acc; + if (need_sq) c->partial_sumsq[idx] = ssq; } } } @@ -7576,7 +7689,8 @@ static ray_t* sg_hint_to_selection(ray_graph_t* g, ray_t* tbl) { * prod (fused product plans). */ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit, - ray_t** agg_vecs, agg_prod_t* prod) { + ray_t** agg_vecs, ray_t** agg_vecs2, + agg_prod_t* prod) { if (group_limit != 0) return false; if (ray_group_emit_filter_get().enabled) return false; ray_op_ext_t* ext = find_ext(g, op->id); @@ -7591,24 +7705,50 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!key_col || key_col != g->sg_col) return false; for (uint32_t a = 0; a < ext->n_aggs; a++) { - if (ext->agg_ins2 && ext->agg_ins2[a] != RAY_OP_NONE) return false; uint16_t aop = ext->agg_ops[a]; if (aop == OP_COUNT) continue; /* group size; input unused */ - if (aop != OP_SUM && aop != OP_AVG) return false; + bool pair = agg_is_binary_agg(aop); + if (!pair && + aop != OP_SUM && aop != OP_AVG && + aop != OP_STDDEV && aop != OP_STDDEV_POP && + aop != OP_VAR && aop != OP_VAR_POP) + return false; ray_op_t* in = op_node(g, ext->agg_ins[a]); if (!in) return false; - if (try_prod_sumavg_input_f64(g, tbl, in, &prod[a])) continue; + if (!pair && (aop == OP_SUM || aop == OP_AVG) && + try_prod_sumavg_input_f64(g, tbl, in, &prod[a])) + continue; ray_op_ext_t* ae = find_ext(g, in->id); if (!ae || ae->base.opcode != OP_SCAN) return false; ray_t* col = ray_table_get_col(tbl, ae->sym); if (!col || ray_is_atom(col)) return false; if (col->attrs & RAY_ATTR_HAS_NULLS) return false; + if (!agg_type_admitted(aop, col->type)) return false; switch (col->type) { case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: case RAY_TIME: case RAY_F64: break; default: return false; /* F32 / exotic → generic path */ } agg_vecs[a] = col; + if (pair) { + if (!ext->agg_ins2 || ext->agg_ins2[a] == RAY_OP_NONE) return false; + ray_op_t* in2 = op_node(g, ext->agg_ins2[a]); + if (!in2) return false; + ray_op_ext_t* ae2 = find_ext(g, in2->id); + if (!ae2 || ae2->base.opcode != OP_SCAN) return false; + ray_t* col2 = ray_table_get_col(tbl, ae2->sym); + if (!col2 || ray_is_atom(col2)) return false; + if (col2->attrs & RAY_ATTR_HAS_NULLS) return false; + if (!agg_type_admitted(aop, col2->type)) return false; + switch (col2->type) { + case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: + case RAY_F64: break; + default: return false; + } + agg_vecs2[a] = col2; + } else if (ext->agg_ins2 && ext->agg_ins2[a] != RAY_OP_NONE) { + return false; + } } return true; } @@ -7623,9 +7763,10 @@ ray_t* ray_group_slice_hint_settle(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) { if (!g->sg_col || !tbl || tbl->type != RAY_TABLE) return NULL; ray_t* agg_vecs[16] = {0}; + ray_t* agg_vecs2[16] = {0}; agg_prod_t prod[16]; memset(prod, 0, sizeof(prod)); - if (sg_shape_eligible(g, op, tbl, group_limit, agg_vecs, prod)) + if (sg_shape_eligible(g, op, tbl, group_limit, agg_vecs, agg_vecs2, prod)) return NULL; /* kernel will consume it */ return sg_hint_to_selection(g, tbl); /* fold; NULL on success */ } @@ -7633,9 +7774,10 @@ ray_t* ray_group_slice_hint_settle(ray_graph_t* g, ray_op_t* op, ray_t* tbl, static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) { ray_t* agg_vecs[16] = {0}; + ray_t* agg_vecs2[16] = {0}; agg_prod_t prod[16]; memset(prod, 0, sizeof(prod)); - if (!sg_shape_eligible(g, op, tbl, group_limit, agg_vecs, prod)) + if (!sg_shape_eligible(g, op, tbl, group_limit, agg_vecs, agg_vecs2, prod)) return NULL; ray_op_ext_t* ext = find_ext(g, op->id); ray_op_ext_t* ke = find_ext(g, ext->keys[0]); @@ -7646,14 +7788,39 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, const ray_idx_slice_t* slices = (K > 0) ? (const ray_idx_slice_t*)ray_data(g->sg_slices_hdr) : NULL; + bool need_sumsq = false, need_pair = false; + for (uint32_t a = 0; a < n_aggs; a++) { + uint16_t aop = ext->agg_ops[a]; + if (aop == OP_STDDEV || aop == OP_STDDEV_POP || + aop == OP_VAR || aop == OP_VAR_POP || + agg_is_binary_agg(aop)) + need_sumsq = true; + if (agg_is_binary_agg(aop)) + need_pair = true; + } + ray_t *sum_hdr = NULL, *cnt_hdr = NULL, *task_hdr = NULL, *part_hdr = NULL; + ray_t *sumsq_hdr = NULL, *sum_y_hdr = NULL, *sumsq_y_hdr = NULL, *sumxy_hdr = NULL; + ray_t *part_sumsq_hdr = NULL, *part_sum_y_hdr = NULL, *part_sumsq_y_hdr = NULL, *part_sumxy_hdr = NULL; da_val_t* sums = NULL; + double *sumsq = NULL, *sum_y = NULL, *sumsq_y = NULL, *sumxy = NULL; int64_t* counts = NULL; if (K > 0) { sums = (da_val_t*)scratch_calloc(&sum_hdr, (size_t)K * n_aggs * sizeof(da_val_t)); counts = (int64_t*)scratch_calloc(&cnt_hdr, (size_t)K * sizeof(int64_t)); + if (need_sumsq) + sumsq = (double*)scratch_calloc(&sumsq_hdr, + (size_t)K * n_aggs * sizeof(double)); + if (need_pair) { + sum_y = (double*)scratch_calloc(&sum_y_hdr, + (size_t)K * n_aggs * sizeof(double)); + sumsq_y = (double*)scratch_calloc(&sumsq_y_hdr, + (size_t)K * n_aggs * sizeof(double)); + sumxy = (double*)scratch_calloc(&sumxy_hdr, + (size_t)K * n_aggs * sizeof(double)); + } /* Chunk slices into row tasks so one dominant key still spreads * across the pool (see sg_task_t). */ int64_t n_tasks = 0; @@ -7661,16 +7828,35 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, n_tasks += (slices[i].n + SG_CHUNK_ROWS - 1) / SG_CHUNK_ROWS; sg_task_t* tasks = NULL; da_val_t* partials = NULL; + double *part_sumsq = NULL, *part_sum_y = NULL, *part_sumsq_y = NULL, *part_sumxy = NULL; if (sums && counts) { task_hdr = ray_alloc((size_t)n_tasks * (int64_t)sizeof(sg_task_t)); tasks = task_hdr ? (sg_task_t*)ray_data(task_hdr) : NULL; partials = (da_val_t*)scratch_calloc(&part_hdr, (size_t)n_tasks * n_aggs * sizeof(da_val_t)); - } - if (!sums || !counts || !tasks || !partials) { + if (need_sumsq) + part_sumsq = (double*)scratch_calloc(&part_sumsq_hdr, + (size_t)n_tasks * n_aggs * sizeof(double)); + if (need_pair) { + part_sum_y = (double*)scratch_calloc(&part_sum_y_hdr, + (size_t)n_tasks * n_aggs * sizeof(double)); + part_sumsq_y = (double*)scratch_calloc(&part_sumsq_y_hdr, + (size_t)n_tasks * n_aggs * sizeof(double)); + part_sumxy = (double*)scratch_calloc(&part_sumxy_hdr, + (size_t)n_tasks * n_aggs * sizeof(double)); + } + } + if (!sums || !counts || !tasks || !partials || + (need_sumsq && (!sumsq || !part_sumsq)) || + (need_pair && (!sum_y || !sumsq_y || !sumxy || + !part_sum_y || !part_sumsq_y || !part_sumxy))) { scratch_free(sum_hdr); scratch_free(cnt_hdr); + scratch_free(sumsq_hdr); scratch_free(sum_y_hdr); + scratch_free(sumsq_y_hdr); scratch_free(sumxy_hdr); if (task_hdr) ray_free(task_hdr); scratch_free(part_hdr); + scratch_free(part_sumsq_hdr); scratch_free(part_sum_y_hdr); + scratch_free(part_sumsq_y_hdr); scratch_free(part_sumxy_hdr); return NULL; /* OOM → generic path via fallback */ } int64_t total_rows = 0, tw = 0; @@ -7682,8 +7868,9 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, tasks[tw].gi = i; tasks[tw].lo = lo; tasks[tw].hi = hi; tw++; } } - sg_ctx_t ctx = { slices, tasks, agg_vecs, prod, ext->agg_ops, - n_aggs, partials, {0}, {0} }; + sg_ctx_t ctx = { slices, tasks, agg_vecs, agg_vecs2, prod, ext->agg_ops, + n_aggs, partials, part_sumsq, part_sum_y, + part_sumsq_y, part_sumxy, {0}, {0} }; /* Shared-stream pairing: a bare-scan SUM/AVG over the same column * a product's int side already streams rides the product loop — * one pass over the column instead of two. */ @@ -7723,16 +7910,25 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, for (uint32_t a = 0; a < n_aggs; a++) { size_t di = (size_t)gi * n_aggs + a; size_t si = (size_t)ti * n_aggs + a; - if (prod[a].enabled || ext->agg_ops[a] == OP_COUNT || + bool pair = agg_is_binary_agg(ext->agg_ops[a]); + if (pair || prod[a].enabled || ext->agg_ops[a] == OP_COUNT || (agg_vecs[a] && agg_vecs[a]->type == RAY_F64)) sums[di].f += partials[si].f; else sums[di].i = (int64_t)((uint64_t)sums[di].i + (uint64_t)partials[si].i); + if (sumsq) sumsq[di] += part_sumsq[si]; + if (pair) { + sum_y[di] += part_sum_y[si]; + sumsq_y[di] += part_sumsq_y[si]; + sumxy[di] += part_sumxy[si]; + } } } ray_free(task_hdr); scratch_free(part_hdr); + scratch_free(part_sumsq_hdr); scratch_free(part_sum_y_hdr); + scratch_free(part_sumsq_y_hdr); scratch_free(part_sumxy_hdr); } /* Emit: key column (domain ids in slice order), then the shared agg @@ -7741,6 +7937,8 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_t* result = ray_table_new(1 + n_aggs); if (!result || RAY_IS_ERR(result)) { scratch_free(sum_hdr); scratch_free(cnt_hdr); + scratch_free(sumsq_hdr); scratch_free(sum_y_hdr); + scratch_free(sumsq_y_hdr); scratch_free(sumxy_hdr); return NULL; } ray_t* kc = col_vec_new(key_col, K > 0 ? K : 1); @@ -7748,6 +7946,8 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (kc) ray_release(kc); ray_release(result); scratch_free(sum_hdr); scratch_free(cnt_hdr); + scratch_free(sumsq_hdr); scratch_free(sum_y_hdr); + scratch_free(sumsq_y_hdr); scratch_free(sumxy_hdr); return NULL; } if (kc->type == RAY_SYM) @@ -7760,15 +7960,19 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_release(kc); if (!result || RAY_IS_ERR(result)) { scratch_free(sum_hdr); scratch_free(cnt_hdr); + scratch_free(sumsq_hdr); scratch_free(sum_y_hdr); + scratch_free(sumsq_y_hdr); scratch_free(sumxy_hdr); return result; } emit_agg_columns(&result, g, ext, agg_vecs, (uint32_t)K, n_aggs, (double*)sums, (int64_t*)sums, NULL, NULL, NULL, NULL, - counts, NULL, prod, NULL, NULL); + counts, NULL, prod, sumsq, NULL, sum_y, sumsq_y, sumxy); scratch_free(sum_hdr); scratch_free(cnt_hdr); + scratch_free(sumsq_hdr); scratch_free(sum_y_hdr); + scratch_free(sumsq_y_hdr); scratch_free(sumxy_hdr); return result; } @@ -8208,7 +8412,8 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { emit_agg_columns(&result, g, ext, agg_vecs, grp_count, n_aggs, (double*)dense_sum, (int64_t*)dense_sum, NULL, NULL, NULL, NULL, - dense_count, agg_affine, agg_prod, NULL, NULL); + dense_count, agg_affine, agg_prod, NULL, NULL, + NULL, NULL, NULL); scratch_free(_h_sum); scratch_free(_h_cnt); scratch_free(range_sum_hdr); scratch_free(cnt_hdr); @@ -8924,7 +9129,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, sums_f64, sums_i64, NULL, NULL, NULL, NULL, counts, agg_affine, agg_prod, - NULL, NULL); + NULL, NULL, NULL, NULL, NULL); scratch_free(sum_hdr); scratch_free(cnt_hdr); ray_release(base_sum_obj); @@ -9158,7 +9363,8 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, (double*)m->sum, (int64_t*)m->sum, (double*)m->min_val, (double*)m->max_val, (int64_t*)m->min_val, (int64_t*)m->max_val, - m->count, agg_affine, agg_prod, m->sumsq_f64, m->nn_count); + m->count, agg_affine, agg_prod, m->sumsq_f64, m->nn_count, + NULL, NULL, NULL); /* Wide-element (STR/GUID) min/max/first/last overflow emit_agg_columns' * fixed-width slots (it truncated them to 1 byte above). Recompute @@ -9960,7 +10166,7 @@ da_path:; (double*)dense_min_val, (double*)dense_max_val, (int64_t*)dense_min_val, (int64_t*)dense_max_val, dense_counts, agg_affine, agg_prod, dense_sumsq, - dense_nn_counts); + dense_nn_counts, NULL, NULL, NULL); scratch_free(_h_dsum); scratch_free(_h_dmin); scratch_free(_h_dmax); @@ -10263,7 +10469,8 @@ da_path:; emit_agg_columns(&result, g, ext, agg_vecs, grp_count, n_aggs, (double*)dense_sum, (int64_t*)dense_sum, NULL, NULL, NULL, NULL, - dense_count, agg_affine, agg_prod, NULL, NULL); + dense_count, agg_affine, agg_prod, NULL, NULL, + NULL, NULL, NULL); scratch_free(_h_sum); scratch_free(_h_cnt); @@ -10489,7 +10696,8 @@ da_path:; emit_agg_columns(&result, g, ext, agg_vecs, grp_count, n_aggs, (double*)dense_sum, (int64_t*)dense_sum, NULL, NULL, NULL, NULL, - dense_count, agg_affine, agg_prod, NULL, NULL); + dense_count, agg_affine, agg_prod, NULL, NULL, + NULL, NULL, NULL); scratch_free(_h_sum); scratch_free(_h_cnt); diff --git a/src/ops/query.c b/src/ops/query.c index 8d3a7fbf4..02c048de6 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -1847,29 +1847,34 @@ static int is_group_dag_agg_expr(ray_t* expr); /* defined below */ * result. Eligibility: >=1 extractable agg; every table-column reference * sits inside one; agg arguments contain no nested aggs; connectives may * be arbitrary calls over literals/globals. */ -/* Slice-group arming gate: the exec_group slice kernel takes only - * SUM/AVG/COUNT over a bare column or a two-column product. Arming the - * hint for other agg shapes is legal (exec_group folds it back into a - * selection) but wasteful — the fold costs a sort of every surviving - * row id. Checked over dag-agg outputs and decomposed hidden slots. */ +/* Slice-group arming gate: keep the hint only for aggregate shapes the + * direct slice path can consume. Arming other shapes is legal, but then + * exec_group must fold the slices back into a selection first. */ static int sg_agg_expr_ok(ray_t* expr) { if (!expr || !is_list(expr) || ray_len(expr) < 2) return 0; ray_t** e = (ray_t**)ray_data(expr); if (!e[0] || e[0]->type != -RAY_SYM) return 0; - static int64_t s_sum = -1, s_avg = -1, s_count = -1, s_mul = -1; - if (s_sum < 0) { - s_sum = ray_sym_intern("sum", 3); - s_avg = ray_sym_intern("avg", 3); - s_count = ray_sym_intern("count", 5); - s_mul = ray_sym_intern("*", 1); - } - int64_t h = e[0]->i64; - if (h != s_sum && h != s_avg && h != s_count) return 0; - if (h == s_count) return 1; /* group size; arg unused */ + uint16_t op = resolve_agg_opcode(e[0]->i64); + if (!op) return 0; + if (op == OP_COUNT) return 1; /* group size; arg unused */ ray_t* arg = e[1]; - if (arg && arg->type == -RAY_SYM && !(arg->attrs & ATTR_QUOTED)) return 1; + bool arg_bare = arg && arg->type == -RAY_SYM && + !(arg->attrs & ATTR_QUOTED); + if (agg_is_binary_agg(op)) { + if (ray_len(expr) < 3) return 0; + ray_t* arg2 = e[2]; + return arg_bare && arg2 && arg2->type == -RAY_SYM && + !(arg2->attrs & ATTR_QUOTED); + } + if (op == OP_STDDEV || op == OP_STDDEV_POP || + op == OP_VAR || op == OP_VAR_POP) + return arg_bare; + if (op != OP_SUM && op != OP_AVG) return 0; + if (arg_bare) return 1; if (arg && is_list(arg) && ray_len(arg) == 3) { ray_t** m = (ray_t**)ray_data(arg); + static int64_t s_mul = -1; + if (s_mul < 0) s_mul = ray_sym_intern("*", 1); if (m[0] && m[0]->type == -RAY_SYM && m[0]->i64 == s_mul && m[1] && m[1]->type == -RAY_SYM && !(m[1]->attrs & ATTR_QUOTED) && m[2] && m[2]->type == -RAY_SYM && !(m[2]->attrs & ATTR_QUOTED)) diff --git a/test/rfl/integration/slice_group.rfl b/test/rfl/integration/slice_group.rfl index fc7bdeabc..99a4252db 100644 --- a/test/rfl/integration/slice_group.rfl +++ b/test/rfl/integration/slice_group.rfl @@ -50,6 +50,24 @@ (min (== (at Q2S 'p) (at Q2P 'p))) -- true (min (== (at Q2S 'sw) (at Q2P 'sw))) -- true +(set Q2BI (select {sd: (stddev f) cp: (pearson_corr f v) cs: (pearson_corr v w) cv: (cov f w) sc: (scov f w) ws: (wsum w f) wa: (wavg w f) from: TI by: s where: (in s ['aa 'cc 'ee])})) +(set Q2BP (select {sd: (stddev f) cp: (pearson_corr f v) cs: (pearson_corr v w) cv: (cov f w) sc: (scov f w) ws: (wsum w f) wa: (wavg w f) from: TP by: s where: (in s ['aa 'cc 'ee])})) +(min (< (abs (- (at Q2BI 'sd) (at Q2BP 'sd))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'cp) (at Q2BP 'cp))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'cs) (at Q2BP 'cs))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'cv) (at Q2BP 'cv))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'sc) (at Q2BP 'sc))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'ws) (at Q2BP 'ws))) 1e-9)) -- true +(min (< (abs (- (at Q2BI 'wa) (at Q2BP 'wa))) 1e-9)) -- true +(set Q2BS (select {sd: (stddev f) cp: (pearson_corr f v) cs: (pearson_corr v w) cv: (cov f w) sc: (scov f w) ws: (wsum w f) wa: (wavg w f) from: TSI by: s where: (in s ['aa 'cc 'ee])})) +(min (< (abs (- (at Q2BS 'sd) (at Q2BP 'sd))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'cp) (at Q2BP 'cp))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'cs) (at Q2BP 'cs))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'cv) (at Q2BP 'cv))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'sc) (at Q2BP 'sc))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'ws) (at Q2BP 'ws))) 1e-9)) -- true +(min (< (abs (- (at Q2BS 'wa) (at Q2BP 'wa))) 1e-9)) -- true + ;; ── shared-stream pairing: sum over the product's int side (q27 shape), ;; incl. arith-of-aggs decomposition wrappers ── (set Q3I (select {wp: (/ (sum (* w f)) (as 'F64 (sum w))) from: TI by: s where: (in s ['aa 'bb 'cc 'dd])})) From 4203946247ecb08ef5768c516a32482ffa1b34ed Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 07:51:27 +0200 Subject: [PATCH 02/10] feat(mem): add scoped query measurements Restore allocator GC and add .mem.objsize and .mem.ts. Share query timing, memory, and worker metrics with profiling and query logging. --- docs/docs/guides/memory.md | 10 +- docs/docs/namespaces/index.md | 3 +- docs/docs/namespaces/mem.md | 71 ++++++ docs/docs/namespaces/sys.md | 8 +- docs/docs/reference/all-functions.md | 4 +- mkdocs.yml | 1 + src/core/ipc.c | 2 +- src/core/qlog.c | 24 +-- src/core/qlog.h | 18 +- src/core/qmeasure.c | 58 +++++ src/core/qmeasure.h | 36 ++++ src/lang/eval.c | 11 +- src/lang/internal.h | 2 + src/mem/heap.c | 88 +++++++- src/mem/heap.h | 14 ++ src/ops/exec.c | 9 +- src/ops/system.c | 310 ++++++++++++++++++++++++++- test/rfl/system/mem_ts.rfl | 29 +++ test/rfl/system/objsize.rfl | 15 ++ 19 files changed, 665 insertions(+), 48 deletions(-) create mode 100644 docs/docs/namespaces/mem.md create mode 100644 src/core/qmeasure.c create mode 100644 src/core/qmeasure.h create mode 100644 test/rfl/system/mem_ts.rfl create mode 100644 test/rfl/system/objsize.rfl diff --git a/docs/docs/guides/memory.md b/docs/docs/guides/memory.md index a2841094a..998c1d2d6 100644 --- a/docs/docs/guides/memory.md +++ b/docs/docs/guides/memory.md @@ -68,7 +68,11 @@ In this example, loading 10 million rows consumed roughly 800 MB of heap memory. ## 3. The `.sys.gc` Function -Call `(.sys.gc 0)` to signal that the runtime should reclaim unused memory. Currently this is a lightweight hook that returns `0` — Rayforce uses deterministic ref counting and eager page release via `madvise` during buddy coalescing, so most memory is reclaimed automatically when references are dropped. +Call `(.sys.gc)` to run allocator maintenance after references have been +dropped. It drains cross-thread frees, flushes slab caches, coalesces free +blocks, reclaims empty oversized pools, and incrementally releases aged free +pages. Values themselves use deterministic reference counting; this is not a +tracing collector. ```lisp (.sys.gc) ; 0 @@ -76,7 +80,9 @@ Call `(.sys.gc 0)` to signal that the runtime should reclaim unused memory. Curr !!! note "Note" - Because Rayforce uses deterministic ref counting (not tracing GC), memory is freed immediately when the last reference is released. The buddy allocator coalesces blocks and releases pages back to the OS automatically. `(.sys.gc 0)` exists as a hook for future use. + Because Rayforce uses deterministic reference counting, objects become + reclaimable immediately when their last reference is released. `(.sys.gc)` + performs the allocator-side consolidation and page-return pass. ## 4. The `.sys.info` Function diff --git a/docs/docs/namespaces/index.md b/docs/docs/namespaces/index.md index 5a6537e2b..4312a61dc 100644 --- a/docs/docs/namespaces/index.md +++ b/docs/docs/namespaces/index.md @@ -15,6 +15,7 @@ Rayfall's builtins are organised under dotted namespaces. Names beginning with ` | [`.idx.*`](idx.md) | Accelerator indexes: bloom, hash, sort, zone. | | [`.ipc.*`](ipc.md) | TCP client IPC and the server connection-hook accessor. | | [`.log.*`](log.md) | Write-ahead log: open, write, sync, snapshot, roll, replay, validate. | +| [`.mem.*`](mem.md) | Value sizing and scoped time/allocation measurement. | | [`.os.*`](os.md) | Process environment: `getenv`, `setenv`. | | [`.repl.*`](repl.md) | Interactive REPL control — attach the local REPL to a remote server. | | [`.sys.*`](sys.md) | System info and shell-style commands: build, info, mem, prof, querylog, gc, exec, listen, timeit, env, cmd. | @@ -33,4 +34,4 @@ When the server is started with `-U `, the following dotted builtins a - `.sys.exec`, `.sys.cmd`, `.sys.listen`, `.sys.querylog.enable` - `.time.timer.set`, `.time.timer.del` -`.attr.*`, `.col.*`, `.graph.*`, `.idx.*`, and pure read/inspect builtins (`.db.splayed.get`, `.db.parted.get`, `.db.parted.tables`, `.fs.size`, `.fs.list`, `.log.write`, `.log.sync`, `.log.validate`, `.sys.build`, `.sys.info`, `.sys.mem`, `.sys.prof`, `.sys.querylog`, `.sys.gc`, `.sys.env`, `.sys.args`, `.sys.timeit`, `.ipc.handle`, `.time.now`) are always available. +`.attr.*`, `.col.*`, `.graph.*`, `.idx.*`, `.mem.*`, and pure read/inspect builtins (`.db.splayed.get`, `.db.parted.get`, `.db.parted.tables`, `.fs.size`, `.fs.list`, `.log.write`, `.log.sync`, `.log.validate`, `.sys.build`, `.sys.info`, `.sys.mem`, `.sys.prof`, `.sys.querylog`, `.sys.gc`, `.sys.env`, `.sys.args`, `.sys.timeit`, `.ipc.handle`, `.time.now`) are always available. diff --git a/docs/docs/namespaces/mem.md b/docs/docs/namespaces/mem.md new file mode 100644 index 000000000..38dfc8f0c --- /dev/null +++ b/docs/docs/namespaces/mem.md @@ -0,0 +1,71 @@ +# `.mem.*` — memory measurement + +Value-level memory inspection and explicitly scoped query measurement. These +operations are unrestricted and do not change the measured value. + +## Reference + +| Function | Arity | Flags | Description | +|---|---|---|---| +| [`.mem.objsize`](#mem-objsize) | unary | — | Logical bytes retained by a value. | +| [`.mem.ts`](#mem-ts) | 1 | special form | Evaluate once and return result plus time, allocation, peak-memory, result-size, and worker statistics. | + +## `.mem.objsize` { #mem-objsize } + +Signature: `(.mem.objsize value)`. + +Returns the logical byte size of the complete `ray_t` graph reachable from +`value`. Each object header and live payload is counted; a shared child is +counted once. The number deliberately excludes buddy-block slack, global +symbol dictionaries, and opaque native handles, making it stable across +allocator tuning. It is an in-memory size, not a serialized-wire size. + +```lisp +(.mem.objsize 42) ;; 32: one ray_t header +(.mem.objsize [1 2 3]) ;; 56: 32-byte header + three I64 cells +``` + +## `.mem.ts` { #mem-ts } + +Signature: `(.mem.ts expression)`. + +`.mem.ts` is a special form: it evaluates `expression` exactly once inside a +process-wide measurement scope. Secondary-worker allocations and worker busy +time are included. Result sizing and construction of the returned statistics +dictionary happen after the time/allocation scope closes. + +```lisp +(set m (.mem.ts (sum (til 1000000)))) +(at m 'result) +(at m 'peak-live-bytes) +``` + +The returned dictionary has these fields: + +| Key | Meaning | +|---|---| +| `result` | The expression's result. | +| `time-ns` | Monotonic wall-clock duration. | +| `memory-bytes` | Net process-memory delta from the shared per-query profiler/query-log measurement. | +| `allocated-bytes` | Sum of allocator block bytes obtained during the scope, including temporary allocations. | +| `freed-bytes` | Sum of allocator block bytes released before the measured evaluation returned. | +| `net-bytes` | Allocated minus freed bytes at the measurement boundary; it may be negative when the expression releases pre-existing values. | +| `peak-live-bytes` | Maximum positive live-byte delta above the starting boundary. This is the closest analogue of q's `.Q.ts` space value. | +| `result-bytes` | Logical `.mem.objsize` of `result`. | +| `alloc-count` | Number of allocator blocks obtained. | +| `free-count` | Number of allocator blocks released. | +| `workers` | Pool workers, including worker 0 (the main thread), that executed at least one parallel task. Zero means no pool dispatch occurred. | +| `worker-busy-ns` | Busy nanoseconds summed across participating workers. | +| `parallelism` | `worker-busy-ns / time-ns`. | + +Only one `.mem.ts` scope may be active in a process, so nested calls return a +`state` error. Allocation instrumentation is normally inactive; ordinary +queries pay only a predicted-not-taken check at allocator entry/exit. While a +scope is active, atomic accounting adds intentional overhead. For performance +benchmarks, time normal executions separately and use `.mem.ts` as an +additional memory-measurement execution. + +The common fields (`time-ns`, `memory-bytes`, `workers`, `worker-busy-ns`, and +`parallelism`) come from the same per-query measurement lifecycle used by the +profiler and query log. The allocation counters and `result-bytes` are the +additional `.mem.ts`-specific detail. diff --git a/docs/docs/namespaces/sys.md b/docs/docs/namespaces/sys.md index 2039d74b2..c8b3ce7a3 100644 --- a/docs/docs/namespaces/sys.md +++ b/docs/docs/namespaces/sys.md @@ -16,7 +16,7 @@ Process-level introspection (build, memory, host info) and command-style operati | [`.sys.prof`](#sys-prof) | variadic | — | Last profiled query's per-step statistics as a table. | | [`.sys.querylog`](#sys-querylog) | variadic | — | Ambient per-query statistics ring as a table. | | [`.sys.querylog.enable`](#sys-querylog-enable) | variadic | restricted | Toggle query-statistics logging. | -| [`.sys.gc`](#sys-gc) | variadic | — | Garbage-collect hint (no-op; returns 0). | +| [`.sys.gc`](#sys-gc) | variadic | — | Run allocator maintenance and return `0`. | | [`.sys.env`](#sys-env) | variadic | — | Count or list of globally bound names. | | [`.sys.exec`](#sys-exec) | unary | restricted | Run a shell command; return its exit code. | | [`.sys.cmd`](#sys-cmd) | unary | restricted | Dispatch a colon-command string. | @@ -209,7 +209,11 @@ The `-Q` startup flag is the equivalent for enabling logging from launch. ## `.sys.gc` { #sys-gc } -Signature: `(.sys.gc)`. No-op stub — Rayforce uses reference counting, so there's no global collector to kick. Kept as a registered builtin so existing scripts that call it don't error. Returns `0`. +Signature: `(.sys.gc)`. Runs allocator maintenance: drains cross-thread frees, +flushes slab caches, coalesces free buddy blocks, reclaims empty oversized +pools, and incrementally returns aged free pages to the operating system. +Rayforce values remain deterministically reference-counted—this is allocator +GC, not a tracing object collector. Returns `0`. ## `.sys.env` { #sys-env } diff --git a/docs/docs/reference/all-functions.md b/docs/docs/reference/all-functions.md index 3870e2563..bbff65b93 100644 --- a/docs/docs/reference/all-functions.md +++ b/docs/docs/reference/all-functions.md @@ -68,7 +68,7 @@ Generated from `src/lang/eval.c` in this checkout. The categorized reference bel `update`, `insert`, `upsert`, `left-join`, `inner-join`, `full-join`, `anti-join`, `window-join`, `window-join1`, `asof-join`, `println`, `show`, `format`, `read-csv`, `write-csv`, `.csv.read`, `.csv.splayed`, `.csv.parted`, `.csv.write`, `resolve`, `timeit`, `enlist`, `map-left`, `map-right`, `.db.splayed.set`, `.db.splayed.get`, -`.db.parted.get`, `.db.parted.tables`, `.db.parted.fill`, `alter`, `print`, `.sys.gc`, `.sys.timeit`, +`.db.parted.get`, `.db.parted.tables`, `.db.parted.fill`, `alter`, `print`, `.sys.gc`, `.mem.objsize`, `.mem.ts`, `.sys.timeit`, `.sys.env`, `.sys.args`, `.ipc.open`, `.ipc.handle`, `.repl.disconnect`, `.log.open`, `.log.roll`, `.log.snapshot`, `.log.sync`, `.log.close`, `.log.purge`, `quote`, `return`, `.time.now`, `.time.timer.set`, `fold-left`, `fold-right`, `scan-left`, `scan-right`, `del`, `.sys.build`, `.sys.mem`, `.sys.prof`, @@ -634,6 +634,8 @@ System interaction, metaprogramming, diagnostics, and runtime inspection. | `eval` | unary | — | Evaluate a parsed Rayfall expression | `(eval (parse "(+ 1 2)"))` → `3` | | `parse` | unary | — | Parse a string into a Rayfall expression tree | `(parse "(+ 1 2)")` | | `.sys.gc` | variadic | — | Trigger GC / heap flush, returns `0` | `(.sys.gc)` | +| `.mem.objsize` | unary | — | Logical bytes retained by an object graph; shared children count once | `(.mem.objsize value)` | +| `.mem.ts` | special form | — | Evaluate once and return result, time, allocation, peak-memory, result-size, and worker statistics as a dict | `(.mem.ts (select {from: trades}))` | | `.sys.exec` | unary | restricted | Execute a shell command, return exit code | `(.sys.exec "ls -la")` | | `.os.getenv` | unary | restricted | Get environment variable value | `(.os.getenv "HOME")` | | `.os.setenv` | binary | restricted | Set environment variable | `(.os.setenv "KEY" "value")` | diff --git a/mkdocs.yml b/mkdocs.yml index 66dadc415..5dc11ad20 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -133,6 +133,7 @@ nav: - .idx — Indexes: docs/namespaces/idx.md - .ipc — IPC client: docs/namespaces/ipc.md - .log — Write-ahead log: docs/namespaces/log.md + - .mem — Measurement: docs/namespaces/mem.md - .os — Process env: docs/namespaces/os.md - .repl — REPL control: docs/namespaces/repl.md - .sys — System info: docs/namespaces/sys.md diff --git a/src/core/ipc.c b/src/core/ipc.c index 7b6aa0b2b..a2fce94f8 100644 --- a/src/core/ipc.c +++ b/src/core/ipc.c @@ -488,7 +488,7 @@ static ray_t* eval_payload_core(uint8_t* payload, size_t payload_len, * is formatted back to source. Only when logging is on. */ char qsrc[RAY_QLOG_QUERY_MAX]; size_t qsrc_len = 0; - if (qc.t0_ns && msg && !RAY_IS_ERR(msg)) { + if (qc.measure.active && msg && !RAY_IS_ERR(msg)) { if (msg->type == -RAY_STR) { qsrc_len = ray_str_len(msg); if (qsrc_len > sizeof(qsrc)) qsrc_len = sizeof(qsrc); diff --git a/src/core/qlog.c b/src/core/qlog.c index c98b33371..d6a17a5cb 100644 --- a/src/core/qlog.c +++ b/src/core/qlog.c @@ -10,24 +10,20 @@ #include #include "core/qlog.h" -#include "core/qstats.h" /* ray_qstats_agg */ /* Single global instance. Zero-initialised: disabled, empty ring. */ ray_qlog_t g_qlog; -void ray_qlog_end(const ray_qlog_ctx_t* c, const char* src, size_t src_len, +void ray_qlog_end(ray_qlog_ctx_t* c, const char* src, size_t src_len, ray_t* result) { - if (c->t0_ns == 0) return; /* logging was off when the query began */ - - int64_t dur_ns = ray_profile_now_ns() - c->t0_ns; - if (dur_ns < 0) dur_ns = 0; - int64_t cur = 0; - ray_sys_get_stat(&cur, NULL); + if (!c->measure.active) return; /* logging was off when query began */ + ray_query_metrics_t metrics; + ray_query_measure_end(&c->measure, &metrics); ray_qlog_row_t* row = &g_qlog.slot[g_qlog.head & (RAY_QLOG_CAP - 1)]; row->time_ns = ray_timestamp_now_ns(); - row->duration_ms = (double)dur_ns / 1e6; - row->memory_kib = (double)(cur - c->mem0) / 1024.0; + row->duration_ms = (double)metrics.time_ns / 1e6; + row->memory_kib = (double)metrics.memory_bytes / 1024.0; /* Result summary + status. ray_len on a scalar atom returns the value, * not a count (the len field aliases i64), so guard atoms as one row. */ @@ -44,12 +40,8 @@ void ray_qlog_end(const ray_qlog_ctx_t* c, const char* src, size_t src_len, : (int64_t)ray_len(result); } - /* Parallelism from the query-scoped qstats aggregate (ray_eval resets the - * slots at each top-level query when capture is armed). */ - uint64_t busy = 0, mx = 0; uint32_t used = 0; - ray_qstats_agg(&used, &busy, &mx); - row->workers = (int32_t)used; - row->parallelism = dur_ns > 0 ? (double)busy / (double)dur_ns : 0.0; + row->workers = (int32_t)metrics.workers; + row->parallelism = metrics.parallelism; /* Bounded copies — no allocation, ring memory stays fixed. */ size_t qn = src ? src_len : 0; diff --git a/src/core/qlog.h b/src/core/qlog.h index d942602cb..4b57c1bb0 100644 --- a/src/core/qlog.h +++ b/src/core/qlog.h @@ -26,8 +26,7 @@ #include #include -#include "core/profile.h" /* ray_profile_now_ns */ -#include "mem/sys.h" /* ray_sys_get_stat */ +#include "core/qmeasure.h" /* Ring capacity (power of two so the write index masks cleanly) and the inline * caps for the two variable-length fields — both bounded so the ring is a @@ -69,23 +68,18 @@ static inline void ray_qlog_set_enabled(bool on) { atomic_store_explicit(&g_qlog.enabled, on ? 1u : 0u, memory_order_relaxed); } -/* Baseline captured before an eval; fed back to ray_qlog_end for the deltas. - * t0_ns == 0 marks "logging was off at begin", so end is a no-op even if the - * flag flips mid-query. */ -typedef struct { int64_t t0_ns; int64_t mem0; } ray_qlog_ctx_t; +/* Query logging wraps the same measurement scope used by `.mem.ts`. */ +typedef struct { ray_query_measure_t measure; } ray_qlog_ctx_t; static inline void ray_qlog_begin(ray_qlog_ctx_t* c) { - if (!ray_qlog_enabled()) { c->t0_ns = 0; c->mem0 = 0; return; } - int64_t cur = 0; - ray_sys_get_stat(&cur, NULL); - c->mem0 = cur; - c->t0_ns = ray_profile_now_ns(); + c->measure = (ray_query_measure_t){0}; + if (ray_qlog_enabled()) ray_query_measure_begin(&c->measure); } /* Close a query and append its summary row. No-op when logging was off at * begin. `src` is the query source (truncated to RAY_QLOG_QUERY_MAX); `result` * is the final, materialized result (an error object sets a non-ok status). */ -void ray_qlog_end(const ray_qlog_ctx_t* c, const char* src, size_t src_len, +void ray_qlog_end(ray_qlog_ctx_t* c, const char* src, size_t src_len, ray_t* result); /* Materialize the ring into a fresh table (oldest row first). Returns an diff --git a/src/core/qmeasure.c b/src/core/qmeasure.c new file mode 100644 index 000000000..1120ce551 --- /dev/null +++ b/src/core/qmeasure.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025-2026 Anton Kundenko + * All rights reserved. + */ + +#include "core/qmeasure.h" + +#include "core/profile.h" +#include "core/qstats.h" +#include "mem/sys.h" + +void ray_query_workers_begin(bool capture, bool progress) { + uint32_t mode = (capture ? RAY_QS_PROF : 0) | + (progress ? RAY_QS_PROGRESS : 0); + ray_qstats_set_mode(mode); + if (mode) ray_qstats_reset(0); +} + +void ray_query_workers_snapshot(uint32_t* workers, uint64_t* busy_ns) { + uint64_t max_busy = 0; + ray_qstats_agg(workers, busy_ns, &max_busy); +} + +void ray_query_measure_begin(ray_query_measure_t* scope) { + if (!scope) return; + scope->prior_qstats_mode = ray_qstats_mode(); + ray_qstats_set_mode(scope->prior_qstats_mode | RAY_QS_PROF); + ray_qstats_reset(0); + ray_sys_get_stat(&scope->memory0, NULL); + scope->t0_ns = ray_profile_now_ns(); + scope->active = true; +} + +void ray_query_measure_end(ray_query_measure_t* scope, ray_query_metrics_t* out) { + if (!scope || !scope->active) { + if (out) *out = (ray_query_metrics_t){0}; + return; + } + + int64_t elapsed = ray_profile_now_ns() - scope->t0_ns; + int64_t memory = 0; + ray_sys_get_stat(&memory, NULL); + uint32_t workers = 0; + uint64_t busy_ns = 0; + ray_query_workers_snapshot(&workers, &busy_ns); + ray_qstats_set_mode(scope->prior_qstats_mode); + scope->active = false; + + if (!out) return; + if (elapsed < 0) elapsed = 0; + *out = (ray_query_metrics_t){ + .time_ns = elapsed, + .memory_bytes = memory - scope->memory0, + .workers = workers, + .worker_busy_ns = busy_ns, + .parallelism = elapsed > 0 ? (double)busy_ns / (double)elapsed : 0.0, + }; +} diff --git a/src/core/qmeasure.h b/src/core/qmeasure.h new file mode 100644 index 000000000..92afe0a60 --- /dev/null +++ b/src/core/qmeasure.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025-2026 Anton Kundenko + * All rights reserved. + * + * Shared per-query measurement lifecycle. Query logging, profiler spans, + * and `.mem.ts` consume this layer so their wall time, process memory, + * worker count, busy time, and parallelism have one definition. + */ + +#ifndef RAY_QMEASURE_H +#define RAY_QMEASURE_H + +#include +#include + +typedef struct { + int64_t time_ns; + int64_t memory_bytes; + uint32_t workers; + uint64_t worker_busy_ns; + double parallelism; +} ray_query_metrics_t; + +typedef struct { + int64_t t0_ns; + int64_t memory0; + uint32_t prior_qstats_mode; + bool active; +} ray_query_measure_t; + +void ray_query_workers_begin(bool capture, bool progress); +void ray_query_workers_snapshot(uint32_t* workers, uint64_t* busy_ns); +void ray_query_measure_begin(ray_query_measure_t* scope); +void ray_query_measure_end(ray_query_measure_t* scope, ray_query_metrics_t* out); + +#endif /* RAY_QMEASURE_H */ diff --git a/src/lang/eval.c b/src/lang/eval.c index ab9c9c1e0..53c5d52b9 100644 --- a/src/lang/eval.c +++ b/src/lang/eval.c @@ -38,6 +38,7 @@ #include "vec/vec.h" #include "core/profile.h" #include "core/qstats.h" /* per-worker parallelism counters for eval-path spans */ +#include "core/qmeasure.h" /* shared per-query worker lifecycle */ #include "core/qlog.h" /* ray_qlog_enabled — arm capture for query logging */ #include "store/serde.h" /* ray_serde_size — result footprint for spans */ #include "table/sym.h" @@ -3236,6 +3237,8 @@ static void ray_register_builtins(void) { * the category. */ register_vary (".sys.gc", RAY_FN_NONE, ray_gc_fn); register_unary(".sys.exec", RAY_FN_RESTRICTED, ray_system_fn); + register_unary(".mem.objsize", RAY_FN_NONE, ray_mem_objsize_fn); + register_vary (".mem.ts", RAY_FN_SPECIAL_FORM, ray_mem_ts_fn); /* Registry-dispatched system commands. `.sys.cmd "name args"` is * the colon-command entry point; the per-command direct builtins below * skip the string parse for callers that already have a typed arg @@ -3454,8 +3457,8 @@ static inline void eval_span_payload(ray_prof_span_t* s, ray_t* result) { int64_t cur = 0; ray_sys_get_stat(&cur, NULL); s->sys_cur = cur; s->qs_rows = ray_qstats_sum_rows(); - uint64_t sum = 0, mx = 0; uint32_t used = 0; - ray_qstats_agg(&used, &sum, &mx); + uint64_t sum = 0; uint32_t used = 0; + ray_query_workers_snapshot(&used, &sum); s->qs_busy_ns = sum; s->qs_workers = used; if (result && !RAY_IS_ERR(result) && !RAY_IS_NULL(result)) { @@ -3488,9 +3491,7 @@ ray_t* ray_eval(ray_t* obj) { * capture strictly off (mode 0) when the profiler is disabled. */ bool capture = g_ray_profile.active || ray_qlog_enabled(); bool progress = ray_progress_active(); - uint32_t qsmode = (capture ? RAY_QS_PROF : 0) | (progress ? RAY_QS_PROGRESS : 0); - ray_qstats_set_mode(qsmode); - if (qsmode) ray_qstats_reset(0); + ray_query_workers_begin(capture, progress); } /* Check for external interrupt (e.g. Ctrl-C from REPL) */ diff --git a/src/lang/internal.h b/src/lang/internal.h index c840447f2..e45a04cf6 100644 --- a/src/lang/internal.h +++ b/src/lang/internal.h @@ -592,6 +592,8 @@ ray_t* ray_eval_builtin_fn(ray_t* x); ray_t* ray_parse_builtin_fn(ray_t* x); ray_t* ray_print_fn(ray_t** args, int64_t n); ray_t* ray_meta_fn(ray_t* x); +ray_t* ray_mem_objsize_fn(ray_t* x); +ray_t* ray_mem_ts_fn(ray_t** args, int64_t n); ray_t* ray_gc_fn(ray_t** args, int64_t n); ray_t* ray_system_fn(ray_t* x); /* `.sys.cmd "name args"` — registry-dispatched system commands with diff --git a/src/mem/heap.c b/src/mem/heap.c index 249eed0dd..eb888bb1a 100644 --- a/src/mem/heap.c +++ b/src/mem/heap.c @@ -242,6 +242,82 @@ static _Atomic(uint64_t) g_heap_id_cursor = 0; static _Atomic(int64_t) g_direct_bytes = 0; static _Atomic(int64_t) g_direct_count = 0; +/* One process-wide, explicitly-scoped allocation trace for `.mem.ts`. + * The inactive hot-path cost is one relaxed load plus a predicted-not-taken + * branch in ray_alloc/ray_free. While active, atomics make allocations from + * the main and every worker heap contribute to the same query measurement. */ +static _Atomic(uint32_t) g_mem_trace_active = 0; +static _Atomic(uint64_t) g_mem_trace_alloc_count = 0; +static _Atomic(uint64_t) g_mem_trace_free_count = 0; +static _Atomic(uint64_t) g_mem_trace_allocated = 0; +static _Atomic(uint64_t) g_mem_trace_freed = 0; +static _Atomic(int64_t) g_mem_trace_live = 0; +static _Atomic(uint64_t) g_mem_trace_peak = 0; + +static inline void mem_trace_note_alloc(size_t bytes) { + if (RAY_LIKELY(atomic_load_explicit(&g_mem_trace_active, + memory_order_relaxed) != 1)) return; + atomic_fetch_add_explicit(&g_mem_trace_alloc_count, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&g_mem_trace_allocated, (uint64_t)bytes, + memory_order_relaxed); + int64_t live = atomic_fetch_add_explicit(&g_mem_trace_live, (int64_t)bytes, + memory_order_relaxed) + (int64_t)bytes; + uint64_t candidate = live > 0 ? (uint64_t)live : 0; + uint64_t peak = atomic_load_explicit(&g_mem_trace_peak, memory_order_relaxed); + while (candidate > peak && + !atomic_compare_exchange_weak_explicit(&g_mem_trace_peak, &peak, + candidate, + memory_order_relaxed, + memory_order_relaxed)) {} +} + +static inline void mem_trace_note_free(size_t bytes) { + if (RAY_LIKELY(atomic_load_explicit(&g_mem_trace_active, + memory_order_relaxed) != 1)) return; + atomic_fetch_add_explicit(&g_mem_trace_free_count, 1, memory_order_relaxed); + atomic_fetch_add_explicit(&g_mem_trace_freed, (uint64_t)bytes, + memory_order_relaxed); + atomic_fetch_sub_explicit(&g_mem_trace_live, (int64_t)bytes, + memory_order_relaxed); +} + +bool ray_mem_trace_begin(void) { + uint32_t expected = 0; + if (!atomic_compare_exchange_strong_explicit(&g_mem_trace_active, &expected, 2, + memory_order_acq_rel, + memory_order_relaxed)) + return false; + atomic_store_explicit(&g_mem_trace_alloc_count, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_free_count, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_allocated, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_freed, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_live, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_peak, 0, memory_order_relaxed); + atomic_store_explicit(&g_mem_trace_active, 1, memory_order_release); + return true; +} + +void ray_mem_trace_end(ray_mem_trace_t* out) { + /* Freeze first: state 2 makes allocation hooks ignore the scope while + * preventing another caller from resetting the counters mid-snapshot. */ + atomic_store_explicit(&g_mem_trace_active, 2, memory_order_release); + if (out) { + out->alloc_count = atomic_load_explicit(&g_mem_trace_alloc_count, + memory_order_relaxed); + out->free_count = atomic_load_explicit(&g_mem_trace_free_count, + memory_order_relaxed); + out->allocated_bytes = atomic_load_explicit(&g_mem_trace_allocated, + memory_order_relaxed); + out->freed_bytes = atomic_load_explicit(&g_mem_trace_freed, + memory_order_relaxed); + out->net_bytes = atomic_load_explicit(&g_mem_trace_live, + memory_order_relaxed); + out->peak_live_bytes = atomic_load_explicit(&g_mem_trace_peak, + memory_order_relaxed); + } + atomic_store_explicit(&g_mem_trace_active, 0, memory_order_release); +} + /* Anonymous (RAM-resident) pool + direct bytes we have committed. File-backed * spill mappings are NOT counted (they can be evicted to disk, never OOM-kill). * When a new anon mapping would push this past the watermark, it is routed to a @@ -1187,7 +1263,11 @@ ray_t* ray_alloc(size_t data_size) { * can still represent the request. */ if (RAY_UNLIKELY(order >= RAY_HEAP_POOL_ORDER)) { ray_t* v = heap_alloc_direct(h, data_size); - if (v) return v; + if (v) { + ray_direct_hdr_t* hdr = (ray_direct_hdr_t*)((char*)v - RAY_DIRECT_HDR); + mem_trace_note_alloc(hdr->map_size); + return v; + } if (order > RAY_HEAP_MAX_ORDER) return NULL; } @@ -1214,6 +1294,7 @@ ray_t* ray_alloc(size_t data_size) { RAY_STAT(h->stats.bytes_allocated += BSIZEOF(order)); RAY_STAT(h->stats.peak_bytes = h->stats.bytes_allocated > h->stats.peak_bytes ? h->stats.bytes_allocated : h->stats.peak_bytes); + mem_trace_note_alloc(BSIZEOF(order)); return v; } } @@ -1275,6 +1356,8 @@ ray_t* ray_alloc(size_t data_size) { RAY_STAT(h->stats.peak_bytes = h->stats.bytes_allocated > h->stats.peak_bytes ? h->stats.bytes_allocated : h->stats.peak_bytes); + mem_trace_note_alloc(BSIZEOF(order)); + return blk; } @@ -1344,6 +1427,7 @@ void ray_free(ray_t* v) { size_t map_size = hdr->map_size; int swap_fd = hdr->swap_fd; char* swap_path = hdr->swap_path; + mem_trace_note_free(map_size); if (h) RAY_STAT(h->stats.free_count++); atomic_fetch_sub_explicit(&g_direct_bytes, (int64_t)map_size, memory_order_relaxed); atomic_fetch_sub_explicit(&g_direct_count, 1, memory_order_relaxed); @@ -1368,6 +1452,8 @@ void ray_free(ray_t* v) { if (order < RAY_ORDER_MIN || order > RAY_HEAP_MAX_ORDER) return; + mem_trace_note_free(BSIZEOF(order)); + #if RAY_MEM_STATS size_t block_size = BSIZEOF(order); #endif diff --git a/src/mem/heap.h b/src/mem/heap.h index 557cc71f9..9d1e0f041 100644 --- a/src/mem/heap.h +++ b/src/mem/heap.h @@ -144,6 +144,18 @@ typedef struct { size_t sys_mapped_peak; /* file-backed mapping high-water mark */ } ray_mem_stats_t; +/* Query-scoped allocation trace. Unlike ray_mem_stats_t (a lifetime snapshot + * of one heap), this aggregates allocation/free activity from every thread + * while active. Only one scope may be active process-wide. */ +typedef struct { + uint64_t alloc_count; + uint64_t free_count; + uint64_t allocated_bytes; + uint64_t freed_bytes; + int64_t net_bytes; + uint64_t peak_live_bytes; +} ray_mem_trace_t; + /* ===== Forward Declarations (internal types) ===== */ typedef struct ray_heap ray_heap_t; @@ -162,6 +174,8 @@ void ray_heap_push_pending(ray_heap_t* heap); void ray_heap_drain_pending(void); uint8_t ray_order_for_size(size_t data_size); void ray_mem_stats(ray_mem_stats_t* out); +bool ray_mem_trace_begin(void); +void ray_mem_trace_end(ray_mem_trace_t* out); void ray_heap_gc(void); void ray_heap_release_pages(void); diff --git a/src/ops/exec.c b/src/ops/exec.c index 53a6631d8..cb0c57e85 100644 --- a/src/ops/exec.c +++ b/src/ops/exec.c @@ -30,6 +30,7 @@ #include "mem/heap.h" #include "mem/sys.h" #include "core/qstats.h" /* per-worker parallelism stats for profile spans */ +#include "core/qmeasure.h" /* shared worker aggregate */ #include "core/runtime.h" /* __VM — filter-compaction projection keep-set */ /* Global profiler instance (zero-initialized = inactive) */ @@ -1769,8 +1770,8 @@ ray_t* exec_node(ray_graph_t* g, ray_op_t* op) { int64_t cur = 0; ray_sys_get_stat(&cur, NULL); sp->sys_cur = cur; sp->qs_rows = ray_qstats_sum_rows(); - uint64_t sum = 0, mx = 0; uint32_t used = 0; - ray_qstats_agg(&used, &sum, &mx); + uint64_t sum = 0; uint32_t used = 0; + ray_query_workers_snapshot(&used, &sum); sp->qs_busy_ns = sum; } } @@ -1785,8 +1786,8 @@ ray_t* exec_node(ray_graph_t* g, ray_op_t* op) { int64_t cur = 0; ray_sys_get_stat(&cur, NULL); ep->sys_cur = cur; ep->qs_rows = ray_qstats_sum_rows(); - uint64_t sum = 0, mx = 0; uint32_t used = 0; - ray_qstats_agg(&used, &sum, &mx); + uint64_t sum = 0; uint32_t used = 0; + ray_query_workers_snapshot(&used, &sum); ep->qs_busy_ns = sum; ep->qs_workers = used; /* Result footprint — rows this operator produced. */ diff --git a/src/ops/system.c b/src/ops/system.c index 38905fef4..516724ac9 100644 --- a/src/ops/system.c +++ b/src/ops/system.c @@ -29,7 +29,10 @@ #include "ops/internal.h" /* ray_group_perpart_runs — (.sys.mem) counter */ #include "mem/heap.h" #include "mem/sys.h" +#include "core/block.h" +#include "ops/idxop.h" #include "core/profile.h" /* g_ray_profile — (.sys.prof) */ +#include "core/qmeasure.h" /* shared per-query measurement lifecycle */ #include "core/qlog.h" /* g_qlog — (.sys.querylog) */ #include "store/serde.h" #include "store/splay.h" @@ -479,9 +482,310 @@ ray_t* ray_meta_fn(ray_t* x) { return ray_dict_new(keys, vals); } -/* (.sys.gc) -- no-op garbage collection trigger, return 0. Variadic - * so the call site can be (.sys.gc) without the dummy-arg ceremony. */ -ray_t* ray_gc_fn(ray_t** args, int64_t n) { (void)args; (void)n; return ray_i64(0); } +/* Object-graph size ------------------------------------------------------- + * + * `.mem.objsize` reports the logical bytes retained by a Rayfall value: + * every reachable ray_t header and its live payload, with shared children + * counted once. It deliberately does not report buddy-block slack, global + * symbol dictionaries, or opaque native handles (graphs/HNSW/lazy plans). + * That makes the result stable across allocator tuning and useful as the + * Rayforce analogue of q's `.mem.objsize` for materialised query results. + * + * The traversal is iterative and pointer-deduplicated. Besides preventing + * double-counting shared table columns, the visited set makes it safe for any + * future compound object that introduces a cycle. */ +typedef struct { + ray_t** stack; + size_t stack_len; + size_t stack_cap; + ray_t** seen; + size_t seen_count; + size_t seen_cap; +} ray_objsize_walk_t; + +static uint64_t objsize_ptr_hash(const ray_t* p) { + uint64_t x = (uint64_t)(uintptr_t)p; + x ^= x >> 33; + x *= UINT64_C(0xff51afd7ed558ccd); + x ^= x >> 33; + x *= UINT64_C(0xc4ceb9fe1a85ec53); + return x ^ (x >> 33); +} + +static bool objsize_stack_push(ray_objsize_walk_t* w, ray_t* child) { + if (!child || RAY_IS_ERR(child)) return true; + if (w->stack_len == w->stack_cap) { + size_t cap = w->stack_cap ? w->stack_cap * 2 : 64; + if (cap < w->stack_cap || cap > SIZE_MAX / sizeof(ray_t*)) return false; + ray_t** grown = (ray_t**)ray_sys_realloc(w->stack, cap * sizeof(ray_t*)); + if (!grown) return false; + w->stack = grown; + w->stack_cap = cap; + } + w->stack[w->stack_len++] = child; + return true; +} + +static bool objsize_seen_grow(ray_objsize_walk_t* w) { + size_t cap = w->seen_cap ? w->seen_cap * 2 : 128; + if (cap < w->seen_cap || cap > SIZE_MAX / sizeof(ray_t*)) return false; + ray_t** grown = (ray_t**)ray_sys_alloc(cap * sizeof(ray_t*)); + if (!grown) return false; + memset(grown, 0, cap * sizeof(ray_t*)); + + if (w->seen) { + size_t mask = cap - 1; + for (size_t i = 0; i < w->seen_cap; i++) { + ray_t* p = w->seen[i]; + if (!p) continue; + size_t slot = (size_t)objsize_ptr_hash(p) & mask; + while (grown[slot]) slot = (slot + 1) & mask; + grown[slot] = p; + } + ray_sys_free(w->seen); + } + w->seen = grown; + w->seen_cap = cap; + return true; +} + +/* Returns true when p was newly inserted, false when already seen. OOM is + * reported separately through *ok so a duplicate is not confused with an + * allocation failure. */ +static bool objsize_seen_insert(ray_objsize_walk_t* w, ray_t* p, bool* ok) { + if (!w->seen_cap || (w->seen_count + 1) * 10 >= w->seen_cap * 7) { + if (!objsize_seen_grow(w)) { *ok = false; return false; } + } + size_t mask = w->seen_cap - 1; + size_t slot = (size_t)objsize_ptr_hash(p) & mask; + while (w->seen[slot]) { + if (w->seen[slot] == p) return false; + slot = (slot + 1) & mask; + } + w->seen[slot] = p; + w->seen_count++; + return true; +} + +static size_t objsize_shallow(ray_t* v) { + if (v->attrs & RAY_ATTR_SLICE) return sizeof(ray_t); + if (v->type == RAY_LAMBDA) + return sizeof(ray_t) + 7 * sizeof(ray_t*); + if (v->type == RAY_INDEX) + return sizeof(ray_t) + sizeof(ray_index_t); + if (RAY_IS_PARTED(v->type)) + return sizeof(ray_t) + (v->len > 0 ? (size_t)v->len * sizeof(ray_t*) : 0); + if (v->type == RAY_MAPCOMMON) + return sizeof(ray_t) + 2 * sizeof(ray_t*); + return ray_block_size(v); +} + +static bool objsize_push_index_children(ray_objsize_walk_t* w, ray_index_t* ix) { +#define OBJSIZE_PUSH(child) do { if (!objsize_stack_push(w, (child))) return false; } while (0) + switch ((ray_idx_kind_t)ix->kind) { + case RAY_IDX_HASH: + OBJSIZE_PUSH(ix->u.hash.table); OBJSIZE_PUSH(ix->u.hash.gkeys); + OBJSIZE_PUSH(ix->u.hash.offs); OBJSIZE_PUSH(ix->u.hash.rows); + break; + case RAY_IDX_SORT: OBJSIZE_PUSH(ix->u.sort.perm); break; + case RAY_IDX_BLOOM: OBJSIZE_PUSH(ix->u.bloom.bits); break; + case RAY_IDX_CHUNK_ZONE: + OBJSIZE_PUSH(ix->u.chunk_zone.mins); + OBJSIZE_PUSH(ix->u.chunk_zone.maxs); + OBJSIZE_PUSH(ix->u.chunk_zone.null_bits); + break; + case RAY_IDX_PART: + OBJSIZE_PUSH(ix->u.part.keys); OBJSIZE_PUSH(ix->u.part.starts); + OBJSIZE_PUSH(ix->u.part.lens); + break; + case RAY_IDX_DICT: + OBJSIZE_PUSH(ix->u.dict.codes); OBJSIZE_PUSH(ix->u.dict.first_occ); + break; + case RAY_IDX_ZONE: + case RAY_IDX_NONE: + break; + } +#undef OBJSIZE_PUSH + return true; +} + +static bool objsize_push_children(ray_objsize_walk_t* w, ray_t* v) { +#define OBJSIZE_PUSH(child) do { if (!objsize_stack_push(w, (child))) return false; } while (0) + if (v->type == RAY_LAMBDA) { + ray_t** slots = (ray_t**)ray_data(v); + for (int i = 0; i < 4; i++) OBJSIZE_PUSH(slots[i]); + OBJSIZE_PUSH(slots[5]); + OBJSIZE_PUSH(slots[6]); + return true; + } + if (ray_is_atom(v)) { + if (v->type == -RAY_GUID && v->obj) OBJSIZE_PUSH(v->obj); + if (v->type == -RAY_STR && + !((v->slen >= 1 && v->slen <= 7) || (v->slen == 0 && !v->obj))) + OBJSIZE_PUSH(v->obj); + return true; + } + if (v->attrs & RAY_ATTR_SLICE) { + OBJSIZE_PUSH(v->slice_parent); + return true; + } + if (v->type == RAY_INDEX) + return objsize_push_index_children(w, ray_index_payload(v)); + if (v->attrs & RAY_ATTR_HAS_INDEX) { + OBJSIZE_PUSH(v->index); + /* An indexed STR vector still owns its external character pool in + * aux bytes 8..15; the index pointer only occupies bytes 0..7. */ + if (v->type == RAY_STR && v->str_pool) + OBJSIZE_PUSH(v->str_pool); + return true; + } + if (v->type == RAY_STR && v->str_pool) + OBJSIZE_PUSH(v->str_pool); + if (RAY_IS_PARTED(v->type)) { + ray_t** segs = (ray_t**)ray_data(v); + for (int64_t i = 0; i < v->len; i++) OBJSIZE_PUSH(segs[i]); + return true; + } + if (v->type == RAY_MAPCOMMON || v->type == RAY_TABLE || v->type == RAY_DICT) { + ray_t** slots = (ray_t**)ray_data(v); + OBJSIZE_PUSH(slots[0]); + OBJSIZE_PUSH(slots[1]); + return true; + } + if (v->type == RAY_LIST) { + ray_t** elems = (ray_t**)ray_data(v); + for (int64_t i = 0; i < v->len; i++) OBJSIZE_PUSH(elems[i]); + } +#undef OBJSIZE_PUSH + return true; +} + +ray_t* ray_mem_objsize_fn(ray_t* root) { + if (!root) return ray_error("type", "objsize expects a value"); + ray_objsize_walk_t w = {0}; + if (!objsize_stack_push(&w, root)) return ray_error("oom", NULL); + + uint64_t total = 0; + bool ok = true; + while (w.stack_len && ok) { + ray_t* v = w.stack[--w.stack_len]; + if (!objsize_seen_insert(&w, v, &ok)) continue; + size_t shallow = objsize_shallow(v); + if (UINT64_MAX - total < shallow) { ok = false; break; } + total += shallow; + if (!objsize_push_children(&w, v)) ok = false; + } + ray_sys_free(w.stack); + ray_sys_free(w.seen); + if (!ok) return ray_error("oom", "objsize traversal overflow or allocation failure"); + if (total > INT64_MAX) return ray_error("limit", "objsize exceeds I64 range"); + return ray_i64((int64_t)total); +} + +/* (.mem.ts expr) — evaluate one expression under a process-wide allocation + * trace and return a named measurement dictionary. SPECIAL_FORM: expr is + * deliberately not pre-evaluated. Result sizing and dictionary construction + * happen after both clocks stop, so they do not contaminate the measured + * expression. */ +ray_t* ray_mem_ts_fn(ray_t** args, int64_t n) { + if (n != 1) + return ray_error("arity", ".mem.ts expects 1 argument, got %lld", + (long long)n); + if (!ray_mem_trace_begin()) + return ray_error("state", ".mem.ts measurement already active"); + + ray_query_measure_t measure; + ray_query_measure_begin(&measure); + ray_t* result = ray_eval(args[0]); + ray_query_metrics_t metrics; + ray_query_measure_end(&measure, &metrics); + + ray_mem_trace_t mt; + ray_mem_trace_end(&mt); + + if (!result) return ray_error("state", ".mem.ts expression returned no value"); + if (RAY_IS_ERR(result)) return result; + + ray_t* result_size = ray_mem_objsize_fn(result); + if (!result_size || RAY_IS_ERR(result_size)) { + ray_release(result); + return result_size ? result_size : ray_error("oom", NULL); + } + + enum { MEM_TS_FIELDS = 13 }; + static const char* names[MEM_TS_FIELDS] = { + "result", "time-ns", "memory-bytes", "allocated-bytes", "freed-bytes", + "net-bytes", "peak-live-bytes", "result-bytes", "alloc-count", + "free-count", "workers", "worker-busy-ns", "parallelism" + }; + static const uint8_t name_lens[MEM_TS_FIELDS] = { + 6, 7, 12, 15, 11, 9, 15, 12, 11, 10, 7, 14, 11 + }; + + ray_t* items[MEM_TS_FIELDS] = { + result, + ray_i64(metrics.time_ns), + ray_i64(metrics.memory_bytes), + ray_i64((int64_t)mt.allocated_bytes), + ray_i64((int64_t)mt.freed_bytes), + ray_i64(mt.net_bytes), + ray_i64((int64_t)mt.peak_live_bytes), + result_size, + ray_i64((int64_t)mt.alloc_count), + ray_i64((int64_t)mt.free_count), + ray_i64((int64_t)metrics.workers), + ray_i64((int64_t)metrics.worker_busy_ns), + ray_f64(metrics.parallelism) + }; + for (int i = 0; i < MEM_TS_FIELDS; i++) { + if (!items[i] || RAY_IS_ERR(items[i])) { + ray_t* err = items[i] ? items[i] : ray_error("oom", NULL); + for (int j = 0; j < MEM_TS_FIELDS; j++) + if (j != i && items[j] && !RAY_IS_ERR(items[j])) ray_release(items[j]); + return err; + } + } + + ray_t* keys = ray_sym_vec_new(RAY_SYM_W64, MEM_TS_FIELDS); + ray_t* vals = ray_list_new(MEM_TS_FIELDS); + if (!keys || RAY_IS_ERR(keys) || !vals || RAY_IS_ERR(vals)) { + ray_t* err = (!keys || RAY_IS_ERR(keys)) + ? (keys ? keys : ray_error("oom", NULL)) + : vals; + if (keys && !RAY_IS_ERR(keys)) ray_release(keys); + if (vals && !RAY_IS_ERR(vals)) ray_release(vals); + for (int i = 0; i < MEM_TS_FIELDS; i++) ray_release(items[i]); + return err; + } + + for (int i = 0; i < MEM_TS_FIELDS; i++) { + int64_t sid = ray_sym_intern(names[i], name_lens[i]); + keys = ray_vec_append(keys, &sid); + if (RAY_IS_ERR(keys)) { + ray_release(vals); + for (int j = i; j < MEM_TS_FIELDS; j++) ray_release(items[j]); + return keys; + } + vals = ray_list_append(vals, items[i]); + ray_release(items[i]); + if (RAY_IS_ERR(vals)) { + ray_release(keys); + for (int j = i + 1; j < MEM_TS_FIELDS; j++) ray_release(items[j]); + return vals; + } + } + return ray_dict_new(keys, vals); +} + +/* (.sys.gc) -- perform the allocator maintenance pass used by v1: drain + * foreign frees, flush slab caches, coalesce blocks and release reclaimable + * pages/pools. Rayforce values are reference-counted, so this is allocator + * GC rather than a tracing collector. Variadic to allow `(.sys.gc)`. */ +ray_t* ray_gc_fn(ray_t** args, int64_t n) { + (void)args; (void)n; + ray_heap_gc(); + return ray_i64(0); +} /* (system cmd) -- run shell command, return exit code */ ray_t* ray_system_fn(ray_t* x) { diff --git a/test/rfl/system/mem_ts.rfl b/test/rfl/system/mem_ts.rfl new file mode 100644 index 000000000..ae1208514 --- /dev/null +++ b/test/rfl/system/mem_ts.rfl @@ -0,0 +1,29 @@ +;; .mem.ts evaluates exactly once and returns named, query-scoped statistics. +(set _mts (.mem.ts (til 10000))) +(count (at _mts 'result)) -- 10000 +(> (at _mts 'time-ns) 0) -- true +(>= (at _mts 'memory-bytes) 0) -- true +(> (at _mts 'allocated-bytes) 0) -- true +(>= (at _mts 'allocated-bytes) (at _mts 'peak-live-bytes)) -- true +(> (at _mts 'peak-live-bytes) 0) -- true +(> (at _mts 'result-bytes) 0) -- true +(> (at _mts 'alloc-count) 0) -- true +(>= (at _mts 'free-count) 0) -- true +(>= (at _mts 'workers) 0) -- true +(>= (at _mts 'worker-busy-ns) 0) -- true +(>= (at _mts 'parallelism) 0.0) -- true + +;; The result sizing is performed after the trace and agrees with the public +;; value-level operation. +(== (at _mts 'result-bytes) (.mem.objsize (at _mts 'result))) -- true + +;; A parallel aggregation must report allocation churn and return its value. +(set _mtsp (.mem.ts (sum (til 1000000)))) +(at _mtsp 'result) -- 499999500000 +(> (at _mtsp 'allocated-bytes) 0) -- true +(> (at _mtsp 'workers) 0) -- true +(> (at _mtsp 'worker-busy-ns) 0) -- true + +;; Measurement scopes are process-global and cannot nest. +(try (.mem.ts (.mem.ts (til 10))) (fn [e] 'nested-rejected)) -- 'nested-rejected +(try (.mem.ts) (fn [e] 'arity-rejected)) -- 'arity-rejected diff --git a/test/rfl/system/objsize.rfl b/test/rfl/system/objsize.rfl new file mode 100644 index 000000000..f5f3c93f1 --- /dev/null +++ b/test/rfl/system/objsize.rfl @@ -0,0 +1,15 @@ +;; .mem.objsize reports logical retained bytes, not serialized size or buddy +;; allocator capacity. Every ray_t contributes its 32-byte header. + +(.mem.objsize 42) -- 32 +(.mem.objsize [1 2 3]) -- 56 + +;; Shared children are counted once: LIST header + two pointer slots + vector. +(set _objsize_shared [1 2 3]) +(.mem.objsize (list _objsize_shared _objsize_shared)) -- 104 + +;; A backing index and its child vectors are part of the retained graph. +(> (.mem.objsize (.attr.set 'parted [1 1 2 2])) (.mem.objsize [1 1 2 2])) -- true + +;; The allocator-maintenance builtin remains callable from Rayfall. +(.sys.gc) -- 0 From ab4e864f8cd0fefecf74fce45fd8a047f43cf528 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 10:34:37 +0200 Subject: [PATCH 03/10] perf(query): accelerate indexed filters and parted groups --- docs/docs/language/functions.md | 2 +- docs/docs/language/math.md | 2 + docs/docs/queries/attributes.md | 16 +- docs/docs/queries/indexes.md | 5 + docs/docs/queries/pivot.md | 15 + docs/docs/reference/all-functions.md | 3 +- src/ops/collection.c | 21 +- src/ops/exec.c | 87 ++++- src/ops/group.c | 21 +- src/ops/idxop.c | 363 ++++++++++++++++++++- src/ops/idxop.h | 51 ++- src/ops/ops.h | 6 +- src/ops/query.c | 55 +++- src/ops/rowsel.c | 103 ++++++ src/ops/rowsel.h | 5 + src/ops/window.c | 300 ++++++++--------- src/vec/vec.c | 3 + test/rfl/collection/time_series_prefix.rfl | 11 + test/rfl/integration/slice_group.rfl | 6 + test/rfl/query/window.rfl | 12 + test/test_idx_route.c | 294 +++++++++++++++++ test/test_window.c | 40 ++- 22 files changed, 1162 insertions(+), 259 deletions(-) diff --git a/docs/docs/language/functions.md b/docs/docs/language/functions.md index 7602735b7..a35f3ef99 100644 --- a/docs/docs/language/functions.md +++ b/docs/docs/language/functions.md @@ -165,7 +165,7 @@ Operations on vectors as collections. | `lead` | unary | Shift values one row forward; last row is null/sentinel | `(lead [10 20 30])` → `[20 30 0Nl]` | | `deltas` | unary | Adjacent differences; first row is null | `(deltas [10 15 13])` → `[0Nl 5 -2]` | | `ratios` | unary | Adjacent ratios as f64; first row is null | `(ratios [2 4 8])` → `[0Nf 2.0 2.0]` | -| `fills` | unary | Forward-fill nullable vectors | `(fills (as 'I64 (list 0N 2 0N)))` → `[0Nl 2 2]` | +| `fills` | unary | Forward-fill nullable vectors, or every column of a table | `(fills (as 'I64 (list 0N 2 0N)))` → `[0Nl 2 2]` | | `sums` | unary | Running sum; nulls are skipped | `(sums [1 2 3])` → `[1 3 6]` | | `avgs` | unary | Running average over non-null values | `(avgs [2 4 6])` → `[2.0 3.0 4.0]` | | `mins` | unary | Running minimum | `(mins [3 1 2])` → `[3 1 1]` | diff --git a/docs/docs/language/math.md b/docs/docs/language/math.md index 0828204bf..8136f7edc 100644 --- a/docs/docs/language/math.md +++ b/docs/docs/language/math.md @@ -578,6 +578,8 @@ For common time-series scans, prefer the built-in DAG forms. They avoid per-row (mins [3 1 2]) ; [3 1 1] (maxs [3 1 2]) ; [3 3 3] (fills (as 'I64 (list 0N 2 0N))) ; [0Nl 2 2] +(fills (table [t value] + (list [1 2 3] (as 'I64 (list 10 0N 30))))) ; fills each column (differ [1 1 2 2]) ; [true false true false] (msum 3 [1 2 3 4]) ; [1 3 6 9] (mavg 3 [1 2 3 4]) ; [1.0 1.5 2.0 3.0] diff --git a/docs/docs/queries/attributes.md b/docs/docs/queries/attributes.md index 63af3eff8..fa6f0290b 100644 --- a/docs/docs/queries/attributes.md +++ b/docs/docs/queries/attributes.md @@ -1,15 +1,15 @@ # Column Attributes -Semantic properties stamped onto numeric columns — `sorted`, `unique`, `grouped`, `parted`. Where an [accelerator index](indexes.md) is a *physical* structure (a hash table, a permutation), a column attribute is an *assertion about meaning*: this column is non-descending, these values are distinct. The `.attr.*` family layers over the same storage as `.idx.*`, but its contract is semantic, not structural. +Semantic properties stamped onto columns — `sorted`, `unique`, `grouped`, `parted`. Where an [accelerator index](indexes.md) is a *physical* structure (a hash table, a permutation), a column attribute is an *assertion about meaning*: this column is non-descending, these values are distinct. The `.attr.*` family layers over the same storage as `.idx.*`, but its contract is semantic, not structural. !!! note "v1 status" - The four attributes, their strict verify-on-set kernels, and the `(.attr.*)` Rayfall surface are shipped, along with the as-of-join fast paths that consume them. All attributes are **numeric-only** in v1 — the same numeric type set as `.idx.*`; symbol / string columns are deferred. + The four attributes, their strict verify-on-set kernels, and the `(.attr.*)` Rayfall surface are shipped. `sorted` and `unique` are numeric; the backing-index attributes `grouped` and `parted` also support symbol columns. ## Why Attributes Are Separate from Indexes An [accelerator index](indexes.md) answers *"what structure is attached?"* — a zonemap, a hash table, a sort permutation. A column attribute answers a different question: *"what is semantically true of these values?"* The two markers (`sorted`, `unique`) carry no allocation at all — they are cheap flags. The two backing-index attributes (`grouped`, `parted`) do reuse the index layer underneath (so a `grouped` or `parted` column also shows up via `.idx.has?` and `.idx.info`), but `.attr.*` exists to assert a *property* the engine can trust, not merely to build a structure. -The payoff is that a consumer — most importantly the [as-of join executor](joins.md#pre-sorted-fast-path) — can trust a stamped attribute unconditionally and skip work it would otherwise have to do defensively. +The payoff is that consumers can trust a stamped attribute unconditionally. Besides the [as-of join executor](joins.md#pre-sorted-fast-path), filters and grouped aggregation use the physical layout directly. ## The Four Attributes @@ -22,7 +22,9 @@ The payoff is that a consumer — most importantly the [as-of join executor](joi The two **markers** are independent and may coexist. A column holds at most **one backing index** at a time: setting `grouped` or `parted` replaces any prior backing index. Markers may coexist with a backing index. -`parted` is **verify-only** — it never reorders data. It checks that the column is already block-laid-out (each distinct value occupies one contiguous, ascending run) and errors if it is not. The caller is responsible for ordering the column first. +`parted` is **verify-only** — it never reorders data. It checks that the column is already block-laid-out (each distinct value occupies one contiguous run; numeric keys must also be ascending) and errors if it is not. The caller is responsible for ordering the column first. + +For `where: (in key set)` on a parted key, the filter builds its row selection from the matching `[start, len)` ranges without scanning the key column or expanding full morsels into row ids. If the same query groups by that key and uses supported aggregates, Rayforce skips the filter and group discovery entirely and streams each selected partition directly into the aggregate kernel. The equivalent `grouped` path consumes hash-index row slices. ## Setting, Reading, and Dropping @@ -30,7 +32,7 @@ The two **markers** are independent and may coexist. A column holds at most **o `(.attr.set 'name v) (.attr.get v) (.attr.drop v)` -- `(.attr.set 'name v)` — assert attribute `name` on numeric vector `v`. Strict verify: it scans `v` and **errors on violation**, so a stamped attribute never lies. Returns the column with the attribute recorded. +- `(.attr.set 'name v)` — assert attribute `name` on vector `v`. Strict verification **errors on violation**, so a stamped attribute never lies. Returns the column with the attribute recorded. - `(.attr.get v)` — return the symbol vector of currently-held attributes (the empty symbol vector when none are set). - `(.attr.drop v)` — remove all attributes from `v`. @@ -85,7 +87,7 @@ An unknown attribute name is rejected the same way: (.attr.set 'bogus [1 2 3]) ; ⇒ error (domain): unknown attribute ``` -Setting `sorted` runs an O(n) ascending scan; `unique` and `grouped` build / consult the hash to detect duplicates; `parted` walks the column verifying that each distinct value forms one contiguous ascending block. +Setting `sorted` runs an O(n) ascending scan; `unique` and `grouped` build / consult the hash to detect duplicates; `parted` walks the column verifying that each distinct value forms one contiguous block (and ascending block order for numeric keys). ## Combining Attributes @@ -160,7 +162,7 @@ For a partitioned join `(asof-join [Key Time] L R)`, if the single numeric equal ## Caveats and Limits -- **Numeric types only (v1).** Attributes accept the same numeric type set as `.idx.*` (boolean, integer, float, and temporal types). Symbol / string columns are deferred — including the `parted` equality key in a partitioned as-of join, which must be a numeric ID. +- **Type support.** `sorted` and `unique` accept numeric types. `grouped` and `parted` also accept `SYM`; `STR` remains unsupported. - **Strict verify, no silent stamping.** `.attr.set` errors on violation rather than recording a property it cannot confirm; this is what lets consumers trust the stamp unconditionally. - **Conservative propagation.** Only copy / rebind preserve attributes; every transform drops them. Re-assert with `.attr.set` after a transform. - **One backing index per column.** `grouped` and `parted` share the [accelerator-index](indexes.md) slot, so a column carries at most one of them at a time; the markers are separate and free. diff --git a/docs/docs/queries/indexes.md b/docs/docs/queries/indexes.md index 4a8a35f18..8a0f97bb1 100644 --- a/docs/docs/queries/indexes.md +++ b/docs/docs/queries/indexes.md @@ -16,8 +16,11 @@ The executor consults indexes at five sites. Each site has a specific consumpti | `.idx.zone` | `filter` — comparison predicates (EQ/NE/LT/LE/GT/GE) | O(1) all/none short-circuit: if the constant falls outside the column's [min, max] range the filter returns 0 rows without scanning; if every row must pass the whole table is returned. Integer and float families both supported. | | `.idx.bloom` | `filter` — EQ on integer-family columns | Definite-absent proof: if all k probe bits are clear the key is absent and the filter returns 0 rows without scanning. False positives fall through to the scan; there are no false absences. | | `.idx.hash` | `filter` — EQ on integer-family columns | Direct rowsel from hash-chain probe: matching rows are collected in O(matches) without a full column scan. Null-free columns only (HAS_NULLS → scan). | +| `part` backing (`.attr.set 'parted`) | `filter` — EQ predicates | Locate the value block and build the rowsel directly from its contiguous `[start, len)` span, including dense blocks. | | `.idx.sort` | `filter` — range predicates (EQ/LT/LE/GT/GE) | Binary search over the row-id permutation to identify the matching span, then a rowsel is built from that span. NE (two disjoint spans) falls back. A selectivity guard fires when the span exceeds n/128 (~0.78%) — see [Performance Characteristics](#performance-characteristics) for the measured rationale. | | `.idx.hash` | `filter` — IN predicates on integer-family columns | Hash-chain probe for each set element; result is the union rowsel over all matches. | +| `part` backing (`.attr.set 'parted`) | `filter` — IN predicates | Match partition keys once and build the rowsel from their contiguous physical spans. | +| `.idx.hash` / `part` backing | `where key IN …` + `group by key` | Resolve only the requested key slices and aggregate them directly, skipping filter scan and group discovery. Part slices stream as contiguous ranges. | | `.idx.sort` | ORDER BY — single ascending key | The pre-built permutation is reused directly rather than re-sorting. Descending ORDER BY falls back to recompute (reversing the perm would swap tie-group positions relative to stable DESC sort). | | `.idx.sort` | `distinct` | Walk the sort permutation once to collect first-occurrence row ids per distinct value — O(n) with no hash table. SYM/GUID/STR columns fall back. | | `.idx.hash` | `find` | Hash-chain probe returns the minimum row id matching the needle, or a provably-absent signal. Integer-family needles only; float and cross-family needles fall through to the scan. | @@ -71,8 +74,10 @@ Set `RAY_IDX_STATS=1` before running a process. At exit, per-site consult and h idx_route filter_zone consults=3 hits=2 idx_route filter_bloom consults=1 hits=1 idx_route filter_hash consults=5 hits=5 +idx_route filter_part consults=3 hits=3 idx_route filter_range consults=4 hits=3 idx_route in consults=2 hits=2 +idx_route group-slice consults=2 hits=2 idx_route find consults=8 hits=7 idx_route sort consults=6 hits=6 idx_route distinct consults=2 hits=2 diff --git a/docs/docs/queries/pivot.md b/docs/docs/queries/pivot.md index 909e70c68..99ecb1558 100644 --- a/docs/docs/queries/pivot.md +++ b/docs/docs/queries/pivot.md @@ -147,6 +147,21 @@ Running computations like cumulative sums can be computed using the DAG-backed t Use `scan` when the accumulator is a custom function. Use `sums`, `avgs`, `mins`, `maxs`, and `prds` for built-in running aggregates, and `msum`, `mavg`, `mmin`, `mmax`, `mcount`, `mvar`, and `mdev` for fixed trailing windows. Constant-window calls are lazy-aware DAG operations and materialize with morsel-based kernels. +When the calculation must restart for each key, use the partitioned `window` +form. A positive `frame:` value is the trailing row count, including the +current row: + +```lisp +(window {from: trades + part: [Sym] + order: [Time] + frame: 3 + funcs: {avg3: (avg Price)}}) +``` + +Use `frame: 'running` for an unbounded cumulative frame and omit `frame:` (or +use `frame: 'whole`) to aggregate over the whole partition. + ### Rank and Order ```lisp diff --git a/docs/docs/reference/all-functions.md b/docs/docs/reference/all-functions.md index bbff65b93..20fab1022 100644 --- a/docs/docs/reference/all-functions.md +++ b/docs/docs/reference/all-functions.md @@ -266,7 +266,7 @@ Operations on vectors and lists as collections — set operations, indexing, sea | `lead` | unary | lazy/DAG | Shift values one row forward; last row is null/sentinel | `(lead [10 20 30])` → `[20 30 0Nl]` | | `deltas` | unary | lazy/DAG | Adjacent differences; first row is null | `(deltas [10 15 13])` → `[0Nl 5 -2]` | | `ratios` | unary | lazy/DAG | Adjacent ratios as f64; first row is null | `(ratios [2 4 8])` → `[0Nf 2.0 2.0]` | -| `fills` | unary | lazy/DAG | Forward-fill nullable vectors | `(fills (as 'I64 (list 0N 2 0N)))` → `[0Nl 2 2]` | +| `fills` | unary | lazy/DAG | Forward-fill nullable vectors, or every column of a table | `(fills (as 'I64 (list 0N 2 0N)))` → `[0Nl 2 2]` | | `sums` | unary | lazy/DAG | Running sum; nulls are skipped | `(sums [1 2 3])` → `[1 3 6]` | | `avgs` | unary | lazy/DAG | Running average over non-null values | `(avgs [2 4 6])` → `[2.0 3.0 4.0]` | | `mins` | unary | lazy/DAG | Running minimum | `(mins [3 1 2])` → `[3 1 1]` | @@ -442,6 +442,7 @@ Special forms that bridge to the Rayforce DAG executor for high-performance colu | Function | Type | Flags | Description | Example | |---|---|---|---|---| | `select` | variadic | special | Query table with optional filter, projection, grouping, and aggregation | `(select {from: t a: a})` | +| `window` | variadic | special | Partitioned window query. `frame:` is `'whole`, `'running`, or a positive trailing row count | `(window {from: t part: [sym] order: [time] frame: 5 funcs: {avg5: (avg price)}})` | | `update` | variadic | special, restricted | Add or modify columns in a table (mutates in-place) | `(update {from: t b: (* a 2)})` | | `insert` | variadic | special, restricted | Insert rows into a table | `(insert t {x: 10 y: 20})` | | `upsert` | variadic | special, restricted | Insert or update rows by key match (target, key, row) | `(upsert t 'x {x: 10 y: 20})` | diff --git a/src/ops/collection.c b/src/ops/collection.c index 4122660ab..056f7dec7 100644 --- a/src/ops/collection.c +++ b/src/ops/collection.c @@ -3851,7 +3851,26 @@ ray_t* ray_ratios_fn(ray_t* x) { ray_t* ray_fills_fn(ray_t* x) { if (ray_is_lazy(x)) return ray_lazy_append(x, OP_FILLS); if (ray_is_vec(x)) return ts_start_lazy_unary(x, ray_fills_op); - return ray_error("type", "fills expects a vector input, got %s", ray_type_name(x ? x->type : 0)); + if (x && x->type == RAY_TABLE) { + int64_t ncols = ray_table_ncols(x); + ray_t* out = ray_table_new(ncols); + if (!out || RAY_IS_ERR(out)) return out ? out : ray_error("oom", NULL); + + for (int64_t c = 0; c < ncols; c++) { + ray_t* col = ray_table_get_col_idx(x, c); + ray_t* filled = fills_vec_eager(col); + if (!filled || RAY_IS_ERR(filled)) { + ray_release(out); + return filled ? filled : ray_error("oom", NULL); + } + + out = ray_table_add_col(out, ray_table_col_name(x, c), filled); + ray_release(filled); + if (!out || RAY_IS_ERR(out)) return out ? out : ray_error("oom", NULL); + } + return out; + } + return ray_error("type", "fills expects a vector or table input, got %s", ray_type_name(x ? x->type : 0)); } ray_t* ray_sums_fn(ray_t* x) { diff --git a/src/ops/exec.c b/src/ops/exec.c index cb0c57e85..15274d4e4 100644 --- a/src/ops/exec.c +++ b/src/ops/exec.c @@ -1192,8 +1192,8 @@ static int idx_filter_decode(ray_graph_t* g, ray_op_t* pred_op, * WHERE filter, after ray_optimize. When the compiled filter's predicate * is exactly membership on the single bare group-key column — (in c SET) * or (== c K), no other conjuncts, no chained filter — and that column - * carries a fresh CSR hash index, the surviving groups are exactly the - * key set and each group's rows are its full CSR slice: resolve the + * carries a fresh hash or part index, the surviving groups are exactly the + * key set and each group's rows are its full index slice: resolve the * slices NOW (freshness verified inside the resolver) and arm the hint * on g. exec_group consumes it — aggregating the slices directly, or * folding them into the selection the skipped filter would have @@ -1207,7 +1207,8 @@ bool ray_slice_group_probe(ray_graph_t* g, ray_op_t* root, ray_t* by_col) { if (!in0 || in0->opcode == OP_FILTER) return false; if (by_col->type != RAY_SYM || ray_is_atom(by_col)) return false; if (by_col->attrs & RAY_ATTR_HAS_NULLS) return false; - if (ray_index_kind(by_col) != RAY_IDX_HASH) return false; + ray_idx_kind_t kind = ray_index_kind(by_col); + if (kind != RAY_IDX_HASH && kind != RAY_IDX_PART) return false; ray_op_t* pred = op_child(g, root, 1); if (!pred) return false; @@ -1236,7 +1237,7 @@ bool ray_slice_group_probe(ray_graph_t* g, ray_op_t* root, ray_t* by_col) { ray_idx_consults[IDX_SITE_GROUP_SLICE]++; ray_idx_slice_t* sl = NULL; ray_t* shdr = NULL; - int64_t k = ray_index_hash_sym_slices(by_col, keys, &sl, &shdr); + int64_t k = ray_index_sym_slices(by_col, keys, &sl, &shdr); if (eq_atom) ray_release(eq_atom); if (k < 0) return false; ray_idx_hits[IDX_SITE_GROUP_SLICE]++; @@ -2072,6 +2073,41 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { ray_t* input = exec_node(g, op_child(g, op, 0)); if (!input || RAY_IS_ERR(input)) return input; + /* A verified physically-sorted column remains routable even when + * this is the outer half of a split AND predicate. Intersect the + * new contiguous range rowsel with the existing selection instead + * of evaluating the second bound over surviving rows. */ + if (input->type == RAY_TABLE) { + ray_t* scol = NULL; + uint16_t sop = 0; + int64_t ski = 0; + double skf = 0.0; + int sisf = 0; + if (idx_filter_decode(g, op_child(g, op, 1), &scol, &sop, + &ski, &skf, &sisf) && + sop != OP_NE && ray_attr_is_sorted(scol)) { + ray_idx_consults[IDX_SITE_FILTER_RANGE]++; + ray_t* ssel = ray_sorted_range_rowsel(scol, sop, ski, skf, + (bool)sisf); + if (ssel) { + if (g->selection) { + ray_t* merged = ray_rowsel_intersect(g->selection, + ssel); + ray_rowsel_release(ssel); + if (merged) { + ray_rowsel_release(g->selection); + g->selection = merged; + ray_idx_hits[IDX_SITE_FILTER_RANGE]++; + return input; + } + } else { + g->selection = ssel; + ray_idx_hits[IDX_SITE_FILTER_RANGE]++; + return input; + } + } + } + } /* Hash-index point-lookup fast path: when the predicate is * `col == K` on a column with RAY_IDX_HASH attached and * built for the column's current length, install the @@ -2124,7 +2160,9 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } } - /* 3. Hash-eq: integer or SYM EQ; re-check HAS_NULLS. */ + /* 3. Hash/part EQ: integer or SYM; re-check HAS_NULLS. + * A part hit is already one contiguous physical span, so + * it stays eligible at any density. */ if (cmp_op == OP_EQ && !is_float && !(col->attrs & RAY_ATTR_HAS_NULLS)) { /* SYM equality: idx_filter_decode passes the global intern id; @@ -2153,21 +2191,33 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } } if (!sym_probe_skip) { - if (ray_index_kind(col) == RAY_IDX_HASH) - ray_idx_consults[IDX_SITE_FILTER_HASH]++; - ray_t* sel = ray_index_hash_eq_rowsel(col, key_i); - if (sel) { - ray_idx_hits[IDX_SITE_FILTER_HASH]++; - g->selection = sel; - return input; + ray_idx_kind_t eq_kind = ray_index_kind(col); + if (eq_kind == RAY_IDX_HASH || + eq_kind == RAY_IDX_PART) { + idx_site_t site = eq_kind == RAY_IDX_HASH + ? IDX_SITE_FILTER_HASH + : IDX_SITE_FILTER_PART; + ray_idx_consults[site]++; + ray_t* sel = eq_kind == RAY_IDX_HASH + ? ray_index_hash_eq_rowsel(col, key_i) + : ray_index_part_eq_rowsel(col, key_i); + if (sel) { + ray_idx_hits[site]++; + g->selection = sel; + return input; + } } /* sel == NULL: ineligible (wrong/stale index kind, * key out of range) or OOM — fall through to scan. */ } } - /* 4. Sort-index range: EQ/LT/LE/GT/GE (NE returns NULL). */ + /* 4. Sort-index range: EQ/LT/LE/GT/GE (NE returns NULL). + * The physical-sorted route runs before this block so it + * can also intersect an existing chained selection. */ if (cmp_op != OP_NE) { + /* Shuffled physical rows need the + * permutation probe and its sparse-span guard. */ if (ray_index_kind(col) == RAY_IDX_SORT) { ray_idx_consults[IDX_SITE_FILTER_RANGE]++; ray_t* sel = ray_index_range_rowsel(col, cmp_op, @@ -2290,17 +2340,18 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } } - /* 5. Hash-index IN probe: FILTER(IN(SCAN col, CONST set_vec)) - * on an integer-family column with a hash index. Probes the - * hash chain for each unique set element; builds one rowsel - * over the union of matches. */ + /* 5. Hash/part-index IN probe: + * FILTER(IN(SCAN col, CONST set_vec)). Hash indexes probe + * each unique set element; part indexes select their + * contiguous physical spans directly. */ { ray_t* in_col = NULL; ray_t* in_set_lit = NULL; if (idx_filter_in_decode(g, op_child(g, op, 1), &in_col, &in_set_lit)) { bool in_col_is_float = (in_col->type == RAY_F32 || in_col->type == RAY_F64); - if (ray_index_kind(in_col) == RAY_IDX_HASH && + ray_idx_kind_t in_kind = ray_index_kind(in_col); + if ((in_kind == RAY_IDX_HASH || in_kind == RAY_IDX_PART) && !in_col_is_float) { ray_idx_consults[IDX_SITE_IN]++; ray_t* sel = ray_index_in_rowsel(in_col, in_set_lit); diff --git a/src/ops/group.c b/src/ops/group.c index 42a83a593..5c8bb310e 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -7370,9 +7370,10 @@ ray_t* exec_group(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) * ============================================================================ * * The probe (exec.c ray_slice_group_probe) resolved the WHERE key set - * against the group-key column's CSR hash index: g->sg_slices_hdr holds - * one (domain-id, rows, n) slice per surviving key, ascending by domain - * id — the DA path's emit order. Aggregate each slice directly: the + * against the group-key column's hash or part index: g->sg_slices_hdr holds + * one slice per surviving key, ascending by domain id — the DA path's emit + * order. Hash slices borrow row-id arrays; part slices carry a contiguous + * physical range. Aggregate each slice directly: the * filter scan, the survivor gather and group discovery all vanish; per * group the accumulation applies the same all_sum recurrence da_accum_row * applies row-by-row (same read helpers, same in-order accumulate), so @@ -7525,12 +7526,12 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { for (int64_t ti = tstart; ti < tend; ti++) { const sg_task_t* tk = &c->tasks[ti]; const ray_idx_slice_t* sl = &c->slices[tk->gi]; - const int64_t* restrict rows = sl->rows + tk->lo; + const int64_t* restrict rows = sl->rows ? sl->rows + tk->lo : NULL; int64_t n = tk->hi - tk->lo; /* Parted layout: each key's rows form one contiguous run — the * accumulate then streams raw column pointers and vectorizes. */ - bool contig = (n > 0 && rows[n - 1] - rows[0] + 1 == n); - int64_t r0 = (n > 0) ? rows[0] : 0; + bool contig = (n > 0 && (!rows || rows[n - 1] - rows[0] + 1 == n)); + int64_t r0 = n > 0 ? (rows ? rows[0] : sl->first + tk->lo) : 0; for (uint32_t a = 0; a < c->n_aggs; a++) { size_t idx = (size_t)ti * c->n_aggs + a; uint16_t op = c->agg_ops[a]; @@ -7667,7 +7668,13 @@ static ray_t* sg_hint_to_selection(ray_graph_t* g, ray_t* tbl) { int64_t* ids = (int64_t*)ray_data(hdr); int64_t w = 0; for (int64_t i = 0; i < K; i++) { - memcpy(ids + w, sl[i].rows, (size_t)sl[i].n * sizeof(int64_t)); + if (sl[i].rows) { + memcpy(ids + w, sl[i].rows, + (size_t)sl[i].n * sizeof(int64_t)); + } else { + for (int64_t j = 0; j < sl[i].n; j++) + ids[w + j] = sl[i].first + j; + } w += sl[i].n; } /* Slices are disjoint and each ascending; a plain sort restores diff --git a/src/ops/idxop.c b/src/ops/idxop.c index ef60aa774..130769648 100644 --- a/src/ops/idxop.c +++ b/src/ops/idxop.c @@ -42,9 +42,9 @@ uint64_t ray_idx_hits[IDX_SITE__N]; static void idx_stats_dump(void) { static const char* names[IDX_SITE__N] = { - "filter-zone", "filter-bloom", "filter-hash", "filter-range", - "in", "find", "sort", "distinct", "asof", "filter-eq-range", - "group-slice", + "filter-zone", "filter-bloom", "filter-hash", "filter-part", + "filter-range", "in", "find", "sort", "distinct", "asof", + "filter-eq-range", "group-slice", }; for (int i = 0; i < IDX_SITE__N; i++) if (ray_idx_consults[i] || ray_idx_hits[i]) @@ -1339,7 +1339,7 @@ ray_t* ray_index_hash_eq_rowsel(ray_t* col, int64_t key) { } /* -------------------------------------------------------------------------- - * Hash-index IN probe + * Hash/part-index IN probe * * For each unique in-range element of set_vec, walk the hash chain and * collect matching row ids into a single shared buffer, then build a @@ -1366,9 +1366,163 @@ static int cmp_i64_plain(const void* a, const void* b) { return (x > y) - (x < y); } +typedef struct { + int64_t start; + int64_t len; +} idx_span_t; + +static bool sorted_i64_contains(const int64_t* values, int64_t n, + int64_t needle) { + int64_t lo = 0, hi = n; + while (lo < hi) { + int64_t mid = lo + (hi - lo) / 2; + if (values[mid] < needle) lo = mid + 1; + else hi = mid; + } + return lo < n && values[lo] == needle; +} + +/* Build a rowsel from sorted, disjoint physical-row spans. Part indexes + * already describe matches this way, so keep full morsels as ALL and store + * row ids only for partial morsels instead of expanding every matching row. */ +static ray_t* rowsel_from_sorted_spans(int64_t n, const idx_span_t* spans, + int64_t nspans, int64_t total) { + if (n < 0 || nspans < 0 || total < 0 || total > n) return NULL; + if (nspans == 0) + return rowsel_from_sorted_ids(n, NULL, 0); + + uint32_t nsegs = n > 0 + ? (uint32_t)((n + RAY_MORSEL_ELEMS - 1) / RAY_MORSEL_ELEMS) + : 0; + ray_t* pop_hdr = ray_alloc((int64_t)nsegs * (int64_t)sizeof(uint32_t)); + if (!pop_hdr) return NULL; + uint32_t* pop = (uint32_t*)ray_data(pop_hdr); + memset(pop, 0, (size_t)nsegs * sizeof(uint32_t)); + + int64_t counted = 0; + int64_t prev_end = 0; + for (int64_t i = 0; i < nspans; i++) { + int64_t start = spans[i].start; + int64_t len = spans[i].len; + if (start < prev_end || len <= 0 || start < 0 || start > n - len) { + ray_release(pop_hdr); + return NULL; + } + int64_t end = start + len; + counted += len; + prev_end = end; + uint32_t first = (uint32_t)(start / RAY_MORSEL_ELEMS); + uint32_t last = (uint32_t)((end - 1) / RAY_MORSEL_ELEMS); + for (uint32_t s = first; s <= last; s++) { + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + int64_t a = start > ss ? start : ss; + int64_t b = end < se ? end : se; + pop[s] += (uint32_t)(b - a); + } + } + if (counted != total) { + ray_release(pop_hdr); + return NULL; + } + + int64_t idx_count = 0; + for (uint32_t s = 0; s < nsegs; s++) { + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + if (pop[s] != 0 && pop[s] != (uint32_t)(se - ss)) + idx_count += pop[s]; + } + + ray_t* block = ray_rowsel_new(n, total, idx_count); + if (!block) { + ray_release(pop_hdr); + return NULL; + } + uint8_t* flags = ray_rowsel_flags(block); + uint32_t* offsets = ray_rowsel_offsets(block); + uint16_t* ids = ray_rowsel_idx(block); + uint32_t cum = 0; + for (uint32_t s = 0; s < nsegs; s++) { + offsets[s] = cum; + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + if (pop[s] == 0) flags[s] = RAY_SEL_NONE; + else if (pop[s] == (uint32_t)(se - ss)) flags[s] = RAY_SEL_ALL; + else { + flags[s] = RAY_SEL_MIX; + cum += pop[s]; + } + } + offsets[nsegs] = cum; + + memset(pop, 0, (size_t)nsegs * sizeof(uint32_t)); + for (int64_t i = 0; i < nspans; i++) { + int64_t start = spans[i].start; + int64_t end = start + spans[i].len; + uint32_t first = (uint32_t)(start / RAY_MORSEL_ELEMS); + uint32_t last = (uint32_t)((end - 1) / RAY_MORSEL_ELEMS); + for (uint32_t s = first; s <= last; s++) { + if (flags[s] != RAY_SEL_MIX) continue; + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + int64_t a = start > ss ? start : ss; + int64_t b = end < se ? end : se; + uint32_t out = offsets[s] + pop[s]; + for (int64_t r = a; r < b; r++) + ids[out++] = (uint16_t)(r - ss); + pop[s] += (uint32_t)(b - a); + } + } + ray_release(pop_hdr); + return block; +} + +ray_t* ray_index_part_eq_rowsel(ray_t* col, int64_t key) { + if (!idx_fresh_nonull(col, RAY_IDX_PART)) return NULL; + if (col->type == RAY_F32 || col->type == RAY_F64) return NULL; + if (col->type != RAY_SYM && !hash_key_in_range(col->type, key)) + return rowsel_from_sorted_ids(col->len, NULL, 0); + + ray_index_t* ix = ray_index_payload(col->index); + ray_t* keys = ix->u.part.keys; + ray_t* starts = ix->u.part.starts; + ray_t* lens = ix->u.part.lens; + int64_t nparts = ix->u.part.n_parts; + if (!keys || !starts || !lens || keys->len != nparts || + starts->len != nparts || lens->len != nparts) + return NULL; + + const uint8_t* kb = (const uint8_t*)ray_data(keys); + const int64_t* st = (const int64_t*)ray_data(starts); + const int64_t* ln = (const int64_t*)ray_data(lens); + for (int64_t p = 0; p < nparts; p++) { + int64_t pkey = keys->type == RAY_SYM + ? ray_read_sym(kb, p, RAY_SYM, keys->attrs) + : set_vec_read_i64(kb, keys->type, p); + if (pkey != key) continue; + /* A direct FILTER rowsel still feeds the generic table compactor. + * For a dense partition that O(selected rows × columns) gather is + * slower than the normal predicate scan. Keep dense partitions on + * the scan path; the dedicated GROUP BY slice path remains dense-safe + * because it streams the range directly into its aggregates. */ + if (ln[p] > 64 && ln[p] > (col->len >> 1)) + return NULL; + idx_span_t span = { .start = st[p], .len = ln[p] }; + return rowsel_from_sorted_spans(col->len, &span, 1, ln[p]); + } + return rowsel_from_sorted_ids(col->len, NULL, 0); +} + ray_t* ray_index_in_rowsel(ray_t* col, ray_t* set_vec) { - /* Gate: integer-family or SYM column with fresh hash index, no nulls. */ - if (!idx_fresh_nonull(col, RAY_IDX_HASH)) return NULL; + /* Gate: integer-family or SYM column with a fresh hash/part index. */ + ray_idx_kind_t kind = ray_index_kind(col); + if ((kind != RAY_IDX_HASH && kind != RAY_IDX_PART) || + !idx_fresh_nonull(col, kind)) return NULL; bool col_is_float = (col->type == RAY_F32 || col->type == RAY_F64); if (col_is_float) return NULL; bool col_is_sym = (col->type == RAY_SYM); @@ -1433,6 +1587,45 @@ ray_t* ray_index_in_rowsel(ray_t* col, ray_t* set_vec) { return rowsel_from_sorted_ids(n, NULL, 0); } + if (kind == RAY_IDX_PART) { + ray_index_t* ix = ray_index_payload(col->index); + ray_t* keys = ix->u.part.keys; + ray_t* starts = ix->u.part.starts; + ray_t* lens = ix->u.part.lens; + int64_t nparts = ix->u.part.n_parts; + if (!keys || !starts || !lens || keys->len != nparts || + starts->len != nparts || lens->len != nparts) { + ray_release(set_hdr); + return NULL; + } + + ray_t* spans_hdr = ray_alloc(set_len * (int64_t)sizeof(idx_span_t)); + if (!spans_hdr) { + ray_release(set_hdr); + return NULL; + } + idx_span_t* spans = (idx_span_t*)ray_data(spans_hdr); + const int64_t* stv = (const int64_t*)ray_data(starts); + const int64_t* lnv = (const int64_t*)ray_data(lens); + const uint8_t* kb = (const uint8_t*)ray_data(keys); + int64_t nspans = 0; + int64_t total = 0; + for (int64_t p = 0; p < nparts; p++) { + int64_t key = col_is_sym + ? ray_read_sym(kb, p, RAY_SYM, keys->attrs) + : set_vec_read_i64(kb, keys->type, p); + if (!sorted_i64_contains(set_scratch, set_len, key)) continue; + spans[nspans].start = stv[p]; + spans[nspans].len = lnv[p]; + total += lnv[p]; + nspans++; + } + ray_release(set_hdr); + ray_t* block = rowsel_from_sorted_spans(n, spans, nspans, total); + ray_release(spans_hdr); + return block; + } + /* Resolve every set element to its GROUP up front (i64 compares against * gkeys — no column reads), so the union size is known EXACTLY before * any collection work: a dense union (where the SIMD membership scan @@ -1547,6 +1740,55 @@ static double sort_read_f64(const uint8_t* base, int8_t t, int64_t rid) { double v; memcpy(&v, base + rid * 8, 8); return v; } +/* Build a rowsel for the contiguous physical-row interval [lo, hi). Full + * morsels are represented by an ALL flag and consume no idx[] space; only + * the two possible boundary morsels contribute local row ids. */ +static ray_t* rowsel_from_contiguous_span(int64_t n, int64_t lo, int64_t hi) { + if (n < 0 || lo < 0 || hi < lo || hi > n) return NULL; + + int64_t total = hi - lo; + uint32_t n_segs = n > 0 + ? (uint32_t)((n + RAY_MORSEL_ELEMS - 1) / RAY_MORSEL_ELEMS) + : 0; + int64_t idx_count = 0; + for (uint32_t s = 0; s < n_segs; s++) { + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + int64_t a = lo > ss ? lo : ss; + int64_t b = hi < se ? hi : se; + int64_t pass = b > a ? b - a : 0; + if (pass > 0 && pass != se - ss) idx_count += pass; + } + + ray_t* block = ray_rowsel_new(n, total, idx_count); + if (!block) return NULL; + uint8_t* flags = ray_rowsel_flags(block); + uint32_t* offsets = ray_rowsel_offsets(block); + uint16_t* ids = ray_rowsel_idx(block); + uint32_t cum = 0; + for (uint32_t s = 0; s < n_segs; s++) { + offsets[s] = cum; + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > n) se = n; + int64_t a = lo > ss ? lo : ss; + int64_t b = hi < se ? hi : se; + int64_t pass = b > a ? b - a : 0; + if (pass == 0) { + flags[s] = RAY_SEL_NONE; + } else if (pass == se - ss) { + flags[s] = RAY_SEL_ALL; + } else { + flags[s] = RAY_SEL_MIX; + for (int64_t r = a; r < b; r++) + ids[cum++] = (uint16_t)(r - ss); + } + } + offsets[n_segs] = cum; + return block; +} + /* lower_bound_i: first sorted position pos where value_at(perm[pos]) >= key. * Positions in [0, n) are searched; perm is the sort permutation. */ static int64_t sort_lower_i(const int64_t* perm, int64_t n, @@ -1666,6 +1908,66 @@ ray_t* ray_index_range_rowsel(ray_t* col, uint16_t cmp_op, return block; } +ray_t* ray_sorted_range_rowsel(ray_t* col, uint16_t cmp_op, + int64_t key_i, double key_f, bool is_float) { + if (!col || RAY_IS_ERR(col) || cmp_op == OP_NE) return NULL; + if (!ray_attr_is_sorted(col) || (col->attrs & RAY_ATTR_HAS_NULLS)) + return NULL; + + bool col_is_float = (col->type == RAY_F32 || col->type == RAY_F64); + bool col_is_int = col->type == RAY_BOOL || col->type == RAY_U8 || + col->type == RAY_I16 || col->type == RAY_I32 || + col->type == RAY_I64 || col->type == RAY_DATE || + col->type == RAY_TIME || col->type == RAY_TIMESTAMP; + if (!col_is_float && !col_is_int) return NULL; + if ((bool)is_float != col_is_float) return NULL; + + int64_t n = col->len; + const uint8_t* base = (const uint8_t*)ray_data(col); + int8_t t = col->type; + int64_t lower = 0, upper = 0; + if (!is_float) { + int64_t hi = n; + while (lower < hi) { + int64_t mid = lower + (hi - lower) / 2; + if (sort_read_i64(base, t, mid) < key_i) lower = mid + 1; + else hi = mid; + } + upper = lower; + hi = n; + while (upper < hi) { + int64_t mid = upper + (hi - upper) / 2; + if (sort_read_i64(base, t, mid) <= key_i) upper = mid + 1; + else hi = mid; + } + } else { + int64_t hi = n; + while (lower < hi) { + int64_t mid = lower + (hi - lower) / 2; + if (sort_read_f64(base, t, mid) < key_f) lower = mid + 1; + else hi = mid; + } + upper = lower; + hi = n; + while (upper < hi) { + int64_t mid = upper + (hi - upper) / 2; + if (sort_read_f64(base, t, mid) <= key_f) upper = mid + 1; + else hi = mid; + } + } + + int64_t lo, hi; + switch (cmp_op) { + case OP_LT: lo = 0; hi = lower; break; + case OP_LE: lo = 0; hi = upper; break; + case OP_GT: lo = upper; hi = n; break; + case OP_GE: lo = lower; hi = n; break; + case OP_EQ: lo = lower; hi = upper; break; + default: return NULL; + } + return rowsel_from_contiguous_span(n, lo, hi); +} + /* -------------------------------------------------------------------------- * Bloom-index definite-absent probe * @@ -2409,12 +2711,14 @@ int64_t ray_index_find_row(ray_t* col, int64_t key) { * sites outside this file (the eq+range conjunction in exec.c) construct * selections from index slices. */ -int64_t ray_index_hash_sym_slices(ray_t* col, ray_t* keys, - ray_idx_slice_t** out, ray_t** hdr_out) { +int64_t ray_index_sym_slices(ray_t* col, ray_t* keys, + ray_idx_slice_t** out, ray_t** hdr_out) { *out = NULL; *hdr_out = NULL; if (!col || !keys) return -1; if (col->type != RAY_SYM) return -1; - if (!idx_fresh_nonull(col, RAY_IDX_HASH)) return -1; + ray_idx_kind_t kind = ray_index_kind(col); + if ((kind != RAY_IDX_HASH && kind != RAY_IDX_PART) || + !idx_fresh_nonull(col, kind)) return -1; bool atom = (keys->type == -RAY_SYM); int64_t nkeys; @@ -2446,13 +2750,40 @@ int64_t ray_index_hash_sym_slices(ray_t* col, ray_t* keys, if (!shdr) { ray_free(dhdr); return -1; } ray_idx_slice_t* sl = (ray_idx_slice_t*)ray_data(shdr); int64_t k = 0; - for (int64_t i = 0; i < nd; i++) { - const int64_t* rows = NULL; - int64_t n = 0; - int hit = ray_index_hash_group(col, doms[i], &rows, &n); - if (hit < 0) { ray_free(dhdr); ray_free(shdr); return -1; } - if (hit == 0 || n == 0) continue; /* in domain, no rows */ - sl[k].dom = doms[i]; sl[k].rows = rows; sl[k].n = n; k++; + if (kind == RAY_IDX_HASH) { + for (int64_t i = 0; i < nd; i++) { + const int64_t* rows = NULL; + int64_t n = 0; + int hit = ray_index_hash_group(col, doms[i], &rows, &n); + if (hit < 0) { ray_free(dhdr); ray_free(shdr); return -1; } + if (hit == 0 || n == 0) continue; + sl[k].dom = doms[i]; sl[k].rows = rows; + sl[k].first = 0; sl[k].n = n; k++; + } + } else { + ray_index_t* ix = ray_index_payload(col->index); + ray_t* pkeys = ix->u.part.keys; + ray_t* starts = ix->u.part.starts; + ray_t* lens = ix->u.part.lens; + int64_t np = ix->u.part.n_parts; + if (!pkeys || !starts || !lens || pkeys->len != np || + starts->len != np || lens->len != np) { + ray_free(dhdr); ray_free(shdr); return -1; + } + const uint8_t* pk = (const uint8_t*)ray_data(pkeys); + const int64_t* ps = (const int64_t*)ray_data(starts); + const int64_t* pl = (const int64_t*)ray_data(lens); + for (int64_t i = 0; i < nd; i++) { + for (int64_t p = 0; p < np; p++) { + int64_t pdom = ray_read_sym(pk, p, RAY_SYM, pkeys->attrs); + if (pdom != doms[i]) continue; + if (pl[p] > 0) { + sl[k].dom = doms[i]; sl[k].rows = NULL; + sl[k].first = ps[p]; sl[k].n = pl[p]; k++; + } + break; + } + } } ray_free(dhdr); if (k == 0) { ray_free(shdr); return 0; } diff --git a/src/ops/idxop.h b/src/ops/idxop.h index 27bcee4b8..973b31976 100644 --- a/src/ops/idxop.h +++ b/src/ops/idxop.h @@ -191,9 +191,9 @@ static inline ray_index_t* ray_index_payload(ray_t* idx) { * Diagnostic, unsynchronized (same caveat as ray_expr_bail_counts). */ typedef enum { IDX_SITE_FILTER_ZONE = 0, IDX_SITE_FILTER_BLOOM, IDX_SITE_FILTER_HASH, - IDX_SITE_FILTER_RANGE, IDX_SITE_IN, IDX_SITE_FIND, IDX_SITE_SORT, - IDX_SITE_DISTINCT, IDX_SITE_ASOF, IDX_SITE_FILTER_EQRANGE, - IDX_SITE_GROUP_SLICE, IDX_SITE__N + IDX_SITE_FILTER_PART, IDX_SITE_FILTER_RANGE, IDX_SITE_IN, IDX_SITE_FIND, + IDX_SITE_SORT, IDX_SITE_DISTINCT, IDX_SITE_ASOF, + IDX_SITE_FILTER_EQRANGE, IDX_SITE_GROUP_SLICE, IDX_SITE__N } idx_site_t; extern uint64_t ray_idx_consults[IDX_SITE__N]; extern uint64_t ray_idx_hits[IDX_SITE__N]; @@ -326,11 +326,17 @@ bool ray_index_bloom_absent(ray_t* col, int64_t key); * has NaN / -0 semantics the unfused compare kernel handles. */ ray_t* ray_index_hash_eq_rowsel(ray_t* col, int64_t key); -/* ===== Hash-index IN probe ===== +/* Direct equality selection for a fresh part index. `key` is a numeric + * value or a column-domain SYM id. Dense matches remain eligible because + * the part payload already stores one contiguous physical span. */ +ray_t* ray_index_part_eq_rowsel(ray_t* col, int64_t key); + +/* ===== Hash/part-index IN probe ===== * - * Build a rowsel of all rows whose value is in set_vec via per-element - * hash-chain probes. Integer-family and SYM columns (same conservatism as - * the hash-eq path: F32/F64 NaN/-0 semantics belong to the scan kernel). + * Build a rowsel of all rows whose value is in set_vec via per-element hash + * probes or direct physical spans from a part index. Integer-family and SYM + * columns (same conservatism as the hash-eq path: F32/F64 NaN/-0 semantics + * belong to the scan kernel). * * set_vec must match the column family: an integer-family typed vec for an * integer column, a SYM vec for a SYM column (each set symbol is resolved @@ -340,10 +346,10 @@ ray_t* ray_index_hash_eq_rowsel(ray_t* col, int64_t key); * Returns: * - A fresh rowsel block (rc=1) on success — install on g->selection. * An empty set yields a valid all-NONE rowsel (NOT NULL). - * - NULL when the column is not eligible (no hash index, stale, float- - * family column, unsupported set type, OOM) or the union is too dense - * (total chain-step budget exceeded — the SIMD membership scan wins at - * that density). Caller falls back to the scan path. */ + * - NULL when the column is not eligible (no hash/part index, stale, + * float-family column, unsupported set type, OOM) or a hash-index union + * is too dense. Part-index unions retain dense full morsels efficiently. + * Caller falls back to the scan path. */ ray_t* ray_index_in_rowsel(ray_t* col, ray_t* set_vec); /* ===== Hash-index find (point lookup) ===== @@ -387,20 +393,23 @@ ray_t* ray_index_rowsel_from_ids(int64_t nrows, const int64_t* ids, int64_t n); * valid while the index is). */ typedef struct { int64_t dom; /* column-domain key id */ - const int64_t* rows; /* ascending row ids (borrowed) */ + const int64_t* rows; /* ascending row ids (borrowed); NULL if contiguous */ + int64_t first; /* first physical row when rows == NULL */ int64_t n; /* slice length (> 0) */ } ray_idx_slice_t; /* Resolve a SYM eq key (sym ATOM, global intern id) or IN set (SYM vec, - * any domain) against `col`'s fresh CSR hash index into per-key slices, + * any domain) against `col`'s fresh hash or part index into per-key slices, * canonicalized ascending by column-domain id (duplicates collapsed; - * absent / rowless keys dropped). On success returns K >= 0 and, for + * absent / rowless keys dropped). Hash slices borrow CSR row ids; part + * slices use rows==NULL plus their contiguous physical first row. On success + * returns K >= 0 and, for * K > 0, *out points into a fresh alloc block returned via *hdr_out * (release with ray_free). K == 0 → provably no surviving rows (*out * and *hdr_out NULL). Returns -1 when ineligible: stale or missing - * hash index, null-bearing or non-SYM column, wrong key shape. */ -int64_t ray_index_hash_sym_slices(ray_t* col, ray_t* keys, - ray_idx_slice_t** out, ray_t** hdr_out); + * hash/part index, null-bearing or non-SYM column, wrong key shape. */ +int64_t ray_index_sym_slices(ray_t* col, ray_t* keys, + ray_idx_slice_t** out, ray_t** hdr_out); /* ===== Sort-index range probe ===== * @@ -421,6 +430,14 @@ int64_t ray_index_hash_sym_slices(ray_t* col, ray_t* keys, ray_t* ray_index_range_rowsel(ray_t* col, uint16_t cmp_op, int64_t key_i, double key_f, bool is_float); +/* Build a rowsel by binary-searching a physically non-descending column. + * Unlike the sort-index route above, matching row ids are already one + * contiguous interval, so dense spans are eligible and the rowsel stores + * only the at-most-two partial boundary morsels. Requires the verified + * RAY_ATTR_SORTED marker and a null-free numeric column. */ +ray_t* ray_sorted_range_rowsel(ray_t* col, uint16_t cmp_op, + int64_t key_i, double key_f, bool is_float); + /* ===== Sort-index ORDER BY permutation probe ===== * * Returns a BORROWED pointer to the ascending permutation vector stored in diff --git a/src/ops/ops.h b/src/ops/ops.h index 20e7c4e81..abbf2d59d 100644 --- a/src/ops/ops.h +++ b/src/ops/ops.h @@ -519,9 +519,9 @@ typedef struct ray_graph { /* Slice-group hint (FILTER(in/eq on c) + GROUP(by c) fusion). * Armed by ray_slice_group_probe (exec.c) INSTEAD of executing the * WHERE filter when the predicate is exactly membership on the - * single bare group-key column and that column carries a fresh CSR - * hash index: surviving groups are then exactly the key set and - * each group's rows are its full CSR slice. sg_col is the retained + * single bare group-key column and that column carries a fresh hash + * or part index: surviving groups are then exactly the key set and + * each group's rows are its full index slice. sg_col is the retained * key column; sg_slices_hdr is an alloc block of ray_idx_slice_t * (idxop.h) resolved at probe time, ascending by column-domain id. * exec_group consumes the hint — aggregating the slices directly or diff --git a/src/ops/query.c b/src/ops/query.c index 02c048de6..7a43fecc6 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -4351,7 +4351,7 @@ static uint8_t win_kind_from_name(const char* s, size_t len) { } /* - * (window {from: T part: [col …] order: [col …] frame: 'whole|'running + * (window {from: T part: [col …] order: [col …] frame: 'whole|'running|N * funcs: {outname: (fn inputcol) …}}) * * Builds a DAG OP_WINDOW node over the table T, executes it, then renames the @@ -4360,7 +4360,9 @@ static uint8_t win_kind_from_name(const char* s, size_t len) { * Special form: args[0] is the unevaluated dict literal AST. * `from:` is the only clause that requires full eval (it's a variable ref). * `part:`/`order:` arrive as RAY_SYM vectors or -RAY_SYM atoms (column refs). - * `frame:` arrives as a quoted -RAY_SYM atom ('whole or 'running). + * `frame:` is either a quoted -RAY_SYM atom ('whole or 'running), or a + * positive integer N for a fixed trailing N-row frame including the current + * row. * `funcs:` is a nested RAY_DICT whose values are list expressions (fn col). */ ray_t* ray_window_fn(ray_t** args, int64_t n) { @@ -4403,22 +4405,41 @@ ray_t* ray_window_fn(ray_t** args, int64_t n) { } /* ── frame: ── */ ray_t* frame_expr = dict_get(dict, "frame"); + uint8_t frame_start = RAY_BOUND_UNBOUNDED_PRECEDING; uint8_t frame_end = RAY_BOUND_UNBOUNDED_FOLLOWING; /* default: whole partition */ + int64_t frame_start_n = 0; if (frame_expr) { - if (frame_expr->type != -RAY_SYM) { - DICT_VIEW_CLOSE(fv); ray_release(tbl); - return ray_error("domain", "window: frame must be 'whole or 'running"); - } - ray_t* fs = ray_sym_str(frame_expr->i64); - if (fs) { - const char* fsp = ray_str_ptr(fs); - size_t fsl = ray_str_len(fs); - if (fsl == 7 && memcmp(fsp, "running", 7) == 0) - frame_end = RAY_BOUND_CURRENT_ROW; - else if (!(fsl == 5 && memcmp(fsp, "whole", 5) == 0)) { + if (frame_expr->type == -RAY_SYM) { + ray_t* fs = ray_sym_str(frame_expr->i64); + if (fs) { + const char* fsp = ray_str_ptr(fs); + size_t fsl = ray_str_len(fs); + if (fsl == 7 && memcmp(fsp, "running", 7) == 0) + frame_end = RAY_BOUND_CURRENT_ROW; + else if (!(fsl == 5 && memcmp(fsp, "whole", 5) == 0)) { + DICT_VIEW_CLOSE(fv); ray_release(tbl); + return ray_error("domain", "window: frame must be 'whole, 'running, or a positive row count"); + } + } + } else { + int64_t width = 0; + switch (frame_expr->type) { + case -RAY_I64: width = frame_expr->i64; break; + case -RAY_I32: width = frame_expr->i32; break; + case -RAY_I16: width = frame_expr->i16; break; + case -RAY_U8: + case -RAY_BOOL: width = frame_expr->u8; break; + default: + DICT_VIEW_CLOSE(fv); ray_release(tbl); + return ray_error("domain", "window: frame must be 'whole, 'running, or a positive row count"); + } + if (width < 1) { DICT_VIEW_CLOSE(fv); ray_release(tbl); - return ray_error("domain", "window: frame must be 'whole or 'running"); + return ray_error("range", "window: trailing frame row count must be >= 1"); } + frame_start = RAY_BOUND_N_PRECEDING; + frame_end = RAY_BOUND_CURRENT_ROW; + frame_start_n = width - 1; } } @@ -4572,8 +4593,8 @@ ray_t* ray_window_fn(ray_t** args, int64_t n) { func_kinds, func_inputs, func_params, (uint32_t)n_funcs, RAY_FRAME_ROWS, - RAY_BOUND_UNBOUNDED_PRECEDING, frame_end, - 0, 0); + frame_start, frame_end, + frame_start_n, 0); scratch_free(funcs_hdr); scratch_free(order_hdr); scratch_free(part_hdr); if (!win_op) { scratch_free(outname_hdr); ray_graph_free(g); ray_release(tbl); @@ -8320,7 +8341,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { root = ray_optimize(g, root); /* Slice-group fusion: when the WHERE predicate is exactly * membership (in/==) on the single bare group-key column - * and that column carries a fresh CSR hash index, skip + * and that column carries a fresh hash or part index, skip * the filter scan entirely — exec_group aggregates the * key slices directly (or folds them into the equivalent * selection). count-distinct outputs stay on the filter diff --git a/src/ops/rowsel.c b/src/ops/rowsel.c index 30d1043ac..42404a7ac 100644 --- a/src/ops/rowsel.c +++ b/src/ops/rowsel.c @@ -578,3 +578,106 @@ ray_t* ray_rowsel_refine(ray_t* existing, ray_t* pred) { ray_release(pop_block); return block; } + +ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b) { + if (!a || !b) return NULL; + ray_rowsel_t* am = ray_rowsel_meta(a); + ray_rowsel_t* bm = ray_rowsel_meta(b); + if (am->nrows != bm->nrows || am->n_segs != bm->n_segs) return NULL; + + uint32_t ns = am->n_segs; + const uint8_t* af = ray_rowsel_flags(a); + const uint8_t* bf = ray_rowsel_flags(b); + const uint32_t* ao = ray_rowsel_offsets(a); + const uint32_t* bo = ray_rowsel_offsets(b); + const uint16_t* ai = ray_rowsel_idx(a); + const uint16_t* bi = ray_rowsel_idx(b); + int64_t total = 0; + int64_t idx_count = 0; + + /* Count the intersection per segment. MIX arrays are sorted by every + * producer, so MIX∩MIX is a linear two-cursor walk. */ + for (uint32_t s = 0; s < ns; s++) { + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > am->nrows) se = am->nrows; + int64_t seg_len = se - ss; + int64_t pc = 0; + if (af[s] == RAY_SEL_NONE || bf[s] == RAY_SEL_NONE) { + pc = 0; + } else if (af[s] == RAY_SEL_ALL && bf[s] == RAY_SEL_ALL) { + pc = seg_len; + } else if (af[s] == RAY_SEL_ALL) { + pc = bo[s + 1] - bo[s]; + } else if (bf[s] == RAY_SEL_ALL) { + pc = ao[s + 1] - ao[s]; + } else { + uint32_t ap = ao[s], ae = ao[s + 1]; + uint32_t bp = bo[s], be = bo[s + 1]; + while (ap < ae && bp < be) { + uint16_t av = ai[ap], bv = bi[bp]; + if (av < bv) ap++; + else if (bv < av) bp++; + else { pc++; ap++; bp++; } + } + } + total += pc; + if (pc > 0 && pc != seg_len) idx_count += pc; + } + + ray_t* out = ray_rowsel_new(am->nrows, total, idx_count); + if (!out) return NULL; + uint8_t* of = ray_rowsel_flags(out); + uint32_t* oo = ray_rowsel_offsets(out); + uint16_t* oi = ray_rowsel_idx(out); + uint32_t cum = 0; + + for (uint32_t s = 0; s < ns; s++) { + oo[s] = cum; + int64_t ss = (int64_t)s * RAY_MORSEL_ELEMS; + int64_t se = ss + RAY_MORSEL_ELEMS; + if (se > am->nrows) se = am->nrows; + int64_t seg_len = se - ss; + + if (af[s] == RAY_SEL_NONE || bf[s] == RAY_SEL_NONE) { + of[s] = RAY_SEL_NONE; + } else if (af[s] == RAY_SEL_ALL && bf[s] == RAY_SEL_ALL) { + of[s] = RAY_SEL_ALL; + } else { + const uint16_t* src = NULL; + uint32_t begin = 0, end = 0; + if (af[s] == RAY_SEL_ALL) { + src = bi; begin = bo[s]; end = bo[s + 1]; + } else if (bf[s] == RAY_SEL_ALL) { + src = ai; begin = ao[s]; end = ao[s + 1]; + } + + uint32_t start = cum; + if (src) { + for (uint32_t p = begin; p < end; p++) oi[cum++] = src[p]; + } else { + uint32_t ap = ao[s], ae = ao[s + 1]; + uint32_t bp = bo[s], be = bo[s + 1]; + while (ap < ae && bp < be) { + uint16_t av = ai[ap], bv = bi[bp]; + if (av < bv) ap++; + else if (bv < av) bp++; + else { oi[cum++] = av; ap++; bp++; } + } + } + uint32_t pc = cum - start; + if (pc == 0) { + of[s] = RAY_SEL_NONE; + } else if ((int64_t)pc == seg_len) { + /* This cannot arise from MIX input under canonical encoding, + * but keep the representation valid defensively. */ + of[s] = RAY_SEL_ALL; + cum = start; + } else { + of[s] = RAY_SEL_MIX; + } + } + } + oo[ns] = cum; + return out; +} diff --git a/src/ops/rowsel.h b/src/ops/rowsel.h index e9fa3c059..a4139e4fb 100644 --- a/src/ops/rowsel.h +++ b/src/ops/rowsel.h @@ -184,6 +184,11 @@ ray_t* ray_rowsel_to_indices(ray_t* sel); * the old selection after replacing it. */ ray_t* ray_rowsel_refine(ray_t* existing, ray_t* pred); +/* Intersect two rowsels over the same source row domain. Both inputs are + * borrowed; the returned rowsel is fresh. Runs in O(n_segs + mixed ids), + * preserving NONE/ALL compression without materializing a bool vector. */ +ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b); + /* ────────────────────────────────────────────────────────────────── * Streaming builder — per-morsel selection emitter * diff --git a/src/ops/window.c b/src/ops/window.c index 081b3a753..fb132935f 100644 --- a/src/ops/window.c +++ b/src/ops/window.c @@ -141,7 +141,7 @@ static void win_compute_partition( ray_t* const* order_vecs, uint32_t n_order, ray_t* const* func_vecs, const uint8_t* func_kinds, const int64_t* func_params, uint32_t n_funcs, - uint8_t frame_start, uint8_t frame_end, + uint8_t frame_start, uint8_t frame_end, int64_t frame_start_n, const int64_t* sorted_idx, int64_t ps, int64_t pe, ray_t* const* result_vecs, const bool* is_f64) { @@ -154,6 +154,8 @@ static void win_compute_partition( ray_t* rvec = result_vecs[f]; bool whole = (frame_start == RAY_BOUND_UNBOUNDED_PRECEDING && frame_end == RAY_BOUND_UNBOUNDED_FOLLOWING); + bool trailing = (frame_start == RAY_BOUND_N_PRECEDING && + frame_end == RAY_BOUND_CURRENT_ROW); switch (kind) { case RAY_WIN_ROW_NUMBER: { @@ -199,6 +201,12 @@ static void win_compute_partition( if (whole) { for (int64_t i = ps; i < pe; i++) out[sorted_idx[i]] = part_len; + } else if (trailing) { + int64_t width = frame_start_n + 1; + for (int64_t i = ps; i < pe; i++) { + int64_t seen = i - ps + 1; + out[sorted_idx[i]] = seen < width ? seen : width; + } } else { for (int64_t i = ps; i < pe; i++) out[sorted_idx[i]] = i - ps + 1; @@ -210,38 +218,44 @@ static void win_compute_partition( if (is_f64[f]) { double* out = (double*)ray_data(rvec); if (whole) { - double t = 0.0; + double total = 0.0; for (int64_t i = ps; i < pe; i++) if (!ray_vec_is_null(fvec, sorted_idx[i])) - t += win_read_f64(fvec, sorted_idx[i]); - /* Single-null float model: window SUM can overflow → ±Inf; - * canonicalize to NULL_F64 (HAS_NULLS via win_finalize_nulls). */ - t = ray_f64_fin(t); - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = t; + total += win_read_f64(fvec, sorted_idx[i]); + total = ray_f64_fin(total); + for (int64_t i = ps; i < pe; i++) out[sorted_idx[i]] = total; } else { double acc = 0.0; for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) - acc += win_read_f64(fvec, sorted_idx[i]); - out[sorted_idx[i]] = ray_f64_fin(acc); + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) acc += win_read_f64(fvec, row); + if (trailing) { + int64_t drop = i - frame_start_n - 1; + if (drop >= ps && !ray_vec_is_null(fvec, sorted_idx[drop])) + acc -= win_read_f64(fvec, sorted_idx[drop]); + } + out[row] = ray_f64_fin(acc); } } } else { int64_t* out = (int64_t*)ray_data(rvec); if (whole) { - int64_t t = 0; + int64_t total = 0; for (int64_t i = ps; i < pe; i++) if (!ray_vec_is_null(fvec, sorted_idx[i])) - t += win_read_i64(fvec, sorted_idx[i]); - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = t; + total += win_read_i64(fvec, sorted_idx[i]); + for (int64_t i = ps; i < pe; i++) out[sorted_idx[i]] = total; } else { int64_t acc = 0; for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) - acc += win_read_i64(fvec, sorted_idx[i]); - out[sorted_idx[i]] = acc; + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) acc += win_read_i64(fvec, row); + if (trailing) { + int64_t drop = i - frame_start_n - 1; + if (drop >= ps && !ray_vec_is_null(fvec, sorted_idx[drop])) + acc -= win_read_i64(fvec, sorted_idx[drop]); + } + out[row] = acc; } } } @@ -251,156 +265,107 @@ static void win_compute_partition( if (!fvec) break; double* out = (double*)ray_data(rvec); if (whole) { - double t = 0.0; - int64_t cnt = 0; - for (int64_t i = ps; i < pe; i++) - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - t += win_read_f64(fvec, sorted_idx[i]); cnt++; - } - if (cnt > 0) { - double avg = ray_f64_fin(t / (double)cnt); - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = avg; - } else { - for (int64_t i = ps; i < pe; i++) - win_set_null(rvec, sorted_idx[i]); - } - } else { - double acc = 0.0; - int64_t cnt = 0; + double total = 0.0; + int64_t count = 0; for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - acc += win_read_f64(fvec, sorted_idx[i]); cnt++; + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) { + total += win_read_f64(fvec, row); + count++; } - if (cnt > 0) - out[sorted_idx[i]] = ray_f64_fin(acc / (double)cnt); - else - win_set_null(rvec, sorted_idx[i]); } - } - break; - } - case RAY_WIN_MIN: { - if (!fvec) break; - if (is_f64[f]) { - double* out = (double*)ray_data(rvec); - if (whole) { - double mn = DBL_MAX; int found = 0; - for (int64_t i = ps; i < pe; i++) { - if (ray_vec_is_null(fvec, sorted_idx[i])) continue; - double v = win_read_f64(fvec, sorted_idx[i]); - if (!found || v < mn) { mn = v; found = 1; } - } - if (found) { - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = mn; - } else { - for (int64_t i = ps; i < pe; i++) - win_set_null(rvec, sorted_idx[i]); - } + if (count > 0) { + double avg = ray_f64_fin(total / (double)count); + for (int64_t i = ps; i < pe; i++) out[sorted_idx[i]] = avg; } else { - double mn = DBL_MAX; int found = 0; - for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - double v = win_read_f64(fvec, sorted_idx[i]); - if (!found || v < mn) { mn = v; found = 1; } - } - if (found) - out[sorted_idx[i]] = mn; - else - win_set_null(rvec, sorted_idx[i]); - } + for (int64_t i = ps; i < pe; i++) win_set_null(rvec, sorted_idx[i]); } } else { - int64_t* out = (int64_t*)ray_data(rvec); - if (whole) { - int64_t mn = INT64_MAX; int found = 0; - for (int64_t i = ps; i < pe; i++) { - if (ray_vec_is_null(fvec, sorted_idx[i])) continue; - int64_t v = win_read_i64(fvec, sorted_idx[i]); - if (!found || v < mn) { mn = v; found = 1; } - } - if (found) { - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = mn; - } else { - for (int64_t i = ps; i < pe; i++) - win_set_null(rvec, sorted_idx[i]); + double acc = 0.0; + int64_t count = 0; + for (int64_t i = ps; i < pe; i++) { + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) { + acc += win_read_f64(fvec, row); + count++; } - } else { - int64_t mn = INT64_MAX; int found = 0; - for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - int64_t v = win_read_i64(fvec, sorted_idx[i]); - if (!found || v < mn) { mn = v; found = 1; } + if (trailing) { + int64_t drop = i - frame_start_n - 1; + if (drop >= ps && !ray_vec_is_null(fvec, sorted_idx[drop])) { + acc -= win_read_f64(fvec, sorted_idx[drop]); + count--; } - if (found) - out[sorted_idx[i]] = mn; - else - win_set_null(rvec, sorted_idx[i]); } + if (count > 0) out[row] = ray_f64_fin(acc / (double)count); + else win_set_null(rvec, row); } } break; } + case RAY_WIN_MIN: case RAY_WIN_MAX: { if (!fvec) break; + bool want_max = kind == RAY_WIN_MAX; if (is_f64[f]) { double* out = (double*)ray_data(rvec); if (whole) { - double mx = -DBL_MAX; int found = 0; + double best = want_max ? -DBL_MAX : DBL_MAX; + int found = 0; for (int64_t i = ps; i < pe; i++) { - if (ray_vec_is_null(fvec, sorted_idx[i])) continue; - double v = win_read_f64(fvec, sorted_idx[i]); - if (!found || v > mx) { mx = v; found = 1; } + int64_t row = sorted_idx[i]; + if (ray_vec_is_null(fvec, row)) continue; + double v = win_read_f64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } } - if (found) { - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = mx; - } else { - for (int64_t i = ps; i < pe; i++) - win_set_null(rvec, sorted_idx[i]); + for (int64_t i = ps; i < pe; i++) { + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } } else { - double mx = -DBL_MAX; int found = 0; for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - double v = win_read_f64(fvec, sorted_idx[i]); - if (!found || v > mx) { mx = v; found = 1; } + int64_t lo = trailing ? i - frame_start_n : ps; + if (lo < ps) lo = ps; + double best = want_max ? -DBL_MAX : DBL_MAX; + int found = 0; + for (int64_t j = lo; j <= i; j++) { + int64_t row = sorted_idx[j]; + if (ray_vec_is_null(fvec, row)) continue; + double v = win_read_f64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } } - if (found) - out[sorted_idx[i]] = mx; - else - win_set_null(rvec, sorted_idx[i]); + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } } } else { int64_t* out = (int64_t*)ray_data(rvec); if (whole) { - int64_t mx = INT64_MIN; int found = 0; + int64_t best = want_max ? INT64_MIN : INT64_MAX; + int found = 0; for (int64_t i = ps; i < pe; i++) { - if (ray_vec_is_null(fvec, sorted_idx[i])) continue; - int64_t v = win_read_i64(fvec, sorted_idx[i]); - if (!found || v > mx) { mx = v; found = 1; } + int64_t row = sorted_idx[i]; + if (ray_vec_is_null(fvec, row)) continue; + int64_t v = win_read_i64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } } - if (found) { - for (int64_t i = ps; i < pe; i++) - out[sorted_idx[i]] = mx; - } else { - for (int64_t i = ps; i < pe; i++) - win_set_null(rvec, sorted_idx[i]); + for (int64_t i = ps; i < pe; i++) { + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } } else { - int64_t mx = INT64_MIN; int found = 0; for (int64_t i = ps; i < pe; i++) { - if (!ray_vec_is_null(fvec, sorted_idx[i])) { - int64_t v = win_read_i64(fvec, sorted_idx[i]); - if (!found || v > mx) { mx = v; found = 1; } + int64_t lo = trailing ? i - frame_start_n : ps; + if (lo < ps) lo = ps; + int64_t best = want_max ? INT64_MIN : INT64_MAX; + int found = 0; + for (int64_t j = lo; j <= i; j++) { + int64_t row = sorted_idx[j]; + if (ray_vec_is_null(fvec, row)) continue; + int64_t v = win_read_i64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } } - if (found) - out[sorted_idx[i]] = mx; - else - win_set_null(rvec, sorted_idx[i]); + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } } } @@ -474,20 +439,23 @@ static void win_compute_partition( } case RAY_WIN_FIRST_VALUE: { if (!fvec) break; - bool first_null = ray_vec_is_null(fvec, sorted_idx[ps]); if (is_f64[f]) { double* out = (double*)ray_data(rvec); - double first = first_null ? 0.0 : win_read_f64(fvec, sorted_idx[ps]); for (int64_t i = ps; i < pe; i++) { - out[sorted_idx[i]] = first; - if (first_null) win_set_null(rvec, sorted_idx[i]); + int64_t src = trailing ? i - frame_start_n : ps; + if (src < ps) src = ps; + bool is_null = ray_vec_is_null(fvec, sorted_idx[src]); + out[sorted_idx[i]] = is_null ? 0.0 : win_read_f64(fvec, sorted_idx[src]); + if (is_null) win_set_null(rvec, sorted_idx[i]); } } else { int64_t* out = (int64_t*)ray_data(rvec); - int64_t first = first_null ? 0 : win_read_i64(fvec, sorted_idx[ps]); for (int64_t i = ps; i < pe; i++) { - out[sorted_idx[i]] = first; - if (first_null) win_set_null(rvec, sorted_idx[i]); + int64_t src = trailing ? i - frame_start_n : ps; + if (src < ps) src = ps; + bool is_null = ray_vec_is_null(fvec, sorted_idx[src]); + out[sorted_idx[i]] = is_null ? 0 : win_read_i64(fvec, sorted_idx[src]); + if (is_null) win_set_null(rvec, sorted_idx[i]); } } break; @@ -533,21 +501,27 @@ static void win_compute_partition( if (!fvec) break; int64_t nth = func_params[f]; if (nth < 1) nth = 1; - bool nth_null = (nth > part_len) || - ray_vec_is_null(fvec, sorted_idx[ps + nth - 1]); if (is_f64[f]) { double* out = (double*)ray_data(rvec); - double val = nth_null ? 0.0 : win_read_f64(fvec, sorted_idx[ps + nth - 1]); for (int64_t i = ps; i < pe; i++) { - out[sorted_idx[i]] = val; - if (nth_null) win_set_null(rvec, sorted_idx[i]); + int64_t lo = trailing ? i - frame_start_n : ps; + if (lo < ps) lo = ps; + int64_t src = lo + nth - 1; + bool is_null = src > (whole ? pe - 1 : i) || + ray_vec_is_null(fvec, sorted_idx[src]); + out[sorted_idx[i]] = is_null ? 0.0 : win_read_f64(fvec, sorted_idx[src]); + if (is_null) win_set_null(rvec, sorted_idx[i]); } } else { int64_t* out = (int64_t*)ray_data(rvec); - int64_t val = nth_null ? 0 : win_read_i64(fvec, sorted_idx[ps + nth - 1]); for (int64_t i = ps; i < pe; i++) { - out[sorted_idx[i]] = val; - if (nth_null) win_set_null(rvec, sorted_idx[i]); + int64_t lo = trailing ? i - frame_start_n : ps; + if (lo < ps) lo = ps; + int64_t src = lo + nth - 1; + bool is_null = src > (whole ? pe - 1 : i) || + ray_vec_is_null(fvec, sorted_idx[src]); + out[sorted_idx[i]] = is_null ? 0 : win_read_i64(fvec, sorted_idx[src]); + if (is_null) win_set_null(rvec, sorted_idx[i]); } } break; @@ -566,6 +540,7 @@ typedef struct { uint32_t n_funcs; uint8_t frame_start; uint8_t frame_end; + int64_t frame_start_n; int64_t* sorted_idx; int64_t* part_offsets; ray_t** result_vecs; @@ -580,7 +555,7 @@ static void win_par_fn(void* arg, uint32_t worker_id, win_compute_partition( ctx->order_vecs, ctx->n_order, ctx->func_vecs, ctx->func_kinds, ctx->func_params, - ctx->n_funcs, ctx->frame_start, ctx->frame_end, + ctx->n_funcs, ctx->frame_start, ctx->frame_end, ctx->frame_start_n, ctx->sorted_idx, ctx->part_offsets[p], ctx->part_offsets[p + 1], ctx->result_vecs, ctx->is_f64); } @@ -670,15 +645,13 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { ray_op_ext_t* ext = find_ext(g, op->id); if (!ext) return ray_error("nyi", NULL); - /* Frame-spec validation (review 2.10): the compute path honors EXACTLY two - * frames — the whole partition (UNBOUNDED PRECEDING .. UNBOUNDED FOLLOWING) - * and the running frame (UNBOUNDED PRECEDING .. CURRENT ROW, ROWS mode). - * Numeric N_PRECEDING/N_FOLLOWING bounds and RANGE-mode CURRENT ROW were - * accepted but SILENTLY degraded to the running frame, yielding the wrong - * window. Reject any unsupported frame LOUDLY instead. - * - whole: end == UNBOUNDED_FOLLOWING (ROWS == RANGE here, mode moot) - * - running: end == CURRENT_ROW and ROWS mode - * Both require start == UNBOUNDED_PRECEDING. */ + /* Frame-spec validation: the compute path honors the whole partition, + * the running frame, and fixed trailing ROWS frames. Other numeric + * bounds and RANGE-mode CURRENT ROW must be rejected loudly rather than + * silently degrading to a different frame. + * - whole: UNBOUNDED PRECEDING .. UNBOUNDED FOLLOWING + * - running: UNBOUNDED PRECEDING .. CURRENT ROW in ROWS mode + * - trailing: N PRECEDING .. CURRENT ROW in ROWS mode */ { uint8_t fs = ext->window.frame_start; uint8_t fe = ext->window.frame_end; @@ -687,10 +660,13 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { bool running_frame = (fs == RAY_BOUND_UNBOUNDED_PRECEDING && fe == RAY_BOUND_CURRENT_ROW && ext->window.frame_type == RAY_FRAME_ROWS); - if (!whole_frame && !running_frame) + bool trailing_frame = (fs == RAY_BOUND_N_PRECEDING && + fe == RAY_BOUND_CURRENT_ROW && + ext->window.frame_type == RAY_FRAME_ROWS && + ext->window.frame_start_n >= 0); + if (!whole_frame && !running_frame && !trailing_frame) return ray_error("nyi", "unsupported window frame: only " - "ROWS BETWEEN UNBOUNDED PRECEDING AND {CURRENT ROW | " - "UNBOUNDED FOLLOWING} are honored"); + "whole, running, and fixed trailing ROWS frames are honored"); } /* Case-a streaming fast-path: if PARTITION BY includes the physical/date @@ -1383,6 +1359,7 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { .func_params = ext->window.func_params, .n_funcs = n_funcs, .frame_start = ext->window.frame_start, .frame_end = ext->window.frame_end, + .frame_start_n = ext->window.frame_start_n, .sorted_idx = sorted_idx, .part_offsets = part_offsets, .result_vecs = result_vecs, .is_f64 = is_f64, }; @@ -1393,6 +1370,7 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { order_vecs, n_order, func_vecs, ext->window.func_kinds, ext->window.func_params, n_funcs, ext->window.frame_start, ext->window.frame_end, + ext->window.frame_start_n, sorted_idx, part_offsets[p], part_offsets[p + 1], result_vecs, is_f64); } diff --git a/src/vec/vec.c b/src/vec/vec.c index 123767854..158fae6e3 100644 --- a/src/vec/vec.c +++ b/src/vec/vec.c @@ -103,6 +103,9 @@ static inline bool vec_any_nulls(const ray_t* v) { * reads them. Detect rc>1 and copy the saved pointers via * ray_index_retain_saved instead of moving them out. */ static inline void vec_drop_index_inplace(ray_t* v) { + /* Any payload/length mutation invalidates the physical-order promise, + * even when no accelerator index is attached. */ + v->attrs &= (uint8_t)~RAY_ATTR_SORTED; if (!(v->attrs & RAY_ATTR_HAS_INDEX)) return; ray_t* idx = v->index; ray_index_t* ix = ray_index_payload(idx); diff --git a/test/rfl/collection/time_series_prefix.rfl b/test/rfl/collection/time_series_prefix.rfl index 9b87448c2..079af346f 100644 --- a/test/rfl/collection/time_series_prefix.rfl +++ b/test/rfl/collection/time_series_prefix.rfl @@ -7,6 +7,17 @@ (fills (as 'F64 (list 0Nf 1.5 0Nf 3.0))) -- (as 'F64 (list 0Nf 1.5 1.5 3.0)) (fills [1 2 3]) -- [1 2 3] +;; Tables are filled column-wise. This is the shape produced by pivot: the +;; index column is already dense while sparse value columns carry forward +;; independently. +(set FT (table [t a b] (list [1 2 3 4] (as 'F64 (list 10.0 0Nf 30.0 0Nf)) (as 'I64 (list 0N 20 0N 40))))) +(set FF (fills FT)) +(key FF) -- [t a b] +(at FF 't) -- [1 2 3 4] +(at FF 'a) -- (as 'F64 [10.0 10.0 30.0 30.0]) +(at FF 'b) -- (as 'I64 (list 0N 20 20 40)) +(type FF) -- 'TABLE + ;; Running aggregates skip null payloads. sum/prod use aggregate identities; ;; avg/min/max stay null until the first non-null value. (sums [1 2 3 4]) -- [1 3 6 10] diff --git a/test/rfl/integration/slice_group.rfl b/test/rfl/integration/slice_group.rfl index 99a4252db..30397c638 100644 --- a/test/rfl/integration/slice_group.rfl +++ b/test/rfl/integration/slice_group.rfl @@ -22,6 +22,9 @@ ;; Sorted twin (contiguous slices — the streaming fast loops). (set TSRT (xasc TP ['s 'v])) (set TSI (table [s v f w] (list (.attr.set 'grouped (at TSRT 's)) (at TSRT 'v) (at TSRT 'f) (at TSRT 'w)))) +;; Same contiguous layout backed by the part index. This must feed the same +;; slice aggregation kernel without expanding each partition into row ids. +(set TPI (table [s v f w] (list (.attr.set 'parted (at TSRT 's)) (at TSRT 'v) (at TSRT 'f) (at TSRT 'w)))) ;; group aa sees v ∈ {0,8,…,3992}: count=500, sum=998000; bb: {1,9,…}: 998500 ;; ── IN set: sums/counts/avg, indexed vs plain vs hand values ── @@ -72,8 +75,11 @@ ;; incl. arith-of-aggs decomposition wrappers ── (set Q3I (select {wp: (/ (sum (* w f)) (as 'F64 (sum w))) from: TI by: s where: (in s ['aa 'bb 'cc 'dd])})) (set Q3P (select {wp: (/ (sum (* w f)) (as 'F64 (sum w))) from: TP by: s where: (in s ['aa 'bb 'cc 'dd])})) +(set Q3PI (select {wp: (/ (sum (* w f)) (as 'F64 (sum w))) from: TPI by: s where: (in s ['aa 'bb 'cc 'dd])})) (min (== (at Q3I 'wp) (at Q3P 'wp))) -- true (min (== (at Q3I 's) (at Q3P 's))) -- true +(min (== (at Q3PI 'wp) (at Q3P 'wp))) -- true +(min (== (at Q3PI 's) (at Q3P 's))) -- true ;; ── eq single key ── (set Q4I (select {sv: (sum v) from: TI by: s where: (== s 'ee)})) diff --git a/test/rfl/query/window.rfl b/test/rfl/query/window.rfl index 0b025a392..1cf618ef4 100644 --- a/test/rfl/query/window.rfl +++ b/test/rfl/query/window.rfl @@ -29,6 +29,16 @@ ;; full result in original row order: [7 2 16 4 8 15] (at (window {from: T part: [k] order: [v] frame: 'running funcs: {cs: (sum v)}}) 'cs) -- [7 2 16 4 8 15] +;; ── fixed trailing frame → resets at each partition boundary ──────────── +;; A positive integer frame is the number of rows including the current row. +(set M (table [k o v] (list ['a 'a 'b 'a 'b 'a] [1 2 1 3 2 4] [10 20 100 30 200 40]))) +(set MW (window {from: M part: [k] order: [o] frame: 3 funcs: {ma: (avg v) ms: (sum v) mn: (min v) mx: (max v) n: (count v)}})) +(at MW 'ma) -- [10.0 15.0 100.0 20.0 150.0 30.0] +(at MW 'ms) -- [10 30 100 60 300 90] +(at MW 'mn) -- [10 10 100 10 100 20] +(at MW 'mx) -- [10 20 100 30 200 40] +(at MW 'n) -- [1 2 1 3 2 3] + ;; ── fby shape: rows equal to their group's min (keeps ties) ───────────────── ;; min over k: a→2 (only row v=2), b→4 (rows v=4,4 both) → 3 rows total (count (select {from: (window {from: T part: [k] funcs: {mn: (min v)}}) where: (== v mn)})) -- 3 @@ -43,6 +53,8 @@ (window {from: T funcs: {x: (median v)}}) !- domain ;; (b) bad frame symbol (window {from: T frame: 'bogus funcs: {mn: (min v)}}) !- domain +;; (b2) trailing row count must be positive +(window {from: T frame: 0 funcs: {mn: (min v)}}) !- range ;; (c) first without order: (window {from: T part: [k] funcs: {f: (first v)}}) !- domain ;; (d) missing funcs: diff --git a/test/test_idx_route.c b/test/test_idx_route.c index 69dbe1ba3..ed205f804 100644 --- a/test/test_idx_route.c +++ b/test/test_idx_route.c @@ -105,6 +105,27 @@ static int attach_hash_to_col(ray_t* tbl, const char* name) { return 0; } +/* Attach a part index using the same retain/write-back cycle. */ +static int attach_part_to_col(ray_t* tbl, const char* name) { + int64_t sym_id = ray_sym_intern(name, (int64_t)strlen(name)); + int64_t ncols = ray_table_ncols(tbl); + int64_t slot = -1; + for (int64_t i = 0; i < ncols; i++) { + if (ray_table_col_name(tbl, i) == sym_id) { slot = i; break; } + } + if (slot < 0) return -1; + + ray_t* col = ray_table_get_col_idx(tbl, slot); + if (!col || RAY_IS_ERR(col)) return -1; + ray_t* w = col; + ray_retain(w); + ray_t* r = ray_index_attach_part(&w); + if (!r || RAY_IS_ERR(r)) { ray_release(w); return -1; } + ray_table_set_col_idx(tbl, slot, w); + ray_release(w); + return 0; +} + /* Attach a zone index to the named column in `tbl`. * Same retain/write-back/release cycle as attach_hash_to_col: * ray_table_set_col_idx retains `w` before releasing the old pointer, @@ -1083,6 +1104,120 @@ static test_result_t test_range_sorted_all_segments(void) { PASS(); } +/* sorted_marker_dense_range: physically sorted k=i with only the verified + * sorted marker (no sort index). (>= k 100000) selects 62% of the table — + * deliberately far beyond the shuffled sort-index selectivity guard. The + * direct path must still hit because the result is one contiguous row span. */ +static ray_op_t* pred_ge_100000(ray_graph_t* g) { + return ray_ge(g, ray_scan(g, "k"), ray_const_i64(g, 100000)); +} + +static test_result_t test_sorted_marker_dense_range(void) { + ray_heap_init(); + (void)ray_sym_init(); + + const int64_t N = 262144; + ray_t* kv = ray_vec_new(RAY_I64, N); + ray_t* vv = ray_vec_new(RAY_I64, N); + TEST_ASSERT_FALSE(RAY_IS_ERR(kv)); + TEST_ASSERT_FALSE(RAY_IS_ERR(vv)); + kv->len = vv->len = N; + int64_t* kd = (int64_t*)ray_data(kv); + int64_t* vd = (int64_t*)ray_data(vv); + for (int64_t i = 0; i < N; i++) kd[i] = vd[i] = i; + kv->attrs |= RAY_ATTR_SORTED; /* fixture is verified by construction */ + + ray_t* tbl = ray_table_new(2); + tbl = ray_table_add_col(tbl, ray_sym_intern("k", 1), kv); + tbl = ray_table_add_col(tbl, ray_sym_intern("v", 1), vv); + ray_release(kv); + ray_release(vv); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl)); + TEST_ASSERT_EQ_I((int)ray_index_kind(ray_table_get_col(tbl, + ray_sym_intern("k", 1))), RAY_IDX_NONE); + + uint64_t cons_before = ray_idx_consults[IDX_SITE_FILTER_RANGE]; + uint64_t hits_before = ray_idx_hits[IDX_SITE_FILTER_RANGE]; + ray_t* r = run_filter(tbl, pred_ge_100000); + TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_EQ_I(ray_table_nrows(r), N - 100000); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_FILTER_RANGE] - cons_before), 1); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_FILTER_RANGE] - hits_before), 1); + + ray_t* rv = ray_table_get_col(r, ray_sym_intern("v", 1)); + TEST_ASSERT_FALSE(!rv || RAY_IS_ERR(rv)); + TEST_ASSERT_EQ_I(ray_vec_get_i64(rv, 0), 100000); + TEST_ASSERT_EQ_I(ray_vec_get_i64(rv, ray_len(rv) - 1), N - 1); + + ray_release(r); + ray_release(tbl); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* Two-sided ranges are split into chained FILTER nodes by the optimizer. + * Both bounds must route and intersect as rowsels; the outer bound must not + * fall back to scanning the inner selection. */ +static test_result_t test_sorted_marker_chained_range(void) { + ray_heap_init(); + (void)ray_sym_init(); + + const int64_t N = 262144; + ray_t* kv = ray_vec_new(RAY_I64, N); + ray_t* vv = ray_vec_new(RAY_I64, N); + TEST_ASSERT_FALSE(RAY_IS_ERR(kv)); + TEST_ASSERT_FALSE(RAY_IS_ERR(vv)); + kv->len = vv->len = N; + int64_t* kd = (int64_t*)ray_data(kv); + int64_t* vd = (int64_t*)ray_data(vv); + for (int64_t i = 0; i < N; i++) kd[i] = vd[i] = i; + kv->attrs |= RAY_ATTR_SORTED; + ray_t* tbl = ray_table_new(2); + tbl = ray_table_add_col(tbl, ray_sym_intern("k", 1), kv); + tbl = ray_table_add_col(tbl, ray_sym_intern("v", 1), vv); + ray_release(kv); ray_release(vv); + + ray_graph_t* g = ray_graph_new(tbl); + ray_op_t* base = ray_const_table(g, tbl); + ray_op_t* ge = ray_ge(g, ray_scan(g, "k"), ray_const_i64(g, 100000)); + ray_op_t* inner = ray_filter(g, base, ge); + ray_op_t* lt = ray_lt(g, ray_scan(g, "k"), ray_const_i64(g, 100100)); + ray_op_t* outer = ray_filter(g, inner, lt); + uint64_t hits_before = ray_idx_hits[IDX_SITE_FILTER_RANGE]; + ray_t* r = ray_execute(g, outer); + TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 100); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_FILTER_RANGE] - hits_before), 2); + ray_t* rv = ray_table_get_col(r, ray_sym_intern("v", 1)); + TEST_ASSERT_EQ_I(ray_vec_get_i64(rv, 0), 100000); + TEST_ASSERT_EQ_I(ray_vec_get_i64(rv, 99), 100099); + + ray_release(r); + ray_graph_free(g); + ray_release(tbl); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* The direct route trusts RAY_ATTR_SORTED, so every payload mutation must + * clear it even when the vector has no attached accelerator index. */ +static test_result_t test_sorted_marker_mutation_invalidates(void) { + ray_heap_init(); + int64_t data[] = { 1, 2, 3 }; + ray_t* v = ray_vec_from_raw(RAY_I64, data, 3); + TEST_ASSERT_FALSE(RAY_IS_ERR(v)); + v->attrs |= RAY_ATTR_SORTED; + int64_t replacement = 0; + v = ray_vec_set(v, 1, &replacement); + TEST_ASSERT_FALSE(RAY_IS_ERR(v)); + TEST_ASSERT_FALSE(ray_attr_is_sorted(v)); + ray_release(v); + ray_heap_destroy(); + PASS(); +} + /* (== k 7) on the dense-dup fixture below. */ static ray_op_t* pred_eq_7_dups(ray_graph_t* g) { return ray_eq(g, ray_scan(g, "k"), ray_const_i64(g, 7)); @@ -1175,6 +1310,69 @@ static ray_op_t* pred_in_100_5(ray_graph_t* g) { return ray_in(g, ray_scan(g, "k"), set_op); } +static ray_t* make_part_sym_table(void) { + (void)ray_sym_init(); + const int64_t n = 4096; + int64_t block_ids[] = { + ray_sym_intern("part_z", 6), + ray_sym_intern("part_a", 6), + ray_sym_intern("part_m", 6), + ray_sym_intern("part_b", 6) + }; + int64_t ends[] = {700, 2000, 2900, 4096}; + ray_t* syms = ray_sym_vec_new(RAY_SYM_W64, n); + ray_t* vals = ray_vec_new(RAY_I64, n); + if (!syms || !vals || RAY_IS_ERR(syms) || RAY_IS_ERR(vals)) { + if (syms && !RAY_IS_ERR(syms)) ray_release(syms); + if (vals && !RAY_IS_ERR(vals)) ray_release(vals); + return ray_error("oom", NULL); + } + syms->len = vals->len = n; + int64_t p = 0; + for (int64_t i = 0; i < n; i++) { + while (i >= ends[p]) p++; + ray_write_sym(ray_data(syms), i, (uint64_t)block_ids[p], + RAY_SYM, syms->attrs); + ((int64_t*)ray_data(vals))[i] = i; + } + ray_t* tbl = ray_table_new(2); + tbl = ray_table_add_col(tbl, ray_sym_intern("sym", 3), syms); + tbl = ray_table_add_col(tbl, ray_sym_intern("v", 1), vals); + ray_release(syms); + ray_release(vals); + return tbl; +} + +/* Select two non-adjacent, dense partitions plus an absent symbol. */ +static ray_op_t* pred_in_part_syms(ray_graph_t* g) { + int64_t ids[] = { + ray_sym_intern("part_a", 6), + ray_sym_intern("part_b", 6), + ray_sym_intern("part_absent", 11) + }; + ray_t* set = ray_sym_vec_new(RAY_SYM_W64, 3); + if (!set || RAY_IS_ERR(set)) return NULL; + for (int64_t i = 0; i < 3; i++) + set = ray_vec_append(set, &ids[i]); + ray_op_t* set_op = ray_const_vec(g, set); + ray_release(set); + return ray_in(g, ray_scan(g, "sym"), set_op); +} + +static ray_op_t* pred_eq_part_a(ray_graph_t* g) { + ray_t* lit = ray_sym(ray_sym_intern("part_a", 6)); + ray_op_t* c = ray_const_atom(g, lit); + ray_release(lit); + return ray_eq(g, ray_scan(g, "sym"), c); +} + +static ray_op_t* pred_eq_part_absent(ray_graph_t* g) { + ray_t* lit = ray_sym(ray_sym_intern("part_absent", 11)); + ray_op_t* c = ray_const_atom(g, lit); + ray_release(lit); + return ray_eq(g, ray_scan(g, "sym"), c); +} + /* ─── IN tests ───────────────────────────────────────────────────── */ /* in_hash: hash on k; FILTER(IN(k, [9 5])) → 4 rows; IDX_SITE_IN @@ -1212,6 +1410,96 @@ static test_result_t test_in_hash(void) { PASS(); } +/* A part index should keep dense unions as physical spans. The chosen + * blocks select 2496/4096 rows, which deliberately exceeds the hash route's + * density guard, and exercise both full and partial morsels. */ +static test_result_t test_in_part_sym_dense(void) { + ray_heap_init(); + ray_t* tbl_a = make_part_sym_table(); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_a)); + TEST_ASSERT(attach_part_to_col(tbl_a, "sym") == 0, + "attach part (in_part_sym_dense)"); + + uint64_t cons_before = ray_idx_consults[IDX_SITE_IN]; + uint64_t hits_before = ray_idx_hits[IDX_SITE_IN]; + ray_t* ra = run_filter(tbl_a, pred_in_part_syms); + TEST_ASSERT_FALSE(RAY_IS_ERR(ra)); + TEST_ASSERT_EQ_I(ray_table_nrows(ra), 2496); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_IN] - cons_before), 1); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_IN] - hits_before), 1); + + ray_t* tbl_b = make_part_sym_table(); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_b)); + ray_t* rb = run_filter(tbl_b, pred_in_part_syms); + TEST_ASSERT_FALSE(RAY_IS_ERR(rb)); + TEST_ASSERT_EQ_I(ray_table_nrows(rb), 2496); + TEST_ASSERT(v_cols_equal(ra, rb), + "v column mismatch between parted IN and scan"); + + ray_release(ra); ray_release(rb); + ray_release(tbl_a); ray_release(tbl_b); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* Equality on a dense part must use its physical span directly rather than + * tripping the hash route's density guard. part_a occupies rows [700,2000). */ +static test_result_t test_eq_part_sym_dense(void) { + ray_heap_init(); + ray_t* tbl_a = make_part_sym_table(); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_a)); + TEST_ASSERT(attach_part_to_col(tbl_a, "sym") == 0, + "attach part (eq_part_sym_dense)"); + + uint64_t cons_before = ray_idx_consults[IDX_SITE_FILTER_PART]; + uint64_t hits_before = ray_idx_hits[IDX_SITE_FILTER_PART]; + ray_t* ra = run_filter(tbl_a, pred_eq_part_a); + TEST_ASSERT_FALSE(RAY_IS_ERR(ra)); + TEST_ASSERT_EQ_I(ray_table_nrows(ra), 1300); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_FILTER_PART] - cons_before), 1); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_FILTER_PART] - hits_before), 1); + + ray_t* tbl_b = make_part_sym_table(); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_b)); + ray_t* rb = run_filter(tbl_b, pred_eq_part_a); + TEST_ASSERT_FALSE(RAY_IS_ERR(rb)); + TEST_ASSERT_EQ_I(ray_table_nrows(rb), 1300); + TEST_ASSERT(v_cols_equal(ra, rb), + "v column mismatch between parted EQ and scan"); + + ray_release(ra); ray_release(rb); + ray_release(tbl_a); ray_release(tbl_b); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* A symbol can exist in the shared domain without having a physical part. + * That is still a successful index proof yielding a valid empty rowsel. */ +static test_result_t test_eq_part_sym_absent(void) { + ray_heap_init(); + ray_t* tbl = make_part_sym_table(); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl)); + (void)ray_sym_intern("part_absent", 11); + TEST_ASSERT(attach_part_to_col(tbl, "sym") == 0, + "attach part (eq_part_sym_absent)"); + + uint64_t cons_before = ray_idx_consults[IDX_SITE_FILTER_PART]; + uint64_t hits_before = ray_idx_hits[IDX_SITE_FILTER_PART]; + ray_t* r = run_filter(tbl, pred_eq_part_absent); + TEST_ASSERT_FALSE(RAY_IS_ERR(r)); + TEST_ASSERT_EQ_I(ray_table_nrows(r), 0); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_FILTER_PART] - cons_before), 1); + TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_FILTER_PART] - hits_before), 1); + + ray_release(r); + ray_release(tbl); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + /* in_dup_set: set [9 9 5] → same 4 rows, no duplicate row ids. */ static test_result_t test_in_dup_set(void) { ray_heap_init(); @@ -2542,8 +2830,14 @@ const test_entry_t idx_route_entries[] = { { "idx_route/range_guard", test_range_guard, NULL, NULL }, { "idx_route/range_f64", test_range_f64, NULL, NULL }, { "idx_route/range_sorted_all_segments", test_range_sorted_all_segments, NULL, NULL }, + { "idx_route/sorted_marker_dense_range", test_sorted_marker_dense_range, NULL, NULL }, + { "idx_route/sorted_marker_chained_range", test_sorted_marker_chained_range, NULL, NULL }, + { "idx_route/sorted_marker_mutation_invalidates", test_sorted_marker_mutation_invalidates, NULL, NULL }, { "idx_route/hash_eq_dense_dups", test_hash_eq_dense_dups, NULL, NULL }, { "idx_route/in_hash", test_in_hash, NULL, NULL }, + { "idx_route/in_part_sym_dense", test_in_part_sym_dense, NULL, NULL }, + { "idx_route/eq_part_sym_dense", test_eq_part_sym_dense, NULL, NULL }, + { "idx_route/eq_part_sym_absent", test_eq_part_sym_absent, NULL, NULL }, { "idx_route/in_dup_set", test_in_dup_set, NULL, NULL }, { "idx_route/in_absent_elems", test_in_absent_elems, NULL, NULL }, { "idx_route/in_float_col_falls_back", test_in_float_col_falls_back, NULL, NULL }, diff --git a/test/test_window.c b/test/test_window.c index ef684588c..2965ba9e5 100644 --- a/test/test_window.c +++ b/test/test_window.c @@ -4196,13 +4196,9 @@ static test_result_t test_window_i32_plain_order_key(void) { PASS(); } -/* Review 2.10 sub-item 4: an unsupported window frame must be a LOUD error, - * not silently degraded to the running frame. The compute path honors only - * the whole partition (UNBOUNDED PRECEDING .. UNBOUNDED FOLLOWING) and the - * running frame (ROWS, UNBOUNDED PRECEDING .. CURRENT ROW). Any other frame - * (N_PRECEDING / N_FOLLOWING bounds, RANGE-mode CURRENT ROW, a non-unbounded - * start) was previously accepted and IGNORED — yielding the wrong window. - * exec_window now rejects these. */ +/* Unsupported frames must be a LOUD error, not silently degraded to the + * running frame. Whole, running, and fixed trailing ROWS frames are honored; + * other N_FOLLOWING/RANGE/non-trailing shapes remain unsupported. */ static test_result_t test_window_unsupported_frame_errors(void) { ray_heap_init(); (void)ray_sym_init(); @@ -4220,8 +4216,6 @@ static test_result_t test_window_unsupported_frame_errors(void) { /* Helper: run a SUM window with the given frame and expect an error. */ struct { uint8_t ftype, fstart, fend; int64_t sn, en; } bad[] = { - /* ROWS BETWEEN 1 PRECEDING AND CURRENT ROW — sliding, NOT honored */ - { RAY_FRAME_ROWS, RAY_BOUND_N_PRECEDING, RAY_BOUND_CURRENT_ROW, 1, 0 }, /* ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING — NOT honored */ { RAY_FRAME_ROWS, RAY_BOUND_UNBOUNDED_PRECEDING, RAY_BOUND_N_FOLLOWING, 0, 1 }, /* RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW — RANGE not honored */ @@ -4247,7 +4241,33 @@ static test_result_t test_window_unsupported_frame_errors(void) { ray_release(result); ray_graph_free(g); } - /* Sanity: the two HONORED frames still succeed. */ + /* ROWS BETWEEN 1 PRECEDING AND CURRENT ROW is a real sliding frame. */ + { + ray_graph_t* g = ray_graph_new(tbl); + ray_op_t* tbl_op = ray_const_table(g, tbl); + ray_op_t* g_op = ray_scan(g, "g"); + ray_op_t* v_op = ray_scan(g, "v"); + ray_op_t* parts[] = { g_op }; + uint8_t kinds[] = { RAY_WIN_SUM }; + ray_op_t* fins[] = { v_op }; + int64_t params[] = { 0 }; + ray_op_t* w = ray_window_op(g, tbl_op, parts, 1, NULL, NULL, 0, + kinds, fins, params, 1, RAY_FRAME_ROWS, + RAY_BOUND_N_PRECEDING, + RAY_BOUND_CURRENT_ROW, 1, 0); + ray_t* result = ray_execute(g, w); + TEST_ASSERT_FALSE(RAY_IS_ERR(result)); + ray_t* sum_col = ray_table_get_col_idx(result, 2); + TEST_ASSERT_NOT_NULL(sum_col); + int64_t* sums = (int64_t*)ray_data(sum_col); + TEST_ASSERT_EQ_I(sums[0], 10); + TEST_ASSERT_EQ_I(sums[1], 30); + TEST_ASSERT_EQ_I(sums[2], 50); + TEST_ASSERT_EQ_I(sums[3], 70); + ray_release(result); ray_graph_free(g); + } + + /* Sanity: the running frame still succeeds. */ { ray_graph_t* g = ray_graph_new(tbl); ray_op_t* tbl_op = ray_const_table(g, tbl); From 6b2182b78d4b95064516ea9f2f54d2bfb784dd11 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 11:16:47 +0200 Subject: [PATCH 04/10] fix(audit): restore linear running windows --- docs/docs/namespaces/mem.md | 2 +- src/ops/rowsel.c | 34 ++++++++++++------- src/ops/system.c | 4 +-- src/ops/window.c | 68 ++++++++++++++++++++++++++----------- 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/docs/docs/namespaces/mem.md b/docs/docs/namespaces/mem.md index 38dfc8f0c..f86d49a93 100644 --- a/docs/docs/namespaces/mem.md +++ b/docs/docs/namespaces/mem.md @@ -50,7 +50,7 @@ The returned dictionary has these fields: | `allocated-bytes` | Sum of allocator block bytes obtained during the scope, including temporary allocations. | | `freed-bytes` | Sum of allocator block bytes released before the measured evaluation returned. | | `net-bytes` | Allocated minus freed bytes at the measurement boundary; it may be negative when the expression releases pre-existing values. | -| `peak-live-bytes` | Maximum positive live-byte delta above the starting boundary. This is the closest analogue of q's `.Q.ts` space value. | +| `peak-live-bytes` | Maximum positive live-byte delta above the starting boundary. | | `result-bytes` | Logical `.mem.objsize` of `result`. | | `alloc-count` | Number of allocator blocks obtained. | | `free-count` | Number of allocator blocks released. | diff --git a/src/ops/rowsel.c b/src/ops/rowsel.c index 42404a7ac..321e8d709 100644 --- a/src/ops/rowsel.c +++ b/src/ops/rowsel.c @@ -652,12 +652,29 @@ ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b) { src = ai; begin = ao[s]; end = ao[s + 1]; } - uint32_t start = cum; if (src) { - for (uint32_t p = begin; p < end; p++) oi[cum++] = src[p]; + uint32_t pc = end - begin; + if ((int64_t)pc == seg_len) { + of[s] = RAY_SEL_ALL; + } else { + for (uint32_t p = begin; p < end; p++) oi[cum++] = src[p]; + of[s] = pc == 0 ? RAY_SEL_NONE : RAY_SEL_MIX; + } } else { uint32_t ap = ao[s], ae = ao[s + 1]; uint32_t bp = bo[s], be = bo[s + 1]; + uint32_t count = 0; + uint32_t cp = ap, dp = bp; + while (cp < ae && dp < be) { + uint16_t av = ai[cp], bv = bi[dp]; + if (av < bv) cp++; + else if (bv < av) dp++; + else { count++; cp++; dp++; } + } + if ((int64_t)count == seg_len) { + of[s] = RAY_SEL_ALL; + continue; + } while (ap < ae && bp < be) { uint16_t av = ai[ap], bv = bi[bp]; if (av < bv) ap++; @@ -665,17 +682,8 @@ ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b) { else { oi[cum++] = av; ap++; bp++; } } } - uint32_t pc = cum - start; - if (pc == 0) { - of[s] = RAY_SEL_NONE; - } else if ((int64_t)pc == seg_len) { - /* This cannot arise from MIX input under canonical encoding, - * but keep the representation valid defensively. */ - of[s] = RAY_SEL_ALL; - cum = start; - } else { - of[s] = RAY_SEL_MIX; - } + if (of[s] != RAY_SEL_ALL) + of[s] = cum == oo[s] ? RAY_SEL_NONE : RAY_SEL_MIX; } } oo[ns] = cum; diff --git a/src/ops/system.c b/src/ops/system.c index 516724ac9..4d0751d39 100644 --- a/src/ops/system.c +++ b/src/ops/system.c @@ -488,8 +488,8 @@ ray_t* ray_meta_fn(ray_t* x) { * every reachable ray_t header and its live payload, with shared children * counted once. It deliberately does not report buddy-block slack, global * symbol dictionaries, or opaque native handles (graphs/HNSW/lazy plans). - * That makes the result stable across allocator tuning and useful as the - * Rayforce analogue of q's `.mem.objsize` for materialised query results. + * That makes the result stable across allocator tuning and useful for + * inspecting materialised query results. * * The traversal is iterative and pointer-deduplicated. Besides preventing * double-counting shared table columns, the visited set makes it safe for any diff --git a/src/ops/window.c b/src/ops/window.c index fb132935f..df4612af4 100644 --- a/src/ops/window.c +++ b/src/ops/window.c @@ -322,19 +322,33 @@ static void win_compute_partition( else win_set_null(rvec, sorted_idx[i]); } } else { - for (int64_t i = ps; i < pe; i++) { - int64_t lo = trailing ? i - frame_start_n : ps; - if (lo < ps) lo = ps; + if (!trailing) { double best = want_max ? -DBL_MAX : DBL_MAX; int found = 0; - for (int64_t j = lo; j <= i; j++) { - int64_t row = sorted_idx[j]; - if (ray_vec_is_null(fvec, row)) continue; - double v = win_read_f64(fvec, row); - if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + for (int64_t i = ps; i < pe; i++) { + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) { + double v = win_read_f64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + } + if (found) out[row] = best; + else win_set_null(rvec, row); + } + } else { + for (int64_t i = ps; i < pe; i++) { + int64_t lo = i - frame_start_n; + if (lo < ps) lo = ps; + double best = want_max ? -DBL_MAX : DBL_MAX; + int found = 0; + for (int64_t j = lo; j <= i; j++) { + int64_t row = sorted_idx[j]; + if (ray_vec_is_null(fvec, row)) continue; + double v = win_read_f64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + } + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } - if (found) out[sorted_idx[i]] = best; - else win_set_null(rvec, sorted_idx[i]); } } } else { @@ -353,19 +367,33 @@ static void win_compute_partition( else win_set_null(rvec, sorted_idx[i]); } } else { - for (int64_t i = ps; i < pe; i++) { - int64_t lo = trailing ? i - frame_start_n : ps; - if (lo < ps) lo = ps; + if (!trailing) { int64_t best = want_max ? INT64_MIN : INT64_MAX; int found = 0; - for (int64_t j = lo; j <= i; j++) { - int64_t row = sorted_idx[j]; - if (ray_vec_is_null(fvec, row)) continue; - int64_t v = win_read_i64(fvec, row); - if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + for (int64_t i = ps; i < pe; i++) { + int64_t row = sorted_idx[i]; + if (!ray_vec_is_null(fvec, row)) { + int64_t v = win_read_i64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + } + if (found) out[row] = best; + else win_set_null(rvec, row); + } + } else { + for (int64_t i = ps; i < pe; i++) { + int64_t lo = i - frame_start_n; + if (lo < ps) lo = ps; + int64_t best = want_max ? INT64_MIN : INT64_MAX; + int found = 0; + for (int64_t j = lo; j <= i; j++) { + int64_t row = sorted_idx[j]; + if (ray_vec_is_null(fvec, row)) continue; + int64_t v = win_read_i64(fvec, row); + if (!found || (want_max ? v > best : v < best)) { best = v; found = 1; } + } + if (found) out[sorted_idx[i]] = best; + else win_set_null(rvec, sorted_idx[i]); } - if (found) out[sorted_idx[i]] = best; - else win_set_null(rvec, sorted_idx[i]); } } } From e0ffa67a3d0520882a2fbcae5bae5f58c2898555 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 15:31:34 +0200 Subject: [PATCH 05/10] perf(aggr): remove fixed grouping limits --- src/ops/agg_engine.c | 489 ++++++++++----------------- src/ops/agg_engine.h | 11 +- src/ops/group.c | 728 +++++++++++++++++++++++------------------ src/ops/internal.h | 6 +- test/test_agg_engine.c | 81 +++-- 5 files changed, 628 insertions(+), 687 deletions(-) diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index 34b4bf447..c4c4091d7 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -85,7 +85,8 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { if (!in || in->opcode != OP_SCAN) return false; ray_op_ext_t* ie = find_ext(g, in->id); ray_t* ic = ie ? ray_table_get_col(tbl, ie->sym) : NULL; - if (!ic || !agg_resolve(ext->agg_ops[a], ic->type)) return false; + const agg_vtable_t* vt = ic ? agg_resolve(ext->agg_ops[a], ic->type) : NULL; + if (!vt || vt->kind != ACC_STREAMING) return false; continue; /* admitted */ } if (ext->agg_ins2 && ext->agg_ins2[a] != RAY_OP_NONE) { @@ -110,7 +111,8 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { ray_op_ext_t* ie = find_ext(g, in->id); ray_t* ic = (ie) ? ray_table_get_col(tbl, ie->sym) : NULL; if (!ic) return false; - if (!agg_resolve(ext->agg_ops[a], ic->type)) return false; + const agg_vtable_t* vt = agg_resolve(ext->agg_ops[a], ic->type); + if (!vt || vt->kind != ACC_STREAMING) return false; } return true; } @@ -119,11 +121,10 @@ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * Decides whether the key tuple packs into a bounded direct-index slot space * (gid = sum_k (key_k - min_k)*strides[k]) so grouping can skip hashing. * Eligible iff: 1..16 keys, every key is an integer/temporal/SYM type with no - * nulls, and the product of per-key ranges stays at or below DENSE_MAX_SLOTS - * (computed overflow-safely). Performs one min/max prescan per key column. + * nulls, and the product of per-key ranges is no larger than the input row + * count (computed overflow-safely). Performs one min/max prescan per key column. * Aggregates may be ACC_STREAMING or ACC_BUFFERED (median/top-k): the dense - * serial driver (agg_run_one) and the dense parallel path both carry the - * per-group destroy lifecycle, so buffered state is handled. */ + * serial driver carries the per-group destroy lifecycle for buffered state. */ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, const agg_vtable_t** vts, uint32_t n_aggs, int64_t nrows, dense_plan_t* out) { @@ -149,15 +150,21 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, if (kc->attrs & RAY_ATTR_HAS_NULLS) return false; if (nrows <= 0) return false; /* empty → no min/max, maxtype == RAY_SYM) { switch (kc->attrs & RAY_SYM_W_MASK) { - case RAY_SYM_W8: out->mins[k] = 0; out->ranges[k] = 256; continue; - case RAY_SYM_W16: out->mins[k] = 0; out->ranges[k] = 65536; continue; + case RAY_SYM_W8: + if (nrows >= (int64_t)UINT8_MAX + 1) { + out->mins[k] = 0; out->ranges[k] = (int64_t)UINT8_MAX + 1; continue; + } + break; + case RAY_SYM_W16: + if (nrows >= (int64_t)UINT16_MAX + 1) { + out->mins[k] = 0; out->ranges[k] = (int64_t)UINT16_MAX + 1; continue; + } + break; default: break; /* W32/W64: try domain bounds, else prescan */ } /* W32/W64 SYM codes are positions in [0, domain_count). When the @@ -167,7 +174,7 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, * domains still scan: their actual used range may be far tighter * than the full count, and the array would otherwise blow the cap. */ int64_t dc = ray_sym_domain_count(ray_sym_vec_domain(kc)); - if (dc > 0 && dc <= 65536 && dc <= nrows) { + if (dc > 0 && dc <= nrows) { out->mins[k] = 0; out->ranges[k] = dc; continue; } } @@ -210,17 +217,17 @@ bool agg_dense_plan(ray_t** key_cols, uint32_t n_keys, out->ranges[k] = mx - mn + 1; } - /* Composite packing; bail if a range is non-positive or the product would - * exceed the cap. Overflow-safe: check before multiplying. */ + /* Composite packing; keep dense state O(input). */ int64_t total = 1; + int64_t dense_limit = nrows < (int64_t)UINT32_MAX + ? nrows : (int64_t)UINT32_MAX; for (uint32_t k = 0; k < n_keys; k++) { int64_t rng = out->ranges[k]; if (rng <= 0) return false; - if (total > DENSE_MAX_SLOTS / rng) return false; /* would exceed cap */ + if (total > dense_limit / rng) return false; out->strides[k] = total; total *= rng; } - if (total > DENSE_MAX_SLOTS) return false; out->total_slots = total; out->ok = true; @@ -690,14 +697,15 @@ static bool agg_dense_plan_sel(ray_t** key_cols, uint32_t n_keys, int64_t n_sel, scratch_free(pre_hdr); int64_t total = 1; + int64_t dense_limit = n_sel < (int64_t)UINT32_MAX + ? n_sel : (int64_t)UINT32_MAX; for (uint32_t k = 0; k < n_keys; k++) { int64_t rng = out->ranges[k]; if (rng <= 0) return false; - if (total > DENSE_MAX_SLOTS / rng) return false; + if (total > dense_limit / rng) return false; out->strides[k] = total; total *= rng; } - if (total > DENSE_MAX_SLOTS) return false; out->total_slots = total; out->ok = true; return true; @@ -806,12 +814,9 @@ typedef struct { agg_local_t* locals; /* [nw] */ } agg_par_ctx_t; -/* (first_row, group idx) pair. The radix path packs (part << 32 | local gid) - * into .idx so finalize can recover both the partition and the in-partition - * state offset for each emitted group; .fr carries the representative row used - * to gather the key columns. Group-by output order is UNSPECIFIED — groups are - * emitted in build order, so there is no sort. */ -typedef struct { int64_t fr; int64_t idx; } agg_fr_pair_t; +/* Radix output order. idx packs (part << 32 | local gid) so finalize can + * recover both the partition and the in-partition state offset. */ +typedef struct { int64_t idx; } agg_radix_order_t; /* Phase A: per-worker local group + accumulate over chunk [start,end). */ static void agg_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { @@ -1408,19 +1413,16 @@ static ray_t* exec_group_v2_parallel_dense( } /* ══════════════════════════════════════════════════════════════════════ - * Parallel SMALL-HASH group-by (high-reduction sparse int/SYM keys). + * Parallel SMALL-HASH fallback for sparse int/SYM keys. * - * Targets the shape the dense path rejects (key range exceeds the dense cap) - * but that still has FEW distinct groups — e.g. 64 groups whose values span - * 0..6.3e9. Routing such a shape to RADIX scatters all N rows into 256 - * partitions for a handful of groups (the ~4.7× "scatter tax"); routing it to - * the generic hash path sizes every per-worker structure to N (htable, states, - * first_row). This path is the missing O(groups)-sized strategy: per-worker - * structures sized to the estimated CARDINALITY, growable on a wrong-low guess. + * This is the allocation-failure fallback for the deterministic radix route. + * Its per-worker structures start at one group and grow from observed + * cardinality, so a sparse, high-reduction input can still complete without a + * sampled cardinality estimate or a cache/RAM-derived routing crossover. * * Streaming aggs only (selector gates ACC_BUFFERED → radix). Per-worker hash: - * open-addressing, next_pow2(4*estimate) with a small floor, GROWABLE (rehash - * on load-factor exceed); states + first_row grow with the hash, never with N. + * open-addressing and growable on load-factor exceed; states + first_row grow + * with the hash, never with N. * Phase A interleaves probe + accumulate in fixed CHUNK_ROWS chunks so the gid * buffer is chunk-sized, not nrows-sized. Phase B merges per-worker hashes into * a global hash (≤ groups×nworkers entries — cheap). Phase C emits in build @@ -1428,92 +1430,6 @@ static ray_t* exec_group_v2_parallel_dense( * row per group, finalize each group's streaming state. * ══════════════════════════════════════════════════════════════════════ */ -/* Cardinality estimate via a bounded sample of the first rows. Inserts key - * tuples into a small open-addressing hash (reusing agg_tuple_hash/agg_tuple_eq - * for byte-consistency with the real grouping) and EARLY-EXITS the moment the - * distinct count exceeds t_route → returns t_route+1 to signal "high card" so - * high-card shapes (q10/S2) pay only the bounded sample, never a full pass. - * On no early-exit, returns the exact distinct count of the sample (capped at - * t_route). sample_rows is clamped to nrows. n_keys is uint32_t (unbounded: - * carries the untruncated ext->n_keys). */ -static int64_t agg_estimate_card(ray_t** key_cols, const void** key_data, - uint32_t n_keys, int64_t nrows, - int64_t sample_rows, int64_t t_route) { - int64_t ns = sample_rows < nrows ? sample_rows : nrows; - if (ns <= 0) return 0; - /* Hash sized to comfortably hold t_route distinct keys at <0.5 load. */ - int64_t htcap = 16; - while (htcap < (t_route + 1) * 4) htcap <<= 1; - uint64_t htmask = (uint64_t)htcap - 1; - int32_t* ht = ray_alloc_raw((size_t)htcap * sizeof(int32_t)); - int64_t* rep = ray_alloc_raw((size_t)(t_route + 2) * sizeof(int64_t)); /* gid -> sample row */ - if (!ht || !rep) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; /* assume high */ } - for (int64_t i = 0; i < htcap; i++) ht[i] = -1; - int64_t distinct = 0; - for (int64_t r = 0; r < ns; r++) { - uint64_t h = agg_tuple_hash(key_cols, key_data, n_keys, r); - uint64_t slot = h & htmask; - for (;;) { - int32_t gp = ht[slot]; - if (gp < 0) { - ht[slot] = (int32_t)distinct; - rep[distinct] = r; - distinct++; - if (distinct > t_route) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - break; - } - if (agg_tuple_eq(key_cols, key_data, n_keys, r, rep[gp])) break; - slot = (slot + 1) & htmask; - } - } - ray_free_raw(ht); ray_free_raw(rep); - return distinct; -} - -/* Sel-mode cardinality estimate: samples the first `sample_rows` SELECTED rows - * (decoded to ORIGINAL indices via the cursor) so the routing reflects the - * cardinality of the rows that actually survive the WHERE — not the full table. - * Same early-exit-above-t_route contract as agg_estimate_card. n_keys is - * uint32_t (unbounded), same contract as agg_estimate_card above. */ -static int64_t agg_estimate_card_sel(ray_t** key_cols, const void** key_data, - uint32_t n_keys, ray_t* sel, - const int64_t* sel_prefix, int64_t n_sel, - int64_t sample_rows, int64_t t_route) { - int64_t ns = sample_rows < n_sel ? sample_rows : n_sel; - if (ns <= 0) return 0; - int64_t htcap = 16; - while (htcap < (t_route + 1) * 4) htcap <<= 1; - uint64_t htmask = (uint64_t)htcap - 1; - int32_t* ht = ray_alloc_raw((size_t)htcap * sizeof(int32_t)); - int64_t* rep = ray_alloc_raw((size_t)(t_route + 2) * sizeof(int64_t)); - if (!ht || !rep) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - for (int64_t i = 0; i < htcap; i++) ht[i] = -1; - int64_t distinct = 0, seen = 0; - int64_t rows[AGG_SEL_CHUNK]; - agg_sel_cursor_t cur; - agg_sel_cursor_init(&cur, sel, sel_prefix, 0, ns); - int64_t cn; - while (seen < ns && (cn = agg_sel_cursor_next(&cur, rows)) > 0) { - for (int64_t i = 0; i < cn && seen < ns; i++, seen++) { - int64_t r = rows[i]; - uint64_t h = agg_tuple_hash(key_cols, key_data, n_keys, r); - uint64_t slot = h & htmask; - for (;;) { - int32_t gp = ht[slot]; - if (gp < 0) { - ht[slot] = (int32_t)distinct; rep[distinct] = r; distinct++; - if (distinct > t_route) { ray_free_raw(ht); ray_free_raw(rep); return t_route + 1; } - break; - } - if (agg_tuple_eq(key_cols, key_data, n_keys, r, rep[gp])) break; - slot = (slot + 1) & htmask; - } - } - } - ray_free_raw(ht); ray_free_raw(rep); - return distinct; -} - /* Per-worker (and global merge) GROWABLE small-hash group table: open-addressing * hash on the tuple-hash → local gid; AoS per-group state blocks of `block` * bytes. Sized to cardinality (not N) and grown by rehash on load-factor. */ @@ -1530,7 +1446,7 @@ typedef struct { static int agg_sh_init(agg_sh_t* sh, int64_t cap, size_t block) { if (cap < 1) cap = 1; - int64_t htcap = 16; + int64_t htcap = 2; while (htcap < cap * 2) htcap <<= 1; /* <0.5 load at full cap */ sh->ng = 0; sh->cap = cap; sh->block = block; sh->oom = 0; sh->htcap = htcap; sh->htmask = (uint64_t)htcap - 1; @@ -1583,15 +1499,6 @@ static int agg_sh_grow_ht(agg_sh_t* sh, ray_t** key_cols, const void** key_data, } #define AGG_SH_CHUNK 4096 - -/* Adaptive promotion: the cardinality estimate is sample-based and can - * underestimate on clustered/ordered data, misrouting a genuinely high-card - * group to this small-hash path where the per-worker hash thrashes cache. - * Guard against it WITHOUT trusting the estimate: when any worker's live group - * count crosses this threshold the per-worker hash no longer fits cache, so we - * abort and re-run on the radix path (data-driven, no sampling bias). Set well - * above the smallhash/radix crossover so medium-card groups are never promoted. */ -#define AGG_SH_PROMOTE (1 << 16) static ray_t* exec_group_v2_parallel_radix( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, @@ -1608,9 +1515,7 @@ typedef struct { uint32_t n_aggs; const void** val_data; const int8_t* val_types; const bool* val_hasnull; const uint8_t* val_esz; const void** val2_data; const int8_t* val2_types; const bool* val2_hasnull; const uint8_t* val2_esz; - int64_t est_card; /* sizing hint for per-worker hashes */ agg_sh_t* locals; /* [nw] */ - _Atomic(int) overflow; /* set when any worker crosses AGG_SH_PROMOTE */ /* Sel-mode (pushed WHERE filter): chunk-iterate selected rows of [0,n_sel). */ ray_t* sel; const int64_t* sel_prefix; @@ -1670,10 +1575,6 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, start, end); int64_t cn; while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) { - /* Adaptive promotion trip-wire: too many distinct keys for cache, or - * another worker already tripped → abort to the radix path. */ - if (sh->ng > AGG_SH_PROMOTE) atomic_store_explicit(&c->overflow, 1, memory_order_relaxed); - if (atomic_load_explicit(&c->overflow, memory_order_relaxed)) { agg_sel_scratch_free(&sc); return; } for (int64_t i = 0; i < cn; i++) { int32_t g = agg_sh_find_or_insert(sh, c->key_cols, c->key_data, c->n_keys, c->vts, c->off, c->n_aggs, rows[i]); @@ -1689,11 +1590,6 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en } for (int64_t cs = start; cs < end; cs += AGG_SH_CHUNK) { - /* Adaptive promotion trip-wire (see AGG_SH_PROMOTE): per-worker hash no - * longer fits cache, or another worker already tripped → abort so the - * caller re-runs on radix. */ - if (sh->ng > AGG_SH_PROMOTE) atomic_store_explicit(&c->overflow, 1, memory_order_relaxed); - if (atomic_load_explicit(&c->overflow, memory_order_relaxed)) return; int64_t ce = cs + AGG_SH_CHUNK; if (ce > end) ce = end; int64_t n = ce - cs; for (int64_t r = cs; r < ce; r++) { @@ -1723,14 +1619,13 @@ static void agg_sh_phaseA_fn(void* vctx, uint32_t wid, int64_t start, int64_t en } } -/* Parallel small-hash path. Precondition: int/SYM keys, all aggs ACC_STREAMING, - * estimated cardinality LOW (caller-gated). est_card sizes the per-worker - * hashes; they grow if the estimate proves too low. */ +/* Parallel small-hash fallback. Precondition: int/SYM keys and streaming + * accumulators. Hashes start at one group and grow from observed cardinality. */ static ray_t* exec_group_v2_parallel_smallhash( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, - int64_t est_card, ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { + ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { ray_op_ext_t* ext = find_ext(g, op->id); uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; ray_pool_t* pool = ray_pool_get(); @@ -1745,18 +1640,11 @@ static ray_t* exec_group_v2_parallel_smallhash( const bool* val2_hasnull = d.val2_hasnull; const uint8_t* val2_esz = d.val2_esz; const int64_t* agg_syms = d.agg_syms; - /* Per-worker hash sized to ~4× the estimated cardinality (floor 256) — the - * cap, not N. Grows by rehash if a worker sees more distinct keys than the - * sample predicted (rare-key tail). Cap is bounded by nrows for safety. */ - int64_t cap0 = est_card * 4; - if (cap0 < 256) cap0 = 256; - if (nrows > 0 && cap0 > nrows) cap0 = nrows; - agg_sh_t* locals = ray_calloc_raw((size_t)((size_t)nw) * (sizeof(agg_sh_t))); if (!locals) { agg_desc_free(&d); return ray_error("oom", NULL); } int alloc_oom = 0; for (uint32_t w = 0; w < nw; w++) - if (agg_sh_init(&locals[w], cap0, block) != 0) { alloc_oom = 1; break; } + if (agg_sh_init(&locals[w], 1, block) != 0) { alloc_oom = 1; break; } if (alloc_oom) { for (uint32_t w = 0; w < nw; w++) agg_sh_destroy(&locals[w]); ray_free_raw(locals); @@ -1771,7 +1659,7 @@ static ray_t* exec_group_v2_parallel_smallhash( .val_hasnull = val_hasnull, .val_esz = val_esz, .val2_data = val2_data, .val2_types = val2_types, .val2_hasnull = val2_hasnull, .val2_esz = val2_esz, - .est_card = est_card, .locals = locals, .overflow = 0, + .locals = locals, .sel = sel, .sel_prefix = sel_prefix, .vd = { .n_aggs = n_aggs, .vts = vts, .off = off, .block = block, .val_data = val_data, .val_types = val_types, .val_hasnull = val_hasnull, .val_esz = val_esz, @@ -1779,17 +1667,6 @@ static ray_t* exec_group_v2_parallel_smallhash( }; ray_pool_dispatch(pool, agg_sh_phaseA_fn, &ctx, sel ? n_sel : nrows); - /* Adaptive promotion: a worker found the live group count too high for the - * per-worker hash (estimate underestimated). Discard the partial small-hash - * state and re-run on the radix path — data-driven, no sampling bias. */ - if (atomic_load_explicit(&ctx.overflow, memory_order_relaxed)) { - for (uint32_t w = 0; w < nw; w++) agg_sh_destroy(&locals[w]); - ray_free_raw(locals); - agg_desc_free(&d); - return exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, - vts, off, block, sel, sel_prefix, n_sel); - } - for (uint32_t w = 0; w < nw; w++) if (locals[w].oom) { for (uint32_t i = 0; i < nw; i++) agg_sh_destroy(&locals[i]); @@ -1874,14 +1751,15 @@ static ray_t* exec_group_v2_parallel_smallhash( /* ══════════════════════════════════════════════════════════════════════ * Parallel RADIX group-by (high-card int/SYM keys, ACC_STREAMING aggs only). * - * Partition rows by key-hash into AGG_RADIX_P disjoint partitions: every row + * Partition rows by key-hash into a worker-derived power-of-two partition set: + * every row * whose tuple-hash maps to partition p lands in p across ALL workers (same key * → same hash → same partition). Partitions therefore hold DISJOINT key sets, * so each can be grouped+accumulated fully in parallel with NO cross-partition * merge — eliminating the serial Phase-B bottleneck of the hash path. * * Phase 1 (parallel over rows): scatter each row's INDEX into a per-(worker, - * partition) growable int64 buffer keyed by RADIX_PART(hash). + * partition) growable int64 buffer keyed by the hash partition. * Phase 2 (parallel over partitions): gather all workers' index buffers for * partition p; build a small open-addressing hash over those rows * (keys disjoint from other partitions); assign partition-local gids @@ -1895,7 +1773,15 @@ static ray_t* exec_group_v2_parallel_smallhash( * Streaming aggs only (selector guarantees it) → no buffered destroy lifecycle. * ══════════════════════════════════════════════════════════════════════ */ -#define AGG_RADIX_P 256 +static uint32_t agg_radix_part_count(uint32_t nworkers, int64_t nrows) { + uint32_t n = 1; + uint64_t rows = nrows > 0 ? (uint64_t)nrows : 1; + while ((n < nworkers || + (uint64_t)n < rows / n + (rows % n != 0)) && + n <= UINT32_MAX / 2) + n <<= 1; + return n; +} /* Growable contiguous PAYLOAD buffer (per worker, per partition). Phase 1 * scatters one fixed-size record per row instead of a bare row index: the @@ -1910,7 +1796,8 @@ typedef struct { char* buf; uint32_t n, cap; } agg_pay_buf_t; /* n = #records * /* Reserve room for one more record of `rec` bytes; returns dest ptr or NULL. */ static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec) { if (b->n == b->cap) { - uint32_t nc = b->cap ? b->cap * 2 : 64; + if (b->cap > UINT32_MAX / 2) return NULL; + uint32_t nc = b->cap ? b->cap * 2 : 1; char* nb = ray_realloc_raw(b->buf, (size_t)nc * rec); if (!nb) return NULL; b->buf = nb; b->cap = nc; @@ -1921,7 +1808,7 @@ static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec) { /* Per-partition result slot (filled by Phase 2). */ typedef struct { char* states; /* [ng * block] AoS group state (every group init'd) */ - int64_t* first_row; /* [ng] MIN row idx per partition-local group */ + int64_t* first_row; /* [ng] MIN logical input position per local group */ int64_t* keys; /* [ng * n_keys] packed keys (build order) for the * representative record of each group — Phase 3 emits * the key columns by un-packing these SEQUENTIALLY @@ -1955,10 +1842,7 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, /* Build a result key column of src_col's type for the radix path by un-packing * the per-group packed key value (column key_idx of each group's representative - * record) SEQUENTIALLY from the per-partition `keys` buffers, in emit order - * (partition-major, then partition-local gid). This replaces the scattered - * gather of the original key column at first_row[gi] — for huge ngroups the - * sequential read is far more cache-friendly. + * record) from the per-partition `keys` buffers in stable first-seen order. * * agg_read_key_i64 widened each key into int64 (sign-extend for signed types, * zero-extend for U8/BOOL/SYM intern ids); write_col_i64 is its exact inverse, @@ -1970,7 +1854,7 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, * agg_v2_can_handle) and key_idx indexes the packed-key stride below. */ static ray_t* agg_unpack_key_col(ray_t* src_col, uint32_t key_idx, uint32_t n_keys, const agg_radix_part_t* parts, uint32_t nparts, - int64_t n) { + const agg_radix_order_t* pairs, int64_t n) { ray_t* out = col_vec_new(src_col, n); if (!out || RAY_IS_ERR(out)) return out; if (out->type == RAY_SYM) @@ -1978,12 +1862,12 @@ static ray_t* agg_unpack_key_col(ray_t* src_col, uint32_t key_idx, uint32_t n_ke out->len = n; int8_t type = src_col->type; uint8_t attrs = src_col->attrs; void* dst = ray_data(out); - int64_t i = 0; - for (uint32_t p = 0; p < nparts; p++) { - const int64_t* keys = parts[p].keys; - int64_t png = parts[p].ng; - for (int64_t gg = 0; gg < png; gg++, i++) - write_col_i64(dst, i, keys[(size_t)gg * n_keys + key_idx], type, attrs); + (void)nparts; + for (int64_t i = 0; i < n; i++) { + uint32_t p = (uint32_t)(pairs[i].idx >> 32); + uint32_t gg = (uint32_t)pairs[i].idx; + write_col_i64(dst, i, + parts[p].keys[(size_t)gg * n_keys + key_idx], type, attrs); } if (src_col->attrs & RAY_ATTR_HAS_NULLS) out->attrs |= RAY_ATTR_HAS_NULLS; return out; @@ -2000,26 +1884,25 @@ typedef struct { const void** val_data; const int8_t* val_types; const bool* val_hasnull; const uint8_t* val_esz; const void** val2_data; const int8_t* val2_types; const bool* val2_hasnull; const uint8_t* val2_esz; uint32_t nw; - agg_pay_buf_t* bufs; /* [nw * AGG_RADIX_P] payload records */ - agg_radix_part_t* parts; /* [AGG_RADIX_P] */ + uint32_t n_parts; + agg_pay_buf_t* bufs; /* [nw * n_parts] payload records */ + agg_radix_part_t* parts; /* [n_parts] */ int phase1_oom; /* set by any Phase-1 worker on push failure */ /* Per-row payload record layout (bytes), computed once by the caller: * [0 .. n_keys*8) packed keys (each widened via agg_read_key_i64) * [val_off[a] ..] agg a's input value at native esz (if val_data[a]) * [val2_off[a] ..] agg a's 2nd input (pearson) at native esz - * [row_off ..] int64 source row index (for first_row) */ + * [row_off ..] int64 logical input position */ size_t rec; /* total record size, 8-aligned */ const size_t* val_off; /* [n_aggs], carved by the caller */ const size_t* val2_off; /* [n_aggs], carved by the caller */ size_t row_off; - /* When false, NO admitted aggregate consults the original row (first_row is - * dead): the record omits its 8-byte row index and Phase-2/3 skip it. Set - * once from the agg kinds; row-dependent aggs (FIRST/LAST) flip it true. */ + /* Stable group ordering records one logical input position per payload. */ bool needs_row; /* Sel-mode (pushed WHERE filter): when sel != NULL, scatter the SELECTED * rows of [0,n_sel) (decoded to ORIGINAL row indices) instead of [start,end). - * The packed record's row_off ALWAYS stores the ORIGINAL decoded row, so - * first_row / key-unpack stay correct in original-row space. */ + * The packed record stores the selected-row ordinal, while column reads use + * the decoded original row. */ ray_t* sel; const int64_t* sel_prefix; /* Per-WORKER key-staging row: [nw * n_keys] int64. The scatter reads all @@ -2033,7 +1916,8 @@ typedef struct { /* Scatter one ORIGINAL row r's packed payload record into the worker's per- * partition buffer. Returns 0 or -1 on push OOM (sets phase1_oom). */ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, - int64_t* kv, int64_t r) { + int64_t* kv, int64_t r, + int64_t input_order) { uint32_t n_keys = c->n_keys, n_aggs = c->n_aggs; /* kv: this worker's key-staging slice (c->kv_scratch + wid*n_keys), sized * for any key count — no fixed [16] cap. */ @@ -2043,7 +1927,7 @@ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, kv[k] = v; h ^= (uint64_t)v; h *= 1099511628211ULL; } - uint32_t p = (uint32_t)(h & (AGG_RADIX_P - 1)); + uint32_t p = (uint32_t)(h & (c->n_parts - 1)); char* rec = agg_pay_reserve(&my[p], c->rec); if (!rec) { c->phase1_oom = 1; return -1; } int64_t* kdst = (int64_t*)rec; @@ -2058,7 +1942,8 @@ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, memcpy(rec + c->val2_off[a], (const char*)c->val2_data[a] + (size_t)r * ez2, ez2); } } - if (c->needs_row) *(int64_t*)(rec + c->row_off) = r; /* loop-invariant branch */ + if (c->needs_row) + *(int64_t*)(rec + c->row_off) = input_order; return 0; } @@ -2077,21 +1962,23 @@ RAY_INLINE void agg_radix_scatter_range(agg_radix_ctx_t* c, agg_pay_buf_t* my, int64_t rows[AGG_SEL_CHUNK]; agg_sel_cursor_t cur; agg_sel_cursor_init(&cur, c->sel, c->sel_prefix, start, end); + int64_t input_order = start; int64_t cn; while ((cn = agg_sel_cursor_next(&cur, rows)) > 0) for (int64_t i = 0; i < cn; i++) - if (agg_radix_scatter_one(c, my, kv, rows[i]) != 0) return; + if (agg_radix_scatter_one(c, my, kv, rows[i], + input_order++) != 0) return; return; } for (int64_t r = start; r < end; r++) - if (agg_radix_scatter_one(c, my, kv, r) != 0) return; + if (agg_radix_scatter_one(c, my, kv, r, r) != 0) return; } /* Phase 1: scatter one packed payload record per row into per-(worker, - * partition) contiguous buffers, keyed by RADIX_PART(tuple hash). */ + * partition) contiguous buffers, keyed by the tuple-hash partition. */ static void agg_radix_scatter_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { agg_radix_ctx_t* c = (agg_radix_ctx_t*)vctx; - agg_pay_buf_t* my = &c->bufs[(size_t)wid * AGG_RADIX_P]; + agg_pay_buf_t* my = &c->bufs[(size_t)wid * c->n_parts]; /* Key-staging row for this worker. The common bounded case (n_keys <= 16) * stages through a STACK array: a compiler-opaque heap slice defeated the * scalarization/register-promotion of the per-row kv[k]=v stage (flat @@ -2127,7 +2014,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t /* Total rows hashing to p across all workers. */ int64_t total = 0; for (uint32_t w = 0; w < c->nw; w++) - total += c->bufs[(size_t)w * AGG_RADIX_P + p].n; + total += c->bufs[(size_t)w * c->n_parts + p].n; if (total == 0) continue; /* Open-addressing hash sized to next_pow2(2*total). */ @@ -2179,7 +2066,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t int64_t ng = 0, ri = 0; for (uint32_t w = 0; w < c->nw; w++) { - agg_pay_buf_t* b = &c->bufs[(size_t)w * AGG_RADIX_P + p]; + agg_pay_buf_t* b = &c->bufs[(size_t)w * c->n_parts + p]; const char* rec = b->buf; for (uint32_t i = 0; i < b->n; i++, rec += c->rec) { const int64_t* keys = (const int64_t*)rec; @@ -2300,7 +2187,7 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t * SERIALLY by the caller (q10 and other scalar shapes get the full win). */ typedef struct { agg_radix_part_t* parts; - const agg_fr_pair_t* pairs; /* [ng], sorted by first_row */ + const agg_radix_order_t* pairs; /* [ng], stable first-seen order */ const agg_vtable_t** vts; const size_t* off; size_t block; @@ -2365,19 +2252,9 @@ static ray_t* exec_group_v2_parallel_radix( const bool* val2_hasnull = d.val2_hasnull; const uint8_t* val2_esz = d.val2_esz; const int64_t* agg_syms = d.agg_syms; - /* Row-dependent aggregates (FIRST/LAST) consult the representative original - * row; every other admitted agg (COUNT/SUM/AVG/MIN/MAX/VAR/STDDEV/PEARSON/ - * MEDIAN/TOP/BOT) accumulates from gathered values and never touches it, and - * the key columns are un-packed from the packed keys — so first_row/row_idx - * are dead weight. Drop them from the record layout when none is needed. - * NOTE: today FIRST/LAST never reach this kernel (agg_resolve() returns NULL - * for them, so the v2 admission gate rejects them), so needs_row is always - * false here — the scan below is a forward-guard, NOT a working restore path: - * admitting a row-dependent agg would also require finalize wiring to read - * the row, not just this layout flag. */ - bool needs_row = false; - for (uint32_t a = 0; a < n_aggs; a++) - if (ext->agg_ops[a] == OP_FIRST || ext->agg_ops[a] == OP_LAST) { needs_row = true; break; } + /* Every group records its earliest source row so emitted order does not + * depend on the number of radix partitions used by this execution. */ + bool needs_row = true; /* Per-row payload record layout: [packed keys][agg values][row_idx?]. * val_off/val2_off are per-dispatch (filled once here, read by workers). @@ -2398,13 +2275,15 @@ static ray_t* exec_group_v2_parallel_radix( size_t row_off = rec_cur; if (needs_row) rec_cur += 8; size_t rec = (rec_cur + 7u) & ~(size_t)7u; /* 8-align records */ - size_t nbuf = (size_t)nw * AGG_RADIX_P; + uint32_t n_parts = agg_radix_part_count(nw, sel ? n_sel : nrows); + size_t nbuf = (size_t)nw * n_parts; agg_pay_buf_t* bufs = ray_calloc_raw((size_t)(nbuf) * (sizeof(agg_pay_buf_t))); - agg_radix_part_t* parts = ray_calloc_raw((size_t)(AGG_RADIX_P) * (sizeof(agg_radix_part_t))); + agg_radix_part_t* parts = ray_calloc_raw((size_t)n_parts * sizeof(agg_radix_part_t)); if (!bufs || !parts) { ray_free_raw(bufs); ray_free_raw(parts); scratch_free(off_hdr); agg_desc_free(&d); - return ray_error("oom", NULL); + return exec_group_v2_parallel_smallhash(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); } agg_radix_ctx_t ctx = { @@ -2412,7 +2291,8 @@ static ray_t* exec_group_v2_parallel_radix( .vts = vts, .off = off, .block = block, .n_aggs = n_aggs, .val_data = val_data, .val_types = val_types, .val_hasnull = val_hasnull, .val_esz = val_esz, .val2_data = val2_data, .val2_types = val2_types, .val2_hasnull = val2_hasnull, .val2_esz = val2_esz, - .nw = nw, .bufs = bufs, .parts = parts, .phase1_oom = 0, + .nw = nw, .n_parts = n_parts, + .bufs = bufs, .parts = parts, .phase1_oom = 0, .rec = rec, .row_off = row_off, .needs_row = needs_row, .sel = sel, .sel_prefix = sel_prefix, .val_off = val_off, .val2_off = val2_off, @@ -2426,55 +2306,71 @@ static ray_t* exec_group_v2_parallel_radix( /* Phase 2: per-partition group+accumulate. */ int oom = ctx.phase1_oom; if (!oom) - ray_pool_dispatch_n(pool, agg_radix_group_fn, &ctx, AGG_RADIX_P); + ray_pool_dispatch_n(pool, agg_radix_group_fn, &ctx, n_parts); if (!oom) - for (uint32_t p = 0; p < AGG_RADIX_P; p++) + for (uint32_t p = 0; p < n_parts; p++) if (parts[p].oom) { oom = 1; break; } /* Phases 1+2 done (both dispatches joined); val_off/val2_off no longer read. */ scratch_free(off_hdr); if (oom) { - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); - return ray_error("oom", NULL); + return exec_group_v2_parallel_smallhash(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); } - /* Phase 3: emit in natural partition/group order. ── - * Group-by output order is UNSPECIFIED, so we emit groups in partition order - * then partition-local build order (no global sort by first_row). The - * (part<<32 | gid) idx encoding locates each group's state during finalize; - * key columns are un-packed sequentially from the contiguous packed-key - * buffers (agg_unpack_key_col), not gathered from scattered first_row[]. */ + /* Phase 3: restore stable first-seen order across partitions. */ int64_t ng = 0; - for (uint32_t p = 0; p < AGG_RADIX_P; p++) ng += parts[p].ng; - - /* (first_row, part, local gid) collected globally in build order. idx packs - * (part << 32 | local gid) so finalize recovers both the partition and the - * in-partition state offset. */ - agg_fr_pair_t* pairs = ray_alloc_raw((size_t)(ng > 0 ? ng : 1) * sizeof(agg_fr_pair_t)); + for (uint32_t p = 0; p < n_parts; p++) ng += parts[p].ng; + + /* Place each group directly at its first logical input position, then + * compact those occupied positions in one linear pass. This restores + * first-seen order without a comparison sort or a routing threshold. */ + int64_t input_count = sel ? n_sel : nrows; + agg_radix_order_t* pairs = ray_alloc_raw( + (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); if (!pairs) { - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); return ray_error("oom", NULL); } - { int64_t i = 0; - for (uint32_t p = 0; p < AGG_RADIX_P; p++) - for (int64_t gg = 0; gg < parts[p].ng; gg++) { - pairs[i].fr = parts[p].first_row ? parts[p].first_row[gg] : 0; /* unused by emit */ - pairs[i].idx = ((int64_t)p << 32) | (uint32_t)gg; /* part | gid */ - i++; - } + for (int64_t i = 0; i < input_count; i++) pairs[i].idx = -1; + bool order_ok = true; + for (uint32_t p = 0; p < n_parts && order_ok; p++) { + for (int64_t gg = 0; gg < parts[p].ng; gg++) { + int64_t first = parts[p].first_row[gg]; + if (first < 0 || first >= input_count || pairs[first].idx != -1) { + order_ok = false; + break; + } + pairs[first].idx = ((int64_t)p << 32) | (uint32_t)gg; + } + } + int64_t ordered = 0; + if (order_ok) { + for (int64_t i = 0; i < input_count; i++) + if (pairs[i].idx != -1) pairs[ordered++].idx = pairs[i].idx; + order_ok = ordered == ng; + } + if (!order_ok) { + ray_free_raw(pairs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); ray_free_raw(parts); + agg_desc_free(&d); + return ray_error("group", "failed to order radix groups"); } ray_t* result = ray_table_new(n_keys + n_aggs); if (!result || RAY_IS_ERR(result)) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2486,10 +2382,11 @@ static ray_t* exec_group_v2_parallel_radix( * original columns at first_row[]. Byte-identical to agg_gather_key_col * (see agg_unpack_key_col), incl. SYM payload + domain. */ for (uint32_t k = 0; k < n_keys; k++) { - ray_t* kc = agg_unpack_key_col(key_cols[k], k, n_keys, parts, AGG_RADIX_P, ng); + ray_t* kc = agg_unpack_key_col(key_cols[k], k, n_keys, parts, n_parts, + pairs, ng); if (!kc || RAY_IS_ERR(kc)) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2507,7 +2404,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_t** outs = (ray_t**)scratch_calloc(&outs_hdr, (size_t)n_aggs * (sizeof(ray_t*) + sizeof(int64_t))); if (!outs) { ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); agg_desc_free(&d); @@ -2522,7 +2419,7 @@ static ray_t* exec_group_v2_parallel_radix( if (!out || RAY_IS_ERR(out)) { for (uint32_t b = 0; b < a; b++) ray_release(outs[b]); ray_free_raw(pairs); - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); scratch_free(outs_hdr); agg_desc_free(&d); @@ -2594,7 +2491,7 @@ static ray_t* exec_group_v2_parallel_radix( ray_free_raw(pairs); /* Finalize is done reading every partition's buffered group state → destroy * exactly once before freeing the partition slabs. */ - agg_radix_parts_destroy(parts, AGG_RADIX_P, vts, off, block, n_aggs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); ray_free_raw(bufs); ray_free_raw(parts); scratch_free(outs_hdr); agg_desc_free(&d); @@ -2680,28 +2577,28 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * count when a filter is active, else the full table. */ int64_t eff_n = sel ? n_sel : nrows; + /* Buffered accumulators retain every contributing value per group. Running + * them through the parallel radix strategy overlaps those buffers with the + * full scatter payload and can exhaust the heap on large inputs. Keep + * buffered shapes on the serial v2 driver; streaming shapes retain all + * parallel strategies below. */ + bool all_streaming = true; + for (uint32_t a = 0; a < ext->n_aggs && all_streaming; a++) + if (vts[a]->kind != ACC_STREAMING) all_streaming = false; + ray_pool_t* pool = ray_pool_get(); - if (pool && eff_n >= RAY_PARALLEL_THRESHOLD) { - /* Per-worker memory budget gate: dense parallel allocates a full - * total_slots*(block + first_row) slab per worker. Low-card (the perf - * target) has tiny total_slots → always within budget → dense parallel. - * Mid-card-over-budget dense plans fall back to hash parallel (correct). */ - /* Compute per_worker_bytes ONLY when the dense plan is valid: a bailed - * plan (dp.ok == false) leaves dp.total_slots unset, and the multiply - * would overflow on garbage (UBSan signed-overflow). Short-circuit on - * dp.ok first. */ - bool dense_par_ok = false; - if (dp.ok) { - int64_t per_worker_bytes = dp.total_slots * (int64_t)(block + 8 /*first_row*/); - dense_par_ok = per_worker_bytes <= (8LL << 20); /* 8 MB/worker cap */ - } + if (pool && eff_n >= RAY_PARALLEL_THRESHOLD && all_streaming) { + /* A parallel dense plan owns one slot slab per worker. Admit it only + * when the aggregate slot count across all workers remains O(input); + * this is data-derived and independent of cache or RAM size. */ + uint32_t dense_workers = ray_pool_total_workers(pool); + bool dense_par_ok = dp.ok && dense_workers > 0 + && dp.total_slots <= eff_n / (int64_t)dense_workers; /* RADIX eligibility: every key an int/SYM type with no nulls (same * type-set check as agg_dense_plan). Radix takes the high-card - * remainder of int-key queries (dense handles the low-card head), - * for BOTH streaming and buffered (median/top) aggs — radix runs the - * full init/update_batch/finalize/destroy lifecycle. Hash handles - * F64 / STR keys. */ + * remainder of streaming int-key queries (dense handles the low-card + * head). Hash handles F64 / STR keys. */ bool keys_intsym = true; for (uint32_t k = 0; k < ext->n_keys && keys_intsym; k++) { ray_t* kc = key_cols[k]; @@ -2714,67 +2611,17 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (kc->attrs & RAY_ATTR_HAS_NULLS) keys_intsym = false; } - /* All aggregates streaming? (Buffered median/top-k keep radix — its - * per-group destroy lifecycle handles them; the small-hash path is - * streaming-only.) */ - bool all_streaming = true; - for (uint32_t a = 0; a < ext->n_aggs && all_streaming; a++) - if (vts[a]->kind != ACC_STREAMING) all_streaming = false; - if (dense_par_ok) { ray_t* r = exec_group_v2_parallel_dense(g, op, tbl, key_cols, key_syms, ext, nrows, pool, &dp, sel, sel_prefix, n_sel); agg_vo_free(&vo); scratch_free(kc_hdr); return r; } if (keys_intsym) { - /* Dense-FAIL int/SYM streaming shapes: probe cardinality on a bounded - * sample. LOW card → small-hash (O(groups) working set, no scatter - * tax); HIGH card → radix. The estimate early-exits above T_route so - * high-card shapes pay only the bounded sample. In sel mode the - * sample is taken over the SELECTED rows (the cardinality that - * actually reaches grouping), so a high-selectivity filter over a - * high-card column still routes to small-hash when the surviving - * rows are few-and-low-card. */ - if (all_streaming) { - /* Exact-size carve (unbounded keys) for the estimate's key-data - * pointers; freed at the smallhash return and on fall-through. */ - ray_t* kd_hdr; - const void** key_data = (const void**)scratch_alloc(&kd_hdr, (size_t)ext->n_keys * sizeof(void*)); - if (!key_data) { agg_vo_free(&vo); scratch_free(kc_hdr); return ray_error("oom", NULL); } - for (uint32_t k = 0; k < ext->n_keys; k++) key_data[k] = ray_data(key_cols[k]); - /* T_ROUTE: the group cardinality where the small-hash - * per-worker working set stops fitting cache and radix's - * partitioned scatter wins. Small-hash keeps one hash - * table of `groups` entries per worker (≈ groups × - * (key + state) bytes) and probes it per input row; while - * that table stays resident in fast cache the probes are - * near-free and small-hash beats radix's extra - * partition/scatter pass. Once `groups` grows past cache - * residency the probes start missing to DRAM and per-row - * cost climbs steeply, whereas radix is cardinality-flat - * (it partitions keys into cache-sized buckets up front). - * Measured 1-key 10M-row sweep on this class of machine: - * the streaming-SUM crossover lands right at ~16384 groups - * (smallhash ≈ radix there; smallhash ~2x faster below, - * ~2x slower above), which is ~0.4 MB of per-worker table — - * the L2-residency edge. Lighter aggs (COUNT) cross a bit - * lower; since radix degrades gracefully past the crossover - * while small-hash blows up, we sit at the high end of that - * band (16384) so a misroute costs little. */ - const int64_t T_ROUTE = 16384; - int64_t est = sel - ? agg_estimate_card_sel(key_cols, key_data, ext->n_keys, sel, sel_prefix, n_sel, 65536, T_ROUTE) - : agg_estimate_card(key_cols, key_data, ext->n_keys, nrows, 65536, T_ROUTE); - if (est <= T_ROUTE) { - ray_t* r = exec_group_v2_parallel_smallhash(g, op, tbl, nrows, - key_cols, key_syms, vts, off, block, est, sel, sel_prefix, n_sel); - scratch_free(kd_hdr); agg_vo_free(&vo); scratch_free(kc_hdr); return r; - } - scratch_free(kd_hdr); /* fall through to radix */ - } - { ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, key_cols, key_syms, vts, off, block, - sel, sel_prefix, n_sel); - agg_vo_free(&vo); scratch_free(kc_hdr); return r; } + /* Sparse integer/SYM ranges use radix deterministically. No sampled + * cardinality or cache-size crossover is baked into routing. */ + ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); + agg_vo_free(&vo); scratch_free(kc_hdr); return r; } /* Hash fallback (F64 / STR keys): not a chunked strategy — compact. */ if (sel) AGG_RUN_COMPACT_FALLBACK(); diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 1868a0bf9..2d66c877c 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -61,10 +61,9 @@ ray_t* agg_run_one(const agg_vtable_t* vt, ray_t* val_col, const uint32_t* gids, int64_t nrows, int64_t ngroups, int64_t kparam); -/* ── Dense grouping eligibility selector (low-cardinality int/SYM keys) ── - * Mirrors the old direct-array group path's cap. When dense applies, a group - * id is the packed key offset (O(1) direct index) rather than a hash slot. */ -#define DENSE_MAX_SLOTS 262144 /* mirror the old DA path's cap */ +/* ── Dense grouping eligibility selector (compact-range int/SYM keys) ── + * When dense applies, a group id is the packed key offset (O(1) direct index) + * rather than a hash slot. */ typedef struct { bool ok; @@ -77,8 +76,8 @@ typedef struct { /* Decide if dense grouping applies to (key_cols, aggs). Eligible iff: * - every key type in {I64,I32,I16,U8,BOOL,DATE,TIME,TIMESTAMP,SYM} and NOT HAS_NULLS - * - every agg vtable is ACC_STREAMING (no buffered median/top) - * - product of per-key ranges <= DENSE_MAX_SLOTS (no overflow) + * - product of per-key ranges is no larger than the contributing row count + * (so dense state is O(input), never controlled by a machine-size budget) * Does one min/max prescan over the key columns. Sets out->ok accordingly. * n_keys is uint32_t (untruncated ext->n_keys): dense direct-index routing * self-limits to <=16 keys here (wider shapes are rejected to v2's unbounded diff --git a/src/ops/group.c b/src/ops/group.c index 5c8bb310e..09d928e02 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -2230,6 +2230,7 @@ typedef struct { const int64_t* offsets; const int64_t* grp_cnt; ray_t* out_list; + _Atomic(int)* oom; } topk_par_ctx_t; /* Read src element as f64 (for the F64 path). Matches med_read_as_f64 @@ -2311,6 +2312,7 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, int64_t K = c->k; uint8_t desc = c->desc; for (int64_t gi = start; gi < end; gi++) { + if (atomic_load_explicit(c->oom, memory_order_relaxed)) break; ray_t* cell = ray_list_get(c->out_list, gi); if (!cell) continue; int64_t cnt = c->grp_cnt[gi]; @@ -2364,11 +2366,26 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, } cell->len = kept; } else { - /* Integer source: stage heap in stack buffer (K <= 1024 → - * 8KB), then narrow back to cell esz on write. */ + /* Integer sources need an I64 working heap before narrowing back + * to the result cell's physical width. Its size follows this + * group's actual min(K, count); no fixed K limit is involved. */ void* dst = ray_data(cell); uint8_t esz = ray_sym_elem_size(t, cell->attrs); - int64_t heap[1024]; + int64_t heap_nmax = cnt < K ? cnt : K; + ray_t* heap_hdr = NULL; + int64_t* heap = NULL; + if (heap_nmax > 0) { + if ((uint64_t)heap_nmax > SIZE_MAX / sizeof(int64_t)) { + atomic_store_explicit(c->oom, 1, memory_order_relaxed); + break; + } + heap = (int64_t*)scratch_alloc(&heap_hdr, + (size_t)heap_nmax * sizeof(int64_t)); + if (!heap) { + atomic_store_explicit(c->oom, 1, memory_order_relaxed); + break; + } + } int64_t kept = 0; int64_t init_end = 0; for (int64_t i = 0; i < cnt && kept < K; i++) { @@ -2399,6 +2416,7 @@ static void topk_per_group_fn(void* ctx_v, uint32_t worker_id, for (int64_t i = 0; i < kept; i++) topk_write_i64(dst, i, heap[i], esz); cell->len = kept; + scratch_free(heap_hdr); } } } @@ -2411,7 +2429,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, const int64_t* grp_cnt, int64_t n_groups) { if (!src || RAY_IS_ERR(src) || n_groups < 0) return NULL; - if (k < 1 || k > 1024) return NULL; + if (k < 1) return NULL; int8_t t = src->type; if (t != RAY_F64 && t != RAY_I64 && t != RAY_I32 && t != RAY_I16 && t != RAY_U8 && t != RAY_BOOL && t != RAY_DATE && t != RAY_TIME && @@ -2445,6 +2463,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, out = new_out; } + _Atomic(int) oom = 0; topk_par_ctx_t ctx = { .base = ray_data(src), .src_type = t, @@ -2455,6 +2474,7 @@ ray_t* ray_topk_per_group_buf(ray_t* src, .offsets = offsets, .grp_cnt = grp_cnt, .out_list = out, + .oom = &oom, }; ray_pool_t* pool = ray_pool_get(); @@ -2470,6 +2490,11 @@ ray_t* ray_topk_per_group_buf(ray_t* src, topk_per_group_fn(&ctx, 0, 0, n_groups); } + if (atomic_load_explicit(&oom, memory_order_relaxed)) { + ray_release(out); + return ray_error("oom", NULL); + } + return out; } @@ -3029,10 +3054,9 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->key_region = (uint16_t)koff; } uint16_t key_region = out->key_region; - /* Entry layout: hash | keys | null_mask | agg_vals | [entry_row?] - * Tail entry_row slot is appended only when any agg is FIRST/LAST, - * carrying the source-row index needed to merge correctly under - * work-stealing dispatch (see radix_phase1_fn / accum_from_entry). + /* Entry layout: hash | keys | null_mask | agg_vals | entry_row + * The tail source-row slot provides stable first-seen group ordering + * across partitions as well as FIRST/LAST merge correctness. * * The agg region is a representational budget, exactly like the key * region above: agg_val_slot is int8 (so ≤ INT8_MAX value slots), and @@ -3042,8 +3066,7 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, * silent wrap here scatters aggs to wrong offsets (the key region was * already budget-guarded; the agg region was not). */ bool has_first_last = (agg_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; - uint32_t entry_tail = has_first_last ? 8u : 0u; - uint32_t entry_stride = 8u + key_region + (uint32_t)nv * 8 + entry_tail; + uint32_t entry_stride = 8u + key_region + (uint32_t)nv * 8 + 8u; uint32_t off = 8u + key_region; uint32_t block = (uint32_t)nv * 8; @@ -3066,6 +3089,7 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->off_sumsq_y = (uint16_t)off; off += block; out->off_sumxy = (uint16_t)off; off += block; } + out->off_group_first = (uint16_t)off; off += 8u; /* Per-slot non-null count block — only when a nullable agg is present. * Null-free shapes leave off_nn == 0 and finalize on the group row count, * so their row_stride and layout are byte-identical to before. */ @@ -3142,7 +3166,7 @@ static bool group_ht_init_sized(group_ht_t* ht, uint32_t cap, } bool group_ht_init(group_ht_t* ht, uint32_t cap, const ght_layout_t* ly) { - return group_ht_init_sized(ht, cap, ly, 256); + return group_ht_init_sized(ht, cap, ly, 1); } /* Populate key_data[k] for wide-key resolution. Called by the HT path @@ -3459,7 +3483,8 @@ static inline void init_accum_from_entry(char* row, const char* entry, uint16_t na = ly->n_aggs; uint8_t nf = ly->need_flags; bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; - /* Entry tail slot carries the source-row index when has_fl. */ + /* The always-present entry tail carries the source row; FIRST/LAST + * consumes it only for those aggregate shapes. */ int64_t entry_row = 0; if (has_fl) memcpy(&entry_row, entry + ly->entry_stride - 8, 8); @@ -3844,6 +3869,8 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, uint64_t hash = *(const uint64_t*)entry; const char* ekeys = entry + 8; uint8_t salt = HT_SALT(hash); + int64_t entry_row; + memcpy(&entry_row, entry + ly->entry_stride - 8, 8); uint32_t slot = (uint32_t)(hash & mask); uint16_t key_bytes = ly->key_region; /* keys + null mask (incl. inline STR descriptors) */ @@ -3870,6 +3897,11 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, memcpy(row + 8, ekeys, key_bytes); if (!accum_skip) init_accum_from_entry(row, entry, ly); + else if (ly->row_stride > 8 + key_bytes) + memset(row + 8 + key_bytes, 0, + ly->row_stride - (8 + key_bytes)); + /* Accumulator initialization clears the row-state region. */ + memcpy(row + ly->off_group_first, &entry_row, 8); ht->slots[slot] = HT_PACK(salt, gid); if (ht->grp_count * 2 > ht->ht_cap) { group_ht_rehash(ht, key_types); @@ -3883,6 +3915,10 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, if (group_keys_equal((const int64_t*)(row + 8), (const int64_t*)ekeys, ly, ht->key_data, ht->key_pool)) { (*(int64_t*)row)++; /* count++ */ + int64_t first; + memcpy(&first, row + ly->off_group_first, 8); + if (entry_row < first) + memcpy(row + ly->off_group_first, &entry_row, 8); if (!accum_skip) accum_from_entry(row, entry, ly); return mask; @@ -3937,10 +3973,9 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, const uint8_t* const kflags = ly->key_flags; const uint8_t* const aflags = ly->agg_flags; uint8_t wide_any = ly->any_wide_key; - bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; uint32_t mask = ht->ht_cap - 1; /* Stack buffer for one entry: hash + (nk+1) key slots + nv agg_vals - * + optional 8-byte source-row tail (FIRST/LAST). The ≤8-key/≤8-agg + * + 8-byte source-row tail. The ≤8-key/≤8-agg * layout fits 8 + 9*8 + 8*8 + 8 = 152 bytes (byte-identical to the legacy * fixed buffer); a wider layout spills to one per-call heap block sized * to entry_stride (carved once here, never per row — unbounded-slots @@ -4044,8 +4079,7 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, /* Tail slot: source row index for FIRST/LAST tie-breaking. Same * layout as the radix path's entries so accum_from_entry can read * it from the same offset. */ - if (has_fl) - memcpy(ebuf + ly->entry_stride - 8, &row, 8); + memcpy(ebuf + ly->entry_stride - 8, &row, 8); mask = group_probe_entry(ht, ebuf, key_types, mask); } @@ -4063,10 +4097,27 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, * Pass 3: Build result columns from inline group rows. * ============================================================================ */ -#define RADIX_BITS 8 -#define RADIX_P (1u << RADIX_BITS) /* 256 partitions */ -#define RADIX_MASK (RADIX_P - 1) -#define RADIX_PART(h) (((uint32_t)((h) >> 16)) & RADIX_MASK) +/* Balance O(P) partition metadata and O(N/P) rows per partition. Both inputs + * come from the execution; there is no fixed fanout or machine-size cap. */ +static uint32_t group_radix_part_count(uint32_t n_workers, int64_t n_rows) { + uint32_t n_parts = 1; + uint64_t rows = n_rows > 0 ? (uint64_t)n_rows : 1; + while ((n_parts < n_workers || + (uint64_t)n_parts < rows / n_parts + (rows % n_parts != 0)) && + n_parts <= UINT32_MAX / 2) + n_parts <<= 1; + return n_parts; +} + +static inline uint32_t group_radix_part(uint64_t hash, uint32_t n_parts) { + return ((uint32_t)(hash >> 16)) & (n_parts - 1); +} + +typedef struct { + int64_t value; + uint32_t part; + uint32_t gid; +} group_topn_item_t; /* Selection-aware group iteration gate. When a WHERE leaves fewer than * nrows >> SEL_MATCH_GATE_SHIFT survivors, the high-card group build iterates @@ -4091,11 +4142,11 @@ typedef struct { static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, uint64_t hash, const int64_t* key_region_buf, const int64_t* agg_vals, uint16_t n_agg_vals, - bool has_first_last, int64_t row, - uint16_t key_region) { + int64_t row, uint16_t key_region) { if (__builtin_expect(buf->count >= buf->cap, 0)) { uint32_t old_cap = buf->cap; - uint32_t new_cap = old_cap * 2; + if (old_cap > UINT32_MAX / 2) { buf->oom = true; return; } + uint32_t new_cap = old_cap ? old_cap * 2 : 1; char* new_data = (char*)scratch_realloc( &buf->_hdr, (size_t)old_cap * entry_stride, (size_t)new_cap * entry_stride); @@ -4108,9 +4159,7 @@ static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, memcpy(dst + 8, key_region_buf, key_region); if (n_agg_vals) memcpy(dst + 8 + key_region, agg_vals, (size_t)n_agg_vals * 8); - /* Tail slot: source row index for FIRST/LAST tie-breaking. */ - if (has_first_last) - memcpy(dst + entry_stride - 8, &row, 8); + memcpy(dst + entry_stride - 8, &row, 8); buf->count++; } @@ -4129,7 +4178,8 @@ typedef struct { ray_t** agg_vecs2; uint8_t* agg_strlen; uint32_t n_workers; - radix_buf_t* bufs; /* [n_workers * RADIX_P] */ + uint32_t n_parts; + radix_buf_t* bufs; /* [n_workers * n_parts] */ ght_layout_t layout; ray_t* rowsel; /* When non-NULL, workers iterate match_idx[start..end) and @@ -4140,7 +4190,7 @@ typedef struct { static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { radix_phase1_ctx_t* c = (radix_phase1_ctx_t*)ctx; const ght_layout_t* ly = &c->layout; - radix_buf_t* my_bufs = &c->bufs[(size_t)worker_id * RADIX_P]; + radix_buf_t* my_bufs = &c->bufs[(size_t)worker_id * c->n_parts]; uint16_t nk = ly->n_keys; uint16_t na = ly->n_aggs; uint16_t nv = ly->n_agg_vals; @@ -4149,7 +4199,6 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ uint8_t wide_any = ly->any_wide_key; uint8_t inline_str = ly->any_inline_str; uint16_t estride = ly->entry_stride; - bool has_fl = (ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST)) != 0; const int64_t* match_idx = c->match_idx; /* Per-worker key/agg staging. The ≤8-key/≤8-agg layout stages through @@ -4249,10 +4298,10 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ } } - uint32_t part = RADIX_PART(h); + uint32_t part = group_radix_part(h, c->n_parts); radix_buf_push(&my_bufs[part], estride, h, inline_str ? (const int64_t*)keybuf : keys, - agg_vals, nv, has_fl, row, ly->key_region); + agg_vals, nv, row, ly->key_region); } scratch_free(stage_hdr); /* NULL (inline staging) → no-op */ } @@ -4302,6 +4351,7 @@ typedef struct { typedef struct { group_ht_t* part_hts; uint32_t* part_offsets; + const uint32_t* group_out; /* flat partition gid -> first-seen output row */ char** key_dsts; int8_t* key_types; uint8_t* key_attrs; @@ -4345,7 +4395,7 @@ static void radix_phase3_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * (→ use cnt) for null-free layouts (byte-identical to before). */ const int64_t* nnbase = ly->off_nn ? (const int64_t*)(const void*)(row + ly->off_nn) : NULL; - uint32_t di = off + gi; + uint32_t di = c->group_out ? c->group_out[off + gi] : off + gi; /* Scatter keys to result columns */ for (uint32_t k = 0; k < nk; k++) { @@ -4539,6 +4589,7 @@ typedef struct { int8_t* key_types; uint32_t n_keys; /* widened from uint8_t (unbounded-ready) */ uint32_t n_workers; + uint32_t n_parts; radix_buf_t* bufs; group_ht_t* part_hts; ght_layout_t layout; @@ -4556,21 +4607,10 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ for (int64_t p = start; p < end; p++) { uint32_t total = 0; for (uint32_t w = 0; w < c->n_workers; w++) - total += c->bufs[(size_t)w * RADIX_P + p].count; + total += c->bufs[(size_t)w * c->n_parts + p].count; if (total == 0) continue; - uint32_t part_ht_cap = 256; - { - uint64_t target = (uint64_t)total * 2; - if (target < 256) target = 256; - while (part_ht_cap < target) part_ht_cap *= 2; - } - /* Pre-size group store to avoid grows. Use next_pow2(total) as upper - * bound on groups. Over-allocation is bounded: worst case total >> groups, - * but total * row_stride is already committed via HT capacity anyway. */ - uint32_t init_grp = 256; - while (init_grp < total && init_grp < 65536) init_grp *= 2; - if (!group_ht_init_sized(&c->part_hts[p], part_ht_cap, &c->layout, init_grp)) + if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) continue; /* Wide keys need source-column resolution during probe/rehash. */ if (c->layout.any_wide_key && c->key_data) { @@ -4579,7 +4619,7 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ } for (uint32_t w = 0; w < c->n_workers; w++) { - radix_buf_t* buf = &c->bufs[(size_t)w * RADIX_P + p]; + radix_buf_t* buf = &c->bufs[(size_t)w * c->n_parts + p]; if (buf->count == 0) continue; group_rows_indirect(&c->part_hts[p], c->key_types, buf->data, buf->count, estride); @@ -4597,13 +4637,9 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * knows how to combine counts; SUM/AVG/MIN/MAX would need their own * state-merge logic (next increment). * - * Per-(worker, partition) HT for a 10M-row count-by-UserID: ~3M distinct - * keys ÷ 256 parts ÷ 8 workers ≈ 1.5K groups → cap ~4K slots → ~64 KB - * row store, L1/L2-resident. Worker w processes its row range; per row - * it hashes keys, computes partition = RADIX_PART(h), probes its local - * HT_p. Phase2 dispatches partitions across workers; each merges the n - * worker HTs for one partition into a final partition HT in part_hts[p]. - * Phase3 (radix_phase3_fn) emits from part_hts[] exactly as before. + * Worker w processes its row ranges; per row it hashes keys, computes the + * execution-derived partition, and probes its local HT_p. Phase2 dispatches + * partitions across workers and merges the worker HTs for each partition. * ============================================================================ */ /* Merge one source group row into the target HT. Hash is recomputed from @@ -4653,6 +4689,11 @@ static inline uint32_t group_merge_row(group_ht_t* ht, if (group_keys_equal((const int64_t*)(row + 8), skeys, ly, ht->key_data, ht->key_pool)) { *(int64_t*)row += src_count; + int64_t dst_first, src_first; + memcpy(&dst_first, row + ly->off_group_first, 8); + memcpy(&src_first, src_row + ly->off_group_first, 8); + if (src_first < dst_first) + memcpy(row + ly->off_group_first, &src_first, 8); if (need_sum) { for (uint32_t a = 0; a < na; a++) { int8_t s = vslot[a]; @@ -4687,7 +4728,8 @@ typedef struct { uint8_t* agg_strlen; uint8_t nullable_mask; uint32_t n_workers; - group_ht_t* wpart_hts; /* [n_workers * RADIX_P] */ + uint32_t n_parts; + group_ht_t* wpart_hts; /* [n_workers * n_parts] */ ght_layout_t layout; ray_t* rowsel; const int64_t* match_idx; @@ -4707,22 +4749,13 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, uint8_t nullable = c->nullable_mask; const int64_t* match_idx = c->match_idx; - group_ht_t* my_hts = &c->wpart_hts[(size_t)worker_id * RADIX_P]; - /* Lazily init this worker's 256 partition HTs. */ - for (uint32_t p = 0; p < RADIX_P; p++) { - if (!my_hts[p].slots) { - if (!group_ht_init_sized(&my_hts[p], 256, ly, 128)) { - atomic_store_explicit(&c->oom, 1, memory_order_relaxed); - return; - } - if (wide_any && c->key_data) { - group_ht_set_key_data(&my_hts[p], c->key_data); - group_ht_set_key_pool(&my_hts[p], c->key_vecs); - } - } - } - uint32_t masks[RADIX_P]; - for (uint32_t p = 0; p < RADIX_P; p++) masks[p] = my_hts[p].ht_cap - 1; + group_ht_t* my_hts = &c->wpart_hts[(size_t)worker_id * c->n_parts]; + uint32_t masks[c->n_parts]; + /* A worker may be called for several morsels. Its partition HTs persist + * across calls, so restore each live probe mask instead of treating every + * invocation as first use. */ + for (uint32_t p = 0; p < c->n_parts; p++) + masks[p] = my_hts[p].slots ? my_hts[p].ht_cap - 1 : 0; /* Stack-resident transient entry + per-key str-pool table, same layout as * group_rows_range. ≤8 keys/aggs stay on the stack (byte-identical); @@ -4812,7 +4845,22 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, } } } - uint32_t p = RADIX_PART(h); + memcpy(ebuf + ly->entry_stride - 8, &row, 8); + uint32_t p = group_radix_part(h, c->n_parts); + if (!my_hts[p].slots) { + /* Demand-driven initialization: the first observed row creates + * the smallest valid table and normal growth follows actual + * cardinality. Empty worker/partition pairs allocate nothing. */ + if (!group_ht_init_sized(&my_hts[p], 2, ly, 1)) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + break; + } + if (wide_any && c->key_data) { + group_ht_set_key_data(&my_hts[p], c->key_data); + group_ht_set_key_pool(&my_hts[p], c->key_vecs); + } + masks[p] = my_hts[p].ht_cap - 1; + } uint32_t new_mask = group_probe_entry(&my_hts[p], ebuf, c->key_types, masks[p]); if (my_hts[p].oom) { @@ -4825,10 +4873,11 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, } typedef struct { - group_ht_t* wpart_hts; /* [n_workers * RADIX_P] — input */ - group_ht_t* part_hts; /* [RADIX_P] — output */ + group_ht_t* wpart_hts; /* [n_workers * n_parts] — input */ + group_ht_t* part_hts; /* [n_parts] — output */ int8_t* key_types; uint32_t n_workers; + uint32_t n_parts; ght_layout_t layout; void** key_data; const void** key_pool; /* [n_keys] str-pool base per wide STR key */ @@ -4847,17 +4896,9 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, * fold those, so the final grp_count is ≤ this sum). */ uint32_t total_grps = 0; for (uint32_t w = 0; w < c->n_workers; w++) - total_grps += c->wpart_hts[(size_t)w * RADIX_P + p].grp_count; + total_grps += c->wpart_hts[(size_t)w * c->n_parts + p].grp_count; if (total_grps == 0) continue; - uint32_t ht_cap = 256; - { - uint64_t target = (uint64_t)total_grps * 2; - if (target < 256) target = 256; - while (ht_cap < target) ht_cap *= 2; - } - uint32_t init_grp = 256; - while (init_grp < total_grps && init_grp < 65536) init_grp *= 2; - if (!group_ht_init_sized(&c->part_hts[p], ht_cap, &c->layout, init_grp)) { + if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) { atomic_store_explicit(&c->oom, 1, memory_order_relaxed); return; } @@ -4867,7 +4908,7 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, } uint32_t mask = c->part_hts[p].ht_cap - 1; for (uint32_t w = 0; w < c->n_workers; w++) { - group_ht_t* src = &c->wpart_hts[(size_t)w * RADIX_P + p]; + group_ht_t* src = &c->wpart_hts[(size_t)w * c->n_parts + p]; if (src->grp_count == 0) continue; const char* rows = src->rows; for (uint32_t gi = 0; gi < src->grp_count; gi++) { @@ -4897,10 +4938,8 @@ typedef struct { uint32_t n_workers; const int64_t* match_idx; /* NULL = no selection */ ray_t* rowsel; - /* DA-path early-out: once any worker observes a key span wider than - * span_budget the direct-array path is provably infeasible (its slot - * count would exceed DA_MAX_COMPOSITE_SLOTS), so the whole scan can - * stop instead of reading the rest of a 10M-row column for nothing. */ + /* DA-path early-out: once any worker observes a key span wider than the + * contributing row count, direct-array state would be superlinear. */ int64_t span_budget; _Atomic(int)* abort_flag; } minmax_ctx_t; @@ -5517,8 +5556,7 @@ static bool sparse_i64_touch(sparse_i64_ht_t* ht, int64_t key, uint8_t n_aggs, return true; } -/* Composite GID from multi-key. Arithmetic overflow is prevented in practice - * by the DA budget check (DA_PER_WORKER_MAX) which limits total_slots to 262K. */ +/* Composite GID from multi-key. Planning bounds total_slots to INT32_MAX. */ static inline int32_t da_composite_gid(da_ctx_t* c, int64_t r) { int32_t gid = 0; for (uint32_t k = 0; k < c->n_keys; k++) { @@ -6349,7 +6387,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { * part_hts[] + part_offsets[], we still need to materialize per-group * value slices to feed the holistic quickselect kernel. This pass: * - * 1. Re-probe each source row against part_hts[RADIX_PART(h)] to + * 1. Re-probe each source row against its hash partition to * recover its global gid (parallel, lookup-only — no inserts). * Writes row_gid[r] = part_offsets[p] + local_gid. * 2. Build idx_buf + offsets via the idxbuf hist/scat pattern over @@ -6402,31 +6440,18 @@ typedef struct { const void** key_pool; /* [n_keys] str-pool base per wide STR key */ group_ht_t* part_hts; const uint32_t* part_offsets; + const uint32_t* group_out; + uint32_t n_parts; int64_t* row_gid; /* output [nrows] */ const int64_t* match_idx; ray_t* rowsel; /* non-NULL when selection carried as rowsel */ } reprobe_ctx_t; -static void reprobe_rows_fn(void* vctx, uint32_t worker_id, - int64_t start, int64_t end) { - (void)worker_id; - reprobe_ctx_t* c = (reprobe_ctx_t*)vctx; +/* Resolve one source row to the flat partition-local group id. The caller + * owns reusable key staging so a scan performs no per-row allocation. */ +static inline int64_t reprobe_flat_gid(const reprobe_ctx_t* c, int64_t row, + int64_t* ek_buf, char* keybuf) { uint32_t nk = c->n_keys; - /* Key lookup staging: ≤8 keys stay on the stack (ek_buf: 8 keys + 1 null - * word; keybuf: inline-STR region), wider layouts carve one per-worker - * heap block ONCE (never per row — unbounded-slots cut 4). */ - int64_t ek_buf_stk[9]; - char keybuf_stk[136]; - int64_t* ek_buf = ek_buf_stk; - char* keybuf = keybuf_stk; - ray_t* rp_stage_hdr = NULL; - if ((size_t)c->layout->key_region > sizeof(ek_buf_stk)) { - size_t kb = c->layout->key_region; - char* blk = (char*)scratch_alloc(&rp_stage_hdr, kb + kb); - if (!blk) return; /* OOM on a pathologically wide layout: drop worker */ - keybuf = blk; - ek_buf = (int64_t*)(blk + kb); - } int8_t* key_types = c->key_types; void** key_data = c->key_data; uint8_t* key_attrs = c->key_attrs; @@ -6434,26 +6459,15 @@ static void reprobe_rows_fn(void* vctx, uint32_t worker_id, uint8_t nullable = c->nullable_mask; const uint8_t* const kflags = c->layout->key_flags; uint8_t wide_any = c->layout->any_wide_key; - const int64_t* match_idx = c->match_idx; - ray_t* rowsel = c->rowsel; - for (int64_t i = start; i < end; i++) { - if (((i - start) & 65535) == 0 && ray_interrupted()) break; - int64_t row = match_idx ? match_idx[i] : i; - /* Honor a rowsel-carried selection: filtered-out rows must not map to - * any group, else the holistic (median/top) idx_buf would include - * them. -1 is the same "no group" sentinel the HT-miss case uses. */ - if (!match_idx && rowsel && !group_rowsel_pass(rowsel, row)) { - c->row_gid[row] = -1; - continue; - } - uint64_t h = 0; - const int64_t* lookup_keys; - if (c->layout->any_inline_str) { - h = inline_build_keys(c->layout, key_types, key_data, key_attrs, - c->key_pool, key_vecs, nullable, row, keybuf); - lookup_keys = (const int64_t*)keybuf; - } else { - int64_t* nullw = ek_buf + nk; /* null-mask words at key_off[nk]==nk*8 */ + uint64_t h = 0; + const int64_t* lookup_keys; + + if (c->layout->any_inline_str) { + h = inline_build_keys(c->layout, key_types, key_data, key_attrs, + c->key_pool, key_vecs, nullable, row, keybuf); + lookup_keys = (const int64_t*)keybuf; + } else { + int64_t* nullw = ek_buf + nk; uint32_t null_words = c->layout->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; for (uint32_t k = 0; k < nk; k++) { @@ -6482,15 +6496,53 @@ static void reprobe_rows_fn(void* vctx, uint32_t worker_id, } h = ght_hash_null_words(h, nullw, null_words); lookup_keys = ek_buf; - } + } - uint32_t part = RADIX_PART(h); - uint32_t local = group_ht_lookup_gid(&c->part_hts[part], h, - lookup_keys, key_types); - if (local == UINT32_MAX || local >= c->part_hts[part].grp_count) { + uint32_t part = group_radix_part(h, c->n_parts); + uint32_t local = group_ht_lookup_gid(&c->part_hts[part], h, + lookup_keys, key_types); + if (local == UINT32_MAX || local >= c->part_hts[part].grp_count) + return -1; + return (int64_t)c->part_offsets[part] + local; +} + +static void reprobe_rows_fn(void* vctx, uint32_t worker_id, + int64_t start, int64_t end) { + (void)worker_id; + reprobe_ctx_t* c = (reprobe_ctx_t*)vctx; + /* Key lookup staging: ≤8 keys stay on the stack (ek_buf: 8 keys + 1 null + * word; keybuf: inline-STR region), wider layouts carve one per-worker + * heap block ONCE (never per row — unbounded-slots cut 4). */ + int64_t ek_buf_stk[9]; + char keybuf_stk[136]; + int64_t* ek_buf = ek_buf_stk; + char* keybuf = keybuf_stk; + ray_t* rp_stage_hdr = NULL; + if ((size_t)c->layout->key_region > sizeof(ek_buf_stk)) { + size_t kb = c->layout->key_region; + char* blk = (char*)scratch_alloc(&rp_stage_hdr, kb + kb); + if (!blk) return; /* OOM on a pathologically wide layout: drop worker */ + keybuf = blk; + ek_buf = (int64_t*)(blk + kb); + } + const int64_t* match_idx = c->match_idx; + ray_t* rowsel = c->rowsel; + for (int64_t i = start; i < end; i++) { + if (((i - start) & 65535) == 0 && ray_interrupted()) break; + int64_t row = match_idx ? match_idx[i] : i; + /* Honor a rowsel-carried selection: filtered-out rows must not map to + * any group, else the holistic (median/top) idx_buf would include + * them. -1 is the same "no group" sentinel the HT-miss case uses. */ + if (!match_idx && rowsel && !group_rowsel_pass(rowsel, row)) { + c->row_gid[row] = -1; + continue; + } + int64_t flat = reprobe_flat_gid(c, row, ek_buf, keybuf); + if (flat < 0) { c->row_gid[row] = -1; } else { - c->row_gid[row] = (int64_t)c->part_offsets[part] + (int64_t)local; + c->row_gid[row] = c->group_out ? (int64_t)c->group_out[flat] + : flat; } } scratch_free(rp_stage_hdr); /* NULL (inline staging) → no-op */ @@ -9479,22 +9531,14 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, da_path:; /* ---- Direct-array fast path for low-cardinality integer keys ---- */ - /* Supports multi-key via composite index: product of ranges <= MAX */ - #define DA_MAX_COMPOSITE_SLOTS 262144 /* 256K slots max */ - #define DA_MEM_BUDGET (256ULL << 20) /* 256 MB total across all workers */ - #define DA_PER_WORKER_MAX (6ULL << 20) /* 6 MB per-worker max */ + /* Supports multi-key via a composite index when its state remains O(input). */ { /* n_aggs <= 64 is not a slot cap — it is the width of the da_ctx_t * agg_f64_mask/agg_int_null_mask bitmasks (uint64_t, one bit per * agg). Wider agg lists route to the hash (HT) path instead, which * carries per-agg arrays rather than a fixed-width bitmask. */ - bool da_eligible = (nrows > 0 && n_keys > 0 && n_keys <= 8 && + bool da_eligible = (n_scan > 0 && n_keys > 0 && n_keys <= 8 && n_aggs <= 64); - if (da_eligible && rowsel && n_keys == 1) { - ray_rowsel_t* sm = ray_rowsel_meta(rowsel); - if (sm && sm->total_pass * 4 < nrows) - da_eligible = false; - } /* Binary aggregators (OP_PEARSON_CORR) are not wired into the * dense-array accumulator's per-worker da_accum_t struct — force * the HT path which has the row-layout offsets allocated. @@ -9537,7 +9581,8 @@ da_path:; * a known distinct-count (dense codes 0..n_distinct-1 over the full * column), so over the filtered survivors n_distinct is an upper bound * on its slot span. If the product of such KNOWN bounds already - * exceeds the DA budget, the composite is infeasible — reject now and + * exceeds the contributing row count, dense state would be superlinear + * in the input — reject now and * skip the min/max prescan (a full gather-scan of the survivors) * entirely. Conservative in the CORRECTNESS direction: keys without a * known count contribute factor 1, and rejecting only ever SKIPS the @@ -9550,12 +9595,12 @@ da_path:; if (key_scan_sym[k] < 0) continue; int64_t card = ray_grp_card_lookup(key_scan_sym[k]); if (card <= 1) continue; - if ((uint64_t)card > (uint64_t)DA_MAX_COMPOSITE_SLOTS / known_ub) { + if ((uint64_t)card > (uint64_t)n_scan / known_ub) { da_eligible = false; break; } known_ub *= (uint64_t)card; - if (known_ub > DA_MAX_COMPOSITE_SLOTS) { da_eligible = false; break; } + if (known_ub > (uint64_t)n_scan) { da_eligible = false; break; } } } @@ -9589,7 +9634,7 @@ da_path:; .n_workers = mm_n, .match_idx = match_idx, .rowsel = rowsel, - .span_budget = DA_MAX_COMPOSITE_SLOTS, + .span_budget = n_scan, .abort_flag = &mm_abort, }; if (mm_n > 1) { @@ -9613,35 +9658,15 @@ da_path:; if (span > (uint64_t)INT64_MAX) { da_fits = false; break; } da_key_range[k] = (int64_t)span; if (da_key_range[k] <= 0) { da_fits = false; break; } - total_slots *= (uint64_t)da_key_range[k]; - if (total_slots > DA_MAX_COMPOSITE_SLOTS) da_fits = false; - } - } - - if (da_fits) { - /* Compute which accumulator arrays we actually need */ - uint8_t need_flags = DA_NEED_COUNT; /* always need count */ - for (uint32_t a = 0; a < n_aggs; a++) { - uint16_t aop = ext->agg_ops[a]; - if (aop == OP_SUM || aop == OP_PROD || aop == OP_AVG || aop == OP_ALL || aop == OP_ANY || aop == OP_FIRST || aop == OP_LAST) need_flags |= DA_NEED_SUM; - else if (aop == OP_STDDEV || aop == OP_STDDEV_POP || aop == OP_VAR || aop == OP_VAR_POP) - { need_flags |= DA_NEED_SUM; need_flags |= DA_NEED_SUMSQ; } - else if (aop == OP_MIN) need_flags |= DA_NEED_MIN; - else if (aop == OP_MAX) need_flags |= DA_NEED_MAX; + uint64_t range = (uint64_t)da_key_range[k]; + uint64_t slot_limit = (uint64_t)n_scan < INT32_MAX + ? (uint64_t)n_scan : INT32_MAX; + if (total_slots > slot_limit / range) { + da_fits = false; + break; + } + total_slots *= range; } - - /* Compute per-worker memory budget. Actual allocation is 1 union - * array per type, but MIN/MAX use conditional random writes that - * perform worse than radix-partitioned HT at high group counts. - * Weight MIN/MAX at 2x to keep those queries on the HT path. */ - uint32_t arrays_per_agg = 0; - if (need_flags & DA_NEED_SUM) arrays_per_agg += 1; - if (need_flags & DA_NEED_MIN) arrays_per_agg += 2; /* 2x: DA MIN slow at high cardinality */ - if (need_flags & DA_NEED_MAX) arrays_per_agg += 2; /* 2x: DA MAX slow at high cardinality */ - if (need_flags & DA_NEED_SUMSQ) arrays_per_agg += 1; - uint64_t per_worker = total_slots * (arrays_per_agg * n_aggs + 1u) * 8u; - if (per_worker > DA_PER_WORKER_MAX) - da_fits = false; } if (da_fits) { @@ -9736,19 +9761,22 @@ da_path:; uint32_t da_n_workers = (da_pool && nrows >= RAY_PARALLEL_THRESHOLD) ? ray_pool_total_workers(da_pool) : 1; - /* Check memory budget — need one accumulator set per worker. - * Weight MIN/MAX at 2x in budget (same as eligibility check) to - * keep MIN/MAX-heavy queries on the faster radix-HT path. */ + /* Bound aggregate state structurally: across workers, direct-array + * cells must remain O(contributing rows). */ uint32_t arrays_per_agg = 0; if (need_flags & DA_NEED_SUM) arrays_per_agg += 1; - if (need_flags & DA_NEED_MIN) arrays_per_agg += 2; - if (need_flags & DA_NEED_MAX) arrays_per_agg += 2; + if (need_flags & DA_NEED_MIN) arrays_per_agg += 1; + if (need_flags & DA_NEED_MAX) arrays_per_agg += 1; if (need_flags & DA_NEED_SUMSQ) arrays_per_agg += 1; /* Nullable aggs add a per-(group, agg) non-null count array. * ~8 bytes per (group, agg). */ if (da_any_nullable) arrays_per_agg += 1; - uint64_t per_worker_bytes = (uint64_t)n_slots * (arrays_per_agg * n_aggs + 1u) * 8u; - if ((uint64_t)da_n_workers * per_worker_bytes > DA_MEM_BUDGET) + uint64_t cells_per_worker = (uint64_t)n_slots + * ((uint64_t)arrays_per_agg * n_aggs + 1u); + uint64_t max_workers = cells_per_worker + ? (uint64_t)n_scan / cells_per_worker : 1; + if (max_workers < 1) max_workers = 1; + if ((uint64_t)da_n_workers > max_workers) da_n_workers = 1; ray_t* accums_hdr; @@ -10774,17 +10802,13 @@ ht_path:; return ray_error("limit", "group: layout stride/slot budget exceeded (or OOM)"); } - /* Right-sized hash table: start small, rehash on load > 0.5 */ - uint32_t ht_cap = 256; - { - uint64_t target = (uint64_t)nrows < 65536 ? (uint64_t)nrows : 65536; - if (target < 256) target = 256; - while (ht_cap < target) ht_cap *= 2; - } + /* Sequential fallback grows strictly from observed groups. */ + uint32_t ht_cap = 2; /* Parallel path: radix-partitioned group-by */ ray_pool_t* pool = ray_pool_get(); uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; + uint32_t radix_n_parts = group_radix_part_count(n_total, n_scan); group_ht_t single_ht; group_ht_t top_ht; @@ -10796,6 +10820,10 @@ ht_path:; radix_buf_t* radix_bufs = NULL; ray_t* part_hts_hdr = NULL; group_ht_t* part_hts = NULL; + ray_t* part_offsets_hdr = NULL; + uint32_t* part_offsets = NULL; + ray_t* group_out_hdr = NULL; + uint32_t* group_out = NULL; /* Top-N-by-count (`select … by … desc:c take:N`) is served by the * parallel radix_v2 path below: phase1/phase2 build per-partition HTs @@ -10805,7 +10833,8 @@ ht_path:; * across all input sizes (measured), so there is no separate * size-gated shortcut here. */ - if (pool && nrows >= RAY_PARALLEL_THRESHOLD && n_total > 1) { + if (pool && n_total > 1 && + (nrows >= RAY_PARALLEL_THRESHOLD || use_topn_filter)) { /* Filtered (rowsel/match_idx) group-bys are allowed on the parallel * paths: both radix_v2_phase1_fn and radix_phase1_fn honour c->rowsel * (skip non-passing rows during scatter), so partitioned data holds @@ -10855,13 +10884,13 @@ ht_path:; if (v2_ok && !(ght_layout.agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST | GHT_AF_HOLISTIC | GHT_AF_BINARY))) { ray_t* wpart_hdr = NULL; - size_t v2_n_w = (size_t)n_total * RADIX_P; + size_t v2_n_w = (size_t)n_total * radix_n_parts; group_ht_t* wpart_hts = (group_ht_t*)scratch_calloc( &wpart_hdr, v2_n_w * sizeof(group_ht_t)); ray_t* v2_part_hdr = NULL; group_ht_t* v2_part_hts = wpart_hts ? (group_ht_t*)scratch_calloc(&v2_part_hdr, - RADIX_P * sizeof(group_ht_t)) + (size_t)radix_n_parts * sizeof(group_ht_t)) : NULL; if (!wpart_hts || !v2_part_hts) { if (wpart_hts) scratch_free(wpart_hdr); @@ -10904,6 +10933,7 @@ ht_path:; .agg_strlen = agg_strlen, .nullable_mask = v2_nullable, .n_workers = n_total, + .n_parts = radix_n_parts, .wpart_hts = wpart_hts, .rowsel = rowsel, .match_idx = sel_match, @@ -10929,6 +10959,7 @@ ht_path:; .part_hts = v2_part_hts, .key_types = key_types, .n_workers = n_total, + .n_parts = radix_n_parts, .key_data = key_data, .oom = 0, }; @@ -10952,7 +10983,8 @@ ht_path:; } derive_key_pool(&ght_layout, key_vecs, v2p2.key_pool); } - ray_pool_dispatch_n(pool, radix_v2_phase2_fn, &v2p2, RADIX_P); + ray_pool_dispatch_n(pool, radix_v2_phase2_fn, &v2p2, + radix_n_parts); scratch_free(v2p2_kp_hdr); CHECK_CANCEL_GOTO(pool, cleanup); /* Worker HTs are no longer needed once the merge is done. */ @@ -10960,7 +10992,7 @@ ht_path:; group_ht_free(&wpart_hts[i]); scratch_free(wpart_hdr); if (atomic_load_explicit(&v2p2.oom, memory_order_relaxed)) { - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < radix_n_parts; p++) group_ht_free(&v2_part_hts[p]); scratch_free(v2_part_hdr); goto v2_done; @@ -10971,31 +11003,12 @@ ht_path:; goto v2_emit; } v2_done:; - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * radix_n_parts; radix_bufs = (radix_buf_t*)scratch_calloc(&radix_bufs_hdr, n_bufs * sizeof(radix_buf_t)); if (!radix_bufs) goto sequential_fallback; - /* Pre-size each buffer: 1.5x expected, capped so total ≤ 2 GB. - * Buffers grow on demand via radix_buf_push doubling. */ - uint32_t buf_init = (uint32_t)((uint64_t)nrows / (RADIX_P * n_total)); - if (buf_init < 64) buf_init = 64; - buf_init = buf_init + buf_init / 2; /* 1.5x headroom */ - uint16_t estride = ght_layout.entry_stride; - { - /* Cap: total pre-alloc ≤ 2 GB */ - size_t total_pre = (size_t)n_bufs * buf_init * estride; - if (total_pre > (size_t)2 << 30) { - buf_init = (uint32_t)(((size_t)2 << 30) / ((size_t)n_bufs * estride)); - if (buf_init < 64) buf_init = 64; - } - } - for (size_t i = 0; i < n_bufs; i++) { - radix_bufs[i].data = (char*)scratch_alloc( - &radix_bufs[i]._hdr, (size_t)buf_init * estride); - radix_bufs[i].count = 0; - radix_bufs[i].cap = buf_init; - } + /* Buffers start empty and grow from observed partition demand. */ /* Any key column that may hold nulls — lets phase1 skip the per-key * null check on the common (no-nulls) fast path. 0/1 summary so key @@ -11015,6 +11028,7 @@ v2_done:; .agg_vecs2 = agg_vecs2, .agg_strlen = agg_strlen, .n_workers = n_total, + .n_parts = radix_n_parts, .bufs = radix_bufs, .rowsel = rowsel, .match_idx = match_idx, @@ -11049,7 +11063,7 @@ v2_done:; /* Pass 2: parallel per-partition aggregation (no column access) */ part_hts = (group_ht_t*)scratch_calloc(&part_hts_hdr, - RADIX_P * sizeof(group_ht_t)); + (size_t)radix_n_parts * sizeof(group_ht_t)); if (!part_hts) { for (size_t i = 0; i < n_bufs; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); @@ -11061,6 +11075,7 @@ v2_done:; .key_types = key_types, .n_keys = n_keys, .n_workers = n_total, + .n_parts = radix_n_parts, .bufs = radix_bufs, .part_hts = part_hts, .key_data = key_data, @@ -11076,12 +11091,12 @@ v2_done:; if (!p2ctx.key_pool) { result = ray_error("oom", NULL); goto cleanup; } derive_key_pool(&ght_layout, key_vecs, p2ctx.key_pool); } - ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, radix_n_parts); scratch_free(p2_kp_hdr); CHECK_CANCEL_GOTO(pool, cleanup); if (radix_bufs) { - size_t n_bufs_free = (size_t)n_total * RADIX_P; + size_t n_bufs_free = (size_t)n_total * radix_n_parts; for (size_t i = 0; i < n_bufs_free; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); @@ -11099,7 +11114,7 @@ v2_emit:; * partition's row array to contain only globally-surviving rows. * Phase3 below then emits N rows total instead of total_grps — * the major win for high-cardinality keys like UserID/URL where - * total_grps is in the millions but N is ≤ 1024. + * total_grps is in the millions but N is much smaller. * * Implementation notes: * - The bounded heap orders by count (the agg at COUNT slot, the @@ -11113,8 +11128,8 @@ v2_emit:; * min_count_exclusive-only filters fall through unchanged. */ if (use_topn_filter) { int64_t k_take = emit_filter.top_count_take; - uint32_t total_pre = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + uint64_t total_pre = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) total_pre += part_hts[p].grp_count; /* Resolve the in-row offset of the order-by agg's value. For * COUNT it's the leading int64 at offset 0; for SUM/MIN/MAX @@ -11158,15 +11173,16 @@ v2_emit:; * isn't set (old single-bit filter shape). Producer code in * query.c sets it explicitly. */ if (order_op == OP_COUNT && !emit_filter.desc) desc_dir = 1; - if ((int64_t)total_pre > k_take && k_take > 0 && k_take <= 1024) { - /* Stack heap: (val, part, gid) triples. k_take ≤ 1024 - * caps the footprint at 1024 * 16 B = 16 KiB. The heap - * invariant flips by direction: min-heap for desc (we - * evict the smallest to keep the largest N), max-heap - * for asc (evict the largest to keep the smallest N). */ - int64_t hval[1024]; - uint32_t hpart[1024]; - uint32_t hgid[1024]; + if (total_pre > (uint64_t)k_take && k_take > 0 && + (uint64_t)k_take <= SIZE_MAX / sizeof(group_topn_item_t)) { + /* The heap is sized by the requested result cardinality. + * There is no machine-tuned maximum: allocation failure only + * skips this optional compaction, leaving the normal final + * sort/take path to produce the same answer. */ + ray_t* heap_hdr = NULL; + group_topn_item_t* heap = (group_topn_item_t*)scratch_alloc( + &heap_hdr, (size_t)k_take * sizeof(group_topn_item_t)); + if (!heap) goto topn_compact_skip; int64_t hn = 0; /* For top-N largest (desc=1): min-heap. Root is smallest; * incoming v replaces root iff v > root. Heap invariant: @@ -11182,7 +11198,7 @@ v2_emit:; (desc_dir ? ((parent) > (child)) : ((parent) < (child))) #define TOPN_SHOULD_REPLACE(new_v, root_v) \ (desc_dir ? ((new_v) > (root_v)) : ((new_v) < (root_v))) - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < radix_n_parts; p++) { group_ht_t* ph = &part_hts[p]; uint16_t rs = ph->layout.row_stride; uint32_t gc = ph->grp_count; @@ -11192,31 +11208,32 @@ v2_emit:; (row + order_off); if (hn < k_take) { int64_t j = hn++; - hval[j] = v; hpart[j] = p; hgid[j] = gi; + heap[j] = (group_topn_item_t){v, p, gi}; /* Sift up: bubble new entry toward root while * parent violates invariant. */ while (j > 0) { int64_t pr = (j - 1) >> 1; - if (!TOPN_NEEDS_SWAP(hval[pr], hval[j])) break; - int64_t tc = hval[pr]; hval[pr] = hval[j]; hval[j] = tc; - uint32_t tp = hpart[pr]; hpart[pr] = hpart[j]; hpart[j] = tp; - uint32_t tg = hgid[pr]; hgid[pr] = hgid[j]; hgid[j] = tg; + if (!TOPN_NEEDS_SWAP(heap[pr].value, + heap[j].value)) break; + group_topn_item_t tmp = heap[pr]; + heap[pr] = heap[j]; heap[j] = tmp; j = pr; } - } else if (TOPN_SHOULD_REPLACE(v, hval[0])) { - hval[0] = v; hpart[0] = p; hgid[0] = gi; + } else if (TOPN_SHOULD_REPLACE(v, heap[0].value)) { + heap[0] = (group_topn_item_t){v, p, gi}; int64_t j = 0; /* Sift down: find the child that should be * promoted (the one most violating the * invariant) and swap. */ for (;;) { int64_t l = j * 2 + 1, r = l + 1, m = j; - if (l < hn && TOPN_NEEDS_SWAP(hval[m], hval[l])) m = l; - if (r < hn && TOPN_NEEDS_SWAP(hval[m], hval[r])) m = r; + if (l < hn && TOPN_NEEDS_SWAP(heap[m].value, + heap[l].value)) m = l; + if (r < hn && TOPN_NEEDS_SWAP(heap[m].value, + heap[r].value)) m = r; if (m == j) break; - int64_t tc = hval[m]; hval[m] = hval[j]; hval[j] = tc; - uint32_t tp = hpart[m]; hpart[m] = hpart[j]; hpart[j] = tp; - uint32_t tg = hgid[m]; hgid[m] = hgid[j]; hgid[j] = tg; + group_topn_item_t tmp = heap[m]; + heap[m] = heap[j]; heap[j] = tmp; j = m; } } @@ -11225,56 +11242,124 @@ v2_emit:; #undef TOPN_NEEDS_SWAP #undef TOPN_SHOULD_REPLACE if (hn > 0) { - /* Build per-partition keep lists (sorted asc by gid so - * the in-place compact below is a single forward sweep). */ - uint16_t keep_n[RADIX_P]; - for (uint32_t p = 0; p < RADIX_P; p++) keep_n[p] = 0; - /* Cap per-partition kept count at hn (≤ k_take ≤ 1024). */ - uint32_t kgid[RADIX_P][1024]; - for (int64_t i = 0; i < hn; i++) { - uint32_t p = hpart[i]; - uint16_t kn = keep_n[p]; - /* Insertion-sort into kgid[p][] keeping asc order. */ - uint16_t j = kn; - while (j > 0 && kgid[p][j - 1] > hgid[i]) { - kgid[p][j] = kgid[p][j - 1]; - j--; + uint64_t all_groups = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) + all_groups += part_hts[p].grp_count; + ray_t* keep_hdr = NULL; + ray_t* off_hdr = NULL; + uint8_t* keep = all_groups <= SIZE_MAX + ? (uint8_t*)scratch_calloc(&keep_hdr, + (size_t)all_groups) + : NULL; + uint32_t* keep_off = + (size_t)radix_n_parts + 1 <= SIZE_MAX / sizeof(uint32_t) + ? (uint32_t*)scratch_alloc(&off_hdr, + ((size_t)radix_n_parts + 1) * sizeof(uint32_t)) + : NULL; + if (keep && keep_off) { + keep_off[0] = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) + keep_off[p + 1] = keep_off[p] + + part_hts[p].grp_count; + for (int64_t i = 0; i < hn; i++) + keep[keep_off[heap[i].part] + heap[i].gid] = 1; + + /* In-place compact each partition in original gid + * order, preserving deterministic tie behaviour. */ + bool rebuilt_slots = false; + for (uint32_t p = 0; p < radix_n_parts; p++) { + group_ht_t* ph = &part_hts[p]; + uint16_t rs = ph->layout.row_stride; + uint32_t old_n = ph->grp_count; + uint32_t kn = 0; + for (uint32_t gi = 0; gi < old_n; gi++) { + if (!keep[keep_off[p] + gi]) continue; + if (gi != kn) + memmove(ph->rows + (size_t)kn * rs, + ph->rows + (size_t)gi * rs, rs); + kn++; + } + if (kn == ph->grp_count) continue; + rebuilt_slots = true; + ph->grp_count = kn; } - kgid[p][j] = hgid[i]; - keep_n[p] = (uint16_t)(kn + 1); - } - /* In-place compact each partition. */ - bool rebuilt_slots = false; - for (uint32_t p = 0; p < RADIX_P; p++) { - group_ht_t* ph = &part_hts[p]; - uint16_t rs = ph->layout.row_stride; - uint16_t kn = keep_n[p]; - if (kn == ph->grp_count) continue; /* all kept */ - rebuilt_slots = true; - if (kn == 0) { ph->grp_count = 0; continue; } - for (uint16_t i = 0; i < kn; i++) { - uint32_t src = kgid[p][i]; - if (src == (uint32_t)i) continue; - memmove(ph->rows + (size_t)i * rs, - ph->rows + (size_t)src * rs, rs); + if (rebuilt_slots) { + for (uint32_t p = 0; p < radix_n_parts; p++) + group_ht_rebuild_slots(&part_hts[p], key_types); } - ph->grp_count = kn; - } - if (rebuilt_slots) { - for (uint32_t p = 0; p < RADIX_P; p++) - group_ht_rebuild_slots(&part_hts[p], key_types); } + scratch_free(off_hdr); + scratch_free(keep_hdr); } + scratch_free(heap_hdr); } topn_compact_skip:; } /* Prefix offsets */ - uint32_t part_offsets[RADIX_P + 1]; + part_offsets = (uint32_t*)scratch_alloc(&part_offsets_hdr, + ((size_t)radix_n_parts + 1) * sizeof(uint32_t)); + if (!part_offsets) { result = ray_error("oom", NULL); goto cleanup; } part_offsets[0] = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < radix_n_parts; p++) part_offsets[p + 1] = part_offsets[p] + part_hts[p].grp_count; - uint32_t total_grps = part_offsets[RADIX_P]; + uint32_t total_grps = part_offsets[radix_n_parts]; + + /* Radix partitioning is an execution detail, not an ordering rule. + * Index each group by its earliest source row, then scan that row + * domain once to assign stable output ids. This is a single linear + * algorithm: no comparison sort, crossover, or fixed fanout. */ + if (total_grps > 0) { + group_out = (uint32_t*)scratch_alloc(&group_out_hdr, + (size_t)total_grps * sizeof(uint32_t)); + if (!group_out) { + result = ray_error("oom", NULL); + goto cleanup; + } + uint64_t row_count = nrows > 0 ? (uint64_t)nrows : 0; + if (row_count > SIZE_MAX / sizeof(uint32_t)) { + result = ray_error("limit", "group row index is too large"); + goto cleanup; + } + ray_t* first_hdr = NULL; + uint32_t* first_to_flat = (uint32_t*)scratch_alloc( + &first_hdr, (size_t)row_count * sizeof(uint32_t)); + if (!first_to_flat) { + result = ray_error("oom", NULL); + goto cleanup; + } + memset(first_to_flat, 0xff, + (size_t)row_count * sizeof(uint32_t)); + bool order_ok = true; + for (uint32_t p = 0; p < radix_n_parts && order_ok; p++) { + group_ht_t* ph = &part_hts[p]; + for (uint32_t gi = 0; gi < ph->grp_count; gi++) { + uint32_t flat = part_offsets[p] + gi; + const char* group_row = ph->rows + + (size_t)gi * ph->layout.row_stride; + int64_t first; + memcpy(&first, + group_row + ph->layout.off_group_first, 8); + if (first < 0 || first >= nrows || + first_to_flat[first] != UINT32_MAX) { + order_ok = false; + break; + } + first_to_flat[first] = flat; + } + } + uint32_t out = 0; + if (order_ok) + for (int64_t row = 0; row < nrows; row++) { + uint32_t flat = first_to_flat[row]; + if (flat != UINT32_MAX) group_out[flat] = out++; + } + scratch_free(first_hdr); + if (!order_ok || out != total_grps) { + result = ray_error("group", "failed to order all groups"); + goto cleanup; + } + } /* Build result directly from partition HTs */ int64_t total_cols = n_keys + n_aggs; @@ -11389,6 +11474,7 @@ v2_emit:; radix_phase3_ctx_t p3ctx = { .part_hts = part_hts, .part_offsets = part_offsets, + .group_out = group_out, .key_dsts = key_dsts, .key_types = key_out_types, .key_attrs = key_attrs, @@ -11399,7 +11485,8 @@ v2_emit:; .n_aggs = n_aggs, .key_src_data = key_data, }; - ray_pool_dispatch_n(pool, radix_phase3_fn, &p3ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase3_fn, &p3ctx, + radix_n_parts); } /* Post-radix holistic fill: OP_MEDIAN slots need a per-group @@ -11431,6 +11518,8 @@ v2_emit:; .layout = &ght_layout, .part_hts = part_hts, .part_offsets = part_offsets, + .group_out = group_out, + .n_parts = radix_n_parts, .row_gid = row_gid, .match_idx = match_idx, .rowsel = rowsel, @@ -12204,16 +12293,18 @@ sequential_fallback:; group_ht_free(&top_ht); } if (radix_bufs) { - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * radix_n_parts; for (size_t i = 0; i < n_bufs; i++) scratch_free(radix_bufs[i]._hdr); scratch_free(radix_bufs_hdr); } if (part_hts) { - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < radix_n_parts; p++) { if (part_hts[p].rows) group_ht_free(&part_hts[p]); } scratch_free(part_hts_hdr); } + scratch_free(part_offsets_hdr); + scratch_free(group_out_hdr); /* Master layout owns any spill block; every by-value copy borrowed it. * cleanup: is reached only after ght_layout is initialised (all gotos to * it are below the ght_compute_layout call). NULL-safe / no-op inline. */ @@ -13144,7 +13235,7 @@ exec_group_per_partition(ray_graph_t* g, ray_t* parted_tbl, ray_op_ext_t* ext, * the result in per-partition HTs with prefix offsets so the caller * can iterate grouped rows without knowing about the radix internals. * Falls back to a single sequential HT for tiny inputs or when no - * pool is available — the caller iterates n_parts ∈ {1, RADIX_P}. + * pool is available — the caller iterates the returned n_parts value. * ══════════════════════════════════════════════════════════════════════ */ static void pivot_ingest_sequential(pivot_ingest_t* out, const ght_layout_t* ly, @@ -13172,9 +13263,15 @@ bool pivot_ingest_run(pivot_ingest_t* out, memset(out, 0, sizeof(*out)); out->row_stride = ly->row_stride; - /* Allocate a small offsets buffer up front (RADIX_P+1 is the max). */ + ray_pool_t* pool = ray_pool_get(); + uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; + bool parallel_ok = (pool && n_scan >= RAY_PARALLEL_THRESHOLD && n_total > 1); + uint32_t n_parts = parallel_ok + ? group_radix_part_count(n_total, n_scan) : 1; + + /* Prefix offsets are sized by the execution-derived partition count. */ out->part_offsets = (uint32_t*)scratch_alloc(&out->_offsets_hdr, - (size_t)(RADIX_P + 1) * sizeof(uint32_t)); + ((size_t)n_parts + 1) * sizeof(uint32_t)); if (!out->part_offsets) return false; /* uint32_t, not uint8_t: ly->n_keys is uint16_t (unbounded-ready, Task @@ -13187,10 +13284,6 @@ bool pivot_ingest_run(pivot_ingest_t* out, * index key is wide (RAY_GUID/RAY_STR) and n_keys > 255. */ uint32_t n_keys = ly->n_keys; - ray_pool_t* pool = ray_pool_get(); - uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; - bool parallel_ok = (pool && n_scan >= RAY_PARALLEL_THRESHOLD && n_total > 1); - if (!parallel_ok) { /* Sequential single-HT path — allocate the HT in its own scratch * block and wire part_hts/n_parts immediately so every failure @@ -13200,10 +13293,7 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (!seq) return false; out->part_hts = seq; out->n_parts = 1; - uint32_t seq_cap = 1024; - uint64_t target = (uint64_t)n_scan * 2; - while ((uint64_t)seq_cap < target && seq_cap < (1u << 24)) seq_cap <<= 1; - if (!group_ht_init(seq, seq_cap, ly)) return false; + if (!group_ht_init(seq, 2, ly)) return false; pivot_ingest_sequential(out, ly, key_data, key_types, key_attrs, key_vecs, agg_vecs, n_scan, seq); /* Surface grow-path OOM from group_probe_entry so callers don't @@ -13213,30 +13303,14 @@ bool pivot_ingest_run(pivot_ingest_t* out, } /* ═════ Parallel radix path ═════ */ - size_t n_bufs = (size_t)n_total * RADIX_P; + size_t n_bufs = (size_t)n_total * n_parts; out->_n_bufs = n_bufs; radix_buf_t* radix_bufs = (radix_buf_t*)scratch_calloc(&out->_radix_bufs_hdr, n_bufs * sizeof(radix_buf_t)); if (!radix_bufs) return false; out->_radix_bufs = radix_bufs; - uint32_t buf_init = (uint32_t)((uint64_t)n_scan / (RADIX_P * n_total)); - if (buf_init < 64) buf_init = 64; - buf_init = buf_init + buf_init / 2; - uint16_t estride = ly->entry_stride; - { - size_t total_pre = (size_t)n_bufs * buf_init * estride; - if (total_pre > (size_t)2 << 30) { - buf_init = (uint32_t)(((size_t)2 << 30) / ((size_t)n_bufs * estride)); - if (buf_init < 64) buf_init = 64; - } - } - for (size_t i = 0; i < n_bufs; i++) { - radix_bufs[i].data = (char*)scratch_alloc(&radix_bufs[i]._hdr, - (size_t)buf_init * estride); - radix_bufs[i].count = 0; - radix_bufs[i].cap = buf_init; - } + /* Buffers start empty and grow from observed partition demand. */ uint8_t p1_nullable = 0; /* 0/1: any key may be null (per-key re-checked in phase1) */ for (uint32_t k = 0; k < n_keys; k++) @@ -13251,6 +13325,7 @@ bool pivot_ingest_run(pivot_ingest_t* out, .agg_vecs = agg_vecs, .agg_vecs2 = NULL, /* this scratch path doesn't use binary aggs */ .n_workers = n_total, + .n_parts = n_parts, .bufs = radix_bufs, .match_idx = NULL, }; @@ -13273,18 +13348,19 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (radix_bufs[i].oom) return false; group_ht_t* part_hts = (group_ht_t*)scratch_calloc(&out->_part_hts_hdr, - RADIX_P * sizeof(group_ht_t)); + (size_t)n_parts * sizeof(group_ht_t)); if (!part_hts) return false; /* Wire out->part_hts/n_parts at alloc time (like the sequential path) so * every failure below funnels through pivot_ingest_free — the calloc-zeroed * HTs are skipped by its rows||slots guard until phase2 fills them. */ out->part_hts = part_hts; - out->n_parts = RADIX_P; + out->n_parts = n_parts; radix_phase2_ctx_t p2ctx = { .key_types = key_types, .n_keys = n_keys, .n_workers = n_total, + .n_parts = n_parts, .bufs = radix_bufs, .part_hts = part_hts, .key_data = key_data, @@ -13298,11 +13374,11 @@ bool pivot_ingest_run(pivot_ingest_t* out, if (!p2ctx.key_pool) return false; derive_key_pool(ly, key_vecs, p2ctx.key_pool); } - ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, RADIX_P); + ray_pool_dispatch_n(pool, radix_phase2_fn, &p2ctx, n_parts); scratch_free(p2_kp_hdr); if (ray_interrupted()) return true; - /* Sync point — partitions materialized; show RADIX_P/RADIX_P. */ - ray_progress_update(NULL, "per-partition aggregate", RADIX_P, RADIX_P); + /* Sync point — every execution-derived partition is materialized. */ + ray_progress_update(NULL, "per-partition aggregate", n_parts, n_parts); /* OOM detection for the parallel path. Two distinct failure modes * must be caught here so callers never see a silently-truncated @@ -13314,19 +13390,19 @@ bool pivot_ingest_run(pivot_ingest_t* out, * (b) grow-path OOM — group_probe_entry sets part_hts[p].oom * on scratch_realloc failure and returns without inserting * the key, silently truncating later groups. */ - for (uint32_t p = 0; p < RADIX_P; p++) { + for (uint32_t p = 0; p < n_parts; p++) { if (part_hts[p].oom) return false; if (part_hts[p].rows) continue; uint32_t pcount = 0; for (uint32_t w = 0; w < n_total; w++) - pcount += radix_bufs[(size_t)w * RADIX_P + p].count; + pcount += radix_bufs[(size_t)w * n_parts + p].count; if (pcount) return false; } out->part_offsets[0] = 0; - for (uint32_t p = 0; p < RADIX_P; p++) + for (uint32_t p = 0; p < n_parts; p++) out->part_offsets[p + 1] = out->part_offsets[p] + part_hts[p].grp_count; - out->total_grps = out->part_offsets[RADIX_P]; + out->total_grps = out->part_offsets[n_parts]; return true; } diff --git a/src/ops/internal.h b/src/ops/internal.h index ac2493ab3..ac0094eb9 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -1116,6 +1116,10 @@ typedef struct { uint16_t off_sum_y; uint16_t off_sumsq_y; uint16_t off_sumxy; + /* Earliest contributing source row for this group. Every packed entry + * carries its source row in the tail slot; partition merges retain the + * minimum so output order is independent of radix partition count. */ + uint16_t off_group_first; /* Per-slot non-null count block (n_agg_vals int64 slots), allocated only * when any_agg_null (any agg input column HAS_NULLS). Zero otherwise — * finalize then divides by the group row count exactly as before, so @@ -1320,7 +1324,7 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, typedef struct { group_ht_t* part_hts; /* n_parts entries */ uint32_t* part_offsets; /* n_parts+1 entries (prefix sums of grp_counts) */ - uint32_t n_parts; /* 1 when sequential, RADIX_P when parallel */ + uint32_t n_parts; /* 1 sequential; execution-derived when parallel */ uint32_t total_grps; uint16_t row_stride; diff --git a/test/test_agg_engine.c b/test/test_agg_engine.c index 9c4c9eff0..c66147cbd 100644 --- a/test/test_agg_engine.c +++ b/test/test_agg_engine.c @@ -208,7 +208,7 @@ static test_result_t test_dense_plan_single_i64(void) { PASS(); } -/* (b) two I64 keys, ranges 100 and 50 → dense, slots 5000, strides {1,100}. */ +/* (b) two I64 keys, ranges 10 and 10 → dense, slots equal input rows. */ static test_result_t test_dense_plan_two_keys(void) { ray_heap_init(); (void)ray_sym_init(); @@ -217,7 +217,7 @@ static test_result_t test_dense_plan_two_keys(void) { ray_t* k2 = ray_vec_new(RAY_I64, 100); k2->len = 100; int64_t* d1 = (int64_t*)ray_data(k1); int64_t* d2 = (int64_t*)ray_data(k2); - for (int64_t i = 0; i < 100; i++) { d1[i] = i; d2[i] = i % 50; } /* ranges 100, 50 */ + for (int64_t i = 0; i < 100; i++) { d1[i] = i % 10; d2[i] = i / 10; } const agg_vtable_t* vt = dense_sum_i64_vt(); TEST_ASSERT_NOT_NULL(vt); @@ -229,11 +229,11 @@ static test_result_t test_dense_plan_two_keys(void) { TEST_ASSERT_TRUE(ok); TEST_ASSERT_TRUE(pl.ok); - TEST_ASSERT_EQ_I(pl.total_slots, 5000); - TEST_ASSERT_EQ_I(pl.ranges[0], 100); - TEST_ASSERT_EQ_I(pl.ranges[1], 50); + TEST_ASSERT_EQ_I(pl.total_slots, 100); + TEST_ASSERT_EQ_I(pl.ranges[0], 10); + TEST_ASSERT_EQ_I(pl.ranges[1], 10); TEST_ASSERT_EQ_I(pl.strides[0], 1); - TEST_ASSERT_EQ_I(pl.strides[1], 100); + TEST_ASSERT_EQ_I(pl.strides[1], 10); ray_release(k1); ray_release(k2); @@ -242,14 +242,14 @@ static test_result_t test_dense_plan_two_keys(void) { PASS(); } -/* (c) single I64 key with huge range (0 and 10,000,000) → not dense (over cap). */ +/* (c) A two-row input cannot justify a 10,000,001-slot dense state. */ static test_result_t test_dense_plan_huge_range(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* k = ray_vec_new(RAY_I64, 2); k->len = 2; int64_t* kd = (int64_t*)ray_data(k); - kd[0] = 0; kd[1] = 10000000; /* range > DENSE_MAX_SLOTS */ + kd[0] = 0; kd[1] = 10000000; /* dense state would exceed input size */ const agg_vtable_t* vt = dense_sum_i64_vt(); TEST_ASSERT_NOT_NULL(vt); @@ -862,14 +862,13 @@ static ray_t* diff_make_2i64_hc(int64_t n, int64_t nk) { /* ── RADIX-FORCING builders (Phase 1c radix) ───────────────────────────── * The radix path is selected for high-card STREAMING int/SYM-key queries whose - * DENSE plan is rejected (total_slots > DENSE_MAX_SLOTS=262144, or per-worker - * slab > 8MB). These builders give a WIDE key value range so the dense plan - * fails and the selector routes to exec_group_v2_parallel_radix; the resulting - * table is still compared byte-for-byte (as a key-sorted multiset) vs the old - * engine. At N=70000 (≥ RAY_PARALLEL_THRESHOLD) the parallel dispatch runs. */ - -/* Single I64 key "k" = (i % 50000) * 7 (≈50000 groups, range ≈350000 > - * DENSE_MAX_SLOTS → dense plan rejected → radix). I64 value "v" = i. */ + * dense representation would require more slots than the contributing input + * has rows. These builders give a wide key range so the structural dense-plan + * rule selects radix; results are compared byte-for-byte (as a key-sorted + * multiset) with the old engine. */ + +/* Single I64 key "k" = (i % 50000) * 7: its range needs more dense slots than + * the input has rows, so it follows the radix representation. */ static ray_t* diff_make_i64_radix(int64_t n, int64_t nk) { (void)nk; ray_t* kvec = ray_vec_new(RAY_I64, n); kvec->len = n; @@ -888,8 +887,8 @@ static ray_t* diff_make_i64_radix(int64_t n, int64_t nk) { /* Two I64 keys with WIDE composite range → dense plan rejected → radix. * k1 = (i%4000)*5 (range ≈20000), k2 = ((i/4000)%4000)*5 → product of ranges - * ≫ DENSE_MAX_SLOTS. ≈ up to 16M composite slots but only ~17500 live tuples - * at N=70000. I64 value "v" = i. */ + * greatly exceeds the input row count. There are up to 16M composite slots + * but only about 17500 live tuples at N=70000. */ static ray_t* diff_make_2i64_radix(int64_t n, int64_t nk) { (void)nk; ray_t* k1 = ray_vec_new(RAY_I64, n); k1->len = n; @@ -939,7 +938,7 @@ static ray_t* diff_make_pearson_radix(int64_t n, int64_t nk) { double* xd = (double*)ray_data(xvec); double* yd = (double*)ray_data(yvec); for (int64_t i = 0; i < n; i++) { - kd[i] = (i % 20000) * 17; /* range ≈340000 > DENSE_MAX_SLOTS */ + kd[i] = (i % 20000) * 17; /* dense range exceeds input rows */ xd[i] = (double)((i % 89) * 1.0); yd[i] = (double)(((i * 7) % 83) * 1.0); } @@ -953,7 +952,7 @@ static ray_t* diff_make_pearson_radix(int64_t n, int64_t nk) { /* Q10-shape 6-key tuple (high card → radix): six int/SYM keys k1..k6 + I64 "v". * Keys vary at independent rates so the 6-tuple space is broad (≈ thousands of * live tuples at N=70000) and the composite range is astronomically larger than - * DENSE_MAX_SLOTS → dense rejected → radix exercises the 6-key tuple hash/eq. */ + * the input row count, so radix exercises the 6-key tuple hash/eq. */ static ray_t* diff_make_6keys_radix(int64_t n, int64_t nk) { (void)nk; ray_t* k1 = ray_vec_new(RAY_I64, n); k1->len = n; @@ -1427,6 +1426,12 @@ static ray_op_t* gb_top100(ray_graph_t* g) { int64_t kk[] = { 100 }; ray_op_t* keys[] = { k }; return ray_group_build(g, keys, 1, ops, ins, NULL, kk, 1); } +static ray_op_t* gb_top2048(ray_graph_t* g) { + ray_op_t* k = ray_scan(g, "k"); ray_op_t* v = ray_scan(g, "v"); + uint16_t ops[] = { OP_TOP_N }; ray_op_t* ins[] = { v }; + int64_t kk[] = { 2048 }; ray_op_t* keys[] = { k }; + return ray_group_build(g, keys, 1, ops, ins, NULL, kk, 1); +} /* heterogeneous: sum + top2 (LIST col) + count, single I64 key. */ static ray_op_t* gb_sum_top2_count(ray_graph_t* g) { ray_op_t* k = ray_scan(g, "k"); ray_op_t* v = ray_scan(g, "v"); @@ -1603,6 +1608,17 @@ static test_result_t test_diff_group_top100(void) { return r; } +/* K above the former fixed stack ceiling. One 3000-row group ensures both + * engines materialize and compare a full 2048-element typed result cell. */ +static test_result_t test_diff_group_top2048(void) { + ray_heap_init(); (void)ray_sym_init(); + ray_t* tbl = diff_make_topk_i64_distinctv(3000, 1); + test_result_t r = diff_group(tbl, gb_top2048, 1); + ray_release(tbl); + ray_sym_destroy(); ray_heap_destroy(); + return r; +} + /* ══════════════════════════════════════════════════════════════════════ * WHERE-FILTERED differentials (architectural enabler: g->selection routes * through exec_group_v2's compact-table prologue). Each runs OP_GROUP with a @@ -1679,10 +1695,10 @@ DIFF_SHAPE(test_diff_group_2k_avg, diff_make_2i64_f64, gb_2k_avg, 2) DIFF_SHAPE(test_diff_group_2k_keynull, diff_make_2i64_keynull, gb_2k_sum, 2) /* ══════════════════════════════════════════════════════════════════════ - * HIGH-CARDINALITY differentials (Phase 1c). N=70000 (≥ RAY_PARALLEL_THRESHOLD - * 65536) with the v2 flag ON forces the parallel two-phase path; MANY groups - * mean each per-worker local table holds many groups and Phase B merges real - * work. Compared as MULTISET (key-sorted) vs the OLD engine. + * HIGH-CARDINALITY differentials (Phase 1c). These inputs are large enough to + * exercise the engine's parallel two-phase path; many groups make every local + * table and the merge do real work. Compared as a key-sorted multiset with the + * old engine. * ══════════════════════════════════════════════════════════════════════ */ #define HC_N 70000 @@ -1777,8 +1793,8 @@ static test_result_t test_diff_group_radix_2k_sum(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* big = diff_make_2i64_radix(HC_N, 0); int64_t ng = v2_group_count(big, gb_2k_sum); - /* All (i%4000, i/4000) pairs distinct at N=70000 → 70000 groups; the wide - * composite range (≈1.7M slots > DENSE_MAX_SLOTS) is what forces radix. */ + /* All (i%4000, i/4000) pairs are distinct at N=70000; their composite + * dense range exceeds the input size, which structurally selects radix. */ TEST_ASSERT_FMT(ng == 70000, "expected 70000 groups, got %lld", (long long)ng); test_result_t r = diff_group(big, gb_2k_sum, 2); ray_release(big); @@ -1811,14 +1827,12 @@ static test_result_t test_diff_group_radix_6k(void) { } /* ══════════════════════════════════════════════════════════════════════ - * SMALL-HASH differential: HIGH-REDUCTION sparse-low-card shape. + * SPARSE LOW-CARDINALITY differential. * * Few groups (64) whose key VALUES are wide/sparse: k=(i%64)*100000001 spans - * 0..~6.3e9, so the dense plan is rejected (range ≫ DENSE_MAX_SLOTS), yet the - * cardinality is tiny. N=70000 ≥ RAY_PARALLEL_THRESHOLD so it runs parallel, - * and the streaming int-key + LOW-estimate routing sends it to the O(groups) - * small-hash path. sum/count/min/max must match the OLD engine exactly - * (key-sorted multiset). This correctness-gates the new routing. + * 0..~6.3e9, so a dense representation would be superlinear in the input even + * though cardinality is tiny. The deterministic sparse-key route must match + * the old engine exactly for sum/count/min/max (key-sorted multiset). * ══════════════════════════════════════════════════════════════════════ */ static ray_t* diff_make_i64_sparse_lowcard(int64_t n, int64_t nk) { (void)nk; @@ -1836,7 +1850,7 @@ static ray_t* diff_make_i64_sparse_lowcard(int64_t n, int64_t nk) { return tbl; } -/* 64 groups, sparse keys, SUM+COUNT+MIN+MAX → small-hash path. */ +/* 64 groups, sparse keys, SUM+COUNT+MIN+MAX. */ static test_result_t test_diff_group_smallhash_sparse_lowcard(void) { ray_heap_init(); (void)ray_sym_init(); ray_t* big = diff_make_i64_sparse_lowcard(HC_N, 0); @@ -2045,6 +2059,7 @@ const test_entry_t agg_engine_entries[] = { { "diff_group_2k_top2", test_diff_group_2k_top2, NULL, NULL }, { "diff_group_mixed_top2", test_diff_group_mixed_top2, NULL, NULL }, { "diff_group_top100", test_diff_group_top100, NULL, NULL }, + { "diff_group_top2048", test_diff_group_top2048, NULL, NULL }, { "diff_group_2k_sum", test_diff_group_2k_sum, NULL, NULL }, { "diff_group_2k_four", test_diff_group_2k_four, NULL, NULL }, { "diff_group_3k_count", test_diff_group_3k_count, NULL, NULL }, From 48eefc8672e8a32c41239fe1ad17f9bb0f0b6f6a Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 19:39:42 +0200 Subject: [PATCH 06/10] fix(core): make persisted F32 queryable Preserve explicit F32 columns through CSV and splayed storage, and execute their scalar, vector, optimizer, and aggregate paths with consistent type promotion. Remove fixed density and compiler-placement shortcuts from the affected routing paths. --- src/io/csv.c | 18 ++ src/io/csv.h | 1 + src/lang/internal.h | 6 + src/ops/agg.c | 53 ++-- src/ops/arith.c | 38 ++- src/ops/expr.c | 129 +++++++- src/ops/graph.c | 18 +- src/ops/group.c | 289 +++++++++--------- src/ops/idxop.c | 7 - src/ops/internal.h | 3 +- src/ops/opt.c | 45 ++- src/ops/pivot.c | 3 +- src/store/col.c | 2 +- .../rfl/system/csv_explicit_numeric_types.rfl | 48 +++ test/test_group_extra.c | 4 +- test/test_idx_route.c | 16 +- test/test_store.c | 38 +++ 17 files changed, 498 insertions(+), 220 deletions(-) create mode 100644 test/rfl/system/csv_explicit_numeric_types.rfl diff --git a/src/io/csv.c b/src/io/csv.c index 25a92c1c7..089158994 100644 --- a/src/io/csv.c +++ b/src/io/csv.c @@ -1037,6 +1037,7 @@ static void csv_parse_fn(void* arg, uint32_t worker_id, case CSV_TYPE_I16: ((int16_t*)ctx->col_data[c])[row] = NULL_I16; break; case CSV_TYPE_I32: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; case CSV_TYPE_I64: ((int64_t*)ctx->col_data[c])[row] = NULL_I64; break; + case CSV_TYPE_F32: ((float*)ctx->col_data[c])[row] = NULL_F32; break; case CSV_TYPE_F64: ((double*)ctx->col_data[c])[row] = NULL_F64; break; case CSV_TYPE_DATE: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; case CSV_TYPE_TIME: ((int32_t*)ctx->col_data[c])[row] = NULL_I32; break; @@ -1114,6 +1115,13 @@ static void csv_parse_fn(void* arg, uint32_t worker_id, if (is_null) my_had_null[c] = true; break; } + case CSV_TYPE_F32: { + bool is_null; + double v = fast_f64(fld, flen, &is_null); + ((float*)ctx->col_data[c])[row] = is_null ? NULL_F32 : (float)v; + if (is_null) my_had_null[c] = true; + break; + } case CSV_TYPE_DATE: { bool is_null; int32_t v = fast_date(fld, flen, &is_null); @@ -1206,6 +1214,7 @@ static void csv_parse_serial(const char* buf, size_t buf_size, case CSV_TYPE_I16: ((int16_t*)col_data[c])[row] = NULL_I16; break; case CSV_TYPE_I32: ((int32_t*)col_data[c])[row] = NULL_I32; break; case CSV_TYPE_I64: ((int64_t*)col_data[c])[row] = NULL_I64; break; + case CSV_TYPE_F32: ((float*)col_data[c])[row] = NULL_F32; break; case CSV_TYPE_F64: ((double*)col_data[c])[row] = NULL_F64; break; case CSV_TYPE_DATE: ((int32_t*)col_data[c])[row] = NULL_I32; break; case CSV_TYPE_TIME: ((int32_t*)col_data[c])[row] = NULL_I32; break; @@ -1281,6 +1290,13 @@ static void csv_parse_serial(const char* buf, size_t buf_size, if (is_null) col_had_null[c] = true; break; } + case CSV_TYPE_F32: { + bool is_null; + double v = fast_f64(fld, flen, &is_null); + ((float*)col_data[c])[row] = is_null ? NULL_F32 : (float)v; + if (is_null) col_had_null[c] = true; + break; + } case CSV_TYPE_DATE: { bool is_null; int32_t v = fast_date(fld, flen, &is_null); @@ -1629,6 +1645,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, case RAY_I16: parse_types[c] = CSV_TYPE_I16; break; case RAY_I32: parse_types[c] = CSV_TYPE_I32; break; case RAY_I64: parse_types[c] = CSV_TYPE_I64; break; + case RAY_F32: parse_types[c] = CSV_TYPE_F32; break; case RAY_F64: parse_types[c] = CSV_TYPE_F64; break; case RAY_DATE: parse_types[c] = CSV_TYPE_DATE; break; case RAY_TIME: parse_types[c] = CSV_TYPE_TIME; break; @@ -2037,6 +2054,7 @@ ray_t* ray_read_csv_named_opts(const char* path, char delimiter, bool header, case RAY_I16: parse_types[c] = CSV_TYPE_I16; break; case RAY_I32: parse_types[c] = CSV_TYPE_I32; break; case RAY_I64: parse_types[c] = CSV_TYPE_I64; break; + case RAY_F32: parse_types[c] = CSV_TYPE_F32; break; case RAY_F64: parse_types[c] = CSV_TYPE_F64; break; case RAY_DATE: parse_types[c] = CSV_TYPE_DATE; break; case RAY_TIME: parse_types[c] = CSV_TYPE_TIME; break; diff --git a/src/io/csv.h b/src/io/csv.h index f77b2af79..6b6b4c270 100644 --- a/src/io/csv.h +++ b/src/io/csv.h @@ -47,6 +47,7 @@ typedef enum { CSV_TYPE_U8, CSV_TYPE_I16, CSV_TYPE_I32, + CSV_TYPE_F32, /* Marker for "pick the narrowest integer width automatically" — used only * during schema processing, never returned by the resolver. */ CSV_TYPE_AUTO diff --git a/src/lang/internal.h b/src/lang/internal.h index e45a04cf6..6c030de3f 100644 --- a/src/lang/internal.h +++ b/src/lang/internal.h @@ -138,6 +138,10 @@ static inline int is_float_op(ray_t* a, ray_t* b) { a->type == -RAY_F32 || b->type == -RAY_F32; } +static inline int both_f32_atoms(ray_t* a, ray_t* b) { + return a->type == -RAY_F32 && b->type == -RAY_F32; +} + /* ══════════════════════════════════════════ * Null/type helpers * ══════════════════════════════════════════ */ @@ -148,6 +152,8 @@ static inline int is_float_op(ray_t* a, ray_t* b) { static inline ray_t* null_for_promoted(ray_t* a, ray_t* b) { if (a->type == -RAY_F64 || b->type == -RAY_F64) return ray_typed_null(-RAY_F64); + if (a->type == -RAY_F32 || b->type == -RAY_F32) + return ray_typed_null(both_f32_atoms(a, b) ? -RAY_F32 : -RAY_F64); if (a->type == -RAY_I64 || b->type == -RAY_I64) return ray_typed_null(-RAY_I64); if (a->type == -RAY_I32 || b->type == -RAY_I32) diff --git a/src/ops/agg.c b/src/ops/agg.c index 0c1d5f7ea..6c42d299e 100644 --- a/src/ops/agg.c +++ b/src/ops/agg.c @@ -93,7 +93,7 @@ static void nth_element_dbl(double* a, int64_t lo, int64_t hi, int64_t k) { static int agg_parted_numeric_base(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64 || + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64 || t == RAY_DATE || t == RAY_TIME || t == RAY_TIMESTAMP; } @@ -130,15 +130,17 @@ static ray_t* agg_parted_sum(ray_t* x) { if (!agg_parted_numeric_base(base) || base == RAY_DATE) return ray_error("type", "sum expects a numeric or time-duration parted column, got %s", ray_type_name(base)); ray_t** segs = (ray_t**)ray_data(x); - if (base == RAY_F64) { + if (base == RAY_F64 || base == RAY_F32) { double sum = 0.0; for (int64_t s = 0; s < x->len; s++) { ray_t* seg = segs[s]; if (!seg) continue; - double* d = (double*)ray_data(seg); int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; for (int64_t i = 0; i < seg->len; i++) - if (!has_nulls || !ray_vec_is_null(seg, i)) sum += d[i]; + if (!has_nulls || !ray_vec_is_null(seg, i)) { + if (base == RAY_F64) sum += ((double*)ray_data(seg))[i]; + else sum += (double)((float*)ray_data(seg))[i]; + } } return make_f64(sum); } @@ -165,11 +167,12 @@ static ray_t* agg_parted_avg(ray_t* x) { ray_t* seg = segs[s]; if (!seg) continue; int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; - if (base == RAY_F64) { - double* d = (double*)ray_data(seg); + if (base == RAY_F64 || base == RAY_F32) { for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - sum += d[i]; cnt++; + if (base == RAY_F64) sum += ((double*)ray_data(seg))[i]; + else sum += (double)((float*)ray_data(seg))[i]; + cnt++; } } else { for (int64_t i = 0; i < seg->len; i++) { @@ -189,16 +192,16 @@ static ray_t* agg_parted_prod(ray_t* x) { return ray_error("type", "prod expects a numeric parted column, got %s", ray_type_name(base)); } ray_t** segs = (ray_t**)ray_data(x); - if (base == RAY_F64) { + if (base == RAY_F64 || base == RAY_F32) { double prod = 1.0; for (int64_t s = 0; s < x->len; s++) { ray_t* seg = segs[s]; if (!seg) continue; - double* d = (double*)ray_data(seg); int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - prod *= d[i]; + if (base == RAY_F64) prod *= ((double*)ray_data(seg))[i]; + else prod *= (double)((float*)ray_data(seg))[i]; } } return make_f64(prod); @@ -228,11 +231,12 @@ static ray_t* agg_parted_minmax(ray_t* x, int want_max) { ray_t* seg = segs[s]; if (!seg) continue; int has_nulls = (seg->attrs & RAY_ATTR_HAS_NULLS) != 0; - if (base == RAY_F64) { - double* d = (double*)ray_data(seg); + if (base == RAY_F64 || base == RAY_F32) { for (int64_t i = 0; i < seg->len; i++) { if (has_nulls && ray_vec_is_null(seg, i)) continue; - double v = d[i]; + double v = base == RAY_F64 + ? ((double*)ray_data(seg))[i] + : (double)((float*)ray_data(seg))[i]; if (!found || (want_max ? v > best_f : v < best_f)) { best_f = v; found = 1; } @@ -249,18 +253,20 @@ static ray_t* agg_parted_minmax(ray_t* x, int want_max) { } if (!found) return ray_typed_null(-base); if (base == RAY_F64) return make_f64(best_f); + if (base == RAY_F32) return ray_f32((float)best_f); return agg_atom_i64_for_type(base, best_i); } static int agg_numeric_flat_type(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } static int agg_read_vec_f64(ray_t* v, int64_t i, double* out) { void* d = ray_data(v); switch (v->type) { case RAY_F64: *out = ((double*)d)[i]; return 1; + case RAY_F32: *out = (double)((float*)d)[i]; return 1; case RAY_I64: *out = (double)((int64_t*)d)[i]; return 1; case RAY_I32: *out = (double)((int32_t*)d)[i]; return 1; case RAY_I16: *out = (double)((int16_t*)d)[i]; return 1; @@ -362,7 +368,9 @@ static ray_t* agg_pair_vec(ray_t* x, ray_t* y, uint16_t op) { double xv = 0.0, yv = 0.0; agg_read_vec_f64(x, i, &xv); agg_read_vec_f64(y, i, &yv); - sx += xv; sy += yv; sxx += xv * xv; syy += yv * yv; sxy += xv * yv; cnt++; + sx += xv; sy += yv; sxx += xv * xv; syy += yv * yv; + sxy += xv * yv; + cnt++; } if (op == OP_WSUM) return ray_f64(ray_f64_fin(sxy)); @@ -449,10 +457,10 @@ ray_t* ray_sum_fn(ray_t* x) { for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "sum: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); if (RAY_ATOM_IS_NULL(elems[i])) { - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; continue; } - if (elems[i]->type == -RAY_F64) { has_float = 1; fsum += elems[i]->f64; } + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) { has_float = 1; fsum += elems[i]->f64; } else if (elems[i]->type == -RAY_I64) { isum += elems[i]->i64; fsum += (double)elems[i]->i64; } else { int64_t v = (int64_t)as_f64(elems[i]); isum += v; fsum += (double)v; } } @@ -479,10 +487,10 @@ ray_t* ray_prod_fn(ray_t* x) { if (!is_numeric(elems[i])) return ray_error("type", "prod: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); if (RAY_ATOM_IS_NULL(elems[i])) { - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; continue; } - if (elems[i]->type == -RAY_F64) { + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) { has_float = 1; fprod *= elems[i]->f64; } else { @@ -613,7 +621,7 @@ ray_t* ray_min_fn(ray_t* x) { double fmin = 0; int64_t imin = 0; for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "min: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; if (RAY_ATOM_IS_NULL(elems[i])) continue; double v = as_f64(elems[i]); if (!found || v < fmin) { fmin = v; imin = elems[i]->type == -RAY_I64 ? elems[i]->i64 : 0; found = 1; } @@ -667,7 +675,7 @@ ray_t* ray_max_fn(ray_t* x) { double fmax = 0; int64_t imax = 0; for (int64_t i = 0; i < len; i++) { if (!is_numeric(elems[i])) return ray_error("type", "max: list elements must be numeric, got %s", ray_type_name(elems[i]->type)); - if (elems[i]->type == -RAY_F64) has_float = 1; + if (elems[i]->type == -RAY_F64 || elems[i]->type == -RAY_F32) has_float = 1; if (RAY_ATOM_IS_NULL(elems[i])) continue; double v = as_f64(elems[i]); if (!found || v > fmax) { fmax = v; imax = elems[i]->type == -RAY_I64 ? elems[i]->i64 : 0; found = 1; } @@ -770,6 +778,9 @@ static ray_t* vec_to_f64_scratch(ray_t* x, double** out_vals) { } else if (x->type == RAY_F64) { double* d = (double*)ray_data(x); for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = d[i]; } + } else if (x->type == RAY_F32) { + float* d = (float*)ray_data(x); + for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = (double)d[i]; } } else if (x->type == RAY_I32) { int32_t* d = (int32_t*)ray_data(x); for (int64_t i = 0; i < len; i++) { if (!ray_vec_is_null(x, i)) vals[cnt++] = (double)d[i]; } diff --git a/src/ops/arith.c b/src/ops/arith.c index 03cd50bc2..6c1e7c17d 100644 --- a/src/ops/arith.c +++ b/src/ops/arith.c @@ -32,7 +32,7 @@ typedef ray_op_t* (*arith_unary_ctor)(ray_graph_t*, ray_op_t*); static bool arith_numeric_vec_type(int8_t t) { if (RAY_IS_PARTED(t)) t = (int8_t)RAY_PARTED_BASETYPE(t); return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } static ray_t* arith_unary_dag_vec(ray_t* x, arith_unary_ctor ctor) { @@ -74,7 +74,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { /* Vector fast path — only when at least one operand is a typed vector */ /* Temporal + integer arithmetic (only int types, not float) */ - if (is_temporal(a) && is_numeric(b) && b->type != -RAY_F64) { + if (is_temporal(a) && is_numeric(b) && + b->type != -RAY_F64 && b->type != -RAY_F32) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(a->type); @@ -83,7 +84,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { if (a->type == -RAY_TIME) return ray_time(a->i64 + v); if (a->type == -RAY_TIMESTAMP) return ray_timestamp(a->i64 + v); } - if (is_numeric(a) && a->type != -RAY_F64 && is_temporal(b)) { + if (is_numeric(a) && a->type != -RAY_F64 && + a->type != -RAY_F32 && is_temporal(b)) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(b->type); @@ -93,7 +95,8 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { if (b->type == -RAY_TIMESTAMP) return ray_timestamp(b->i64 + v); } /* Reject float + temporal */ - if ((a->type == -RAY_F64 && is_temporal(b)) || (is_temporal(a) && b->type == -RAY_F64)) + if (((a->type == -RAY_F64 || a->type == -RAY_F32) && is_temporal(b)) || + (is_temporal(a) && (b->type == -RAY_F64 || b->type == -RAY_F32))) return ray_error("type", "add: temporal arithmetic requires integer offsets, got %s and %s", ray_type_name(a->type), ray_type_name(b->type)); /* Reject null_numeric + temporal (for null floats etc) */ @@ -132,6 +135,7 @@ ray_t* ray_add_fn(ray_t* a, ray_t* b) { ray_type_name(a->type), ray_type_name(b->type)); /* Null propagation */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return null_for_promoted(a, b); + if (both_f32_atoms(a, b)) return ray_f32((float)(as_f64(a) + as_f64(b))); if (is_float_op(a, b)) return make_f64(as_f64(a) + as_f64(b)); int8_t rt = promote_int_type(a, b); return make_typed_int(rt, as_i64(a) + as_i64(b)); @@ -204,6 +208,7 @@ ray_t* ray_sub_fn(ray_t* a, ray_t* b) { if (is_float_op(a, b)) { double r = as_f64(a) - as_f64(b); if (r == 0.0) r = 0.0; /* normalize -0.0 to +0.0 */ + if (both_f32_atoms(a, b)) return ray_f32((float)r); return make_f64(r); } int8_t rt = promote_int_type_right(a, b); @@ -233,6 +238,7 @@ ray_t* ray_mul_fn(ray_t* a, ray_t* b) { ray_type_name(a->type), ray_type_name(b->type)); /* Null propagation */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return null_for_promoted(a, b); + if (both_f32_atoms(a, b)) return ray_f32((float)(as_f64(a) * as_f64(b))); if (is_float_op(a, b)) return make_f64(as_f64(a) * as_f64(b)); int8_t rt = promote_int_type(a, b); return make_typed_int(rt, as_i64(a) * as_i64(b)); @@ -275,8 +281,8 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(a->type); int64_t bv; - if (b->type == -RAY_F64) { - double bvf = b->f64; + if (b->type == -RAY_F64 || b->type == -RAY_F32) { + double bvf = as_f64(b); if (bvf == 0.0) return ray_typed_null(a->type); bv = (int64_t)bvf; @@ -310,21 +316,21 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { /* Null propagation and division by zero: null type follows RIGHT operand */ if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) { - int8_t rt = (b->type == -RAY_F64 || a->type == -RAY_F64) ? -RAY_F64 : b->type; - return ray_typed_null(rt); + return null_for_promoted(a, b); } /* Float modulo: result = a - b * floor(a/b), type follows RIGHT or f64 */ if (is_float_op(a, b)) { double av = as_f64(a), bv = as_f64(b); if (bv == 0.0) { - int8_t rt = (b->type == -RAY_F64 || a->type == -RAY_F64) ? -RAY_F64 : b->type; - return ray_typed_null(rt); + return null_for_promoted(a, b); } double result = av - bv * floor(av / bv); /* Snap tiny residual to 0 */ if (fabs(result) < 1e-12 || fabs(result - fabs(bv)) < 1e-12) result = bv > 0 ? 0.0 : -0.0; - if (b->type == -RAY_F64 || a->type == -RAY_F64) return make_f64(result); + if (both_f32_atoms(a, b)) return ray_f32((float)result); + if (b->type == -RAY_F64 || a->type == -RAY_F64 || + b->type == -RAY_F32 || a->type == -RAY_F32) return make_f64(result); if (b->type == -RAY_I32) return make_i32((int32_t)(int64_t)result); if (b->type == -RAY_I16) return make_i16((int16_t)(int64_t)result); return make_i64((int64_t)result); @@ -348,6 +354,7 @@ ray_t* ray_mod_fn(ray_t* a, ray_t* b) { ray_t* ray_neg_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(-x->f64); + if (x->type == -RAY_F32) return ray_f32((float)-x->f64); /* INT_MIN is the lone overflow case for signed negation: -INT_MIN * doesn't fit in the same width. By convention, surface this * as a typed null of the same width — preserving type, avoiding UB, @@ -370,8 +377,12 @@ ray_t* ray_neg_fn(ray_t* x) { /* round: round to nearest integer (ties go away from zero), returns f64 */ ray_t* ray_round_fn(ray_t* x) { - if (RAY_ATOM_IS_NULL(x)) return ray_typed_null(-RAY_F64); + if (RAY_ATOM_IS_NULL(x)) { + if (x->type == -RAY_F32) return ray_typed_null(-RAY_F32); + return ray_typed_null(-RAY_F64); + } if (x->type == -RAY_F64) return make_f64(round(x->f64)); + if (x->type == -RAY_F32) return ray_f32(roundf((float)x->f64)); if (is_numeric(x)) return make_f64(round(as_f64(x))); return ray_error("type", "round: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -380,6 +391,7 @@ ray_t* ray_round_fn(ray_t* x) { ray_t* ray_floor_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(floor(x->f64)); + if (x->type == -RAY_F32) return ray_f32(floorf((float)x->f64)); if (is_numeric(x)) { ray_retain(x); return x; } return ray_error("type", "floor: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -388,6 +400,7 @@ ray_t* ray_floor_fn(ray_t* x) { ray_t* ray_ceil_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(ceil(x->f64)); + if (x->type == -RAY_F32) return ray_f32(ceilf((float)x->f64)); if (is_numeric(x)) { ray_retain(x); return x; } return ray_error("type", "ceil: expects a numeric argument, got %s", ray_type_name(x->type)); } @@ -399,6 +412,7 @@ ray_t* ray_ceil_fn(ray_t* x) { ray_t* ray_abs_fn(ray_t* x) { if (RAY_ATOM_IS_NULL(x)) { ray_retain(x); return x; } if (x->type == -RAY_F64) return make_f64(fabs(x->f64)); + if (x->type == -RAY_F32) return ray_f32(fabsf((float)x->f64)); if (x->type == -RAY_I64) { if (RAY_UNLIKELY(x->i64 == INT64_MIN)) return ray_typed_null(-RAY_I64); return make_i64(x->i64 < 0 ? -x->i64 : x->i64); diff --git a/src/ops/expr.c b/src/ops/expr.c index b99e9bb6a..5d71f64db 100644 --- a/src/ops/expr.c +++ b/src/ops/expr.c @@ -37,6 +37,11 @@ static bool atom_to_numeric(ray_t* atom, double* out_f, int64_t* out_i, bool* ou *out_i = (int64_t)atom->f64; *out_is_f64 = true; return true; + case -RAY_F32: + *out_f = (double)(float)atom->f64; + *out_i = (int64_t)(float)atom->f64; + *out_is_f64 = true; + return true; case -RAY_I64: case -RAY_SYM: case -RAY_DATE: @@ -673,7 +678,8 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].is_parted = true; out->regs[r].col_obj = NULL; /* parted: per-segment, no zone-skip */ out->regs[r].parted_col = col; - out->regs[r].type = (base == RAY_F64) ? RAY_F64 : RAY_I64; + out->regs[r].type = (base == RAY_F64 || base == RAY_F32) + ? RAY_F64 : RAY_I64; out->regs[r].nullable = col_nulls; out->has_parted = true; } else { @@ -683,7 +689,8 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].is_parted = false; out->regs[r].col_obj = col; out->regs[r].parted_col = NULL; - out->regs[r].type = (col->type == RAY_F64) ? RAY_F64 : RAY_I64; + out->regs[r].type = (col->type == RAY_F64 || col->type == RAY_F32) + ? RAY_F64 : RAY_I64; out->regs[r].nullable = col_nulls; } } else if (node->opcode == OP_CONST) { @@ -712,6 +719,12 @@ bool expr_compile(ray_graph_t* g, ray_t* tbl, ray_op_t* root, ray_expr_t* out) { out->regs[r].const_f64 = cf; out->regs[r].const_i64 = ci; } else if (expr_is_elementwise(node->opcode)) { + /* The fused VM has F64/I64 lanes only. F32 inputs can be read + * exactly through the F64 lane, but a node whose result is F32 + * must round and store at F32 width. Keep every such node on + * the typed executor until the VM has a real F32 lane. */ + if (node->out_type == RAY_F32) + EXPR_BAIL(EXPR_BAIL_OTHER); if (!op_child(g, node, 0)) EXPR_BAIL(EXPR_BAIL_OTHER); uint8_t s1 = node_reg[node->in_id[0]]; if (s1 == 0xFF) EXPR_BAIL(EXPR_BAIL_OTHER); @@ -912,6 +925,14 @@ static void expr_load_f64(double* dst, const void* data, int8_t col_type, /* NaN already canonical in place — direct copy */ memcpy(dst, (const double*)data + start, (size_t)n * 8); break; + case RAY_F32: { + const float* s = (const float*)data + start; + if (has_nulls) + for (int64_t j = 0; j < n; j++) + dst[j] = (s[j] != s[j]) ? NULL_F64 : (double)s[j]; + else + for (int64_t j = 0; j < n; j++) dst[j] = (double)s[j]; + } break; case RAY_I64: case RAY_TIMESTAMP: { const int64_t* s = (const int64_t*)data + start; if (has_nulls) @@ -2193,7 +2214,7 @@ static int8_t ew_value_base_type(ray_t* x) { static bool ew_pow_type_admitted(int8_t t) { return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } /* Resolve data pointer for a vector, accounting for slices. @@ -2333,7 +2354,7 @@ static bool ew_unary_math_type_admitted(uint16_t opcode, int8_t t) { if (opcode != OP_SIGNUM && !expr_is_f64_unary_math(opcode)) return true; if (RAY_IS_PARTED(t)) t = (int8_t)RAY_PARTED_BASETYPE(t); return t == RAY_BOOL || t == RAY_U8 || t == RAY_I16 || - t == RAY_I32 || t == RAY_I64 || t == RAY_F64; + t == RAY_I32 || t == RAY_I64 || t == RAY_F32 || t == RAY_F64; } ray_t* exec_elementwise_unary(ray_graph_t* g, ray_op_t* op, ray_t* input) { @@ -2360,7 +2381,55 @@ ray_t* exec_elementwise_unary(ray_graph_t* g, ray_op_t* op, ray_t* input) { int64_t out_off = 0; uint16_t opc = op->opcode; - if (in_type == RAY_F64 && out_type == RAY_F64) { + if (in_type == RAY_F32 && out_type == RAY_F32) { + while (ray_morsel_next(&m)) { + int64_t n = m.morsel_len; + float* src = (float*)m.morsel_ptr; + float* dst = (float*)ray_data(result) + out_off; + switch (opc) { + case OP_NEG: for (int64_t i=0;i 0.0f) - (src[i] < 0.0f); + else + for (int64_t i=0;itype, lhs->attrs); if (lhs->type == RAY_F64) lp_f64 = (double*)lbase; + else if (lhs->type == RAY_F32) lp_f32 = (float*)lbase; else if (lhs->type == RAY_I64 || lhs->type == RAY_TIMESTAMP) lp_i64 = (int64_t*)lbase; else if (RAY_IS_SYM(lhs->type)) { uint8_t w = lhs->attrs & RAY_SYM_W_MASK; @@ -2960,6 +3032,7 @@ static void binary_range(ray_op_t* op, int8_t out_type, void* r_data = resolve_vec_data(rhs, &r_off); void* rbase = (char*)r_data + r_off * ray_sym_elem_size(rhs->type, rhs->attrs); if (rhs->type == RAY_F64) rp_f64 = (double*)rbase; + else if (rhs->type == RAY_F32) rp_f32 = (float*)rbase; else if (rhs->type == RAY_I64 || rhs->type == RAY_TIMESTAMP) rp_i64 = (int64_t*)rbase; else if (RAY_IS_SYM(rhs->type)) { uint8_t w = rhs->attrs & RAY_SYM_W_MASK; @@ -2993,12 +3066,16 @@ static void binary_range(ray_op_t* op, int8_t out_type, /* Resolve the lhs/rhs reader to a single decision made once before the loop. * Each of these yields a double for the given index. */ -#define LV_READ(i) (lp_f64 ? lp_f64[i] : lp_i64 ? (double)lp_i64[i] : lp_i32 ? (double)lp_i32[i] : lp_u32 ? (double)lp_u32[i] : lp_i16 ? (double)lp_i16[i] : lp_bool ? (double)lp_bool[i] : (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64)) ? l_f64 : (double)l_i64) -#define RV_READ(i) (rp_f64 ? rp_f64[i] : rp_i64 ? (double)rp_i64[i] : rp_i32 ? (double)rp_i32[i] : rp_u32 ? (double)rp_u32[i] : rp_i16 ? (double)rp_i16[i] : rp_bool ? (double)rp_bool[i] : (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64)) ? r_f64 : (double)r_i64) +#define LV_READ(i) (lp_f64 ? lp_f64[i] : lp_f32 ? (double)lp_f32[i] : lp_i64 ? (double)lp_i64[i] : lp_i32 ? (double)lp_i32[i] : lp_u32 ? (double)lp_u32[i] : lp_i16 ? (double)lp_i16[i] : lp_bool ? (double)lp_bool[i] : (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64 || lhs->type == -RAY_F32 || lhs->type == RAY_F32)) ? l_f64 : (double)l_i64) +#define RV_READ(i) (rp_f64 ? rp_f64[i] : rp_f32 ? (double)rp_f32[i] : rp_i64 ? (double)rp_i64[i] : rp_i32 ? (double)rp_i32[i] : rp_u32 ? (double)rp_u32[i] : rp_i16 ? (double)rp_i16[i] : rp_bool ? (double)rp_bool[i] : (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || rhs->type == -RAY_F32 || rhs->type == RAY_F32)) ? r_f64 : (double)r_i64) /* Compute once: is lhs/rhs integer-family (not float)? Used by BOOL path. */ - int l_is_int = !(lp_f64 || (l_scalar && (lhs->type == -RAY_F64 || lhs->type == RAY_F64))); - int r_is_int = !(rp_f64 || (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64))); + int l_is_int = !(lp_f64 || lp_f32 || (l_scalar && + (lhs->type == -RAY_F64 || lhs->type == RAY_F64 || + lhs->type == -RAY_F32 || lhs->type == RAY_F32))); + int r_is_int = !(rp_f64 || rp_f32 || (r_scalar && + (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || + rhs->type == -RAY_F32 || rhs->type == RAY_F32))); int src_is_i64_all = l_is_int && r_is_int; /* Hoist out_type outside the loop. Each branch is a tight per-element kernel. */ @@ -3035,6 +3112,25 @@ static void binary_range(ray_op_t* op, int8_t out_type, if (any_nan) __atomic_fetch_or(&result->attrs, (uint8_t)RAY_ATTR_HAS_NULLS, __ATOMIC_RELAXED); } + } else if (out_type == RAY_F32) { + float* odst = (float*)dst; + switch (op->opcode) { + case OP_ADD: for (int64_t i=0;i0)!=(rv>0)))v+=rv;v=isfinite(v)?v:NULL_F32;}else v=NULL_F32;odst[i]=v; } break; + case OP_MIN2: for (int64_t i=0;irv?lv:rv; } break; + default: for (int64_t i=0;iopcode == OP_ADD || op->opcode == OP_SUB || + op->opcode == OP_MUL || op->opcode == OP_MOD) { + int any_nan = 0; + for (int64_t i = 0; i < n; i++) any_nan |= (odst[i] != odst[i]); + if (any_nan) + __atomic_fetch_or(&result->attrs, (uint8_t)RAY_ATTR_HAS_NULLS, + __ATOMIC_RELAXED); + } } else if (out_type == RAY_I64 || out_type == RAY_TIMESTAMP) { int64_t* odst = (int64_t*)dst; switch (op->opcode) { @@ -3296,7 +3392,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* if (str_resolved && lhs->type == -RAY_STR) l_i64_val = resolved_sym_id; else if (ray_is_atom(lhs)) { - if (lhs->type == -RAY_F64) l_f64_val = lhs->f64; + if (lhs->type == -RAY_F64 || lhs->type == -RAY_F32) + l_f64_val = lhs->f64; else if (lhs->type == -RAY_I32 || lhs->type == -RAY_DATE || lhs->type == -RAY_TIME) l_i64_val = (int64_t)lhs->i32; else if (lhs->type == -RAY_I16) l_i64_val = (int64_t)lhs->i16; @@ -3307,6 +3404,7 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* int64_t elem = 0; void* data = resolve_vec_data(lhs, &elem); if (t == RAY_F64) l_f64_val = ((double*)data)[elem]; + else if (t == RAY_F32) l_f64_val = (double)((float*)data)[elem]; else l_i64_val = read_col_i64(data, elem, t, lhs->attrs); } } @@ -3314,7 +3412,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* if (str_resolved && rhs->type == -RAY_STR) r_i64_val = resolved_sym_id; else if (ray_is_atom(rhs)) { - if (rhs->type == -RAY_F64) r_f64_val = rhs->f64; + if (rhs->type == -RAY_F64 || rhs->type == -RAY_F32) + r_f64_val = rhs->f64; else if (rhs->type == -RAY_I32 || rhs->type == -RAY_DATE || rhs->type == -RAY_TIME) r_i64_val = (int64_t)rhs->i32; else if (rhs->type == -RAY_I16) r_i64_val = (int64_t)rhs->i16; @@ -3325,6 +3424,7 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* int64_t elem = 0; void* data = resolve_vec_data(rhs, &elem); if (t == RAY_F64) r_f64_val = ((double*)data)[elem]; + else if (t == RAY_F32) r_f64_val = (double)((float*)data)[elem]; else r_i64_val = read_col_i64(data, elem, t, rhs->attrs); } } @@ -3424,7 +3524,8 @@ ray_t* exec_elementwise_binary(ray_graph_t* g, ray_op_t* op, ray_t* lhs, ray_t* } else { /* Scalar divisor: check for zero using the correct type */ bool is_zero = false; - if (rhs->type == -RAY_F64 || rhs->type == RAY_F64) + if (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || + rhs->type == -RAY_F32 || rhs->type == RAY_F32) is_zero = (r_f64_val == 0.0); else is_zero = (r_i64_val == 0); diff --git a/src/ops/graph.c b/src/ops/graph.c index 1e14f3cd9..9775a6920 100644 --- a/src/ops/graph.c +++ b/src/ops/graph.c @@ -403,11 +403,15 @@ static ray_op_t* make_binary(ray_graph_t* g, uint16_t opcode, ray_op_t* a, ray_o return n; } -/* Type promotion: BOOL < U8 < I16 < I32 < I64 < F64. +/* Type promotion: BOOL < U8 < I16 < I32 < I64 < F32 < F64. + * Two F32 operands retain F32 arithmetic and storage; mixing F32 with another + * numeric width promotes to F64 so no integer precision is silently narrowed. * RAY_STR is its own type class — not promotable to numeric types. */ static int8_t promote(int8_t a, int8_t b) { if (a == RAY_STR || b == RAY_STR) return RAY_STR; if (a == RAY_F64 || b == RAY_F64) return RAY_F64; + if (a == RAY_F32 || b == RAY_F32) + return (a == RAY_F32 && b == RAY_F32) ? RAY_F32 : RAY_F64; if (a == RAY_I64 || b == RAY_I64 || a == RAY_SYM || b == RAY_SYM || a == RAY_TIMESTAMP || b == RAY_TIMESTAMP) return RAY_I64; if (a == RAY_I32 || b == RAY_I32 || @@ -624,16 +628,16 @@ ray_op_t* ray_concat(ray_graph_t* g, ray_op_t** args, int n) { * Reduction ops * -------------------------------------------------------------------------- */ -ray_op_t* ray_sum(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_SUM, a, a->out_type == RAY_F64 ? RAY_F64 : RAY_I64); } -ray_op_t* ray_prod(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_PROD, a, a->out_type == RAY_F64 ? RAY_F64 : RAY_I64); } +ray_op_t* ray_sum(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_SUM, a, (a->out_type == RAY_F64 || a->out_type == RAY_F32) ? RAY_F64 : RAY_I64); } +ray_op_t* ray_prod(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_PROD, a, (a->out_type == RAY_F64 || a->out_type == RAY_F32) ? RAY_F64 : RAY_I64); } ray_op_t* ray_all(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ALL, a, RAY_BOOL); } ray_op_t* ray_any(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ANY, a, RAY_BOOL); } -ray_op_t* ray_min_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MIN, a, a->out_type); } -ray_op_t* ray_max_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MAX, a, a->out_type); } +ray_op_t* ray_min_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MIN, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } +ray_op_t* ray_max_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_MAX, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } ray_op_t* ray_count(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_COUNT, a, RAY_I64); } ray_op_t* ray_avg(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_AVG, a, RAY_F64); } -ray_op_t* ray_first(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_FIRST, a, a->out_type); } -ray_op_t* ray_last(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_LAST, a, a->out_type); } +ray_op_t* ray_first(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_FIRST, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } +ray_op_t* ray_last(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_LAST, a, a->out_type == RAY_F32 ? RAY_F64 : a->out_type); } ray_op_t* ray_count_distinct(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_COUNT_DISTINCT, a, RAY_I64); } ray_op_t* ray_distinct_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_DISTINCT, a, a->out_type); } ray_op_t* ray_asc_op(ray_graph_t* g, ray_op_t* a) { return make_unary(g, OP_ASC, a, a->out_type); } diff --git a/src/ops/group.c b/src/ops/group.c index 09d928e02..dc6dff256 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -35,6 +35,18 @@ #include +/* Group accumulators use F64 lanes for both floating storage widths. Keep the + * source read width explicit so F32 payload bits are never mistaken for an + * integer or loaded through a double pointer. */ +static inline bool group_fp_type(int8_t t) { + return t == RAY_F32 || t == RAY_F64; +} + +static inline double group_fp_at(const void* data, int8_t t, int64_t row) { + return t == RAY_F64 ? ((const double*)data)[row] + : (double)((const float*)data)[row]; +} + /* Monotonic count of exec_group_per_partition entries (the streaming parted * GROUP kernel). Bumped ONCE at the kernel entry — O(1) per query, never * per-row, so it does not violate the "instrumentation never costs O(data)" @@ -227,13 +239,13 @@ static ray_t* agg_wide_reduce(ray_t* input, uint16_t op, } while (0) /* Float reduction loop — see REDUCE_LOOP_I for HAS_NULLS/HAS_IDX semantics. - * F64 null = NaN (NULL_F64); detect via v != v (only NaN fails self-equality). */ -#define REDUCE_LOOP_F(base, start, end, acc, HAS_NULLS, HAS_IDX, idx) \ + * F32/F64 null = NaN; both accumulate in the shared F64 reduction lanes. */ +#define REDUCE_LOOP_F(T, base, start, end, acc, HAS_NULLS, HAS_IDX, idx) \ do { \ - const double* d = (const double*)(base); \ + const T* d = (const T*)(base); \ for (int64_t i = start; i < end; i++) { \ int64_t row = (HAS_IDX) ? (idx)[i] : i; \ - double v = d[row]; \ + double v = (double)d[row]; \ if ((HAS_NULLS) && v != v) { (acc)->null_count++; continue; } \ (acc)->sum_f += v; (acc)->sum_sq_f += v * v; (acc)->prod_f *= v; \ if (v == 0.0) (acc)->zero_count++; \ @@ -259,40 +271,18 @@ static ray_t* agg_wide_reduce(ray_t* input, uint16_t op, REDUCE_LOOP_I(T, NULL_SENT, base, start, end, acc, 1, 1, idx); \ } while (0) -#define DISPATCH_F(base, start, end, acc, has_nulls, idx) \ +#define DISPATCH_F(T, base, start, end, acc, has_nulls, idx) \ do { \ if (!(has_nulls) && !(idx)) \ - REDUCE_LOOP_F(base, start, end, acc, 0, 0, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 0, 0, idx); \ else if (!(has_nulls)) \ - REDUCE_LOOP_F(base, start, end, acc, 0, 1, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 0, 1, idx); \ else if (!(idx)) \ - REDUCE_LOOP_F(base, start, end, acc, 1, 0, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 1, 0, idx); \ else \ - REDUCE_LOOP_F(base, start, end, acc, 1, 1, idx); \ + REDUCE_LOOP_F(T, base, start, end, acc, 1, 1, idx); \ } while (0) -/* Pin the keyless reduction kernel's hot-loop/jump alignment. This kernel's - * tight inner reduction loops are alignment-fragile on 32-byte-fetch cores (DSB - * / JCC-erratum class): as the branch as a whole grows, .o files linked - * before group.o shift reduce_range's ABSOLUTE address (it is early in the TU, - * so the shift is cumulative binary-layout, not a group.c edit), which moved - * its hot I64 loop off its 32-byte boundary and dropped IPC 1.56→1.01 on q03/ - * q02 (+48% cycles, byte-identical instructions) — RCA task-9. Entry alignment - * is NOT the lever (the loop's offset from entry is fixed, so its absolute - * alignment tracks the entry's low bits). `align-loops=32` re-pins every loop - * head AND `align-jumps=32` re-pins the branch targets inside the unrolled - * reduction (the min/max-update return path) — both are needed: loops alone - * only recovered ~⅔ of the gap (IPC 1.37); adding jump alignment restores full - * parity. Now the hot loop's internal alignment is invariant to reduce_range's - * absolute address, so future binary-layout churn can no longer reshuffle it - * onto a bad boundary. aligned(64) keeps the entry cacheline-stable too. - * Per-function attributes only — no global codegen flag, no -m target hack. - * GCC-only: clang has no `optimize` function attribute and -Werror promotes - * the unknown-attribute warning to an error; the pin is performance-only, - * so clang builds simply go without it (the pre-pin status quo). */ -#if defined(__GNUC__) && !defined(__clang__) -__attribute__((aligned(64), optimize("align-loops=32","align-jumps=32"))) -#endif static void reduce_range(ray_t* input, int64_t start, int64_t end, reduce_acc_t* acc, bool has_nulls, const int64_t* idx) { @@ -324,8 +314,10 @@ static void reduce_range(ray_t* input, int64_t start, int64_t end, DISPATCH_I(int32_t, NULL_I32, base, start, end, acc, has_nulls, idx); break; case RAY_I64: case RAY_TIMESTAMP: DISPATCH_I(int64_t, NULL_I64, base, start, end, acc, has_nulls, idx); break; + case RAY_F32: + DISPATCH_F(float, base, start, end, acc, has_nulls, idx); break; case RAY_F64: - DISPATCH_F(base, start, end, acc, has_nulls, idx); break; + DISPATCH_F(double, base, start, end, acc, has_nulls, idx); break; case RAY_SYM: { /* Adaptive-width SYM columns — read_col_i64 produces the i64 * sym id; id 0 is the canonical null sym (interned empty string @@ -405,7 +397,7 @@ static void par_reduce_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t static void reduce_merge(reduce_acc_t* dst, const reduce_acc_t* src, int8_t in_type, struct ray_sym_domain_s* sym_dom) { - if (in_type == RAY_F64) { + if (in_type == RAY_F64 || in_type == RAY_F32) { dst->sum_f += src->sum_f; dst->sum_sq_f += src->sum_sq_f; dst->prod_f *= src->prod_f; @@ -2568,6 +2560,7 @@ static ray_t* reduction_extreme_result(ray_op_t* op, int8_t in_type, bool found, /* Single-null float model: min/max of finite inputs is finite, but guard * against an ±Inf init sentinel surfacing as a value. */ if (out_type == RAY_F64) return ray_f64(ray_f64_fin(fval)); + if (out_type == RAY_F32) return ray_f32((float)fval); return reduction_i64_result(ival, out_type, out_type == RAY_SYM ? src : NULL); } @@ -2664,7 +2657,7 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { * full reduction pass. Non-numeric types (STR, GUID) fall through * to the serial reduction path below. */ if ((op->opcode == OP_FIRST || op->opcode == OP_LAST) && - (in_type == RAY_I64 || in_type == RAY_F64 || in_type == RAY_I32 || + (in_type == RAY_I64 || in_type == RAY_F64 || in_type == RAY_F32 || in_type == RAY_I32 || in_type == RAY_I16 || in_type == RAY_BOOL || in_type == RAY_U8 || in_type == RAY_TIMESTAMP || in_type == RAY_DATE || in_type == RAY_TIME || in_type == RAY_SYM)) { @@ -2685,6 +2678,7 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { return ray_typed_null(-in_type); void* base = ray_data(input); if (in_type == RAY_F64) return ray_f64(((const double*)base)[row]); + if (in_type == RAY_F32) return ray_f64((double)((const float*)base)[row]); return reduction_i64_result(read_col_i64(base, row, in_type, input->attrs), in_type, in_type == RAY_SYM ? input : NULL); } @@ -2712,14 +2706,14 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* first = accs[first worker with data], last = accs[last worker with data] */ for (uint32_t i = 0; i < nw; i++) { if (accs[i].has_first) { - if (in_type == RAY_F64) merged.first_f = accs[i].first_f; + if (in_type == RAY_F64 || in_type == RAY_F32) merged.first_f = accs[i].first_f; else merged.first_i = accs[i].first_i; break; } } for (int32_t i = (int32_t)nw - 1; i >= 0; i--) { if (accs[i].has_first) { - if (in_type == RAY_F64) merged.last_f = accs[i].last_f; + if (in_type == RAY_F64 || in_type == RAY_F32) merged.last_f = accs[i].last_f; else merged.last_i = accs[i].last_i; break; } @@ -2727,8 +2721,8 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { ray_t* result; switch (op->opcode) { - case OP_SUM: result = in_type == RAY_F64 ? ray_f64(ray_f64_fin(merged.sum_f)) : (in_type == RAY_TIME ? ray_time(merged.sum_i) : ray_i64(merged.sum_i)); break; - case OP_PROD: result = in_type == RAY_F64 ? ray_f64(ray_f64_fin(merged.prod_f)) : ray_i64(merged.prod_i); break; + case OP_SUM: result = (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(merged.sum_f)) : (in_type == RAY_TIME ? ray_time(merged.sum_i) : ray_i64(merged.sum_i)); break; + case OP_PROD: result = (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(merged.prod_f)) : ray_i64(merged.prod_i); break; case OP_ALL: result = ray_bool(merged.zero_count == 0); break; case OP_ANY: result = ray_bool(merged.cnt > merged.zero_count); break; case OP_MIN: result = reduction_extreme_result(op, in_type, merged.cnt > 0, merged.min_f, merged.min_i, input); break; @@ -2736,15 +2730,15 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* COUNT returns total length including nulls — matches ray_count_fn's * "count all elements" semantics, not SQL's COUNT(col) non-null count. */ case OP_COUNT: result = ray_i64(scan_n); break; - case OP_AVG: result = merged.cnt > 0 ? ray_f64(ray_f64_fin(in_type == RAY_F64 ? merged.sum_f / merged.cnt : merged.sum_d / merged.cnt)) : ray_typed_null(-RAY_F64); break; - case OP_FIRST: result = merged.has_first ? (in_type == RAY_F64 ? ray_f64(merged.first_f) : reduction_i64_result(merged.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); break; - case OP_LAST: result = merged.has_first ? (in_type == RAY_F64 ? ray_f64(merged.last_f) : reduction_i64_result(merged.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); break; + case OP_AVG: result = merged.cnt > 0 ? ray_f64(ray_f64_fin((in_type == RAY_F64 || in_type == RAY_F32) ? merged.sum_f / merged.cnt : merged.sum_d / merged.cnt)) : ray_typed_null(-RAY_F64); break; + case OP_FIRST: result = merged.has_first ? (group_fp_type(in_type) ? ray_f64(merged.first_f) : reduction_i64_result(merged.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); break; + case OP_LAST: result = merged.has_first ? (group_fp_type(in_type) ? ray_f64(merged.last_f) : reduction_i64_result(merged.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); break; case OP_VAR: case OP_VAR_POP: case OP_STDDEV: case OP_STDDEV_POP: { bool insufficient = (op->opcode == OP_VAR || op->opcode == OP_STDDEV) ? merged.cnt <= 1 : merged.cnt <= 0; if (insufficient) { result = ray_typed_null(-RAY_F64); break; } double mean, var_pop; - if (in_type == RAY_F64) { mean = merged.sum_f / merged.cnt; var_pop = merged.sum_sq_f / merged.cnt - mean * mean; } + if (in_type == RAY_F64 || in_type == RAY_F32) { mean = merged.sum_f / merged.cnt; var_pop = merged.sum_sq_f / merged.cnt - mean * mean; } else { mean = merged.sum_d / merged.cnt; var_pop = (double)merged.sum_sq_i / merged.cnt - mean * mean; } if (var_pop < 0) var_pop = 0; double val; @@ -2768,8 +2762,8 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { if (sel_idx_block) ray_release(sel_idx_block); switch (op->opcode) { - case OP_SUM: return in_type == RAY_F64 ? ray_f64(ray_f64_fin(acc.sum_f)) : (in_type == RAY_TIME ? ray_time(acc.sum_i) : ray_i64(acc.sum_i)); - case OP_PROD: return in_type == RAY_F64 ? ray_f64(ray_f64_fin(acc.prod_f)) : ray_i64(acc.prod_i); + case OP_SUM: return (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(acc.sum_f)) : (in_type == RAY_TIME ? ray_time(acc.sum_i) : ray_i64(acc.sum_i)); + case OP_PROD: return (in_type == RAY_F64 || in_type == RAY_F32) ? ray_f64(ray_f64_fin(acc.prod_f)) : ray_i64(acc.prod_i); case OP_ALL: return ray_bool(acc.zero_count == 0); case OP_ANY: return ray_bool(acc.cnt > acc.zero_count); case OP_MIN: return reduction_extreme_result(op, in_type, acc.cnt > 0, acc.min_f, acc.min_i, input); @@ -2777,15 +2771,15 @@ ray_t* exec_reduction(ray_graph_t* g, ray_op_t* op, ray_t* input) { /* COUNT returns total length including nulls — matches ray_count_fn's * "count all elements" semantics, not SQL's COUNT(col) non-null count. */ case OP_COUNT: return ray_i64(scan_n); - case OP_AVG: return acc.cnt > 0 ? ray_f64(ray_f64_fin(in_type == RAY_F64 ? acc.sum_f / acc.cnt : acc.sum_d / acc.cnt)) : ray_typed_null(-RAY_F64); - case OP_FIRST: return acc.has_first ? (in_type == RAY_F64 ? ray_f64(acc.first_f) : reduction_i64_result(acc.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); - case OP_LAST: return acc.has_first ? (in_type == RAY_F64 ? ray_f64(acc.last_f) : reduction_i64_result(acc.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-in_type); + case OP_AVG: return acc.cnt > 0 ? ray_f64(ray_f64_fin((in_type == RAY_F64 || in_type == RAY_F32) ? acc.sum_f / acc.cnt : acc.sum_d / acc.cnt)) : ray_typed_null(-RAY_F64); + case OP_FIRST: return acc.has_first ? (group_fp_type(in_type) ? ray_f64(acc.first_f) : reduction_i64_result(acc.first_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); + case OP_LAST: return acc.has_first ? (group_fp_type(in_type) ? ray_f64(acc.last_f) : reduction_i64_result(acc.last_i, in_type, in_type == RAY_SYM ? input : NULL)) : ray_typed_null(-(op->out_type ? op->out_type : in_type)); case OP_VAR: case OP_VAR_POP: case OP_STDDEV: case OP_STDDEV_POP: { bool insufficient = (op->opcode == OP_VAR || op->opcode == OP_STDDEV) ? acc.cnt <= 1 : acc.cnt <= 0; if (insufficient) return ray_typed_null(-RAY_F64); double mean, var_pop; - if (in_type == RAY_F64) { mean = acc.sum_f / acc.cnt; var_pop = acc.sum_sq_f / acc.cnt - mean * mean; } + if (in_type == RAY_F64 || in_type == RAY_F32) { mean = acc.sum_f / acc.cnt; var_pop = acc.sum_sq_f / acc.cnt - mean * mean; } else { mean = acc.sum_d / acc.cnt; var_pop = (double)acc.sum_sq_i / acc.cnt - mean * mean; } if (var_pop < 0) var_pop = 0; double val; @@ -2904,7 +2898,8 @@ void ght_layout_copy(ght_layout_t* dst, const ght_layout_t* src) { } bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, - ray_t** agg_vecs, uint8_t need_flags, + ray_t** agg_vecs, ray_t** agg_vecs2, + uint8_t need_flags, const uint16_t* agg_ops, const int8_t* key_types) { memset(out, 0, sizeof(*out)); @@ -2970,7 +2965,10 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->agg_val_slot[a] = -1; } else if (agg_vecs[a]) { out->agg_val_slot[a] = (int8_t)nv; - if (agg_vecs[a]->type == RAY_F64) + bool pair = agg_ops && agg_is_binary_agg(agg_ops[a]); + bool pair_fp = pair && agg_vecs2 && agg_vecs2[a] && + group_fp_type(agg_vecs2[a]->type); + if (group_fp_type(agg_vecs[a]->type) || pair_fp) af |= GHT_AF_F64; if (agg_vecs[a]->type == RAY_SYM) { af |= GHT_AF_SYM; @@ -3458,7 +3456,7 @@ static void accum_from_entry_nullable(char* row, const char* entry, /* Initialize accumulators for a new group from entry's inline agg values. * Each unified block has n_agg_vals slots of 8 bytes, typed by agg_is_f64. */ static inline void init_accum_from_entry(char* row, const char* entry, - const ght_layout_t* ly) { + const ght_layout_t* ly) { if (ly->any_agg_null) { init_accum_from_entry_nullable(row, entry, ly); return; } /* key_region-based (mirrors init_accum_from_entry_nullable): the old * `8 + (n_keys+1)*8` assumed exactly one trailing null-mask word. For @@ -3564,7 +3562,7 @@ static inline void init_accum_from_entry(char* row, const char* entry, /* Accumulate into existing group from entry's inline agg values */ static inline void accum_from_entry(char* row, const char* entry, - const ght_layout_t* ly) { + const ght_layout_t* ly) { if (ly->any_agg_null) { accum_from_entry_nullable(row, entry, ly); return; } const char* agg_data = entry + 8 + (size_t)ly->key_region; uint16_t na = ly->n_aggs; @@ -4061,16 +4059,24 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, if (!ac) continue; if (agg_strlen && agg_strlen[a]) ev[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; /* Binary aggregator: pack y after x in the same entry. */ if ((af & GHT_AF_BINARY) && agg_vecs2 && agg_vecs2[a]) { ray_t* ay = agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; @@ -4278,8 +4284,12 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ if (!ac) continue; if (c->agg_strlen && c->agg_strlen[a]) agg_vals[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&agg_vals[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } else agg_vals[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; @@ -4290,8 +4300,12 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ * support F64 inputs cleanly — i64 path is a perf followup). */ if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { ray_t* ay = c->agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&agg_vals[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } else agg_vals[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; @@ -4830,15 +4844,23 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, if (!ac) continue; if (c->agg_strlen && c->agg_strlen[a]) ev[vi] = group_strlen_at(ac, row); - else if (ac->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ac))[row], 8); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : (double)read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); vi++; if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { ray_t* ay = c->agg_vecs2[a]; - if (ay->type == RAY_F64) - memcpy(&ev[vi], &((double*)ray_data(ay))[row], 8); + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : (double)read_col_i64(ray_data(ay), row, ay->type, ay->attrs); + memcpy(&ev[vi], &v, sizeof(v)); + } else ev[vi] = read_col_i64(ray_data(ay), row, ay->type, ay->attrs); vi++; @@ -5071,7 +5093,7 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* * product accumulates F64; the linear i64 plan accumulates i64 * even when its compiled expression promotes to F64. */ bool is_f64 = agg_col - ? (agg_col->type == RAY_F64) + ? group_fp_type(agg_col->type) : (prod && prod[a].enabled); int8_t out_type; switch (agg_op) { @@ -5102,7 +5124,8 @@ static void emit_agg_columns(ray_t** result, ray_graph_t* g, const ray_op_ext_t* case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col = ray_vec_new(out_type, (int64_t)grp_count); if (!new_col || RAY_IS_ERR(new_col)) continue; @@ -5584,8 +5607,8 @@ DEFINE_DA_COMPOSITE_GID_TYPED(i64, int64_t) static inline void da_read_val(const void* ptr, int8_t type, uint8_t attrs, int64_t r, double* out_f64, int64_t* out_i64) { - if (type == RAY_F64) { - *out_f64 = ((const double*)ptr)[r]; + if (group_fp_type(type)) { + *out_f64 = group_fp_at(ptr, type, r); *out_i64 = (int64_t)*out_f64; } else { *out_i64 = read_col_i64(ptr, r, type, attrs); @@ -5911,7 +5934,7 @@ static inline void scalar_accum_row(scalar_ctx_t* c, da_accum_t* acc, int64_t r) } } uint16_t op = c->agg_ops[a]; - bool is_f = (c->agg_types[a] == RAY_F64); + bool is_f = group_fp_type(c->agg_types[a]); /* NULL_I* sentinel = null. */ bool int_null = !is_f && c->agg_int_null_has[a] && iv == c->agg_int_null_sentinel[a]; @@ -6033,7 +6056,7 @@ static inline void da_accum_row(da_ctx_t* c, da_accum_t* acc, int32_t gid, int64 if (nn) nn[idx]++; } else if (f64m & ((uint64_t)1 << a)) { /* NaN payload = null, skip from sum. */ - double v = ((const double*)c->agg_ptrs[a])[r]; + double v = group_fp_at(c->agg_ptrs[a], c->agg_types[a], r); if (RAY_LIKELY(v == v)) { acc->sum[idx].f += v; if (nn) nn[idx]++; } } else { /* NULL_I* sentinel = null, skip from sum. Only paid when @@ -6088,7 +6111,7 @@ static inline void da_accum_row(da_ctx_t* c, da_accum_t* acc, int32_t gid, int64 da_read_val(c->agg_ptrs[a], c->agg_types[a], attrs, r, &fv, &iv); } uint16_t op = c->agg_ops[a]; - bool is_f = (c->agg_types[a] == RAY_F64); + bool is_f = group_fp_type(c->agg_types[a]); /* NULL_I* sentinel = null. Bit set in agg_int_null_mask AND * value equal to per-agg sentinel means this row is null for * an integer aggregation column. */ @@ -6324,12 +6347,12 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); } - } else if (agg_types[a] == RAY_F64) + } else if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; @@ -6338,7 +6361,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (c->need_flags & DA_NEED_MIN) { for (uint32_t a = 0; a < n_aggs; a++) { size_t idx = base + a; - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[idx].f < merged->min_val[idx].f) merged->min_val[idx].f = wa->min_val[idx].f; } else if (agg_types[a] == RAY_SYM) { @@ -6356,7 +6379,7 @@ static void da_merge_fn(void* ctx, uint32_t wid, int64_t start, int64_t end) { if (c->need_flags & DA_NEED_MAX) { for (uint32_t a = 0; a < n_aggs; a++) { size_t idx = base + a; - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[idx].f > merged->max_val[idx].f) merged->max_val[idx].f = wa->max_val[idx].f; } else if (agg_types[a] == RAY_SYM) { @@ -7542,15 +7565,17 @@ static inline double sg_prod_range(const agg_prod_t* p, int64_t r0, int64_t n, } static inline double sg_num_at(ray_t* v, int64_t row) { - if (v->type == RAY_F64) - return ((const double*)ray_data(v))[row]; + if (group_fp_type(v->type)) + return group_fp_at(ray_data(v), v->type, row); return (double)read_col_i64(ray_data(v), row, v->type, v->attrs); } -static inline void sg_pair_accum(ray_t* x, ray_t* y, const int64_t* rows, +static inline void sg_pair_accum(ray_t* x, ray_t* y, uint16_t op, + const int64_t* rows, int64_t n, bool contig, int64_t r0, double* sx, double* sy, double* sxx, double* syy, double* sxy) { + (void)op; double ax = 0.0, ay = 0.0, axx = 0.0, ayy = 0.0, axy = 0.0; if (contig) { for (int64_t j = 0; j < n; j++) { @@ -7589,7 +7614,7 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { uint16_t op = c->agg_ops[a]; if (agg_is_binary_agg(op)) { double sx = 0.0, sy = 0.0, sxx = 0.0, syy = 0.0, sxy = 0.0; - sg_pair_accum(c->agg_vecs[a], c->agg_vecs2[a], rows, n, + sg_pair_accum(c->agg_vecs[a], c->agg_vecs2[a], op, rows, n, contig, r0, &sx, &sy, &sxx, &syy, &sxy); c->partials[idx].f = sx; if (c->partial_sumsq) c->partial_sumsq[idx] = sxx; @@ -7620,19 +7645,17 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { * directly whichever order they appear in). */ } else if (op == OP_COUNT) { /* counts[gi] above suffices. */ - } else if (c->agg_vecs[a]->type == RAY_F64) { + } else if (group_fp_type(c->agg_vecs[a]->type)) { /* NaN payload = null, skip from sum (mirror all_sum). */ - const double* restrict d = - (const double*)ray_data(c->agg_vecs[a]); double acc = 0.0; double ssq = 0.0; bool need_sq = c->partial_sumsq && (op == OP_STDDEV || op == OP_STDDEV_POP || op == OP_VAR || op == OP_VAR_POP); if (contig) { - const double* restrict dr = d + r0; for (int64_t j = 0; j < n; j++) { - double v = dr[j]; + double v = group_fp_at(ray_data(c->agg_vecs[a]), + c->agg_vecs[a]->type, r0 + j); if (RAY_LIKELY(v == v)) { acc += v; if (need_sq) ssq += v * v; @@ -7640,7 +7663,8 @@ static void sg_accum_fn(void* raw, uint32_t wid, int64_t tstart, int64_t tend) { } } else { for (int64_t j = 0; j < n; j++) { - double v = d[rows[j]]; + double v = group_fp_at(ray_data(c->agg_vecs[a]), + c->agg_vecs[a]->type, rows[j]); if (RAY_LIKELY(v == v)) { acc += v; if (need_sq) ssq += v * v; @@ -7785,8 +7809,8 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!agg_type_admitted(aop, col->type)) return false; switch (col->type) { case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: - case RAY_TIME: case RAY_F64: break; - default: return false; /* F32 / exotic → generic path */ + case RAY_TIME: case RAY_F32: case RAY_F64: break; + default: return false; } agg_vecs[a] = col; if (pair) { @@ -7801,7 +7825,7 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (!agg_type_admitted(aop, col2->type)) return false; switch (col2->type) { case RAY_U8: case RAY_I16: case RAY_I32: case RAY_I64: - case RAY_F64: break; + case RAY_F32: case RAY_F64: break; default: return false; } agg_vecs2[a] = col2; @@ -7971,7 +7995,7 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, size_t si = (size_t)ti * n_aggs + a; bool pair = agg_is_binary_agg(ext->agg_ops[a]); if (pair || prod[a].enabled || ext->agg_ops[a] == OP_COUNT || - (agg_vecs[a] && agg_vecs[a]->type == RAY_F64)) + (agg_vecs[a] && group_fp_type(agg_vecs[a]->type))) sums[di].f += partials[si].f; else sums[di].i = (int64_t)((uint64_t)sums[di].i + @@ -8292,7 +8316,7 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { sums[a].i += group_strlen_at_cached( \ agg_vecs[a], dyn_row, strlen_sym_strings, strlen_sym_count); \ else if (agg_f64_mask & ((uint64_t)1 << a)) \ - sums[a].f += ((const double*)agg_ptrs[a])[dyn_row]; \ + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], dyn_row); \ else \ sums[a].i += read_col_i64(agg_ptrs[a], dyn_row, agg_types[a], 0); \ } \ @@ -8425,7 +8449,7 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { sums[a].i += group_strlen_at_cached( \ agg_vecs[a], dyn_row, strlen_sym_strings, strlen_sym_count); \ else if (agg_f64_mask & ((uint64_t)1 << a)) \ - sums[a].f += ((const double*)agg_ptrs[a])[dyn_row]; \ + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], dyn_row); \ else \ sums[a].i += read_col_i64(agg_ptrs[a], dyn_row, agg_types[a], 0);\ } \ @@ -8492,26 +8516,6 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { return NULL; } -/* Pin the DA-prescan dense-group finalize alignment. This function owns the - * dict/dense-array (DA) group path whose finalize loop (the total_groups / - * keep_min / range_count compaction region below, ~line 8395-8475, family - * da_count_emit_keep_min_u32) is the dominant symbol for high-cardinality - * group-by + count + desc-sort + take shapes. The unbounded-slots work - * added lines elsewhere in group.c - * (the legacy ght_layout_t hash path — unrelated, byte-identical to baseline - * at the hot lines), shifting this loop's absolute address/alignment and - * dropping IPC 2.07→1.96 (+7.2% cycles, +1.6% instructions — a placement - * artifact, not new work; RCA in .superpowers/sdd/task-2-bench-checkpoint.md). - * Same remedy as reduce_range's cut-3 pin: aligned(64) stabilizes the entry - * cacheline; align-loops=32 re-pins every loop head and align-jumps=32 the - * unrolled branch targets, making the finalize loop's internal alignment - * invariant to future whole-TU layout churn. Per-function attributes only — - * no global codegen flag. GCC-only: clang lacks the `optimize` attribute and - * -Werror would promote the unknown-attribute warning; the pin is - * performance-only, so clang builds go without it. */ -#if defined(__GNUC__) && !defined(__clang__) -__attribute__((aligned(64), optimize("align-loops=32","align-jumps=32"))) -#endif static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit) { if (!tbl || RAY_IS_ERR(tbl)) return tbl; @@ -9054,7 +9058,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, t == RAY_DATE || t == RAY_TIME || t == RAY_TIMESTAMP); sc_int_null_has[a] = is_sentinel_typed && (agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS); if ((agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS) && - (agg_vecs[a]->type == RAY_F64 || is_sentinel_typed)) + (group_fp_type(agg_vecs[a]->type) || is_sentinel_typed)) sc_any_nullable = true; } else { agg_ptrs[a] = NULL; @@ -9238,7 +9242,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, n_aggs * sizeof(da_val_t)); if (!sc_acc[w].min_val) { alloc_ok = false; break; } for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) sc_acc[w].min_val[a].f = DBL_MAX; + if (group_fp_type(agg_types[a])) sc_acc[w].min_val[a].f = DBL_MAX; else sc_acc[w].min_val[a].i = INT64_MAX; } } @@ -9247,7 +9251,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, n_aggs * sizeof(da_val_t)); if (!sc_acc[w].max_val) { alloc_ok = false; break; } for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) sc_acc[w].max_val[a].f = -DBL_MAX; + if (group_fp_type(agg_types[a])) sc_acc[w].max_val[a].f = -DBL_MAX; else sc_acc[w].max_val[a].i = INT64_MIN; } } @@ -9304,7 +9308,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, scalar_fn_t sc_fn = scalar_accum_fn; bool agg0_has_nulls = n_aggs > 0 && (sc_int_null_has[0] || - (agg_vecs[0] && agg_vecs[0]->type == RAY_F64 && + (agg_vecs[0] && group_fp_type(agg_vecs[0]->type) && (agg_vecs[0]->attrs & RAY_ATTR_HAS_NULLS))); if (n_aggs == 1 && !match_idx && !rowsel && agg_ptrs[0] != NULL && !agg0_has_nulls) { uint16_t op0 = ext->agg_ops[0]; @@ -9346,13 +9350,13 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, if (wnn > 0) { if (mnn == 0) m->sum[a] = wa->sum[a]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) m->sum[a].f *= wa->sum[a].f; else m->sum[a].i = (int64_t)((uint64_t)m->sum[a].i * (uint64_t)wa->sum[a].i); } } else { - if (agg_types[a] == RAY_F64) + if (group_fp_type(agg_types[a])) m->sum[a].f += wa->sum[a].f; else m->sum[a].i += wa->sum[a].i; @@ -9365,7 +9369,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, } if (need_flags & DA_NEED_MIN) { for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[a].f < m->min_val[a].f) m->min_val[a].f = wa->min_val[a].f; } else if (agg_types[a] == RAY_SYM) { @@ -9382,7 +9386,7 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, } if (need_flags & DA_NEED_MAX) { for (uint32_t a = 0; a < n_aggs; a++) { - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[a].f > m->max_val[a].f) m->max_val[a].f = wa->max_val[a].f; } else if (agg_types[a] == RAY_SYM) { @@ -9734,7 +9738,7 @@ da_path:; if (agg_vecs[a]) { agg_ptrs[a] = ray_data(agg_vecs[a]); agg_types[a] = agg_vecs[a]->type; - if (agg_vecs[a]->type == RAY_F64) + if (group_fp_type(agg_vecs[a]->type)) agg_f64_mask |= ((uint64_t)1 << a); da_int_null_sentinel[a] = agg_int_null_sentinel_for(agg_vecs[a]->type); /* Only set the int-null mask bit for storage types whose @@ -9748,7 +9752,7 @@ da_path:; if (is_sentinel_typed && (agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS)) da_int_null_mask |= ((uint64_t)1 << a); if ((agg_vecs[a]->attrs & RAY_ATTR_HAS_NULLS) && - (agg_vecs[a]->type == RAY_F64 || is_sentinel_typed)) + (group_fp_type(agg_vecs[a]->type) || is_sentinel_typed)) da_any_nullable = true; } else { agg_ptrs[a] = NULL; @@ -9802,7 +9806,7 @@ da_path:; if (!accums[w].min_val) { alloc_ok = false; break; } for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) accums[w].min_val[i].f = DBL_MAX; + if (group_fp_type(agg_types[a])) accums[w].min_val[i].f = DBL_MAX; else accums[w].min_val[i].i = INT64_MAX; } } @@ -9812,7 +9816,7 @@ da_path:; if (!accums[w].max_val) { alloc_ok = false; break; } for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) accums[w].max_val[i].f = -DBL_MAX; + if (group_fp_type(agg_types[a])) accums[w].max_val[i].f = -DBL_MAX; else accums[w].max_val[i].i = INT64_MIN; } } @@ -9930,7 +9934,7 @@ da_path:; size_t idx = base + a; uint16_t aop = ext->agg_ops[a]; if (aop == OP_SUM || aop == OP_AVG || aop == OP_ALL || aop == OP_ANY || aop == OP_STDDEV || aop == OP_STDDEV_POP || aop == OP_VAR || aop == OP_VAR_POP) { - if (agg_types[a] == RAY_F64) merged->sum[idx].f += wa->sum[idx].f; + if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; } else if (aop == OP_PROD) { /* Use per-(group, agg) non-null counts when @@ -9941,7 +9945,7 @@ da_path:; if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); @@ -9959,7 +9963,7 @@ da_path:; if (need_flags & DA_NEED_MIN) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[i].f < merged->min_val[i].f) merged->min_val[i].f = wa->min_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -9979,7 +9983,7 @@ da_path:; if (need_flags & DA_NEED_MAX) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[i].f > merged->max_val[i].f) merged->max_val[i].f = wa->max_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10041,12 +10045,12 @@ da_path:; if (wnn > 0) { if (mnn == 0) merged->sum[idx] = wa->sum[idx]; - else if (agg_types[a] == RAY_F64) + else if (group_fp_type(agg_types[a])) merged->sum[idx].f *= wa->sum[idx].f; else merged->sum[idx].i = (int64_t)((uint64_t)merged->sum[idx].i * (uint64_t)wa->sum[idx].i); } - } else if (agg_types[a] == RAY_F64) + } else if (group_fp_type(agg_types[a])) merged->sum[idx].f += wa->sum[idx].f; else merged->sum[idx].i += wa->sum[idx].i; @@ -10056,7 +10060,7 @@ da_path:; if (need_flags & DA_NEED_MIN) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->min_val[i].f < merged->min_val[i].f) merged->min_val[i].f = wa->min_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10076,7 +10080,7 @@ da_path:; if (need_flags & DA_NEED_MAX) { for (size_t i = 0; i < total; i++) { uint8_t a = (uint8_t)(i % n_aggs); - if (agg_types[a] == RAY_F64) { + if (group_fp_type(agg_types[a])) { if (wa->max_val[i].f > merged->max_val[i].f) merged->max_val[i].f = wa->max_val[i].f; } else if (agg_types[a] == RAY_SYM) { @@ -10273,7 +10277,7 @@ da_path:; if (agg_vecs[a]) { agg_ptrs[a] = ray_data(agg_vecs[a]); agg_types[a] = agg_vecs[a]->type; - if (agg_vecs[a]->type == RAY_F64) + if (group_fp_type(agg_vecs[a]->type)) agg_f64_mask |= ((uint64_t)1 << a); } else { agg_ptrs[a] = NULL; @@ -10375,7 +10379,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); @@ -10483,7 +10487,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); @@ -10572,7 +10576,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); } @@ -10713,7 +10717,7 @@ da_path:; agg_vecs[a], r, strlen_sym_strings, strlen_sym_count); else if (agg_f64_mask & ((uint64_t)1 << a)) - sums[a].f += ((const double*)agg_ptrs[a])[r]; + sums[a].f += group_fp_at(agg_ptrs[a], agg_types[a], r); else sums[a].i += read_col_i64(agg_ptrs[a], r, agg_types[a], 0); } @@ -10791,7 +10795,8 @@ ht_path:; * stride budgets or the int8 value-slot count — none of which a ≤8 shape * can hit. */ ght_layout_t ght_layout; - if (!ght_compute_layout(&ght_layout, n_keys, n_aggs, agg_vecs, ght_need, + if (!ght_compute_layout(&ght_layout, n_keys, n_aggs, agg_vecs, agg_vecs2, + ght_need, ext->agg_ops, key_types)) { for (uint32_t a = 0; a < n_aggs; a++) { if (agg_owned[a] && agg_vecs[a]) ray_release(agg_vecs[a]); @@ -11418,7 +11423,7 @@ v2_emit:; for (uint32_t a = 0; a < n_aggs; a++) { uint16_t agg_op = ext->agg_ops[a]; ray_t* agg_col = agg_vecs[a]; - bool is_f64 = agg_col && agg_col->type == RAY_F64; + bool is_f64 = (ght_layout.agg_flags[a] & GHT_AF_F64) != 0; int8_t out_type; switch (agg_op) { case OP_AVG: @@ -11444,7 +11449,8 @@ v2_emit:; case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col = ray_vec_new(out_type, (int64_t)total_grps); if (!new_col || RAY_IS_ERR(new_col)) { @@ -12041,7 +12047,7 @@ sequential_fallback:; for (uint32_t a = 0; a < n_aggs; a++) { uint16_t agg_op = ext->agg_ops[a]; ray_t* agg_col = agg_vecs[a]; - bool is_f64 = agg_col && agg_col->type == RAY_F64; + bool is_f64 = (ly->agg_flags[a] & GHT_AF_F64) != 0; int8_t out_type; switch (agg_op) { case OP_AVG: @@ -12073,7 +12079,8 @@ sequential_fallback:; case OP_PROD: out_type = is_f64 ? RAY_F64 : RAY_I64; break; default: - out_type = agg_col ? agg_col->type : RAY_I64; break; + out_type = is_f64 ? RAY_F64 + : (agg_col ? agg_col->type : RAY_I64); break; } ray_t* new_col; /* Drive off the layout bitmask, not the op literal: wide-element diff --git a/src/ops/idxop.c b/src/ops/idxop.c index 130769648..7d150db54 100644 --- a/src/ops/idxop.c +++ b/src/ops/idxop.c @@ -1505,13 +1505,6 @@ ray_t* ray_index_part_eq_rowsel(ray_t* col, int64_t key) { ? ray_read_sym(kb, p, RAY_SYM, keys->attrs) : set_vec_read_i64(kb, keys->type, p); if (pkey != key) continue; - /* A direct FILTER rowsel still feeds the generic table compactor. - * For a dense partition that O(selected rows × columns) gather is - * slower than the normal predicate scan. Keep dense partitions on - * the scan path; the dedicated GROUP BY slice path remains dense-safe - * because it streams the range directly into its aggregates. */ - if (ln[p] > 64 && ln[p] > (col->len >> 1)) - return NULL; idx_span_t span = { .start = st[p], .len = ln[p] }; return rowsel_from_sorted_spans(col->len, &span, 1, ln[p]); } diff --git a/src/ops/internal.h b/src/ops/internal.h index ac0094eb9..488124302 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -1251,7 +1251,8 @@ static inline int64_t agg_int_null_sentinel_for(int8_t t) { } bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, - ray_t** agg_vecs, uint8_t need_flags, + ray_t** agg_vecs, ray_t** agg_vecs2, + uint8_t need_flags, const uint16_t* agg_ops, const int8_t* key_types); /* By-value copy that fixes the base pointers. Dispatches on STORAGE, not diff --git a/src/ops/opt.c b/src/ops/opt.c index 369356c28..208416c9c 100644 --- a/src/ops/opt.c +++ b/src/ops/opt.c @@ -79,6 +79,8 @@ bool ray_opt_no_group_pushdown = false; static int8_t promote_type(int8_t a, int8_t b) { if (a == RAY_STR || b == RAY_STR) return RAY_STR; if (a == RAY_F64 || b == RAY_F64) return RAY_F64; + if (a == RAY_F32 || b == RAY_F32) + return (a == RAY_F32 && b == RAY_F32) ? RAY_F32 : RAY_F64; /* Treat SYM/TIMESTAMP/DATE/TIME as integer-class types */ if (a == RAY_I64 || b == RAY_I64 || a == RAY_SYM || b == RAY_SYM || a == RAY_TIMESTAMP || b == RAY_TIMESTAMP) return RAY_I64; @@ -293,6 +295,11 @@ static bool atom_to_numeric(ray_t* v, double* out_f, int64_t* out_i, bool* is_f6 *out_i = (int64_t)v->f64; *is_f64 = true; return true; + case -RAY_F32: + *out_f = (double)(float)v->f64; + *out_i = (int64_t)(float)v->f64; + *is_f64 = true; + return true; case -RAY_I64: case -RAY_SYM: case -RAY_DATE: @@ -327,7 +334,8 @@ static bool pow_atom_type_admitted(ray_t* v) { if (!v || !ray_is_atom(v)) return false; return v->type == -RAY_BOOL || v->type == -RAY_U8 || v->type == -RAY_I16 || v->type == -RAY_I32 || - v->type == -RAY_I64 || v->type == -RAY_F64; + v->type == -RAY_I64 || v->type == -RAY_F32 || + v->type == -RAY_F64; } static bool replace_with_const(ray_graph_t* g, ray_op_t* node, ray_t* literal) { @@ -367,12 +375,15 @@ static bool fold_unary_const(ray_graph_t* g, ray_op_t* node) { ray_t* folded = NULL; switch (node->opcode) { case OP_NEG: - if (is_f64) folded = ray_f64(-vf); + if (node->out_type == RAY_F32) folded = ray_f32((float)-vf); + else if (is_f64) folded = ray_f64(-vf); else if (vi == INT64_MIN) return false; /* -INT64_MIN overflows */ else folded = ray_i64(-vi); break; case OP_ABS: - if (is_f64) + if (node->out_type == RAY_F32) + folded = ray_f32(fabsf((float)vf)); + else if (is_f64) folded = ray_f64(fabs(vf)); else if (vi == INT64_MIN) return false; /* -INT64_MIN overflows */ else folded = ray_i64(vi < 0 ? -vi : vi); @@ -418,10 +429,12 @@ static bool fold_unary_const(ray_graph_t* g, ray_op_t* node) { break; } case OP_CEIL: - folded = is_f64 ? ray_f64(ceil(vf)) : ray_i64(vi); + folded = node->out_type == RAY_F32 ? ray_f32(ceilf((float)vf)) + : is_f64 ? ray_f64(ceil(vf)) : ray_i64(vi); break; case OP_FLOOR: - folded = is_f64 ? ray_f64(floor(vf)) : ray_i64(vi); + folded = node->out_type == RAY_F32 ? ray_f32(floorf((float)vf)) + : is_f64 ? ray_f64(floor(vf)) : ray_i64(vi); break; default: return false; @@ -453,6 +466,28 @@ static bool fold_binary_const(ray_graph_t* g, ray_op_t* node) { ray_t* folded = NULL; switch (node->out_type) { + case RAY_F32: { + float lv = (float)(l_is_f64 ? lf : (double)li); + float rv = (float)(r_is_f64 ? rf : (double)ri); + float r = 0.0f; + switch (node->opcode) { + case OP_ADD: r = lv + rv; break; + case OP_SUB: r = lv - rv; break; + case OP_MUL: r = lv * rv; break; + case OP_MOD: + if (rv == 0.0f) r = NULL_F32; + else { + r = fmodf(lv, rv); + if (r != 0.0f && ((r > 0.0f) != (rv > 0.0f))) r += rv; + } + break; + case OP_MIN2: r = lv < rv ? lv : rv; break; + case OP_MAX2: r = lv > rv ? lv : rv; break; + default: return false; + } + folded = ray_f32(isfinite(r) ? r : NULL_F32); + break; + } case RAY_F64: { double lv = l_is_f64 ? lf : (double)li; double rv = r_is_f64 ? rf : (double)ri; diff --git a/src/ops/pivot.c b/src/ops/pivot.c index 7ef5a0e06..4b8f14457 100644 --- a/src/ops/pivot.c +++ b/src/ops/pivot.c @@ -937,7 +937,8 @@ ray_t* exec_pivot(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * point; every exit below (early return AND pivot_cleanup) must call * it exactly once so a spilled layout's heap block is never leaked. */ ght_layout_t ly; - if (!ght_compute_layout(&ly, n_keys, 1, agg_vecs, need_flags, agg_ops, key_types)) { + if (!ght_compute_layout(&ly, n_keys, 1, agg_vecs, NULL, + need_flags, agg_ops, key_types)) { scratch_free(key_hdr); return ray_error("limit", "pivot: key stride budget exceeded"); } diff --git a/src/store/col.c b/src/store/col.c index 7e8b9ca08..b6af7d00c 100644 --- a/src/store/col.c +++ b/src/store/col.c @@ -146,7 +146,7 @@ static size_t col_str_pool_payload_len(const ray_t* vec); static bool is_serializable_type(int8_t t) { switch (t) { case RAY_BOOL: case RAY_U8: case RAY_I16: - case RAY_I32: case RAY_I64: case RAY_F64: + case RAY_I32: case RAY_I64: case RAY_F32: case RAY_F64: case RAY_DATE: case RAY_TIME: case RAY_TIMESTAMP: case RAY_GUID: case RAY_SYM: case RAY_STR: return true; diff --git a/test/rfl/system/csv_explicit_numeric_types.rfl b/test/rfl/system/csv_explicit_numeric_types.rfl new file mode 100644 index 000000000..d053c5209 --- /dev/null +++ b/test/rfl/system/csv_explicit_numeric_types.rfl @@ -0,0 +1,48 @@ +;; Explicit numeric schemas must survive the streaming .csv.splayed writer +;; and the mmap-backed .db.splayed.get reload without widening. + +(.sys.exec "rm -rf rf_test_explicit_numeric rf_test_explicit_numeric.csv") -- 0 +(.sys.exec "printf 'f32a,f32b,i32,i16\n1.25,0.25,100000,32000\n-2.5,1.5,-7,-300\n3.75,-0.25,42,9\n16777216,1,100001,9\n' > rf_test_explicit_numeric.csv") -- 0 + +(set T (.csv.splayed [F32 F32 I32 I16] "rf_test_explicit_numeric.csv" "rf_test_explicit_numeric/")) +(count T) -- 4 +(type (at T 'f32a)) -- 'F32 +(type (at T 'f32b)) -- 'F32 +(type (at T 'i32)) -- 'I32 +(type (at T 'i16)) -- 'I16 +(at (at T 'f32a) 0) -- 1.25 +(at (at T 'f32a) 1) -- -2.5 +(at (at (select {x: (- f32a f32b) from: T}) 'x) 0) -- 1.0 +(at (at (select {x: (- f32a f32b) from: T}) 'x) 2) -- 4.0 +(type (at (select {x: (+ f32a f32b) from: T}) 'x)) -- 'F32 +;; 2^24 + 1 rounds back to 2^24 at F32 precision. +(at (at (select {x: (+ f32a f32b) from: T}) 'x) 3) -- 16777216.0 +(type (at (at T 'f32a) 0)) -- 'f32 +(type (at (at T 'f32b) 0)) -- 'f32 +(type (+ (at (at T 'f32a) 0) (at (at T 'f32b) 0))) -- 'f32 +(+ (at (at T 'f32a) 0) (at (at T 'f32b) 0)) -- 1.5 +(type (at (select {x: (neg f32a) from: T}) 'x)) -- 'F32 +(at (at (select {x: (neg f32a) from: T}) 'x) 1) -- 2.5 +(type (at (select {x: (+ f32a i32) from: T}) 'x)) -- 'F64 +(at (at (select {x: (+ f32a i32) from: T}) 'x) 0) -- 100001.25 +(type (wavg (at T 'i32) (at T 'f32b))) -- 'f64 +(set G (select {mx: (max f32a) wa: (wavg i32 f32b) by: {k: i16} from: T})) +(count G) -- 3 +(type (at G 'mx)) -- 'F64 +(type (at G 'wa)) -- 'F64 +(set G1 (select {mx: (max f32a) wa: (wavg i32 f32b) from: T where: (== i16 32000)})) +(at (at G1 'mx) 0) -- 1.25 +(at (at G1 'wa) 0) -- 0.25 +(at (at T 'i32) 0) -- 100000 +(at (at T 'i16) 1) -- -300 + +(set R (.db.splayed.get "rf_test_explicit_numeric/")) +(type (at R 'f32a)) -- 'F32 +(type (at R 'f32b)) -- 'F32 +(type (at R 'i32)) -- 'I32 +(type (at R 'i16)) -- 'I16 +(at (at R 'f32a) 2) -- 3.75 +(at (at R 'i32) 1) -- -7 +(at (at R 'i16) 2) -- 9 + +(.sys.exec "rm -rf rf_test_explicit_numeric rf_test_explicit_numeric.csv") -- 0 diff --git a/test/test_group_extra.c b/test/test_group_extra.c index 4b957c064..3b3154a27 100644 --- a/test/test_group_extra.c +++ b/test/test_group_extra.c @@ -2319,7 +2319,7 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { } ght_layout_t master; - TEST_ASSERT_TRUE(ght_compute_layout(&master, NK, NA, agg_vecs, + TEST_ASSERT_TRUE(ght_compute_layout(&master, NK, NA, agg_vecs, NULL, GHT_NEED_SUM, agg_ops, key_types)); /* > GHT_INLINE on both axes: this must be a real, owned spill block. */ TEST_ASSERT_NOT_NULL(master.spill_hdr); @@ -2375,7 +2375,7 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { * must have its copy RE-POINTED at the destination's own inline arrays, * never left aliasing the source's — the mirror of the spill leg above. */ ght_layout_t inl; - TEST_ASSERT_TRUE(ght_compute_layout(&inl, 2, 2, agg_vecs, + TEST_ASSERT_TRUE(ght_compute_layout(&inl, 2, 2, agg_vecs, NULL, GHT_NEED_SUM, agg_ops, key_types)); TEST_ASSERT_NULL(inl.spill_hdr); TEST_ASSERT_TRUE(inl.agg_val_slot == inl.agg_val_slot_in); diff --git a/test/test_idx_route.c b/test/test_idx_route.c index ed205f804..7131694e8 100644 --- a/test/test_idx_route.c +++ b/test/test_idx_route.c @@ -1319,7 +1319,7 @@ static ray_t* make_part_sym_table(void) { ray_sym_intern("part_m", 6), ray_sym_intern("part_b", 6) }; - int64_t ends[] = {700, 2000, 2900, 4096}; + int64_t ends[] = {500, 2800, 3400, 4096}; ray_t* syms = ray_sym_vec_new(RAY_SYM_W64, n); ray_t* vals = ray_vec_new(RAY_I64, n); if (!syms || !vals || RAY_IS_ERR(syms) || RAY_IS_ERR(vals)) { @@ -1411,7 +1411,7 @@ static test_result_t test_in_hash(void) { } /* A part index should keep dense unions as physical spans. The chosen - * blocks select 2496/4096 rows, which deliberately exceeds the hash route's + * blocks select 2996/4096 rows, which deliberately exceeds the hash route's * density guard, and exercise both full and partial morsels. */ static test_result_t test_in_part_sym_dense(void) { ray_heap_init(); @@ -1424,7 +1424,7 @@ static test_result_t test_in_part_sym_dense(void) { uint64_t hits_before = ray_idx_hits[IDX_SITE_IN]; ray_t* ra = run_filter(tbl_a, pred_in_part_syms); TEST_ASSERT_FALSE(RAY_IS_ERR(ra)); - TEST_ASSERT_EQ_I(ray_table_nrows(ra), 2496); + TEST_ASSERT_EQ_I(ray_table_nrows(ra), 2996); TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_IN] - cons_before), 1); TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_IN] - hits_before), 1); @@ -1432,7 +1432,7 @@ static test_result_t test_in_part_sym_dense(void) { TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_b)); ray_t* rb = run_filter(tbl_b, pred_in_part_syms); TEST_ASSERT_FALSE(RAY_IS_ERR(rb)); - TEST_ASSERT_EQ_I(ray_table_nrows(rb), 2496); + TEST_ASSERT_EQ_I(ray_table_nrows(rb), 2996); TEST_ASSERT(v_cols_equal(ra, rb), "v column mismatch between parted IN and scan"); @@ -1443,8 +1443,8 @@ static test_result_t test_in_part_sym_dense(void) { PASS(); } -/* Equality on a dense part must use its physical span directly rather than - * tripping the hash route's density guard. part_a occupies rows [700,2000). */ +/* Equality on a part larger than half the table must still use its physical + * span directly. part_a occupies rows [500,2800). */ static test_result_t test_eq_part_sym_dense(void) { ray_heap_init(); ray_t* tbl_a = make_part_sym_table(); @@ -1456,7 +1456,7 @@ static test_result_t test_eq_part_sym_dense(void) { uint64_t hits_before = ray_idx_hits[IDX_SITE_FILTER_PART]; ray_t* ra = run_filter(tbl_a, pred_eq_part_a); TEST_ASSERT_FALSE(RAY_IS_ERR(ra)); - TEST_ASSERT_EQ_I(ray_table_nrows(ra), 1300); + TEST_ASSERT_EQ_I(ray_table_nrows(ra), 2300); TEST_ASSERT_EQ_I((int64_t)(ray_idx_consults[IDX_SITE_FILTER_PART] - cons_before), 1); TEST_ASSERT_EQ_I((int64_t)(ray_idx_hits[IDX_SITE_FILTER_PART] - hits_before), 1); @@ -1464,7 +1464,7 @@ static test_result_t test_eq_part_sym_dense(void) { TEST_ASSERT_FALSE(RAY_IS_ERR(tbl_b)); ray_t* rb = run_filter(tbl_b, pred_eq_part_a); TEST_ASSERT_FALSE(RAY_IS_ERR(rb)); - TEST_ASSERT_EQ_I(ray_table_nrows(rb), 1300); + TEST_ASSERT_EQ_I(ray_table_nrows(rb), 2300); TEST_ASSERT(v_cols_equal(ra, rb), "v column mismatch between parted EQ and scan"); diff --git a/test/test_store.c b/test/test_store.c index 9888e96c5..4edb0210b 100644 --- a/test/test_store.c +++ b/test/test_store.c @@ -141,6 +141,43 @@ static test_result_t test_col_mmap_f64(void) { PASS(); } +/* ---- test_col_mmap_f32 ------------------------------------------------- */ + +static test_result_t test_col_mmap_f32(void) { + float raw[] = {1.25f, -2.5f, 3.75f, 0.0f}; + ray_t* vec = ray_vec_from_raw(RAY_F32, raw, 4); + TEST_ASSERT_NOT_NULL(vec); + TEST_ASSERT_FALSE(RAY_IS_ERR(vec)); + + ray_err_t err = ray_col_save(vec, TMP_COL_PATH); + TEST_ASSERT_EQ_I(err, RAY_OK); + + ray_t* loaded = ray_col_load(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(loaded); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + TEST_ASSERT_EQ_I(loaded->type, RAY_F32); + TEST_ASSERT_EQ_I(loaded->len, 4); + float* loaded_data = (float*)ray_data(loaded); + for (int i = 0; i < 4; i++) + TEST_ASSERT(loaded_data[i] == raw[i], "f32 load value mismatch"); + ray_release(loaded); + + ray_t* mapped = ray_col_mmap(TMP_COL_PATH); + TEST_ASSERT_NOT_NULL(mapped); + TEST_ASSERT_FALSE(RAY_IS_ERR(mapped)); + TEST_ASSERT_EQ_U(mapped->mmod, 1); + TEST_ASSERT_EQ_I(mapped->type, RAY_F32); + TEST_ASSERT_EQ_I(mapped->len, 4); + float* mapped_data = (float*)ray_data(mapped); + for (int i = 0; i < 4; i++) + TEST_ASSERT(mapped_data[i] == raw[i], "f32 mmap value mismatch"); + + ray_release(mapped); + ray_release(vec); + unlink(TMP_COL_PATH); + PASS(); +} + /* ---- test_col_mmap_cow ------------------------------------------------- */ static test_result_t test_col_mmap_cow(void) { @@ -4909,6 +4946,7 @@ static test_result_t test_col_save_nyi_type(void) { const test_entry_t store_entries[] = { { "store/col_mmap_i64", test_col_mmap_i64, store_setup, store_teardown }, { "store/col_mmap_f64", test_col_mmap_f64, store_setup, store_teardown }, + { "store/col_mmap_f32", test_col_mmap_f32, store_setup, store_teardown }, { "store/col_mmap_cow", test_col_mmap_cow, store_setup, store_teardown }, { "store/col_mmap_refcount", test_col_mmap_refcount, store_setup, store_teardown }, { "store/col_mmap_corrupt", test_col_mmap_corrupt, store_setup, store_teardown }, From 57636455f53d25b3a17e2b44a1c197367d76885c Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 20:15:10 +0200 Subject: [PATCH 07/10] fix(rowsel): initialize mixed intersections --- src/ops/rowsel.c | 3 +-- test/test_rowsel.c | 55 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/ops/rowsel.c b/src/ops/rowsel.c index 321e8d709..0dc883587 100644 --- a/src/ops/rowsel.c +++ b/src/ops/rowsel.c @@ -681,9 +681,8 @@ ray_t* ray_rowsel_intersect(ray_t* a, ray_t* b) { else if (bv < av) bp++; else { oi[cum++] = av; ap++; bp++; } } + of[s] = count == 0 ? RAY_SEL_NONE : RAY_SEL_MIX; } - if (of[s] != RAY_SEL_ALL) - of[s] = cum == oo[s] ? RAY_SEL_NONE : RAY_SEL_MIX; } } oo[ns] = cum; diff --git a/test/test_rowsel.c b/test/test_rowsel.c index 1f83c2569..8a28a0a3e 100644 --- a/test/test_rowsel.c +++ b/test/test_rowsel.c @@ -422,6 +422,59 @@ static test_result_t test_rowsel_refine_null_existing(void) { PASS(); } +/* MIX∩MIX must initialize its output flag even when the recycled rowsel + * payload previously held ALL. The allocator zeroes object headers, not + * payload bytes, so reading the output flag before assigning it can turn a + * partial intersection into an entire morsel. */ +static test_result_t test_rowsel_intersect_mixed_reused_block(void) { + ray_heap_init(); + enum { N = RAY_MORSEL_ELEMS }; + uint8_t a_bits[N]; + uint8_t b_bits[N]; + memset(a_bits, 0, sizeof(a_bits)); + memset(b_bits, 0, sizeof(b_bits)); + + const int64_t a_start = N / 4; + const int64_t b_end = (N * 5) / 8; + for (int64_t i = a_start; i < N; i++) a_bits[i] = 1; + for (int64_t i = 0; i < b_end; i++) b_bits[i] = 1; + + ray_t* a_pred = make_pred(a_bits, N); + ray_t* b_pred = make_pred(b_bits, N); + TEST_ASSERT_NOT_NULL(a_pred); + TEST_ASSERT_NOT_NULL(b_pred); + ray_t* a = ray_rowsel_from_pred(a_pred); + ray_t* b = ray_rowsel_from_pred(b_pred); + TEST_ASSERT_NOT_NULL(a); + TEST_ASSERT_NOT_NULL(b); + TEST_ASSERT_EQ_I(ray_rowsel_flags(a)[0], RAY_SEL_MIX); + TEST_ASSERT_EQ_I(ray_rowsel_flags(b)[0], RAY_SEL_MIX); + + const int64_t expected = b_end - a_start; + ray_t* poison = ray_rowsel_new(N, expected, expected); + TEST_ASSERT_NOT_NULL(poison); + ray_rowsel_flags(poison)[0] = RAY_SEL_ALL; + ray_rowsel_release(poison); + + ray_t* out = ray_rowsel_intersect(a, b); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQ_I(ray_rowsel_meta(out)->total_pass, expected); + TEST_ASSERT_EQ_I(ray_rowsel_flags(out)[0], RAY_SEL_MIX); + int64_t rows[N]; + int64_t count = reconstruct(out, rows); + TEST_ASSERT_EQ_I(count, expected); + TEST_ASSERT_EQ_I(rows[0], a_start); + TEST_ASSERT_EQ_I(rows[count - 1], b_end - 1); + + ray_rowsel_release(out); + ray_rowsel_release(a); + ray_rowsel_release(b); + ray_release(a_pred); + ray_release(b_pred); + ray_heap_destroy(); + PASS(); +} + /* Streaming emitter must produce a block byte-identical to the * whole-vec ray_rowsel_from_pred over the same bools. Pattern spans * >1 morsel with a NONE seg, an ALL seg, and MIX segs. */ @@ -486,8 +539,8 @@ const test_entry_t rowsel_entries[] = { { "rowsel/to_indices", test_rowsel_to_indices, NULL, NULL }, { "rowsel/refine", test_rowsel_refine, NULL, NULL }, { "rowsel/refine_null_existing", test_rowsel_refine_null_existing, NULL, NULL }, + { "rowsel/intersect_mixed_reused_block", test_rowsel_intersect_mixed_reused_block, NULL, NULL }, { "rowsel/emit_equiv", test_rowsel_emit_equiv, NULL, NULL }, { NULL, NULL, NULL, NULL }, }; - From 8b063d9a9d084a2d51e02fc876e9e9c7c0cd0074 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 20:25:03 +0200 Subject: [PATCH 08/10] fix(csv): serialize F32 columns --- src/io/csv.c | 3 +++ test/test_csv.c | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/io/csv.c b/src/io/csv.c index 089158994..5f766db72 100644 --- a/src/io/csv.c +++ b/src/io/csv.c @@ -3146,6 +3146,9 @@ static void csv_write_cell(csv_writer_t* w, const csv_col_info_t* ci, int64_t r) case RAY_F64: csv_write_f64(w, ((const double*)d)[dr]); break; + case RAY_F32: + csv_write_f64(w, (double)((const float*)d)[dr]); + break; case RAY_DATE: csv_write_date(w, ((const int32_t*)d)[dr]); break; diff --git a/test/test_csv.c b/test/test_csv.c index cc9031882..9d0ac1f4e 100644 --- a/test/test_csv.c +++ b/test/test_csv.c @@ -111,6 +111,41 @@ static test_result_t test_csv_roundtrip_f64(void) { PASS(); } +static test_result_t test_csv_roundtrip_f32(void) { + ray_heap_init(); + (void)ray_sym_init(); + + float vals[] = {41.87f, -0.125f, 0.0f}; + ray_t* vec = ray_vec_from_raw(RAY_F32, vals, 3); + int64_t name = ray_sym_intern("price", 5); + ray_t* tbl = ray_table_new(1); + tbl = ray_table_add_col(tbl, name, vec); + ray_release(vec); + + ray_err_t err = ray_write_csv(tbl, TMP_CSV); + TEST_ASSERT_EQ_I(err, RAY_OK); + + int8_t schema[] = {RAY_F32}; + ray_t* loaded = ray_read_csv_opts(TMP_CSV, ',', true, schema, 1); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + TEST_ASSERT_EQ_I(ray_table_nrows(loaded), 3); + + ray_t* col = ray_table_get_col_idx(loaded, 0); + TEST_ASSERT_NOT_NULL(col); + TEST_ASSERT_EQ_I(col->type, RAY_F32); + float* loaded_data = (float*)ray_data(col); + TEST_ASSERT_EQ_F(loaded_data[0], vals[0], 1e-6); + TEST_ASSERT_EQ_F(loaded_data[1], vals[1], 1e-6); + TEST_ASSERT_EQ_F(loaded_data[2], vals[2], 1e-6); + + ray_release(loaded); + ray_release(tbl); + unlink(TMP_CSV); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + static test_result_t test_csv_multi_column(void) { ray_heap_init(); (void)ray_sym_init(); @@ -1513,6 +1548,7 @@ const test_entry_t csv_entries[] = { { "csv/roundtrip_i64", test_csv_roundtrip_i64, NULL, NULL }, { "csv/roundtrip_guid", test_csv_guid_roundtrip, NULL, NULL }, { "csv/roundtrip_f64", test_csv_roundtrip_f64, NULL, NULL }, + { "csv/roundtrip_f32", test_csv_roundtrip_f32, NULL, NULL }, { "csv/multi_column", test_csv_multi_column, NULL, NULL }, { "csv/empty_table", test_csv_empty_table, NULL, NULL }, { "csv/null_i64", test_csv_null_i64, NULL, NULL }, From 6879ff35598adf2ebf32ffd8e03a0317877295cc Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 22:51:24 +0200 Subject: [PATCH 09/10] fix(ops): complete F32 query execution --- src/ops/expr.c | 37 ++++++++++++++++- src/ops/internal.h | 2 +- src/ops/sort.c | 41 ++++++++++++++++++- src/ops/window.c | 10 ++++- .../query/query_select_fastpath_coverage.rfl | 9 ++++ test/rfl/query/window.rfl | 26 ++++++++++++ test/rfl/sort/sort_coverage2.rfl | 12 ++++++ 7 files changed, 132 insertions(+), 5 deletions(-) diff --git a/src/ops/expr.c b/src/ops/expr.c index 5d71f64db..332747f6d 100644 --- a/src/ops/expr.c +++ b/src/ops/expr.c @@ -2381,7 +2381,42 @@ ray_t* exec_elementwise_unary(ray_graph_t* g, ray_op_t* op, ray_t* input) { int64_t out_off = 0; uint16_t opc = op->opcode; - if (in_type == RAY_F32 && out_type == RAY_F32) { + if (opc == OP_CAST && out_type == RAY_F32) { + /* The query compiler admits `(as 'F32 expr)` for every numeric input. + * Keep that surface complete in the typed fallback. Null propagation + * below rewrites source sentinels to NULL_F32 after conversion. */ + if (in_type != RAY_F64 && in_type != RAY_F32 && + in_type != RAY_I64 && in_type != RAY_TIMESTAMP && + in_type != RAY_I32 && in_type != RAY_DATE && in_type != RAY_TIME && + in_type != RAY_I16 && in_type != RAY_U8 && in_type != RAY_BOOL) { + ray_release(result); + return ray_error("type", "cast: cannot convert %s to F32", + ray_type_name(in_type)); + } + while (ray_morsel_next(&m)) { + int64_t n = m.morsel_len; + float* dst = (float*)ray_data(result) + out_off; + if (in_type == RAY_F64) { + const double* src = (const double*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_F32) { + memcpy(dst, m.morsel_ptr, (size_t)n * sizeof(float)); + } else if (in_type == RAY_I64 || in_type == RAY_TIMESTAMP) { + const int64_t* src = (const int64_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_I32 || in_type == RAY_DATE || in_type == RAY_TIME) { + const int32_t* src = (const int32_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else if (in_type == RAY_I16) { + const int16_t* src = (const int16_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } else { + const uint8_t* src = (const uint8_t*)m.morsel_ptr; + for (int64_t i = 0; i < n; i++) dst[i] = (float)src[i]; + } + out_off += n; + } + } else if (in_type == RAY_F32 && out_type == RAY_F32) { while (ray_morsel_next(&m)) { int64_t n = m.morsel_len; float* src = (float*)m.morsel_ptr; diff --git a/src/ops/internal.h b/src/ops/internal.h index 488124302..3c6034bc4 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -804,7 +804,7 @@ static inline uint8_t radix_key_bytes(int8_t type) { switch (type) { case RAY_BOOL: case RAY_U8: return 1; case RAY_I16: return 2; - case RAY_I32: case RAY_DATE: case RAY_TIME: return 4; + case RAY_I32: case RAY_F32: case RAY_DATE: case RAY_TIME: return 4; default: return 8; /* I64, F64, TIMESTAMP, SYM */ } } diff --git a/src/ops/sort.c b/src/ops/sort.c index 09a0738ca..b50abf921 100644 --- a/src/ops/sort.c +++ b/src/ops/sort.c @@ -55,6 +55,11 @@ int sort_cmp(const sort_cmp_ctx_t* ctx, int64_t a, int64_t b) { double vb = ((double*)ray_data(col))[b]; if (va < vb) cmp = -1; else if (va > vb) cmp = 1; + } else if (col->type == RAY_F32) { + float va = ((float*)ray_data(col))[a]; + float vb = ((float*)ray_data(col))[b]; + if (va < vb) cmp = -1; + else if (va > vb) cmp = 1; } else if (col->type == RAY_I64 || col->type == RAY_TIMESTAMP) { int64_t va = ((int64_t*)ray_data(col))[a]; int64_t vb = ((int64_t*)ray_data(col))[b]; @@ -997,6 +1002,26 @@ void radix_encode_fn(void* arg, uint32_t wid, int64_t start, int64_t end) { } break; } + case RAY_F32: { + const float* d = (const float*)c->data; + bool nf = c->nulls_first; + bool desc = c->desc; + uint32_t nan_e = (nf ^ desc) ? 0U : UINT32_MAX; + for (int64_t i = start; i < end; i++) { + uint32_t bits; + memcpy(&bits, &d[i], 4); + uint32_t e; + if ((bits & 0x7F800000U) == 0x7F800000U && + (bits & 0x007FFFFFU)) { + e = nan_e; + } else { + uint32_t mask = -(bits >> 31) | ((uint32_t)1 << 31); + e = bits ^ mask; + } + c->keys[i] = desc ? (uint64_t)(~e) : (uint64_t)e; + } + break; + } case RAY_I32: case RAY_DATE: case RAY_TIME: { const int32_t* d = (const int32_t*)c->data; bool has_nulls = c->col && (c->col->attrs & RAY_ATTR_HAS_NULLS); @@ -2215,6 +2240,15 @@ static void radix_decode_into(void* dst, int8_t type, const uint64_t* sorted_key uint64_t bits = k ^ mask; memcpy(&d[i], &bits, 8); } + } else if (type == RAY_F32) { + float* d = (float*)dst; + for (int64_t i = 0; i < n; i++) { + uint32_t k = (uint32_t)sorted_keys[i]; + if (desc) k = ~k; + uint32_t mask = (k >> 31) ? ((uint32_t)1 << 31) : ~(uint32_t)0; + uint32_t bits = k ^ mask; + memcpy(&d[i], &bits, 4); + } } else if (type == RAY_I32 || type == RAY_DATE || type == RAY_TIME) { int32_t* d = (int32_t*)dst; if (desc) @@ -2308,7 +2342,9 @@ static ray_t* sort_indices_ex(ray_t** cols, uint8_t* descs, uint8_t* nulls_first if (!cols[k]) { can_radix = false; break; } int8_t t = cols[k]->type; if (t == RAY_STR || t == RAY_GUID) { has_wide_key = true; continue; } - if (t != RAY_I64 && t != RAY_F64 && t != RAY_I32 && t != RAY_I16 && + if (t == RAY_F32 && n_cols != 1) { can_radix = false; break; } + if (t != RAY_I64 && t != RAY_F64 && t != RAY_F32 && + t != RAY_I32 && t != RAY_I16 && t != RAY_BOOL && t != RAY_U8 && t != RAY_SYM && t != RAY_DATE && t != RAY_TIME && t != RAY_TIMESTAMP) { can_radix = false; break; @@ -3113,7 +3149,8 @@ static ray_t* topk_indices_single(ray_t* col, uint8_t desc, uint8_t nf, int8_t type = col->type; /* Whitelist of types where radix_encode_fn produces an order-preserving * uint64 — exactly the cases topk can handle without a comparator. */ - bool ok = (type == RAY_I64 || type == RAY_TIMESTAMP || type == RAY_F64 || + bool ok = (type == RAY_I64 || type == RAY_TIMESTAMP || + type == RAY_F64 || type == RAY_F32 || type == RAY_I32 || type == RAY_DATE || type == RAY_TIME || type == RAY_SYM || type == RAY_STR || type == RAY_GUID || type == RAY_I16 || type == RAY_BOOL || type == RAY_U8); diff --git a/src/ops/window.c b/src/ops/window.c index df4612af4..78f154d35 100644 --- a/src/ops/window.c +++ b/src/ops/window.c @@ -45,6 +45,12 @@ static inline bool win_keys_differ(ray_t* const* vecs, uint32_t n_keys, if (a != b) return true; break; } + case RAY_F32: { + float a = ((const float*)ray_data(col))[ra]; + float b = ((const float*)ray_data(col))[rb]; + if (a != b) return true; + break; + } case RAY_I32: case RAY_DATE: case RAY_TIME: if (((const int32_t*)ray_data(col))[ra] != ((const int32_t*)ray_data(col))[rb]) return true; @@ -89,6 +95,7 @@ static inline bool win_keys_differ(ray_t* const* vecs, uint32_t n_keys, static inline double win_read_f64(ray_t* col, int64_t row) { switch (col->type) { case RAY_F64: return ((const double*)ray_data(col))[row]; + case RAY_F32: return (double)((const float*)ray_data(col))[row]; case RAY_I64: case RAY_TIMESTAMP: return (double)((const int64_t*)ray_data(col))[row]; case RAY_I32: case RAY_DATE: case RAY_TIME: @@ -110,6 +117,7 @@ static inline int64_t win_read_i64(ray_t* col, int64_t row) { case RAY_SYM: return sym_cell_runtime_id(col, row); case RAY_F64: return (int64_t)((const double*)ray_data(col))[row]; + case RAY_F32: return (int64_t)((const float*)ray_data(col))[row]; case RAY_I16: return (int64_t)((const int16_t*)ray_data(col))[row]; case RAY_BOOL: case RAY_U8: return (int64_t)((const uint8_t*)ray_data(col))[row]; default: return 0; @@ -1346,7 +1354,7 @@ ray_t* exec_window(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { kind == RAY_WIN_MAX || kind == RAY_WIN_LAG || kind == RAY_WIN_LEAD || kind == RAY_WIN_FIRST_VALUE || kind == RAY_WIN_LAST_VALUE || kind == RAY_WIN_NTH_VALUE) { - out_f64 = fvec && fvec->type == RAY_F64; + out_f64 = fvec && (fvec->type == RAY_F64 || fvec->type == RAY_F32); } is_f64[f] = out_f64; diff --git a/test/rfl/query/query_select_fastpath_coverage.rfl b/test/rfl/query/query_select_fastpath_coverage.rfl index d635bc9ca..a281eaa11 100644 --- a/test/rfl/query/query_select_fastpath_coverage.rfl +++ b/test/rfl/query/query_select_fastpath_coverage.rfl @@ -316,6 +316,15 @@ (set Tint5 (table [v] (list [1 2 3 4 5]))) (at (at (select {f: (as 'F64 v) from: Tint5}) 'f) 0) -- 1.0 +;; Explicit F64 -> F32 projection cast uses the typed executor fallback. +;; It must round the expression result, not return an uninitialised vector. +(set Tf32proj (select {f: (as 'F32 (/ (+ v 0.3) 2.0)) from: Tint5})) +(type (at Tf32proj 'f)) -- 'F32 +(at (at Tf32proj 'f) 0) -- 0.65e +(at (at (select {f: (as 'F32 v) from: Tint5}) 'f) 4) -- 5.0e +(set Tf32i32 (table [v] (list (as 'I32 [7 8])))) +(at (at (select {f: (as 'F32 v) from: Tf32i32}) 'f) 1) -- 8.0e + ;; as cast I64→I32 in DAG (at (at (select {i: (as 'I32 v) from: Tint5}) 'i) 0) -- 1i diff --git a/test/rfl/query/window.rfl b/test/rfl/query/window.rfl index 1cf618ef4..ba65e8ffc 100644 --- a/test/rfl/query/window.rfl +++ b/test/rfl/query/window.rfl @@ -39,6 +39,32 @@ (at MW 'mx) -- [10 20 100 30 200 40] (at MW 'n) -- [1 2 1 3 2 3] +;; ── F32 inputs promote numeric window results to F64 ────────────────────── +;; F32 used to fall through the I64 readers and silently broadcast zeros. +(.sys.exec "printf 'k,o,v\\na,1,1.5\\na,2,2.5\\nb,1,10.0\\na,3,3.5\\nb,2,20.0\\n' > /tmp/rfl_window_f32.csv") +(set F (.csv.read [SYMBOL I64 F32] "/tmp/rfl_window_f32.csv")) +(set FW (window {from: F part: [k] funcs: {sm: (sum v) av: (avg v) mn: (min v) mx: (max v)}})) +(type (at FW 'sm)) -- 'F64 +(type (at FW 'av)) -- 'F64 +(type (at FW 'mn)) -- 'F64 +(type (at FW 'mx)) -- 'F64 +(at FW 'sm) -- [7.5 7.5 30.0 7.5 30.0] +(at FW 'av) -- [2.5 2.5 15.0 2.5 15.0] +(at FW 'mn) -- [1.5 1.5 10.0 1.5 10.0] +(at FW 'mx) -- [3.5 3.5 20.0 3.5 20.0] + +;; The same readers are used by running/trailing frames and value functions. +(set FT (window {from: F part: [k] order: [o] frame: 2 funcs: {sm: (sum v) av: (avg v) mn: (min v) mx: (max v)}})) +(at FT 'sm) -- [1.5 4.0 10.0 6.0 30.0] +(at FT 'av) -- [1.5 2.0 10.0 3.0 15.0] +(at FT 'mn) -- [1.5 1.5 10.0 2.5 10.0] +(at FT 'mx) -- [1.5 2.5 10.0 3.5 20.0] +(set FV (window {from: F part: [k] order: [o] funcs: {f: (first v) l: (last v)}})) +(type (at FV 'f)) -- 'F64 +(at FV 'f) -- [1.5 1.5 10.0 1.5 10.0] +(at FV 'l) -- [3.5 3.5 20.0 3.5 20.0] +(.sys.exec "rm -f /tmp/rfl_window_f32.csv") + ;; ── fby shape: rows equal to their group's min (keeps ties) ───────────────── ;; min over k: a→2 (only row v=2), b→4 (rows v=4,4 both) → 3 rows total (count (select {from: (window {from: T part: [k] funcs: {mn: (min v)}}) where: (== v mn)})) -- 3 diff --git a/test/rfl/sort/sort_coverage2.rfl b/test/rfl/sort/sort_coverage2.rfl index beca3bbf1..ffe0fb8b8 100644 --- a/test/rfl/sort/sort_coverage2.rfl +++ b/test/rfl/sort/sort_coverage2.rfl @@ -250,6 +250,18 @@ (at (at (xasc Txasc2 'k) 'k) 0) -- 1 (at (at (xdesc Txasc2 'k) 'k) 0) -- 20000000 +;; F32 single-key sort: query projections can explicitly narrow an F64 +;; expression to F32. Cover both the order-preserving F32 radix encoding and +;; stable equal-key order on a table large enough to take the radix path. +(set Tf32base (table [i v] (list (til 4097) (take [9.0 1.0 5.0 1.0 7.0] 4097)))) +(set Tf32sort (select {i: i v: (as 'F32 v) from: Tf32base})) +(type (at Tf32sort 'v)) -- 'F32 +(at (at (xasc Tf32sort 'v) 'v) 0) -- 1.0e +(at (at (xdesc Tf32sort 'v) 'v) 0) -- 9.0e +(take (at (xdesc Tf32sort 'v) 'i) 3) -- [0 5 10] +;; <=64 rows uses the stable comparison-sort path and its F32 comparator. +(take (at (xasc (take Tf32sort 5) 'v) 'i) 2) -- [1 3] + ;; ──────────────────────────────────────────────────────────────────── ;; 11. Multi-column composite sort with small n (< SMALL_POOL_THRESHOLD=8192) ;; but > RADIX_SORT_THRESHOLD=4096 — hits the else-branch From e5741a4b64f10fa7fd5e6d9ea733f57910211780 Mon Sep 17 00:00:00 2001 From: Anton Date: Fri, 10 Jul 2026 23:04:00 +0200 Subject: [PATCH 10/10] fix(lang): compile null keyword in lambdas --- src/lang/compile.c | 13 +++++++++++-- test/rfl/lang/null_keyword.rfl | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/lang/compile.c b/src/lang/compile.c index 909e87e11..7475c5053 100644 --- a/src/lang/compile.c +++ b/src/lang/compile.c @@ -192,7 +192,7 @@ static void patch_jump(compiler_t *c, int32_t pos) { } /* Cached sym IDs for special forms */ -static _Thread_local int64_t sf_set = -1, sf_let = -1, sf_if = -1, sf_do = -1, sf_fn = -1, sf_self = -1, sf_try = -1, sf_return = -1; +static _Thread_local int64_t sf_set = -1, sf_let = -1, sf_if = -1, sf_do = -1, sf_fn = -1, sf_self = -1, sf_try = -1, sf_return = -1, sf_null = -1; static _Thread_local int64_t sf_eval = -1, sf_resolve = -1; static void init_sf_syms(void) { @@ -205,6 +205,7 @@ static void init_sf_syms(void) { sf_self = ray_sym_intern("self", 4); sf_try = ray_sym_intern("try", 3); sf_return = ray_sym_intern("return", 6); + sf_null = ray_sym_intern("null", 4); sf_eval = ray_sym_intern("eval", 4); sf_resolve = ray_sym_intern("resolve", 7); } @@ -522,6 +523,14 @@ static void compile_expr(compiler_t *c, ray_t *ast) { if (ray_is_atom(ast)) { if (ast->type == -RAY_SYM && !(ast->attrs & ATTR_QUOTED)) { + /* The tree walker treats bare `null` as the singleton value, not + * an environment lookup. Compiled lambdas must do the same; + * otherwise any branch returning null fails with a name error. */ + init_sf_syms(); + if (ast->i64 == sf_null) { + emit_const(c, add_constant(c, RAY_NULL_OBJ)); + return; + } int32_t slot = find_local(c, ast->i64); if (slot >= 0) { emit(c, OP_LOADENV); @@ -639,6 +648,6 @@ ray_span_t ray_bc_dbg_get(ray_t* dbg, int32_t ip) { } void ray_compile_reset(void) { - sf_set = sf_let = sf_if = sf_do = sf_fn = sf_self = sf_try = sf_return = -1; + sf_set = sf_let = sf_if = sf_do = sf_fn = sf_self = sf_try = sf_return = sf_null = -1; sf_eval = sf_resolve = -1; } diff --git a/test/rfl/lang/null_keyword.rfl b/test/rfl/lang/null_keyword.rfl index 73ff8305f..10e03d5b1 100644 --- a/test/rfl/lang/null_keyword.rfl +++ b/test/rfl/lang/null_keyword.rfl @@ -13,6 +13,10 @@ ;; null is falsy in a condition → else branch, no crash (if null 1 2) -- 2 + +;; Compiled lambdas use the same null-keyword semantics as the tree walker. +(nil? ((fn [] null))) -- true +(nil? ((fn [x] (if x 1 null)) false)) -- true (if 1 10 20) -- 10 ;; producer parity: split-of-empty, ser/de round-trip, resolve-of-unbound