From 80c81fbfc52612092774ae10e0436f64847f2dfe Mon Sep 17 00:00:00 2001 From: John Yang Date: Mon, 22 Jun 2026 16:44:40 -0400 Subject: [PATCH] package: report mean_score over the full benchmark (200), not attempted Matches the leaderboard, where unattempted tasks count as 0; keeps the console summary from overstating a partial submission's mean. --- src/programbench/submission.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/programbench/submission.py b/src/programbench/submission.py index 396a9ec..7534682 100644 --- a/src/programbench/submission.py +++ b/src/programbench/submission.py @@ -183,10 +183,10 @@ def aggregate(scores: dict[str, float], n_total: int) -> Headline: if not values: raise ValueError("No scored instances found") n = len(values) - # mean is over attempted instances; resolved/near are over the full benchmark - # (an unattempted task counts as unresolved). + # mean, resolved, and near are all over the full benchmark — an unattempted task counts + # as 0, matching how the leaderboard scores partial submissions. return Headline( - mean_score=round(sum(values) / n, 4), + mean_score=round(sum(values) / n_total, 4), resolved_pct=round(100 * sum(s >= RESOLVED_THRESHOLD for s in values) / n_total, 1), near_resolved_pct=round(100 * sum(s >= NEAR_RESOLVED_THRESHOLD for s in values) / n_total, 1), n_instances_attempted=n,