diff --git a/.claude/skills/create-ladder/SKILL.md b/.claude/skills/create-ladder/SKILL.md
index f8819ab8..1e92ec4e 100644
--- a/.claude/skills/create-ladder/SKILL.md
+++ b/.claude/skills/create-ladder/SKILL.md
@@ -104,9 +104,14 @@ Local validation is necessary but NOT sufficient — a local shim skips Docker,
`GITHUB_TOKEN=$(gh auth token) uv run codeclash ladder make configs/ablations/ladder/make_.yaml --workers N`
→ logs land under `logs/ladder//`. Fast arenas run on a laptop; big ones (e.g. SCML's
~1275 pairs) want an AWS box under `tmux`/`nohup`.
-4. Rank: `python -m codeclash.analysis.metrics.elo -d logs/ladder/ --output-dir assets/_elo`
+4. Rank: `uv run python -m codeclash.analysis.metrics.elo -d logs/ladder/ --include-round-0 --output-dir assets/_elo`
→ prints the Bradley-Terry/Elo order (weakest → strongest); that ordering IS the ladder.
- Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
+ - **`--include-round-0` is REQUIRED for ladder construction.** `ladder make` uses
+ `tournament.rounds: 0`, so round 0 IS the match; without the flag every tournament is
+ dropped (round 0 is normally the excluded identical-codebases baseline) and the fit
+ crashes on an empty matrix. Do NOT pass it for normal multi-round PvP/climbing Elo.
+ - Use `uv run python`, not bare `python` — the analysis deps (matplotlib) live in the uv venv.
+ - Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
## Phase 5 — Assemble the ranked configs
diff --git a/codeclash/analysis/metrics/elo.py b/codeclash/analysis/metrics/elo.py
index 929d3154..5d7a9ef6 100644
--- a/codeclash/analysis/metrics/elo.py
+++ b/codeclash/analysis/metrics/elo.py
@@ -40,6 +40,7 @@ def __init__(
score_type: SCORING_TYPES = "per_round_tertiary",
max_round: int = 15,
only_specific_round: bool = False,
+ include_round_0: bool = False,
):
"""This class builds a win matrix from a log directory, it doesn't fit anything yet.
It also adds a "ALL" game to the win matrix, which is the sum of all games.
@@ -62,6 +63,9 @@ def __init__(
The `max_round` parameter controls the maximum number of rounds to include in the score calculation (default: 15).
The `only_specific_round` parameter controls whether to only include the specific round (True) or all rounds up to max_round (False).
+ The `include_round_0` parameter controls whether round 0 is counted. In normal PvP/climbing
+ tournaments round 0 is the identical-codebases baseline and is excluded. For ladder
+ construction (`ladder make`, `tournament.rounds: 0`) round 0 IS the match, so set this True.
"""
self.win_matrix: dict[str, dict[tuple[str, str], list[float]]] = defaultdict(
lambda: defaultdict(lambda: [0.0, 0.0])
@@ -71,6 +75,7 @@ def __init__(
self.score_type = score_type
self.max_round = max_round
self.only_specific_round = only_specific_round
+ self.include_round_0 = include_round_0
self._samples: dict[str, dict[tuple[str, str], list[tuple[float, float]]]] = defaultdict(
lambda: defaultdict(list)
)
@@ -154,13 +159,19 @@ def _process_tournament(self, metadata_path: Path) -> None:
return
player_names = [p["name"] for p in players]
- models = [p["config"]["model"]["model_name"].strip("@") for p in players]
+ models = []
+ for p in players:
+ try:
+ models.append(p["config"]["model"]["model_name"].strip("@"))
+ except KeyError:
+ # Ladder bots have no model config; identify by branch (flatten "/" to keep years distinct).
+ models.append(p["name"].removeprefix("human/").replace("/", "__"))
# Aggregate scores for each round
p1_round_scores = []
p2_round_scores = []
for idx, stats in metadata["round_stats"].items():
- if idx == "0":
+ if idx == "0" and not self.include_round_0:
continue
round_num = int(idx)
@@ -1547,6 +1558,12 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
parser = argparse.ArgumentParser(description="Build win matrix and fit Bradley-Terry model")
parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR)
parser.add_argument("--print-matrix", action="store_true", help="Print win matrix")
+ parser.add_argument(
+ "--include-round-0",
+ action="store_true",
+ help="Count round 0 (normally the excluded identical-codebases baseline). REQUIRED for "
+ "ladder construction (`ladder make` uses tournament.rounds: 0, so round 0 IS the match).",
+ )
parser.add_argument(
"-s",
"--score-type",
@@ -1578,7 +1595,9 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
args = parser.parse_args()
builder = ScoreMatrixBuilder(
- all_games_normalization_scheme=args.all_normalization_scheme, score_type=args.score_type
+ all_games_normalization_scheme=args.all_normalization_scheme,
+ score_type=args.score_type,
+ include_round_0=args.include_round_0,
)
builder.build(args.log_dir)
@@ -1632,22 +1651,26 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
).run()
write_bootstrap_metrics_table(bootstrap_results, args.output_dir, game="ALL")
- logger.info("Running EloVsMaxRounds analysis")
- EloVsMaxRounds(
- log_dir=args.log_dir,
- max_rounds=15,
- all_games_normalization_scheme=args.all_normalization_scheme,
- score_type=args.score_type,
- regularization=args.regularization,
- output_dir=args.output_dir,
- ).run()
-
- logger.info("Running EloOnlyAtRound analysis")
- EloOnlyAtRound(
- log_dir=args.log_dir,
- max_rounds=15,
- all_games_normalization_scheme=args.all_normalization_scheme,
- score_type=args.score_type,
- regularization=args.regularization,
- output_dir=args.output_dir,
- ).run()
+ # Max-round analyses are multi-round-only; skip them for single-round ladder round-robins.
+ if not args.include_round_0:
+ logger.info("Running EloVsMaxRounds analysis")
+ EloVsMaxRounds(
+ log_dir=args.log_dir,
+ max_rounds=15,
+ all_games_normalization_scheme=args.all_normalization_scheme,
+ score_type=args.score_type,
+ regularization=args.regularization,
+ output_dir=args.output_dir,
+ ).run()
+
+ logger.info("Running EloOnlyAtRound analysis")
+ EloOnlyAtRound(
+ log_dir=args.log_dir,
+ max_rounds=15,
+ all_games_normalization_scheme=args.all_normalization_scheme,
+ score_type=args.score_type,
+ regularization=args.regularization,
+ output_dir=args.output_dir,
+ ).run()
+ else:
+ logger.info("Skipping EloVsMaxRounds / EloOnlyAtRound (ladder mode: single round-0 round-robin)")
diff --git a/codeclash/arenas/scml/SCML.Dockerfile b/codeclash/arenas/scml/SCML.Dockerfile
index 94c14350..b7edb8fa 100644
--- a/codeclash/arenas/scml/SCML.Dockerfile
+++ b/codeclash/arenas/scml/SCML.Dockerfile
@@ -4,6 +4,13 @@ ENV DEBIAN_FRONTEND=noninteractive \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1
+# Pin numpy/BLAS to a single math thread per container. Each SCML simulation is
+# single-threaded compute (measured: pinned solo == unpinned solo)
+ENV OMP_NUM_THREADS=1 \
+ OPENBLAS_NUM_THREADS=1 \
+ MKL_NUM_THREADS=1 \
+ NUMEXPR_NUM_THREADS=1
+
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates git build-essential jq \
@@ -12,12 +19,10 @@ RUN apt-get update \
RUN python -m pip install --upgrade pip \
&& python -m pip install scml==0.8.2
+# Clone the arena repo so `origin` is set for branch_init / push (matches the other
+# arenas). Default branch holds the runtime; human/* branches overlay scml_agent.py.
+RUN git clone https://github.com/CodeClash-ai/SCML.git /workspace \
+ && cd /workspace \
+ && git config user.email "player@codeclash.com" \
+ && git config user.name "Player"
WORKDIR /workspace
-
-COPY codeclash/arenas/scml/runtime/ /workspace/
-
-RUN git init \
- && git config user.email "player@codeclash.com" \
- && git config user.name "Player" \
- && git add . \
- && git commit -m "Initial SCML workspace"
diff --git a/configs/ladder/make_scml.yaml b/configs/ladder/make_scml.yaml
new file mode 100644
index 00000000..250d7a9e
--- /dev/null
+++ b/configs/ladder/make_scml.yaml
@@ -0,0 +1,129 @@
+# Round-robin over the SCML OneShot human bots (ported from yasserfarouk/scml-agents +
+# built-in baselines), used to RANK them via Elo/Bradley-Terry -> see scml.yaml for the ladder.
+# rounds:0 = dummy players just play the baseline SCML2024OneShot game (no code edits).
+# 51 bots -> 51*50/2 = 1275 pairwise tournaments. Bots live on CodeClash-ai/SCML; Docker required.
+#
+# COST (measured ~4.8s/sim + ~3s overhead per pair; pairs are single-threaded, so wall =
+# core-hours / workers): at sims_per_round=200 a pair takes ~16 min -> ~341 core-hours total.
+# * 32-core box (--workers 30): ~11 h * 64 vCPU (-w 62): ~5.5 h * 96 vCPU (-w 90): ~3.8 h
+# Keep --workers <= (cores - 2): the decide() call has a 3s timeout that CPU oversubscription
+# can trip. Resumable: reruns skip pairs already logged, so it's safe to stop/resume.
+#
+# Full run (200 sims = publication-quality Elo, sized for SCML's high per-sim variance):
+# uv run codeclash ladder make configs/ladder/make_scml.yaml --workers 30
+# Cheap SANITY pilot first (~1 h @ 32 cores): temporarily set sims_per_round: 30, run, rank, and
+# eyeball the ordering (baselines near the bottom) before committing to the full 200-sim pass.
+tournament:
+ rounds: 0
+game:
+ name: SCML
+ sims_per_round: 200
+ timeout: 10800 # 3h per-pair game cap; 200 sims needs ~1-1.5h (default 180s kills the run)
+ args:
+ n_steps: 10
+ n_lines: 2
+players:
+- agent: dummy
+ branch_init: human/scml-baselines/greedy
+- agent: dummy
+ branch_init: human/scml-baselines/nice
+- agent: dummy
+ branch_init: human/scml-baselines/random
+- agent: dummy
+ branch_init: human/scml2021/staghunter
+- agent: dummy
+ branch_init: human/scml2021/team_50
+- agent: dummy
+ branch_init: human/scml2021/team_51
+- agent: dummy
+ branch_init: human/scml2021/team_54
+- agent: dummy
+ branch_init: human/scml2021/team_55
+- agent: dummy
+ branch_init: human/scml2021/team_61
+- agent: dummy
+ branch_init: human/scml2021/team_62
+- agent: dummy
+ branch_init: human/scml2021/team_72
+- agent: dummy
+ branch_init: human/scml2021/team_73
+- agent: dummy
+ branch_init: human/scml2021/team_86
+- agent: dummy
+ branch_init: human/scml2021/team_90
+- agent: dummy
+ branch_init: human/scml2021/team_corleone
+- agent: dummy
+ branch_init: human/scml2022/team_102
+- agent: dummy
+ branch_init: human/scml2022/team_103
+- agent: dummy
+ branch_init: human/scml2022/team_105
+- agent: dummy
+ branch_init: human/scml2022/team_106
+- agent: dummy
+ branch_init: human/scml2022/team_107
+- agent: dummy
+ branch_init: human/scml2022/team_123
+- agent: dummy
+ branch_init: human/scml2022/team_124
+- agent: dummy
+ branch_init: human/scml2022/team_126
+- agent: dummy
+ branch_init: human/scml2022/team_131
+- agent: dummy
+ branch_init: human/scml2022/team_134
+- agent: dummy
+ branch_init: human/scml2022/team_94
+- agent: dummy
+ branch_init: human/scml2022/team_96
+- agent: dummy
+ branch_init: human/scml2023/team_102
+- agent: dummy
+ branch_init: human/scml2023/team_123
+- agent: dummy
+ branch_init: human/scml2023/team_126
+- agent: dummy
+ branch_init: human/scml2023/team_127
+- agent: dummy
+ branch_init: human/scml2023/team_134
+- agent: dummy
+ branch_init: human/scml2023/team_143
+- agent: dummy
+ branch_init: human/scml2023/team_144
+- agent: dummy
+ branch_init: human/scml2023/team_145
+- agent: dummy
+ branch_init: human/scml2023/team_148
+- agent: dummy
+ branch_init: human/scml2023/team_149
+- agent: dummy
+ branch_init: human/scml2023/team_151
+- agent: dummy
+ branch_init: human/scml2023/team_poli_usp
+- agent: dummy
+ branch_init: human/scml2024/coyoteteam
+- agent: dummy
+ branch_init: human/scml2024/ozug4
+- agent: dummy
+ branch_init: human/scml2024/team_144
+- agent: dummy
+ branch_init: human/scml2024/team_164
+- agent: dummy
+ branch_init: human/scml2024/team_171
+- agent: dummy
+ branch_init: human/scml2024/team_172
+- agent: dummy
+ branch_init: human/scml2024/team_174
+- agent: dummy
+ branch_init: human/scml2024/team_193
+- agent: dummy
+ branch_init: human/scml2024/team_238
+- agent: dummy
+ branch_init: human/scml2024/team_abc
+- agent: dummy
+ branch_init: human/scml2024/team_miyajima
+- agent: dummy
+ branch_init: human/scml2024/teamyuzuru
+prompts:
+ game_description: SCML OneShot ladder