ci: pin the data-ingestors schema ref + add a coverage floor (#1009 P0.3)

.github/workflows/build.yml

@@ -23,11 +23,13 @@ jobs:
     timeout-minutes: 10
     name: Schema drift check
     # Verifies the embedded internal/schema/ingest.v1.json matches
-    # tracebloc/data-ingestors' master. A green PR that silently
-    # diverges from upstream is a real correctness hazard — a
-    # customer's YAML could pass `tracebloc ingest validate` locally
-    # but be rejected by jobs-manager (or vice versa). Forcing the
-    # sync as a PR step keeps drift visible.
+    # tracebloc/data-ingestors at the PINNED ref (scripts/.data-ingestors-ref),
+    # not a floating branch. A green PR that silently diverges from the schema
+    # jobs-manager enforces is a real correctness hazard — a customer's YAML
+    # could pass `tracebloc ingest validate` locally but be rejected in-cluster
+    # (or vice versa). Pinning stops an unrelated upstream commit from redding
+    # every open CLI PR; adopting upstream is a deliberate SHA bump + re-sync
+    # (backend#1009).
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -77,6 +79,14 @@ jobs:
         # us from having to retrofit it later.
         run: go test -race -cover ./...
 
+      - name: Coverage floor (internal/cli, internal/submit must not rot)
+        # `go test -cover` above prints numbers but asserts nothing. This
+        # enforces a per-package floor on the two load-bearing, historically
+        # thin-tested packages (the money path + submit orchestration) so a
+        # test deletion can't silently drop coverage. Floors ratchet UP only —
+        # see scripts/coverage-floor.sh (backend#1009).
+        run: ./scripts/coverage-floor.sh
+
   lint:
     timeout-minutes: 10
     name: Lint

scripts/.data-ingestors-ref

@@ -0,0 +1,12 @@
+# Pinned tracebloc/data-ingestors commit the CLI's embedded schema (and, when
+# the goldens drift job lands, the validator goldens) are synced from. CI
+# checks against THIS ref, not a floating branch — so an unrelated upstream
+# commit can't red every open CLI PR (backend#1009).
+#
+# To adopt upstream changes: bump this SHA, then run `scripts/sync-schema.sh`
+# (and `scripts/sync-validator-goldens.sh` once its CI job exists) and commit
+# the regenerated files together in one deliberate PR.
+#
+# Format: the first non-comment, non-blank line is the ref (a full commit SHA
+# preferred; a branch name works but reintroduces floating drift).
+0de1f148f9f19c8838d275ab9e5295ae224385c2

scripts/coverage-floor.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Fail if a load-bearing package's statement coverage drops below its floor.
+#
+# internal/cli (the money path — submit → classify → JSON → reclaim) and
+# internal/submit (the jobs-manager orchestration) were historically the
+# thinnest-tested, highest-stakes code in the CLI (backend#1009). A bare
+# `go test -cover` prints the number and asserts nothing, so coverage could
+# silently rot. This gate makes a regression loud.
+#
+# The floors are a RATCHET: set just under the current numbers, and bumped UP
+# as coverage improves — never silently down. Lowering a floor must be a
+# deliberate, reviewed edit here (with a reason), not a side effect of deleting
+# tests. Current (develop): internal/cli ~70%, internal/submit ~75%.
+#
+# Usage: scripts/coverage-floor.sh   (run from the repo root)
+#
+# Portable to bash 3.2 (macOS default): no associative arrays.
+set -euo pipefail
+
+# "package:floor" entries. Keep floors integers; coverage is compared as a
+# float against them.
+FLOORS="
+internal/cli:68
+internal/submit:72
+"
+
+status=0
+for entry in $FLOORS; do
+  pkg="${entry%%:*}"
+  min="${entry##*:}"
+  # A malformed entry (no ":floor", or a non-integer floor) must fail loudly,
+  # not slip through. Without this, a dropped colon leaves min="$entry" (the
+  # whole token); the awk comparison below then errors on that as bare source
+  # and exits non-zero — which `if awk` reads as "not below floor", prints a
+  # bogus "ok", and turns the ratchet into a silent no-op for that package.
+  if [ "$pkg" = "$entry" ] || ! printf '%s' "$min" | grep -qE '^[0-9]+$'; then
+    echo "::error::malformed FLOORS entry '$entry' (want 'package:INT') — fix scripts/coverage-floor.sh" >&2
+    status=1
+    continue
+  fi
+  line="$(go test -cover "./$pkg/" 2>/dev/null | grep -E 'coverage: [0-9]' || true)"
+  pct="$(printf '%s\n' "$line" | sed -nE 's/.*coverage: ([0-9]+(\.[0-9]+)?)% of statements.*/\1/p' | head -1)"
+  if [ -z "$pct" ]; then
+    echo "::error::could not read coverage for ./$pkg/ (did any test run?)" >&2
+    status=1
+    continue
+  fi
+  # awk exits 0 when pct < min (i.e. below the floor → failure).
+  if awk "BEGIN{exit !($pct < $min)}"; then
+    echo "::error::./$pkg/ coverage ${pct}% is below the floor ${min}% — add tests, or (with a reason) lower the floor in scripts/coverage-floor.sh" >&2
+    status=1
+  else
+    echo "ok: ./$pkg/ ${pct}% >= ${min}%"
+  fi
+done
+
+exit "$status"

scripts/sync-schema.sh

@@ -16,8 +16,15 @@
 #   scripts/sync-schema.sh --check        # verify in-tree copy matches upstream; exit non-zero on drift
 #
 # Env knobs:
-#   SCHEMA_SOURCE_URL  override the upstream URL (default: data-ingestors' master)
-#   SCHEMA_OUT         override the in-tree destination (default: internal/schema/ingest.v1.json)
+#   SCHEMA_SOURCE_URL   override the upstream URL (default: built from the
+#                       pinned ref below)
+#   DATA_INGESTORS_REF  override the data-ingestors ref (default: the pinned
+#                       SHA in scripts/.data-ingestors-ref, else master)
+#   SCHEMA_OUT          override the in-tree destination (default: internal/schema/ingest.v1.json)
+#
+# The ref is PINNED (scripts/.data-ingestors-ref), not a floating branch, so an
+# unrelated upstream commit doesn't red every open CLI PR — adopting upstream
+# is a deliberate SHA bump + re-sync (backend#1009).
 #
 # Future: when we cut a v2 schema, this script will need to learn
 # about multiple versions (e.g. embed v1 AND v2 side-by-side, picked
@@ -26,7 +33,28 @@
 
 set -euo pipefail
 
-readonly DEFAULT_URL="https://raw.githubusercontent.com/tracebloc/data-ingestors/master/tracebloc_ingestor/schema/ingest.v1.json"
+# The pinned data-ingestors ref: first non-comment, non-blank line of the ref
+# file (a full commit SHA), overridable via DATA_INGESTORS_REF, falling back to
+# master if the file is somehow absent.
+REF_FILE="$(cd "$(dirname "$0")" && pwd)/.data-ingestors-ref"
+readonly REF_FILE
+_pinned_ref="$(grep -vE '^[[:space:]]*(#|$)' "$REF_FILE" 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
+DATA_INGESTORS_REF="${DATA_INGESTORS_REF:-${_pinned_ref:-master}}"
+
+# The ref is interpolated into a download URL, so validate it before use
+# (like scripts/install.sh does for its release tag): a crafted ref — most
+# plausibly via the DATA_INGESTORS_REF override — could otherwise inject path
+# traversal ("../..") or extra segments into the raw.githubusercontent path.
+# Allow only a SHA / branch / tag shape: alnum start, then alnum . _ - / and
+# no ".." component.
+if ! printf '%s' "$DATA_INGESTORS_REF" | grep -qE '^[A-Za-z0-9][A-Za-z0-9._/-]*$' \
+   || printf '%s' "$DATA_INGESTORS_REF" | grep -q '\.\.'; then
+  echo "error: invalid data-ingestors ref '$DATA_INGESTORS_REF' — expected a commit SHA, branch, or tag" >&2
+  echo "(set it in scripts/.data-ingestors-ref or via DATA_INGESTORS_REF)" >&2
+  exit 2
+fi
+
+readonly DEFAULT_URL="https://raw.githubusercontent.com/tracebloc/data-ingestors/${DATA_INGESTORS_REF}/tracebloc_ingestor/schema/ingest.v1.json"
 readonly DEFAULT_OUT="internal/schema/ingest.v1.json"
 
 SCHEMA_SOURCE_URL="${SCHEMA_SOURCE_URL:-$DEFAULT_URL}"