Merge pull request #567 from gittensor-ai-lab/chore/tighten-h2-longctx-bars

chore(eval): tighten H2 long-context bars (top1>=0.90, KL<=0.5)

Author: skyrocket2026

bench/scripts/accuracy.sh

@@ -61,23 +61,20 @@ echo ">> eval prompt: seed=$SEED tokens=$(echo "$IDS" | wc -w)"
 # diverges from llama more than bf16 does. They come from a 27-seed sweep on RTX 5090 (Qwen3.6, 8k,
 # tail-128), across all 3-seed combinations of 16 correct (#517-fixed) and 11 broken (#517-live)
 # builds:
-#     metric   correct 3-seed range   broken 3-seed range   bar     false-reject / false-accept
-#     top-1     [0.870, 0.974]         [0.568, 0.786]        >=0.85       0/560   /   0/165
-#     KL        [0.215, 1.367]         [1.444, 2.461]        <=1.0        0/560   /   0/165
+#     metric   correct 3-seed range   broken 3-seed range   (historical cal bars)
+#     top-1     [0.870, 0.974]         [0.568, 0.786]        was >=0.85
+#     KL        [0.215, 1.367]         [1.444, 2.461]        was <=1.0
 # Single-seed the two KL distributions OVERLAP (correct up to 1.66, broken down to 1.25) and no KL
-# bar works — averaging LONG_SEEDS is what opens the gap. The KL bar (1.0) is far looser than the
-# short pass's (0.20) precisely to allow int8's honest quantization loss: reading that loss as
-# "corruption" and tightening the bar is what turned this probe off in #227 and created the blind
-# spot. In this data top-1 alone already separates cleanly (broken tops out at 0.786), so the KL
-# veto is defense-in-depth against a future bug that spares top-1 but wrecks KL. Reject on EITHER.
-# Margins are ~0.08 (top-1) and ~0.13 (KL) from this corpus/model; re-run the sweep before trusting
-# them on a new model. All bars, the seed count, window, and token length are env-overridable.
+# bar works — averaging LONG_SEEDS is what opens the gap. Bars below are tighter than that
+# historical calibration (top1>=0.90, KL<=0.50) to demand closer int8↔llama agreement; override via
+# env if a model/corpus needs looser margins. Reject on EITHER. All bars, the seed count, window,
+# and token length are env-overridable.
 LONGCTX="${SPARKINFER_EVAL_LONGCTX:-1}"
 LONG_N="${SPARKINFER_EVAL_LONGCTX_TOKENS:-8448}"
 LONG_TAIL="${SPARKINFER_EVAL_LONGCTX_TAIL:-128}"
 LONG_SEEDS="${SPARKINFER_EVAL_LONGCTX_SEEDS:-3}"
-LONG_TOP1_BAR="${SPARKINFER_EVAL_LONGCTX_TOP1_BAR:-0.85}"
-LONG_KL_BAR="${SPARKINFER_EVAL_LONGCTX_KL_BAR:-1.0}"
+LONG_TOP1_BAR="${SPARKINFER_EVAL_LONGCTX_TOP1_BAR:-0.90}"
+LONG_KL_BAR="${SPARKINFER_EVAL_LONGCTX_KL_BAR:-0.5}"
 if [ "$LONGCTX" = "1" ]; then
   for j in $(seq 0 $((LONG_SEEDS - 1))); do
     python3 "$HERE/gen_eval_prompt.py" "${SEED}:L${j}" "$MODELS_DIR/tokenizer.json" "$HERE/eval_corpus.txt" \
@@ -139,8 +136,8 @@ fi
 echo
 LONG_TOP1_BAR="$LONG_TOP1_BAR" LONG_KL_BAR="$LONG_KL_BAR" python3 - /tmp/acc_short.txt /tmp/acc_long.txt <<'PY'
 import re, sys, os
-t1_bar = float(os.environ.get("LONG_TOP1_BAR", "0.85"))
-kl_bar = float(os.environ.get("LONG_KL_BAR", "1.0"))
+t1_bar = float(os.environ.get("LONG_TOP1_BAR", "0.90"))
+kl_bar = float(os.environ.get("LONG_KL_BAR", "0.5"))
 def grab_short(path):
     for line in open(path):
         m = re.match(r'^METRIC_SHORT top1=([\d.]+) kl=([\d.]+)', line)