{ "meta": { "date_utc": "2026-09-24T07:42:28Z", "git_sha": "8028a10", "host": "m3-max-metal", "profile": "release", "laya_feature": true, "laya_max_questions": 0, "corpus_cap_mode": "registry defaults", "corpus_protocol": "one engine domain per label; corpus = train-split docs of that label, capped per label (registry), self-doc fallback for labels absent from the fetched train rows", "calibration_protocol": "sigmoid-gate fit on a train-tail calibration slice (same builder over the train rows); fused-gate thresholds fitted per suite at the cal-slice 30th percentile (the T1.6 arena posture rho=30%; the birth constants do not transfer \u2014 measured: 100% abstain on several suites at defaults); raw-vs-calibrated readout ECE compared per the G1 gate; no calibration claim when the calibrator never moved", "floor_definition": "conformal-naive floor = split-conformal recalibration of the raw readout confidence, c'(s) = (1 + #{cal s_i <= s}) / (n_cal + 1) \u2014 the exchangeability-valid baseline (G1 fails unless the calibrated ECE beats it)", "determinism_scoping": "the bit-identity claim is the MODELLESS lane's (plan caveat 3); the laya lane gets the same observed repeat check and it is reported, not claimed", "laya_device": "metal (LAYA_DEVICE or the build's macOS default)", "laya_python_lane": "on \u2014 the ORIGINAL torch reference as a JSONL subprocess oracle: same cases, the reference's own rounded-4 probabilities, latency = subprocess round-trip (IPC included)", "box_state": { "start": { "power": "AC Power", "power_mode": "high", "load_1m": 6.31, "swap_used_mb": 2563.94, "latency_quotable": false, "refusals": [ "load 6.31 > 6 \u2014 a sibling job is on the box" ] }, "end": { "power": "AC Power", "power_mode": "high", "load_1m": 6.97, "swap_used_mb": 2563.94, "latency_quotable": false, "refusals": [ "load 6.97 > 6 \u2014 a sibling job is on the box" ] } }, "divergences": [ "massive option sampling: SplitMix64 (fixed seed), not CPython MT19937 \u2014 both lanes see byte-identical questions, which is the integrity that matters", "banking77: mteb/banking77 mirror (the reference's own bench_apps variant); PolyAI/banking77 is script-based and unservable", "engine context = state + prompt (wire criteria None) \u2014 the modelless serving path" ], "hosts": [ { "host": "m3-max-metal", "date_utc": "2026-09-24T07:42:28Z", "git_sha": "8028a10", "profile": "release", "laya_feature": true, "laya_device": "metal (LAYA_DEVICE or the build's macOS default)", "laya_python_lane": "on \u2014 the ORIGINAL torch reference as a JSONL subprocess oracle: same cases, the reference's own rounded-4 probabilities, latency = subprocess round-trip (IPC included)", "laya_max_questions": 0, "corpus_cap_mode": "registry defaults", "lane_sources": { "modelless": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" }, "laya:english": { "git_sha": "8ca8770", "date_utc": "2026-10-02T12:01:18Z" }, "laya:multilingual": { "git_sha": "5ada17a", "date_utc": "2026-09-27T08:00:55Z" }, "laya:py/english": { "git_sha": "8ca8770", "date_utc": "2026-10-02T12:01:18Z" }, "laya:py/multilingual": { "git_sha": "5ada17a", "date_utc": "2026-09-27T08:00:55Z" }, "laya:py/typed": { "git_sha": "5ada17a", "date_utc": "2026-09-27T08:00:55Z" }, "laya:typed": { "git_sha": "5ada17a", "date_utc": "2026-09-27T08:00:55Z" }, "hybrid": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" }, "paw": { "git_sha": "e45c828", "date_utc": "2026-10-03T21:54:51Z" }, "openthai": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" }, "encoder": { "git_sha": "e493c56", "date_utc": "2026-10-07T01:37:41Z" }, "bekko": { "git_sha": "e8328de", "date_utc": "2026-10-03T16:34:28Z" }, "paw_local": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:14:24Z" }, "clef": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:33:58Z" }, "pplx": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" }, "drex": { "git_sha": "0a3c79e", "date_utc": "2026-10-08T01:38:13Z" } }, "head_posture": "OFF (head_scale 0 \u2014 the published baseline posture)" }, { "host": "m3-max-ane", "date_utc": "2026-09-24T07:16:02Z", "git_sha": "afacc3a", "profile": "release", "laya_feature": true, "laya_device": "ane (LAYA_DEVICE=ane \u2192 load_ane; whole-graph CoreML encoder, Plan 002)", "laya_python_lane": "off (pass --laya-python to add the reference lane)", "laya_max_questions": 0, "corpus_cap_mode": "registry defaults", "lane_sources": { "laya:english": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } }, { "host": "4090-win", "date_utc": "2026-09-24T07:11:03Z", "git_sha": "afacc3a", "profile": "release", "laya_feature": true, "laya_device": "cuda (LAYA_DEVICE or the build's non-macOS CUDA default)", "laya_python_lane": "off (pass --laya-python to add the reference lane)", "laya_max_questions": 0, "corpus_cap_mode": "registry defaults", "lane_sources": { "modelless": { "git_sha": "741bda3", "date_utc": "2026-10-09T00:13:55Z" }, "laya:english": { "git_sha": "3cad008", "date_utc": "2026-10-02T20:23:08Z" }, "laya:multilingual": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" }, "laya:typed": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" }, "clm": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" }, "gliner": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" }, "agentjev": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" }, "paw_local": { "git_sha": "634093f", "date_utc": "2026-09-26T20:05:28Z" }, "openthai": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" }, "drex": { "git_sha": "7162840", "date_utc": "2026-10-08T02:36:48Z" } }, "head_posture": "OFF (head_scale 0 \u2014 the published baseline posture)" } ], "head_posture": "OFF (head_scale 0 \u2014 the published baseline posture)", "edition": "2026-10-11" }, "suites": [ { "name": "typed_decisions", "n_cases": 400, "n_questions": 2000, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 2000, "accuracy": 0.57, "macro_f1": 0.539329054680424, "ece": 0.21543478833883997, "brier": 0.6434320700139581, "nll": 1.146723112360243, "aurc": 0.30591289718569137, "mean_confidence": 0.35456521166116, "acc_at_50_coverage": 0.668, "acc_at_80_coverage": 0.58 }, "readout_ece": 0.09456632691621782, "readout_brier": 0.24279730191318755, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.632, "selective_accuracy": 0.5747282608695652, "selective_n": 736 }, "abstain_causes": { "score_gate": 947, "distance_gate": 317, "grammar_invalid": 0 }, "readout_ece_raw": 0.5644833512306213, "readout_ece_calibrated": 0.09456632691621782, "floor_ece": 0.21644411177644715, "g1_pass": true, "g1_verdict": "pass", "by_question_type": { "choice": { "n": 600, "accuracy": 0.5483333333333333, "macro_f1": 0.4427516704324253, "ece": 0.2756351305296024, "brier": 0.7261415492313711, "nll": 1.3760498373068872, "aurc": 0.43141318071378293, "mean_confidence": 0.27269820280373097, "acc_at_50_coverage": 0.4766666666666667, "acc_at_80_coverage": 0.5041666666666667 }, "noul": { "n": 600, "accuracy": 0.7433333333333333, "macro_f1": 0.7371535205498156, "ece": 0.20128869409362474, "brier": 0.45091075422633403, "nll": 0.6436496199143352, "aurc": 0.16217189041911362, "mean_confidence": 0.5420446392397086, "acc_at_50_coverage": 0.8266666666666667, "acc_at_80_coverage": 0.775 }, "score": { "n": 800, "accuracy": 0.45625, "macro_f1": 0.36099755761350183, "ece": 0.18089410237967968, "brier": 0.7257909474416057, "nll": 1.3520331879846887, "aurc": 0.46530446693557165, "mean_confidence": 0.2753558976203203, "acc_at_50_coverage": 0.5225, "acc_at_80_coverage": 0.4734375 } }, "soft_acc": 0.33301113778941527, "brier_soft": 0.2106513235327601, "score_mae": 0.6697793511115953, "within_1": 0.76, "latency_p50_ms": 0.762, "latency_p99_ms": 1.623, "latency_tail_support": 5, "latency_extremes": { "first_ms": 0.204, "max_ms": 6.173, "argmax_case": 200 }, "determinism_ok": true, "determinism_n": 10, "seconds": 6.5459362500000005, "n_cases": 400, "n_questions": 2000, "score_threshold": 0.56503725, "distance_threshold": 0.91222644, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.56503725, "status": "fitted", "targetAccuracy": 0.4, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } }, "distance": { "threshold": 0.91222644, "status": "fitted", "targetAccuracy": 0.47142857, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } } }, "corpus_cap": { "effective": 48, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 } ] }, "oc_selection": { "selected_scale": 0.25, "candidates": [ { "scale": 0.0, "cal_acc": 0.318 }, { "scale": 0.25, "cal_acc": 0.54 }, { "scale": 0.5, "cal_acc": 0.532 }, { "scale": 1.0, "cal_acc": 0.536 }, { "scale": 2.0, "cal_acc": 0.536 }, { "scale": 4.0, "cal_acc": 0.538 }, { "scale": 8.0, "cal_acc": 0.536 } ] }, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.54 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "Elevated; should be handled within the same week.", "pred": "Critical; requires action within the same day.", "count": 55, "share_of_errors": 0.07790368271954674 }, { "gold": "Elevated; should be handled within the same week.", "pred": "Routine; handle within the normal queue.", "count": 52, "share_of_errors": 0.07365439093484419 }, { "gold": "success", "pred": "partial", "count": 50, "share_of_errors": 0.0708215297450425 }, { "gold": "No time pressure; can wait indefinitely.", "pred": "Routine; handle within the normal queue.", "count": 36, "share_of_errors": 0.05099150141643059 }, { "gold": "Material: a large or unexplained difference.", "pred": "None: everything reconciles.", "count": 35, "share_of_errors": 0.049575070821529746 }, { "gold": "Benign: read-only or clearly safe actions.", "pred": "Low: routine writes within scope.", "count": 33, "share_of_errors": 0.046742209631728045 }, { "gold": "Moderate: access to internal systems or non-public data.", "pred": "High: access to production, secrets or customer data.", "count": 30, "share_of_errors": 0.042492917847025496 }, { "gold": "Critical; requires action within the same day.", "pred": "Routine; handle within the normal queue.", "count": 28, "share_of_errors": 0.039660056657223795 }, { "gold": "Imminent: threatening to cancel, dispute or leave.", "pred": "Clearly unhappy; repeat problems or explicit complaints.", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "Mild frustration, but the relationship is intact.", "pred": "Clearly unhappy; repeat problems or explicit complaints.", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "request_information", "pred": "escalate_to_human", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "Routine; handle within the normal queue.", "pred": "Critical; requires action within the same day.", "count": 22, "share_of_errors": 0.031161473087818695 } ], "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.39222614840989395, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.5529305105209351 }, { "mode": "max_prob", "cal_ece": 0.20307282534241677 }, { "mode": "inv_entropy", "cal_ece": 0.5529305105209351 } ] }, "corpus_digest": "fnv1a64-2fe9ec3132dbc924", "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 2000, "accuracy": 0.3575, "macro_f1": 0.32620639534238516, "ece": 0.2112706999999999, "brier": 0.7684491694499974, "nll": 1.3593418376936985, "aurc": 0.5626661372266888, "mean_confidence": 0.5687707000000009, "acc_at_50_coverage": 0.429, "acc_at_80_coverage": 0.38125 }, "readout_ece": 0.19182040000000003, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.28833333333333333, "macro_f1": 0.2958479995436556, "ece": 0.16805866666666666, "brier": 0.8079409903499993, "nll": 1.5603174844492513, "aurc": 0.6171637396940312, "mean_confidence": 0.4527049999999996, "acc_at_50_coverage": 0.38666666666666666, "acc_at_80_coverage": 0.325 }, "noul": { "n": 600, "accuracy": 0.47333333333333333, "macro_f1": 0.46994095544820186, "ece": 0.2808480000000001, "brier": 0.6693728044000007, "nll": 0.9693957725723233, "aurc": 0.4926269983870734, "mean_confidence": 0.7541813333333328, "acc_at_50_coverage": 0.4866666666666667, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.3225, "macro_f1": 0.24129818605373515, "ece": 0.2064690000000001, "brier": 0.8131375775625017, "nll": 1.5010696514680573, "aurc": 0.6560603649790228, "mean_confidence": 0.5167620000000003, "acc_at_50_coverage": 0.335, "acc_at_80_coverage": 0.3125 } }, "soft_acc": 0.33458230168279346, "brier_soft": 0.3348088366904328, "score_mae": 0.6937235449999997, "within_1": 0.7525, "latency_p50_ms": 195.0, "latency_p99_ms": 358.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 92.0, "max_ms": 401.0, "argmax_case": 120 }, "determinism_ok": true, "seconds": 81.877849, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true }, "multilingual": { "lane": "laya (rust)", "model": "multilingual", "hard": { "n": 2000, "accuracy": 0.349, "macro_f1": 0.3498994829046305, "ece": 0.32280229999999993, "brier": 0.9056679379849982, "nll": 1.9436809234449117, "aurc": 0.554820549344135, "mean_confidence": 0.6687793000000012, "acc_at_50_coverage": 0.404, "acc_at_80_coverage": 0.36375 }, "readout_ece": 0.25666635, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.295, "macro_f1": 0.3242803546952691, "ece": 0.362346, "brier": 0.9884960357500008, "nll": 2.5096392870563395, "aurc": 0.5968789601913673, "mean_confidence": 0.6559423333333334, "acc_at_50_coverage": 0.31666666666666665, "acc_at_80_coverage": 0.28541666666666665 }, "noul": { "n": 600, "accuracy": 0.4866666666666667, "macro_f1": 0.4534096824570536, "ece": 0.40840199999999977, "brier": 0.874118264733335, "nll": 1.6946655329027263, "aurc": 0.4613874178933604, "mean_confidence": 0.8919399999999996, "acc_at_50_coverage": 0.45, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.28625, "macro_f1": 0.2554689567204614, "ece": 0.23129124999999998, "brier": 0.8672091196000032, "nll": 1.7059736936429828, "aurc": 0.7179156295538103, "mean_confidence": 0.5110365000000006, "acc_at_50_coverage": 0.26, "acc_at_80_coverage": 0.2765625 } }, "soft_acc": 0.3271328198665314, "brier_soft": 0.4737295325604735, "score_mae": 0.7603941325000002, "within_1": 0.7, "latency_p50_ms": 92.0, "latency_p99_ms": 163.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 40.0, "max_ms": 174.0, "argmax_case": 151 }, "determinism_ok": true, "seconds": 38.118442625, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true }, "typed": { "lane": "laya (rust)", "model": "typed", "hard": { "n": 2000, "accuracy": 0.7445, "macro_f1": 0.7349433525103708, "ece": 0.1923918, "brier": 0.40988110527999805, "nll": 0.7184640377885018, "aurc": 0.12634017215900445, "mean_confidence": 0.5521081999999994, "acc_at_50_coverage": 0.859, "acc_at_80_coverage": 0.793125 }, "readout_ece": 0.3952849499999999, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.7333333333333333, "macro_f1": 0.6943336435123875, "ece": 0.25484566666666664, "brier": 0.46509025648333213, "nll": 0.8649966941522675, "aurc": 0.11299371741797679, "mean_confidence": 0.47848766666666676, "acc_at_50_coverage": 0.8966666666666666, "acc_at_80_coverage": 0.8 }, "noul": { "n": 600, "accuracy": 0.785, "macro_f1": 0.7833092099185031, "ece": 0.12942600000000015, "brier": 0.3215147912666666, "nll": 0.4946194012650314, "aurc": 0.08280543014765192, "mean_confidence": 0.6624203333333335, "acc_at_50_coverage": 0.93, "acc_at_80_coverage": 0.8541666666666666 }, "score": { "n": 800, "accuracy": 0.7225, "macro_f1": 0.7150339821484726, "ece": 0.19867975000000002, "brier": 0.4347489773875002, "nll": 0.7764480229082796, "aurc": 0.1370651831938734, "mean_confidence": 0.5245894999999998, "acc_at_50_coverage": 0.87, "acc_at_80_coverage": 0.7671875 } }, "soft_acc": 0.4668227307218911, "brier_soft": 0.06772113657369036, "score_mae": 0.24240826250000005, "within_1": 0.995, "latency_p50_ms": 286.0, "latency_p99_ms": 484.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 93.0, "max_ms": 532.0, "argmax_case": 150 }, "determinism_ok": true, "seconds": 109.645121, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 2000, "accuracy": 0.3575, "macro_f1": 0.32620639534238516, "ece": 0.2112708999999999, "brier": 0.7684491959949972, "nll": 1.3593406048220138, "aurc": 0.5626626389078578, "mean_confidence": 0.5687709000000007, "acc_at_50_coverage": 0.429, "acc_at_80_coverage": 0.38125 }, "readout_ece": 0.19182075000000007, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.28833333333333333, "macro_f1": 0.2958479995436556, "ece": 0.16805883333333335, "brier": 0.8079407131999993, "nll": 1.5603165852368217, "aurc": 0.6171637396940312, "mean_confidence": 0.4527051666666662, "acc_at_50_coverage": 0.38666666666666666, "acc_at_80_coverage": 0.325 }, "noul": { "n": 600, "accuracy": 0.47333333333333333, "macro_f1": 0.46994095544820186, "ece": 0.28084866666666675, "brier": 0.6693738674000009, "nll": 0.9693971360440156, "aurc": 0.49261854857569193, "mean_confidence": 0.7541819999999997, "acc_at_50_coverage": 0.4866666666666667, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.3225, "macro_f1": 0.24129818605373515, "ece": 0.20646887500000002, "brier": 0.8131370545375017, "nll": 1.5010662210943988, "aurc": 0.6560603649790228, "mean_confidence": 0.5167618750000003, "acc_at_50_coverage": 0.335, "acc_at_80_coverage": 0.3125 } }, "soft_acc": 0.3345823158039128, "brier_soft": 0.334808769875873, "score_mae": 0.6937230449999998, "within_1": 0.7525, "latency_p50_ms": 328.0, "latency_p99_ms": 557.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 104.0, "max_ms": 693.0, "argmax_case": 183 }, "determinism_ok": true, "seconds": 141.410336666, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true }, "py/multilingual": { "lane": "laya (python)", "model": "multilingual", "hard": { "n": 2000, "accuracy": 0.349, "macro_f1": 0.3498994829046305, "ece": 0.3228019999999999, "brier": 0.9056681466699985, "nll": 1.9436844062731538, "aurc": 0.5548199956342791, "mean_confidence": 0.6687790000000011, "acc_at_50_coverage": 0.404, "acc_at_80_coverage": 0.36375 }, "readout_ece": 0.2566663, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.295, "macro_f1": 0.3242803546952691, "ece": 0.3623458333333333, "brier": 0.9884961356000006, "nll": 2.509649991474549, "aurc": 0.5968789601913673, "mean_confidence": 0.6559421666666665, "acc_at_50_coverage": 0.31666666666666665, "acc_at_80_coverage": 0.28541666666666665 }, "noul": { "n": 600, "accuracy": 0.4866666666666667, "macro_f1": 0.4534096824570536, "ece": 0.40840183333333313, "brier": 0.8741184182333351, "nll": 1.6946657494232418, "aurc": 0.4613874178933604, "mean_confidence": 0.8919398333333328, "acc_at_50_coverage": 0.45, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.28625, "macro_f1": 0.2554689567204614, "ece": 0.23129074999999996, "brier": 0.8672094513000029, "nll": 1.7059742100095436, "aurc": 0.7179156295538103, "mean_confidence": 0.5110360000000005, "acc_at_50_coverage": 0.26, "acc_at_80_coverage": 0.2765625 } }, "soft_acc": 0.32713275531640584, "brier_soft": 0.47372954021453173, "score_mae": 0.7603942575000003, "within_1": 0.7, "latency_p50_ms": 173.0, "latency_p99_ms": 327.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 50.0, "max_ms": 393.0, "argmax_case": 144 }, "determinism_ok": true, "seconds": 73.405661625, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true }, "py/typed": { "lane": "laya (python)", "model": "typed", "hard": { "n": 2000, "accuracy": 0.7445, "macro_f1": 0.7349433525103708, "ece": 0.19239184999999998, "brier": 0.4098811812149981, "nll": 0.7184641706793969, "aurc": 0.12633869093638475, "mean_confidence": 0.5521081499999994, "acc_at_50_coverage": 0.859, "acc_at_80_coverage": 0.793125 }, "readout_ece": 0.39528484999999997, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.7333333333333333, "macro_f1": 0.6943336435123875, "ece": 0.2548458333333333, "brier": 0.46509071859999873, "nll": 0.864997551289102, "aurc": 0.11299371741797679, "mean_confidence": 0.47848750000000007, "acc_at_50_coverage": 0.8966666666666666, "acc_at_80_coverage": 0.8 }, "noul": { "n": 600, "accuracy": 0.785, "macro_f1": 0.7833092099185031, "ece": 0.1294261666666668, "brier": 0.32151443403333324, "nll": 0.49461904218570935, "aurc": 0.08280543014765192, "mean_confidence": 0.6624201666666668, "acc_at_50_coverage": 0.93, "acc_at_80_coverage": 0.8541666666666666 }, "score": { "n": 800, "accuracy": 0.7225, "macro_f1": 0.7150339821484726, "ece": 0.198679625, "brier": 0.4347490885625001, "nll": 0.776447981592383, "aurc": 0.1370651831938734, "mean_confidence": 0.5245896249999998, "acc_at_50_coverage": 0.87, "acc_at_80_coverage": 0.7671875 } }, "soft_acc": 0.46682272509426403, "brier_soft": 0.06772116115371934, "score_mae": 0.2424075125000001, "within_1": 0.995, "latency_p50_ms": 352.0, "latency_p99_ms": 774.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 105.0, "max_ms": 822.0, "argmax_case": 119 }, "determinism_ok": true, "seconds": 148.247935, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": {}, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 2000, "accuracy": 0.57, "macro_f1": 0.539329054680424, "ece": 0.2154347883686423, "brier": 0.643432070067824, "nll": 1.1467231124146129, "aurc": 0.30591289718569137, "mean_confidence": 0.3545652116313577, "acc_at_50_coverage": 0.668, "acc_at_80_coverage": 0.58 }, "readout_ece": 0.09456630986928942, "readout_brier": 0.24279729675048764, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.632, "selective_accuracy": 0.5747282608695652, "selective_n": 736 }, "abstain_causes": { "score_gate": 947, "distance_gate": 317, "grammar_invalid": 0 }, "readout_ece_raw": 0.5644833512902259, "readout_ece_calibrated": 0.09456630986928942, "floor_ece": 0.21644411177644715, "g1_pass": true, "g1_verdict": "pass", "by_question_type": { "choice": { "n": 600, "accuracy": 0.5483333333333333, "macro_f1": 0.4427516704324253, "ece": 0.2756351305296024, "brier": 0.7261415492313711, "nll": 1.3760498373068872, "aurc": 0.43141318071378293, "mean_confidence": 0.27269820280373097, "acc_at_50_coverage": 0.4766666666666667, "acc_at_80_coverage": 0.5041666666666667 }, "noul": { "n": 600, "accuracy": 0.7433333333333333, "macro_f1": 0.7371535205498156, "ece": 0.20128869419296583, "brier": 0.4509107544058871, "nll": 0.6436496200955683, "aurc": 0.16217189041911362, "mean_confidence": 0.5420446391403675, "acc_at_50_coverage": 0.8266666666666667, "acc_at_80_coverage": 0.775 }, "score": { "n": 800, "accuracy": 0.45625, "macro_f1": 0.36099755761350183, "ece": 0.18089410237967968, "brier": 0.7257909474416057, "nll": 1.3520331879846887, "aurc": 0.46530446693557165, "mean_confidence": 0.2753558976203203, "acc_at_50_coverage": 0.5225, "acc_at_80_coverage": 0.4734375 } }, "soft_acc": 0.3330111377725273, "brier_soft": 0.2106513235607973, "score_mae": 0.6697793511115953, "within_1": 0.76, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 2000, "score_threshold": 0.56503725, "distance_threshold": 0.91222644, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.56503725, "status": "fitted", "targetAccuracy": 0.4, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } }, "distance": { "threshold": 0.91222644, "status": "fitted", "targetAccuracy": 0.47142857, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } } }, "corpus_cap": { "effective": 48, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.318 } ] }, "oc_selection": { "selected_scale": 0.25, "candidates": [ { "scale": 0.0, "cal_acc": 0.318 }, { "scale": 0.25, "cal_acc": 0.54 }, { "scale": 0.5, "cal_acc": 0.532 }, { "scale": 1.0, "cal_acc": 0.536 }, { "scale": 2.0, "cal_acc": 0.536 }, { "scale": 4.0, "cal_acc": 0.538 }, { "scale": 8.0, "cal_acc": 0.536 } ] }, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.54 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "Elevated; should be handled within the same week.", "pred": "Critical; requires action within the same day.", "count": 55, "share_of_errors": 0.07790368271954674 }, { "gold": "Elevated; should be handled within the same week.", "pred": "Routine; handle within the normal queue.", "count": 52, "share_of_errors": 0.07365439093484419 }, { "gold": "success", "pred": "partial", "count": 50, "share_of_errors": 0.0708215297450425 }, { "gold": "No time pressure; can wait indefinitely.", "pred": "Routine; handle within the normal queue.", "count": 36, "share_of_errors": 0.05099150141643059 }, { "gold": "Material: a large or unexplained difference.", "pred": "None: everything reconciles.", "count": 35, "share_of_errors": 0.049575070821529746 }, { "gold": "Benign: read-only or clearly safe actions.", "pred": "Low: routine writes within scope.", "count": 33, "share_of_errors": 0.046742209631728045 }, { "gold": "Moderate: access to internal systems or non-public data.", "pred": "High: access to production, secrets or customer data.", "count": 30, "share_of_errors": 0.042492917847025496 }, { "gold": "Critical; requires action within the same day.", "pred": "Routine; handle within the normal queue.", "count": 28, "share_of_errors": 0.039660056657223795 }, { "gold": "Imminent: threatening to cancel, dispute or leave.", "pred": "Clearly unhappy; repeat problems or explicit complaints.", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "Mild frustration, but the relationship is intact.", "pred": "Clearly unhappy; repeat problems or explicit complaints.", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "request_information", "pred": "escalate_to_human", "count": 24, "share_of_errors": 0.0339943342776204 }, { "gold": "Routine; handle within the normal queue.", "pred": "Critical; requires action within the same day.", "count": 22, "share_of_errors": 0.031161473087818695 } ], "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.39222614840989395, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.5529305106401444 }, { "mode": "max_prob", "cal_ece": 0.2030728253722191 }, { "mode": "inv_entropy", "cal_ece": 0.5529305106401444 } ] }, "corpus_digest": "fnv1a64-2fe9ec3132dbc924", "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 2000, "accuracy": 0.3575, "macro_f1": 0.32620639534238516, "ece": 0.21127114999999994, "brier": 0.7684488626149973, "nll": 1.3593395233945826, "aurc": 0.5626661372266888, "mean_confidence": 0.5687711500000007, "acc_at_50_coverage": 0.429, "acc_at_80_coverage": 0.38125 }, "readout_ece": 0.19182065, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.28833333333333333, "macro_f1": 0.2958479995436556, "ece": 0.16805983333333332, "brier": 0.8079412361333324, "nll": 1.5603165852368217, "aurc": 0.6171637396940312, "mean_confidence": 0.45270616666666624, "acc_at_50_coverage": 0.38666666666666666, "acc_at_80_coverage": 0.325 }, "noul": { "n": 600, "accuracy": 0.47333333333333333, "macro_f1": 0.46994095544820186, "ece": 0.2808481666666668, "brier": 0.6693727361666675, "nll": 0.9693957264213799, "aurc": 0.4926269983870734, "mean_confidence": 0.7541814999999997, "acc_at_50_coverage": 0.4866666666666667, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.3225, "macro_f1": 0.24129818605373515, "ece": 0.20646887500000005, "brier": 0.8131366773125016, "nll": 1.501064574742798, "aurc": 0.6560603649790228, "mean_confidence": 0.5167621250000003, "acc_at_50_coverage": 0.335, "acc_at_80_coverage": 0.3125 } }, "soft_acc": 0.3345824092731849, "brier_soft": 0.33480864483416156, "score_mae": 0.6937211699999999, "within_1": 0.7525, "latency_p50_ms": 168.0, "latency_p99_ms": 252.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 56.0, "max_ms": 284.0, "argmax_case": 184 }, "determinism_ok": true, "seconds": 61.4124903, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } }, "multilingual": { "lane": "laya (rust)", "model": "multilingual", "hard": { "n": 2000, "accuracy": 0.349, "macro_f1": 0.3498994829046305, "ece": 0.3228023499999999, "brier": 0.9056686497349985, "nll": 1.9436850897748563, "aurc": 0.5548208277405716, "mean_confidence": 0.6687793500000011, "acc_at_50_coverage": 0.404, "acc_at_80_coverage": 0.36375 }, "readout_ece": 0.2566663, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.295, "macro_f1": 0.3242803546952691, "ece": 0.36234616666666664, "brier": 0.988496027433334, "nll": 2.5096494088603847, "aurc": 0.5968789601913673, "mean_confidence": 0.6559424999999999, "acc_at_50_coverage": 0.31666666666666665, "acc_at_80_coverage": 0.28541666666666665 }, "noul": { "n": 600, "accuracy": 0.4866666666666667, "macro_f1": 0.4534096824570536, "ece": 0.4084023333333331, "brier": 0.8741193240000018, "nll": 1.6946673304708622, "aurc": 0.4613874178933604, "mean_confidence": 0.8919403333333329, "acc_at_50_coverage": 0.45, "acc_at_80_coverage": 0.48333333333333334 }, "score": { "n": 800, "accuracy": 0.28625, "macro_f1": 0.2554689567204614, "ece": 0.23129099999999994, "brier": 0.8672101107625029, "nll": 1.7059751699387093, "aurc": 0.7179176231742249, "mean_confidence": 0.5110362500000005, "acc_at_50_coverage": 0.26, "acc_at_80_coverage": 0.2765625 } }, "soft_acc": 0.32713270906562, "brier_soft": 0.47372988499163493, "score_mae": 0.7603950075000003, "within_1": 0.7, "latency_p50_ms": 82.0, "latency_p99_ms": 126.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 32.0, "max_ms": 139.0, "argmax_case": 183 }, "determinism_ok": true, "seconds": 33.5396411, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } }, "typed": { "lane": "laya (rust)", "model": "typed", "hard": { "n": 2000, "accuracy": 0.7445, "macro_f1": 0.7349433525103708, "ece": 0.1923919, "brier": 0.40988108352999814, "nll": 0.7184638360675057, "aurc": 0.12634017215900445, "mean_confidence": 0.5521080999999993, "acc_at_50_coverage": 0.859, "acc_at_80_coverage": 0.793125 }, "readout_ece": 0.39528484999999997, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.7333333333333333, "macro_f1": 0.6943336435123875, "ece": 0.25484549999999995, "brier": 0.4650906017499988, "nll": 0.8649971870301854, "aurc": 0.11299371741797679, "mean_confidence": 0.4784878333333334, "acc_at_50_coverage": 0.8966666666666666, "acc_at_80_coverage": 0.8 }, "noul": { "n": 600, "accuracy": 0.785, "macro_f1": 0.7833092099185031, "ece": 0.1294261666666668, "brier": 0.32151493469999987, "nll": 0.4946195947319354, "aurc": 0.08280543014765192, "mean_confidence": 0.6624198333333333, "acc_at_50_coverage": 0.93, "acc_at_80_coverage": 0.8541666666666666 }, "score": { "n": 800, "accuracy": 0.7225, "macro_f1": 0.7150339821484726, "ece": 0.19867975000000002, "brier": 0.4347485564875002, "nll": 0.7764470038471731, "aurc": 0.1370651831938734, "mean_confidence": 0.5245894999999998, "acc_at_50_coverage": 0.87, "acc_at_80_coverage": 0.7671875 } }, "soft_acc": 0.46682272260714175, "brier_soft": 0.06772116157482795, "score_mae": 0.2424063875000001, "within_1": 0.995, "latency_p50_ms": 164.0, "latency_p99_ms": 294.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 57.0, "max_ms": 340.0, "argmax_case": 120 }, "determinism_ok": true, "seconds": 63.1842924, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 2000, "accuracy": 0.3465, "macro_f1": 0.3465884147936963, "ece": 0.4643837817120366, "brier": 1.0082961386202958, "nll": 2.4269882116481853, "aurc": 0.5984239220064476, "mean_confidence": 0.8108837817120366, "acc_at_50_coverage": 0.378, "acc_at_80_coverage": 0.37625 }, "readout_ece": 0.3830076462700963, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.25166666666666665, "macro_f1": 0.23483052745038466, "ece": 0.5743500151236852, "brier": 1.216538822586063, "nll": 3.0169599585756712, "aurc": 0.7233244062498014, "mean_confidence": 0.8260166817903518, "acc_at_50_coverage": 0.29, "acc_at_80_coverage": 0.25625 }, "noul": { "n": 600, "accuracy": 0.465, "macro_f1": 0.4529636814464935, "ece": 0.28523051866485427, "brier": 0.6404771234026876, "nll": 0.9156384618856787, "aurc": 0.46262538166555617, "mean_confidence": 0.7502305186648542, "acc_at_50_coverage": 0.5466666666666666, "acc_at_80_coverage": 0.51875 }, "score": { "n": 800, "accuracy": 0.32875, "macro_f1": 0.26377036202682874, "ece": 0.5162740539386869, "brier": 1.12797838705916, "nll": 3.118021713774438, "aurc": 0.5530030336742611, "mean_confidence": 0.8450240539386868, "acc_at_50_coverage": 0.4375, "acc_at_80_coverage": 0.359375 } }, "soft_acc": 0.3553442714733309, "brier_soft": 0.5839153738645226, "score_mae": 0.9178445641411309, "within_1": 0.61, "latency_p50_ms": 105.0, "latency_p99_ms": 2123.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 235.0, "max_ms": 3259.0, "argmax_case": 252 }, "determinism_ok": true, "determinism_n": 10, "seconds": 64.131059, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.07632508833922261, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 2000, "accuracy": 0.528, "macro_f1": 0.5198316028118526, "ece": 0.09588082381940428, "brier": 0.6057403150456773, "nll": 1.0662487183760088, "aurc": 0.36563423961725566, "mean_confidence": 0.43790600669969004, "acc_at_50_coverage": 0.634, "acc_at_80_coverage": 0.571875 }, "readout_ece": 0.09588082381940428, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.49166666666666664, "macro_f1": 0.49646777652104745, "ece": 0.09388112721498426, "brier": 0.6416265634210919, "nll": 1.2105778934133675, "aurc": 0.3608241499380246, "mean_confidence": 0.40650598964619045, "acc_at_50_coverage": 0.6, "acc_at_80_coverage": 0.5375 }, "noul": { "n": 600, "accuracy": 0.6183333333333333, "macro_f1": 0.6177716700373604, "ece": 0.019429831672370917, "brier": 0.4718547816588068, "nll": 0.6648422845239664, "aurc": 0.34594609102918034, "mean_confidence": 0.6082711974065109, "acc_at_50_coverage": 0.6566666666666666, "acc_at_80_coverage": 0.6354166666666666 }, "score": { "n": 800, "accuracy": 0.4875, "macro_f1": 0.4032685049617461, "ece": 0.159997498668753, "brier": 0.6792397788042708, "nll": 1.259056662487016, "aurc": 0.3783267125021683, "mean_confidence": 0.3336821264596977, "acc_at_50_coverage": 0.58, "acc_at_80_coverage": 0.5234375 } }, "soft_acc": 0.36544210737156896, "brier_soft": 0.18658760669736948, "score_mae": 0.5665355150813695, "within_1": 0.87625, "latency_p50_ms": 29.0, "latency_p99_ms": 44.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 102.0, "max_ms": 102.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 22.2471268, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.3328621908127209, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 2000, "accuracy": 0.772, "macro_f1": 0.7698450800890898, "ece": 0.10872479543508962, "brier": 0.33713667548089904, "nll": 0.5770177699477649, "aurc": 0.1004260172185894, "mean_confidence": 0.6645093024033122, "acc_at_50_coverage": 0.901, "acc_at_80_coverage": 0.8175 }, "readout_ece": 0.10872479543508962, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.7766666666666666, "macro_f1": 0.7469692067667553, "ece": 0.13647663856546083, "brier": 0.34545076123644186, "nll": 0.6237740710141895, "aurc": 0.0829784382569066, "mean_confidence": 0.6443036875625452, "acc_at_50_coverage": 0.9466666666666667, "acc_at_80_coverage": 0.8354166666666667 }, "noul": { "n": 600, "accuracy": 0.7616666666666667, "macro_f1": 0.7576757204063591, "ece": 0.041415465388757476, "brier": 0.2948595198409241, "nll": 0.4390444316191804, "aurc": 0.08630257205218163, "mean_confidence": 0.7624051355679209, "acc_at_50_coverage": 0.93, "acc_at_80_coverage": 0.8291666666666667 }, "score": { "n": 800, "accuracy": 0.77625, "macro_f1": 0.7726608596817798, "ece": 0.17000836133956915, "brier": 0.36260897789421875, "nll": 0.6454305478943873, "aurc": 0.09781931118883022, "mean_confidence": 0.6062416386604309, "acc_at_50_coverage": 0.925, "acc_at_80_coverage": 0.828125 } }, "soft_acc": 0.5271786083410424, "brier_soft": 0.06133552698090245, "score_mae": 0.21679321854850095, "within_1": 0.9925, "latency_p50_ms": 87.0, "latency_p99_ms": 178.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 439.0, "max_ms": 439.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 43.4238091, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.6777385159010602, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 2000, "accuracy": 0.536, "macro_f1": 0.539188499881466, "ece": 0.2274510137247853, "brier": 0.6464800785384385, "nll": 1.176883793206239, "aurc": 0.32452831651455416, "mean_confidence": 0.7620469068097881, "acc_at_50_coverage": 0.634, "acc_at_80_coverage": 0.56 }, "readout_ece": 0.15663834208015734, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.545, "macro_f1": 0.5550637131854588, "ece": 0.2832088556389014, "brier": 0.726042368799983, "nll": 1.4632473185020318, "aurc": 0.28980483340473656, "mean_confidence": 0.8276060942312081, "acc_at_50_coverage": 0.6433333333333333, "acc_at_80_coverage": 0.5708333333333333 }, "noul": { "n": 600, "accuracy": 0.5166666666666667, "macro_f1": 0.5152879301123194, "ece": 0.2366313620765383, "brier": 0.5719942126358104, "nll": 0.8068832841381363, "aurc": 0.3256950147452612, "mean_confidence": 0.7457535545388236, "acc_at_50_coverage": 0.6333333333333333, "acc_at_80_coverage": 0.5354166666666667 }, "score": { "n": 800, "accuracy": 0.54375, "macro_f1": 0.5207694222850818, "ece": 0.18621573891490698, "brier": 0.6426727602692369, "nll": 1.2396115310354738, "aurc": 0.33882020236382976, "mean_confidence": 0.7250975304469466, "acc_at_50_coverage": 0.65, "acc_at_80_coverage": 0.5609375 } }, "soft_acc": 0.4559382317932149, "brier_soft": 0.31618158912991967, "score_mae": 0.4994295016957298, "within_1": 0.89375, "latency_p50_ms": 78.0, "latency_p99_ms": 102.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 232.0, "max_ms": 232.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 38.4093595, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.3441696113074206, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } }, "drex": { "lane": "drex (reference)", "model": "drex-dlm", "hard": { "n": 2000, "accuracy": 0.5865, "macro_f1": 0.5802987467662611, "ece": 0.10819204999999996, "brier": 0.5438868918900003, "nll": 0.9371348971982383, "aurc": 0.2847039238141745, "mean_confidence": 0.6808192500000004, "acc_at_50_coverage": 0.691, "acc_at_80_coverage": 0.6225 }, "readout_ece": 0.1928557500000001, "readout_brier": 0.2663099949850001, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "drex_conf_readout": { "choice": { "n": 600, "ece": 0.1471298333333334, "brier": 0.23891225341666658 }, "score": { "n": 800, "ece": 0.24774887500000006, "brier": 0.2876990959624999 }, "noul": null, "floor_ece": 0.12199103321785201, "floor_n_cal": 700, "floor_n_test": 700, "floor_posture": "split-half conformal (first half cal, second half test) \u2014 this lane has no dedicated cal slice" }, "by_question_type": { "choice": { "n": 600, "accuracy": 0.6166666666666667, "macro_f1": 0.5986425345380547, "ece": 0.12052533333333346, "brier": 0.5410055013166669, "nll": 1.0193013141720357, "aurc": 0.24645347271504708, "mean_confidence": 0.7359016666666661, "acc_at_50_coverage": 0.7233333333333334, "acc_at_80_coverage": 0.66875 }, "noul": { "n": 600, "accuracy": 0.5716666666666667, "macro_f1": 0.5187250896538379, "ece": 0.1874288333333333, "brier": 0.5303778705000013, "nll": 0.7878827847309335, "aurc": 0.2961348428659781, "mean_confidence": 0.7590955000000001, "acc_at_50_coverage": 0.69, "acc_at_80_coverage": 0.6 }, "score": { "n": 800, "accuracy": 0.575, "macro_f1": 0.6013650035333423, "ece": 0.06176450000000007, "brier": 0.5561797008624992, "nll": 0.9874491688183723, "aurc": 0.28973144355001207, "mean_confidence": 0.5808002499999998, "acc_at_50_coverage": 0.6675, "acc_at_80_coverage": 0.6140625 } }, "soft_acc": 0.461749687739992, "brier_soft": 0.22050576685344442, "score_mae": 0.3906310500000003, "within_1": 0.92125, "latency_p50_ms": 106.0, "latency_p99_ms": 167.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 165.0, "max_ms": 170.0, "argmax_case": 119 }, "determinism_ok": true, "determinism_n": 10, "seconds": 50.7448774, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.41554770318021206, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "7162840", "date_utc": "2026-10-08T02:36:48Z" } } } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-6e37760ee2a5b6c9" }, "py": { "kind": "digest", "id": "fnv1a64-6e37760ee2a5b6c9" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-6e37760ee2a5b6c9" }, "rust": { "kind": "digest", "id": "fnv1a64-6e37760ee2a5b6c9" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "H2(\u03b2=0.5,nmin=2,\u03c4=2)", "hard": { "n": 2000, "accuracy": 0.6475, "ece": 0.18986412296281885, "mean_confidence": 0.8235891979113439, "acc_at_50_coverage": 0.754, "macro_f1": 0.6052308469932142, "jdi_chance": 0.2925, "jdi_skill": 0.5017667844522968 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.003166, "latency_p99_ms": 0.012083, "latency_tail_support": 21, "serves": "H2(\u03b2=0.5,nmin=2,\u03c4=2)", "gate": "certified (paired LB95 +0.0580, mean +0.0750)", "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 2000, "accuracy": 0.5345, "macro_f1": 0.5371199035294525, "ece": 0.2274083137717098, "brier": 0.6460861957061594, "nll": 1.1753930534378259, "aurc": 0.3246053071014397, "mean_confidence": 0.7618385489489883, "acc_at_50_coverage": 0.633, "acc_at_80_coverage": 0.559375 }, "readout_ece": 0.15472393500125628, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.545, "macro_f1": 0.554520813706687, "ece": 0.28237459932764375, "brier": 0.7253539905037218, "nll": 1.4589747245828801, "aurc": 0.29013083854246025, "mean_confidence": 0.8268162489434083, "acc_at_50_coverage": 0.6466666666666666, "acc_at_80_coverage": 0.5666666666666667 }, "noul": { "n": 600, "accuracy": 0.5166666666666667, "macro_f1": 0.5151096298112829, "ece": 0.234377175240467, "brier": 0.5712280944901422, "nll": 0.805847085584037, "aurc": 0.32524087769305254, "mean_confidence": 0.7453727274822692, "acc_at_50_coverage": 0.6333333333333333, "acc_at_80_coverage": 0.5375 }, "score": { "n": 800, "accuracy": 0.54, "macro_f1": 0.5138024878101111, "ece": 0.185966830663383, "brier": 0.6427789255199906, "nll": 1.2398662759693813, "aurc": 0.33940640735851063, "mean_confidence": 0.7254546400532127, "acc_at_50_coverage": 0.6525, "acc_at_80_coverage": 0.5578125 } }, "soft_acc": 0.45594216445138996, "brier_soft": 0.31585399196464964, "score_mae": 0.4991223548573117, "within_1": 0.895, "latency_p50_ms": 497.0, "latency_p99_ms": 779.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 392.0, "max_ms": 787.0, "argmax_case": 151 }, "determinism_ok": true, "seconds": 208.132836417, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "73c963ebfd40a8d25288,f1d08b30f8c84c2fec6c,04a3b05698d9f7afc8cd,3f8d28006aa102c9df04,0189d0989f5222454bea,5c74a4fca552e2471669,0c366580689869c1f689,0522b1a4023281b00c19,c4bf7c8af208cf4374af,19114692867411036ed5,3466ab136a4aa4ae6bdf,84d32e31a96917d47a0b,eeecbb0dbfd3bba96bdb,d8564cd360417f43590c,492c5669e60b7b07a7db", "spec_file": "scripts/paw_specs/typed_decisions.action.txt,scripts/paw_specs/typed_decisions.needs_review.txt,scripts/paw_specs/typed_decisions.outcome.txt,scripts/paw_specs/typed_decisions.risk.txt,scripts/paw_specs/typed_decisions.urgency.txt,scripts/paw_specs/typed_decisions.category.txt,scripts/paw_specs/typed_decisions.churn_risk.txt,scripts/paw_specs/typed_decisions.needs_human.txt,scripts/paw_specs/typed_decisions.discrepancy_severity.txt,scripts/paw_specs/typed_decisions.disposition.txt,scripts/paw_specs/typed_decisions.duplicate.txt,scripts/paw_specs/typed_decisions.matches_order.txt,scripts/paw_specs/typed_decisions.credential_compromise.txt,scripts/paw_specs/typed_decisions.severity.txt,scripts/paw_specs/typed_decisions.true_positive.txt", "spec_blake3": "72055396b7348aa43c7fcc2f56d7617e9d6448defdcd1a37fe46680a29c3973c,c7a474c539477d43c9b1de0d464f695836ecf7744fd3021c64dbf166a482451b,b18771699ebb4d643ef2a95de0890fc3f735e090e58bda0afb4eee4115ee4cd5,8d5ae953d07c1ba75f325e3cee87eec54cccf4a3a2adb05221e287b7c6852a46,dadfd9808e4aa4edbcdcb1ca8445b5070ff132d48e5c6a1d0ce082a80b2e68ea,1e91ffffbd7ecc71492da3dc24f9552cd08d48e162ab1b7923b7cdf1ba42bf66,b3d2713c4a8bea1a38b46ed2c26188ac506c54712b06736daa90240516cad297,0ac1f08efb9a0e0c4bfe8d80cadf12b90221ad20ad3965d5e99685aff3f7c75b,6a6ed8cbcc154f47435e9258a84d560d8fd1292a51edc639be5218d3dcd84016,eb7abb02fb227a9ccd04136c7b46ac9ffd72553780da57e608ddd232cf92212d,bdcd453d1d3e321e39d3ab77685d1e0f00b959f0bdc9c1df59b031f90b91eaf6,175eb2d46596884958f2bafc6786a0945aa8982e08270b81dcb50e05471a9f19,b0edaed6fff9b21a2d8d94406acc82fb7b2d2fda05751eb62346f6e105ff1c61,2173eeb74a3af2ea95857ad6ad8d5963cdab904921d1d42abc96b9784bebe6d9,87c8f166174f846c75f6ec2170e031e3e34dff569e9edb99112c2ecd6da3ca06", "compile_cache_hit": true, "compile_wall_s": 2645.5270762899995, "n_cases": 400, "n_questions": 2000, "n_answered": 1998, "refusals": 2, "quote_stripped": 0, "accuracy": 0.5925, "answered_accuracy": 0.5930930930930931, "refusal_rate": 0.001, "score_mae_answered": 0.6463642800000002, "within_1_answered": 0.79625, "refusal_samples": [ "data", "observe" ], "latency_p50_ms": 1123.103, "latency_p99_ms": 2108.0, "latency_tail_support": 21, "server_latency_p50_ms": 94.6, "determinism_ok": true, "seconds": 2361.861139041, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T21:54:51Z" } }, "encoder": { "lane": "Rethink", "model": "ENC-typed_encoder_v2 (laya-typed encoder + NLEH v2 per-option head)", "hard": { "n": 2000, "accuracy": 0.755, "macro_f1": 0.7445330507017898, "jdi_chance": 0.2925, "jdi_skill": 0.6537102473498233, "ece": 0.05995282201096416, "mean_confidence": 0.7145579666681587, "acc_at_50_coverage": 0.852 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 47.335, "latency_p99_ms": 74.193, "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.7550 (1510/2000 question rows) \u00b7 paired vs the incumbent arm (H2(\u03b2=0.5,nmin=2,\u03c4=2)): mean +0.1075 \u00b7 LB95 +0.0862 \u00b7 per-row p50 47.335 ms (metal device, laya-typed). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017).", "device": "metal", "head_kind": "v2 per-option", "ckpt": "typed", "shape_desc": "d 1024 \u00b7 hidden 128 \u00b7 widths 2/4/5", "record_only": true, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "e493c56", "date_utc": "2026-10-07T01:37:41Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 2000, "accuracy": 0.6235, "macro_f1": 0.6118980306296695, "ece": 0.07355827322602271, "brier": 0.5079956884791216, "nll": 0.8702929329235691, "aurc": 0.27857286503863055, "mean_confidence": 0.6615492779910565, "acc_at_50_coverage": 0.722, "acc_at_80_coverage": 0.6575 }, "readout_ece": 0.07355827322602271, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.6466666666666666, "macro_f1": 0.6369820906714498, "ece": 0.06022603397568067, "brier": 0.4684623287214902, "nll": 0.8409224754314247, "aurc": 0.1903552858511572, "mean_confidence": 0.6070189823210239, "acc_at_50_coverage": 0.8066666666666666, "acc_at_80_coverage": 0.7020833333333333 }, "noul": { "n": 600, "accuracy": 0.6583333333333333, "macro_f1": 0.6571667479618145, "ece": 0.14293939212958015, "brier": 0.49694519340378807, "nll": 0.7986957730217075, "aurc": 0.28931944597954057, "mean_confidence": 0.7990485644340515, "acc_at_50_coverage": 0.6866666666666666, "acc_at_80_coverage": 0.6833333333333333 }, "score": { "n": 800, "accuracy": 0.58, "macro_f1": 0.5806196861293669, "ece": 0.06745047811418772, "brier": 0.5459335796038359, "nll": 0.9460186459690771, "aurc": 0.35564505152085907, "mean_confidence": 0.5993225349113345, "acc_at_50_coverage": 0.645, "acc_at_80_coverage": 0.6078125 } }, "soft_acc": 0.4717161859231123, "brier_soft": 0.17086411334936705, "score_mae": 0.3657786200978856, "within_1": 0.95375, "latency_p50_ms": 726.0, "latency_p99_ms": 1223.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 551.0, "max_ms": 1380.0, "argmax_case": 120 }, "determinism_ok": true, "determinism_n": 10, "seconds": 312.211692375, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.46784452296819795, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T16:34:28Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "73c963ebfd40a8d25288,f1d08b30f8c84c2fec6c,04a3b05698d9f7afc8cd,3f8d28006aa102c9df04,0189d0989f5222454bea,5c74a4fca552e2471669,0c366580689869c1f689,0522b1a4023281b00c19,c4bf7c8af208cf4374af,19114692867411036ed5,3466ab136a4aa4ae6bdf,84d32e31a96917d47a0b,eeecbb0dbfd3bba96bdb,d8564cd360417f43590c,492c5669e60b7b07a7db", "spec_file": "scripts/paw_specs/typed_decisions.action.txt,scripts/paw_specs/typed_decisions.needs_review.txt,scripts/paw_specs/typed_decisions.outcome.txt,scripts/paw_specs/typed_decisions.risk.txt,scripts/paw_specs/typed_decisions.urgency.txt,scripts/paw_specs/typed_decisions.category.txt,scripts/paw_specs/typed_decisions.churn_risk.txt,scripts/paw_specs/typed_decisions.needs_human.txt,scripts/paw_specs/typed_decisions.discrepancy_severity.txt,scripts/paw_specs/typed_decisions.disposition.txt,scripts/paw_specs/typed_decisions.duplicate.txt,scripts/paw_specs/typed_decisions.matches_order.txt,scripts/paw_specs/typed_decisions.credential_compromise.txt,scripts/paw_specs/typed_decisions.severity.txt,scripts/paw_specs/typed_decisions.true_positive.txt", "spec_blake3": "72055396b7348aa43c7fcc2f56d7617e9d6448defdcd1a37fe46680a29c3973c,c7a474c539477d43c9b1de0d464f695836ecf7744fd3021c64dbf166a482451b,b18771699ebb4d643ef2a95de0890fc3f735e090e58bda0afb4eee4115ee4cd5,8d5ae953d07c1ba75f325e3cee87eec54cccf4a3a2adb05221e287b7c6852a46,dadfd9808e4aa4edbcdcb1ca8445b5070ff132d48e5c6a1d0ce082a80b2e68ea,1e91ffffbd7ecc71492da3dc24f9552cd08d48e162ab1b7923b7cdf1ba42bf66,b3d2713c4a8bea1a38b46ed2c26188ac506c54712b06736daa90240516cad297,0ac1f08efb9a0e0c4bfe8d80cadf12b90221ad20ad3965d5e99685aff3f7c75b,6a6ed8cbcc154f47435e9258a84d560d8fd1292a51edc639be5218d3dcd84016,eb7abb02fb227a9ccd04136c7b46ac9ffd72553780da57e608ddd232cf92212d,bdcd453d1d3e321e39d3ab77685d1e0f00b959f0bdc9c1df59b031f90b91eaf6,175eb2d46596884958f2bafc6786a0945aa8982e08270b81dcb50e05471a9f19,b0edaed6fff9b21a2d8d94406acc82fb7b2d2fda05751eb62346f6e105ff1c61,2173eeb74a3af2ea95857ad6ad8d5963cdab904921d1d42abc96b9784bebe6d9,87c8f166174f846c75f6ec2170e031e3e34dff569e9edb99112c2ecd6da3ca06", "compile_cache_hit": true, "compile_wall_s": 2645.5270762899995, "n_cases": 400, "n_questions": 2000, "n_answered": 1998, "refusals": 2, "quote_stripped": 0, "accuracy": 0.5955, "answered_accuracy": 0.5960960960960962, "refusal_rate": 0.001, "score_mae_answered": 0.6386642800000001, "within_1_answered": 0.79625, "refusal_samples": [ "data", "observe" ], "latency_p50_ms": 65.167, "latency_p99_ms": 143.836, "latency_tail_support": 21, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 158.784585083, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:14:24Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 2000, "accuracy": 0.6955, "macro_f1": 0.674190133619508, "ece": 0.0214991500000001, "brier": 0.4010809480199996, "nll": 0.703460507964248, "aurc": 0.15453834199524571, "mean_confidence": 0.6938984499999992, "acc_at_50_coverage": 0.861, "acc_at_80_coverage": 0.749375 }, "readout_ece": 0.0214991500000001, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.68, "macro_f1": 0.6654819049959715, "ece": 0.043095666666666685, "brier": 0.4278200046333323, "nll": 0.7826123445028398, "aurc": 0.16201834458694875, "mean_confidence": 0.6941556666666667, "acc_at_50_coverage": 0.8333333333333334, "acc_at_80_coverage": 0.7270833333333333 }, "noul": { "n": 600, "accuracy": 0.79, "macro_f1": 0.7895416685225602, "ece": 0.05168566666666666, "brier": 0.2857085801333334, "nll": 0.45048747196241257, "aurc": 0.09918285591452777, "mean_confidence": 0.8146023333333334, "acc_at_50_coverage": 0.9233333333333333, "acc_at_80_coverage": 0.8583333333333333 }, "score": { "n": 800, "accuracy": 0.63625, "macro_f1": 0.6135190738880458, "ece": 0.043644624999999944, "brier": 0.4675559314749991, "nll": 0.8338264075616842, "aurc": 0.20723428476578787, "mean_confidence": 0.6031776249999998, "acc_at_50_coverage": 0.7675, "acc_at_80_coverage": 0.678125 } }, "soft_acc": 0.5150087633859379, "brier_soft": 0.11176883383455492, "score_mae": 0.2956587900000002, "within_1": 0.975, "latency_p50_ms": 1690.0, "latency_p99_ms": 2554.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 1691.0, "max_ms": 2606.0, "argmax_case": 399 }, "determinism_ok": true, "determinism_n": 10, "seconds": 724.068979125, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.5696113074204947, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "latency_quotable": true, "source_run": { "git_sha": "91dd0cb", "date_utc": "2026-10-03T20:18:10Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 2000, "accuracy": 0.735, "macro_f1": 0.7219508452915961, "ece": 0.07999853553377613, "brier": 0.3730202811735602, "nll": 0.6533279436680843, "aurc": 0.12563936761624653, "mean_confidence": 0.8109832233224313, "acc_at_50_coverage": 0.88, "acc_at_80_coverage": 0.794375 }, "readout_ece": 0.06699284937669191, "readout_brier": 0.1726679498349063, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": { "choice": { "n": 600, "accuracy": 0.68, "macro_f1": 0.6714846501772543, "ece": 0.09919407403004055, "brier": 0.4247064127825122, "nll": 0.7303590904814541, "aurc": 0.14561867897705566, "mean_confidence": 0.7751998195064207, "acc_at_50_coverage": 0.8566666666666667, "acc_at_80_coverage": 0.7520833333333333 }, "noul": { "n": 600, "accuracy": 0.84, "macro_f1": 0.8398861412560042, "ece": 0.07004587443062507, "brier": 0.2465721595982678, "nll": 0.4439093407281271, "aurc": 0.06347971560925621, "mean_confidence": 0.9067888301944846, "acc_at_50_coverage": 0.9433333333333334, "acc_at_80_coverage": 0.8895833333333333 }, "score": { "n": 800, "accuracy": 0.6975, "macro_f1": 0.6841403664572553, "ece": 0.0756673390543597, "brier": 0.42909177364831186, "nll": 0.7526185357630252, "aurc": 0.1938021997111977, "mean_confidence": 0.7659665710304011, "acc_at_50_coverage": 0.8125, "acc_at_80_coverage": 0.7390625 } }, "soft_acc": 0.5631760183336487, "brier_soft": 0.16464016855256186, "score_mae": 0.3280957041724921, "within_1": 0.965, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.6254416961130742, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } }, "drex": { "lane": "drex (reference)", "model": "drex-dlm", "hard": { "n": 2000, "accuracy": 0.5855, "macro_f1": 0.5788517787061889, "ece": 0.10553430000000033, "brier": 0.5443953298449985, "nll": 0.937999554928506, "aurc": 0.2846085209990927, "mean_confidence": 0.6800911999999993, "acc_at_50_coverage": 0.689, "acc_at_80_coverage": 0.6225 }, "readout_ece": 0.10553430000000033, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "drex_conf_readout": { "choice": { "n": 600, "ece": 0.14601283333333334, "brier": 0.2376375493833334 }, "score": { "n": 800, "ece": 0.2494627499999999, "brier": 0.2894427336249999 }, "noul": null, "floor_ece": 0.11581618096596694, "floor_n_cal": 700, "floor_n_test": 700, "floor_posture": "split-half conformal (first half cal, second half test) \u2014 this lane has no dedicated cal slice" }, "by_question_type": { "choice": { "n": 600, "accuracy": 0.6166666666666667, "macro_f1": 0.599781605844419, "ece": 0.12936550000000008, "brier": 0.5395994695999993, "nll": 1.0156477654914284, "aurc": 0.24501003549092526, "mean_confidence": 0.7349198333333333, "acc_at_50_coverage": 0.7266666666666667, "acc_at_80_coverage": 0.66875 }, "noul": { "n": 600, "accuracy": 0.575, "macro_f1": 0.5224704196954423, "ece": 0.18530716666666663, "brier": 0.5322140476999991, "nll": 0.7914105865373074, "aurc": 0.2965439337753797, "mean_confidence": 0.7603071666666673, "acc_at_50_coverage": 0.6866666666666666, "acc_at_80_coverage": 0.6 }, "score": { "n": 800, "accuracy": 0.57, "macro_f1": 0.5979793787732504, "ece": 0.059286999999999965, "brier": 0.5571281866375002, "nll": 0.9897051232997081, "aurc": 0.2906095294764057, "mean_confidence": 0.5788077499999997, "acc_at_50_coverage": 0.6625, "acc_at_80_coverage": 0.61875 } }, "soft_acc": 0.46141827546461234, "brier_soft": 0.22035116608827357, "score_mae": 0.39117767749999943, "within_1": 0.92125, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 2000, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2925, "jdi_skill": 0.4141342756183746, "cases_digest": "fnv1a64-6e37760ee2a5b6c9", "source_run": { "git_sha": "0a3c79e", "date_utc": "2026-10-08T01:38:13Z" } } }, { "name": "ag_news", "n_cases": 400, "n_questions": 400, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 400, "accuracy": 0.8825, "macro_f1": 0.8741933646432797, "ece": 0.47827894262969495, "brier": 0.5053158266072506, "nll": 0.9629571322547602, "aurc": 0.04689348589556988, "mean_confidence": 0.40422105737030506, "acc_at_50_coverage": 0.94, "acc_at_80_coverage": 0.9375 }, "readout_ece": 0.04585413459688425, "readout_brier": 0.09073375505306376, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.49, "selective_accuracy": 0.9509803921568627, "selective_n": 204 }, "abstain_causes": { "score_gate": 60, "distance_gate": 136, "grammar_invalid": 0 }, "readout_ece_raw": 0.8304792892932893, "readout_ece_calibrated": 0.04585413459688425, "floor_ece": 0.24822139303482577, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.143, "latency_p99_ms": 0.24, "latency_tail_support": 5, "latency_extremes": { "first_ms": 0.238, "max_ms": 0.251, "argmax_case": 5 }, "determinism_ok": true, "determinism_n": 10, "seconds": 4.300983583, "n_cases": 400, "n_questions": 400, "score_threshold": 0.8358875, "distance_threshold": 0.41053298, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.8358875, "status": "fitted", "targetAccuracy": 0.9357143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.41053298, "status": "fitted", "targetAccuracy": 0.8857143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.39 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.7 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.7 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.745 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.74 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.74 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.8825, "honest_accuracy": 0.8825, "n_pseudo": 400, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "business", "pred": "sci_tech", "count": 10, "share_of_errors": 0.2127659574468085 }, { "gold": "sci_tech", "pred": "business", "count": 8, "share_of_errors": 0.1702127659574468 }, { "gold": "world", "pred": "business", "count": 7, "share_of_errors": 0.14893617021276595 }, { "gold": "world", "pred": "sports", "count": 6, "share_of_errors": 0.1276595744680851 }, { "gold": "sports", "pred": "sci_tech", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "sports", "pred": "world", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "world", "pred": "sci_tech", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "business", "pred": "world", "count": 2, "share_of_errors": 0.0425531914893617 }, { "gold": "sci_tech", "pred": "sports", "count": 2, "share_of_errors": 0.0425531914893617 }, { "gold": "business", "pred": "sports", "count": 1, "share_of_errors": 0.02127659574468085 }, { "gold": "sci_tech", "pred": "world", "count": 1, "share_of_errors": 0.02127659574468085 }, { "gold": "sports", "pred": "business", "count": 1, "share_of_errors": 0.02127659574468085 } ], "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8303249097472923, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.8204048293828965 }, { "mode": "max_prob", "cal_ece": 0.48570768773555756 }, { "mode": "inv_entropy", "cal_ece": 0.8204048293828965 } ] }, "corpus_digest": "fnv1a64-58e669a512102140", "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 400, "accuracy": 0.95, "macro_f1": 0.9439373207318056, "ece": 0.03155950000000057, "brier": 0.08078989162499997, "nll": 0.16374224530654963, "aurc": 0.006681571222221135, "mean_confidence": 0.9220469999999991, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.990625 }, "readout_ece": 0.14119374999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 23.0, "latency_p99_ms": 76.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 39.0, "max_ms": 95.0, "argmax_case": 333 }, "determinism_ok": true, "seconds": 18.214984708, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 400, "accuracy": 0.95, "macro_f1": 0.9439373207318056, "ece": 0.03155950000000057, "brier": 0.08078967889999995, "nll": 0.16374224530654963, "aurc": 0.006681571222221135, "mean_confidence": 0.9220469999999991, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.990625 }, "readout_ece": 0.14119374999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 34.0, "latency_p99_ms": 84.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 20.0, "max_ms": 98.0, "argmax_case": 33 }, "determinism_ok": true, "seconds": 22.166054834, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": { "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 365, "accuracy": 0.9452054794520548, "macro_f1": 0.9373782486886029, "ece": 0.038674246575342346, "brier": 0.08643471446575335, "nll": 0.17340547255438327, "aurc": 0.007544982421972155, "mean_confidence": 0.9196917808219178, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9897260273972602 }, "readout_ece": 0.14208958904109587, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 28.0, "latency_p99_ms": 34.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 27.0, "max_ms": 36.0, "argmax_case": 50 }, "determinism_ok": true, "seconds": 48.386979667, "n_cases": 365, "n_questions": 365, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } } }, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 400, "accuracy": 0.8825, "macro_f1": 0.8741933646432797, "ece": 0.4782789427042008, "brier": 0.5053158268175713, "nll": 0.9629571323951134, "aurc": 0.04689348589556988, "mean_confidence": 0.40422105729579927, "acc_at_50_coverage": 0.94, "acc_at_80_coverage": 0.9375 }, "readout_ece": 0.04585413459688427, "readout_brier": 0.09073375498350682, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.49, "selective_accuracy": 0.9509803921568627, "selective_n": 204 }, "abstain_causes": { "score_gate": 60, "distance_gate": 136, "grammar_invalid": 0 }, "readout_ece_raw": 0.8304792888462544, "readout_ece_calibrated": 0.04585413459688427, "floor_ece": 0.24822139303482577, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 400, "score_threshold": 0.8358875, "distance_threshold": 0.41053298, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.8358875, "status": "fitted", "targetAccuracy": 0.9357143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.41053298, "status": "fitted", "targetAccuracy": 0.8857143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.39 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.7 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.7 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.745 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.745 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.74 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.74 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.8825, "honest_accuracy": 0.8825, "n_pseudo": 400, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "business", "pred": "sci_tech", "count": 10, "share_of_errors": 0.2127659574468085 }, { "gold": "sci_tech", "pred": "business", "count": 8, "share_of_errors": 0.1702127659574468 }, { "gold": "world", "pred": "business", "count": 7, "share_of_errors": 0.14893617021276595 }, { "gold": "world", "pred": "sports", "count": 6, "share_of_errors": 0.1276595744680851 }, { "gold": "sports", "pred": "sci_tech", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "sports", "pred": "world", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "world", "pred": "sci_tech", "count": 3, "share_of_errors": 0.06382978723404255 }, { "gold": "business", "pred": "world", "count": 2, "share_of_errors": 0.0425531914893617 }, { "gold": "sci_tech", "pred": "sports", "count": 2, "share_of_errors": 0.0425531914893617 }, { "gold": "business", "pred": "sports", "count": 1, "share_of_errors": 0.02127659574468085 }, { "gold": "sci_tech", "pred": "world", "count": 1, "share_of_errors": 0.02127659574468085 }, { "gold": "sports", "pred": "business", "count": 1, "share_of_errors": 0.02127659574468085 } ], "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8303249097472923, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.8204048293828965 }, { "mode": "max_prob", "cal_ece": 0.48570768788456914 }, { "mode": "inv_entropy", "cal_ece": 0.8204048293828965 } ] }, "corpus_digest": "fnv1a64-58e669a512102140", "cases_digest": "fnv1a64-07eff23c06505584", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 400, "accuracy": 0.95, "macro_f1": 0.9439373207318056, "ece": 0.03155925000000057, "brier": 0.08078943217499995, "nll": 0.16374175177185255, "aurc": 0.006681571222221135, "mean_confidence": 0.9220472499999991, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.990625 }, "readout_ece": 0.14119374999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 13.0, "latency_p99_ms": 18.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 13.0, "max_ms": 20.0, "argmax_case": 301 }, "determinism_ok": true, "seconds": 7.852086, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 400, "accuracy": 0.4025, "macro_f1": 0.3577063186309196, "ece": 0.29118786707520483, "brier": 0.8605258386571991, "nll": 2.1582092945212086, "aurc": 0.4310711176964101, "mean_confidence": 0.6936878670752048, "acc_at_50_coverage": 0.535, "acc_at_80_coverage": 0.446875 }, "readout_ece": 0.19723441418260335, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 33.0, "latency_p99_ms": 77.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 468.0, "max_ms": 468.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 16.6127995, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.13718411552346574, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 400, "accuracy": 0.7025, "macro_f1": 0.6590344066076197, "ece": 0.12537422353601715, "brier": 0.4871705710256062, "nll": 1.0569244178165471, "aurc": 0.20165422979334033, "mean_confidence": 0.8141866097185432, "acc_at_50_coverage": 0.77, "acc_at_80_coverage": 0.725 }, "readout_ece": 0.12537422353601715, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 22.0, "latency_p99_ms": 25.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 93.0, "max_ms": 93.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 19.3007944, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.5703971119133574, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 400, "accuracy": 0.7975, "macro_f1": 0.777482531847409, "ece": 0.13593083105981352, "brier": 0.3157601633175491, "nll": 0.6135953169728172, "aurc": 0.08339255712786313, "mean_confidence": 0.6615691689401865, "acc_at_50_coverage": 0.945, "acc_at_80_coverage": 0.853125 }, "readout_ece": 0.13593083105981352, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 29.0, "latency_p99_ms": 35.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 86.0, "max_ms": 86.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 12.7561706, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.7075812274368231, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "35a7963d2d18aac54616", "spec_file": "scripts/paw_specs\\ag_news.txt", "spec_blake3": "c5d5b475b88da357364a5f3ae3d87b02424a72bb4b16331fb9b59b0d1d543063", "compile_cache_hit": true, "compile_wall_s": 11.7595367, "n_cases": 400, "n_questions": 400, "n_answered": 400, "refusals": 0, "quote_stripped": 0, "accuracy": 0.8, "answered_accuracy": 0.8, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 264.0, "latency_p99_ms": 1563.774, "latency_tail_support": 5, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 241.0684133, "latency_quotable": null }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.89, "macro_f1": 0.882644318569296, "ece": 0.047420404031872684, "brier": 0.1550014348564587, "nll": 0.28137258555480354, "aurc": 0.02023301346851728, "mean_confidence": 0.9207828643172979, "acc_at_50_coverage": 0.985, "acc_at_80_coverage": 0.96875 }, "readout_ece": 0.06789136253893188, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 53.0, "latency_p99_ms": 70.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 52.0, "max_ms": 87.0, "argmax_case": 1 }, "determinism_ok": true, "determinism_n": 10, "seconds": 23.4496954, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8411552346570398, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 400, "exact": 1, "near": 26 }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "35a7963d2d18aac54616", "spec_file": "scripts/paw_specs/ag_news.txt", "spec_blake3": "c5d5b475b88da357364a5f3ae3d87b02424a72bb4b16331fb9b59b0d1d543063", "compile_cache_hit": true, "compile_wall_s": 1.72005275, "n_cases": 400, "n_questions": 400, "n_answered": 400, "refusals": 0, "quote_stripped": 0, "accuracy": 0.79, "answered_accuracy": 0.79, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 1049.536, "latency_p99_ms": 1644.348, "latency_tail_support": 5, "server_latency_p50_ms": 94.0, "determinism_ok": true, "seconds": 451.193453708, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T20:53:41Z" } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-07eff23c06505584" }, "py": { "kind": "digest", "id": "fnv1a64-07eff23c06505584" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-07eff23c06505584" }, "rust": { "kind": "digest", "id": "fnv1a64-07eff23c06505584" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "H2(\u03b2=0.25,nmin=2,\u03c4=2)", "hard": { "n": 400, "accuracy": 0.8975, "ece": 0.013315694555954022, "mean_confidence": 0.888912138562494, "acc_at_50_coverage": 0.99, "macro_f1": 0.8878275558568157, "jdi_chance": 0.3075, "jdi_skill": 0.8519855595667869 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.0015, "latency_p99_ms": 0.003125, "latency_tail_support": 5, "serves": "H2(\u03b2=0.25,nmin=2,\u03c4=2)", "gate": "best measured, T2-uncertified (paired LB95 -0.0100) \u2014 certification is more questions, not a posture rollback", "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.89, "macro_f1": 0.882644318569296, "ece": 0.051162975877523394, "brier": 0.15519215674552342, "nll": 0.2817015835268838, "aurc": 0.020399605241929217, "mean_confidence": 0.9206447131931782, "acc_at_50_coverage": 0.985, "acc_at_80_coverage": 0.96875 }, "readout_ece": 0.07099155767576987, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 113.0, "latency_p99_ms": 226.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 127.0, "max_ms": 237.0, "argmax_case": 5 }, "determinism_ok": true, "seconds": 51.154000542, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "encoder": { "lane": "Rethink", "model": "ENC-ag_news_encoder_v1 (laya-english encoder + NLEH v1 head)", "hard": { "n": 400, "accuracy": 0.9475, "ece": 0.03893379544839259, "mean_confidence": 0.980914422031492, "acc_at_50_coverage": 0.995 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 16.983, "latency_p99_ms": 27.313, "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.9475 (379/400 rows) \u00b7 paired vs the incumbent A1: mean +0.0600 \u00b7 LB95 +0.0320 \u00b7 per-row p50 16.983 ms (metal device). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017). The head reads one question under its own reference (riir-train 600 T1).", "device": "metal", "record_only": true, "latency_quotable": true, "source_run": { "git_sha": "0f81b54", "date_utc": "2026-10-01T11:20:42Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 400, "accuracy": 0.915, "macro_f1": 0.9088637583052136, "ece": 0.035219853520393384, "brier": 0.1230511733458803, "nll": 0.24119856360330336, "aurc": 0.01498919760080796, "mean_confidence": 0.9212963885068893, "acc_at_50_coverage": 0.99, "acc_at_80_coverage": 0.96875 }, "readout_ece": 0.035219853520393384, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 148.0, "latency_p99_ms": 191.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 133.0, "max_ms": 202.0, "argmax_case": 6 }, "determinism_ok": true, "determinism_n": 10, "seconds": 70.808643666, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8772563176895307, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:09:55Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "35a7963d2d18aac54616", "spec_file": "scripts/paw_specs/ag_news.txt", "spec_blake3": "c5d5b475b88da357364a5f3ae3d87b02424a72bb4b16331fb9b59b0d1d543063", "compile_cache_hit": true, "compile_wall_s": 1.72005275, "n_cases": 400, "n_questions": 400, "n_answered": 400, "refusals": 0, "quote_stripped": 0, "accuracy": 0.7925, "answered_accuracy": 0.7925, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 31.092, "latency_p99_ms": 48.64, "latency_tail_support": 5, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 13.927686667, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:15:01Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 400, "accuracy": 0.9, "macro_f1": 0.8917746035059467, "ece": 0.03992650000000026, "brier": 0.1696214880000008, "nll": 0.3335992007998545, "aurc": 0.03126012430935035, "mean_confidence": 0.9217579999999992, "acc_at_50_coverage": 0.985, "acc_at_80_coverage": 0.95 }, "readout_ece": 0.03992650000000026, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 456.0, "latency_p99_ms": 1045.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 386.0, "max_ms": 1361.0, "argmax_case": 301 }, "determinism_ok": true, "determinism_n": 10, "seconds": 199.255336334, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8555956678700362, "cases_digest": "fnv1a64-07eff23c06505584", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:29:34Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 400, "accuracy": 0.89, "macro_f1": 0.8807080810359893, "ece": 0.05604267134801089, "brier": 0.16837005809027647, "nll": 0.35469409878055763, "aurc": 0.03328516636848431, "mean_confidence": 0.9277402427217694, "acc_at_50_coverage": 0.965, "acc_at_80_coverage": 0.946875 }, "readout_ece": 0.07183897108149026, "readout_brier": 0.08100043333889548, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.8411552346570398, "cases_digest": "fnv1a64-07eff23c06505584", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } }, "drex": { "lane": "drex (reference)", "model": "drex-dlm", "hard": { "n": 400, "accuracy": 0.895, "macro_f1": 0.8859395200866416, "ece": 0.0575575000000001, "brier": 0.16363134985000038, "nll": 0.34518013062910813, "aurc": 0.0298156889884093, "mean_confidence": 0.9398579999999999, "acc_at_50_coverage": 0.97, "acc_at_80_coverage": 0.959375 }, "readout_ece": 0.0575575000000001, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "drex_conf_readout": { "choice": { "n": 400, "ece": 0.058848750000000484, "brier": 0.07801197997500009 }, "score": null, "noul": null, "floor_ece": 0.383134328358209, "floor_n_cal": 200, "floor_n_test": 200, "floor_posture": "split-half conformal (first half cal, second half test) \u2014 this lane has no dedicated cal slice" }, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3075, "jdi_skill": 0.848375451263538, "cases_digest": "fnv1a64-07eff23c06505584", "source_run": { "git_sha": "0a3c79e", "date_utc": "2026-10-08T01:38:13Z" } } }, { "name": "emotion", "n_cases": 400, "n_questions": 400, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 400, "accuracy": 0.885, "macro_f1": 0.8014284327939634, "ece": 0.6182363981008531, "brier": 0.6593071434500918, "nll": 1.3642793452306783, "aurc": 0.035998860173968965, "mean_confidence": 0.26676360189914705, "acc_at_50_coverage": 0.975, "acc_at_80_coverage": 0.940625 }, "readout_ece": 0.04880640662275256, "readout_brier": 0.09032840869404729, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.4625, "selective_accuracy": 0.9348837209302325, "selective_n": 215 }, "abstain_causes": { "score_gate": 80, "distance_gate": 105, "grammar_invalid": 0 }, "readout_ece_raw": 0.8621306203305722, "readout_ece_calibrated": 0.04880640662275256, "floor_ece": 0.27810945273631843, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.11, "latency_p99_ms": 0.121, "latency_tail_support": 5, "latency_extremes": { "first_ms": 0.116, "max_ms": 0.125, "argmax_case": 154 }, "determinism_ok": true, "determinism_n": 10, "seconds": 10.029595333, "n_cases": 400, "n_questions": 400, "score_threshold": 0.7631245, "distance_threshold": 0.4265421, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.7631245, "status": "fitted", "targetAccuracy": 0.89285713, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.4265421, "status": "fitted", "targetAccuracy": 0.73571426, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.195 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.535 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.465 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 8.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.55 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.58 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.64 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.69 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.76 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.8875, "honest_accuracy": 0.885, "n_pseudo": 400, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "love", "pred": "joy", "count": 15, "share_of_errors": 0.32608695652173914 }, { "gold": "anger", "pred": "sadness", "count": 5, "share_of_errors": 0.10869565217391304 }, { "gold": "fear", "pred": "joy", "count": 5, "share_of_errors": 0.10869565217391304 }, { "gold": "fear", "pred": "sadness", "count": 4, "share_of_errors": 0.08695652173913043 }, { "gold": "surprise", "pred": "joy", "count": 4, "share_of_errors": 0.08695652173913043 }, { "gold": "fear", "pred": "anger", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "joy", "pred": "sadness", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "joy", "pred": "surprise", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "sadness", "pred": "joy", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "surprise", "pred": "fear", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "anger", "pred": "joy", "count": 1, "share_of_errors": 0.021739130434782608 }, { "gold": "anger", "pred": "surprise", "count": 1, "share_of_errors": 0.021739130434782608 } ], "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.8362989323843416, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.7589061281085013 }, { "mode": "max_prob", "cal_ece": 0.527420337125659 }, { "mode": "inv_entropy", "cal_ece": 0.7589061281085013 } ] }, "corpus_digest": "fnv1a64-dfde0f5a6091008c", "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 400, "accuracy": 0.5925, "macro_f1": 0.47086603548842415, "ece": 0.308786, "brier": 0.6962648099500026, "nll": 2.184045176210792, "aurc": 0.2657102861338114, "mean_confidence": 0.9012859999999996, "acc_at_50_coverage": 0.715, "acc_at_80_coverage": 0.6375 }, "readout_ece": 0.2570897499999998, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 14.0, "latency_p99_ms": 20.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 12.0, "max_ms": 23.0, "argmax_case": 22 }, "determinism_ok": true, "seconds": 9.871729916, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 400, "accuracy": 0.5925, "macro_f1": 0.47086603548842415, "ece": 0.308786, "brier": 0.6962642192000027, "nll": 2.184042555956118, "aurc": 0.2657102861338114, "mean_confidence": 0.9012859999999996, "acc_at_50_coverage": 0.715, "acc_at_80_coverage": 0.6375 }, "readout_ece": 0.2570897499999998, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 24.0, "latency_p99_ms": 58.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 17.0, "max_ms": 67.0, "argmax_case": 143 }, "determinism_ok": true, "seconds": 16.902667542, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": { "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 400, "accuracy": 0.5975, "macro_f1": 0.47767899336751946, "ece": 0.3040499999999997, "brier": 0.6963723941000033, "nll": 2.1820143189369885, "aurc": 0.2658936866009599, "mean_confidence": 0.9015499999999996, "acc_at_50_coverage": 0.715, "acc_at_80_coverage": 0.6375 }, "readout_ece": 0.2546720000000001, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 19.0, "latency_p99_ms": 24.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 21.0, "max_ms": 25.0, "argmax_case": 119 }, "determinism_ok": true, "seconds": 11.97681375, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } } }, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 400, "accuracy": 0.885, "macro_f1": 0.8014284327939634, "ece": 0.6182363714277744, "brier": 0.6593071014744065, "nll": 1.3642792551882197, "aurc": 0.035998860173968965, "mean_confidence": 0.2667636285722256, "acc_at_50_coverage": 0.975, "acc_at_80_coverage": 0.940625 }, "readout_ece": 0.048806075789034384, "readout_brier": 0.09032823103134206, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.4625, "selective_accuracy": 0.9348837209302325, "selective_n": 215 }, "abstain_causes": { "score_gate": 80, "distance_gate": 105, "grammar_invalid": 0 }, "readout_ece_raw": 0.8621306075155736, "readout_ece_calibrated": 0.048806075789034384, "floor_ece": 0.27812189054726366, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 400, "score_threshold": 0.76312464, "distance_threshold": 0.4265421, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.76312464, "status": "fitted", "targetAccuracy": 0.89285713, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.4265421, "status": "fitted", "targetAccuracy": 0.73571426, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.195 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.535 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.55 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.465 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.485 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 8.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.55 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.58 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.64 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.69 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.735 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.76 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.8875, "honest_accuracy": 0.885, "n_pseudo": 400, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "love", "pred": "joy", "count": 15, "share_of_errors": 0.32608695652173914 }, { "gold": "anger", "pred": "sadness", "count": 5, "share_of_errors": 0.10869565217391304 }, { "gold": "fear", "pred": "joy", "count": 5, "share_of_errors": 0.10869565217391304 }, { "gold": "fear", "pred": "sadness", "count": 4, "share_of_errors": 0.08695652173913043 }, { "gold": "surprise", "pred": "joy", "count": 4, "share_of_errors": 0.08695652173913043 }, { "gold": "fear", "pred": "anger", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "joy", "pred": "sadness", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "joy", "pred": "surprise", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "sadness", "pred": "joy", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "surprise", "pred": "fear", "count": 2, "share_of_errors": 0.043478260869565216 }, { "gold": "anger", "pred": "joy", "count": 1, "share_of_errors": 0.021739130434782608 }, { "gold": "anger", "pred": "surprise", "count": 1, "share_of_errors": 0.021739130434782608 } ], "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.8362989323843416, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.7589061298966407 }, { "mode": "max_prob", "cal_ece": 0.5274203329533338 }, { "mode": "inv_entropy", "cal_ece": 0.7589061298966407 } ] }, "corpus_digest": "fnv1a64-dfde0f5a6091008c", "cases_digest": "fnv1a64-6feefd1c947dd008", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 400, "accuracy": 0.5925, "macro_f1": 0.47086603548842415, "ece": 0.3087865, "brier": 0.6962649922250027, "nll": 2.1840444269226102, "aurc": 0.2657102861338114, "mean_confidence": 0.9012864999999995, "acc_at_50_coverage": 0.715, "acc_at_80_coverage": 0.6375 }, "readout_ece": 0.25708974999999984, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 11.0, "latency_p99_ms": 14.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 11.0, "max_ms": 14.0, "argmax_case": 7 }, "determinism_ok": true, "seconds": 6.9131029, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 400, "accuracy": 0.295, "macro_f1": 0.09440569730143315, "ece": 0.37592108093202115, "brier": 1.0979565177781476, "nll": 4.844648455500639, "aurc": 0.6125397107064762, "mean_confidence": 0.6706122183054686, "acc_at_50_coverage": 0.375, "acc_at_80_coverage": 0.325 }, "readout_ece": 0.30973466217517853, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 34.0, "latency_p99_ms": 80.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 425.0, "max_ms": 425.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 16.855636, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.0, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 400, "accuracy": 0.565, "macro_f1": 0.5279099010442294, "ece": 0.14034680133039706, "brier": 0.629630034182988, "nll": 1.4156947177261885, "aurc": 0.33930307457825565, "mean_confidence": 0.635604412016868, "acc_at_50_coverage": 0.665, "acc_at_80_coverage": 0.609375 }, "readout_ece": 0.14034680133039706, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 21.0, "latency_p99_ms": 28.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 96.0, "max_ms": 96.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 19.2474774, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.38078291814946613, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 400, "accuracy": 0.42, "macro_f1": 0.3821987247436258, "ece": 0.05576842796057464, "brier": 0.6894387204818822, "nll": 1.4365399879231027, "aurc": 0.4350399766342879, "mean_confidence": 0.4396525963768363, "acc_at_50_coverage": 0.555, "acc_at_80_coverage": 0.475 }, "readout_ece": 0.05576842796057464, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 33.0, "latency_p99_ms": 43.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 53.0, "max_ms": 53.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 14.4977176, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.17437722419928825, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "703c4b3c3acf668f8ad8", "spec_file": "scripts/paw_specs\\emotion.txt", "spec_blake3": "ab8a01c6e1d96ca06ce9d40c5c57bd491e7ab3d53b18a8b3e7d51ac8a794baf4", "compile_cache_hit": true, "compile_wall_s": 11.6640664, "n_cases": 400, "n_questions": 400, "n_answered": 399, "refusals": 1, "quote_stripped": 0, "accuracy": 0.4875, "answered_accuracy": 0.48872180451127817, "refusal_rate": 0.0025, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "sleepiness" ], "latency_p50_ms": 113.736, "latency_p99_ms": 213.371, "latency_tail_support": 5, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 54.6759834, "latency_quotable": null }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.595, "macro_f1": 0.48767525539925655, "ece": 0.2124154042452574, "brier": 0.6033856957178485, "nll": 1.3950646286483295, "aurc": 0.22055689638200907, "mean_confidence": 0.8071955532580614, "acc_at_50_coverage": 0.775, "acc_at_80_coverage": 0.646875 }, "readout_ece": 0.11384338745045525, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 51.0, "latency_p99_ms": 66.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 57.0, "max_ms": 70.0, "argmax_case": 36 }, "determinism_ok": true, "determinism_n": 10, "seconds": 22.0915601, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.42348754448398573, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 400, "exact": 0, "near": 0 }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "703c4b3c3acf668f8ad8", "spec_file": "scripts/paw_specs/emotion.txt", "spec_blake3": "ab8a01c6e1d96ca06ce9d40c5c57bd491e7ab3d53b18a8b3e7d51ac8a794baf4", "compile_cache_hit": true, "compile_wall_s": 0.924536416, "n_cases": 400, "n_questions": 400, "n_answered": 400, "refusals": 0, "quote_stripped": 0, "accuracy": 0.495, "answered_accuracy": 0.495, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 1047.608, "latency_p99_ms": 1448.602, "latency_tail_support": 5, "server_latency_p50_ms": 76.4, "determinism_ok": true, "seconds": 437.893380667, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T20:45:58Z" } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-6feefd1c947dd008" }, "py": { "kind": "digest", "id": "fnv1a64-6feefd1c947dd008" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-6feefd1c947dd008" }, "rust": { "kind": "digest", "id": "fnv1a64-6feefd1c947dd008" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "A0", "hard": { "n": 400, "accuracy": 0.885, "ece": 0.8622394363582135, "mean_confidence": 0.022760563641786576, "acc_at_50_coverage": 0.97, "macro_f1": 0.8014284327939634, "jdi_chance": 0.2975, "jdi_skill": 0.8362989323843416 }, "consult_rate": 0.0, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": 0.115, "latency_p99_ms": 0.129, "latency_tail_support": 5, "serves": "A0", "gate": "served by the reflex half \u2014 the best measured arm on this suite", "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.59, "macro_f1": 0.4844213484626461, "ece": 0.21647185690701007, "brier": 0.602934554245623, "nll": 1.3914321551837918, "aurc": 0.21994493595638656, "mean_confidence": 0.8064718569070101, "acc_at_50_coverage": 0.775, "acc_at_80_coverage": 0.65 }, "readout_ece": 0.111127159642895, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 78.0, "latency_p99_ms": 149.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 75.0, "max_ms": 163.0, "argmax_case": 378 }, "determinism_ok": true, "seconds": 38.124298167, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 400, "accuracy": 0.5825, "macro_f1": 0.4664338387697607, "ece": 0.15611889574676752, "brier": 0.6065817658058879, "nll": 1.4556839894437805, "aurc": 0.23935113232761343, "mean_confidence": 0.7376930480077862, "acc_at_50_coverage": 0.725, "acc_at_80_coverage": 0.64375 }, "readout_ece": 0.15611889574676752, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 146.0, "latency_p99_ms": 172.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 147.0, "max_ms": 181.0, "argmax_case": 119 }, "determinism_ok": true, "determinism_n": 10, "seconds": 73.320062084, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.40569395017793597, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:20:33Z" } }, "encoder": { "lane": "Rethink", "model": "A0", "hard": { "n": 400, "accuracy": 0.885, "ece": 0.8622394363582135, "mean_confidence": 0.022760563641786576, "acc_at_50_coverage": 0.97 }, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": 0.141, "latency_p99_ms": 0.281, "latency_tail_support": 5, "serves": "tier-fallback", "served_by": "Instinct (A0)", "fallback_note": "screened \u2014 no head earned: 6 gold-only fits (the 5-seed T7 sweep + 1 fresh seed) all refused on holdout; the encoder reference reads 0.5950 vs the incumbent 0.8850 (instinct issue 016 T7/T8, riir-train t599/t7 + t8 record)", "derived": true, "source_run": { "git_sha": "8bbff09", "date_utc": "2026-09-28T10:13:40Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "703c4b3c3acf668f8ad8", "spec_file": "scripts/paw_specs/emotion.txt", "spec_blake3": "ab8a01c6e1d96ca06ce9d40c5c57bd491e7ab3d53b18a8b3e7d51ac8a794baf4", "compile_cache_hit": true, "compile_wall_s": 0.924536416, "n_cases": 400, "n_questions": 400, "n_answered": 400, "refusals": 0, "quote_stripped": 0, "accuracy": 0.49, "answered_accuracy": 0.49, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 28.514, "latency_p99_ms": 35.327, "latency_tail_support": 5, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 12.374872792, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:15:33Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 400, "accuracy": 0.5925, "macro_f1": 0.4914428254411061, "ece": 0.22750299999999998, "brier": 0.6258503445999996, "nll": 1.3155490101052263, "aurc": 0.28487971165470716, "mean_confidence": 0.8095199999999996, "acc_at_50_coverage": 0.73, "acc_at_80_coverage": 0.615625 }, "readout_ece": 0.22750299999999998, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 416.0, "latency_p99_ms": 582.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 468.0, "max_ms": 612.0, "argmax_case": 7 }, "determinism_ok": true, "determinism_n": 10, "seconds": 180.761833625, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.41992882562277584, "cases_digest": "fnv1a64-6feefd1c947dd008", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:26:05Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 400, "accuracy": 0.64, "macro_f1": 0.5364780003205954, "ece": 0.07491385941093745, "brier": 0.5005753878430622, "nll": 1.0192086304308254, "aurc": 0.1886208212578937, "mean_confidence": 0.7115529372890493, "acc_at_50_coverage": 0.805, "acc_at_80_coverage": 0.696875 }, "readout_ece": 0.08786401394714044, "readout_brier": 0.19824695821849192, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2975, "jdi_skill": 0.48754448398576516, "cases_digest": "fnv1a64-6feefd1c947dd008", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "sst5", "n_cases": 600, "n_questions": 600, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 600, "accuracy": 0.39666666666666667, "macro_f1": 0.32101890185812104, "ece": 0.13891353478034335, "brier": 0.7604214524982581, "nll": 1.5156292710507784, "aurc": 0.5364704218034313, "mean_confidence": 0.2577531318863233, "acc_at_50_coverage": 0.47, "acc_at_80_coverage": 0.42083333333333334 }, "readout_ece": 0.10858877512315908, "readout_brier": 0.2494353212266079, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.5316666666666666, "selective_accuracy": 0.40569395017793597, "selective_n": 281 }, "abstain_causes": { "score_gate": 188, "distance_gate": 131, "grammar_invalid": 0 }, "readout_ece_raw": 0.3849011408289273, "readout_ece_calibrated": 0.10858877512315908, "floor_ece": 0.22199004975124378, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": 1.1170251806701224, "within_1": 0.54, "latency_p50_ms": 0.09, "latency_p99_ms": 0.094, "latency_tail_support": 7, "latency_extremes": { "first_ms": 0.091, "max_ms": 0.098, "argmax_case": 181 }, "determinism_ok": true, "determinism_n": 10, "seconds": 3.83934575, "n_cases": 600, "n_questions": 600, "score_threshold": 0.28131148, "distance_threshold": 0.41663086, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.28131148, "status": "fitted", "targetAccuracy": 0.27142859, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.41663086, "status": "fitted", "targetAccuracy": 0.30714285, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 16.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.24 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.355 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.315 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.335 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.335 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.36 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.355 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.36 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.37 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.39 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.4 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.39666666666666667, "honest_accuracy": 0.39666666666666667, "n_pseudo": 0, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "very positive", "pred": "positive", "count": 62, "share_of_errors": 0.1712707182320442 }, { "gold": "very negative", "pred": "negative", "count": 49, "share_of_errors": 0.13535911602209943 }, { "gold": "neutral", "pred": "negative", "count": 47, "share_of_errors": 0.1298342541436464 }, { "gold": "positive", "pred": "negative", "count": 36, "share_of_errors": 0.09944751381215469 }, { "gold": "negative", "pred": "positive", "count": 28, "share_of_errors": 0.07734806629834254 }, { "gold": "neutral", "pred": "positive", "count": 28, "share_of_errors": 0.07734806629834254 }, { "gold": "very positive", "pred": "negative", "count": 24, "share_of_errors": 0.06629834254143646 }, { "gold": "negative", "pred": "neutral", "count": 17, "share_of_errors": 0.04696132596685083 }, { "gold": "positive", "pred": "very positive", "count": 17, "share_of_errors": 0.04696132596685083 }, { "gold": "very negative", "pred": "positive", "count": 11, "share_of_errors": 0.03038674033149171 }, { "gold": "very negative", "pred": "neutral", "count": 10, "share_of_errors": 0.027624309392265192 }, { "gold": "very positive", "pred": "neutral", "count": 9, "share_of_errors": 0.024861878453038673 } ], "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.16972477064220184, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.2754612743854523 }, { "mode": "max_prob", "cal_ece": 0.04647017426788808 }, { "mode": "inv_entropy", "cal_ece": 0.2754612743854523 } ] }, "corpus_digest": "fnv1a64-726f717634ca24fb", "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 600, "accuracy": 0.37166666666666665, "macro_f1": 0.32915341205482057, "ece": 0.2477898333333333, "brier": 0.8171809703500024, "nll": 1.729144921851138, "aurc": 0.5279207298357831, "mean_confidence": 0.6194564999999997, "acc_at_50_coverage": 0.45666666666666667, "acc_at_80_coverage": 0.4041666666666667 }, "readout_ece": 0.08109283333333335, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 18.0, "latency_p99_ms": 24.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 14.0, "max_ms": 26.0, "argmax_case": 19 }, "determinism_ok": true, "seconds": 14.787980458, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 600, "accuracy": 0.37166666666666665, "macro_f1": 0.32915341205482057, "ece": 0.24778999999999993, "brier": 0.8171821949666691, "nll": 1.7291473733884497, "aurc": 0.5279207298357831, "mean_confidence": 0.6194566666666664, "acc_at_50_coverage": 0.45666666666666667, "acc_at_80_coverage": 0.4041666666666667 }, "readout_ece": 0.0810926666666667, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 27.0, "latency_p99_ms": 63.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 16.0, "max_ms": 70.0, "argmax_case": 85 }, "determinism_ok": true, "seconds": 23.306305667, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": { "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 600, "accuracy": 0.37333333333333335, "macro_f1": 0.33034260369116886, "ece": 0.24612783333333332, "brier": 0.8179473067833364, "nll": 1.7314241878126222, "aurc": 0.5289789146967602, "mean_confidence": 0.6194611666666664, "acc_at_50_coverage": 0.4533333333333333, "acc_at_80_coverage": 0.4041666666666667 }, "readout_ece": 0.08072583333333337, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 23.0, "latency_p99_ms": 27.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 23.0, "max_ms": 27.0, "argmax_case": 1 }, "determinism_ok": true, "seconds": 17.972210625, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } } }, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 600, "accuracy": 0.39666666666666667, "macro_f1": 0.32101890185812104, "ece": 0.13891353500386078, "brier": 0.7604214525872166, "nll": 1.515629271034501, "aurc": 0.5364704218034313, "mean_confidence": 0.2577531316628059, "acc_at_50_coverage": 0.47, "acc_at_80_coverage": 0.42083333333333334 }, "readout_ece": 0.10858877954383692, "readout_brier": 0.24943532260349655, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.5316666666666666, "selective_accuracy": 0.40569395017793597, "selective_n": 281 }, "abstain_causes": { "score_gate": 188, "distance_gate": 131, "grammar_invalid": 0 }, "readout_ece_raw": 0.38490114152431476, "readout_ece_calibrated": 0.10858877954383692, "floor_ece": 0.22199004975124378, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": 1.1170251805707812, "within_1": 0.54, "determinism_ok": true, "determinism_n": 10, "n_cases": 600, "n_questions": 600, "score_threshold": 0.2813115, "distance_threshold": 0.41663086, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.2813115, "status": "fitted", "targetAccuracy": 0.27142859, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.41663086, "status": "fitted", "targetAccuracy": 0.30714285, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 16.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.24 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.355 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.36 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.315 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.335 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.335 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.36 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.355 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.36 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.37 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.39 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.4 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.39666666666666667, "honest_accuracy": 0.39666666666666667, "n_pseudo": 0, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "very positive", "pred": "positive", "count": 62, "share_of_errors": 0.1712707182320442 }, { "gold": "very negative", "pred": "negative", "count": 49, "share_of_errors": 0.13535911602209943 }, { "gold": "neutral", "pred": "negative", "count": 47, "share_of_errors": 0.1298342541436464 }, { "gold": "positive", "pred": "negative", "count": 36, "share_of_errors": 0.09944751381215469 }, { "gold": "negative", "pred": "positive", "count": 28, "share_of_errors": 0.07734806629834254 }, { "gold": "neutral", "pred": "positive", "count": 28, "share_of_errors": 0.07734806629834254 }, { "gold": "very positive", "pred": "negative", "count": 24, "share_of_errors": 0.06629834254143646 }, { "gold": "negative", "pred": "neutral", "count": 17, "share_of_errors": 0.04696132596685083 }, { "gold": "positive", "pred": "very positive", "count": 17, "share_of_errors": 0.04696132596685083 }, { "gold": "very negative", "pred": "positive", "count": 11, "share_of_errors": 0.03038674033149171 }, { "gold": "very negative", "pred": "neutral", "count": 10, "share_of_errors": 0.027624309392265192 }, { "gold": "very positive", "pred": "neutral", "count": 9, "share_of_errors": 0.024861878453038673 } ], "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.16972477064220184, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.27546127498149875 }, { "mode": "max_prob", "cal_ece": 0.046470174267888076 }, { "mode": "inv_entropy", "cal_ece": 0.27546127498149875 } ] }, "corpus_digest": "fnv1a64-726f717634ca24fb", "cases_digest": "fnv1a64-ad64a6cb0039e916", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 600, "accuracy": 0.37166666666666665, "macro_f1": 0.32915341205482057, "ece": 0.24779033333333325, "brier": 0.8171819570333358, "nll": 1.7291456252344222, "aurc": 0.5279207298357831, "mean_confidence": 0.6194569999999998, "acc_at_50_coverage": 0.45666666666666667, "acc_at_80_coverage": 0.4041666666666667 }, "readout_ece": 0.08109316666666669, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 12.0, "latency_p99_ms": 17.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 11.0, "max_ms": 20.0, "argmax_case": 80 }, "determinism_ok": true, "seconds": 10.109242, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 600, "accuracy": 0.235, "macro_f1": 0.16563654685925427, "ece": 0.5211544505258401, "brier": 1.2170341258480324, "nll": 3.957467651071437, "aurc": 0.75573835807425, "mean_confidence": 0.7561544505258401, "acc_at_50_coverage": 0.22666666666666666, "acc_at_80_coverage": 0.21875 }, "readout_ece": 0.46188401788473127, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 37.0, "latency_p99_ms": 78.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 305.0, "max_ms": 305.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 27.9364001, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.0, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 600, "accuracy": 0.43833333333333335, "macro_f1": 0.4179638590590876, "ece": 0.07646240903830624, "brier": 0.6840256496462226, "nll": 1.2902129384345649, "aurc": 0.4884865918387397, "mean_confidence": 0.49827637203100417, "acc_at_50_coverage": 0.5333333333333333, "acc_at_80_coverage": 0.47291666666666665 }, "readout_ece": 0.07646240903830624, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 21.0, "latency_p99_ms": 25.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 90.0, "max_ms": 90.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 23.523033, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.2270642201834863, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 600, "accuracy": 0.43833333333333335, "macro_f1": 0.33224573059683615, "ece": 0.02100397316118083, "brier": 0.675163589042319, "nll": 1.2886992397519743, "aurc": 0.49059739311584727, "mean_confidence": 0.434176149542133, "acc_at_50_coverage": 0.5066666666666667, "acc_at_80_coverage": 0.4666666666666667 }, "readout_ece": 0.02100397316118083, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 29.0, "latency_p99_ms": 35.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 79.0, "max_ms": 79.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 18.6767961, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.2270642201834863, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "a0fb83a0afc58a1ce2b9", "spec_file": "scripts/paw_specs\\sst5.txt", "spec_blake3": "03ec007949af34227f6e88bc5d008ac7c6825f6fde9f6aed344aa643493fa71f", "compile_cache_hit": true, "compile_wall_s": 11.6851636, "n_cases": 600, "n_questions": 600, "n_answered": 600, "refusals": 0, "quote_stripped": 0, "accuracy": 0.4166666666666667, "answered_accuracy": 0.4166666666666667, "refusal_rate": 0.0, "score_mae_answered": 0.86, "within_1_answered": 0.8166666666666667, "refusal_samples": [], "latency_p50_ms": 118.293, "latency_p99_ms": 210.725, "latency_tail_support": 7, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 78.9310791, "latency_quotable": null }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 600, "accuracy": 0.43333333333333335, "macro_f1": 0.41443478670275447, "ece": 0.17377723559737204, "brier": 0.6929953722067029, "nll": 1.3198731713128236, "aurc": 0.4601873382005463, "mean_confidence": 0.60707742040356, "acc_at_50_coverage": 0.5233333333333333, "acc_at_80_coverage": 0.4625 }, "readout_ece": 0.08253375576529452, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 52.0, "latency_p99_ms": 69.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 74.0, "max_ms": 98.0, "argmax_case": 1 }, "determinism_ok": true, "determinism_n": 10, "seconds": 33.6061884, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.22018348623853215, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 600, "exact": 1, "near": 0 }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "a0fb83a0afc58a1ce2b9", "spec_file": "scripts/paw_specs/sst5.txt", "spec_blake3": "03ec007949af34227f6e88bc5d008ac7c6825f6fde9f6aed344aa643493fa71f", "compile_cache_hit": true, "compile_wall_s": 0.942867667, "n_cases": 600, "n_questions": 600, "n_answered": 600, "refusals": 0, "quote_stripped": 0, "accuracy": 0.3983333333333333, "answered_accuracy": 0.3983333333333333, "refusal_rate": 0.0, "score_mae_answered": 0.8783333333333333, "within_1_answered": 0.8183333333333334, "refusal_samples": [], "latency_p50_ms": 1048.593, "latency_p99_ms": 2102.464, "latency_tail_support": 7, "server_latency_p50_ms": 76.4, "determinism_ok": true, "seconds": 663.656717083, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T21:04:53Z" } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-ad64a6cb0039e916" }, "py": { "kind": "digest", "id": "fnv1a64-ad64a6cb0039e916" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-ad64a6cb0039e916" }, "rust": { "kind": "digest", "id": "fnv1a64-ad64a6cb0039e916" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "A1", "hard": { "n": 600, "accuracy": 0.4216666666666667, "ece": 0.10076249030729133, "mean_confidence": 0.4711805712431669, "acc_at_50_coverage": 0.5, "macro_f1": 0.3645178225884921, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.20412844036697253 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.000833, "latency_p99_ms": 0.001792, "latency_tail_support": 8, "serves": "A1", "gate": "best measured, T2-uncertified (paired LB95 -0.0129) \u2014 certification is more questions, not a posture rollback", "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 600, "accuracy": 0.43, "macro_f1": 0.41049715078874643, "ece": 0.17805990467468896, "brier": 0.6935595157268799, "nll": 1.321830326901058, "aurc": 0.4612050619400724, "mean_confidence": 0.608059904674689, "acc_at_50_coverage": 0.53, "acc_at_80_coverage": 0.46458333333333335 }, "readout_ece": 0.09205816870891721, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 203.0, "latency_p99_ms": 283.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 114.0, "max_ms": 300.0, "argmax_case": 359 }, "determinism_ok": true, "seconds": 124.272431334, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "encoder": { "lane": "Rethink", "model": "ENC-t6s0 (laya-english encoder + NLEH v1 head)", "hard": { "n": 600, "accuracy": 0.5266666666666666, "acc_at_50_coverage": 0.64, "ece": 0.06626914332310359, "mean_confidence": 0.4954945766925812 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 14.708917, "latency_p99_ms": 17.452375, "serves": "\u2717 (encoder class refused at serve \u2014 A1 serves; instinct issue 014 C1)", "gate": "T2-certified above the incumbent A1 (paired LB95 +0.0535, mean +0.1050) \u2014 serve REFUSED on the latency class (14.7 ms/row vs the ~0.3 ms provisional bar; issue 014 decision 1)", "device": "metal", "record_only": true, "source_run": { "git_sha": "211c68d", "date_utc": "2026-09-30T02:01:58Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 600, "accuracy": 0.455, "macro_f1": 0.30583133122619166, "ece": 0.1911103808383147, "brier": 0.6737527244270393, "nll": 1.1754999466411433, "aurc": 0.48222009446122455, "mean_confidence": 0.639093052794536, "acc_at_50_coverage": 0.51, "acc_at_80_coverage": 0.46458333333333335 }, "readout_ece": 0.1911103808383147, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 129.0, "latency_p99_ms": 152.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 126.0, "max_ms": 166.0, "argmax_case": 452 }, "determinism_ok": true, "determinism_n": 10, "seconds": 89.262115208, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.25000000000000006, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:11:29Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "a0fb83a0afc58a1ce2b9", "spec_file": "scripts/paw_specs/sst5.txt", "spec_blake3": "03ec007949af34227f6e88bc5d008ac7c6825f6fde9f6aed344aa643493fa71f", "compile_cache_hit": true, "compile_wall_s": 0.942867667, "n_cases": 600, "n_questions": 600, "n_answered": 600, "refusals": 0, "quote_stripped": 0, "accuracy": 0.3983333333333333, "answered_accuracy": 0.3983333333333333, "refusal_rate": 0.0, "score_mae_answered": 0.8816666666666667, "within_1_answered": 0.815, "refusal_samples": [], "latency_p50_ms": 27.147, "latency_p99_ms": 35.374, "latency_tail_support": 7, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 18.01071375, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:15:55Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 600, "accuracy": 0.6033333333333334, "macro_f1": 0.5752050497694366, "ece": 0.08852866666666666, "brier": 0.5538565256499991, "nll": 0.9846725971763867, "aurc": 0.34716630317106256, "mean_confidence": 0.6810806666666661, "acc_at_50_coverage": 0.6666666666666666, "acc_at_80_coverage": 0.6291666666666667 }, "readout_ece": 0.08852866666666666, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 409.0, "latency_p99_ms": 497.0, "latency_tail_support": 7, "latency_extremes": { "first_ms": 373.0, "max_ms": 568.0, "argmax_case": 312 }, "determinism_ok": true, "determinism_n": 10, "seconds": 254.562425084, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.4541284403669726, "cases_digest": "fnv1a64-ad64a6cb0039e916", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:33:58Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 600, "accuracy": 0.57, "macro_f1": 0.5137560655946953, "ece": 0.05736454698077388, "brier": 0.5377834597652177, "nll": 0.9124600823718955, "aurc": 0.33888394116185194, "mean_confidence": 0.6220284529784216, "acc_at_50_coverage": 0.65, "acc_at_80_coverage": 0.6125 }, "readout_ece": 0.08035967002983975, "readout_brier": 0.23991928155349418, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 600, "n_questions": 600, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.4082568807339449, "cases_digest": "fnv1a64-ad64a6cb0039e916", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "prompt_injections", "n_cases": 116, "n_questions": 116, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 116, "accuracy": 0.7672413793103449, "macro_f1": 0.7642808760442538, "ece": 0.13358817981748744, "brier": 0.33647229928899053, "nll": 0.5194112460029578, "aurc": 0.09487560056959717, "mean_confidence": 0.6544157581339622, "acc_at_50_coverage": 0.896551724137931, "acc_at_80_coverage": 0.8369565217391305 }, "readout_ece": 0.09491884348721336, "readout_brier": 0.16565858443344492, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.6810344827586207, "selective_accuracy": 0.9459459459459459, "selective_n": 37 }, "abstain_causes": { "score_gate": 52, "distance_gate": 27, "grammar_invalid": 0 }, "readout_ece_raw": 0.674452191282963, "readout_ece_calibrated": 0.09491884348721336, "floor_ece": 0.3622396722430863, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.076, "latency_p99_ms": 0.085, "latency_tail_support": 2, "latency_extremes": { "first_ms": 0.084, "max_ms": 0.09, "argmax_case": 8 }, "determinism_ok": true, "determinism_n": 10, "seconds": 1.512909667, "n_cases": 116, "n_questions": 116, "score_threshold": 0.7991401, "distance_threshold": 0.46189553, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.7991401, "status": "fitted", "targetAccuracy": 0.8, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } }, "distance": { "threshold": 0.46189553, "status": "fitted", "targetAccuracy": 0.78571427, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": 1, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.5 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.5 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.61 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.7672413793103449, "honest_accuracy": 0.7672413793103449, "n_pseudo": 0, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [], "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.5178571428571429, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.6830641734600067 }, { "mode": "max_prob", "cal_ece": 0.1164907401800156 }, { "mode": "inv_entropy", "cal_ece": 0.6830641734600067 } ] }, "corpus_digest": "fnv1a64-91324bedb87d226f", "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 116, "accuracy": 0.6982758620689655, "macro_f1": 0.6750700280112045, "ece": 0.2620396551724138, "brier": 0.5259102800000001, "nll": 3.1496688651163907, "aurc": 0.10727341980747937, "mean_confidence": 0.9603155172413793, "acc_at_50_coverage": 0.9655172413793104, "acc_at_80_coverage": 0.7717391304347826 }, "readout_ece": 0.2620396551724138, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 18.0, "latency_p99_ms": 37.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 22.0, "max_ms": 38.0, "argmax_case": 37 }, "determinism_ok": true, "seconds": 6.029472583, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 116, "accuracy": 0.6982758620689655, "macro_f1": 0.6750700280112045, "ece": 0.2620387931034483, "brier": 0.5259077681034483, "nll": 3.149665690493024, "aurc": 0.10727341980747937, "mean_confidence": 0.9603146551724138, "acc_at_50_coverage": 0.9655172413793104, "acc_at_80_coverage": 0.7717391304347826 }, "readout_ece": 0.2620387931034483, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 30.0, "latency_p99_ms": 73.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 30.0, "max_ms": 84.0, "argmax_case": 84 }, "determinism_ok": true, "seconds": 10.115332209, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": { "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 107, "accuracy": 0.6915887850467289, "macro_f1": 0.6522210184182016, "ece": 0.2668280373831776, "brier": 0.5192283248598133, "nll": 2.8819367108393252, "aurc": 0.10464502378900169, "mean_confidence": 0.9584168224299067, "acc_at_50_coverage": 0.9811320754716981, "acc_at_80_coverage": 0.7764705882352941 }, "readout_ece": 0.26682803738317756, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 23.0, "latency_p99_ms": 30.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 23.0, "max_ms": 31.0, "argmax_case": 40 }, "determinism_ok": true, "seconds": 6.534780375, "n_cases": 107, "n_questions": 107, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } } }, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 116, "accuracy": 0.7672413793103449, "macro_f1": 0.7642808760442538, "ece": 0.13358817981748744, "brier": 0.33647229928899053, "nll": 0.5194112460029578, "aurc": 0.09487560056959717, "mean_confidence": 0.6544157581339622, "acc_at_50_coverage": 0.896551724137931, "acc_at_80_coverage": 0.8369565217391305 }, "readout_ece": 0.09491884348721336, "readout_brier": 0.16565858443344492, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.6810344827586207, "selective_accuracy": 0.9459459459459459, "selective_n": 37 }, "abstain_causes": { "score_gate": 52, "distance_gate": 27, "grammar_invalid": 0 }, "readout_ece_raw": 0.674452191282963, "readout_ece_calibrated": 0.09491884348721336, "floor_ece": 0.3622396722430863, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 116, "n_questions": 116, "score_threshold": 0.7991401, "distance_threshold": 0.46189553, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.7991401, "status": "fitted", "targetAccuracy": 0.8, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } }, "distance": { "threshold": 0.46189553, "status": "fitted", "targetAccuracy": 0.78571427, "support": { "n": 100, "nPass": 70, "nAbstain": 30 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": 1, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.5 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.39 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.5 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.49 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.61 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.51 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.61 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.61 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.7672413793103449, "honest_accuracy": 0.7672413793103449, "n_pseudo": 0, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [], "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.5178571428571429, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.6830641734600067 }, { "mode": "max_prob", "cal_ece": 0.1164907401800156 }, { "mode": "inv_entropy", "cal_ece": 0.6830641734600067 } ] }, "corpus_digest": "fnv1a64-91324bedb87d226f", "cases_digest": "fnv1a64-5080803ec29f2423", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 116, "accuracy": 0.6982758620689655, "macro_f1": 0.6750700280112045, "ece": 0.26203793103448275, "brier": 0.5259087506896553, "nll": 3.1496668960995784, "aurc": 0.10727341980747937, "mean_confidence": 0.9603137931034482, "acc_at_50_coverage": 0.9655172413793104, "acc_at_80_coverage": 0.7717391304347826 }, "readout_ece": 0.26203793103448275, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 12.0, "latency_p99_ms": 18.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 16.0, "max_ms": 37.0, "argmax_case": 37 }, "determinism_ok": true, "seconds": 3.6467929999999997, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 116, "accuracy": 0.5344827586206896, "macro_f1": 0.37931034482758624, "ece": 0.24657633129892678, "brier": 0.5952651188638927, "nll": 0.8098890507578412, "aurc": 0.32781787446407185, "mean_confidence": 0.7686865329742432, "acc_at_50_coverage": 0.6379310344827587, "acc_at_80_coverage": 0.5543478260869565 }, "readout_ece": 0.15592285322731939, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 41.0, "latency_p99_ms": 101.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 188.0, "max_ms": 188.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 6.3293031, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.03571428571428556, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 116, "accuracy": 0.6810344827586207, "macro_f1": 0.6529473599094364, "ece": 0.11716827177463737, "brier": 0.4537946656520319, "nll": 0.698787636455864, "aurc": 0.3443143397382513, "mean_confidence": 0.7570098172019883, "acc_at_50_coverage": 0.7586206896551724, "acc_at_80_coverage": 0.717391304347826 }, "readout_ece": 0.11716827177463737, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 21.0, "latency_p99_ms": 28.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 92.0, "max_ms": 92.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 12.552222, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.3392857142857142, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 116, "accuracy": 0.4827586206896552, "macro_f1": 0.32558139534883723, "ece": 0.38324982364629884, "brier": 0.7988302137025399, "nll": 1.1787828483921383, "aurc": 0.5450165091728463, "mean_confidence": 0.8660084443359539, "acc_at_50_coverage": 0.5, "acc_at_80_coverage": 0.4891304347826087 }, "readout_ece": 0.38324982364629884, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 33.0, "latency_p99_ms": 36.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 38.0, "max_ms": 38.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 4.6361419, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.0, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 116, "accuracy": 0.6293103448275862, "macro_f1": 0.5820695433598659, "ece": 0.21178160022125295, "brier": 0.4788486284339183, "nll": 0.6802795495321566, "aurc": 0.15917213216918957, "mean_confidence": 0.8337148148446055, "acc_at_50_coverage": 0.8620689655172413, "acc_at_80_coverage": 0.6847826086956522 }, "readout_ece": 0.21178160022125295, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 51.0, "latency_p99_ms": 71.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 70.0, "max_ms": 71.0, "argmax_case": 12 }, "determinism_ok": true, "determinism_n": 10, "seconds": 7.2874413, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.23214285714285704, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 546, "n_eval": 116, "exact": 0, "near": 2 }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-5080803ec29f2423" }, "py": { "kind": "digest", "id": "fnv1a64-5080803ec29f2423" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-5080803ec29f2423" }, "rust": { "kind": "digest", "id": "fnv1a64-5080803ec29f2423" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "A1", "hard": { "n": 116, "accuracy": 0.853448275862069, "ece": 0.0823830797754485, "mean_confidence": 0.8039373015535289, "acc_at_50_coverage": 0.9655172413793104, "macro_f1": 0.8529126575669427, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.6964285714285714 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.000417, "latency_p99_ms": 0.003, "latency_tail_support": 2, "serves": "A1", "gate": "certified (paired LB95 +0.0082, mean +0.0862)", "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 116, "accuracy": 0.6293103448275862, "macro_f1": 0.5820695433598659, "ece": 0.20458659990530076, "brier": 0.47757539131823673, "nll": 0.6793071528875683, "aurc": 0.15964234121544751, "mean_confidence": 0.833896944732887, "acc_at_50_coverage": 0.8620689655172413, "acc_at_80_coverage": 0.6847826086956522 }, "readout_ece": 0.20458659990530076, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 101.0, "latency_p99_ms": 277.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 277.0, "max_ms": 284.0, "argmax_case": 8 }, "determinism_ok": true, "seconds": 17.660868166, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "d9cfba100af1e46f97cd", "spec_file": "scripts/paw_specs/prompt_injections.injection.txt", "spec_blake3": "13ef1026c9cecf47fc9b17a084855d57125af8862018ab6892ceda48275bd51a", "compile_cache_hit": true, "compile_wall_s": 77.575057042, "n_cases": 116, "n_questions": 116, "n_answered": 116, "refusals": 0, "quote_stripped": 0, "accuracy": 0.6379310344827587, "answered_accuracy": 0.6379310344827587, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 1043.931, "latency_p99_ms": 1374.191, "latency_tail_support": 2, "server_latency_p50_ms": 76.9, "determinism_ok": true, "seconds": 133.407218042, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T16:53:09Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 116, "accuracy": 0.5086206896551724, "macro_f1": 0.3789799943646098, "ece": 0.3121914329199955, "brier": 0.6778031947429162, "nll": 0.9820978261541736, "aurc": 0.40268594067501284, "mean_confidence": 0.820812122575168, "acc_at_50_coverage": 0.5862068965517241, "acc_at_80_coverage": 0.5543478260869565 }, "readout_ece": 0.3121914329199955, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 146.0, "latency_p99_ms": 214.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 180.0, "max_ms": 216.0, "argmax_case": 37 }, "determinism_ok": true, "determinism_n": 10, "seconds": 30.62635, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.0, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:29:54Z" } }, "encoder": { "lane": "Rethink", "model": "A1", "hard": { "n": 116, "accuracy": 0.853448275862069, "ece": 0.0823830797754485, "mean_confidence": 0.8039373015535289, "acc_at_50_coverage": 0.9655172413793104 }, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.0005420000000000001, "latency_p99_ms": 0.0039169999999999995, "latency_tail_support": 2, "serves": "tier-fallback", "served_by": "Instinct (A1)", "fallback_note": "best-of-family (owner 2026-10-02): the encoder arm's own measured read is 0.8017, 0.0517 under the served arm \u2014 refused at serve, so the served answer is shown; the refused arm's full record rides the cell (displaced_record)", "derived": true, "source_run": { "git_sha": "8bbff09", "date_utc": "2026-09-28T10:13:40Z" }, "displaced_record": { "lane": "Rethink", "model": "ENC-prompt_encoder_v1 (laya-english encoder + NLEH v1 head)", "hard": { "n": 116, "accuracy": 0.8017241379310345 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.8017 (93/116 question rows) \u00b7 paired vs the incumbent A1: mean -0.0517 \u00b7 LB95 -0.1343 \u00b7 per-row p50 13.873 ms (metal device, laya-english). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017).", "device": "metal", "head_kind": "v1", "ckpt": "english", "shape_desc": "2 classes \u00b7 feat 3072", "record_only": true, "latency_quotable": true, "source_run": { "git_sha": "e0440e3", "date_utc": "2026-10-01T18:51:14Z" } } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "d9cfba100af1e46f97cd", "spec_file": "scripts/paw_specs/prompt_injections.injection.txt", "spec_blake3": "13ef1026c9cecf47fc9b17a084855d57125af8862018ab6892ceda48275bd51a", "compile_cache_hit": true, "compile_wall_s": 77.575057042, "n_cases": 116, "n_questions": 116, "n_answered": 116, "refusals": 0, "quote_stripped": 0, "accuracy": 0.6379310344827587, "answered_accuracy": 0.6379310344827587, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 24.36, "latency_p99_ms": 38.947, "latency_tail_support": 2, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 4.005738959, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:09:51Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 116, "accuracy": 0.5862068965517241, "macro_f1": 0.5166666666666666, "ece": 0.3627982758620688, "brier": 0.7202440972413787, "nll": 1.3115656635238997, "aurc": 0.17774181693710567, "mean_confidence": 0.9406706896551722, "acc_at_50_coverage": 0.8620689655172413, "acc_at_80_coverage": 0.6304347826086957 }, "readout_ece": 0.3627982758620688, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 330.0, "latency_p99_ms": 523.0, "latency_tail_support": 2, "latency_extremes": { "first_ms": 336.0, "max_ms": 864.0, "argmax_case": 8 }, "determinism_ok": true, "determinism_n": 10, "seconds": 46.17145675, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.14285714285714268, "cases_digest": "fnv1a64-5080803ec29f2423", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:14:49Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 116, "accuracy": 0.9224137931034483, "macro_f1": 0.9222693768148313, "ece": 0.05892976281315476, "brier": 0.12575097420856407, "nll": 0.2872869806599655, "aurc": 0.02121076376069987, "mean_confidence": 0.9602459462785918, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9565217391304348 }, "readout_ece": 0.05892976281315476, "readout_brier": 0.06287548710428202, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 116, "n_questions": 116, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.8392857142857143, "cases_digest": "fnv1a64-5080803ec29f2423", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "xnli_en", "n_cases": 300, "n_questions": 300, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 300, "accuracy": 0.5233333333333333, "macro_f1": 0.5061129338341462, "ece": 0.13834670235713323, "brier": 0.6178344532049044, "nll": 1.0286514608348634, "aurc": 0.3119835082997881, "mean_confidence": 0.3849866309762001, "acc_at_50_coverage": 0.64, "acc_at_80_coverage": 0.5791666666666667 }, "readout_ece": 0.11191444128751758, "readout_brier": 0.23493533027637814, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.54, "selective_accuracy": 0.6086956521739131, "selective_n": 138 }, "abstain_causes": { "score_gate": 90, "distance_gate": 72, "grammar_invalid": 0 }, "readout_ece_raw": 0.5133298230171204, "readout_ece_calibrated": 0.11191444128751758, "floor_ece": 0.14800995024875624, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.084, "latency_p99_ms": 0.088, "latency_tail_support": 4, "latency_extremes": { "first_ms": 0.089, "max_ms": 0.089, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 3.495122333, "n_cases": 300, "n_questions": 300, "score_threshold": 0.55691224, "distance_threshold": 0.51104105, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.55691224, "status": "fitted", "targetAccuracy": 0.6357143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.51104105, "status": "fitted", "targetAccuracy": 0.62142855, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "fixed-1", "selected_noul_domain": null, "selected_view": "pair", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.365 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.31 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.32 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.31 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.56 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.56 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.58 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.57 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.565 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.555 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.58 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.555 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.505 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.455 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.44 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.5033333333333333, "honest_accuracy": 0.5233333333333333, "n_pseudo": 300, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "entailment", "pred": "neutral", "count": 46, "share_of_errors": 0.32167832167832167 }, { "gold": "neutral", "pred": "contradiction", "count": 27, "share_of_errors": 0.1888111888111888 }, { "gold": "entailment", "pred": "contradiction", "count": 26, "share_of_errors": 0.18181818181818182 }, { "gold": "contradiction", "pred": "neutral", "count": 21, "share_of_errors": 0.14685314685314685 }, { "gold": "neutral", "pred": "entailment", "count": 18, "share_of_errors": 0.1258741258741259 }, { "gold": "contradiction", "pred": "entailment", "count": 5, "share_of_errors": 0.03496503496503497 } ], "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.285, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.5868099159002303 }, { "mode": "max_prob", "cal_ece": 0.21351082310080524 }, { "mode": "inv_entropy", "cal_ece": 0.5868099159002303 } ] }, "corpus_digest": "fnv1a64-ec79d4c41953d288", "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 300, "accuracy": 0.86, "macro_f1": 0.8611876188602533, "ece": 0.06853533333333349, "brier": 0.22044518360000007, "nll": 0.39874945916910204, "aurc": 0.0380700868516878, "mean_confidence": 0.920768666666667, "acc_at_50_coverage": 0.98, "acc_at_80_coverage": 0.9208333333333333 }, "readout_ece": 0.10435199999999993, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 19.0, "latency_p99_ms": 25.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 17.0, "max_ms": 27.0, "argmax_case": 13 }, "determinism_ok": true, "seconds": 9.95162, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 300, "accuracy": 0.86, "macro_f1": 0.8611876188602533, "ece": 0.06853533333333349, "brier": 0.2204451122333334, "nll": 0.39874945916910204, "aurc": 0.0380700868516878, "mean_confidence": 0.920768666666667, "acc_at_50_coverage": 0.98, "acc_at_80_coverage": 0.9208333333333333 }, "readout_ece": 0.10435099999999992, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 32.0, "latency_p99_ms": 71.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 23.0, "max_ms": 83.0, "argmax_case": 209 }, "determinism_ok": true, "seconds": 16.605432417, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": { "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 298, "accuracy": 0.8590604026845637, "macro_f1": 0.860324679891885, "ece": 0.06803288590604022, "brier": 0.22218071946308734, "nll": 0.4016795391289157, "aurc": 0.03854688249403715, "mean_confidence": 0.9204087248322141, "acc_at_50_coverage": 0.9798657718120806, "acc_at_80_coverage": 0.9159663865546218 }, "readout_ece": 0.10385805369127518, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 28.0, "latency_p99_ms": 32.0, "latency_tail_support": 3, "latency_extremes": { "first_ms": 31.0, "max_ms": 32.0, "argmax_case": 3 }, "determinism_ok": true, "seconds": 12.437068125, "n_cases": 298, "n_questions": 298, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "2f6b58c", "date_utc": "2026-09-30T10:35:35Z" } } } }, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 300, "accuracy": 0.5233333333333333, "macro_f1": 0.5061129338341462, "ece": 0.13834670225779216, "brier": 0.6178344533893235, "nll": 1.0286514608348634, "aurc": 0.3119835082997881, "mean_confidence": 0.38498663107554115, "acc_at_50_coverage": 0.64, "acc_at_80_coverage": 0.5791666666666667 }, "readout_ece": 0.11191444128751758, "readout_brier": 0.23493533027637814, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.54, "selective_accuracy": 0.6086956521739131, "selective_n": 138 }, "abstain_causes": { "score_gate": 90, "distance_gate": 72, "grammar_invalid": 0 }, "readout_ece_raw": 0.5133298230171204, "readout_ece_calibrated": 0.11191444128751758, "floor_ece": 0.14800995024875624, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 300, "n_questions": 300, "score_threshold": 0.55691224, "distance_threshold": 0.51104105, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.55691224, "status": "fitted", "targetAccuracy": 0.6357143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.51104105, "status": "fitted", "targetAccuracy": 0.62142855, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "fixed-1", "selected_noul_domain": null, "selected_view": "pair", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.365 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.31 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.34 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.32 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.31 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.325 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.33 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.56 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "pair", "cal_acc": 0.55 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.56 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.58 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.57 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.565 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "pair", "cal_acc": 0.555 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.58 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.555 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.54 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.505 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.455 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.44 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.5033333333333333, "honest_accuracy": 0.5233333333333333, "n_pseudo": 300, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "entailment", "pred": "neutral", "count": 46, "share_of_errors": 0.32167832167832167 }, { "gold": "neutral", "pred": "contradiction", "count": 27, "share_of_errors": 0.1888111888111888 }, { "gold": "entailment", "pred": "contradiction", "count": 26, "share_of_errors": 0.18181818181818182 }, { "gold": "contradiction", "pred": "neutral", "count": 21, "share_of_errors": 0.14685314685314685 }, { "gold": "neutral", "pred": "entailment", "count": 18, "share_of_errors": 0.1258741258741259 }, { "gold": "contradiction", "pred": "entailment", "count": 5, "share_of_errors": 0.03496503496503497 } ], "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.285, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.5868099159002303 }, { "mode": "max_prob", "cal_ece": 0.213510822802782 }, { "mode": "inv_entropy", "cal_ece": 0.5868099159002303 } ] }, "corpus_digest": "fnv1a64-ec79d4c41953d288", "cases_digest": "fnv1a64-b136750e5af1d412", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 300, "accuracy": 0.86, "macro_f1": 0.8611876188602533, "ece": 0.06853533333333349, "brier": 0.22044494230000009, "nll": 0.39874945916910204, "aurc": 0.0380700868516878, "mean_confidence": 0.920768666666667, "acc_at_50_coverage": 0.98, "acc_at_80_coverage": 0.9208333333333333 }, "readout_ece": 0.10435199999999992, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 13.0, "latency_p99_ms": 16.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 13.0, "max_ms": 17.0, "argmax_case": 90 }, "determinism_ok": true, "seconds": 6.2867043, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 300, "accuracy": 0.62, "macro_f1": 0.5683608058608058, "ece": 0.07387009759744005, "brier": 0.5381085066281244, "nll": 0.9764965986694375, "aurc": 0.2616815291414599, "mean_confidence": 0.6104464075962702, "acc_at_50_coverage": 0.7266666666666667, "acc_at_80_coverage": 0.6875 }, "readout_ece": 0.21147348757212361, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 43.0, "latency_p99_ms": 85.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 382.0, "max_ms": 382.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 15.8184836, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.43, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 300, "accuracy": 0.4766666666666667, "macro_f1": 0.40968489334530006, "ece": 0.05632179826268047, "brier": 0.6175729560703562, "nll": 1.025190509719224, "aurc": 0.44561388263656226, "mean_confidence": 0.46186815969647116, "acc_at_50_coverage": 0.56, "acc_at_80_coverage": 0.5166666666666667 }, "readout_ece": 0.05632179826268047, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 21.0, "latency_p99_ms": 23.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 92.0, "max_ms": 92.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 16.8306283, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.21500000000000002, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 300, "accuracy": 0.4533333333333333, "macro_f1": 0.43954330270119746, "ece": 0.14924911528825763, "brier": 0.6499242464555377, "nll": 1.0949265007405775, "aurc": 0.45766445896980285, "mean_confidence": 0.6025824486215909, "acc_at_50_coverage": 0.5666666666666667, "acc_at_80_coverage": 0.4875 }, "readout_ece": 0.14924911528825763, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 29.0, "latency_p99_ms": 35.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 81.0, "max_ms": 81.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 9.7988033, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.17999999999999997, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 300, "accuracy": 0.9, "macro_f1": 0.8999672827033166, "ece": 0.04532527506351472, "brier": 0.16604580727055215, "nll": 0.3076956385611185, "aurc": 0.023331961958249815, "mean_confidence": 0.8594962954521179, "acc_at_50_coverage": 0.9866666666666667, "acc_at_80_coverage": 0.9458333333333333 }, "readout_ece": 0.2503718494966851, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 52.0, "latency_p99_ms": 69.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 64.0, "max_ms": 75.0, "argmax_case": 289 }, "determinism_ok": true, "determinism_n": 10, "seconds": 17.5712424, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.8499999999999999, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 300, "exact": 0, "near": 0 }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-b136750e5af1d412" }, "py": { "kind": "digest", "id": "fnv1a64-b136750e5af1d412" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-b136750e5af1d412" }, "rust": { "kind": "digest", "id": "fnv1a64-b136750e5af1d412" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "A0", "hard": { "n": 300, "accuracy": 0.5233333333333333, "ece": 0.5026737833023072, "mean_confidence": 0.020659550031026205, "acc_at_50_coverage": 0.6466666666666666, "macro_f1": 0.5070059473023049, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.285 }, "consult_rate": 0.0, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": 0.087, "latency_p99_ms": 0.102, "latency_tail_support": 5, "serves": "A0", "gate": "served by the reflex half \u2014 the best measured arm on this suite", "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 300, "accuracy": 0.8966666666666666, "macro_f1": 0.8966195977749251, "ece": 0.044656349023183224, "brier": 0.1663415748887174, "nll": 0.30846237893560924, "aurc": 0.023442450106287704, "mean_confidence": 0.8592826906840006, "acc_at_50_coverage": 0.9866666666666667, "acc_at_80_coverage": 0.9458333333333333 }, "readout_ece": 0.24790328881071622, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 111.0, "latency_p99_ms": 164.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 125.0, "max_ms": 178.0, "argmax_case": 4 }, "determinism_ok": true, "seconds": 41.818258584, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "3a7be30", "date_utc": "2026-09-28T15:36:40Z" } }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "1949b801cf9746028d56", "spec_file": "scripts/paw_specs/xnli_en.relation.txt", "spec_blake3": "ead96a846bbe50430b7d9c6c635c17ecea8c59a2bd43e36210e93e7574d17c31", "compile_cache_hit": true, "compile_wall_s": 120.965073917, "n_cases": 300, "n_questions": 300, "n_answered": 300, "refusals": 0, "quote_stripped": 0, "accuracy": 0.7233333333333334, "answered_accuracy": 0.7233333333333334, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 1048.078, "latency_p99_ms": 2091.855, "latency_tail_support": 4, "server_latency_p50_ms": 96.1, "determinism_ok": true, "seconds": 338.3810435, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:02:32Z" } }, "encoder": { "lane": "Rethink", "model": "ENC-xnli_en_encoder_v1 (laya-english encoder + NLEH v1 head)", "hard": { "n": 300, "accuracy": 0.86, "ece": 0.09030396024386092, "mean_confidence": 0.947332436243693, "acc_at_50_coverage": 0.9733333333333334 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 16.513, "latency_p99_ms": 18.634, "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.8600 (258/300 rows) \u00b7 paired vs the incumbent A1: mean +0.4500 \u00b7 LB95 +0.3832 \u00b7 per-row p50 16.513 ms (metal device). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017). Reference-identified: the head ties the laya-english forward 300/300 picks (riir-train 599 T5a) \u2014 the cell is the encoder lane's own frozen read, not the trainer's lift.", "device": "metal", "record_only": true, "latency_quotable": true, "source_run": { "git_sha": "0f81b54", "date_utc": "2026-10-01T11:20:17Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 300, "accuracy": 0.86, "macro_f1": 0.858750405020872, "ece": 0.054230172932148026, "brier": 0.20355339462624883, "nll": 0.36150648236447586, "aurc": 0.04480278953978929, "mean_confidence": 0.8634677937626839, "acc_at_50_coverage": 0.96, "acc_at_80_coverage": 0.9333333333333333 }, "readout_ece": 0.054230172932148026, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 150.0, "latency_p99_ms": 179.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 161.0, "max_ms": 201.0, "argmax_case": 11 }, "determinism_ok": true, "determinism_n": 10, "seconds": 57.386010375, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.7899999999999998, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:12:30Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "1949b801cf9746028d56", "spec_file": "scripts/paw_specs/xnli_en.relation.txt", "spec_blake3": "ead96a846bbe50430b7d9c6c635c17ecea8c59a2bd43e36210e93e7574d17c31", "compile_cache_hit": true, "compile_wall_s": 120.965073917, "n_cases": 300, "n_questions": 300, "n_answered": 300, "refusals": 0, "quote_stripped": 0, "accuracy": 0.7133333333333334, "answered_accuracy": 0.7133333333333334, "refusal_rate": 0.0, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [], "latency_p50_ms": 35.877, "latency_p99_ms": 44.891, "latency_tail_support": 4, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 10.9374175, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:10:22Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 300, "accuracy": 0.8133333333333334, "macro_f1": 0.8150782669962471, "ece": 0.09094333333333339, "brier": 0.2610078611333333, "nll": 0.4771099683991806, "aurc": 0.06807677861824893, "mean_confidence": 0.7622119999999997, "acc_at_50_coverage": 0.9533333333333334, "acc_at_80_coverage": 0.9 }, "readout_ece": 0.09094333333333339, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 351.0, "latency_p99_ms": 426.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 399.0, "max_ms": 539.0, "argmax_case": 32 }, "determinism_ok": true, "determinism_n": 10, "seconds": 115.912468667, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.72, "cases_digest": "fnv1a64-b136750e5af1d412", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:17:04Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 300, "accuracy": 0.9566666666666667, "macro_f1": 0.9568105065666042, "ece": 0.0412692383139186, "brier": 0.08378799680663888, "nll": 0.16575445091689936, "aurc": 0.007702602669968632, "mean_confidence": 0.9153974283527483, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9791666666666666 }, "readout_ece": 0.08548499591989234, "readout_brier": 0.05759809975306793, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.9349999999999998, "cases_digest": "fnv1a64-b136750e5af1d412", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "massive_intent_en", "n_cases": 300, "n_questions": 300, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 300, "accuracy": 0.78, "macro_f1": 0.7698039912157559, "ece": 0.635231661486129, "brier": 0.7856433271131125, "nll": 2.059894100161934, "aurc": 0.055327402112393974, "mean_confidence": 0.14476833851387103, "acc_at_50_coverage": 0.9666666666666667, "acc_at_80_coverage": 0.8916666666666667 }, "readout_ece": 0.07900598575671515, "readout_brier": 0.10938055445024435, "raw_abstain": { "abstain_rate": 0.5133333333333333, "selective_accuracy": 0.910958904109589, "selective_n": 146 }, "calibrated_abstain": { "abstain_rate": 0.31, "selective_accuracy": 0.7681159420289855, "selective_n": 207 }, "abstain_causes": { "score_gate": 0, "distance_gate": 93, "grammar_invalid": 0 }, "readout_ece_raw": 0.635231661486129, "readout_ece_calibrated": 0.07900598575671515, "floor_ece": 0.11626865671641791, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.111, "latency_p99_ms": 0.133, "latency_tail_support": 4, "latency_extremes": { "first_ms": 0.122, "max_ms": 0.15, "argmax_case": 94 }, "determinism_ok": true, "determinism_n": 10, "seconds": 37.828576583, "n_cases": 300, "n_questions": 300, "score_threshold": 0.11058739, "distance_threshold": 0.5259382, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.11058739, "status": "fitted", "targetAccuracy": 0.545, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.5259382, "status": "fitted", "targetAccuracy": 0.55714285, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 48, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.445 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.58 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.6 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.545 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.58 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.605 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.625 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.615 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.7833333333333333, "honest_accuracy": 0.78, "n_pseudo": 300, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "music_dislikeness", "pred": "play_music", "count": 3, "share_of_errors": 0.045454545454545456 }, { "gold": "audio_volume_other", "pred": "audio_volume_up", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "datetime_convert", "pred": "datetime_query", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "music_likeness", "pred": "play_music", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "music_settings", "pred": "play_music", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "alarm_query", "pred": "alarm_set", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_down", "pred": "audio_volume_up", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_mute", "pred": "calendar_query", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_mute", "pred": "general_quirky", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_other", "pred": "alarm_set", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_other", "pred": "qa_maths", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_up", "pred": "play_audiobook", "count": 1, "share_of_errors": 0.015151515151515152 } ], "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.759124087591241, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.43910876892507067 }, { "mode": "max_prob", "cal_ece": 0.43910876892507067 }, { "mode": "inv_entropy", "cal_ece": 0.5072379374504089 } ] }, "corpus_digest": "fnv1a64-8e7cc94319af1108", "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 300, "accuracy": 0.6933333333333334, "macro_f1": 0.6936125577989888, "ece": 0.24830266666666645, "brier": 0.540529478666667, "nll": 3.5499559002853465, "aurc": 0.14436891325283494, "mean_confidence": 0.9300260000000002, "acc_at_50_coverage": 0.9133333333333333, "acc_at_80_coverage": 0.775 }, "readout_ece": 0.24406833333333328, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 31.0, "latency_p99_ms": 38.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 25.0, "max_ms": 40.0, "argmax_case": 6 }, "determinism_ok": true, "seconds": 13.264258166, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 300, "accuracy": 0.6933333333333334, "macro_f1": 0.6936125577989888, "ece": 0.24830299999999972, "brier": 0.5405290377000004, "nll": 3.5499541563128743, "aurc": 0.14436891325283494, "mean_confidence": 0.9300263333333335, "acc_at_50_coverage": 0.9133333333333333, "acc_at_80_coverage": 0.775 }, "readout_ece": 0.2440686666666666, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 48.0, "latency_p99_ms": 81.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 33.0, "max_ms": 89.0, "argmax_case": 42 }, "determinism_ok": true, "seconds": 21.45423025, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": {}, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 300, "accuracy": 0.78, "macro_f1": 0.7698039912157559, "ece": 0.6352316615109643, "brier": 0.7856433271606119, "nll": 2.0598941003679703, "aurc": 0.055327402112393974, "mean_confidence": 0.14476833848903575, "acc_at_50_coverage": 0.9666666666666667, "acc_at_80_coverage": 0.8916666666666667 }, "readout_ece": 0.07900598575671515, "readout_brier": 0.10938055445024435, "raw_abstain": { "abstain_rate": 0.5133333333333333, "selective_accuracy": 0.910958904109589, "selective_n": 146 }, "calibrated_abstain": { "abstain_rate": 0.31, "selective_accuracy": 0.7681159420289855, "selective_n": 207 }, "abstain_causes": { "score_gate": 0, "distance_gate": 93, "grammar_invalid": 0 }, "readout_ece_raw": 0.6352316615109643, "readout_ece_calibrated": 0.07900598575671515, "floor_ece": 0.11626865671641791, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 300, "n_questions": 300, "score_threshold": 0.11058739, "distance_threshold": 0.5259382, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.11058739, "status": "fitted", "targetAccuracy": 0.545, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.5259382, "status": "fitted", "targetAccuracy": 0.55714285, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 48, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 4.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.445 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.58 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.6 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.59 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.545 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.58 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.56 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.6 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.605 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.625 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.615 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.7833333333333333, "honest_accuracy": 0.78, "n_pseudo": 300, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "music_dislikeness", "pred": "play_music", "count": 3, "share_of_errors": 0.045454545454545456 }, { "gold": "audio_volume_other", "pred": "audio_volume_up", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "datetime_convert", "pred": "datetime_query", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "music_likeness", "pred": "play_music", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "music_settings", "pred": "play_music", "count": 2, "share_of_errors": 0.030303030303030304 }, { "gold": "alarm_query", "pred": "alarm_set", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_down", "pred": "audio_volume_up", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_mute", "pred": "calendar_query", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_mute", "pred": "general_quirky", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_other", "pred": "alarm_set", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_other", "pred": "qa_maths", "count": 1, "share_of_errors": 0.015151515151515152 }, { "gold": "audio_volume_up", "pred": "play_audiobook", "count": 1, "share_of_errors": 0.015151515151515152 } ], "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.759124087591241, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.4391087687760591 }, { "mode": "max_prob", "cal_ece": 0.4391087687760591 }, { "mode": "inv_entropy", "cal_ece": 0.5072379374504089 } ] }, "corpus_digest": "fnv1a64-8e7cc94319af1108", "cases_digest": "fnv1a64-49390c8b80475849", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 300, "accuracy": 0.6933333333333334, "macro_f1": 0.6936125577989888, "ece": 0.24830266666666645, "brier": 0.540529420866667, "nll": 3.549952987360154, "aurc": 0.14436891325283494, "mean_confidence": 0.9300260000000002, "acc_at_50_coverage": 0.9133333333333333, "acc_at_80_coverage": 0.775 }, "readout_ece": 0.24406899999999992, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 16.0, "latency_p99_ms": 19.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 17.0, "max_ms": 21.0, "argmax_case": 267 }, "determinism_ok": true, "seconds": 7.3354609, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 300, "accuracy": 0.23, "macro_f1": 0.21782100151124775, "ece": 0.20043704390525818, "brier": 0.9186053140645308, "nll": 2.966549632494973, "aurc": 0.6306888412086249, "mean_confidence": 0.42867544730504353, "acc_at_50_coverage": 0.30666666666666664, "acc_at_80_coverage": 0.2708333333333333 }, "readout_ece": 0.1738587213307619, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 71.0, "latency_p99_ms": 114.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 311.0, "max_ms": 311.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 19.6580532, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.15693430656934307, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 300, "accuracy": 0.7266666666666667, "macro_f1": 0.7249218986097294, "ece": 0.16077978400453152, "brier": 0.386352050736128, "nll": 0.989236671120893, "aurc": 0.07486567841510584, "mean_confidence": 0.578327801604251, "acc_at_50_coverage": 0.9533333333333334, "acc_at_80_coverage": 0.85 }, "readout_ece": 0.16077978400453152, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 26.0, "latency_p99_ms": 40.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 97.0, "max_ms": 97.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 18.3951428, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.7007299270072993, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 300, "accuracy": 0.55, "macro_f1": 0.5386535385794329, "ece": 0.22304864831268786, "brier": 0.6325827756779955, "nll": 1.559629982376817, "aurc": 0.22594612111102766, "mean_confidence": 0.3366849787781636, "acc_at_50_coverage": 0.78, "acc_at_80_coverage": 0.6291666666666667 }, "readout_ece": 0.22304864831268786, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 56.0, "latency_p99_ms": 64.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 154.0, "max_ms": 154.0, "argmax_case": 0 }, "determinism_ok": false, "determinism_n": 10, "seconds": 18.630934, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.5072992700729928, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 300, "accuracy": 0.92, "macro_f1": 0.9142354668289988, "ece": 0.04438563970228031, "brier": 0.10702258525710472, "nll": 0.28892249486105676, "aurc": 0.008952395232507604, "mean_confidence": 0.9417286148418983, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9875 }, "readout_ece": 0.038264626647573526, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 60.0, "latency_p99_ms": 76.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 228.0, "max_ms": 228.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 20.182809, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.9124087591240876, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 300, "exact": 4, "near": 17 }, "hybrid": { "lane": "Instinct", "model": "H2(\u03b2=1,nmin=2,\u03c4=4)", "hard": { "n": 300, "accuracy": 0.84, "ece": 0.06870501239437647, "mean_confidence": 0.8517494499514889, "acc_at_50_coverage": 0.9933333333333333, "macro_f1": 0.8317455255770921, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.8248175182481752 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.001083, "latency_p99_ms": 0.001958, "latency_tail_support": 4, "serves": "H2(\u03b2=1,nmin=2,\u03c4=4)", "gate": "best measured, T2-uncertified (paired LB95 -0.0039) \u2014 certification is more questions, not a posture rollback", "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-49390c8b80475849" }, "py": { "kind": "digest", "id": "fnv1a64-49390c8b80475849" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-49390c8b80475849" }, "rust": { "kind": "digest", "id": "fnv1a64-49390c8b80475849" }, "status": "same" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 300, "accuracy": 0.92, "macro_f1": 0.9142354668289988, "ece": 0.04578136064112186, "brier": 0.1068547925077969, "nll": 0.2889260038103776, "aurc": 0.009152540920493028, "mean_confidence": 0.941057560518384, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9875 }, "readout_ece": 0.038352326947816914, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 1710.0, "latency_p99_ms": 2384.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 1705.0, "max_ms": 2410.0, "argmax_case": 182 }, "determinism_ok": true, "seconds": 567.472480792, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "3a7be30", "date_utc": "2026-09-28T15:36:40Z" } }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "1b5d4b654f60b508602c", "spec_file": "scripts/paw_specs/massive_intent_en.intent.txt", "spec_blake3": "10b2865efdaaeff8129e625e18a36b141e07fd7f5fea6407c759487c31096f17", "compile_cache_hit": true, "compile_wall_s": 88.633114917, "n_cases": 300, "n_questions": 300, "n_answered": 182, "refusals": 118, "quote_stripped": 0, "accuracy": 0.5133333333333333, "answered_accuracy": 0.8461538461538461, "refusal_rate": 0.3933333333333333, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "qa_factoid", "iot_hue_lightchange", "iot_hue_sound", "ui_status", "iot_hue_lightdown", "iot_hue_lightoff", "iot_hub_power", "toaster_query" ], "latency_p50_ms": 1049.427, "latency_p99_ms": 2345.279, "latency_tail_support": 4, "server_latency_p50_ms": 91.0, "determinism_ok": false, "seconds": 344.786368584, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T20:38:17Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 300, "accuracy": 0.91, "macro_f1": 0.9056898356680406, "ece": 0.03461575806140901, "brier": 0.13930689146055505, "nll": 0.31711430182399747, "aurc": 0.011505582712503468, "mean_confidence": 0.9334321061770121, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9833333333333333 }, "readout_ece": 0.03461575806140901, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 255.0, "latency_p99_ms": 274.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 257.0, "max_ms": 508.0, "argmax_case": 217 }, "determinism_ok": true, "determinism_n": 10, "seconds": 91.283758625, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.9014598540145986, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T15:29:21Z" } }, "encoder": { "lane": "Rethink", "model": "H2(\u03b2=1,nmin=2,\u03c4=8)", "hard": { "n": 300, "accuracy": 0.8266666666666667, "ece": 0.06070700170483837, "mean_confidence": 0.8355173326634961, "acc_at_50_coverage": 0.9933333333333333 }, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.001042, "latency_p99_ms": 0.002125, "latency_tail_support": 4, "serves": "tier-fallback", "served_by": "Instinct (H2(\u03b2=1,nmin=2,\u03c4=8))", "fallback_note": "best-of-family (owner 2026-10-02): the encoder arm's own measured read is 0.6567, 0.1700 under the served arm \u2014 refused at serve, so the served answer is shown; the refused arm's full record rides the cell (displaced_record)", "derived": true, "source_run": { "git_sha": "8bbff09", "date_utc": "2026-09-28T10:13:40Z" }, "displaced_record": { "lane": "Rethink", "model": "ENC-massive_encoder_v1_probe (laya-english encoder + NLEH v1 head)", "hard": { "n": 300, "accuracy": 0.6566666666666666 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.6567 (197/300 question rows) \u00b7 paired vs the incumbent A1: mean -0.1600 \u00b7 LB95 -0.2194 \u00b7 per-row p50 31.332 ms (metal device, laya-english). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017).", "device": "metal", "head_kind": "v1", "ckpt": "english", "shape_desc": "20 classes \u00b7 feat 21504", "record_only": true, "latency_quotable": false, "source_run": { "git_sha": "e0440e3", "date_utc": "2026-10-01T17:58:51Z" } } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "1b5d4b654f60b508602c", "spec_file": "scripts/paw_specs/massive_intent_en.intent.txt", "spec_blake3": "10b2865efdaaeff8129e625e18a36b141e07fd7f5fea6407c759487c31096f17", "compile_cache_hit": true, "compile_wall_s": 88.633114917, "n_cases": 300, "n_questions": 300, "n_answered": 182, "refusals": 118, "quote_stripped": 0, "accuracy": 0.5133333333333333, "answered_accuracy": 0.8461538461538461, "refusal_rate": 0.3933333333333333, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "qa_factoid", "iot_hue_lightchange", "iot_hue_sound", "screen_query", "iot_hue_lightdown", "iot_hue_lightoff", "iot_hub_power", "toaster_query" ], "latency_p50_ms": 32.166, "latency_p99_ms": 50.58, "latency_tail_support": 4, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 11.568108375, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:11:20Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 300, "accuracy": 0.9333333333333333, "macro_f1": 0.9284468303231186, "ece": 0.036306333333333295, "brier": 0.10314332356666697, "nll": 0.2514852520445878, "aurc": 0.007203424882594562, "mean_confidence": 0.9498336666666667, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9833333333333333 }, "readout_ece": 0.036306333333333295, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 903.0, "latency_p99_ms": 1262.0, "latency_tail_support": 4, "latency_extremes": { "first_ms": 1279.0, "max_ms": 1279.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 300.746141917, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.927007299270073, "cases_digest": "fnv1a64-49390c8b80475849", "latency_quotable": true, "source_run": { "git_sha": "fec0d7c", "date_utc": "2026-10-03T22:22:48Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 300, "accuracy": 0.93, "macro_f1": 0.9268372783806814, "ece": 0.03325216180977414, "brier": 0.09811877597387934, "nll": 0.20649707517769392, "aurc": 0.005786967452330837, "mean_confidence": 0.9509246800206347, "acc_at_50_coverage": 1.0, "acc_at_80_coverage": 0.9958333333333333 }, "readout_ece": 0.03130831007363033, "readout_brier": 0.0411940360656454, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 300, "n_questions": 300, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.9233576642335767, "cases_digest": "fnv1a64-49390c8b80475849", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "banking77", "n_cases": 500, "n_questions": 500, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 500, "accuracy": 0.842, "macro_f1": 0.8362254543690752, "ece": 0.8193658229298889, "brier": 0.9687678887240521, "nll": 3.821497518480265, "aurc": 0.05987299049258608, "mean_confidence": 0.022634177070111037, "acc_at_50_coverage": 0.972, "acc_at_80_coverage": 0.9075 }, "readout_ece": 0.051719533622264856, "readout_brier": 0.11249431446039657, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.478, "selective_accuracy": 0.9348659003831418, "selective_n": 261 }, "abstain_causes": { "score_gate": 130, "distance_gate": 109, "grammar_invalid": 0 }, "readout_ece_raw": 0.8193658229298889, "readout_ece_calibrated": 0.051719533622264856, "floor_ece": 0.29331343283582084, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.357, "latency_p99_ms": 0.666, "latency_tail_support": 6, "latency_extremes": { "first_ms": 0.312, "max_ms": 0.681, "argmax_case": 72 }, "determinism_ok": true, "determinism_n": 10, "seconds": 50.875362959, "n_cases": 500, "n_questions": 500, "score_threshold": 0.74778104, "distance_threshold": 0.55494976, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.74778104, "status": "fitted", "targetAccuracy": 0.9, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.55494976, "status": "fitted", "targetAccuracy": 0.8142857, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 40, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.465 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.81 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.8 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.795 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.775 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.77 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.76 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.76 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.81 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.845 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.855 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.855 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.845 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.825 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.836, "honest_accuracy": 0.842, "n_pseudo": 500, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "exchange charge", "pred": "exchange rate", "count": 3, "share_of_errors": 0.0379746835443038 }, { "gold": "card delivery estimate", "pred": "activate my card", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "card delivery estimate", "pred": "card arrival", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "card delivery estimate", "pred": "transfer timing", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "contactless not working", "pred": "declined transfer", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "fiat currency support", "pred": "supported cards and currencies", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "pending top up", "pred": "top up reverted", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "top up reverted", "pred": "Refund not showing up", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "wrong exchange rate for cash withdrawal", "pred": "wrong amount of cash received", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "balance not updated after bank transfer", "pred": "transfer timing", "count": 1, "share_of_errors": 0.012658227848101266 }, { "gold": "beneficiary not allowed", "pred": "declined transfer", "count": 1, "share_of_errors": 0.012658227848101266 }, { "gold": "card acceptance", "pred": "atm support", "count": 1, "share_of_errors": 0.012658227848101266 } ], "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.8397565922920892, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.7729256937466562 }, { "mode": "max_prob", "cal_ece": 0.7729256937466562 }, { "mode": "inv_entropy", "cal_ece": 0.7924418762326241 } ] }, "corpus_digest": "fnv1a64-07e3c974b5ae1bda", "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 500, "accuracy": 0.422, "macro_f1": 0.37652146392023184, "ece": 0.38134340000000017, "brier": 0.9326880786800371, "nll": 6.121876279293705, "aurc": 0.4066392903778559, "mean_confidence": 0.7951093999999999, "acc_at_50_coverage": 0.6, "acc_at_80_coverage": 0.475 }, "readout_ece": 0.3940097999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 48.0, "latency_p99_ms": 58.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 37.0, "max_ms": 66.0, "argmax_case": 4 }, "determinism_ok": true, "seconds": 28.820647875, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true }, "py/english": { "lane": "laya (python)", "model": "english", "hard": { "n": 500, "accuracy": 0.422, "macro_f1": 0.37652146392023184, "ece": 0.3813438000000001, "brier": 0.9326873171600369, "nll": 6.121874967045045, "aurc": 0.4066392903778559, "mean_confidence": 0.7951097999999999, "acc_at_50_coverage": 0.6, "acc_at_80_coverage": 0.475 }, "readout_ece": 0.3940095999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 76.0, "latency_p99_ms": 110.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 52.0, "max_ms": 172.0, "argmax_case": 120 }, "determinism_ok": true, "seconds": 46.196261833, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true } }, "extra_host_lanes": { "m3-max-ane": {}, "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 500, "accuracy": 0.842, "macro_f1": 0.8362254543690752, "ece": 0.8193658229261637, "brier": 0.9687678887201593, "nll": 3.821497518287094, "aurc": 0.05987299049258608, "mean_confidence": 0.022634177073836328, "acc_at_50_coverage": 0.972, "acc_at_80_coverage": 0.9075 }, "readout_ece": 0.05171953397989272, "readout_brier": 0.11249431439803001, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.478, "selective_accuracy": 0.9348659003831418, "selective_n": 261 }, "abstain_causes": { "score_gate": 130, "distance_gate": 109, "grammar_invalid": 0 }, "readout_ece_raw": 0.8193658229261637, "readout_ece_calibrated": 0.05171953397989272, "floor_ece": 0.29331343283582084, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 500, "n_questions": 500, "score_threshold": 0.74778104, "distance_threshold": 0.55494976, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.74778104, "status": "fitted", "targetAccuracy": 0.9, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.55494976, "status": "fitted", "targetAccuracy": 0.8142857, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 40, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.465 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.81 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.805 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.8 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.795 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.775 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.77 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.76 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.76 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.81 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.845 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.855 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.855 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.845 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.825 } ] }, "genome_selection": null, "transductive": { "accuracy": 0.836, "honest_accuracy": 0.842, "n_pseudo": 500, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "exchange charge", "pred": "exchange rate", "count": 3, "share_of_errors": 0.0379746835443038 }, { "gold": "card delivery estimate", "pred": "activate my card", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "card delivery estimate", "pred": "card arrival", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "card delivery estimate", "pred": "transfer timing", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "contactless not working", "pred": "declined transfer", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "fiat currency support", "pred": "supported cards and currencies", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "pending top up", "pred": "top up reverted", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "top up reverted", "pred": "Refund not showing up", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "wrong exchange rate for cash withdrawal", "pred": "wrong amount of cash received", "count": 2, "share_of_errors": 0.02531645569620253 }, { "gold": "balance not updated after bank transfer", "pred": "transfer timing", "count": 1, "share_of_errors": 0.012658227848101266 }, { "gold": "beneficiary not allowed", "pred": "declined transfer", "count": 1, "share_of_errors": 0.012658227848101266 }, { "gold": "card acceptance", "pred": "atm support", "count": 1, "share_of_errors": 0.012658227848101266 } ], "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.8397565922920892, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.7729256937466562 }, { "mode": "max_prob", "cal_ece": 0.7729256937466562 }, { "mode": "inv_entropy", "cal_ece": 0.7924418762326241 } ] }, "corpus_digest": "fnv1a64-07e3c974b5ae1bda", "cases_digest": "fnv1a64-dd8ab35333abb82a", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 500, "accuracy": 0.422, "macro_f1": 0.37652146392023184, "ece": 0.38134320000000016, "brier": 0.9326872987800371, "nll": 6.1218761064175515, "aurc": 0.4066392903778559, "mean_confidence": 0.7951087999999998, "acc_at_50_coverage": 0.6, "acc_at_80_coverage": 0.475 }, "readout_ece": 0.3940095999999995, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 32.0, "latency_p99_ms": 47.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 26.0, "max_ms": 48.0, "argmax_case": 290 }, "determinism_ok": true, "seconds": 19.9027001, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": null, "source_run": { "git_sha": "97c2b1f", "date_utc": "2026-09-28T07:13:04Z" } } }, "clm": { "lane": "clm (reference)", "model": "clm-latest", "hard": { "n": 500, "accuracy": 0.02, "macro_f1": 0.007109419470566462, "ece": 0.24596123208105564, "brier": 1.0819508418215247, "nll": 5.795338553297388, "aurc": 0.9663799479334131, "mean_confidence": 0.26498144306242466, "acc_at_50_coverage": 0.016, "acc_at_80_coverage": 0.02 }, "readout_ece": 0.2363028274029493, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 28.0, "latency_p99_ms": 228.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 494.0, "max_ms": 494.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 20.3961333, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.006085192697768763, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:47:55Z" } }, "gliner": { "lane": "gliner (reference)", "model": "fastino/GLiNER2.5-Decide", "hard": { "n": 500, "accuracy": 0.71, "macro_f1": 0.6953814027600415, "ece": 0.09249225624340321, "brier": 0.4388267952845511, "nll": 1.3199787805005347, "aurc": 0.11985960849314305, "mean_confidence": 0.7818954451636425, "acc_at_50_coverage": 0.904, "acc_at_80_coverage": 0.7975 }, "readout_ece": 0.09249225624340321, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 23.0, "latency_p99_ms": 30.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 98.0, "max_ms": 98.0, "argmax_case": 0 }, "determinism_ok": true, "determinism_n": 10, "seconds": 21.944307, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.7058823529411764, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:55:50Z" } }, "agentjev": { "lane": "agentjev (reference)", "model": "AgentJev-0.6B@68883998", "hard": { "n": 500, "accuracy": 0.552, "macro_f1": 0.5365576159059401, "ece": 0.34275407325476404, "brier": 0.7471535546351183, "nll": 2.2722377994844196, "aurc": 0.24047110198673846, "mean_confidence": 0.20924592674523593, "acc_at_50_coverage": 0.776, "acc_at_80_coverage": 0.6375 }, "readout_ece": 0.34275407325476404, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 134.0, "latency_p99_ms": 226.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 247.0, "max_ms": 253.0, "argmax_case": 375 }, "determinism_ok": false, "determinism_n": 10, "seconds": 73.6444655, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.5456389452332657, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T18:08:47Z" } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "ebbfe5a01ae044052526", "spec_file": "scripts/paw_specs\\banking77.txt", "spec_blake3": "387c823cb86efe63d4cbba2e8b299f213f5a2487f3b7b049fe07315348ff2d43", "compile_cache_hit": true, "compile_wall_s": 11.9880512, "n_cases": 500, "n_questions": 500, "n_answered": 323, "refusals": 177, "quote_stripped": 0, "accuracy": 0.412, "answered_accuracy": 0.6377708978328174, "refusal_rate": 0.354, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "locate my card", "wrong exchange rate for foreign exchange", "top up by day", "exchange currency", "contactless", "pending gas", "cancel transaction", "top up limit" ], "latency_p50_ms": 131.696, "latency_p99_ms": 236.735, "latency_tail_support": 6, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 72.0437115, "latency_quotable": null }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 500, "accuracy": 0.654, "macro_f1": 0.6330925363531608, "ece": 0.08950261621177197, "brier": 0.5155436575799948, "nll": 1.9273258252069845, "aurc": 0.16820807721410694, "mean_confidence": 0.5716962376385927, "acc_at_50_coverage": 0.884, "acc_at_80_coverage": 0.74 }, "readout_ece": 0.11409038517030952, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 114.0, "latency_p99_ms": 127.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 122.0, "max_ms": 132.0, "argmax_case": 122 }, "determinism_ok": true, "determinism_n": 10, "seconds": 60.5626093, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.6490872210953347, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "leak": { "threshold": 0.8, "n_reference": 4000, "n_eval": 500, "exact": 0, "near": 17 }, "paw": { "lane": "paw (hosted)", "model": "paw-ft-bs48-20260530", "posture": "hosted-anonymous", "program_id": "ebbfe5a01ae044052526", "spec_file": "scripts/paw_specs/banking77.txt", "spec_blake3": "387c823cb86efe63d4cbba2e8b299f213f5a2487f3b7b049fe07315348ff2d43", "compile_cache_hit": true, "compile_wall_s": 0.975116792, "n_cases": 500, "n_questions": 500, "n_answered": 327, "refusals": 173, "quote_stripped": 0, "accuracy": 0.418, "answered_accuracy": 0.6391437308868502, "refusal_rate": 0.346, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "locate my card", "wrong exchange rate for foreign exchange", "request to hold funds", "top up by day", "exchange currency", "contactless", "pending gas", "cancel transaction" ], "latency_p50_ms": 1059.219, "latency_p99_ms": 1570.816, "latency_tail_support": 6, "server_latency_p50_ms": 112.2, "determinism_ok": true, "seconds": 566.16626575, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "e45c828", "date_utc": "2026-10-03T21:15:17Z" } }, "pairing": { "rust_vs_py": { "rust": { "kind": "digest", "id": "fnv1a64-dd8ab35333abb82a" }, "py": { "kind": "digest", "id": "fnv1a64-dd8ab35333abb82a" }, "status": "same" }, "km_vs_laya": { "modelless": { "kind": "digest", "id": "fnv1a64-dd8ab35333abb82a" }, "rust": { "kind": "digest", "id": "fnv1a64-dd8ab35333abb82a" }, "status": "same" } }, "hybrid": { "lane": "Instinct", "model": "H2(\u03b2=2,nmin=8,\u03c4=8)", "hard": { "n": 500, "accuracy": 0.854, "ece": 0.08412793359878214, "mean_confidence": 0.9374190933679326, "acc_at_50_coverage": 0.996, "macro_f1": 0.8527493620885521, "jdi_chance": 0.014, "jdi_skill": 0.8519269776876267 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.007292, "latency_p99_ms": 0.026709, "latency_tail_support": 6, "serves": "H2(\u03b2=2,nmin=8,\u03c4=8)", "gate": "best measured, T2-uncertified (paired LB95 -0.0013) \u2014 certification is more questions, not a posture rollback", "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "eef77d1", "date_utc": "2026-10-02T10:01:40Z" } }, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 500, "accuracy": 0.656, "macro_f1": 0.6355435167453176, "ece": 0.09753488361835477, "brier": 0.515381702952365, "nll": 1.925635515579029, "aurc": 0.16795993855145483, "mean_confidence": 0.5719192961454391, "acc_at_50_coverage": 0.884, "acc_at_80_coverage": 0.74 }, "readout_ece": 0.12044315707662828, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 3056.0, "latency_p99_ms": 4084.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 4535.0, "max_ms": 4535.0, "argmax_case": 0 }, "determinism_ok": true, "seconds": 1650.295442791, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "00bddd1", "date_utc": "2026-09-28T16:27:36Z" } }, "bekko": { "lane": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "hard": { "n": 500, "accuracy": 0.792, "macro_f1": 0.7805128981026606, "ece": 0.04694363492727279, "brier": 0.30066090506936505, "nll": 0.765191381825856, "aurc": 0.05976304691192087, "mean_confidence": 0.7859104135632515, "acc_at_50_coverage": 0.98, "acc_at_80_coverage": 0.87 }, "readout_ece": 0.04694363492727279, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 645.0, "latency_p99_ms": 696.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 630.0, "max_ms": 705.0, "argmax_case": 297 }, "determinism_ok": true, "determinism_n": 10, "seconds": 341.598648083, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.7890466531440162, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "e8328de", "date_utc": "2026-10-03T16:34:28Z" } }, "encoder": { "lane": "Rethink", "model": "H2(\u03b2=2,nmin=8,\u03c4=8)", "hard": { "n": 500, "accuracy": 0.854, "ece": 0.08412793359878214, "mean_confidence": 0.9374190933679326, "acc_at_50_coverage": 0.996 }, "latency_scope": "arm-only", "latency_rows": "questions", "latency_p50_ms": 0.008541, "latency_p99_ms": 0.06775, "latency_tail_support": 6, "serves": "tier-fallback", "served_by": "Instinct (H2(\u03b2=2,nmin=8,\u03c4=8))", "fallback_note": "best-of-family (owner 2026-10-02): the encoder arm's own measured read is 0.4420, 0.4120 under the served arm \u2014 refused at serve, so the served answer is shown; the refused arm's full record rides the cell (displaced_record)", "derived": true, "source_run": { "git_sha": "8bbff09", "date_utc": "2026-09-28T10:13:40Z" }, "displaced_record": { "lane": "Rethink", "model": "ENC-banking77_encoder_v1 (laya-english encoder + NLEH v1 head)", "hard": { "n": 500, "accuracy": 0.442 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "gate": "acc 0.4420 (221/500 question rows) \u00b7 paired vs the incumbent A1: mean -0.3860 \u00b7 LB95 -0.4363 \u00b7 per-row p50 42.983 ms (metal device, laya-english). Serve REFUSED (014 class-wide latency class); record-only cell, seated per the owner call 2026-10-01 (instinct issue 017).", "device": "metal", "head_kind": "v1", "ckpt": "english", "shape_desc": "77 classes \u00b7 feat 79872", "record_only": true, "latency_quotable": true, "source_run": { "git_sha": "e0440e3", "date_utc": "2026-10-01T18:50:28Z" } } }, "paw_local": { "lane": "paw (local)", "model": "paw-ft-bs48-20260530", "posture": "local-subprocess", "program_id": "ebbfe5a01ae044052526", "spec_file": "scripts/paw_specs/banking77.txt", "spec_blake3": "387c823cb86efe63d4cbba2e8b299f213f5a2487f3b7b049fe07315348ff2d43", "compile_cache_hit": true, "compile_wall_s": 0.975116792, "n_cases": 500, "n_questions": 500, "n_answered": 323, "refusals": 177, "quote_stripped": 0, "accuracy": 0.418, "answered_accuracy": 0.6470588235294118, "refusal_rate": 0.354, "score_mae_answered": null, "within_1_answered": null, "refusal_samples": [ "locate my card", "wrong exchange rate for foreign exchange", "top up by day", "exchange currency", "contactless", "pending gas", "cancel transaction", "top up limit" ], "latency_p50_ms": 38.986, "latency_p99_ms": 60.087, "latency_tail_support": 6, "server_latency_p50_ms": null, "determinism_ok": true, "seconds": 20.905661667, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "0dc2c20", "date_utc": "2026-10-03T17:17:16Z" } }, "clef": { "lane": "clef (local)", "model": "clef-flash-4bit", "hard": { "n": 500, "accuracy": 0.954, "macro_f1": 0.9536241896631502, "ece": 0.034862000000000185, "brier": 0.07688410772000609, "nll": 0.22256338872472112, "aurc": 0.010036258534506482, "mean_confidence": 0.9304940000000004, "acc_at_50_coverage": 0.988, "acc_at_80_coverage": 0.985 }, "readout_ece": 0.034862000000000185, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 3368.0, "latency_p99_ms": 4371.0, "latency_tail_support": 6, "latency_extremes": { "first_ms": 2331.0, "max_ms": 4632.0, "argmax_case": 126 }, "determinism_ok": true, "determinism_n": 10, "seconds": 1775.382709042, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.9533468559837728, "cases_digest": "fnv1a64-dd8ab35333abb82a", "latency_quotable": true, "source_run": { "git_sha": "91dd0cb", "date_utc": "2026-10-03T20:05:58Z" } }, "pplx": { "lane": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "hard": { "n": 500, "accuracy": 0.804, "macro_f1": 0.7897567672968737, "ece": 0.06527229835170956, "brier": 0.30025348438374594, "nll": 0.886750909518558, "aurc": 0.06106060322176921, "mean_confidence": 0.8559988388277652, "acc_at_50_coverage": 0.952, "acc_at_80_coverage": 0.89 }, "readout_ece": 0.058423230572243634, "readout_brier": 0.11893338446597215, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 500, "n_questions": 500, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.014, "jdi_skill": 0.8012170385395538, "cases_digest": "fnv1a64-dd8ab35333abb82a", "source_run": { "git_sha": "0413dcd", "date_utc": "2026-10-08T21:33:44Z" } } }, { "name": "thai_wisesight", "n_questions": 400, "n_cases": 400, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.475, "macro_f1": 0.4011828455419324, "ece": 0.36124100334942344, "brier": 0.841119386323466, "nll": 1.7111618732338834, "aurc": 0.4940801666417539, "mean_confidence": 0.8349697407335043, "acc_at_50_coverage": 0.52, "acc_at_80_coverage": 0.484375 }, "readout_ece": 0.25726806574518457, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 107.0, "latency_p99_ms": 393.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 119.0, "max_ms": 946.0, "argmax_case": 123 }, "determinism_ok": true, "seconds": 46.735881167, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-fb7285428b60cfa1", "latency_quotable": true, "source_run": { "git_sha": "3a7be30", "date_utc": "2026-09-28T15:36:40Z" } }, "extra_host_lanes": { "4090-win": { "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 400, "accuracy": 0.4675, "macro_f1": 0.39621142694817335, "ece": 0.36819564446806907, "brier": 0.8414794855041283, "nll": 1.713083528154888, "aurc": 0.49315841829612755, "mean_confidence": 0.8356956444680691, "acc_at_50_coverage": 0.515, "acc_at_80_coverage": 0.484375 }, "readout_ece": 0.26103172260204727, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 52.0, "latency_p99_ms": 73.0, "latency_tail_support": 5, "latency_extremes": { "first_ms": 59.0, "max_ms": 100.0, "argmax_case": 123 }, "determinism_ok": true, "determinism_n": 10, "seconds": 22.8172719, "n_cases": 400, "n_questions": 400, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.545, "jdi_skill": 0.0, "cases_digest": "fnv1a64-fb7285428b60cfa1", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "encoder": { "lane": "Rethink", "model": "ENC-ref (laya-multilingual encoder, reference logits \u2014 no head earned)", "hard": { "n": 400, "accuracy": 0.4075 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "serves": "\u2717 (record-only; no head earned and the suite is unsold in every instinct lane)", "gate": "acc 0.4075 (163/400 rows) \u2014 the laya-multilingual REFERENCE read (untrained logits, gold-only eval, class-relative gate REFUSED the head: holdout 0.4213 < ref holdout 0.4550). Below the openthai comparison lane's 0.475 \u2014 the multilingual encoder is not yet competitive here. Train-side protocol (case wire mirrors reflex build_thai_wisesight verbatim; 4000-row train pool, 800 holdout, fresh NLEH fit); encode wall 6.3 s / 400 rows (M3 metal, loaded box).", "device": "metal", "record_only": true, "latency_quotable": false, "source_run": { "git_sha": "e0440e3", "date_utc": "2026-10-01T19:20:00Z" } } }, { "name": "thai_sib200", "n_questions": 204, "n_cases": 204, "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 204, "accuracy": 0.8382352941176471, "macro_f1": 0.8178733804911904, "ece": 0.06707383619219648, "brier": 0.2748790805227198, "nll": 0.5618465140813915, "aurc": 0.07182042533936529, "mean_confidence": 0.848810897592236, "acc_at_50_coverage": 0.9411764705882353, "acc_at_80_coverage": 0.8773006134969326 }, "readout_ece": 0.09746096452409891, "raw_abstain": null, "calibrated_abstain": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 112.0, "latency_p99_ms": 178.0, "latency_tail_support": 3, "latency_extremes": { "first_ms": 116.0, "max_ms": 227.0, "argmax_case": 102 }, "determinism_ok": true, "seconds": 27.023358125, "n_cases": 204, "n_questions": 204, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "cases_digest": "fnv1a64-b6e532604b031e8d", "latency_quotable": true, "source_run": { "git_sha": "3a7be30", "date_utc": "2026-09-28T15:36:40Z" } }, "extra_host_lanes": { "4090-win": { "openthai": { "lane": "openthai (reference)", "model": "openthai-systemone", "hard": { "n": 204, "accuracy": 0.8382352941176471, "macro_f1": 0.8178733804911904, "ece": 0.06511045860893587, "brier": 0.27483704765782135, "nll": 0.5621628036142624, "aurc": 0.07176961221068476, "mean_confidence": 0.8493531900001507, "acc_at_50_coverage": 0.9411764705882353, "acc_at_80_coverage": 0.8773006134969326 }, "readout_ece": 0.09236668563918003, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 54.0, "latency_p99_ms": 69.0, "latency_tail_support": 3, "latency_extremes": { "first_ms": 57.0, "max_ms": 72.0, "argmax_case": 118 }, "determinism_ok": true, "determinism_n": 10, "seconds": 12.4727798, "n_cases": 204, "n_questions": 204, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.25, "jdi_skill": 0.7843137254901961, "cases_digest": "fnv1a64-b6e532604b031e8d", "latency_quotable": true, "source_run": { "git_sha": "5a52ff9", "date_utc": "2026-10-04T17:23:40Z" } } } }, "encoder": { "lane": "Rethink", "model": "ENC-ref (laya-multilingual encoder, reference logits \u2014 no head earned)", "hard": { "n": 204, "accuracy": 0.7843137254901961 }, "consult_rate": 1.0, "latency_scope": "arm-only", "latency_rows": "questions", "serves": "\u2717 (record-only; no head earned and the suite is unsold in every instinct lane)", "gate": "acc 0.7843 (160/204 rows) \u2014 the laya-multilingual REFERENCE read (untrained logits, gold-only eval, class-relative gate REFUSED the head: holdout 0.6800 < ref holdout 0.7200). Below the openthai comparison lane's 0.838. Train-side protocol (case wire mirrors reflex build_thai_sib200 verbatim; 701-row train pool, 200 holdout, fresh NLEH fit); encode wall 3.6 s / 204 rows (M3 metal, loaded box).", "device": "metal", "record_only": true, "latency_quotable": false, "source_run": { "git_sha": "e0440e3", "date_utc": "2026-10-01T19:20:00Z" } } }, { "name": "s1mb_choice", "n_questions": 4329, "n_cases": 4329, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 4329, "accuracy": 0.22961422961422961, "macro_f1": 0.05728477985464882, "ece": 0.04368157599910449, "brier": 0.7720725868820079, "nll": 1.843002186277205, "aurc": 0.6642965880999749, "mean_confidence": 0.24446859121964598, "acc_at_50_coverage": 0.3202402957486137, "acc_at_80_coverage": 0.2682645105399942 }, "readout_ece": 0.20603674965280053, "readout_brier": 0.22520657170020122, "raw_abstain": { "abstain_rate": 0.7803187803187803, "selective_accuracy": 0.138801261829653, "selective_n": 951 }, "calibrated_abstain": { "abstain_rate": 0.7803187803187803, "selective_accuracy": 0.138801261829653, "selective_n": 951 }, "abstain_causes": { "score_gate": 2686, "distance_gate": 692, "grammar_invalid": 0 }, "readout_ece_raw": 0.20603674965280053, "readout_ece_calibrated": 0.20603674965280053, "floor_ece": 0.40613977927410666, "g1_pass": false, "g1_verdict": "fail", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.373, "latency_p99_ms": 1.819, "latency_tail_support": 44, "latency_extremes": { "first_ms": 0.432, "max_ms": 9.735, "argmax_case": 351 }, "determinism_ok": true, "determinism_n": 10, "seconds": 558.808362584, "n_cases": 4329, "n_questions": 4329, "score_threshold": 0.010039693, "distance_threshold": 0.5756306, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.010039693, "status": "fitted", "targetAccuracy": 0.15, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.5756306, "status": "fitted", "targetAccuracy": 0.16428572, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 16, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.155 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.185 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.18 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.18 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.17 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.165 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.16 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.16 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.155 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.17 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.19 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.195 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.185 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.195 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "not_mentioned", "pred": "contradiction", "count": 312, "share_of_errors": 0.09355322338830585 }, { "gold": "entailment", "pred": "contradiction", "count": 257, "share_of_errors": 0.07706146926536732 }, { "gold": "not_mentioned", "pred": "entailment", "count": 108, "share_of_errors": 0.032383808095952024 }, { "gold": "entailment", "pred": "neutral", "count": 80, "share_of_errors": 0.0239880059970015 }, { "gold": "contradiction", "pred": "entailment", "count": 61, "share_of_errors": 0.018290854572713643 }, { "gold": "contradiction", "pred": "neutral", "count": 53, "share_of_errors": 0.015892053973013492 }, { "gold": "option_1", "pred": "option_0", "count": 53, "share_of_errors": 0.015892053973013492 }, { "gold": "neutral", "pred": "entailment", "count": 46, "share_of_errors": 0.013793103448275862 }, { "gold": "option_2", "pred": "option_3", "count": 26, "share_of_errors": 0.007796101949025487 }, { "gold": "A", "pred": "B", "count": 25, "share_of_errors": 0.0074962518740629685 }, { "gold": "2", "pred": "3", "count": 24, "share_of_errors": 0.00719640179910045 }, { "gold": "option_0", "pred": "option_1", "count": 22, "share_of_errors": 0.006596701649175412 } ], "pair_head_ab": null, "jdi_chance": 0.23331023331023332, "jdi_skill": 0.0, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.13536291792988778 }, { "mode": "max_prob", "cal_ece": 0.09499485112726685 }, { "mode": "inv_entropy", "cal_ece": 0.15231328934431076 } ] }, "corpus_fallbacks": [ "getting_spare_card", "verify_my_identity", "virtual_card_not_working", "disposable_card_limits", "reverted_card_payment?", "pending_cash_withdrawal", "get_physical_card", "lost_or_stolen_card", "top_up_by_card_charge", "extra_charge_on_statement", "automatic_top_up", "get_disposable_virtual_card", "transfer_not_received_by_recipient", "request_refund", "pending_card_payment", "declined_transfer", "age_limit", "card_payment_fee_charged", "verify_source_of_funds", "declined_card_payment", "card_payment_wrong_exchange_rate", "wrong_amount_of_cash_received", "atm_support", "transaction_charged_twice", "wrong_exchange_rate_for_cash_withdrawal", "supported_cards_and_currencies", "balance_not_updated_after_bank_transfer", "card_swallowed", "failed_transfer", "cancel_transfer", "direct_debit_payment_not_recognised", "top_up_limits", "exchange_via_app", "passcode_forgotten", "top_up_by_cash_or_cheque", "top_up_by_bank_transfer_charge", "beneficiary_not_allowed", "compromised_card", "unable_to_verify_identity", "card_not_working", "exchange_rate", "card_payment_not_recognised", "transfer_into_account", "cash_withdrawal_charge", "card_arrival", "card_acceptance", "activate_my_card", "transfer_timing", "terminate_account", "topping_up_by_card", "cash_withdrawal_not_recognised", "getting_virtual_card", "country_support", "Refund_not_showing_up", "balance_not_updated_after_cheque_or_cash_deposit", "change_pin", "pending_transfer", "edit_personal_details", "verify_top_up", "pending_top_up", "why_verify_identity", "card_about_to_expire", "exchange_charge", "transfer_fee_charged", "top_up_failed", "fiat_currency_support", "card_delivery_estimate", "rewards_balance", "translate", "cancel", "maybe", "restaurant_reviews", "repeat", "todo_list", "flight_status", "last_maintenance", "pto_balance", "directions", "restaurant_reservation", "confirm_reservation", "order_status", "tell_joke", "oos", "calculator", "gas_type", "travel_suggestion", "share_location", "shopping_list", "change_language", "timezone", "yes", "book_flight", "payday", "no", "meeting_schedule", "do_you_have_pets", "order", "credit_limit_change", "recipe", "calendar_update", "improve_credit_score", "pin_change", "update_playlist", "cancel_reservation", "date", "measurement_conversion", "definition", "oil_change_how", "whisper_mode", "flip_coin", "find_phone", "todo_list_update", "redeem_rewards", "change_user_name", "change_ai_name", "min_payment", "spelling", "how_busy", "change_accent", "schedule_meeting", "food_last", "international_fees", "book_hotel", "where_are_you_from", "change_volume", "are_you_a_bot", "tire_change", "ingredients_list", "current_location", "goodbye", "insurance_change", "w2", "pto_used", "new_card", "traffic", "balance", "fun_fact", "ingredient_substitution", "transfer", "freeze_account", "schedule_maintenance", "thank_you", "what_is_your_name", "gas", "vaccines", "accept_reservations", "transactions", "cook_time", "distance", "routing", "report_lost_card", "next_song", "carry_on", "what_song", "travel_notification", "apr", "nutrition_info", "who_made_you", "next_holiday", "bill_balance", "reminder_update", "how_old_are_you", "mpg", "rollover_401k", "account_blocked", "plug_type", "card_declined", "smart_home", "meal_suggestion", "direct_deposit", "sync_device", "pay_bill", "uber", "pto_request_status", "taxes", "order_checks", "greeting", "income", "spending_history", "calories", "reset_settings", "expiration_date", "tire_pressure", "meaning_of_life", "make_call", "insurance", "restaurant_suggestion", "interest_rate", "credit_limit", "bill_due", "who_do_you_work_for", "report_fraud", "roll_dice", "change_speed", "damaged_card", "shopping_list_update", "what_are_your_hobbies", "lost_luggage", "oil_change_when", "text", "travel_alert", "car_rental", "international_visa", "application_status", "jump_start", "credit_score", "what_can_i_ask_you", "replacement_card_duration", "pto_request", "national_id", "company_name", "account_number", "password", "url", "-47", "-4", "14", "34", "24", "5760", "5756", "5750", "5765", "33", "-2", "8", "12", "29", "16", "52", "-5", "17", "72", "80", "78", "103", "98", "48", "200", "150", "180", "196", "64", "114", "59", "84", "38", "63", "43", "32", "21", "20", "-18", "27", "149", "197", "172", "147", "88", "93", "-1", "-3", "350", "397", "400", "375", "248", "254", "255", "251", "6580", "6598", "6600", "6593", "293", "305", "288", "13997", "14000", "13993", "14007", "-33", "475", "504", "501", "500", "11", "40", "90", "70", "89", "1198", "1194", "1202", "1173", "297", "310", "-15", "-10", "71", "121", "-21", "36", "122", "128", "125", "190", "207", "47", "39", "92", "130", "76", "195", "41", "46", "283", "273", "44", "18", "-8", "62", "5", "55", "play game", "iot coffee", "transport traffic", "calendar query", "qa factoid", "play audiobook", "iot cleaning", "alarm remove", "general negate", "recommendation events", "alarm set", "play podcasts", "transport query", "general commandstop", "iot wemo off", "music settings", "general explain", "lists query", "general joke", "iot hue lightup", "email query", "takeaway query", "play radio", "lists remove", "music query", "recommendation movies", "close_no_action", "delivery", "stop", "harmful", "qa_stock", "recommendation_movies", "audio_volume_up", "play_audiobook", "recommendation_locations", "cooking_recipe", "datetime_convert", "cooking_query", "qa_currency", "weather_query", "music_query", "iot_hue_lightdim", "iot_hue_lightup", "qa_definition", "audio_volume_down", "general_greet", "iot_hue_lightoff", "alarm_set", "general_quirky", "play_podcasts", "email_addcontact", "cooking", "IN:LIKE_MUSIC", "IN:GET_RECIPES", "IN:GET_TIMER", "IN:CREATE_PLAYLIST_MUSIC", "IN:PAUSE_MUSIC", "IN:SET_DEFAULT_PROVIDER_CALLING", "IN:SWITCH_CALL", "IN:SET_UNAVAILABLE", "IN:GET_EDUCATION_DEGREE", "IN:GET_MAJOR", "IN:GET_TRACK_INFO_MUSIC", "IN:GET_ATTENDEE_EVENT", "IN:UPDATE_METHOD_CALL", "IN:GET_AVAILABILITY", "IN:GET_EMPLOYMENT_TIME", "IN:GET_REMINDER_DATE_TIME", "IN:CREATE_REMINDER", "IN:SKIP_TRACK_MUSIC", "IN:GET_AGE", "IN:GET_LIFE_EVENT", "IN:STOP_SHUFFLE_MUSIC", "IN:GET_CALL", "IN:MERGE_CALL", "IN:DELETE_ALARM", "IN:UNLOOP_MUSIC", "IN:SET_DEFAULT_PROVIDER_MUSIC", "IN:RESUME_CALL", "IN:CREATE_CALL", "IN:GET_CALL_TIME", "IN:GET_CONTACT_METHOD", "IN:GET_UNDERGRAD", "IN:GET_GENDER", "IN:SHARE_EVENT", "IN:RESTART_TIMER", "IN:SET_RSVP_INTERESTED", "IN:QUESTION_NEWS", "IN:GET_DETAILS_NEWS", "IN:ADD_TIME_TIMER", "IN:GET_MESSAGE_CONTACT", "IN:ANSWER_CALL", "IN:SET_AVAILABLE", "IN:UPDATE_REMINDER_LOCATION", "IN:REMOVE_FROM_PLAYLIST_MUSIC", "IN:PLAY_MEDIA", "IN:HOLD_CALL", "IN:GET_STORIES_NEWS", "IN:GET_EVENT", "IN:FOLLOW_MUSIC", "IN:GET_AIRQUALITY", "IN:CANCEL_CALL", "IN:GET_REMINDER_AMOUNT", "IN:GET_WEATHER", "IN:UPDATE_REMINDER_DATE_TIME", "IN:CANCEL_MESSAGE", "IN:IS_TRUE_RECIPES", "IN:GET_GROUP", "IN:PAUSE_TIMER", "IN:DISPREFER", "IN:CREATE_TIMER", "option_9", "option_10", "option_11", "option_12", "option_13", "option_14", "option_15", "mixed", "SearchHotel", "MakePayment", "RequestPayment", "GetTimesForMovie", "BuyMovieTickets", "FindRestaurants", "SearchHouse", "SearchOnewayFlight", "LookupMusic", "AddAlarm", "GetWeather", "GetRide", "ShareLocation", "teacher", "supervisor", "clerk", "lawyer", "baker", "carpenter", "developer", "auditor" ], "corpus_digest": "fnv1a64-e5cf849bd33ea8e1", "cases_digest": "fnv1a64-46281107df052d0e", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "hybrid": { "lane": "Instinct", "model": "A0", "hard": { "n": 4329, "accuracy": 0.22961422961422961, "ece": 0.20603674965280053, "mean_confidence": 0.021036477420426566, "acc_at_50_coverage": 0.17236598890942698 }, "consult_rate": 0.0, "latency_scope": "seat+arm", "latency_rows": "questions", "serves": "A0", "gate": "served by the reflex half \u2014 the best measured arm on this suite", "source_run": { "git_sha": "b6425bf", "date_utc": "2026-10-03T09:03:51Z" } }, "extra_host_lanes": { "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 4329, "accuracy": 0.22961422961422961, "macro_f1": 0.05728477985464882, "ece": 0.043681576005988834, "brier": 0.7720725868744388, "nll": 1.8430021862719967, "aurc": 0.6642965880999749, "mean_confidence": 0.2444685912265303, "acc_at_50_coverage": 0.3202402957486137, "acc_at_80_coverage": 0.2682645105399942 }, "readout_ece": 0.2060367496941066, "readout_brier": 0.22520657170172276, "raw_abstain": { "abstain_rate": 0.7803187803187803, "selective_accuracy": 0.138801261829653, "selective_n": 951 }, "calibrated_abstain": { "abstain_rate": 0.7803187803187803, "selective_accuracy": 0.138801261829653, "selective_n": 951 }, "abstain_causes": { "score_gate": 2686, "distance_gate": 692, "grammar_invalid": 0 }, "readout_ece_raw": 0.2060367496941066, "readout_ece_calibrated": 0.2060367496941066, "floor_ece": 0.40613977927410666, "g1_pass": false, "g1_verdict": "fail", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 4329, "n_questions": 4329, "score_threshold": 0.010039693, "distance_threshold": 0.5756306, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.010039693, "status": "fitted", "targetAccuracy": 0.15, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } }, "distance": { "threshold": 0.5756306, "status": "fitted", "targetAccuracy": 0.16428572, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 16, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.155 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.185 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.18 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.18 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.17 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.175 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.165 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.16 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.16 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.155 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.17 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.19 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.195 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.185 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.195 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "not_mentioned", "pred": "contradiction", "count": 312, "share_of_errors": 0.09355322338830585 }, { "gold": "entailment", "pred": "contradiction", "count": 257, "share_of_errors": 0.07706146926536732 }, { "gold": "not_mentioned", "pred": "entailment", "count": 108, "share_of_errors": 0.032383808095952024 }, { "gold": "entailment", "pred": "neutral", "count": 80, "share_of_errors": 0.0239880059970015 }, { "gold": "contradiction", "pred": "entailment", "count": 61, "share_of_errors": 0.018290854572713643 }, { "gold": "contradiction", "pred": "neutral", "count": 53, "share_of_errors": 0.015892053973013492 }, { "gold": "option_1", "pred": "option_0", "count": 53, "share_of_errors": 0.015892053973013492 }, { "gold": "neutral", "pred": "entailment", "count": 46, "share_of_errors": 0.013793103448275862 }, { "gold": "option_2", "pred": "option_3", "count": 26, "share_of_errors": 0.007796101949025487 }, { "gold": "A", "pred": "B", "count": 25, "share_of_errors": 0.0074962518740629685 }, { "gold": "2", "pred": "3", "count": 24, "share_of_errors": 0.00719640179910045 }, { "gold": "option_0", "pred": "option_1", "count": 22, "share_of_errors": 0.006596701649175412 } ], "pair_head_ab": null, "jdi_chance": 0.23331023331023332, "jdi_skill": 0.0, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.13536291792988778 }, { "mode": "max_prob", "cal_ece": 0.09499485112726685 }, { "mode": "inv_entropy", "cal_ece": 0.15231328934431076 } ] }, "corpus_fallbacks": [ "getting_spare_card", "verify_my_identity", "virtual_card_not_working", "disposable_card_limits", "reverted_card_payment?", "pending_cash_withdrawal", "get_physical_card", "lost_or_stolen_card", "top_up_by_card_charge", "extra_charge_on_statement", "automatic_top_up", "get_disposable_virtual_card", "transfer_not_received_by_recipient", "request_refund", "pending_card_payment", "declined_transfer", "age_limit", "card_payment_fee_charged", "verify_source_of_funds", "declined_card_payment", "card_payment_wrong_exchange_rate", "wrong_amount_of_cash_received", "atm_support", "transaction_charged_twice", "wrong_exchange_rate_for_cash_withdrawal", "supported_cards_and_currencies", "balance_not_updated_after_bank_transfer", "card_swallowed", "failed_transfer", "cancel_transfer", "direct_debit_payment_not_recognised", "top_up_limits", "exchange_via_app", "passcode_forgotten", "top_up_by_cash_or_cheque", "top_up_by_bank_transfer_charge", "beneficiary_not_allowed", "compromised_card", "unable_to_verify_identity", "card_not_working", "exchange_rate", "card_payment_not_recognised", "transfer_into_account", "cash_withdrawal_charge", "card_arrival", "card_acceptance", "activate_my_card", "transfer_timing", "terminate_account", "topping_up_by_card", "cash_withdrawal_not_recognised", "getting_virtual_card", "country_support", "Refund_not_showing_up", "balance_not_updated_after_cheque_or_cash_deposit", "change_pin", "pending_transfer", "edit_personal_details", "verify_top_up", "pending_top_up", "why_verify_identity", "card_about_to_expire", "exchange_charge", "transfer_fee_charged", "top_up_failed", "fiat_currency_support", "card_delivery_estimate", "rewards_balance", "translate", "cancel", "maybe", "restaurant_reviews", "repeat", "todo_list", "flight_status", "last_maintenance", "pto_balance", "directions", "restaurant_reservation", "confirm_reservation", "order_status", "tell_joke", "oos", "calculator", "gas_type", "travel_suggestion", "share_location", "shopping_list", "change_language", "timezone", "yes", "book_flight", "payday", "no", "meeting_schedule", "do_you_have_pets", "order", "credit_limit_change", "recipe", "calendar_update", "improve_credit_score", "pin_change", "update_playlist", "cancel_reservation", "date", "measurement_conversion", "definition", "oil_change_how", "whisper_mode", "flip_coin", "find_phone", "todo_list_update", "redeem_rewards", "change_user_name", "change_ai_name", "min_payment", "spelling", "how_busy", "change_accent", "schedule_meeting", "food_last", "international_fees", "book_hotel", "where_are_you_from", "change_volume", "are_you_a_bot", "tire_change", "ingredients_list", "current_location", "goodbye", "insurance_change", "w2", "pto_used", "new_card", "traffic", "balance", "fun_fact", "ingredient_substitution", "transfer", "freeze_account", "schedule_maintenance", "thank_you", "what_is_your_name", "gas", "vaccines", "accept_reservations", "transactions", "cook_time", "distance", "routing", "report_lost_card", "next_song", "carry_on", "what_song", "travel_notification", "apr", "nutrition_info", "who_made_you", "next_holiday", "bill_balance", "reminder_update", "how_old_are_you", "mpg", "rollover_401k", "account_blocked", "plug_type", "card_declined", "smart_home", "meal_suggestion", "direct_deposit", "sync_device", "pay_bill", "uber", "pto_request_status", "taxes", "order_checks", "greeting", "income", "spending_history", "calories", "reset_settings", "expiration_date", "tire_pressure", "meaning_of_life", "make_call", "insurance", "restaurant_suggestion", "interest_rate", "credit_limit", "bill_due", "who_do_you_work_for", "report_fraud", "roll_dice", "change_speed", "damaged_card", "shopping_list_update", "what_are_your_hobbies", "lost_luggage", "oil_change_when", "text", "travel_alert", "car_rental", "international_visa", "application_status", "jump_start", "credit_score", "what_can_i_ask_you", "replacement_card_duration", "pto_request", "national_id", "company_name", "account_number", "password", "url", "-47", "-4", "14", "34", "24", "5760", "5756", "5750", "5765", "33", "-2", "8", "12", "29", "16", "52", "-5", "17", "72", "80", "78", "103", "98", "48", "200", "150", "180", "196", "64", "114", "59", "84", "38", "63", "43", "32", "21", "20", "-18", "27", "149", "197", "172", "147", "88", "93", "-1", "-3", "350", "397", "400", "375", "248", "254", "255", "251", "6580", "6598", "6600", "6593", "293", "305", "288", "13997", "14000", "13993", "14007", "-33", "475", "504", "501", "500", "11", "40", "90", "70", "89", "1198", "1194", "1202", "1173", "297", "310", "-15", "-10", "71", "121", "-21", "36", "122", "128", "125", "190", "207", "47", "39", "92", "130", "76", "195", "41", "46", "283", "273", "44", "18", "-8", "62", "5", "55", "play game", "iot coffee", "transport traffic", "calendar query", "qa factoid", "play audiobook", "iot cleaning", "alarm remove", "general negate", "recommendation events", "alarm set", "play podcasts", "transport query", "general commandstop", "iot wemo off", "music settings", "general explain", "lists query", "general joke", "iot hue lightup", "email query", "takeaway query", "play radio", "lists remove", "music query", "recommendation movies", "close_no_action", "delivery", "stop", "harmful", "qa_stock", "recommendation_movies", "audio_volume_up", "play_audiobook", "recommendation_locations", "cooking_recipe", "datetime_convert", "cooking_query", "qa_currency", "weather_query", "music_query", "iot_hue_lightdim", "iot_hue_lightup", "qa_definition", "audio_volume_down", "general_greet", "iot_hue_lightoff", "alarm_set", "general_quirky", "play_podcasts", "email_addcontact", "cooking", "IN:LIKE_MUSIC", "IN:GET_RECIPES", "IN:GET_TIMER", "IN:CREATE_PLAYLIST_MUSIC", "IN:PAUSE_MUSIC", "IN:SET_DEFAULT_PROVIDER_CALLING", "IN:SWITCH_CALL", "IN:SET_UNAVAILABLE", "IN:GET_EDUCATION_DEGREE", "IN:GET_MAJOR", "IN:GET_TRACK_INFO_MUSIC", "IN:GET_ATTENDEE_EVENT", "IN:UPDATE_METHOD_CALL", "IN:GET_AVAILABILITY", "IN:GET_EMPLOYMENT_TIME", "IN:GET_REMINDER_DATE_TIME", "IN:CREATE_REMINDER", "IN:SKIP_TRACK_MUSIC", "IN:GET_AGE", "IN:GET_LIFE_EVENT", "IN:STOP_SHUFFLE_MUSIC", "IN:GET_CALL", "IN:MERGE_CALL", "IN:DELETE_ALARM", "IN:UNLOOP_MUSIC", "IN:SET_DEFAULT_PROVIDER_MUSIC", "IN:RESUME_CALL", "IN:CREATE_CALL", "IN:GET_CALL_TIME", "IN:GET_CONTACT_METHOD", "IN:GET_UNDERGRAD", "IN:GET_GENDER", "IN:SHARE_EVENT", "IN:RESTART_TIMER", "IN:SET_RSVP_INTERESTED", "IN:QUESTION_NEWS", "IN:GET_DETAILS_NEWS", "IN:ADD_TIME_TIMER", "IN:GET_MESSAGE_CONTACT", "IN:ANSWER_CALL", "IN:SET_AVAILABLE", "IN:UPDATE_REMINDER_LOCATION", "IN:REMOVE_FROM_PLAYLIST_MUSIC", "IN:PLAY_MEDIA", "IN:HOLD_CALL", "IN:GET_STORIES_NEWS", "IN:GET_EVENT", "IN:FOLLOW_MUSIC", "IN:GET_AIRQUALITY", "IN:CANCEL_CALL", "IN:GET_REMINDER_AMOUNT", "IN:GET_WEATHER", "IN:UPDATE_REMINDER_DATE_TIME", "IN:CANCEL_MESSAGE", "IN:IS_TRUE_RECIPES", "IN:GET_GROUP", "IN:PAUSE_TIMER", "IN:DISPREFER", "IN:CREATE_TIMER", "option_9", "option_10", "option_11", "option_12", "option_13", "option_14", "option_15", "mixed", "SearchHotel", "MakePayment", "RequestPayment", "GetTimesForMovie", "BuyMovieTickets", "FindRestaurants", "SearchHouse", "SearchOnewayFlight", "LookupMusic", "AddAlarm", "GetWeather", "GetRide", "ShareLocation", "teacher", "supervisor", "clerk", "lawyer", "baker", "carpenter", "developer", "auditor" ], "corpus_digest": "fnv1a64-e5cf849bd33ea8e1", "cases_digest": "fnv1a64-46281107df052d0e", "source_run": { "git_sha": "741bda3", "date_utc": "2026-10-09T00:13:55Z" }, "latency_provenance": { "note": "latency cells carried from the host's incumbent run; accuracy is this lane's own (LANE-CARRY, Issue 032)" } } } }, "encoder": { "lane": "Rethink", "model": "A0", "hard": { "n": 4329, "accuracy": 0.22961422961422961, "ece": 0.20603674965280053, "mean_confidence": 0.021036477420426566, "acc_at_50_coverage": 0.17236598890942698 }, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": null, "latency_p99_ms": null, "latency_tail_support": null, "serves": "tier-fallback", "served_by": "Instinct (A0)", "fallback_note": "no seated arm for this lane on this suite \u2014 the served answer is the tier shown (the full-coverage serving law, instinct 057d31a)", "derived": true, "source_run": { "git_sha": "b6425bf", "date_utc": "2026-10-03T09:03:51Z" } } }, { "name": "s1mb_noul", "n_questions": 6173, "n_cases": 6173, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 6173, "accuracy": 0.7054916572169123, "macro_f1": 0.42913836905658626, "ece": 0.20416973503486036, "brier": 0.5001310888121616, "nll": 0.6932784373256037, "aurc": 0.37506022202551853, "mean_confidence": 0.501321922182052, "acc_at_50_coverage": 0.6970187945560596, "acc_at_80_coverage": 0.6871202916160389 }, "readout_ece": 0.006089712481202825, "readout_brier": 0.25001118921980225, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.4676818402721529, "selective_accuracy": 0.30401704199634816, "selective_n": 3286 }, "abstain_causes": { "score_gate": 0, "distance_gate": 2887, "grammar_invalid": 0 }, "readout_ece_raw": 0.10980077466141282, "readout_ece_calibrated": 0.006089712481202825, "floor_ece": 0.06142461191525924, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.291, "latency_p99_ms": 0.431, "latency_tail_support": 62, "latency_extremes": { "first_ms": 0.279, "max_ms": 1.038, "argmax_case": 3626 }, "determinism_ok": true, "determinism_n": 10, "seconds": 5.9449105, "n_cases": 6173, "n_questions": 6173, "score_threshold": 0.45501745, "distance_threshold": 0.6012192, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.45501745, "status": "fitted", "targetAccuracy": 0.455, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.6012192, "status": "fitted", "targetAccuracy": 0.47142857, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.5 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.5 } ] }, "genome_selection": null, "transductive": null, "confusion": [], "pair_head_ab": null, "jdi_chance": 0.7357848695933906, "jdi_skill": 0.0, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.11997524559497832 }, { "mode": "max_prob", "cal_ece": 0.0015637028217315718 }, { "mode": "inv_entropy", "cal_ece": 0.11997524559497832 } ] }, "corpus_digest": "fnv1a64-0c8778c6d51742f4", "cases_digest": "fnv1a64-a887ca350a4a8b7c", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "hybrid": { "lane": "Instinct", "model": "A0", "hard": { "n": 6173, "accuracy": 0.7054916572169123, "ece": 0.10980077466141282, "mean_confidence": 3.2369676834370933e-05, "acc_at_50_coverage": 0.6970187945560596 }, "consult_rate": 0.0, "latency_scope": "seat+arm", "latency_rows": "questions", "serves": "A0", "gate": "served by the reflex half (A0 is the argmax)", "source_run": { "git_sha": "b6425bf", "date_utc": "2026-10-03T09:03:51Z" } }, "extra_host_lanes": { "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 6173, "accuracy": 0.7054916572169123, "macro_f1": 0.42913836905658626, "ece": 0.20416973503486036, "brier": 0.5001310888121616, "nll": 0.6932784373256037, "aurc": 0.37506022202551853, "mean_confidence": 0.501321922182052, "acc_at_50_coverage": 0.6970187945560596, "acc_at_80_coverage": 0.6871202916160389 }, "readout_ece": 0.006089712481202825, "readout_brier": 0.25001118921980225, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.4676818402721529, "selective_accuracy": 0.30401704199634816, "selective_n": 3286 }, "abstain_causes": { "score_gate": 0, "distance_gate": 2887, "grammar_invalid": 0 }, "readout_ece_raw": 0.10980077466141282, "readout_ece_calibrated": 0.006089712481202825, "floor_ece": 0.06142461191525924, "g1_pass": true, "g1_verdict": "pass", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 6173, "n_questions": 6173, "score_threshold": 0.45501745, "distance_threshold": 0.6012192, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.45501745, "status": "fitted", "targetAccuracy": 0.455, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.6012192, "status": "fitted", "targetAccuracy": 0.47142857, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 64, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.5 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 0, "view": "bag", "cal_acc": 0.515 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 0, "view": "bag", "cal_acc": 0.53 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": 1, "view": "bag", "cal_acc": 0.485 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": 1, "view": "bag", "cal_acc": 0.47 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.5 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.5 } ] }, "genome_selection": null, "transductive": null, "confusion": [], "pair_head_ab": null, "jdi_chance": 0.7357848695933906, "jdi_skill": 0.0, "readout_report": { "best_on_cal": "max_prob", "candidates": [ { "mode": "dispatch", "cal_ece": 0.11997524559497832 }, { "mode": "max_prob", "cal_ece": 0.0015637028217315718 }, { "mode": "inv_entropy", "cal_ece": 0.11997524559497832 } ] }, "corpus_digest": "fnv1a64-0c8778c6d51742f4", "cases_digest": "fnv1a64-a887ca350a4a8b7c", "source_run": { "git_sha": "741bda3", "date_utc": "2026-10-09T00:13:55Z" }, "latency_provenance": { "note": "latency cells carried from the host's incumbent run; accuracy is this lane's own (LANE-CARRY, Issue 032)" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 6173, "accuracy": 0.6964198930827799, "macro_f1": 0.6277703084304177, "ece": 0.09739886602948292, "brier": 0.405517668958367, "nll": 0.6462810084974737, "aurc": 0.18015165205991684, "mean_confidence": 0.7911902478535553, "acc_at_50_coverage": 0.8204795852235904, "acc_at_80_coverage": 0.7353179424868368 }, "readout_ece": 0.09739886602948293, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 18.0, "latency_p99_ms": 32.0, "latency_tail_support": 62, "latency_extremes": { "first_ms": 17.0, "max_ms": 43.0, "argmax_case": 3626 }, "determinism_ok": true, "determinism_n": 10, "seconds": 126.8324202, "n_cases": 6173, "n_questions": 6173, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.7357848695933906, "jdi_skill": 0.0, "cases_digest": "fnv1a64-a887ca350a4a8b7c", "latency_quotable": null, "source_run": { "git_sha": "3cad008", "date_utc": "2026-10-02T20:23:08Z" } } } } }, "encoder": { "lane": "Rethink", "model": "A0", "hard": { "n": 6173, "accuracy": 0.7054916572169123, "ece": 0.10980077466141282, "mean_confidence": 3.2369676834370933e-05, "acc_at_50_coverage": 0.6970187945560596 }, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": null, "latency_p99_ms": null, "latency_tail_support": null, "serves": "tier-fallback", "served_by": "Instinct (A0)", "fallback_note": "no seated arm for this lane on this suite \u2014 the served answer is the tier shown (the full-coverage serving law, instinct 057d31a)", "derived": true, "source_run": { "git_sha": "b6425bf", "date_utc": "2026-10-03T09:03:51Z" } } }, { "name": "s1mb_score", "n_questions": 2571, "n_cases": 2571, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 2571, "accuracy": 0.4982497082847141, "macro_f1": 0.10160467092004079, "ece": 0.29225885901978305, "brier": 0.7940581511281869, "nll": 1.5851819124303006, "aurc": 0.5903998829836306, "mean_confidence": 0.20599084926493108, "acc_at_50_coverage": 0.4350194552529183, "acc_at_80_coverage": 0.4771400778210117 }, "readout_ece": 0.26810998820530585, "readout_brier": 0.32188089003429515, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.07701283547257877, "selective_accuracy": 0.5128529287821323, "selective_n": 2373 }, "abstain_causes": { "score_gate": 0, "distance_gate": 198, "grammar_invalid": 0 }, "readout_ece_raw": 0.004096552471690769, "readout_ece_calibrated": 0.26810998820530585, "floor_ece": 0.4869681154709195, "g1_pass": false, "g1_verdict": "fail", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": 1.4454512679107152, "within_1": 0.26059898872034226, "latency_p50_ms": 0.245, "latency_p99_ms": 0.379, "latency_tail_support": 26, "latency_extremes": { "first_ms": 0.069, "max_ms": 0.438, "argmax_case": 2040 }, "determinism_ok": true, "determinism_n": 10, "seconds": 8.995764375, "n_cases": 2571, "n_questions": 2571, "score_threshold": 0.23013774, "distance_threshold": 0.41533497, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.23013774, "status": "fitted", "targetAccuracy": 0.23, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.41533497, "status": "fitted", "targetAccuracy": 0.27857143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 16, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "1.0", "pred": "0.0", "count": 499, "share_of_errors": 0.3868217054263566 }, { "gold": "3.0", "pred": "0.0", "count": 354, "share_of_errors": 0.2744186046511628 }, { "gold": "2.0", "pred": "0.0", "count": 241, "share_of_errors": 0.1868217054263566 }, { "gold": "4.0", "pred": "0.0", "count": 135, "share_of_errors": 0.10465116279069768 }, { "gold": "0.5", "pred": "0.0", "count": 11, "share_of_errors": 0.008527131782945736 }, { "gold": "0.75", "pred": "0.0", "count": 9, "share_of_errors": 0.0069767441860465115 }, { "gold": "1.0", "pred": "0.3333333333333333", "count": 8, "share_of_errors": 0.006201550387596899 }, { "gold": "0.25", "pred": "0.0", "count": 7, "share_of_errors": 0.005426356589147287 }, { "gold": "0.6666666666666666", "pred": "0.3333333333333333", "count": 6, "share_of_errors": 0.004651162790697674 }, { "gold": "0.3333333333333333", "pred": "0.0", "count": 5, "share_of_errors": 0.003875968992248062 }, { "gold": "0.0", "pred": "0.3333333333333333", "count": 3, "share_of_errors": 0.002325581395348837 }, { "gold": "0.6666666666666666", "pred": "0.0", "count": 2, "share_of_errors": 0.0015503875968992248 } ], "pair_head_ab": null, "jdi_chance": 0.4955270322831583, "jdi_skill": 0.005397070161912056, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.0006615148484706879 }, { "mode": "max_prob", "cal_ece": 0.05418207250535489 }, { "mode": "inv_entropy", "cal_ece": 1.7529428005218504e-05 } ] }, "corpus_fallbacks": [ "5", "6", "7", "8", "9" ], "corpus_digest": "fnv1a64-f5d9f67a1fd69443", "cases_digest": "fnv1a64-81f40d7a9b57e9b6", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "hybrid": { "lane": "Instinct", "model": "A0", "hard": { "n": 2571, "accuracy": 0.4982497082847141, "ece": 0.00638683968796335, "mean_confidence": 0.0002253734586722004, "acc_at_50_coverage": 0.4217898832684825 }, "consult_rate": 0.0, "latency_scope": "seat+arm", "latency_rows": "questions", "serves": "A0", "gate": "served by the reflex half \u2014 the best measured arm on this suite", "source_run": { "git_sha": "60f373c", "date_utc": "2026-10-09T02:05:00Z" } }, "extra_host_lanes": { "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 2571, "accuracy": 0.4982497082847141, "macro_f1": 0.10160467092004079, "ece": 0.29225885901978305, "brier": 0.7940581511281869, "nll": 1.5851819124303006, "aurc": 0.5903998829836306, "mean_confidence": 0.20599084926493108, "acc_at_50_coverage": 0.4350194552529183, "acc_at_80_coverage": 0.4771400778210117 }, "readout_ece": 0.26810998820530585, "readout_brier": 0.32188089003429515, "raw_abstain": { "abstain_rate": 1.0, "selective_accuracy": 0.0, "selective_n": 0 }, "calibrated_abstain": { "abstain_rate": 0.07701283547257877, "selective_accuracy": 0.5128529287821323, "selective_n": 2373 }, "abstain_causes": { "score_gate": 0, "distance_gate": 198, "grammar_invalid": 0 }, "readout_ece_raw": 0.004096552471690769, "readout_ece_calibrated": 0.26810998820530585, "floor_ece": 0.4869681154709195, "g1_pass": false, "g1_verdict": "fail", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": 1.4454512679107152, "within_1": 0.26059898872034226, "determinism_ok": true, "determinism_n": 10, "n_cases": 2571, "n_questions": 2571, "score_threshold": 0.23013774, "distance_threshold": 0.41533497, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.23013774, "status": "fitted", "targetAccuracy": 0.23, "support": { "n": 200, "nPass": 200, "nAbstain": 0 } }, "distance": { "threshold": 0.41533497, "status": "fitted", "targetAccuracy": 0.27857143, "support": { "n": 200, "nPass": 140, "nAbstain": 60 } } }, "corpus_cap": { "effective": 16, "source": "registry", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 0.0, "selected_alpha": "off", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.21717171717171718 } ] }, "oc_selection": null, "ridge_selection": { "selected_scale": 0.0, "selected_lambda": 10.0, "candidates": [ { "scale": 0.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 0.5, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 1.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 2.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 4.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 }, { "scale": 8.0, "lambda": 10.0, "cal_acc": 0.21717171717171718 } ] }, "genome_selection": null, "transductive": null, "confusion": [ { "gold": "1.0", "pred": "0.0", "count": 499, "share_of_errors": 0.3868217054263566 }, { "gold": "3.0", "pred": "0.0", "count": 354, "share_of_errors": 0.2744186046511628 }, { "gold": "2.0", "pred": "0.0", "count": 241, "share_of_errors": 0.1868217054263566 }, { "gold": "4.0", "pred": "0.0", "count": 135, "share_of_errors": 0.10465116279069768 }, { "gold": "0.5", "pred": "0.0", "count": 11, "share_of_errors": 0.008527131782945736 }, { "gold": "0.75", "pred": "0.0", "count": 9, "share_of_errors": 0.0069767441860465115 }, { "gold": "1.0", "pred": "0.3333333333333333", "count": 8, "share_of_errors": 0.006201550387596899 }, { "gold": "0.25", "pred": "0.0", "count": 7, "share_of_errors": 0.005426356589147287 }, { "gold": "0.6666666666666666", "pred": "0.3333333333333333", "count": 6, "share_of_errors": 0.004651162790697674 }, { "gold": "0.3333333333333333", "pred": "0.0", "count": 5, "share_of_errors": 0.003875968992248062 }, { "gold": "0.0", "pred": "0.3333333333333333", "count": 3, "share_of_errors": 0.002325581395348837 }, { "gold": "0.6666666666666666", "pred": "0.0", "count": 2, "share_of_errors": 0.0015503875968992248 } ], "pair_head_ab": null, "jdi_chance": 0.4955270322831583, "jdi_skill": 0.005397070161912056, "readout_report": { "best_on_cal": "dispatch", "candidates": [ { "mode": "dispatch", "cal_ece": 0.0006615148484706879 }, { "mode": "max_prob", "cal_ece": 0.05418207250535489 }, { "mode": "inv_entropy", "cal_ece": 1.7529428005218504e-05 } ] }, "corpus_fallbacks": [ "5", "6", "7", "8", "9" ], "corpus_digest": "fnv1a64-f5d9f67a1fd69443", "cases_digest": "fnv1a64-81f40d7a9b57e9b6", "source_run": { "git_sha": "741bda3", "date_utc": "2026-10-09T00:13:55Z" }, "latency_provenance": { "note": "latency cells carried from the host's incumbent run; accuracy is this lane's own (LANE-CARRY, Issue 032)" } }, "laya": { "english": { "lane": "laya (rust)", "model": "english", "hard": { "n": 2571, "accuracy": 0.2590431738623104, "macro_f1": 0.12722887517853823, "ece": 0.18714566316608316, "brier": 0.8355351283547244, "nll": 1.7237879218114467, "aurc": 0.6122756154382043, "mean_confidence": 0.44618883702839446, "acc_at_50_coverage": 0.3478599221789883, "acc_at_80_coverage": 0.28745136186770426 }, "readout_ece": 0.07684850252819911, "raw_abstain": null, "calibrated_abstain": null, "abstain_causes": null, "readout_ece_raw": null, "readout_ece_calibrated": null, "floor_ece": null, "g1_pass": null, "g1_verdict": null, "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 24.0, "latency_p99_ms": 31.0, "latency_tail_support": 26, "latency_extremes": { "first_ms": 23.0, "max_ms": 33.0, "argmax_case": 2024 }, "determinism_ok": true, "determinism_n": 10, "seconds": 65.2643958, "n_cases": 2571, "n_questions": 2571, "score_threshold": null, "distance_threshold": null, "threshold_recommendation": null, "corpus_cap": null, "head_selection": null, "nb_selection": null, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": null, "confusion": null, "pair_head_ab": null, "jdi_chance": 0.4955270322831583, "jdi_skill": 0.0, "cases_digest": "fnv1a64-81f40d7a9b57e9b6", "latency_quotable": null, "source_run": { "git_sha": "3cad008", "date_utc": "2026-10-02T20:23:08Z" } } } } }, "encoder": { "lane": "Rethink", "model": "A0", "hard": { "n": 2571, "accuracy": 0.4982497082847141, "ece": 0.00638683968796335, "mean_confidence": 0.0002253734586722004, "acc_at_50_coverage": 0.4217898832684825 }, "latency_scope": "seat+arm", "latency_rows": "questions", "latency_p50_ms": null, "latency_p99_ms": null, "latency_tail_support": null, "serves": "tier-fallback", "served_by": "Instinct (A0)", "fallback_note": "no seated arm for this lane on this suite \u2014 the served answer is the tier shown (the full-coverage serving law, instinct 057d31a)", "derived": true, "source_run": { "git_sha": "60f373c", "date_utc": "2026-10-09T02:05:00Z" } } }, { "name": "semantic_defects", "n_questions": 102, "n_cases": 102, "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 102, "accuracy": 0.4411764705882353, "macro_f1": 0.4334818023351805, "ece": 0.23470770494610654, "brier": 0.79874137265555, "nll": 1.6978339787256655, "aurc": 0.430125277309569, "mean_confidence": 0.20646876564212874, "acc_at_50_coverage": 0.49019607843137253, "acc_at_80_coverage": 0.49382716049382713 }, "readout_ece": 0.43579113892480437, "readout_brier": 0.43534595726314507, "raw_abstain": { "abstain_rate": 0.5, "selective_accuracy": 0.43137254901960786, "selective_n": 51 }, "calibrated_abstain": { "abstain_rate": 0.5, "selective_accuracy": 0.43137254901960786, "selective_n": 51 }, "abstain_causes": { "score_gate": 32, "distance_gate": 19, "grammar_invalid": 0 }, "readout_ece_raw": 0.43579113892480437, "readout_ece_calibrated": 0.43579113892480437, "floor_ece": 0.19453044375645, "g1_pass": null, "g1_verdict": "no_claim", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "latency_p50_ms": 0.024, "latency_p99_ms": 0.026, "latency_tail_support": 2, "latency_extremes": { "first_ms": 0.025, "max_ms": 0.026, "argmax_case": 5 }, "determinism_ok": true, "determinism_n": 10, "seconds": 0.128910583, "n_cases": 102, "n_questions": 102, "score_threshold": 0.0026944876, "distance_threshold": 0.52091897, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.0026944876, "status": "fitted", "targetAccuracy": 0.53846157, "support": { "n": 18, "nPass": 13, "nAbstain": 5 } }, "distance": { "threshold": 0.52091897, "status": "fitted", "targetAccuracy": 0.53846157, "support": { "n": 18, "nPass": 13, "nAbstain": 5 } } }, "corpus_cap": { "effective": 18446744073709551615, "source": "registry (selection n/a: self-corpora)", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.2777777777777778 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3333333333333333 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 } ] }, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": { "accuracy": 0.43137254901960786, "honest_accuracy": 0.4411764705882353, "n_pseudo": 102, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "clean", "pred": "off_by_one", "count": 8, "share_of_errors": 0.14035087719298245 }, { "gold": "off_by_one", "pred": "inverted_condition", "count": 5, "share_of_errors": 0.08771929824561403 }, { "gold": "swallowed_error", "pred": "swapped_lookup", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "unwrapped_none", "pred": "clean", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "unwrapped_none", "pred": "swapped_lookup", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "clean", "pred": "inverted_condition", "count": 3, "share_of_errors": 0.05263157894736842 }, { "gold": "clean", "pred": "swapped_lookup", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "inverted_condition", "pred": "off_by_one", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "off_by_one", "pred": "swapped_lookup", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swallowed_error", "pred": "clean", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swallowed_error", "pred": "off_by_one", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swapped_lookup", "pred": "clean", "count": 2, "share_of_errors": 0.03508771929824561 } ], "pair_head_ab": null, "jdi_chance": 0.16666666666666666, "jdi_skill": 0.32941176470588235, "corpus_digest": "fnv1a64-d0a395518fca2bdb", "cases_digest": "fnv1a64-8032fe33f25653bc", "latency_quotable": true, "source_run": { "git_sha": "197e855", "date_utc": "2026-10-10T06:52:15Z" } }, "extra_host_lanes": { "4090-win": { "modelless": { "lane": "KatGPT", "model": "modelless", "hard": { "n": 102, "accuracy": 0.4411764705882353, "macro_f1": 0.4334818023351805, "ece": 0.2347077048000167, "brier": 0.79874137265555, "nll": 1.69783397782216, "aurc": 0.430125277309569, "mean_confidence": 0.20646876578821854, "acc_at_50_coverage": 0.49019607843137253, "acc_at_80_coverage": 0.49382716049382713 }, "readout_ece": 0.43579113892480437, "readout_brier": 0.43534595726314507, "raw_abstain": { "abstain_rate": 0.5, "selective_accuracy": 0.43137254901960786, "selective_n": 51 }, "calibrated_abstain": { "abstain_rate": 0.5, "selective_accuracy": 0.43137254901960786, "selective_n": 51 }, "abstain_causes": { "score_gate": 32, "distance_gate": 19, "grammar_invalid": 0 }, "readout_ece_raw": 0.43579113892480437, "readout_ece_calibrated": 0.43579113892480437, "floor_ece": 0.19453044375645, "g1_pass": null, "g1_verdict": "no_claim", "by_question_type": null, "soft_acc": null, "brier_soft": null, "score_mae": null, "within_1": null, "determinism_ok": true, "determinism_n": 10, "n_cases": 102, "n_questions": 102, "score_threshold": 0.0026944876, "distance_threshold": 0.52091897, "threshold_recommendation": { "posture": { "posture": "percentile", "rho": 0.3 }, "score": { "threshold": 0.0026944876, "status": "fitted", "targetAccuracy": 0.53846157, "support": { "n": 18, "nPass": 13, "nAbstain": 5 } }, "distance": { "threshold": 0.52091897, "status": "fitted", "targetAccuracy": 0.53846157, "support": { "n": 18, "nPass": 13, "nAbstain": 5 } } }, "corpus_cap": { "effective": 18446744073709551615, "source": "registry (selection n/a: self-corpora)", "selection": null }, "head_selection": null, "nb_selection": { "selected_scale": 1.0, "selected_alpha": "observed-laplace", "selected_noul_domain": null, "selected_view": "bag", "candidates": [ { "scale": 0.0, "alpha": "off", "noul_domain": null, "view": "bag", "cal_acc": 0.2777777777777778 }, { "scale": 1.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 4.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 16.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 32.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 64.0, "alpha": "observed-laplace", "noul_domain": null, "view": "bag", "cal_acc": 0.4444444444444444 }, { "scale": 1.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3333333333333333 }, { "scale": 4.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 16.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 32.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 }, { "scale": 64.0, "alpha": "fixed-1", "noul_domain": null, "view": "bag", "cal_acc": 0.3888888888888889 } ] }, "oc_selection": null, "ridge_selection": null, "genome_selection": null, "transductive": { "accuracy": 0.43137254901960786, "honest_accuracy": 0.4411764705882353, "n_pseudo": 102, "protocol": "TRANSDUCTIVE \u2014 unlabeled TEST text joins the count tables, labelled by the honest engine's own forced picks (gold never read); 2-fold cross-fit (half A's pseudo-docs re-score half B and vice versa, so no case scores against its own text); drafter corpora, route, heads and posture unchanged. Not comparable to the headline acc" }, "confusion": [ { "gold": "clean", "pred": "off_by_one", "count": 8, "share_of_errors": 0.14035087719298245 }, { "gold": "off_by_one", "pred": "inverted_condition", "count": 5, "share_of_errors": 0.08771929824561403 }, { "gold": "swallowed_error", "pred": "swapped_lookup", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "unwrapped_none", "pred": "clean", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "unwrapped_none", "pred": "swapped_lookup", "count": 4, "share_of_errors": 0.07017543859649122 }, { "gold": "clean", "pred": "inverted_condition", "count": 3, "share_of_errors": 0.05263157894736842 }, { "gold": "clean", "pred": "swapped_lookup", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "inverted_condition", "pred": "off_by_one", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "off_by_one", "pred": "swapped_lookup", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swallowed_error", "pred": "clean", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swallowed_error", "pred": "off_by_one", "count": 2, "share_of_errors": 0.03508771929824561 }, { "gold": "swapped_lookup", "pred": "clean", "count": 2, "share_of_errors": 0.03508771929824561 } ], "pair_head_ab": null, "jdi_chance": 0.16666666666666666, "jdi_skill": 0.32941176470588235, "corpus_digest": "fnv1a64-d0a395518fca2bdb", "cases_digest": "fnv1a64-8032fe33f25653bc", "source_run": { "git_sha": "70f2b2c", "date_utc": "2026-10-08T15:40:17Z" } } } }, "hybrid": { "lane": "Instinct", "model": "modelless", "hard": { "n": 102, "accuracy": 0.4411764705882353, "macro_f1": 0.4334818023351805, "ece": 0.23470770494610654, "brier": 0.79874137265555, "nll": 1.6978339787256655, "aurc": 0.430125277309569, "mean_confidence": 0.20646876564212874, "acc_at_50_coverage": 0.49019607843137253, "acc_at_80_coverage": 0.49382716049382713 }, "latency_scope": null, "latency_rows": null, "latency_p50_ms": null, "latency_p99_ms": null, "latency_tail_support": null, "serves": "tier-fallback", "served_by": "Reflex (the modelless tier)", "fallback_note": "no seated arm for this lane on this suite \u2014 the served answer is the tier shown (the full-coverage serving law, instinct 057d31a)", "derived": true, "source_run": { "git_sha": "f068ae6", "date_utc": "2026-10-08T15:09:13Z" } }, "encoder": { "lane": "Rethink", "model": "modelless", "hard": { "n": 102, "accuracy": 0.4411764705882353, "macro_f1": 0.4334818023351805, "ece": 0.23470770494610654, "brier": 0.79874137265555, "nll": 1.6978339787256655, "aurc": 0.430125277309569, "mean_confidence": 0.20646876564212874, "acc_at_50_coverage": 0.49019607843137253, "acc_at_80_coverage": 0.49382716049382713 }, "latency_scope": null, "latency_rows": null, "latency_p50_ms": null, "latency_p99_ms": null, "latency_tail_support": null, "serves": "tier-fallback", "served_by": "Reflex (the modelless tier)", "fallback_note": "no seated arm for this lane on this suite \u2014 the served answer is the tier shown (the full-coverage serving law, instinct 057d31a)", "derived": true, "source_run": { "git_sha": "f068ae6", "date_utc": "2026-10-08T15:09:13Z" } } } ], "areas": { "version": 4, "edition": "2026-10-11", "chance_digest": "61c5096331d22d6383de84f51a20f717", "scale": "chance-corrected accuracy: cc = (acc - chance) / (1 - chance); 0 = random guessing, 1 = every question right; per-suite chance = the mean per-question random-pick probability of the harness's own option construction (a dataset fact, not a measurement); values below 0 are BELOW CHANCE and stay negative on purpose \u2014 a lane scoring under random guessing must read that way, and negatives pull area means and the index down by design", "suites": { "ag_news": { "area": "language", "chance": 0.25 }, "massive_intent_en": { "area": "language", "chance": 0.05 }, "banking77": { "area": "language", "chance": 0.012987 }, "sst5": { "area": "sentiment", "chance": 0.2 }, "emotion": { "area": "sentiment", "chance": 0.166667 }, "xnli_en": { "area": "reasoning", "chance": 0.333333 }, "prompt_injections": { "area": "reasoning", "chance": 0.5 }, "typed_decisions": { "area": "decisions", "chance": 0.3175 } }, "areas": [ { "id": "language", "label": "Language & intent", "suites": [ "ag_news", "massive_intent_en", "banking77" ] }, { "id": "sentiment", "label": "Sentiment", "suites": [ "sst5", "emotion" ] }, { "id": "reasoning", "label": "Reasoning & safety", "suites": [ "xnli_en", "prompt_injections" ] }, { "id": "decisions", "label": "Decisions", "suites": [ "typed_decisions" ] } ], "lanes": { "modelless": { "display": "Reflex", "color_key": "katgpt", "kind": "modelless-in-process", "per_suite": { "ag_news": { "acc": 0.8825, "cc": 0.843333 }, "massive_intent_en": { "acc": 0.78, "cc": 0.768421 }, "banking77": { "acc": 0.842, "cc": 0.839921 }, "sst5": { "acc": 0.396667, "cc": 0.245833 }, "emotion": { "acc": 0.885, "cc": 0.862 }, "xnli_en": { "acc": 0.523333, "cc": 0.285 }, "prompt_injections": { "acc": 0.767241, "cc": 0.534483 }, "typed_decisions": { "acc": 0.57, "cc": 0.369963 } }, "areas": { "language": 0.817225, "sentiment": 0.553917, "reasoning": 0.409741, "decisions": 0.369963 }, "index": 0.537712, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "hybrid": { "display": "Instinct", "color_key": "instinct", "kind": "trained-head", "per_suite": { "ag_news": { "acc": 0.8975, "cc": 0.863333 }, "massive_intent_en": { "acc": 0.84, "cc": 0.831579 }, "banking77": { "acc": 0.854, "cc": 0.852079 }, "sst5": { "acc": 0.421667, "cc": 0.277083 }, "emotion": { "acc": 0.885, "cc": 0.862 }, "xnli_en": { "acc": 0.523333, "cc": 0.285 }, "prompt_injections": { "acc": 0.853448, "cc": 0.706897 }, "typed_decisions": { "acc": 0.6475, "cc": 0.483516 } }, "areas": { "language": 0.848997, "sentiment": 0.569542, "reasoning": 0.495949, "decisions": 0.483516 }, "index": 0.599501, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "encoder": { "display": "Rethink", "color_key": "instinct-encoder", "kind": "encoder", "per_suite": { "ag_news": { "acc": 0.9475, "cc": 0.93 }, "massive_intent_en": { "acc": 0.826667, "cc": 0.817544, "fallback": true, "served_by": "Instinct (H2(\u03b2=1,nmin=2,\u03c4=8))", "record_acc": 0.656667 }, "banking77": { "acc": 0.854, "cc": 0.852079, "fallback": true, "served_by": "Instinct (H2(\u03b2=2,nmin=8,\u03c4=8))", "record_acc": 0.442 }, "sst5": { "acc": 0.526667, "cc": 0.408333 }, "emotion": { "acc": 0.885, "cc": 0.862, "fallback": true, "served_by": "Instinct (A0)" }, "xnli_en": { "acc": 0.86, "cc": 0.79 }, "prompt_injections": { "acc": 0.853448, "cc": 0.706897, "fallback": true, "served_by": "Instinct (A1)", "record_acc": 0.801724 }, "typed_decisions": { "acc": 0.755, "cc": 0.641026 } }, "areas": { "language": 0.866541, "sentiment": 0.635166, "reasoning": 0.748449, "decisions": 0.641026 }, "index": 0.722796, "coverage": { "suites": 4, "of": 8 }, "complete": false, "fallback_suites": [ "banking77", "emotion", "massive_intent_en", "prompt_injections" ], "served_coverage": { "suites": 8, "of": 8 } }, "laya": { "display": "laya (rust)", "color_key": "rust", "kind": "encoder", "per_suite": { "ag_news": { "acc": 0.95, "cc": 0.933333, "ck": "english" }, "massive_intent_en": { "acc": 0.693333, "cc": 0.677193, "ck": "english" }, "banking77": { "acc": 0.422, "cc": 0.414395, "ck": "english" }, "sst5": { "acc": 0.371667, "cc": 0.214583, "ck": "english" }, "emotion": { "acc": 0.5925, "cc": 0.511, "ck": "english" }, "xnli_en": { "acc": 0.86, "cc": 0.79, "ck": "english" }, "prompt_injections": { "acc": 0.698276, "cc": 0.396552, "ck": "english" }, "typed_decisions": { "acc": 0.7445, "cc": 0.625641, "ck": "typed" } }, "areas": { "language": 0.674974, "sentiment": 0.362791, "reasoning": 0.593276, "decisions": 0.625641 }, "index": 0.564171, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "python": { "display": "laya (python)", "color_key": "python", "kind": "encoder", "per_suite": { "ag_news": { "acc": 0.95, "cc": 0.933333, "ck": "english" }, "massive_intent_en": { "acc": 0.693333, "cc": 0.677193, "ck": "english" }, "banking77": { "acc": 0.422, "cc": 0.414395, "ck": "english" }, "sst5": { "acc": 0.371667, "cc": 0.214583, "ck": "english" }, "emotion": { "acc": 0.5925, "cc": 0.511, "ck": "english" }, "xnli_en": { "acc": 0.86, "cc": 0.79, "ck": "english" }, "prompt_injections": { "acc": 0.698276, "cc": 0.396552, "ck": "english" }, "typed_decisions": { "acc": 0.7445, "cc": 0.625641, "ck": "typed" } }, "areas": { "language": 0.674974, "sentiment": 0.362791, "reasoning": 0.593276, "decisions": 0.625641 }, "index": 0.564171, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "bekko": { "display": "bekko", "color_key": "bekko", "kind": "python-subprocess", "per_suite": { "ag_news": { "acc": 0.915, "cc": 0.886667 }, "massive_intent_en": { "acc": 0.91, "cc": 0.905263 }, "banking77": { "acc": 0.792, "cc": 0.789263 }, "sst5": { "acc": 0.455, "cc": 0.31875 }, "emotion": { "acc": 0.5825, "cc": 0.499 }, "xnli_en": { "acc": 0.86, "cc": 0.79 }, "prompt_injections": { "acc": 0.508621, "cc": 0.017241 }, "typed_decisions": { "acc": 0.6235, "cc": 0.448352 } }, "areas": { "language": 0.860398, "sentiment": 0.408875, "reasoning": 0.40362, "decisions": 0.448352 }, "index": 0.530311, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "openthai": { "display": "openthai", "color_key": "openthai", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.89, "cc": 0.853333 }, "massive_intent_en": { "acc": 0.92, "cc": 0.915789 }, "banking77": { "acc": 0.656, "cc": 0.651474 }, "sst5": { "acc": 0.43, "cc": 0.2875 }, "emotion": { "acc": 0.59, "cc": 0.508 }, "xnli_en": { "acc": 0.896667, "cc": 0.845 }, "prompt_injections": { "acc": 0.62931, "cc": 0.258621 }, "typed_decisions": { "acc": 0.5345, "cc": 0.317949 } }, "areas": { "language": 0.806865, "sentiment": 0.39775, "reasoning": 0.55181, "decisions": 0.317949 }, "index": 0.518594, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "paw": { "display": "paw (hosted)", "color_key": "paw", "kind": "compiled-program", "per_suite": { "ag_news": { "acc": 0.79, "cc": 0.72 }, "massive_intent_en": { "acc": 0.513333, "cc": 0.487719 }, "banking77": { "acc": 0.418, "cc": 0.410342 }, "sst5": { "acc": 0.398333, "cc": 0.247917 }, "emotion": { "acc": 0.495, "cc": 0.394 }, "xnli_en": { "acc": 0.723333, "cc": 0.585 }, "prompt_injections": { "acc": 0.637931, "cc": 0.275862 }, "typed_decisions": { "acc": 0.5925, "cc": 0.40293 } }, "areas": { "language": 0.539354, "sentiment": 0.320959, "reasoning": 0.430431, "decisions": 0.40293 }, "index": 0.423419, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "paw_local": { "display": "paw (local)", "color_key": "paw", "kind": "compiled-program", "per_suite": { "ag_news": { "acc": 0.7925, "cc": 0.723333 }, "massive_intent_en": { "acc": 0.513333, "cc": 0.487719 }, "banking77": { "acc": 0.418, "cc": 0.410342 }, "sst5": { "acc": 0.398333, "cc": 0.247917 }, "emotion": { "acc": 0.49, "cc": 0.388 }, "xnli_en": { "acc": 0.713333, "cc": 0.57 }, "prompt_injections": { "acc": 0.637931, "cc": 0.275862 }, "typed_decisions": { "acc": 0.5955, "cc": 0.407326 } }, "areas": { "language": 0.540465, "sentiment": 0.317959, "reasoning": 0.422931, "decisions": 0.407326 }, "index": 0.42217, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "clef": { "display": "clef (local)", "color_key": "clef", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.9, "cc": 0.866667 }, "massive_intent_en": { "acc": 0.933333, "cc": 0.929825 }, "banking77": { "acc": 0.954, "cc": 0.953395 }, "sst5": { "acc": 0.603333, "cc": 0.504167 }, "emotion": { "acc": 0.5925, "cc": 0.511 }, "xnli_en": { "acc": 0.813333, "cc": 0.72 }, "prompt_injections": { "acc": 0.586207, "cc": 0.172414 }, "typed_decisions": { "acc": 0.6955, "cc": 0.553846 } }, "areas": { "language": 0.916629, "sentiment": 0.507583, "reasoning": 0.446207, "decisions": 0.553846 }, "index": 0.606066, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "pplx": { "display": "pplx (local)", "color_key": "pplx", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.89, "cc": 0.853333 }, "massive_intent_en": { "acc": 0.93, "cc": 0.926316 }, "banking77": { "acc": 0.804, "cc": 0.801421 }, "sst5": { "acc": 0.57, "cc": 0.4625 }, "emotion": { "acc": 0.64, "cc": 0.568 }, "xnli_en": { "acc": 0.956667, "cc": 0.935 }, "prompt_injections": { "acc": 0.922414, "cc": 0.844828 }, "typed_decisions": { "acc": 0.735, "cc": 0.611722 } }, "areas": { "language": 0.860357, "sentiment": 0.51525, "reasoning": 0.889914, "decisions": 0.611722 }, "index": 0.719311, "coverage": { "suites": 8, "of": 8 }, "complete": true }, "drex": { "display": "drex", "color_key": "drex", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.895, "cc": 0.86 }, "typed_decisions": { "acc": 0.5855, "cc": 0.392674 } }, "areas": { "language": 0.86, "decisions": 0.392674 }, "index": 0.626337, "coverage": { "suites": 2, "of": 8 }, "complete": false }, "clm@4090-win": { "display": "clm", "color_key": "clm", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.4025, "cc": 0.203333 }, "massive_intent_en": { "acc": 0.23, "cc": 0.189474 }, "banking77": { "acc": 0.02, "cc": 0.007105 }, "sst5": { "acc": 0.235, "cc": 0.04375 }, "emotion": { "acc": 0.295, "cc": 0.154 }, "xnli_en": { "acc": 0.62, "cc": 0.43 }, "prompt_injections": { "acc": 0.534483, "cc": 0.068966 }, "typed_decisions": { "acc": 0.3465, "cc": 0.042491 } }, "areas": { "language": 0.133304, "sentiment": 0.098875, "reasoning": 0.249483, "decisions": 0.042491 }, "index": 0.131038, "coverage": { "suites": 8, "of": 8 }, "complete": true, "host": "4090-win" }, "gliner@4090-win": { "display": "gliner", "color_key": "gliner", "kind": "python-subprocess", "per_suite": { "ag_news": { "acc": 0.7025, "cc": 0.603333 }, "massive_intent_en": { "acc": 0.726667, "cc": 0.712281 }, "banking77": { "acc": 0.71, "cc": 0.706184 }, "sst5": { "acc": 0.438333, "cc": 0.297917 }, "emotion": { "acc": 0.565, "cc": 0.478 }, "xnli_en": { "acc": 0.476667, "cc": 0.215 }, "prompt_injections": { "acc": 0.681034, "cc": 0.362069 }, "typed_decisions": { "acc": 0.528, "cc": 0.308425 } }, "areas": { "language": 0.673933, "sentiment": 0.387958, "reasoning": 0.288534, "decisions": 0.308425 }, "index": 0.414713, "coverage": { "suites": 8, "of": 8 }, "complete": true, "host": "4090-win" }, "agentjev@4090-win": { "display": "agentjev", "color_key": "agentjev", "kind": "http-oracle", "per_suite": { "ag_news": { "acc": 0.7975, "cc": 0.73 }, "massive_intent_en": { "acc": 0.55, "cc": 0.526316 }, "banking77": { "acc": 0.552, "cc": 0.546105 }, "sst5": { "acc": 0.438333, "cc": 0.297917 }, "emotion": { "acc": 0.42, "cc": 0.304 }, "xnli_en": { "acc": 0.453333, "cc": 0.18 }, "prompt_injections": { "acc": 0.482759, "cc": -0.034483 }, "typed_decisions": { "acc": 0.772, "cc": 0.665934 } }, "areas": { "language": 0.600807, "sentiment": 0.300959, "reasoning": 0.072759, "decisions": 0.665934 }, "index": 0.410115, "coverage": { "suites": 8, "of": 8 }, "complete": true, "host": "4090-win" } }, "timing": { "modelless": { "clock": "in-process", "method": "Rust in-process per-decision read (the engine's own decision-time measurement; zero network, zero IPC)", "suites": 8, "p50_geomean_ms": 0.1512, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "hybrid": { "clock": "in-process", "method": "Rust in-process; seat+arm cells time the full composed path (modelless seat + arm), arm-only cells (A1/H2) time the composition forward alone - the modelless seat is NOT in that number (instinct Issue 007)", "suites": 8, "p50_geomean_ms": 0.1, "n_used": 2, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted); 6 arm-only cell(s) are the composition marginal (the fusion forward over the modelless seat - the seat solve is not in that number), counted, never plotted", "n_arm_only": 6 }, "encoder": { "clock": "in-process", "method": "Rust in-process GPU encode+head forward (Metal on the m3 hosts; the cell's `device` field names it)", "suites": 8, "p50_geomean_ms": 23.6778, "n_used": 3, "n_unquotable": 0, "n_unjudged": 5, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted); 4 fallback spoke(s) carry the answering tier's timing, unjudged here \u2014 shown in the suite tables, never plotted" }, "laya": { "clock": "in-process", "method": "Rust in-process forward (device per host row: laya_device \u2014 metal/cuda/cpu)", "suites": 8, "p50_geomean_ms": 30.9575, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "python": { "clock": "subprocess-jsonl", "method": "JSONL subprocess round-trip to the reference torch runtime \u2014 IPC included, by the lane's own protocol", "suites": 8, "p50_geomean_ms": 47.7794, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "bekko": { "clock": "subprocess-jsonl", "method": "JSONL subprocess round-trip to their BekkoSentenceTransformer runtime", "suites": 8, "p50_geomean_ms": 227.9479, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "openthai": { "clock": "http", "method": "HTTP round-trip to their OpenThai-SystemOne teacher over a loopback FastAPI subprocess", "suites": 8, "p50_geomean_ms": 291.4757, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "paw": { "clock": "http", "method": "HTTP round-trip to their hosted REST compile+answer service (curl subprocess per question; the cell also discloses their server-side latency separately)", "suites": 8, "p50_geomean_ms": 1058.4084, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "paw_local": { "clock": "local-runtime", "method": "Local llama.cpp runtime behind a Python subprocess (their programasweights paw.function; warm cache)", "suites": 8, "p50_geomean_ms": 33.8382, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "clef": { "clock": "http", "method": "HTTP round-trip to a local clef-flash-4bit (community 4-bit quant of their 9B) served on the m3 behind a loopback shim; the hosted 27B posture is owner-gated \u2014 never pooled with these cells", "suites": 8, "p50_geomean_ms": 680.877, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "pplx": { "clock": "http", "method": "HTTP round-trip to a local pplx-decider-v1.1-27b (the community MLX 4-bit quant of their 27B, readout bf16; fidelity vs their bf16: 33/33 argmax, mean KL 0.0014 \u2014 the port's own validation) served on the m3 behind the reference-wire adapter; their bf16 hosted posture is a different row, never pooled", "suites": 8, "p50_geomean_ms": null, "n_used": 0, "n_unquotable": 0, "n_unjudged": 8, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "drex": { "clock": "http", "method": "HTTP round-trip to their Drex-DLM SystemOne server (4090: their Python serve.py, BF16 CUDA \u2014 quotable timing; m3: their llama.cpp edlm fork, Q8_0 Metal \u2014 accuracy-only, its preflight REFUSED and the ms figures are never quoted; the postures agree to 0.001 on the headline). Weights CC BY-NC 4.0 \u2014 measurement-only, never a distill teacher, never a product", "suites": 2, "p50_geomean_ms": null, "n_used": 0, "n_unquotable": 0, "n_unjudged": 2, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "clm@4090-win": { "clock": "http", "method": "HTTP round-trip to their served reference (vLLM pooling on the 4090 window)", "suites": 8, "p50_geomean_ms": 44.4638, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "gliner@4090-win": { "clock": "subprocess-python", "method": "Python subprocess round-trip (their gliner2 package answers per batch; interpreter load excluded)", "suites": 8, "p50_geomean_ms": 22.8455, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" }, "agentjev@4090-win": { "clock": "http", "method": "HTTP round-trip to their jev_service (step-600 tensors; bf16 wobble disclosed in the bench record)", "suites": 8, "p50_geomean_ms": 45.173, "n_used": 8, "n_unquotable": 0, "n_unjudged": 0, "note": "population = the suites behind the lane's index; p50 geometric mean over latency_quotable cells only (unfit timing is shown in the tables, never plotted)" } }, "scope": "primary-host rows; a lane the primary host never ran rolls up from its serving host under a host-tagged lane key (clm@4090-win); a product lane's fallback spokes roll up from the answering tier (marked fallback, drawn as triangles \u2014 coverage stays the lane's own measured count)" }, "s1mb": { "version": 1, "suites": [ "s1mb_noul", "s1mb_score", "s1mb_choice" ], "lanes": [ { "key": "modelless", "display": "Reflex", "model": "the modelless engine (count tables + sigmoid calibration)", "note": "the free floor \u2014 runs in the browser tab", "cells": { "s1mb_noul": { "n": 6173, "acc": 0.7055, "latency_quotable": true }, "s1mb_score": { "n": 2571, "acc": 0.4982, "latency_quotable": true }, "s1mb_choice": { "n": 4329, "acc": 0.2296, "latency_quotable": true } }, "avg": 0.4778, "coverage": { "suites": 3, "of": 3 } }, { "key": "hybrid", "display": "Instinct", "model": "the served hybrid arm over the same seat", "note": "the served arm is the modelless half (A0) on every s1mb suite today \u2014 no specialist arm cleared its certification gate, so the lane's cells are the A0 reads (shown, labeled, per the show-losses law)", "cells": { "s1mb_noul": { "n": 6173, "acc": 0.7055, "serves": "A0" }, "s1mb_score": { "n": 2571, "acc": 0.4982, "serves": "A0" }, "s1mb_choice": { "n": 4329, "acc": 0.2296, "serves": "A0" } }, "avg": 0.4778, "coverage": { "suites": 3, "of": 3 } }, { "key": "laya", "display": "laya (rust)", "model": "the zero-shot encoder lane (english checkpoint, round-1 breadth)", "note": "the encoder family's measured zero-shot reading on this benchmark \u2014 the s1mb suites route the english checkpoint, measured on the GPU serving host; the encoder head budget refuses the pick-an-answer suite's option list (it runs to hundreds of rendered options per question), so that cell is an honest gap, not a zero", "cells": { "s1mb_noul": { "n": 6173, "acc": 0.6964 }, "s1mb_score": { "n": 2571, "acc": 0.259 } }, "avg": 0.4777, "coverage": { "suites": 2, "of": 3 }, "gap_suites": [ "s1mb_choice" ] }, { "key": "encoder", "display": "Rethink", "model": "trained per-option encoder head", "cells": { "s1mb_noul": { "n": 6173, "acc": 0.7055, "serves": "tier-fallback", "fallback": true }, "s1mb_score": { "n": 2571, "acc": 0.4982, "serves": "tier-fallback", "fallback": true }, "s1mb_choice": { "n": 4329, "acc": 0.2296, "serves": "tier-fallback", "fallback": true } }, "avg": null, "coverage": { "suites": 0, "of": 3 }, "note": "no trained head has passed its earn gate on these suites yet, so the lane serves the answering tier (shown above as the served-by reading) \u2014 the gap is the disclosure, never a padded number" } ], "disclosure": "forced-pick accuracy on our own 50/50 corpus/test split of the benchmark (every lane reads the same halves) \u2014 NOT comparable to the benchmark's own leaderboard, whose skill score is a different scale we do not publish. avg is the unweighted mean over the three suite accuracies. A missing cell is a disclosed gap, never a zero." }, "abstention": { "version": 1, "edition": "2026-10-11", "lanes": { "modelless": { "causes": { "distance_gate": 4786, "score_gate": 4265 }, "total": 9051, "shares": { "distance_gate": 0.5288, "score_gate": 0.4712 }, "suites_with_stamps": 12, "suites_covered": 12 } }, "scope": "per-lane abstain-cause rollups over the primary host's own cells, all suites (dataset + decision-set); counts are the harness's own abstain-cause stamps, never derived from rates; the harness stamps causes on the modelless lane's results (reflex a7475c7), so other lanes are absent here \u2014 never zeros" }, "crosswalk": { "caveat": "JDI-comparable crosswalk, NOT a JDI board row \u2014 different corpus, protocol caps, hardware; board membership requires their full frozen suite.", "suites": [ { "name": "typed_decisions", "n_cases": 400, "n_questions": 2000, "pin": "fnv1a64-6e37760ee2a5b6c9", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 2000, "accuracy": 0.6955, "macro_f1": 0.674190133619508, "jdi_chance": 0.2925, "jdi_skill": 0.5696113074204947, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 2000, "accuracy": 0.735, "macro_f1": 0.7219508452915961, "jdi_chance": 0.2925, "jdi_skill": 0.6254416961130742, "serves": null, "record_only": false }, { "lane": "drex", "display": "drex (reference)", "model": "drex-dlm", "n": 2000, "accuracy": 0.5855, "macro_f1": 0.5788517787061889, "jdi_chance": 0.2925, "jdi_skill": 0.4141342756183746, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "H2(\u03b2=0.5,nmin=2,\u03c4=2)", "n": 2000, "accuracy": 0.6475, "macro_f1": 0.6052308469932142, "jdi_chance": 0.2925, "jdi_skill": 0.5017667844522968, "serves": "H2(\u03b2=0.5,nmin=2,\u03c4=2)", "record_only": false }, { "lane": "encoder", "display": "Rethink", "model": "ENC-typed_encoder_v2 (laya-typed encoder + NLEH v2 per-option head)", "n": 2000, "accuracy": 0.755, "macro_f1": 0.7445330507017898, "jdi_chance": 0.2925, "jdi_skill": 0.6537102473498233, "serves": "\u2717 (encoder class refused at serve \u2014 the incumbent arm serves; instinct issue 014 C1, seating per issue 017)", "record_only": true }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 2000, "accuracy": 0.57, "macro_f1": 0.539329054680424, "jdi_chance": 0.2925, "jdi_skill": 0.39222614840989395, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 2000, "accuracy": 0.5345, "macro_f1": 0.5371199035294525, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 2000, "accuracy": 0.6235, "macro_f1": 0.6118980306296695, "jdi_chance": 0.2925, "jdi_skill": 0.46784452296819795, "serves": null, "record_only": false } ], "excluded": [ "paw_local@typed_decisions: no measured accuracy on this suite", "paw@typed_decisions: no measured accuracy on this suite" ] }, { "name": "ag_news", "n_cases": 400, "n_questions": 400, "pin": "fnv1a64-07eff23c06505584", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 400, "accuracy": 0.9, "macro_f1": 0.8917746035059467, "jdi_chance": 0.3075, "jdi_skill": 0.8555956678700362, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 400, "accuracy": 0.89, "macro_f1": 0.8807080810359893, "jdi_chance": 0.3075, "jdi_skill": 0.8411552346570398, "serves": null, "record_only": false }, { "lane": "drex", "display": "drex (reference)", "model": "drex-dlm", "n": 400, "accuracy": 0.895, "macro_f1": 0.8859395200866416, "jdi_chance": 0.3075, "jdi_skill": 0.848375451263538, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "H2(\u03b2=0.25,nmin=2,\u03c4=2)", "n": 400, "accuracy": 0.8975, "macro_f1": 0.8878275558568157, "jdi_chance": 0.3075, "jdi_skill": 0.8519855595667869, "serves": "H2(\u03b2=0.25,nmin=2,\u03c4=2)", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 400, "accuracy": 0.8825, "macro_f1": 0.8741933646432797, "jdi_chance": 0.3075, "jdi_skill": 0.8303249097472923, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 400, "accuracy": 0.89, "macro_f1": 0.882644318569296, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 400, "accuracy": 0.915, "macro_f1": 0.9088637583052136, "jdi_chance": 0.3075, "jdi_skill": 0.8772563176895307, "serves": null, "record_only": false } ], "excluded": [ "encoder@ag_news: digest (absent) != the pin fnv1a64-07eff23c06505584 (different population \u2014 excluded, never mixed)", "paw_local@ag_news: no measured accuracy on this suite", "paw@ag_news: no measured accuracy on this suite" ] }, { "name": "emotion", "n_cases": 400, "n_questions": 400, "pin": "fnv1a64-6feefd1c947dd008", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 400, "accuracy": 0.5925, "macro_f1": 0.4914428254411061, "jdi_chance": 0.2975, "jdi_skill": 0.41992882562277584, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 400, "accuracy": 0.64, "macro_f1": 0.5364780003205954, "jdi_chance": 0.2975, "jdi_skill": 0.48754448398576516, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "A0", "n": 400, "accuracy": 0.885, "macro_f1": 0.8014284327939634, "jdi_chance": 0.2975, "jdi_skill": 0.8362989323843416, "serves": "A0", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 400, "accuracy": 0.885, "macro_f1": 0.8014284327939634, "jdi_chance": 0.2975, "jdi_skill": 0.8362989323843416, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 400, "accuracy": 0.59, "macro_f1": 0.4844213484626461, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 400, "accuracy": 0.5825, "macro_f1": 0.4664338387697607, "jdi_chance": 0.2975, "jdi_skill": 0.40569395017793597, "serves": null, "record_only": false } ], "excluded": [ "encoder@emotion: tier-fallback (the answering tier's read, not this lane's own)", "paw_local@emotion: no measured accuracy on this suite", "paw@emotion: no measured accuracy on this suite" ] }, { "name": "sst5", "n_cases": 600, "n_questions": 600, "pin": "fnv1a64-ad64a6cb0039e916", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 600, "accuracy": 0.6033333333333334, "macro_f1": 0.5752050497694366, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.4541284403669726, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 600, "accuracy": 0.57, "macro_f1": 0.5137560655946953, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.4082568807339449, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "A1", "n": 600, "accuracy": 0.4216666666666667, "macro_f1": 0.3645178225884921, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.20412844036697253, "serves": "A1", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 600, "accuracy": 0.39666666666666667, "macro_f1": 0.32101890185812104, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.16972477064220184, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 600, "accuracy": 0.43, "macro_f1": 0.41049715078874643, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 600, "accuracy": 0.455, "macro_f1": 0.30583133122619166, "jdi_chance": 0.2733333333333333, "jdi_skill": 0.25000000000000006, "serves": null, "record_only": false } ], "excluded": [ "encoder@sst5: digest (absent) != the pin fnv1a64-ad64a6cb0039e916 (different population \u2014 excluded, never mixed)", "paw_local@sst5: no measured accuracy on this suite", "paw@sst5: no measured accuracy on this suite" ] }, { "name": "prompt_injections", "n_cases": 116, "n_questions": 116, "pin": "fnv1a64-5080803ec29f2423", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 116, "accuracy": 0.5862068965517241, "macro_f1": 0.5166666666666666, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.14285714285714268, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 116, "accuracy": 0.9224137931034483, "macro_f1": 0.9222693768148313, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.8392857142857143, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "A1", "n": 116, "accuracy": 0.853448275862069, "macro_f1": 0.8529126575669427, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.6964285714285714, "serves": "A1", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 116, "accuracy": 0.7672413793103449, "macro_f1": 0.7642808760442538, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.5178571428571429, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 116, "accuracy": 0.6293103448275862, "macro_f1": 0.5820695433598659, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 116, "accuracy": 0.5086206896551724, "macro_f1": 0.3789799943646098, "jdi_chance": 0.5172413793103449, "jdi_skill": 0.0, "serves": null, "record_only": false } ], "excluded": [ "encoder@prompt_injections: tier-fallback (the answering tier's read, not this lane's own)", "paw_local@prompt_injections: no measured accuracy on this suite", "paw@prompt_injections: no measured accuracy on this suite" ] }, { "name": "xnli_en", "n_cases": 300, "n_questions": 300, "pin": "fnv1a64-b136750e5af1d412", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 300, "accuracy": 0.8133333333333334, "macro_f1": 0.8150782669962471, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.72, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 300, "accuracy": 0.9566666666666667, "macro_f1": 0.9568105065666042, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.9349999999999998, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "A0", "n": 300, "accuracy": 0.5233333333333333, "macro_f1": 0.5070059473023049, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.285, "serves": "A0", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 300, "accuracy": 0.5233333333333333, "macro_f1": 0.5061129338341462, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.285, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 300, "accuracy": 0.8966666666666666, "macro_f1": 0.8966195977749251, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 300, "accuracy": 0.86, "macro_f1": 0.858750405020872, "jdi_chance": 0.3333333333333333, "jdi_skill": 0.7899999999999998, "serves": null, "record_only": false } ], "excluded": [ "encoder@xnli_en: digest (absent) != the pin fnv1a64-b136750e5af1d412 (different population \u2014 excluded, never mixed)", "paw_local@xnli_en: no measured accuracy on this suite", "paw@xnli_en: no measured accuracy on this suite" ] }, { "name": "massive_intent_en", "n_cases": 300, "n_questions": 300, "pin": "fnv1a64-49390c8b80475849", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 300, "accuracy": 0.9333333333333333, "macro_f1": 0.9284468303231186, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.927007299270073, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 300, "accuracy": 0.93, "macro_f1": 0.9268372783806814, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.9233576642335767, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "H2(\u03b2=1,nmin=2,\u03c4=4)", "n": 300, "accuracy": 0.84, "macro_f1": 0.8317455255770921, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.8248175182481752, "serves": "H2(\u03b2=1,nmin=2,\u03c4=4)", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 300, "accuracy": 0.78, "macro_f1": 0.7698039912157559, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.759124087591241, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 300, "accuracy": 0.92, "macro_f1": 0.9142354668289988, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 300, "accuracy": 0.91, "macro_f1": 0.9056898356680406, "jdi_chance": 0.08666666666666667, "jdi_skill": 0.9014598540145986, "serves": null, "record_only": false } ], "excluded": [ "encoder@massive_intent_en: tier-fallback (the answering tier's read, not this lane's own)", "paw_local@massive_intent_en: no measured accuracy on this suite", "paw@massive_intent_en: no measured accuracy on this suite" ] }, { "name": "banking77", "n_cases": 500, "n_questions": 500, "pin": "fnv1a64-dd8ab35333abb82a", "rows": [ { "lane": "clef", "display": "clef (local)", "model": "clef-flash-4bit", "n": 500, "accuracy": 0.954, "macro_f1": 0.9536241896631502, "jdi_chance": 0.014, "jdi_skill": 0.9533468559837728, "serves": null, "record_only": false }, { "lane": "pplx", "display": "pplx (local)", "model": "autojev-qwen3.8-27b@pplx-decider-v1.1-27b-4bit", "n": 500, "accuracy": 0.804, "macro_f1": 0.7897567672968737, "jdi_chance": 0.014, "jdi_skill": 0.8012170385395538, "serves": null, "record_only": false }, { "lane": "hybrid", "display": "Instinct", "model": "H2(\u03b2=2,nmin=8,\u03c4=8)", "n": 500, "accuracy": 0.854, "macro_f1": 0.8527493620885521, "jdi_chance": 0.014, "jdi_skill": 0.8519269776876267, "serves": "H2(\u03b2=2,nmin=8,\u03c4=8)", "record_only": false }, { "lane": "modelless", "display": "KatGPT", "model": "modelless", "n": 500, "accuracy": 0.842, "macro_f1": 0.8362254543690752, "jdi_chance": 0.014, "jdi_skill": 0.8397565922920892, "serves": null, "record_only": false }, { "lane": "openthai", "display": "openthai (reference)", "model": "openthai-systemone", "n": 500, "accuracy": 0.656, "macro_f1": 0.6355435167453176, "jdi_chance": null, "jdi_skill": null, "serves": null, "record_only": false }, { "lane": "bekko", "display": "bekko (reference)", "model": "hotchpotch/bekko-system-one-v0-400m", "n": 500, "accuracy": 0.792, "macro_f1": 0.7805128981026606, "jdi_chance": 0.014, "jdi_skill": 0.7890466531440162, "serves": null, "record_only": false } ], "excluded": [ "encoder@banking77: tier-fallback (the answering tier's read, not this lane's own)", "paw_local@banking77: no measured accuracy on this suite", "paw@banking77: no measured accuracy on this suite" ] } ], "board_reference": { "note": "The JDI board's own rows \u2014 their frozen corpus, their protocol, their hardware (edition release-v4 / Decision Index 0.4.1, generated 2026-10-22, pinned in rethink .research/data/ + Issue 032 T10; Full (balanced_skill) is now 100% PRIVATE \u2014 0.60 private-same-skills + 0.40 private-new-domains, stock-model-mapped \u2014 while public benchmarks no longer count toward it, so the two columns are rendered SEPARATE and never blended or compared across editions; the instrument repair: rethink bench 072 proved public_skill == the per-area category prints and bounded the per-case divergence class). REFERENCE ONLY: no number here is comparable with the measured rows above without the caveat's full weight. reflex-27b-pmscr is a THIRD-PARTY fork (kshetrajna12) wearing our lane's name \u2014 never a reflex row.", "columns": [ "entrant", "balanced_raw", "full_skill", "public_skill", "median_ms" ], "rows": [ { "entrant": "mammoth-1 (Base Labs, 27B)", "balanced_raw": 81.19, "full_skill": 68.44, "public_skill": 75.34, "median_ms": 220.1, "note": "board-#1 BOTH aggregates; MIT - the first permissively-licensed #1" }, { "entrant": "StartLux-Decision v1.1 Instant (27B)", "balanced_raw": 79.39, "full_skill": 67.92, "public_skill": 72.88, "median_ms": 79.8, "note": "our startlux lane's model; CC BY-NC 4.0 (measurement-only posture)" }, { "entrant": "elephant-1 (Base Labs, 9B)", "balanced_raw": 78.88, "full_skill": 60.26, "public_skill": 72.24, "median_ms": 113.7, "note": "public #4; the M5 sub-15B bar holder; MIT" }, { "entrant": "Fastino GLiDE v1.2 (28B)", "balanced_raw": 78.89, "full_skill": 71.52, "public_skill": 71.99, "median_ms": 92.9, "note": "" }, { "entrant": "Perplexity Decider v1.2 (27B)", "balanced_raw": 75.59, "full_skill": 67.98, "public_skill": 67.74, "median_ms": 144.4, "note": "our pplx lane's model (v1.1 on the old pin, now v1.2)" }, { "entrant": "Drex max v1", "balanced_raw": 75.4, "full_skill": 66.6, "public_skill": 67.52, "median_ms": 103.6, "note": "our drex lane's family" }, { "entrant": "Torchcast Decision 27B", "balanced_raw": 73.64, "full_skill": 61.49, "public_skill": 65.1, "median_ms": 99.9, "note": "was the 0.3 public-only #1 (65.10 - the same print carries over)" }, { "entrant": "Cloudflare clef (27B)", "balanced_raw": 71.07, "full_skill": 54.92, "public_skill": 61.71, "median_ms": 102.1, "note": "our clef lane's model - the SAME public print as the 0.3 pin" }, { "entrant": "JEV (jev-1.13.0)", "balanced_raw": 68.55, "full_skill": 59.22, "public_skill": 57.96, "median_ms": 524.0, "note": "the M1 pass-Jev bar is jev's 0.4.1 PRINT: 60.70" }, { "entrant": "reflex-27b-pmscr (kshetrajna12 - THIRD-PARTY, NOT OURS)", "balanced_raw": 67.88, "full_skill": 58.77, "public_skill": 57.34, "median_ms": 148.5, "note": "a fork wearing our lane's name - never quoted as a reflex row" }, { "entrant": "Cloudflare clef-flash (9B)", "balanced_raw": 66.53, "full_skill": 44.43, "public_skill": 56.15, "median_ms": 53.4, "note": "our clef lane's model" }, { "entrant": "Winnow-12B (Q8 GGUF/llama.cpp)", "balanced_raw": 62.84, "full_skill": 46.57, "public_skill": 50.66, "median_ms": 72.5, "note": "" }, { "entrant": "GLiNER2.5-Decide (our GLiNER lane)", "balanced_raw": 33.08, "full_skill": 10.14, "public_skill": 11.48, "median_ms": 23.3, "note": "" }, { "entrant": "CLM-v0.1-8B (our CLM lane)", "balanced_raw": 28.52, "full_skill": 6.49, "public_skill": 7.41, "median_ms": 46.8, "note": "" }, { "entrant": "laya (our laya lane's source)", "balanced_raw": 28.26, "full_skill": 9.21, "public_skill": 6.19, "median_ms": 5.8, "note": "still among the fastest on board" } ] }, "blog_banking77_mf1": { "note": "Clef's own blog table on BANKING77 (their run, their corpus \u2014 macro-F1 % and median ms; pinned in reflex .research/005). Our banking77 row above is OUR split \u2014 the same suite family, never the same population.", "rows": [ { "model": "Clef", "macro_f1_pct": 94.2, "median_ms": 209.3 }, { "model": "Clef-flash", "macro_f1_pct": 90.93, "median_ms": 38.8 }, { "model": "Kev-9B", "macro_f1_pct": 84.83, "median_ms": null }, { "model": "Jev", "macro_f1_pct": 79.74, "median_ms": 524.1 }, { "model": "laya", "macro_f1_pct": 14.29, "median_ms": 5.8 } ] } } }