{ "schema": "research-release-practical-aggregate-v1", "status": "Local draft \u2014 practical guidance successor; independent scientific/editorial review and browser QA pending; not published", "new_jev_1_14": { "status": "reading_and_one_step_complete_only", "model": "jev-1.14.0", "aggregates": { "schema": "jev114-independent-results-v1", "stages": { "reading": { "assigned": 60, "valid": 60, "invalid": 0, "correct": 60, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "classes": { "0": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 }, "1": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 } }, "probability_denominator": 60, "brier_valid_only": 0.0015700000000000004, "auc_valid_only": 1.0, "mean_p_valid_only": 0.5193333333333333, "predicted_one_valid_only": 30, "gate": "PASS" }, "one_step": { "assigned": 529, "valid": 527, "invalid": 2, "correct": 356, "accuracy_assigned": 0.6729678638941399, "accuracy_valid_only": 0.6755218216318786, "classes": { "0": { "assigned": 265, "valid": 265, "correct": 116, "accuracy_assigned": 0.4377358490566038, "accuracy_valid_only": 0.4377358490566038 }, "1": { "assigned": 264, "valid": 262, "correct": 240, "accuracy_assigned": 0.9090909090909091, "accuracy_valid_only": 0.916030534351145 } }, "probability_denominator": 527, "brier_valid_only": 0.21656812144212523, "auc_valid_only": 0.8276537519804119, "mean_p_valid_only": 0.5400759013282732, "predicted_one_valid_only": 389, "gate": "FAIL" } }, "historical_jev113": { "reading": { "assigned": 60, "valid": 60, "invalid": 0, "correct": 60, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "classes": { "0": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 }, "1": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 } }, "probability_denominator": 60, "brier_valid_only": 0.0017600000000000003, "auc_valid_only": 1.0, "mean_p_valid_only": 0.524, "predicted_one_valid_only": 30, "gate": "PASS" }, "one_step": { "assigned": 529, "valid": 529, "invalid": 0, "correct": 377, "accuracy_assigned": 0.7126654064272212, "accuracy_valid_only": 0.7126654064272212, "classes": { "0": { "assigned": 265, "valid": 265, "correct": 184, "accuracy_assigned": 0.6943396226415094, "accuracy_valid_only": 0.6943396226415094 }, "1": { "assigned": 264, "valid": 264, "correct": 193, "accuracy_assigned": 0.7310606060606061, "accuracy_valid_only": 0.7310606060606061 } }, "probability_denominator": 529, "brier_valid_only": 0.20950132325141776, "auc_valid_only": 0.8104988564894225, "mean_p_valid_only": 0.5046502835538752, "predicted_one_valid_only": 274, "gate": "FAIL" } }, "paired_114_vs_historical_113": { "reading": { "all_assigned": { "n": 60, "both_correct": 60, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 }, "both_valid": { "n": 60, "both_correct": 60, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 }, "truth_classes": { "0": { "all_assigned": { "n": 30, "both_correct": 30, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 }, "both_valid": { "n": 30, "both_correct": 30, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 } }, "1": { "all_assigned": { "n": 30, "both_correct": 30, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 }, "both_valid": { "n": 30, "both_correct": 30, "only_114_correct": 0, "only_comparator_correct": 0, "neither_correct": 0 } } }, "historical_metrics_on_both_valid": { "assigned": 60, "valid": 60, "invalid": 0, "correct": 60, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "classes": { "0": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 }, "1": { "assigned": 30, "valid": 30, "correct": 30, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0 } }, "probability_denominator": 60, "brier_valid_only": 0.0017600000000000003, "auc_valid_only": 1.0, "mean_p_valid_only": 0.524, "predicted_one_valid_only": 30, "gate": "PASS" } }, "one_step": { "all_assigned": { "n": 529, "both_correct": 297, "only_114_correct": 59, "only_comparator_correct": 80, "neither_correct": 93 }, "both_valid": { "n": 527, "both_correct": 297, "only_114_correct": 59, "only_comparator_correct": 79, "neither_correct": 92 }, "truth_classes": { "0": { "all_assigned": { "n": 265, "both_correct": 110, "only_114_correct": 6, "only_comparator_correct": 74, "neither_correct": 75 }, "both_valid": { "n": 265, "both_correct": 110, "only_114_correct": 6, "only_comparator_correct": 74, "neither_correct": 75 } }, "1": { "all_assigned": { "n": 264, "both_correct": 187, "only_114_correct": 53, "only_comparator_correct": 6, "neither_correct": 18 }, "both_valid": { "n": 262, "both_correct": 187, "only_114_correct": 53, "only_comparator_correct": 5, "neither_correct": 17 } } }, "historical_metrics_on_both_valid": { "assigned": 527, "valid": 527, "invalid": 0, "correct": 376, "accuracy_assigned": 0.713472485768501, "accuracy_valid_only": 0.713472485768501, "classes": { "0": { "assigned": 265, "valid": 265, "correct": 184, "accuracy_assigned": 0.6943396226415094, "accuracy_valid_only": 0.6943396226415094 }, "1": { "assigned": 262, "valid": 262, "correct": 192, "accuracy_assigned": 0.732824427480916, "accuracy_valid_only": 0.732824427480916 } }, "probability_denominator": 527, "brier_valid_only": 0.2093089184060721, "auc_valid_only": 0.8114503816793893, "mean_p_valid_only": 0.5047058823529412, "predicted_one_valid_only": 273, "gate": "FAIL" } } }, "usage": { "reading": { "attempts": 60, "known": 60, "missing": 0, "input_tokens_known_sum": 50136, "output_tokens_known_sum": 2520, "billing_unknown": true, "billed_usd": null }, "one_step": { "attempts": 529, "known": 528, "missing": 1, "input_tokens_known_sum": 589200, "output_tokens_known_sum": 47512, "billing_unknown": true, "billed_usd": null } }, "latency_by_phase": { "jev114-public-one-step-fast-v1": { "n": 18, "sum_s": 4.190995743963867, "mean_s": 0.23283309688688153, "median_s": 0.19776205974631011, "min_s": 0.157082200050354, "max_s": 0.42606170009821653, "first_utc": "2026-10-16T17:33:07.243681+00:00", "last_utc": "2026-10-16T17:33:28.389775+00:00" }, "jev114-public-one-step-final-recovery-v1": { "n": 99, "sum_s": 37.204905355814844, "mean_s": 0.37580712480621054, "median_s": 0.27031416865065694, "min_s": 0.1757841627113521, "max_s": 2.0468590576201677, "first_utc": "2026-10-16T18:04:44.540540+00:00", "last_utc": "2026-10-16T18:13:33.650499+00:00" }, "jev114-public-one-step-overload-recovery-v1": { "n": 186, "sum_s": 53.22997603658587, "mean_s": 0.28618266686336485, "median_s": 0.21604815870523453, "min_s": 0.1663033477962017, "max_s": 11.486157171428204, "first_utc": "2026-10-16T17:40:31.953642+00:00", "last_utc": "2026-10-16T17:56:52.364177+00:00" }, "jev114-public-one-step-v1": { "n": 226, "sum_s": 52.25931499246508, "mean_s": 0.23123590704630567, "median_s": 0.21897016395814717, "min_s": 0.16338064521551132, "max_s": 0.5061764009296894, "first_utc": "2026-10-16T15:31:39.851769+00:00", "last_utc": "2026-10-16T17:25:05.231610+00:00" }, "jev114-public-reading-v1": { "n": 60, "sum_s": 12.535191596020013, "mean_s": 0.20891985993366688, "median_s": 0.20174817345105112, "min_s": 0.16451005125418305, "max_s": 0.37486856197938323, "first_utc": "2026-10-15T23:20:48.412718+00:00", "last_utc": "2026-10-15T23:50:31.897845+00:00" } }, "current_hosted_comparison": { "microsoft/microsoft-decision-1": { "reading": { "assigned": 60, "valid": 60.0, "correct": 60.0, "classes": { "0": { "assigned": 30, "valid": 30.0, "correct": 30.0 }, "1": { "assigned": 30, "valid": 30.0, "correct": 30.0 } }, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "provider": "Azure", "gate": "PASS", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" }, "one_step": { "assigned": 529, "valid": 529.0, "correct": 285.0, "classes": { "0": { "assigned": 265, "valid": 265.0, "correct": 238.0 }, "1": { "assigned": 264, "valid": 264.0, "correct": 47.0 } }, "accuracy_assigned": 0.5387523629489603, "accuracy_valid_only": 0.5387523629489603, "provider": "Azure", "gate": "FAIL", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" } }, "cloudflare/clef-flash": { "reading": { "assigned": 60, "valid": 60.0, "correct": 60.0, "classes": { "0": { "assigned": 30, "valid": 30.0, "correct": 30.0 }, "1": { "assigned": 30, "valid": 30.0, "correct": 30.0 } }, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "provider": "PrimeIntellect", "gate": "PASS", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" }, "one_step": { "assigned": 529, "valid": 529.0, "correct": 291.0, "classes": { "0": { "assigned": 265, "valid": 265.0, "correct": 183.0 }, "1": { "assigned": 264, "valid": 264.0, "correct": 108.0 } }, "accuracy_assigned": 0.5500945179584121, "accuracy_valid_only": 0.5500945179584121, "provider": "PrimeIntellect", "gate": "FAIL", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" } }, "cloudflare/clef": { "reading": { "assigned": 60, "valid": 60.0, "correct": 60.0, "classes": { "0": { "assigned": 30, "valid": 30.0, "correct": 30.0 }, "1": { "assigned": 30, "valid": 30.0, "correct": 30.0 } }, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "provider": "PrimeIntellect", "gate": "PASS", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" }, "one_step": { "assigned": 529, "valid": 529.0, "correct": 304.0, "classes": { "0": { "assigned": 265, "valid": 265.0, "correct": 248.0 }, "1": { "assigned": 264, "valid": 264.0, "correct": 56.0 } }, "accuracy_assigned": 0.5746691871455577, "accuracy_valid_only": 0.5746691871455577, "provider": "PrimeIntellect", "gate": "FAIL", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" } }, "perplexity/pplx-decider-v1.1-27b": { "reading": { "assigned": 60, "valid": 60.0, "correct": 60.0, "classes": { "0": { "assigned": 30, "valid": 30.0, "correct": 30.0 }, "1": { "assigned": 30, "valid": 30.0, "correct": 30.0 } }, "accuracy_assigned": 1.0, "accuracy_valid_only": 1.0, "provider": "Perplexity", "gate": "PASS", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" }, "one_step": { "assigned": 529, "valid": 529.0, "correct": 459.0, "classes": { "0": { "assigned": 265, "valid": 265.0, "correct": 226.0 }, "1": { "assigned": 264, "valid": 264.0, "correct": 233.0 } }, "accuracy_assigned": 0.8676748582230623, "accuracy_valid_only": 0.8676748582230623, "provider": "Perplexity", "gate": "FAIL", "evidence": "accepted full export cross-checked against successor report; not newly revalidated raw comparator responses" } } }, "accounting": { "local_allowance_usd": "59.970399942", "approved_cap_usd": "60", "remaining_allowance_usd": "0.029600058", "next_attempt_allowance_usd": "0.10", "sufficient_for_next_attempt": false, "billed_usd": null, "billing_unknown": true }, "limitations": [ "Historical development cohorts, not fresh held-out evaluation.", "Jev 1.13 comparison is historical, not contemporaneous: renderer/provider/time confounds preclude causal version or general regression claims.", "Probability metrics exclude invalid responses without imputation; assigned accuracy counts failures as incorrect.", "Only new reading and one_step complete; all other Jev 1.14 families pending.", "No page publication or whole-benchmark completion.", "Latency is phase-specific recorded transport timing, not throughput or provider compute time." ], "invalid_summary": [ { "http_status": 529, "reason": "HTTP_OR_CAPTURE_FAILURE", "usage_known": false }, { "http_status": 200, "reason": "NORMALIZATION_CONFIDENCE", "usage_known": true } ] }, "unrun_families": { "sufficiency": { "status": "unrun_unavailable", "metrics": null }, "news": { "status": "unrun_unavailable", "metrics": null }, "domain_representation": { "status": "unrun_unavailable", "metrics": null }, "menu_escape": { "status": "unrun_unavailable", "metrics": null }, "exit": { "status": "unrun_unavailable", "metrics": null }, "coverage": { "status": "unrun_unavailable", "metrics": null } } }, "tables": { "models": { "headers": [ "Arm", "Requested canonical ID", "Serving provider" ], "rows": [ [ "Microsoft Decision 1", "microsoft/microsoft-decision-1-20261009", "Azure" ], [ "Clef Flash", "cloudflare/clef-flash", "PrimeIntellect" ], [ "Clef", "cloudflare/clef", "PrimeIntellect" ], [ "Perplexity Decider V1.1 27B", "perplexity/pplx-decider-v1.1-27b-20261006", "Perplexity" ] ] }, "prerequisites": { "headers": [ "Model / track", "Reading correct", "One-step correct", "One-step accuracy", "Current computation gate" ], "rows": [ [ "Jev 1.13 (historical)", "60/60", "377/529", "71.3%", "historical" ], [ "Microsoft Decision 1", "60/60", "285/529", "53.9%", "failed" ], [ "Clef Flash", "60/60", "291/529", "55.0%", "failed" ], [ "Clef", "60/60", "304/529", "57.5%", "failed" ], [ "Perplexity Decider V1.1 27B", "60/60", "459/529", "86.8%", "failed" ] ] }, "sufficiency": { "headers": [ "Model / track", "Valid / 180 assigned", "Sufficiency AUC", "Determined recognized", "False certainty", "Sufficient + correct bit" ], "rows": [ [ "Jev 1.13 (historical)", 180, "0.812", "3/120", "0/60", "3/120" ], [ "Microsoft Decision 1", 180, "0.861", "44/120", "4/60", "29/120" ], [ "Clef Flash", 180, "0.911", "0/120", "0/60", "0/120" ], [ "Clef", 180, "0.875", "5/120", "0/60", "3/120" ], [ "Perplexity Decider V1.1 27B", 179, "0.904", "100/120", "12/60", "73/120" ] ] }, "controls": { "headers": [ "Model", "Symbolic reading", "Symbolic one-step", "Categorical reading", "Categorical one-step" ], "rows": [ [ "Microsoft Decision 1", "6/8", "6/8", "8/8", "8/8" ], [ "Clef Flash", "4/8", "6/8", "8/8", "8/8" ], [ "Clef", "6/8", "6/8", "8/8", "8/8" ], [ "Perplexity Decider V1.1 27B", "7/8", "6/8", "8/8", "8/8" ] ] }, "small_rules": { "headers": [ "Model", "Lookup", "Extraction", "Combined", "Multistep", "Partial" ], "rows": [ [ "Microsoft Decision 1", "32/32", "29/32", "32/32", "18/32", "19/32" ], [ "Clef Flash", "32/32", "18/32", "31/32", "20/32", "20/32" ], [ "Clef", "32/32", "26/32", "32/32", "19/32", "21/32" ], [ "Perplexity Decider V1.1 27B", "32/32", "32/32", "32/32", "20/32", "25/32" ] ] }, "representation": { "headers": [ "Model", "Complete / assigned seed groups", "All four correct", "Same semantic choice", "Consistent but wrong" ], "rows": [ [ "Microsoft Decision 1", "40/40", 27, 30, 3 ], [ "Clef Flash", "39/40", 24, 27, 3 ], [ "Clef", "40/40", 26, 30, 4 ], [ "Perplexity Decider V1.1 27B", "40/40", 32, 34, 2 ] ] }, "menu": { "headers": [ "Model", "Full-menu correct", "Omitted + escape correct", "Forced confidence >=.9", "Coverage full", "Coverage omitted + escape", "Coverage forced" ], "rows": [ [ "Microsoft Decision 1", "24/24", "24/24", "0/24", "17/24", "5/24", "9/24" ], [ "Clef Flash", "24/24", "24/24", "0/24", "24/24", "13/24", "24/24" ], [ "Clef", "24/24", "24/24", "0/24", "24/24", "22/24", "24/24" ], [ "Perplexity Decider V1.1 27B", "24/24", "24/24", "0/24", "15/24", "9/24", "9/24" ] ] }, "exits": { "headers": [ "Model", "Answer present", "Known absent: correct exit", "Insufficient: correct exit", "Absent: content errors", "Absent: exit confusions" ], "rows": [ [ "Microsoft Decision 1", "12/12", "12/12", "12/12", 0, 0 ], [ "Clef Flash", "12/12", "12/12", "12/12", 0, 0 ], [ "Clef", "12/12", "4/12", "12/12", 3.0, 5.0 ], [ "Perplexity Decider V1.1 27B", "12/12", "9/12", "12/12", 0, 3.0 ] ] }, "coverage": { "headers": [ "Model", "Initial coverage correct", "Complete repeat triplets", "Mean signed-p spread", "Max signed-p spread" ], "rows": [ [ "Microsoft Decision 1", "46/48", 16, "0.0106968", "0.0532620" ], [ "Clef Flash", "48/48", 16, "0.0000625", "0.0010000" ], [ "Clef", "34/48", 16, "0.0000000", "0.0000000" ], [ "Perplexity Decider V1.1 27B", "48/48", 16, "0.0000000", "0.0000000" ] ] }, "news": { "headers": [ "Model", "Valid test", "Accuracy at .5", "AUC", "TP", "FP", "FN", "Brier" ], "rows": [ [ "Microsoft Decision 1", 220, "53.6%", "0.660", 68, 91, 11, "0.364" ], [ "Clef Flash", 220, "48.6%", "0.693", 76, 110, 3, "0.345" ], [ "Clef", 220, "53.2%", "0.700", 74, 98, 5, "0.309" ], [ "Perplexity Decider V1.1 27B", 220, "56.4%", "0.732", 74, 91, 5, "0.324" ] ] }, "routing": { "headers": [ "Model", "Frozen threshold", "Autonomous /220", "Fallback /220", "Routed disagreement", "Important miss rate" ], "rows": [ [ "Microsoft Decision 1", 0.8, 157, 63, "45.2%", "1.3%" ], [ "Clef Flash", 0.7, 180, 40, "49.4%", "0.0%" ], [ "Clef", 0.75, 142, 78, "41.5%", "0.0%" ], [ "Perplexity Decider V1.1 27B", 0.9, 121, 99, "36.4%", "0.0%" ] ] }, "utility": { "headers": [ "Ranking", "Review slots", "Proxy-important caught", "Proxy-negative slots", "Recall" ], "rows": [ [ "historical_significance_path", 110, "64/79", 46, "81.0%" ], [ "jev_semantic_v3", 110, "49/79", 61, "62.0%" ], [ "perplexity", 110, "56/79", 54, "70.9%" ], [ "stored_phase_a_heuristic", 110, "57/79", 53, "72.2%" ] ] }, "laya": { "headers": [ "Historical untuned Laya task", "n", "Accuracy" ], "rows": [ [ "symbolic/full_execution", 400, "50.0%" ], [ "symbolic/partial_sufficiency", 200, "42.5%" ], [ "categorical/full_execution", 300, "62.3%" ], [ "categorical/partial_reachable_set", 200, "50.0%" ], [ "symbolic/reading_control", 8, "50.0%" ], [ "symbolic/one_step_control", 8, "50.0%" ], [ "categorical/reading_control", 8, "37.5%" ], [ "categorical/one_step_control", 8, "25.0%" ] ] }, "inventory": { "headers": [ "Cohort (four arms)", "Assigned", "Recorded outcomes", "Valid", "Invalid", "Uncalled", "Source units / dependence" ], "rows": [ [ "Reading", 240, 240, 240, 0, 0, "60 shared cases" ], [ "One-step", 2116, 2116, 2116, 0, 0, "529 shared cases" ], [ "Exploratory sufficiency", 720, 720, 719, 1, 0, "180 shared cases; dependent CA families" ], [ "Original domain controls", 128, 128, 128, 0, 0, "32 shared fixtures" ], [ "Paired mini-fixtures", 640, 640, 639, 1, 0, "40 seed groups; four siblings" ], [ "Menu probe", 288, 288, 288, 0, 0, "8 seeds; menus and presentations" ], [ "Two-exit follow-up", 144, 144, 144, 0, 0, "12 matched seed groups" ], [ "News calibration + test", 2200, 2200, 2199, 1, 0, "550 shared historical rows; possible event clusters" ], [ "Coverage diagnostic", 320, 320, 320, 0, 0, "4 seeds; 192 initial + 128 planned repeats" ] ] }, "accounting": { "headers": [ "Accounting quantity", "Value" ], "rows": [ [ "attempts", 6798 ], [ "charged_input_tokens", 32035672 ], [ "cap_charge_usd", "0.970399942" ], [ "estimated_usd", "0.970399942" ], [ "billed_known_usd", "0.140790438" ], [ "pending", 0 ], [ "unknown_usage", 3 ] ] } }, "table_evidence_status": { "models": "Canonical requested route and provider metadata are accepted supplemental evidence, not full-export fields.", "prerequisites": "Counts and accuracy regenerated; frozen gate status retained from accepted protocol evidence, not inferred from aggregate accuracy alone.", "sufficiency": "All numerical cells regenerated; exploratory-only after failed prerequisite.", "controls": "Numerators regenerated; eight assigned per task retained from accepted cohort contract.", "small_rules": "Numerators regenerated; 32 assigned per task, not valid-only by_task n.", "representation": "Complete-group counts regenerated from exported semantic pairs; no incomplete-group imputation.", "menu": "Semantic and coverage counts regenerated. Forced confidence >=.9 is absent from full export: retained only as accepted supplemental source evidence, not replay-verified here.", "exits": "All numerical cells regenerated from condition groups.", "coverage": "All numerical cells regenerated from initial cells and scheduled repeat spreads.", "news": "Valid/accuracy/AUC/Brier regenerated. TP/FP/FN absent from full export: retained only as accepted supplemental comparison evidence.", "routing": "All numerical cells regenerated. Current-arm research policy scores; not production authorization.", "utility": "Review-budget endpoints absent from both integrated exports. Entire table retained as accepted supplemental practical-shadow evidence, not full-suite replay parity; no measured time saved.", "laya": "Historical summary-only values regenerated; no case parity or fresh fifth arm.", "inventory": "Assigned/valid/invalid regenerated. Recorded-outcome, uncalled and dependence descriptions remain accepted supplemental evidence.", "accounting": "Billing/token/reserve quantities absent from full export. Accepted historical snapshot retained, not a current bill or new replay measurement. 6798 reserves = 6796 task instances + two smoke attempts." }, "inventory": { "task_instances": 6796, "families": { "exact": 2116, "reading": 240, "menu": 288, "news": 2200, "sufficiency": 720, "domain": 768, "exits": 144, "coverage": 320 }, "valid": 6793, "invalid": 3, "historical_reserves": 6798, "non_task_smoke_attempts": 2 }, "historical_jev": { "model": "jev-1.13.0", "families": { "reading": { "availability": "available_case_matched", "metrics": { "assigned": 60, "valid": 60, "invalid": 0, "correct": 60, "accuracy_assigned": 1.0, "confusion": { "tp": 30, "fp": 0, "tn": 30, "fn": 0 }, "auc_valid_only": 1.0, "brier_valid_only": 0.0017600000000000003 } }, "one_step": { "availability": "available_case_matched", "metrics": { "assigned": 529, "valid": 529, "invalid": 0, "correct": 377, "accuracy_assigned": 0.7126654064272212, "confusion": { "tp": 193, "fp": 81, "tn": 184, "fn": 71 }, "auc_valid_only": 0.8104988564894225, "brier_valid_only": 0.20950132325141776 } }, "sufficiency": { "availability": "available_case_matched", "metrics": { "assigned": 180, "valid": 180, "invalid": 0, "correct": 63, "accuracy_assigned": 0.35, "confusion": { "tp": 3, "fp": 0, "tn": 60, "fn": 117 }, "auc_valid_only": 0.8122222222222222, "brier_valid_only": 0.33173722222222224, "determined_assigned": 120, "ambiguous_assigned": 60, "joint_sufficient_correct": 3 } }, "news_calibration": { "availability": "available_case_matched", "metrics": { "assigned": 330, "valid": 330, "invalid": 0, "correct": 233, "accuracy_assigned": 0.706060606060606, "confusion": { "tp": 108, "fp": 50, "tn": 125, "fn": 47 }, "auc_valid_only": 0.7380829493087557, "brier_valid_only": 0.22727 } }, "news_test": { "availability": "available_case_matched", "metrics": { "assigned": 220, "valid": 220, "invalid": 0, "correct": 128, "accuracy_assigned": 0.5818181818181818, "confusion": { "tp": 49, "fp": 62, "tn": 79, "fn": 30 }, "auc_valid_only": 0.6484424095520244, "brier_valid_only": 0.29598772727272726 } }, "original_domain_controls": { "availability": "unavailable_matched_cases", "metrics": null }, "paired_representations": { "availability": "unavailable_matched_cases", "metrics": null }, "menu_escape": { "availability": "unavailable_matched_cases", "metrics": null }, "exit_discrimination": { "availability": "unavailable_matched_cases", "metrics": null }, "coverage_wording_structure_repeats": { "availability": "unavailable_matched_cases", "metrics": null } }, "news_policy": { "qualifies": false, "eligible_for_autonomous_routing": false, "routing_metrics": null } }, "historical_laya": { "availability": "summary_only", "case_parity": false, "scores": { "symbolic/full_execution": { "n": 400, "accuracy": 0.5 }, "symbolic/partial_sufficiency": { "n": 200, "accuracy": 0.425 }, "categorical/full_execution": { "n": 300, "accuracy": 0.6233333333333333 }, "categorical/partial_reachable_set": { "n": 200, "accuracy": 0.5 }, "symbolic/reading_control": { "n": 8, "accuracy": 0.5 }, "symbolic/one_step_control": { "n": 8, "accuracy": 0.5 }, "categorical/reading_control": { "n": 8, "accuracy": 0.375 }, "categorical/one_step_control": { "n": 8, "accuracy": 0.25 } } }, "public_citations": "pending", "source_sha256": { "benchmark_export": "1b8c119f9c4e49f38aca14045614c6a98b28c491d41bbe859df1f657763e2ee4", "historical_comparison": "13cf0d8581c7353a62d2cf479e16534a6157457b222854f281995a0c8a9220a9", "accepted_tables": "eaa8860cb4ea2693fa054814838ce307fa38a320cc7925e8ca22039f9d27952b" }, "addendum_tables": { "jev114_prerequisites": { "headers": [ "Track / stage", "Assigned", "Valid", "Invalid", "Correct", "Assigned accuracy", "Valid-only accuracy", "Gate" ], "rows": [ [ "Jev 1.14 native / reading", 60, 60, 0, 60, "100.0000%", "100.0000%", "PASS" ], [ "Jev 1.14 native / one_step", 529, 527, 2, 356, "67.2968%", "67.5522%", "FAIL" ], [ "Jev 1.13 historical / reading", 60, 60, 0, 60, "100.0000%", "100.0000%", "PASS" ], [ "Jev 1.13 historical / one_step", 529, 529, 0, 377, "71.2665%", "71.2665%", "FAIL" ] ] }, "jev114_historical_overlap": { "headers": [ "Subset", "Assigned pairs", "Both correct", "Only 1.14 correct", "Only historical 1.13 correct", "Neither correct" ], "rows": [ [ "All assigned", 529, 297, 59, 80, 93 ], [ "Both valid", 527, 297, 59, 79, 92 ], [ "Truth 0, all assigned", 265, 110, 6, 74, 75 ], [ "Truth 0, both valid", 265, 110, 6, 74, 75 ], [ "Truth 1, all assigned", 264, 187, 53, 6, 18 ], [ "Truth 1, both valid", 262, 187, 53, 5, 17 ] ] }, "jev114_usage": { "headers": [ "Stage", "Attempts", "Usage known", "Usage missing", "Known input tokens", "Known output tokens" ], "rows": [ [ "reading", 60, 60, 0, 50136, 2520 ], [ "one_step", 529, 528, 1, 589200, 47512 ] ] }, "jev114_allowance": { "headers": [ "Accounting quantity (not billing)", "USD" ], "rows": [ [ "Local allowance", "59.970399942" ], [ "Approved cap", "60" ], [ "Remaining allowance", "0.029600058" ], [ "Next-attempt allowance", "0.10" ] ] } }, "review_status": { "historical_suite": "ACCEPTED_OFFLINE_ONLY", "predecessor_page": "ACCEPTED_LOCAL_RELEASE_ONLY", "jev_results": "independently_audited", "this_page": "pending_independent_scientific_editorial_review", "this_html_browser_qa": "pending", "publication": "not_authorized" } }