| { |
| "date": "2026-10-03", |
| "metric": "argmax_accuracy", |
| "models": { |
| "jev": { |
| "description": "Hosted service; JevBench tested locally at Jev 1.13.0, other suites from Kev authors" |
| }, |
| "kev_9b": { |
| "revision": "2629c06a", |
| "code_revision": "557598fced1dada75dfbf36ed144dce309ac6ceb", |
| "dtype": "fp32", |
| "temperature": 1.0, |
| "source": "JevBench locally rerun; other suites from author reports" |
| }, |
| "qev_9b": { |
| "repo_id": "AustinFu/Qev-9B", |
| "revision": "v0.3.0", |
| "base": "Qwen/Qwen3.5-9B-Base", |
| "base_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404", |
| "step": 2658, |
| "dtype": "bf16", |
| "execution": "reference", |
| "seed": 17, |
| "temperature": 1.0 |
| }, |
| "qwen35_9b_base": { |
| "description": "Frozen native LM head; zero-shot candidate code probabilities", |
| "dtype": "bf16" |
| }, |
| "qev_2b": { |
| "base": "Qwen/Qwen3.5-2B-Base", |
| "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c", |
| "dtype": "bf16", |
| "description": "Probability and representation-response distillation from Qev-9B v0.1.0", |
| "seed": 17, |
| "continuation_steps": 800, |
| "teacher_revision": "v0.1.0" |
| }, |
| "qwen35_2b_base": { |
| "description": "Frozen native LM head; zero-shot candidate code probabilities", |
| "dtype": "bf16", |
| "base": "Qwen/Qwen3.5-2B-Base", |
| "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c" |
| }, |
| "qev_4b": { |
| "repo_id": "AustinFu/Qev-4B", |
| "revision": "v0.1.0", |
| "base": "Qwen/Qwen3.5-4B-Base", |
| "base_revision": "710fd005d44d55ee27b7ad5147e318e546efdbfe", |
| "step": 2786, |
| "dtype": "bf16", |
| "execution": "reference", |
| "seed": 17, |
| "temperature": 1.0, |
| "description": "Initialized directly from the Qwen base; teacher-probability cross entropy", |
| "teacher_repo_id": "AustinFu/Qev-9B", |
| "teacher_revision": "v0.3.0" |
| }, |
| "qwen35_4b_base": { |
| "repo_id": "Qwen/Qwen3.5-4B-Base", |
| "revision": "710fd005d44d55ee27b7ad5147e318e546efdbfe", |
| "dtype": "bf16", |
| "description": "Frozen native LM head; zero-shot candidate code probabilities" |
| }, |
| "kev_4b": { |
| "repo_id": "jaredpalmer/kev-4b", |
| "revision": "139fdd9", |
| "code_revision": "5920c5f", |
| "dtype": "fp32", |
| "temperature": 1.0, |
| "description": "Author inference code, unmerged adapter; Qwen3.5-4B-Base" |
| }, |
| "jevany_4b_pointer": { |
| "repo_id": "SimpleJev/JevAny-Qwen3.5-4B-LoRA", |
| "code_revision": "33cb677", |
| "dtype": "fp32", |
| "temperature": 1.0, |
| "description": "Pointer head on post-trained Qwen3.5-4B, author exact inference path" |
| }, |
| "jevany_4b_direct": { |
| "repo_id": "SimpleJev/JevAny-Qwen3.5-4B-Direct-Token-LoRA", |
| "code_revision": "33cb677", |
| "dtype": "fp32", |
| "temperature": 1.0, |
| "description": "Direct-token readout on post-trained Qwen3.5-4B, author exact inference path" |
| } |
| }, |
| "results": { |
| "decision_dev_all": { |
| "total": 1468, |
| "models": { |
| "jev": { |
| "correct": 1221, |
| "accuracy": 0.8317438692098093 |
| }, |
| "kev_9b": { |
| "correct": 1289, |
| "accuracy": 0.8780653950953679 |
| }, |
| "qev_9b": { |
| "correct": 1296, |
| "accuracy": 0.8828337874659401 |
| }, |
| "qwen35_9b_base": { |
| "correct": 1112, |
| "accuracy": 0.7574931880108992 |
| }, |
| "qev_2b": { |
| "correct": 1266, |
| "accuracy": 0.8623978201634878 |
| }, |
| "qwen35_2b_base": { |
| "correct": 927, |
| "accuracy": 0.6314713896457765 |
| }, |
| "qev_4b": { |
| "correct": 1286, |
| "accuracy": 0.8760217983651226 |
| }, |
| "qwen35_4b_base": { |
| "correct": 1050, |
| "accuracy": 0.715258855585831 |
| }, |
| "kev_4b": { |
| "correct": 1293, |
| "accuracy": 0.8807901907356949 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 1281, |
| "accuracy": 0.8726158038147139 |
| }, |
| "jevany_4b_direct": { |
| "correct": 1278, |
| "accuracy": 0.8705722070844687 |
| } |
| } |
| }, |
| "decision_dev_clean": { |
| "total": 1264, |
| "models": { |
| "jev": { |
| "correct": 1068, |
| "accuracy": 0.8449367088607594 |
| }, |
| "kev_9b": { |
| "correct": 1102, |
| "accuracy": 0.8718354430379747 |
| }, |
| "qev_9b": { |
| "correct": 1107, |
| "accuracy": 0.8757911392405063 |
| }, |
| "qwen35_9b_base": { |
| "correct": 982, |
| "accuracy": 0.7768987341772152 |
| }, |
| "qev_2b": { |
| "correct": 1079, |
| "accuracy": 0.8536392405063291 |
| }, |
| "qwen35_2b_base": { |
| "correct": 827, |
| "accuracy": 0.6542721518987342 |
| }, |
| "qev_4b": { |
| "correct": 1099, |
| "accuracy": 0.8694620253164557 |
| }, |
| "qwen35_4b_base": { |
| "correct": 932, |
| "accuracy": 0.7373417721518988 |
| }, |
| "kev_4b": { |
| "correct": 1103, |
| "accuracy": 0.872626582278481 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 1095, |
| "accuracy": 0.8662974683544303 |
| }, |
| "jevany_4b_direct": { |
| "correct": 1090, |
| "accuracy": 0.8623417721518988 |
| } |
| } |
| }, |
| "transfer_dev_all": { |
| "total": 764, |
| "models": { |
| "jev": { |
| "correct": 647, |
| "accuracy": 0.8468586387434555 |
| }, |
| "kev_9b": { |
| "correct": 620, |
| "accuracy": 0.8115183246073299 |
| }, |
| "qev_9b": { |
| "correct": 629, |
| "accuracy": 0.8232984293193717 |
| }, |
| "qwen35_9b_base": { |
| "correct": 561, |
| "accuracy": 0.7342931937172775 |
| }, |
| "qev_2b": { |
| "correct": 584, |
| "accuracy": 0.7643979057591623 |
| }, |
| "qwen35_2b_base": { |
| "correct": 493, |
| "accuracy": 0.6452879581151832 |
| }, |
| "qev_4b": { |
| "correct": 618, |
| "accuracy": 0.8089005235602095 |
| }, |
| "qwen35_4b_base": { |
| "correct": 537, |
| "accuracy": 0.7028795811518325 |
| }, |
| "kev_4b": { |
| "correct": 615, |
| "accuracy": 0.8049738219895288 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 654, |
| "accuracy": 0.856020942408377 |
| }, |
| "jevany_4b_direct": { |
| "correct": 660, |
| "accuracy": 0.8638743455497382 |
| } |
| } |
| }, |
| "transfer_dev_clean": { |
| "total": 656, |
| "models": { |
| "jev": { |
| "correct": 562, |
| "accuracy": 0.8567073170731707 |
| }, |
| "kev_9b": { |
| "correct": 539, |
| "accuracy": 0.8216463414634146 |
| }, |
| "qev_9b": { |
| "correct": 549, |
| "accuracy": 0.836890243902439 |
| }, |
| "qwen35_9b_base": { |
| "correct": 488, |
| "accuracy": 0.7439024390243902 |
| }, |
| "qev_2b": { |
| "correct": 507, |
| "accuracy": 0.7728658536585366 |
| }, |
| "qwen35_2b_base": { |
| "correct": 427, |
| "accuracy": 0.6509146341463414 |
| }, |
| "qev_4b": { |
| "correct": 538, |
| "accuracy": 0.8201219512195121 |
| }, |
| "qwen35_4b_base": { |
| "correct": 469, |
| "accuracy": 0.7149390243902439 |
| }, |
| "kev_4b": { |
| "correct": 536, |
| "accuracy": 0.8170731707317073 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 555, |
| "accuracy": 0.8460365853658537 |
| }, |
| "jevany_4b_direct": { |
| "correct": 561, |
| "accuracy": 0.8551829268292683 |
| } |
| } |
| }, |
| "mmlupro": { |
| "total": 1000, |
| "models": { |
| "jev": { |
| "correct": 835, |
| "accuracy": 0.835 |
| }, |
| "kev_9b": { |
| "correct": 511, |
| "accuracy": 0.511 |
| }, |
| "qev_9b": { |
| "correct": 565, |
| "accuracy": 0.565 |
| }, |
| "qwen35_9b_base": { |
| "correct": 504, |
| "accuracy": 0.504 |
| }, |
| "qev_2b": { |
| "correct": 387, |
| "accuracy": 0.387 |
| }, |
| "qwen35_2b_base": { |
| "correct": 312, |
| "accuracy": 0.312 |
| }, |
| "qev_4b": { |
| "correct": 503, |
| "accuracy": 0.503 |
| }, |
| "qwen35_4b_base": { |
| "correct": 435, |
| "accuracy": 0.435 |
| }, |
| "kev_4b": { |
| "correct": 524, |
| "accuracy": 0.524 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 523, |
| "accuracy": 0.523 |
| }, |
| "jevany_4b_direct": { |
| "correct": 511, |
| "accuracy": 0.511 |
| } |
| } |
| }, |
| "semif_handwritten": { |
| "total": 144, |
| "models": { |
| "jev": { |
| "correct": 139, |
| "accuracy": 0.9652777777777778 |
| }, |
| "kev_9b": { |
| "correct": 131, |
| "accuracy": 0.9097222222222222 |
| }, |
| "qev_9b": { |
| "correct": 135, |
| "accuracy": 0.9375 |
| }, |
| "qwen35_9b_base": { |
| "correct": 130, |
| "accuracy": 0.9027777777777778 |
| }, |
| "qev_2b": { |
| "correct": 119, |
| "accuracy": 0.8263888888888888 |
| }, |
| "qwen35_2b_base": { |
| "correct": 92, |
| "accuracy": 0.6388888888888888 |
| }, |
| "qev_4b": { |
| "correct": 130, |
| "accuracy": 0.9027777777777778 |
| }, |
| "qwen35_4b_base": { |
| "correct": 111, |
| "accuracy": 0.7708333333333334 |
| }, |
| "kev_4b": { |
| "correct": 128, |
| "accuracy": 0.8888888888888888 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 130, |
| "accuracy": 0.9027777777777778 |
| }, |
| "jevany_4b_direct": { |
| "correct": 130, |
| "accuracy": 0.9027777777777778 |
| } |
| } |
| }, |
| "scienthoon": { |
| "total": 873, |
| "models": { |
| "jev": { |
| "correct": 657, |
| "accuracy": 0.7525773195876289 |
| }, |
| "kev_9b": { |
| "correct": 659, |
| "accuracy": 0.7548682703321878 |
| }, |
| "qev_9b": { |
| "correct": 653, |
| "accuracy": 0.7479954180985109 |
| }, |
| "qwen35_9b_base": { |
| "correct": 601, |
| "accuracy": 0.6884306987399771 |
| }, |
| "qev_2b": { |
| "correct": 628, |
| "accuracy": 0.7193585337915235 |
| }, |
| "qwen35_2b_base": { |
| "correct": 470, |
| "accuracy": 0.5383734249713631 |
| }, |
| "qev_4b": { |
| "correct": 670, |
| "accuracy": 0.7674684994272624 |
| }, |
| "qwen35_4b_base": { |
| "correct": 649, |
| "accuracy": 0.7434135166093929 |
| }, |
| "kev_4b": { |
| "correct": 631, |
| "accuracy": 0.722794959908362 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 605, |
| "accuracy": 0.693012600229095 |
| }, |
| "jevany_4b_direct": { |
| "correct": 601, |
| "accuracy": 0.6884306987399771 |
| } |
| } |
| }, |
| "wanli": { |
| "total": 256, |
| "models": { |
| "jev": { |
| "correct": 194, |
| "accuracy": 0.7578125 |
| }, |
| "kev_9b": { |
| "correct": 180, |
| "accuracy": 0.703125 |
| }, |
| "qev_9b": { |
| "correct": 192, |
| "accuracy": 0.75 |
| }, |
| "qwen35_9b_base": { |
| "correct": 174, |
| "accuracy": 0.6796875 |
| }, |
| "qev_2b": { |
| "correct": 173, |
| "accuracy": 0.67578125 |
| }, |
| "qwen35_2b_base": { |
| "correct": 129, |
| "accuracy": 0.50390625 |
| }, |
| "qev_4b": { |
| "correct": 188, |
| "accuracy": 0.734375 |
| }, |
| "qwen35_4b_base": { |
| "correct": 155, |
| "accuracy": 0.60546875 |
| }, |
| "kev_4b": { |
| "correct": 176, |
| "accuracy": 0.6875 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 182, |
| "accuracy": 0.7109375 |
| }, |
| "jevany_4b_direct": { |
| "correct": 182, |
| "accuracy": 0.7109375 |
| } |
| } |
| }, |
| "jevbench_public": { |
| "total": 231, |
| "models": { |
| "jev": { |
| "correct": 198, |
| "accuracy": 0.8571428571428571 |
| }, |
| "kev_9b": { |
| "correct": 175, |
| "accuracy": 0.7575757575757576 |
| }, |
| "qev_9b": { |
| "correct": 187, |
| "accuracy": 0.8095238095238095 |
| }, |
| "qwen35_9b_base": { |
| "correct": 175, |
| "accuracy": 0.7575757575757576 |
| }, |
| "qev_2b": { |
| "correct": 172, |
| "accuracy": 0.7445887445887446 |
| }, |
| "qwen35_2b_base": { |
| "correct": 146, |
| "accuracy": 0.6320346320346321 |
| }, |
| "qev_4b": { |
| "correct": 190, |
| "accuracy": 0.8225108225108225 |
| }, |
| "qwen35_4b_base": { |
| "correct": 155, |
| "accuracy": 0.670995670995671 |
| }, |
| "kev_4b": { |
| "correct": 175, |
| "accuracy": 0.7575757575757576 |
| }, |
| "jevany_4b_pointer": { |
| "correct": 181, |
| "accuracy": 0.7835497835497836 |
| }, |
| "jevany_4b_direct": { |
| "correct": 184, |
| "accuracy": 0.7965367965367965 |
| } |
| } |
| } |
| }, |
| "jevbench_subsets": { |
| "original": { |
| "total": 72, |
| "correct": { |
| "jev": 71, |
| "kev_9b": 65, |
| "qev_9b": 67, |
| "qwen35_9b_base": 59, |
| "qev_2b": 64, |
| "qwen35_2b_base": 44, |
| "qev_4b": 69, |
| "qwen35_4b_base": 53, |
| "kev_4b": 67, |
| "jevany_4b_pointer": 69, |
| "jevany_4b_direct": 71 |
| } |
| }, |
| "easy": { |
| "total": 48, |
| "correct": { |
| "jev": 48, |
| "kev_9b": 48, |
| "qev_9b": 48, |
| "qwen35_9b_base": 48, |
| "qev_2b": 48, |
| "qwen35_2b_base": 48, |
| "qev_4b": 48, |
| "qwen35_4b_base": 48, |
| "kev_4b": 48, |
| "jevany_4b_pointer": 48, |
| "jevany_4b_direct": 48 |
| } |
| }, |
| "hard": { |
| "total": 111, |
| "correct": { |
| "jev": 79, |
| "kev_9b": 62, |
| "qev_9b": 72, |
| "qwen35_9b_base": 68, |
| "qev_2b": 60, |
| "qwen35_2b_base": 54, |
| "qev_4b": 73, |
| "qwen35_4b_base": 54, |
| "kev_4b": 60, |
| "jevany_4b_pointer": 64, |
| "jevany_4b_direct": 65 |
| } |
| } |
| }, |
| "notes": [ |
| "Public JevBench accuracy is not the official composite score.", |
| "MMLU-Pro: Kev answered 992/1000; its 8 unanswered items count as wrong.", |
| "SemIf: the 144 handwritten questions, excluding the 108 perturbations.", |
| "Cross-model precisions and execution implementations differ.", |
| "The Qev release is one seed, and these comparisons do not isolate architecture gains.", |
| "The 2B columns use the same clean development and 144-question SemIf subsets as the 9B comparison." |
| ] |
| } |
|
|
