{
  "date": "2026-10-03",
  "metric": "argmax_accuracy",
  "models": {
    "jev": {
      "description": "Hosted service; JevBench tested locally at Jev 1.13.0, other suites from Kev authors"
    },
    "kev_9b": {
      "revision": "2629c06a",
      "code_revision": "557598fced1dada75dfbf36ed144dce309ac6ceb",
      "dtype": "fp32",
      "temperature": 1.0,
      "source": "JevBench locally rerun; other suites from author reports"
    },
    "qev_9b": {
      "repo_id": "AustinFu/Qev-9B",
      "revision": "v0.3.0",
      "base": "Qwen/Qwen3.5-9B-Base",
      "base_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404",
      "step": 2658,
      "dtype": "bf16",
      "execution": "reference",
      "seed": 17,
      "temperature": 1.0
    },
    "qwen35_9b_base": {
      "description": "Frozen native LM head; zero-shot candidate code probabilities",
      "dtype": "bf16"
    },
    "qev_2b": {
      "base": "Qwen/Qwen3.5-2B-Base",
      "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c",
      "dtype": "bf16",
      "description": "Probability and representation-response distillation from Qev-9B v0.1.0",
      "seed": 17,
      "continuation_steps": 800,
      "teacher_revision": "v0.1.0"
    },
    "qwen35_2b_base": {
      "description": "Frozen native LM head; zero-shot candidate code probabilities",
      "dtype": "bf16",
      "base": "Qwen/Qwen3.5-2B-Base",
      "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c"
    },
    "qev_4b": {
      "repo_id": "AustinFu/Qev-4B",
      "revision": "v0.1.0",
      "base": "Qwen/Qwen3.5-4B-Base",
      "base_revision": "710fd005d44d55ee27b7ad5147e318e546efdbfe",
      "step": 2786,
      "dtype": "bf16",
      "execution": "reference",
      "seed": 17,
      "temperature": 1.0,
      "description": "Initialized directly from the Qwen base; teacher-probability cross entropy",
      "teacher_repo_id": "AustinFu/Qev-9B",
      "teacher_revision": "v0.3.0"
    },
    "qwen35_4b_base": {
      "repo_id": "Qwen/Qwen3.5-4B-Base",
      "revision": "710fd005d44d55ee27b7ad5147e318e546efdbfe",
      "dtype": "bf16",
      "description": "Frozen native LM head; zero-shot candidate code probabilities"
    },
    "kev_4b": {
      "repo_id": "jaredpalmer/kev-4b",
      "revision": "139fdd9",
      "code_revision": "5920c5f",
      "dtype": "fp32",
      "temperature": 1.0,
      "description": "Author inference code, unmerged adapter; Qwen3.5-4B-Base"
    },
    "jevany_4b_pointer": {
      "repo_id": "SimpleJev/JevAny-Qwen3.5-4B-LoRA",
      "code_revision": "33cb677",
      "dtype": "fp32",
      "temperature": 1.0,
      "description": "Pointer head on post-trained Qwen3.5-4B, author exact inference path"
    },
    "jevany_4b_direct": {
      "repo_id": "SimpleJev/JevAny-Qwen3.5-4B-Direct-Token-LoRA",
      "code_revision": "33cb677",
      "dtype": "fp32",
      "temperature": 1.0,
      "description": "Direct-token readout on post-trained Qwen3.5-4B, author exact inference path"
    }
  },
  "results": {
    "decision_dev_all": {
      "total": 1468,
      "models": {
        "jev": {
          "correct": 1221,
          "accuracy": 0.8317438692098093
        },
        "kev_9b": {
          "correct": 1289,
          "accuracy": 0.8780653950953679
        },
        "qev_9b": {
          "correct": 1296,
          "accuracy": 0.8828337874659401
        },
        "qwen35_9b_base": {
          "correct": 1112,
          "accuracy": 0.7574931880108992
        },
        "qev_2b": {
          "correct": 1266,
          "accuracy": 0.8623978201634878
        },
        "qwen35_2b_base": {
          "correct": 927,
          "accuracy": 0.6314713896457765
        },
        "qev_4b": {
          "correct": 1286,
          "accuracy": 0.8760217983651226
        },
        "qwen35_4b_base": {
          "correct": 1050,
          "accuracy": 0.715258855585831
        },
        "kev_4b": {
          "correct": 1293,
          "accuracy": 0.8807901907356949
        },
        "jevany_4b_pointer": {
          "correct": 1281,
          "accuracy": 0.8726158038147139
        },
        "jevany_4b_direct": {
          "correct": 1278,
          "accuracy": 0.8705722070844687
        }
      }
    },
    "decision_dev_clean": {
      "total": 1264,
      "models": {
        "jev": {
          "correct": 1068,
          "accuracy": 0.8449367088607594
        },
        "kev_9b": {
          "correct": 1102,
          "accuracy": 0.8718354430379747
        },
        "qev_9b": {
          "correct": 1107,
          "accuracy": 0.8757911392405063
        },
        "qwen35_9b_base": {
          "correct": 982,
          "accuracy": 0.7768987341772152
        },
        "qev_2b": {
          "correct": 1079,
          "accuracy": 0.8536392405063291
        },
        "qwen35_2b_base": {
          "correct": 827,
          "accuracy": 0.6542721518987342
        },
        "qev_4b": {
          "correct": 1099,
          "accuracy": 0.8694620253164557
        },
        "qwen35_4b_base": {
          "correct": 932,
          "accuracy": 0.7373417721518988
        },
        "kev_4b": {
          "correct": 1103,
          "accuracy": 0.872626582278481
        },
        "jevany_4b_pointer": {
          "correct": 1095,
          "accuracy": 0.8662974683544303
        },
        "jevany_4b_direct": {
          "correct": 1090,
          "accuracy": 0.8623417721518988
        }
      }
    },
    "transfer_dev_all": {
      "total": 764,
      "models": {
        "jev": {
          "correct": 647,
          "accuracy": 0.8468586387434555
        },
        "kev_9b": {
          "correct": 620,
          "accuracy": 0.8115183246073299
        },
        "qev_9b": {
          "correct": 629,
          "accuracy": 0.8232984293193717
        },
        "qwen35_9b_base": {
          "correct": 561,
          "accuracy": 0.7342931937172775
        },
        "qev_2b": {
          "correct": 584,
          "accuracy": 0.7643979057591623
        },
        "qwen35_2b_base": {
          "correct": 493,
          "accuracy": 0.6452879581151832
        },
        "qev_4b": {
          "correct": 618,
          "accuracy": 0.8089005235602095
        },
        "qwen35_4b_base": {
          "correct": 537,
          "accuracy": 0.7028795811518325
        },
        "kev_4b": {
          "correct": 615,
          "accuracy": 0.8049738219895288
        },
        "jevany_4b_pointer": {
          "correct": 654,
          "accuracy": 0.856020942408377
        },
        "jevany_4b_direct": {
          "correct": 660,
          "accuracy": 0.8638743455497382
        }
      }
    },
    "transfer_dev_clean": {
      "total": 656,
      "models": {
        "jev": {
          "correct": 562,
          "accuracy": 0.8567073170731707
        },
        "kev_9b": {
          "correct": 539,
          "accuracy": 0.8216463414634146
        },
        "qev_9b": {
          "correct": 549,
          "accuracy": 0.836890243902439
        },
        "qwen35_9b_base": {
          "correct": 488,
          "accuracy": 0.7439024390243902
        },
        "qev_2b": {
          "correct": 507,
          "accuracy": 0.7728658536585366
        },
        "qwen35_2b_base": {
          "correct": 427,
          "accuracy": 0.6509146341463414
        },
        "qev_4b": {
          "correct": 538,
          "accuracy": 0.8201219512195121
        },
        "qwen35_4b_base": {
          "correct": 469,
          "accuracy": 0.7149390243902439
        },
        "kev_4b": {
          "correct": 536,
          "accuracy": 0.8170731707317073
        },
        "jevany_4b_pointer": {
          "correct": 555,
          "accuracy": 0.8460365853658537
        },
        "jevany_4b_direct": {
          "correct": 561,
          "accuracy": 0.8551829268292683
        }
      }
    },
    "mmlupro": {
      "total": 1000,
      "models": {
        "jev": {
          "correct": 835,
          "accuracy": 0.835
        },
        "kev_9b": {
          "correct": 511,
          "accuracy": 0.511
        },
        "qev_9b": {
          "correct": 565,
          "accuracy": 0.565
        },
        "qwen35_9b_base": {
          "correct": 504,
          "accuracy": 0.504
        },
        "qev_2b": {
          "correct": 387,
          "accuracy": 0.387
        },
        "qwen35_2b_base": {
          "correct": 312,
          "accuracy": 0.312
        },
        "qev_4b": {
          "correct": 503,
          "accuracy": 0.503
        },
        "qwen35_4b_base": {
          "correct": 435,
          "accuracy": 0.435
        },
        "kev_4b": {
          "correct": 524,
          "accuracy": 0.524
        },
        "jevany_4b_pointer": {
          "correct": 523,
          "accuracy": 0.523
        },
        "jevany_4b_direct": {
          "correct": 511,
          "accuracy": 0.511
        }
      }
    },
    "semif_handwritten": {
      "total": 144,
      "models": {
        "jev": {
          "correct": 139,
          "accuracy": 0.9652777777777778
        },
        "kev_9b": {
          "correct": 131,
          "accuracy": 0.9097222222222222
        },
        "qev_9b": {
          "correct": 135,
          "accuracy": 0.9375
        },
        "qwen35_9b_base": {
          "correct": 130,
          "accuracy": 0.9027777777777778
        },
        "qev_2b": {
          "correct": 119,
          "accuracy": 0.8263888888888888
        },
        "qwen35_2b_base": {
          "correct": 92,
          "accuracy": 0.6388888888888888
        },
        "qev_4b": {
          "correct": 130,
          "accuracy": 0.9027777777777778
        },
        "qwen35_4b_base": {
          "correct": 111,
          "accuracy": 0.7708333333333334
        },
        "kev_4b": {
          "correct": 128,
          "accuracy": 0.8888888888888888
        },
        "jevany_4b_pointer": {
          "correct": 130,
          "accuracy": 0.9027777777777778
        },
        "jevany_4b_direct": {
          "correct": 130,
          "accuracy": 0.9027777777777778
        }
      }
    },
    "scienthoon": {
      "total": 873,
      "models": {
        "jev": {
          "correct": 657,
          "accuracy": 0.7525773195876289
        },
        "kev_9b": {
          "correct": 659,
          "accuracy": 0.7548682703321878
        },
        "qev_9b": {
          "correct": 653,
          "accuracy": 0.7479954180985109
        },
        "qwen35_9b_base": {
          "correct": 601,
          "accuracy": 0.6884306987399771
        },
        "qev_2b": {
          "correct": 628,
          "accuracy": 0.7193585337915235
        },
        "qwen35_2b_base": {
          "correct": 470,
          "accuracy": 0.5383734249713631
        },
        "qev_4b": {
          "correct": 670,
          "accuracy": 0.7674684994272624
        },
        "qwen35_4b_base": {
          "correct": 649,
          "accuracy": 0.7434135166093929
        },
        "kev_4b": {
          "correct": 631,
          "accuracy": 0.722794959908362
        },
        "jevany_4b_pointer": {
          "correct": 605,
          "accuracy": 0.693012600229095
        },
        "jevany_4b_direct": {
          "correct": 601,
          "accuracy": 0.6884306987399771
        }
      }
    },
    "wanli": {
      "total": 256,
      "models": {
        "jev": {
          "correct": 194,
          "accuracy": 0.7578125
        },
        "kev_9b": {
          "correct": 180,
          "accuracy": 0.703125
        },
        "qev_9b": {
          "correct": 192,
          "accuracy": 0.75
        },
        "qwen35_9b_base": {
          "correct": 174,
          "accuracy": 0.6796875
        },
        "qev_2b": {
          "correct": 173,
          "accuracy": 0.67578125
        },
        "qwen35_2b_base": {
          "correct": 129,
          "accuracy": 0.50390625
        },
        "qev_4b": {
          "correct": 188,
          "accuracy": 0.734375
        },
        "qwen35_4b_base": {
          "correct": 155,
          "accuracy": 0.60546875
        },
        "kev_4b": {
          "correct": 176,
          "accuracy": 0.6875
        },
        "jevany_4b_pointer": {
          "correct": 182,
          "accuracy": 0.7109375
        },
        "jevany_4b_direct": {
          "correct": 182,
          "accuracy": 0.7109375
        }
      }
    },
    "jevbench_public": {
      "total": 231,
      "models": {
        "jev": {
          "correct": 198,
          "accuracy": 0.8571428571428571
        },
        "kev_9b": {
          "correct": 175,
          "accuracy": 0.7575757575757576
        },
        "qev_9b": {
          "correct": 187,
          "accuracy": 0.8095238095238095
        },
        "qwen35_9b_base": {
          "correct": 175,
          "accuracy": 0.7575757575757576
        },
        "qev_2b": {
          "correct": 172,
          "accuracy": 0.7445887445887446
        },
        "qwen35_2b_base": {
          "correct": 146,
          "accuracy": 0.6320346320346321
        },
        "qev_4b": {
          "correct": 190,
          "accuracy": 0.8225108225108225
        },
        "qwen35_4b_base": {
          "correct": 155,
          "accuracy": 0.670995670995671
        },
        "kev_4b": {
          "correct": 175,
          "accuracy": 0.7575757575757576
        },
        "jevany_4b_pointer": {
          "correct": 181,
          "accuracy": 0.7835497835497836
        },
        "jevany_4b_direct": {
          "correct": 184,
          "accuracy": 0.7965367965367965
        }
      }
    }
  },
  "jevbench_subsets": {
    "original": {
      "total": 72,
      "correct": {
        "jev": 71,
        "kev_9b": 65,
        "qev_9b": 67,
        "qwen35_9b_base": 59,
        "qev_2b": 64,
        "qwen35_2b_base": 44,
        "qev_4b": 69,
        "qwen35_4b_base": 53,
        "kev_4b": 67,
        "jevany_4b_pointer": 69,
        "jevany_4b_direct": 71
      }
    },
    "easy": {
      "total": 48,
      "correct": {
        "jev": 48,
        "kev_9b": 48,
        "qev_9b": 48,
        "qwen35_9b_base": 48,
        "qev_2b": 48,
        "qwen35_2b_base": 48,
        "qev_4b": 48,
        "qwen35_4b_base": 48,
        "kev_4b": 48,
        "jevany_4b_pointer": 48,
        "jevany_4b_direct": 48
      }
    },
    "hard": {
      "total": 111,
      "correct": {
        "jev": 79,
        "kev_9b": 62,
        "qev_9b": 72,
        "qwen35_9b_base": 68,
        "qev_2b": 60,
        "qwen35_2b_base": 54,
        "qev_4b": 73,
        "qwen35_4b_base": 54,
        "kev_4b": 60,
        "jevany_4b_pointer": 64,
        "jevany_4b_direct": 65
      }
    }
  },
  "notes": [
    "Public JevBench accuracy is not the official composite score.",
    "MMLU-Pro: Kev answered 992/1000; its 8 unanswered items count as wrong.",
    "SemIf: the 144 handwritten questions, excluding the 108 perturbations.",
    "Cross-model precisions and execution implementations differ.",
    "The Qev release is one seed, and these comparisons do not isolate architecture gains.",
    "The 2B columns use the same clean development and 144-question SemIf subsets as the 9B comparison."
  ]
}