| { |
| "model": "solintellegence/sol-pro", |
| "tokenizer": "solintellegence/sol-pro", |
| "params": 138037594, |
| "dataset": { |
| "path": "/home/marimo/.cache/huggingface/hub/datasets--AxiomicLabs--ArithMark-3.0/blobs/237840149650455c4c54ff18f02203e2a76c1d75", |
| "sha256": "bf8ab1a5193d52cdf0e05ff0b3ca226bdfcf416cb6e75562dcbe72e7e4559435", |
| "items": 1000 |
| }, |
| "runtime": { |
| "device": "cuda", |
| "dtype": "torch.float32", |
| "batch_size": 32, |
| "max_context": 1024, |
| "attention_implementation": "auto", |
| "compiled": false, |
| "torch_version": "2.11.0+cu130" |
| }, |
| "primary_metric": "acc_norm", |
| "results": { |
| "arithmark-3": { |
| "acc": 37.2, |
| "acc_norm": 36.0, |
| "raw_correct": 372, |
| "norm_correct": 360, |
| "total": 1000, |
| "categories": { |
| "elementary_school_math_continuation::addition::grades_1_2::easy": { |
| "acc": 23.4375, |
| "acc_norm": 24.21875, |
| "raw_correct": 30, |
| "norm_correct": 31, |
| "total": 128 |
| }, |
| "elementary_school_math_continuation::comparison::grades_2_3::medium": { |
| "acc": 20.454545454545457, |
| "acc_norm": 18.181818181818183, |
| "raw_correct": 9, |
| "norm_correct": 8, |
| "total": 44 |
| }, |
| "elementary_school_math_continuation::comparison_difference::grades_2_3::medium": { |
| "acc": 37.5, |
| "acc_norm": 35.41666666666667, |
| "raw_correct": 18, |
| "norm_correct": 17, |
| "total": 48 |
| }, |
| "elementary_school_math_continuation::data::grades_2_3::easy": { |
| "acc": 30.23255813953488, |
| "acc_norm": 37.2093023255814, |
| "raw_correct": 13, |
| "norm_correct": 16, |
| "total": 43 |
| }, |
| "elementary_school_math_continuation::division::grades_3_4::medium": { |
| "acc": 25.925925925925924, |
| "acc_norm": 25.925925925925924, |
| "raw_correct": 14, |
| "norm_correct": 14, |
| "total": 54 |
| }, |
| "elementary_school_math_continuation::fractions_counting::grades_3_4::medium": { |
| "acc": 20.0, |
| "acc_norm": 20.0, |
| "raw_correct": 10, |
| "norm_correct": 10, |
| "total": 50 |
| }, |
| "elementary_school_math_continuation::geometry_area::grades_4_5::medium": { |
| "acc": 57.692307692307686, |
| "acc_norm": 55.769230769230774, |
| "raw_correct": 30, |
| "norm_correct": 29, |
| "total": 52 |
| }, |
| "elementary_school_math_continuation::geometry_perimeter::grades_4_5::medium": { |
| "acc": 62.22222222222222, |
| "acc_norm": 66.66666666666666, |
| "raw_correct": 28, |
| "norm_correct": 30, |
| "total": 45 |
| }, |
| "elementary_school_math_continuation::measurement::grades_2_3::easy": { |
| "acc": 32.89473684210527, |
| "acc_norm": 31.57894736842105, |
| "raw_correct": 25, |
| "norm_correct": 24, |
| "total": 76 |
| }, |
| "elementary_school_math_continuation::money::grades_3_4::medium": { |
| "acc": 28.125, |
| "acc_norm": 29.6875, |
| "raw_correct": 18, |
| "norm_correct": 19, |
| "total": 64 |
| }, |
| "elementary_school_math_continuation::multiplication::grades_3_4::medium": { |
| "acc": 67.56756756756756, |
| "acc_norm": 59.45945945945946, |
| "raw_correct": 50, |
| "norm_correct": 44, |
| "total": 74 |
| }, |
| "elementary_school_math_continuation::patterns::grades_3_4::medium": { |
| "acc": 33.9622641509434, |
| "acc_norm": 30.18867924528302, |
| "raw_correct": 18, |
| "norm_correct": 16, |
| "total": 53 |
| }, |
| "elementary_school_math_continuation::subtraction::grades_1_2::easy": { |
| "acc": 28.205128205128204, |
| "acc_norm": 27.350427350427353, |
| "raw_correct": 33, |
| "norm_correct": 32, |
| "total": 117 |
| }, |
| "elementary_school_math_continuation::time::grades_2_3::easy": { |
| "acc": 100.0, |
| "acc_norm": 87.27272727272727, |
| "raw_correct": 55, |
| "norm_correct": 48, |
| "total": 55 |
| }, |
| "elementary_school_math_continuation::two_step_add_subtract::grades_2_3::medium": { |
| "acc": 17.391304347826086, |
| "acc_norm": 19.565217391304348, |
| "raw_correct": 8, |
| "norm_correct": 9, |
| "total": 46 |
| }, |
| "elementary_school_math_continuation::two_step_addition::grades_2_3::medium": { |
| "acc": 21.052631578947366, |
| "acc_norm": 21.052631578947366, |
| "raw_correct": 4, |
| "norm_correct": 4, |
| "total": 19 |
| }, |
| "elementary_school_math_continuation::two_step_subtraction::grades_2_3::medium": { |
| "acc": 28.125, |
| "acc_norm": 28.125, |
| "raw_correct": 9, |
| "norm_correct": 9, |
| "total": 32 |
| } |
| }, |
| "timing": { |
| "tokenization_seconds": 0.2399261619993922, |
| "evaluation_seconds": 49.24065730300026, |
| "examples_per_second": 20.308421023840992 |
| }, |
| "primary_metric": "acc_norm", |
| "primary_acc": 36.0 |
| } |
| } |
| } |
