onnx/benchmark_results.jsonl

{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 53.0, "p95_ms": 508.9}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 49.4, "p95_ms": 400.8}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "raya-int8.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7904, "changed_vs_pytorch": 53, "p50_ms": 26.7, "p95_ms": 248.2}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "raya-int8-emb.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.794, "changed_vs_pytorch": 47, "p50_ms": 27.0, "p95_ms": 253.9}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "raya-int8-mixed.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7922, "changed_vs_pytorch": 41, "p50_ms": 36.6, "p95_ms": 309.1}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 4, "file": "raya-int8-blockwise.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8153, "changed_vs_pytorch": 5, "p50_ms": 42.7, "p95_ms": 379.1}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 41.7, "p95_ms": 421.2}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 37.8, "p95_ms": 296.8}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "raya-int8.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7904, "changed_vs_pytorch": 53, "p50_ms": 24.2, "p95_ms": 199.5}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "raya-int8-emb.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.794, "changed_vs_pytorch": 47, "p50_ms": 24.0, "p95_ms": 201.2}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "raya-int8-mixed.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7922, "changed_vs_pytorch": 41, "p50_ms": 30.6, "p95_ms": 241.7}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 8, "file": "raya-int8-blockwise.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8153, "changed_vs_pytorch": 5, "p50_ms": 33.7, "p95_ms": 281.9}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 56.6, "p95_ms": 539.5}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 43.3, "p95_ms": 367.9}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "raya-int8.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7904, "changed_vs_pytorch": 53, "p50_ms": 34.3, "p95_ms": 232.8}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "raya-int8-emb.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.794, "changed_vs_pytorch": 47, "p50_ms": 27.7, "p95_ms": 209.7}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "raya-int8-mixed.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.7922, "changed_vs_pytorch": 41, "p50_ms": 34.6, "p95_ms": 252.7}
{"cpu": "Intel Core i9-13900 (AVX-VNNI)", "threads": 16, "file": "raya-int8-blockwise.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8153, "changed_vs_pytorch": 5, "p50_ms": 35.8, "p95_ms": 277.7}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 4, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 105.7, "p95_ms": 844.3, "note": "equivalent build of the same recipe, produced on that node"}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 8, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 59.6, "p95_ms": 486.6, "note": "equivalent build of the same recipe, produced on that node"}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 16, "file": "raya.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8117, "changed_vs_pytorch": 3, "p50_ms": 78.0, "p95_ms": 499.6, "note": "equivalent build of the same recipe, produced on that node"}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 4, "file": "raya-int8-blockwise.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8135, "changed_vs_pytorch": 4, "p50_ms": 148.3, "p95_ms": 1192.1, "note": "equivalent build of the same recipe, produced on that node"}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 8, "file": "raya-int8-blockwise.onnx", "max_tokens": 512, "n": 563, "accuracy": 0.8135, "changed_vs_pytorch": 4, "p50_ms": 127.2, "p95_ms": 975.3, "note": "equivalent build of the same recipe, produced on that node"}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 4, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 155.7, "p95_ms": 1333.2}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 8, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 88.7, "p95_ms": 526.4}
{"cpu": "AMD EPYC 7502P (AVX2, no VNNI)", "threads": 16, "file": "pytorch", "max_tokens": 1024, "n": 563, "accuracy": 0.8099, "changed_vs_pytorch": 0, "p50_ms": 79.5, "p95_ms": 466.2}