Add fresh 100-sample BF16 eval artifact

samples/bf16_100_sample_eval_seed20260515.jsonl

@@ -0,0 +1,100 @@
+{"sample_no": 1, "dataset_index": 2086, "audio_seconds": 5.25, "reference": "Justin Zhuang Cham Tao Soon and Ted De Ponti", "prediction": "Justin Chua, Chan, Tau, Soon and Teck Dee Pon, Teck.", "wer": 0.777778, "zero_wer_after_release_normalization": false}
+{"sample_no": 2, "dataset_index": 779, "audio_seconds": 4.77, "reference": "can you recommend me a restaurant near Brani Marine Fire Station", "prediction": "Can you recommend me a restaurant near Brani Marine Fire Station?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 3, "dataset_index": 1935, "audio_seconds": 5.81, "reference": "Tan Hwa Luck Brigadier Ivan Simson and Kenneth Mitchell", "prediction": "Tan Hwa Luck Brigadier Ivan Simson and Kenneth Mitchell.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 4, "dataset_index": 1977, "audio_seconds": 2.29, "reference": "how much is Sam Lau", "prediction": "How much is Sam Lau?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 5, "dataset_index": 1524, "audio_seconds": 3.09, "reference": "this shop sells Shan Rui Tang", "prediction": "This shop sells Shan Rui Tang.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 6, "dataset_index": 782, "audio_seconds": 4.54, "reference": "Show Me The Way to Ahmadiyya Burial Ground", "prediction": "Show me the way to Amalia Burleigh Ground.", "wer": 0.25, "zero_wer_after_release_normalization": false}
+{"sample_no": 7, "dataset_index": 180, "audio_seconds": 4.55, "reference": "Could YOU Tell Me The Way to Kiew Lee Tong Temple", "prediction": "Could you tell me the way to Kiew Lee Tong Temple?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 8, "dataset_index": 1416, "audio_seconds": 5.03, "reference": "Riverlife Church Dickson Road and Brockhampton Drive", "prediction": "Riverlife Church, Dixon Road and Brockhampton Drive.", "wer": 0.142857, "zero_wer_after_release_normalization": false}
+{"sample_no": 9, "dataset_index": 1646, "audio_seconds": 4.24, "reference": "Can I Take The M R T to Edgware Road", "prediction": "Can I Take The M, R, T to Echowra Road?", "wer": 0.1, "zero_wer_after_release_normalization": false}
+{"sample_no": 10, "dataset_index": 1589, "audio_seconds": 6.04, "reference": "Kirk Terrace Bukit Gombak M R T Station and Thomson Road", "prediction": "Kirk Terrace, Bukit Gombak, M, R, T Station and Thomson Road.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 11, "dataset_index": 449, "audio_seconds": 4.778, "reference": "BudgetOne Hotel Mei Hwan Road and Jalan Selendang Delima", "prediction": "Budget One Hotel, Mei Hwan Road and Jalan Sentang Delima.", "wer": 0.333333, "zero_wer_after_release_normalization": false}
+{"sample_no": 12, "dataset_index": 96, "audio_seconds": 1.87, "reference": "Lambert", "prediction": "Lembah.", "wer": 1.0, "zero_wer_after_release_normalization": false}
+{"sample_no": 13, "dataset_index": 1067, "audio_seconds": 6.11, "reference": "John Fearns Nicoll Thio Chan Bee and Mary Gomes", "prediction": "John Fearns, Nico, Teo, Chan Bee and Mary Gomes.", "wer": 0.222222, "zero_wer_after_release_normalization": false}
+{"sample_no": 14, "dataset_index": 1263, "audio_seconds": 3.137, "reference": "Teochew Porridge IS very tasty", "prediction": "Teochew Porridge is very tasty.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 15, "dataset_index": 2744, "audio_seconds": 3.678, "reference": "I Am waiting For Brendan to Come Back from Hougang Mall", "prediction": "I'm waiting for Brendan to come back from Hougang Mount.", "wer": 0.272727, "zero_wer_after_release_normalization": false}
+{"sample_no": 16, "dataset_index": 1229, "audio_seconds": 5.19, "reference": "I know people who have the exact name as Tan Hwee Hock Julia Tan and Vincent Leow", "prediction": "I know People Who Have The exact same name as Tan Hwee Hock, Julia Tan and Vincent Leow.", "wer": 0.058824, "zero_wer_after_release_normalization": false}
+{"sample_no": 17, "dataset_index": 2839, "audio_seconds": 3.76, "reference": "Pulao Jingisukan Kheema", "prediction": "Pulao, Jingisukan, Kheer.", "wer": 0.333333, "zero_wer_after_release_normalization": false}
+{"sample_no": 18, "dataset_index": 2500, "audio_seconds": 4.27, "reference": "three seven one nine zero nine four five", "prediction": "Three, seven, one, nine, zero, nine, four, five.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 19, "dataset_index": 2463, "audio_seconds": 3.73, "reference": "show me the way to Killiney Road Post Office", "prediction": "Show me the way to Kiliney Road Post Office.", "wer": 0.111111, "zero_wer_after_release_normalization": false}
+{"sample_no": 20, "dataset_index": 1391, "audio_seconds": 4.49, "reference": "One Shenton Tower Pipit Road and Turnhouse Road", "prediction": "One Stanton Tower, Pipit Road and Turnhouse Road.", "wer": 0.125, "zero_wer_after_release_normalization": false}
+{"sample_no": 21, "dataset_index": 2883, "audio_seconds": 3.169, "reference": "How Long Will IT Take to Walk to Waterloo Close", "prediction": "How long will it take to walk to Waterloo Close?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 22, "dataset_index": 731, "audio_seconds": 3.34, "reference": "Tobie Pluma and Rock", "prediction": "Toby Pluma and Rob.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 23, "dataset_index": 2273, "audio_seconds": 3.23, "reference": "Heinz Pasar and Caltex", "prediction": "Hains Pasara and Caltex.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 24, "dataset_index": 1393, "audio_seconds": 3.408, "reference": "Cannie Viney and Eryn", "prediction": "Kenni, Vinnie and Erin.", "wer": 0.75, "zero_wer_after_release_normalization": false}
+{"sample_no": 25, "dataset_index": 810, "audio_seconds": 4.704, "reference": "I like Nasi Goreng Daging Merah very much", "prediction": "I like Nasi Goreng Denggi, Merah very much.", "wer": 0.125, "zero_wer_after_release_normalization": false}
+{"sample_no": 26, "dataset_index": 374, "audio_seconds": 3.46, "reference": "I Am meeting Jinnie At Stadium Place First", "prediction": "I Am meeting Genie At Stadium Place First.", "wer": 0.125, "zero_wer_after_release_normalization": false}
+{"sample_no": 27, "dataset_index": 2831, "audio_seconds": 3.23, "reference": "what's the latest Floxia product", "prediction": "What is the latest Floxia product?", "wer": 0.4, "zero_wer_after_release_normalization": false}
+{"sample_no": 28, "dataset_index": 813, "audio_seconds": 4.97, "reference": "Miso Soup Monsunabe and Kimchi Jjigae", "prediction": "Miso Soup, Monsunabe and Kimchi, Jjigae.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 29, "dataset_index": 32, "audio_seconds": 3.773, "reference": "I am flying to Cambodia now see you soon", "prediction": "I am flying to Cambodia. Now. See you soon.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 30, "dataset_index": 950, "audio_seconds": 4.32, "reference": "Wang Gungwu Charles Emmerson and Lui Pao ** Chuen", "prediction": "Wang Gang, Ooi, Charles, Emerson and Loh Pak Kuan Chan.", "wer": 0.875, "zero_wer_after_release_normalization": false}
+{"sample_no": 31, "dataset_index": 36, "audio_seconds": 4.26, "reference": "Could YOU Tell Me The Way to Frankel Street", "prediction": "Could you tell me the way to Frankel Street?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 32, "dataset_index": 2648, "audio_seconds": 4.469, "reference": "Medu Vada ** and Kheer", "prediction": "Mudo Vada, Jalebi, and Kheema.", "wer": 0.75, "zero_wer_after_release_normalization": false}
+{"sample_no": 33, "dataset_index": 1969, "audio_seconds": 2.62, "reference": "Haresh Aruna Jahangir", "prediction": "Harish, Aruna and Jagannath.", "wer": 1.0, "zero_wer_after_release_normalization": false}
+{"sample_no": 34, "dataset_index": 1233, "audio_seconds": 2.899, "reference": "I can't eat all of this Bun", "prediction": "I can't eat all of this Fun.", "wer": 0.142857, "zero_wer_after_release_normalization": false}
+{"sample_no": 35, "dataset_index": 1232, "audio_seconds": 4.89, "reference": "Woodleigh Lane Rosa Hall and National Neuroscience Institute", "prediction": "Woodleigh Lane, Rosa Hill and National Neuropsychiatric Institute.", "wer": 0.25, "zero_wer_after_release_normalization": false}
+{"sample_no": 36, "dataset_index": 2571, "audio_seconds": 6.24, "reference": "Date Tamarind Chutney Ravioli and Hiyashi Chuka", "prediction": "Edet, Tamarind Chutney, Ravioli and Hiyashi Chuka.", "wer": 0.142857, "zero_wer_after_release_normalization": false}
+{"sample_no": 37, "dataset_index": 1717, "audio_seconds": 3.44, "reference": "Vespa Mug Root Beer and Dettol", "prediction": "Vespa, Mug Root Beer and Detro.", "wer": 0.166667, "zero_wer_after_release_normalization": false}
+{"sample_no": 38, "dataset_index": 2768, "audio_seconds": 13.03, "reference": "number sequence is T nine three four zero five seven two J and date of birth is eleven December nineteen thirty three", "prediction": "Number Sequence is T, nine, three, four, zero, five, seven, two, J, and date of birth is eleven, December, nineteen, thirty three.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 39, "dataset_index": 394, "audio_seconds": 3.41, "reference": "five hundred and four thousand two hundred and ninety eight", "prediction": "Five hundred and four thousand, two hundred and ninety eight.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 40, "dataset_index": 2586, "audio_seconds": 4.638, "reference": "Which Bus should I Take to Singapore Indian Association", "prediction": "Which bus should I take to Singapore Indian Association?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 41, "dataset_index": 496, "audio_seconds": 5.2, "reference": "how long does it take to get to Backpackers Inn Chinatown by taxi", "prediction": "How long does it take to get to Backpackers Inn Chinatown by taxi?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 42, "dataset_index": 1671, "audio_seconds": 4.276, "reference": "six seven two nine one two five four", "prediction": "Six, seven, two, nine, one, two, five, four.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 43, "dataset_index": 1508, "audio_seconds": 5.54, "reference": "Osman Merican Salleh Japar and Dennis Tan Lip Fong", "prediction": "Osman, Braekhan, Saleh, Japar and Dennis Tan Lip Fong.", "wer": 0.222222, "zero_wer_after_release_normalization": false}
+{"sample_no": 44, "dataset_index": 126, "audio_seconds": 6.93, "reference": "Sree Ramar Temple Police Heritage Centre and Istana Sri Temasek", "prediction": "Sri Rama Temple Police Heritage Centre and Istana Sri Temasek.", "wer": 0.2, "zero_wer_after_release_normalization": false}
+{"sample_no": 45, "dataset_index": 26, "audio_seconds": 2.49, "reference": "Campbell loves Chaat Papri", "prediction": "Campbell loves Chap Purée.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 46, "dataset_index": 2633, "audio_seconds": 3.68, "reference": "please tell me how to get to Cairnhill Circle", "prediction": "Please tell me how to get to Cairnhill Circle.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 47, "dataset_index": 2370, "audio_seconds": 7.66, "reference": "twenty fourth July twenty twenty four seventeen hundred hours thirty three minutes and twenty seven seconds", "prediction": "Twenty four, July, twenty, twenty four, seventeen hundred hours, thirty three minutes and twenty seven seconds.", "wer": 0.0625, "zero_wer_after_release_normalization": false}
+{"sample_no": 48, "dataset_index": 1331, "audio_seconds": 2.99, "reference": "Iyanna love Carbonara", "prediction": "Liana, Lofu, Carbonara.", "wer": 0.666667, "zero_wer_after_release_normalization": false}
+{"sample_no": 49, "dataset_index": 1457, "audio_seconds": 3.505, "reference": "This IS The Best Dendeng Paru that I Can Find", "prediction": "This is the best Dendeng Paru that I can find.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 50, "dataset_index": 2396, "audio_seconds": 3.27, "reference": "Jaybird Cheers and Candy Empire", "prediction": "Jaybird Cheers and Candy Empire.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 51, "dataset_index": 2880, "audio_seconds": 8.283, "reference": "I know People Who Have The exact same name as Robin Tessensohn Twisstii and Thomas Braddell", "prediction": "I know People Who Have The exact same name as Robin Tessensohn Twisti and Thomas Braddell.", "wer": 0.0625, "zero_wer_after_release_normalization": false}
+{"sample_no": 52, "dataset_index": 2786, "audio_seconds": 3.03, "reference": "William Kristi and Hughes", "prediction": "William, Christie and Hugh.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 53, "dataset_index": 269, "audio_seconds": 4.27, "reference": "where can I find the best Lemak Siput", "prediction": "Where can I find the best Lemak Siput?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 54, "dataset_index": 194, "audio_seconds": 4.48, "reference": "I knew a person who has met both Shen Xi and Ridzwan Dzafir", "prediction": "I knew a person who has met both Shen Xi and Rizwan Dhafir.", "wer": 0.153846, "zero_wer_after_release_normalization": false}
+{"sample_no": 55, "dataset_index": 74, "audio_seconds": 3.5, "reference": "Safiya Budi and Farish", "prediction": "Safiya, Budi and Farish.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 56, "dataset_index": 314, "audio_seconds": 5.27, "reference": "can I take the M R T to Moonbeam View", "prediction": "Can I Take The M. R, T to Moonbeam View?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 57, "dataset_index": 1507, "audio_seconds": 5.82, "reference": "Champion Way Koek Road and Kampong Glam Constituency Sports Club", "prediction": "Champion Way, Creek Road and Kampong Glam Constituency Sports Club.", "wer": 0.1, "zero_wer_after_release_normalization": false}
+{"sample_no": 58, "dataset_index": 414, "audio_seconds": 6.23, "reference": "R E L C International Hotel Singapore Recreation Club and Lancaster Gate", "prediction": "R, E, L, C International Hotel, Singapore Recreation Club and Lancaster Gate.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 59, "dataset_index": 924, "audio_seconds": 8.25, "reference": "number sequence is S eight five three zero six two one D and date of birth is thirty November nineteen fifty four", "prediction": "Number sequence is S. eight, five, three, zero, six, two, one, D, and date of birth is thirty, November, nineteen, fifty four.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 60, "dataset_index": 1840, "audio_seconds": 5.67, "reference": "Kupang One George Street and Asphodel Inn", "prediction": "Kupang Wan, George Street and Ephraim Inn.", "wer": 0.285714, "zero_wer_after_release_normalization": false}
+{"sample_no": 61, "dataset_index": 1309, "audio_seconds": 3.524, "reference": "Rosaline Chan and Choo Hwee Lim", "prediction": "Roselyn Chan and Chew Kwei Lim.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 62, "dataset_index": 2435, "audio_seconds": 3.82, "reference": "Can YOU recommend Me A Restaurant near Kangkar", "prediction": "Can you recommend me a restaurant near Kankana?", "wer": 0.125, "zero_wer_after_release_normalization": false}
+{"sample_no": 63, "dataset_index": 2072, "audio_seconds": 4.783, "reference": "nine eight one one nine eight Zero three", "prediction": "Nine, eight, one, one, nine, eight, zero, three.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 64, "dataset_index": 179, "audio_seconds": 3.48, "reference": "Mayer Fresh And White and Friskies", "prediction": "Meyer Fresh and White Frisky's.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 65, "dataset_index": 2371, "audio_seconds": 5.21, "reference": "I knew person who has met both Julia Tan and Tan Chin Tuan", "prediction": "I knew a person who has met both Julia Tan and Tan Chin Tuan.", "wer": 0.076923, "zero_wer_after_release_normalization": false}
+{"sample_no": 66, "dataset_index": 501, "audio_seconds": 6.19, "reference": "this is the best Yao Cai Hei Ji Tang that I can find", "prediction": "This is the best Yao Cai Hei Zih Tang that I can find.", "wer": 0.076923, "zero_wer_after_release_normalization": false}
+{"sample_no": 67, "dataset_index": 1092, "audio_seconds": 2.52, "reference": "how much is Ikan Bakar", "prediction": "How much is Ikan Bakar?", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 68, "dataset_index": 1268, "audio_seconds": 4.29, "reference": "how long does it take to get to Singapore Institute of Management by taxi", "prediction": "How long is the ticket to Singapore Institute?", "wer": 0.642857, "zero_wer_after_release_normalization": false}
+{"sample_no": 69, "dataset_index": 199, "audio_seconds": 3.49, "reference": "a quarter to eleven in the morning", "prediction": "A quarter to eleven in The morning.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 70, "dataset_index": 1299, "audio_seconds": 6.59, "reference": "number sequence is S nine four seven zero one five two S and date of birth is seventeen June nineteen seventy two", "prediction": "Number sequence is S. nine, four, seven, zero, one, five, two, S, and the date of birth is seventeen, June, nineteen seventy two.", "wer": 0.045455, "zero_wer_after_release_normalization": false}
+{"sample_no": 71, "dataset_index": 2771, "audio_seconds": 5.12, "reference": "Tan Cheng Kee and Tracie Pang has met this person that I know of", "prediction": "Tan Cheng Kee and Tracy Pang has met this person that I know of.", "wer": 0.071429, "zero_wer_after_release_normalization": false}
+{"sample_no": 72, "dataset_index": 1769, "audio_seconds": 13.38, "reference": "Number sequence IS T one five eight six nine three seven L and Date of birth IS Third February nineteen seventy one", "prediction": "Number sequence is T, one, five, eight, six, nine, three, seven, L, and date of birth is third, February, nineteen, seventy one.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 73, "dataset_index": 2484, "audio_seconds": 6.56, "reference": "Casino Regulatory Authority Kangkar and Presbyterian High", "prediction": "Casino Regulatory Authority, Ganka and Presbyterian High.", "wer": 0.142857, "zero_wer_after_release_normalization": false}
+{"sample_no": 74, "dataset_index": 508, "audio_seconds": 4.9, "reference": "Sumang Link Jalan Mangkok and Qi Tian Gong Temple", "prediction": "Sumang Link, Jalan Mangkok and Chee Tian Gong Temple.", "wer": 0.111111, "zero_wer_after_release_normalization": false}
+{"sample_no": 75, "dataset_index": 1725, "audio_seconds": 4.4, "reference": "Birbal Aruna and Choor", "prediction": "Ribha Aruna and Chor.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 76, "dataset_index": 2777, "audio_seconds": 3.298, "reference": "Patti IS dropping Me off At Coronation Drive", "prediction": "Party is dropping me off at Coronation Drive.", "wer": 0.125, "zero_wer_after_release_normalization": false}
+{"sample_no": 77, "dataset_index": 1135, "audio_seconds": 4.95, "reference": "National Archives of Singapore Turf Club and Wimborne Road", "prediction": "National Archive of Singapore, Turf Club and Winbond Road.", "wer": 0.222222, "zero_wer_after_release_normalization": false}
+{"sample_no": 78, "dataset_index": 723, "audio_seconds": 4.144, "reference": "Arleth IS dropping Me off At Sri Guru Singh Sabha", "prediction": "Arlynn is dropping me off at Sri Guru Sheng Samma.", "wer": 0.3, "zero_wer_after_release_normalization": false}
+{"sample_no": 79, "dataset_index": 2684, "audio_seconds": 4.619, "reference": "Prada Formula one and Flora", "prediction": "Prata Formula One and Flora.", "wer": 0.2, "zero_wer_after_release_normalization": false}
+{"sample_no": 80, "dataset_index": 2894, "audio_seconds": 4.32, "reference": "EcoBrown's Dough Culture and Maybank", "prediction": "Ecolabrons, Doh, Culture and Maybank.", "wer": 0.4, "zero_wer_after_release_normalization": false}
+{"sample_no": 81, "dataset_index": 2720, "audio_seconds": 3.41, "reference": "W S one zero L H", "prediction": "W, S, one Zero, L, H.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 82, "dataset_index": 2652, "audio_seconds": 3.47, "reference": "Lynn recently opened a new chicken liver restaurant", "prediction": "Lynn recently opened a new Chicken Liver Restaurant.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 83, "dataset_index": 2438, "audio_seconds": 3.33, "reference": "how about a Boat Tour in Lesotho", "prediction": "How about a Pork Knuckle?", "wer": 0.571429, "zero_wer_after_release_normalization": false}
+{"sample_no": 84, "dataset_index": 1737, "audio_seconds": 6.195, "reference": "Mushroom Spaghetti Carbonara Salami and Chaat Papri", "prediction": "Mushroom Spaghetti Carbonara, Salami and Chaparain.", "wer": 0.285714, "zero_wer_after_release_normalization": false}
+{"sample_no": 85, "dataset_index": 1461, "audio_seconds": 8.217, "reference": "Goodwood Park Hotel Y W C A Fort Canning Lodge and Jalan Dinding", "prediction": "Goodwood Park Hotel, Y, W, C, A Fort Canning Lodge and Jalan Dinding.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 86, "dataset_index": 1484, "audio_seconds": 3.894, "reference": "six A seven four Zero O", "prediction": "Six, eight, seven, four, zero, O.", "wer": 0.166667, "zero_wer_after_release_normalization": false}
+{"sample_no": 87, "dataset_index": 880, "audio_seconds": 4.998, "reference": "Tan Ban Soon Cheo Kim Ban and Boey Chuan Poh", "prediction": "Tan Ban Soon, Cheo, Kim Ban and Boey Chuan Poh.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 88, "dataset_index": 2118, "audio_seconds": 3.64, "reference": "H D B I D A and A C J C", "prediction": "H, D, P, I, D, A and A, C, J, C.", "wer": 0.090909, "zero_wer_after_release_normalization": false}
+{"sample_no": 89, "dataset_index": 2163, "audio_seconds": 6.7, "reference": "are there food courts or restaurant near Genome Institute of Singapore", "prediction": "Are there food courts or restaurants near Genoy Institute of Singapore?", "wer": 0.181818, "zero_wer_after_release_normalization": false}
+{"sample_no": 90, "dataset_index": 1795, "audio_seconds": 3.44, "reference": "Vespa Mug Root Beer and Dettol", "prediction": "Vespa, Mug Root Beer and Detro.", "wer": 0.166667, "zero_wer_after_release_normalization": false}
+{"sample_no": 91, "dataset_index": 730, "audio_seconds": 2.899, "reference": "I can't eat all of this Bun", "prediction": "I can't eat all of this Fun.", "wer": 0.142857, "zero_wer_after_release_normalization": false}
+{"sample_no": 92, "dataset_index": 570, "audio_seconds": 3.851, "reference": "Nasi Lemak Cheese Naan and Gado Gado", "prediction": "Nasi Lemak, Cheese, Naan and Gado Gado.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 93, "dataset_index": 1922, "audio_seconds": 4.45, "reference": "Elizabeth Drive Peicai Secondary School and Boo Tong San Temple", "prediction": "Elizabeth Drive, Pei Chye Secondary School and Bo Tong San Temple.", "wer": 0.3, "zero_wer_after_release_normalization": false}
+{"sample_no": 94, "dataset_index": 1493, "audio_seconds": 6.14, "reference": "Sigmund recently opened A New Chinese Spinach with Assorted Eggs Restaurant", "prediction": "Sigmund recently opened a new Chinese Spinach Assorted Eggs restaurant.", "wer": 0.090909, "zero_wer_after_release_normalization": false}
+{"sample_no": 95, "dataset_index": 1184, "audio_seconds": 3.43, "reference": "you must try Unagi when you are here", "prediction": "You must try Unagi when you are here.", "wer": 0.0, "zero_wer_after_release_normalization": true}
+{"sample_no": 96, "dataset_index": 1075, "audio_seconds": 8.12, "reference": "Song Hoot Kiam ** Prabhakara Jimmy Quek and Prema Letchumanan", "prediction": "Song Hoot Kiam, Parashu, Pakar, James, Quak and Prema Letchumanan.", "wer": 0.444444, "zero_wer_after_release_normalization": false}
+{"sample_no": 97, "dataset_index": 2779, "audio_seconds": 5.122, "reference": "Scott's Colora Water Colours and Dermaveen", "prediction": "Scotts Colora Water Colours and Devine.", "wer": 0.166667, "zero_wer_after_release_normalization": false}
+{"sample_no": 98, "dataset_index": 1857, "audio_seconds": 7.898, "reference": "Clay Pot Mee Stir Fried Pork with Ginger Onions and Chrysanmumty Fried Grouper", "prediction": "Clay Pot, Mee, Stir Fry, Pork, Ginger, Onions and Kremetete Fried Grouper.", "wer": 0.230769, "zero_wer_after_release_normalization": false}
+{"sample_no": 99, "dataset_index": 1415, "audio_seconds": 4.78, "reference": "Centennial Tower Woodlands Checkpoint and Blessed Sacrament Church", "prediction": "Sandy Shell Tower, Woodlands, Sector and Blessed Sacrament's Church.", "wer": 0.5, "zero_wer_after_release_normalization": false}
+{"sample_no": 100, "dataset_index": 527, "audio_seconds": 5.25, "reference": "Leedon Road City Vibe at Clementi and Pasir Panjang Hill Brethren Church", "prediction": "Ledan Road, Cityscape, Clementi and Pasir Panjang Hill Brethren Church.", "wer": 0.333333, "zero_wer_after_release_normalization": false}

samples/bf16_100_sample_eval_seed20260515.md

@@ -0,0 +1,30 @@
+# Fresh 100-sample BF16 evaluation
+
+- Repository: `majentik/gemma-4-e4b-mlx-elderwise-MERaLiON`
+- Model: Gemma-4-E4B BF16 + MERaLiON speech LoRA release package
+- Dataset: MNSC ASR Part 2 Test (held out)
+- Sampling: random without replacement, seed `20260515`
+- Samples: 100
+- Normalization: lowercase, strip punctuation, collapse whitespace, strip edges
+- Aggregate WER: **15.31%**
+- Baseline WER: 25.78%
+- Full 3000-clip release WER: 16.09%
+- Exact zero-WER clips in this sample: 35 / 100
+- Nonzero clips with WER ≤20%: 31
+
+The companion JSONL contains all 100 references, predictions, dataset indices, audio durations, and per-sample WER values.
+
+## First 10 samples
+
+| # | Dataset index | WER | Reference | Prediction |
+|---:|---:|---:|---|---|
+| 1 | 2086 | 77.8% | Justin Zhuang Cham Tao Soon and Ted De Ponti | Justin Chua, Chan, Tau, Soon and Teck Dee Pon, Teck. |
+| 2 | 779 | 0.0% | can you recommend me a restaurant near Brani Marine Fire Station | Can you recommend me a restaurant near Brani Marine Fire Station? |
+| 3 | 1935 | 0.0% | Tan Hwa Luck Brigadier Ivan Simson and Kenneth Mitchell | Tan Hwa Luck Brigadier Ivan Simson and Kenneth Mitchell. |
+| 4 | 1977 | 0.0% | how much is Sam Lau | How much is Sam Lau? |
+| 5 | 1524 | 0.0% | this shop sells Shan Rui Tang | This shop sells Shan Rui Tang. |
+| 6 | 782 | 25.0% | Show Me The Way to Ahmadiyya Burial Ground | Show me the way to Amalia Burleigh Ground. |
+| 7 | 180 | 0.0% | Could YOU Tell Me The Way to Kiew Lee Tong Temple | Could you tell me the way to Kiew Lee Tong Temple? |
+| 8 | 1416 | 14.3% | Riverlife Church Dickson Road and Brockhampton Drive | Riverlife Church, Dixon Road and Brockhampton Drive. |
+| 9 | 1646 | 10.0% | Can I Take The M R T to Edgware Road | Can I Take The M, R, T to Echowra Road? |
+| 10 | 1589 | 0.0% | Kirk Terrace Bukit Gombak M R T Station and Thomson Road | Kirk Terrace, Bukit Gombak, M, R, T Station and Thomson Road. |

samples/bf16_100_sample_eval_seed20260515_summary.json

@@ -0,0 +1,18 @@
+{
+  "repo_id": "majentik/gemma-4-e4b-mlx-elderwise-MERaLiON",
+  "artifact": "fresh 100-sample BF16 release evaluation",
+  "dataset": "MERaLiON/Multitask-National-Speech-Corpus-v1, ASR Part 2 Test",
+  "sample_count": 100,
+  "sampling": {
+    "method": "random without replacement",
+    "seed": 20260515
+  },
+  "metric": "WER via elderwise.eval_wer release normalization",
+  "wer": 0.153061,
+  "wer_percent": 15.31,
+  "baseline_wer_percent": 25.78,
+  "full_test_set_wer_percent": 16.09,
+  "zero_wer_count": 35,
+  "nonzero_le_20pct_count": 31,
+  "jsonl_file": "bf16_100_sample_eval_seed20260515.jsonl"
+}