File size: 1,895 Bytes
d5b398e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | {
"dataset_revision": "f2fb14e4ec977c420f376c08785664cd38763d7e",
"families": {
"agent_next_action_type": {
"n": 1158,
"majority_correct": 616,
"majority_accuracy": 0.531951640759931,
"majority_labels": [
"text_response"
],
"candidate_count": {
"median": 3.0,
"min": 3,
"max": 3
},
"uniform_expected_accuracy": 0.3333333333333333,
"reference_definition": "Descriptive test-set constant-label majority; no model fitting",
"unseen_task_family": false,
"training_fixed_majority": {
"label": "text_response",
"correct": 616,
"n": 1158,
"majority_label_absent_from_choices": 0
}
},
"tool_selection": {
"n": 542,
"majority_correct": 79,
"majority_accuracy": 0.14575645756457564,
"majority_labels": [
"transfer_to_human_agent"
],
"candidate_count": {
"median": 20.0,
"min": 11,
"max": 32
},
"uniform_expected_accuracy": 0.051859254651728436,
"reference_definition": "Descriptive test-set constant-label majority; no model fitting",
"unseen_task_family": false,
"training_fixed_majority": {
"label": "transfer_to_human_agent",
"correct": 79,
"n": 542,
"majority_label_absent_from_choices": 235
}
},
"when_to_call_tool": {
"n": 3652,
"majority_correct": 1295,
"majority_accuracy": 0.3546002190580504,
"majority_labels": [
"cannot_answer",
"tool_call"
],
"candidate_count": {
"median": 4.0,
"min": 4,
"max": 4
},
"uniform_expected_accuracy": 0.25,
"reference_definition": "Descriptive test-set constant-label majority; no model fitting",
"unseen_task_family": true
}
},
"when2call_training_rows": 0
}
|