| { |
| "dataset_revision": "f2fb14e4ec977c420f376c08785664cd38763d7e", |
| "families": { |
| "agent_next_action_type": { |
| "n": 1158, |
| "majority_correct": 616, |
| "majority_accuracy": 0.531951640759931, |
| "majority_labels": [ |
| "text_response" |
| ], |
| "candidate_count": { |
| "median": 3.0, |
| "min": 3, |
| "max": 3 |
| }, |
| "uniform_expected_accuracy": 0.3333333333333333, |
| "reference_definition": "Descriptive test-set constant-label majority; no model fitting", |
| "unseen_task_family": false, |
| "training_fixed_majority": { |
| "label": "text_response", |
| "correct": 616, |
| "n": 1158, |
| "majority_label_absent_from_choices": 0 |
| } |
| }, |
| "tool_selection": { |
| "n": 542, |
| "majority_correct": 79, |
| "majority_accuracy": 0.14575645756457564, |
| "majority_labels": [ |
| "transfer_to_human_agent" |
| ], |
| "candidate_count": { |
| "median": 20.0, |
| "min": 11, |
| "max": 32 |
| }, |
| "uniform_expected_accuracy": 0.051859254651728436, |
| "reference_definition": "Descriptive test-set constant-label majority; no model fitting", |
| "unseen_task_family": false, |
| "training_fixed_majority": { |
| "label": "transfer_to_human_agent", |
| "correct": 79, |
| "n": 542, |
| "majority_label_absent_from_choices": 235 |
| } |
| }, |
| "when_to_call_tool": { |
| "n": 3652, |
| "majority_correct": 1295, |
| "majority_accuracy": 0.3546002190580504, |
| "majority_labels": [ |
| "cannot_answer", |
| "tool_call" |
| ], |
| "candidate_count": { |
| "median": 4.0, |
| "min": 4, |
| "max": 4 |
| }, |
| "uniform_expected_accuracy": 0.25, |
| "reference_definition": "Descriptive test-set constant-label majority; no model fitting", |
| "unseen_task_family": true |
| } |
| }, |
| "when2call_training_rows": 0 |
| } |
|
|
