Check submissions against every sealed suite; count only failed time-outs

Decontamination and name checks now cover every sealed suite in configs/suites (full TB2 and LHTB,
not only tb2-9b's 32 tasks), once per dataset revision. Recipe v2's per-suite stage markers
(baseline_eval.<suite>.tNN) map to their stage. A task that passes after the time limit is a pass,
not a time-out.

challenges.py

@@ -550,7 +550,7 @@ def parse_log(events):
         stage=stages[current]
         if 'VLLM_HEALTH_FAILED' in line: health_failures+=1;continue
         if line.startswith('[posttrainarena] '):
-            name=line[17:].split(':',1)[0].strip();rollout=ROLLOUT.match(name);key='training' if rollout else STAGE_OF.get(name)
+            name=line[17:].split(':',1)[0].strip();rollout=ROLLOUT.match(name);key='training' if rollout else STAGE_OF.get(name) or STAGE_OF.get(name.split('.',1)[0])  # recipe v2: baseline_eval.<suite>.tNN
             if key:
                 enter(key,t);pipeline_stage='train_grpo' if rollout else name
                 if rollout: train_step=int(rollout.group(1));stages[key]['rollouts']+=1;stages[key]['steps'].add(train_step)
@@ -566,7 +566,7 @@ def parse_log(events):
             kind={'PASS':'pass','FAIL':'fail','ERR':'error'}[verdict.group(1)];stage[kind]+=1
             note=(verdict.group(3) or '').rstrip('…').strip()
             feed.append({'t':t,'stage':current,'event':kind,'task':verdict.group(2),'note':note[:120] or None,'step':train_step if current=='training' else None})
-            if 'wall-clock' in note or 'idle timeout' in note.lower(): stage['timeout']+=1
+            if kind!='pass' and ('wall-clock' in note or 'idle timeout' in note.lower()): stage['timeout']+=1  # a pass after the limit is a pass
             elif kind=='error': note=note[:90] or 'unspecified';stage['reasons'][note]=stage['reasons'].get(note,0)+1
             if current=='training' and train_step is not None:
                 counts=step_verdicts.setdefault(train_step,{'pass':0,'fail':0,'error':0});counts[kind]+=1

pipeline_jobs.py

@@ -4,8 +4,7 @@ The job mirrors pipelines/benchflow-task-posttrain/scripts/bootstrap_gpu.sh (tor
 vLLM 0.23 cu129 wheel, pipeline[train]), serves the policy with the TRL vLLM server on one GPU,
 reaches the model bridge on loopback (no inbound tunnel; rollout ingress goes through the Space relay), and runs
 `posttrainarena-train run` on GPUs 0-3 with Daytona sandboxes. Results upload to RUNS.
-This is the launcher that completed the phase-2 loop; keep it byte-for-byte with /tmp/pta-live/phase2-job.py
-when changing one of them.
+The organizer's baseline grid (~/benchflow/pta-work/launchers/eval_grid.py) reuses this bootstrap; keep them aligned.
 """
 import os
 from pathlib import Path

test_compose.py

@@ -86,3 +86,21 @@ class ComposeTest(unittest.TestCase):
 
 if __name__ == '__main__':
     unittest.main()
+
+
+class SealedCoverageTest(unittest.TestCase):
+    def test_every_sealed_suite_is_checked_once_per_dataset_revision(self):
+        import validation_gates as gates
+        with mock.patch.object(gates, '_sealed_instructions', return_value={}):
+            suites = {s.repo_id: s for s in gates.sealed_suites()}
+        tb2 = suites['benchflow/tb2-benchflow']
+        self.assertEqual(len(tb2.evaluated), 86)  # tb2-32 and tb2 share one revision: the union
+        self.assertEqual(len(suites['benchflow/lhtb-nongame-benchflow'].evaluated), 38)
+
+    def test_recipe_v2_stage_names_and_late_passes(self):
+        import challenges
+        events = [('t1', '[posttrainarena] baseline_eval.lhtb.t02: bench eval run ...'), ('t2', '[PASS] task-a (tools=3) (Agent prompt exceeded wall-clock budget 900s)'),
+                  ('t3', '[FAIL] task-b (tools=2) (Agent prompt exceeded wall-clock budget 900s)')]
+        parsed = challenges.parse_log(events)
+        self.assertIn('baseline', parsed['stages'])
+        self.assertEqual((parsed['stages']['baseline']['pass'], parsed['stages']['baseline']['timeout']), (1, 1))

validation_gates.py

@@ -475,20 +475,27 @@ def _sealed_instructions(repo_id: str, revision: str):
 
 
 def sealed_suites():
-    """Sealed suites of the arena challenges. Instructions come from the private dataset at the pinned revision with
-    the Space token; if they cannot be read, name checks still run against the evaluated task list."""
-    import challenges  # lazy: challenges imports environments, which imports this module
-    suites = []
-    for row in challenges.CHALLENGES:
-        suite = row.get('eval_suite') or {}
-        if not suite.get('sealed'):
+    """Every sealed held-out suite registered in configs/suites, planned ones included, so a submission cannot copy a
+    task a current or future challenge evaluates. Suites on the same dataset revision are checked once, with the union
+    of their task lists. Instructions come from the private dataset with the Space token; if they cannot be read, name
+    checks still run against the task lists."""
+    import compose  # lazy, like the challenge registry this replaced
+    by_source = {}
+    for suite_id in compose.ids('suites'):
+        data = compose.fragment('suites', suite_id)
+        if not data.get('meta', {}).get('sealed'):
             continue
-        evaluated = challenges.suite_task_ids(row)
+        suite = data['suite']
+        entry = by_source.setdefault((suite['repo_id'], suite['revision']), {'ids': [], 'tasks': set()})
+        entry['ids'].append(suite_id)
+        entry['tasks'].update(compose.task_ids(suite_id))
+    suites = []
+    for (repo_id, revision), entry in by_source.items():
         try:
-            instructions, error = _sealed_instructions(suite['repo_id'], suite['revision']), None
+            instructions, error = _sealed_instructions(repo_id, revision), None
         except Exception as exc:  # network, token, or dataset problem: degrade to a review note, never block
             instructions, error = None, type(exc).__name__
-        suites.append(SealedSuite(row['id'], suite['repo_id'], suite['revision'], evaluated, instructions, error))
+        suites.append(SealedSuite(' + '.join(sorted(entry['ids'])), repo_id, revision, sorted(entry['tasks']), instructions, error))
     return suites