Run pinned submitted environments on HF with evidence collection and reviewed result publication

public_results.py

@@ -1,5 +1,9 @@
 """Explicit organizer publication of reviewed, allowlisted result metadata."""
 import json
+import hashlib
+import re
+from pathlib import Path
+from huggingface_hub import CommitOperationAdd, hf_hub_download
 from fastapi import APIRouter, HTTPException, Request
 import collab
 import environments as env
@@ -16,10 +20,50 @@ def sanitized(row):
         'result':{'verification':'valid','reviewed_at':result.get('reviewed_at'),
             'payload':{k:p[k] for k in ('baseline','score','report_url','job_url','adapter_url') if k in p}}}
 
+def summary(row, report, raw_hash):
+    """Public machine evidence without model text, prompts, credentials or private logs."""
+    def evaluation(value):
+        if not isinstance(value,dict):return None
+        safe={k:value[k] for k in ('status','passed_tests','total_tests','score','verifier_reward','sandbox_job_id','sandbox_terminated') if k in value}
+        safe['tests']=[{k:test[k] for k in ('name','status') if k in test} for test in value.get('tests',[]) if isinstance(test,dict)]
+        return safe
+    config=report.get('config',{})
+    return {'schema_version':1,'experiment_id':row['id'],'status':report.get('status'),
+        'scope':report.get('scope'),'verification':'valid','reviewed_at':row['result'].get('reviewed_at'),
+        'source_report':row['result']['payload']['report_url'],'source_report_sha256':raw_hash,
+        'job_url':row['result']['payload'].get('job_url'),
+        'config':{k:config[k] for k in ('run_id','experiment_id','environment_id','package_repo','package_revision','environment_path','model','model_revision','steps','rate','rank','seed','max_new_tokens','sandbox_image','sandbox_image_digest','sandbox_image_revision','profile') if k in config},
+        'decoding':report.get('decoding'),'source_file_sha256':report.get('source_file_sha256'),
+        'global_steps':report.get('global_steps'),'training_loss':report.get('training_loss'),
+        'adapter_reloaded':report.get('adapter_reloaded'),'sandbox_terminated':report.get('sandbox_terminated'),
+        'network_isolation':report.get('network_isolation'),'protocol_deviations':report.get('protocol_deviations'),
+        'controls':{k:evaluation(v) for k,v in report.get('controls',{}).items() if k in ('empty','oracle')},
+        'baseline':evaluation(report.get('baseline')),'final':evaluation(report.get('final'))}
+
 def publish_feed():
     rows=[r for r in collab.experiments() if r.get('public_result') and (r.get('result') or {}).get('verification')=='valid']
-    content={'schema_version':1,'generated_at':collab.now(),'experiments':[sanitized(r) for r in rows]}
-    client=env.api()
+    client=env.api(); public_rows=[sanitized(r) for r in rows]
+    operations=[]; summary_paths={}
+    for row in rows:
+        run_id=(row.get('result') or {}).get('execution_run_id')
+        if not run_id or not re.fullmatch(r'arena-[0-9a-f]{12}',run_id):continue
+        url=row['result']['payload']['report_url']
+        pattern=r'https://huggingface.co/datasets/'+re.escape(env.REPO)+r'/(?:blob|resolve)/([0-9a-f]{40})/(arena/results/'+re.escape(run_id)+r'\.json)'
+        match=re.fullmatch(pattern,url)
+        if not match:raise HTTPException(409,'Collected report does not match its reserved run.')
+        raw=Path(hf_hub_download(env.REPO,match[2],repo_type='dataset',revision=match[1],token=client.token)).read_bytes()
+        if len(raw)>2_000_000:raise HTTPException(409,'Report is too large for public evidence export.')
+        report=json.loads(raw)
+        if report.get('status')!='completed' or report.get('experiment_id')!=row['id']:raise HTTPException(409,'Cannot publish incomplete runner evidence.')
+        path='reports/'+run_id+'.json';summary_paths[row['id']]=path
+        operations.append(CommitOperationAdd(path_in_repo=path,path_or_fileobj=json.dumps(summary(row,report,hashlib.sha256(raw).hexdigest()),indent=2).encode()))
+    if operations:
+        head=client.repo_info(REPO,repo_type='dataset').sha
+        try: receipt=client.create_commit(repo_id=REPO,repo_type='dataset',parent_commit=head,operations=operations,commit_message='Publish sanitized reviewed execution evidence')
+        except Exception:raise HTTPException(503,'Evidence publication could not be confirmed. Retry the same publication request.') from None
+        for row in public_rows:
+            if row['id'] in summary_paths:row['result']['payload']['report_url']=f'https://huggingface.co/datasets/{REPO}/blob/{receipt.oid}/{summary_paths[row["id"]]}'
+    content={'schema_version':1,'generated_at':collab.now(),'experiments':public_rows}
     for _ in range(4):
         head=client.repo_info(REPO,repo_type='dataset').sha
         try:

test_public_results.py

@@ -11,6 +11,15 @@ class Tests(unittest.TestCase):
   value=public.sanitized(self.row)
   self.assertNotIn('notes',value);self.assertNotIn('training_data',value['config']);self.assertNotIn('verification_note',value['result'])
   self.assertIsNone(value['result']['payload']['baseline'])
+ def test_machine_evidence_excludes_generated_text_and_logs(self):
+  report={'status':'completed','experiment_id':self.row['id'],'config':{'model':'org/model','HF_TOKEN':'never-publish'},'baseline':{'passed_tests':8,'total_tests':9,'generated_text':'private-output','verifier_log':'private-log','tests':[{'name':'test_one','status':'passed','message':'private-failure-details'}]},'controls':{'oracle':{'artifacts':{'private':'data'},'passed_tests':9,'total_tests':9}}}
+  value=public.summary(self.row,report,'a'*64)
+  import json
+  text=json.dumps(value)
+  for secret in ('never-publish','private-output','private-log','private-failure-details'):self.assertNotIn(secret,text)
+  self.assertNotIn('artifacts',value['controls']['oracle'])
+  self.assertEqual(value['baseline']['passed_tests'],8)
+  self.assertEqual(value['baseline']['tests'],[{'name':'test_one','status':'passed'}])
  def test_unreviewed_results_rejected(self):
   for status in ('pending','invalid'):
    self.row['result']['verification']=status