arena.py

"""Competition catalog and authenticated, non-ranked community submissions."""
import json, os, re, uuid
from datetime import datetime, timezone
from pathlib import Path
from fastapi import HTTPException
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.errors import EntryNotFoundError
from pydantic import BaseModel, Field

REPO = 'benchflow/posttrain-lab-20260920-artifacts'
PATH = 'arena/submissions-v1.json'
COMPETITION = {
 'id':'skillsbench', 'name':'SkillsBench', 'status':'Practice open',
 'description':'Google Auto repair with Qwen3.6-27B LoRA. One generated repair, three original verifier checks.',
 'tracks':[
  {'id':'practice','name':'Seen-task practice','status':'Accepting artifacts','description':'Training and evaluation may use the same task. Results demonstrate memorization or pipeline behavior, not generalization. BenchFlow editors can launch the fixed verifier. Scores come from its result, never from uploaded claims.'},
  {'id':'ranked','name':'Held-out competition','status':'Not open','description':'Ranked submissions will open after the task split, evaluation protocol, model constraints, compute rules, and deadline are published.'}
 ]
}
class Submission(BaseModel):
 competition_id: str = 'skillsbench'
 track: str = 'practice'
 model_repo: str = Field(min_length=3,max_length=150)
 revision: str = Field(min_length=1,max_length=80)
 title: str = Field(min_length=3,max_length=100)
 notes: str = Field(default='',max_length=1200)

def api(): return HfApi(token=os.environ.get('HF_TOKEN'))
def read(revision=None):
 try:
  p=hf_hub_download(REPO,PATH,repo_type='dataset',revision=revision,token=os.environ.get('HF_TOKEN'),force_download=True)
  return json.loads(Path(p).read_text())
 except EntryNotFoundError:return []

def submit(value,authorization=None,identity=None):
 if value.competition_id!='skillsbench' or value.track!='practice':raise HTTPException(422,'Only the SkillsBench practice track is open.')
 if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+',value.model_repo):raise HTTPException(422,'Enter a Hugging Face model repository as owner/name.')
 if len(value.title.strip())<3:raise HTTPException(422,'Enter a title of at least three characters.')
 token=authorization[7:] if authorization and authorization.startswith('Bearer ') else None
 if identity is None:
  if not token:raise HTTPException(401,'Sign in with Hugging Face or supply a bearer token.')
  try:identity=HfApi(token=token).whoami()
  except Exception:raise HTTPException(401,'Could not verify your HF identity. Sign in again.') from None
 # Public adapters, or BenchFlow adapters already accessible to both parties.
 try: model=HfApi(token=False).model_info(value.model_repo,revision=value.revision)
 except Exception:
  if not value.model_repo.startswith('benchflow/'):raise HTTPException(422,'Use a public adapter or an accessible BenchFlow adapter.') from None
  try:
   if token:model=HfApi(token=token).model_info(value.model_repo,revision=value.revision)
   else:
    if not any(o.get('name')=='benchflow' and o.get('roleInOrg') in ('write','admin') for o in identity.get('orgs',[])):raise ValueError('Private adapter requires BenchFlow editor access.')
    model=api().model_info(value.model_repo,revision=value.revision)
   api().model_info(value.model_repo,revision=model.sha)
  except Exception:raise HTTPException(422,'Both you and this workspace need access to the adapter.') from None
 from arena_jobs import validate_adapter
 validate_adapter(value.model_repo,model.sha)
 record=dict(id='submission-'+uuid.uuid4().hex[:12],competition_id='skillsbench',track='practice',
  author=identity['name'],title=value.title.strip(),model_repo=value.model_repo,revision=model.sha,
  notes=value.notes.strip(),status='Awaiting evaluation',score=None,
  created_at=datetime.now(timezone.utc).isoformat(),model_url=f'https://huggingface.co/{value.model_repo}/tree/{model.sha}')
 client=api()
 for _ in range(3):
  head=client.repo_info(REPO,repo_type='dataset').sha
  rows=read(head)
  duplicate=next((r for r in rows if r['author']==record['author'] and r['model_repo']==record['model_repo'] and r['revision']==record['revision'] and r['track']==record['track']),None)
  if duplicate:return duplicate
  try:
   client.upload_file(repo_id=REPO,repo_type='dataset',path_in_repo=PATH,path_or_fileobj=json.dumps([record]+rows,indent=2).encode(),parent_commit=head,commit_message='Register SkillsBench practice submission')
   return record
  except Exception as e:
   if getattr(getattr(e,'response',None),'status_code',None) in (409,412):continue
   raise HTTPException(503,'Could not confirm registration. Refresh submissions before trying again.') from None
 raise HTTPException(409,'Another submission updated the list. Refresh submissions and try again.')

CATALOG='arena/competitions-v1.json'
def competitions():
 try:
  p=hf_hub_download(REPO,CATALOG,repo_type='dataset',token=os.environ.get('HF_TOKEN'),force_download=True)
  return json.loads(Path(p).read_text())
 except EntryNotFoundError:return [COMPETITION]
class NewCompetition(BaseModel):
 id: str = Field(pattern=r'^[a-z][a-z0-9-]{2,49}$')
 name: str = Field(min_length=3,max_length=80)
 description: str = Field(min_length=10,max_length=800)
 task_reference: str = Field(min_length=3,max_length=300)
 metric: str = Field(min_length=3,max_length=300)
def create_competition(value,authorization):
 from arena_jobs import operator
 author=operator(authorization)
 client=api();head=client.repo_info(REPO,repo_type='dataset').sha
 # Pin catalog to the same head used by the atomic update.
 try:rows=json.loads(Path(hf_hub_download(REPO,CATALOG,repo_type='dataset',revision=head,token=os.environ.get('HF_TOKEN'))).read_text())
 except EntryNotFoundError:rows=[COMPETITION]
 if any(c['id']==value.id for c in rows):raise HTTPException(409,'That competition ID already exists.')
 record={**value.model_dump(),'status':'Draft','created_by':author,'tracks':[]}
 try:client.upload_file(repo_id=REPO,repo_type='dataset',path_in_repo=CATALOG,path_or_fileobj=json.dumps(rows+[record],indent=2).encode(),parent_commit=head,commit_message='Create competition draft')
 except Exception:raise HTTPException(409,'Could not confirm catalog update. Refresh before retrying.') from None
 return record