ThinkingCap Qwen3.8-27B ZeroGPU chat demo

README.md

@@ -1,13 +1,18 @@
 ---
 title: ThinkingCap Qwen3.8 27B
-emoji: 🐨
-colorFrom: pink
-colorTo: pink
+emoji: 🧢
+colorFrom: blue
+colorTo: indigo
 sdk: gradio
 sdk_version: 6.28.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+python_version: "3.12"
+startup_duration_timeout: 1h
+short_description: Qwen3.8-27B reasoning with 37% fewer thinking tokens
+models:
+  - bottlecapai/ThinkingCap-Qwen3.8-27B
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Chat demo for [bottlecapai/ThinkingCap-Qwen3.8-27B](https://huggingface.co/bottlecapai/ThinkingCap-Qwen3.8-27B),
+a token-efficient reasoning fine-tune of Qwen3.8-27B. Runs bf16 transformers on ZeroGPU (xlarge).
+Each reply shows how many tokens the model spent reasoning.

app.py

@@ -0,0 +1,221 @@
+import spaces  # must precede torch: it patches torch.cuda for ZeroGPU
+
+import queue
+import time
+from threading import Thread
+
+import gradio as gr
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from transformers.generation.streamers import BaseStreamer
+
+from chat_utils import notice, render_reply, split_ids, to_messages
+
+MODEL_ID = "bottlecapai/ThinkingCap-Qwen3.8-27B"
+
+# One ZeroGPU reservation. A long answer is generated as a chain of reservations, each
+# re-prefilling prompt + tokens so far, so a small value keeps queue priority high and
+# lets even signed-out visitors (2 min/day, charged 2x on xlarge) run one step.
+GPU_DURATION = 60
+# Headroom inside a reservation for worker start-up and the final yield; generation is
+# cut by max_time before ZeroGPU would abort the worker mid-stream.
+GPU_MARGIN = 15
+
+EFFORT_CHOICES = [("xhigh (recommended)", "xhigh"), ("medium", "medium"), ("low", "low"),
+                  ("off: no thinking", "off")]
+
+processor = AutoProcessor.from_pretrained(MODEL_ID)
+tokenizer = processor.tokenizer
+# bf16 is ~54 GB, so it needs the full 96 GB card (size="xlarge" below).
+model = AutoModelForImageTextToText.from_pretrained(
+    MODEL_ID, dtype=torch.bfloat16, device_map="cuda").eval()
+
+EOS_IDS = set(model.generation_config.eos_token_id)
+THINK_END_ID = tokenizer.convert_tokens_to_ids("</think>")
+
+
+class IdStreamer(BaseStreamer):
+    """Hands generated token ids to the consumer thread; skips the prompt."""
+
+    def __init__(self):
+        self.queue = queue.Queue()
+        self.prompt_seen = False
+
+    def put(self, value):
+        if not self.prompt_seen:
+            self.prompt_seen = True
+            return
+        self.queue.put(value.flatten().tolist())
+
+    def end(self):
+        self.queue.put(None)
+
+    def __iter__(self):
+        while (ids := self.queue.get()) is not None:
+            yield ids
+
+
+def extend_inputs(prompt, generated):
+    """Append already generated tokens to the processor output so generation resumes."""
+    prompt_ids = prompt["input_ids"]
+    extra = torch.tensor([generated], dtype=prompt_ids.dtype)
+    inputs = {}
+    for key, value in prompt.items():
+        if key == "input_ids":
+            value = torch.cat([value, extra], dim=1)
+        elif torch.is_tensor(value) and value.shape == prompt_ids.shape:
+            # Per-token side inputs: attention mask continues with 1s, type ids with 0s
+            # (generated tokens are text).
+            fill = 1 if key == "attention_mask" else 0
+            value = torch.cat([value, torch.full_like(extra, fill, dtype=value.dtype)], dim=1)
+        inputs[key] = value
+    return inputs
+
+
+@spaces.GPU(duration=GPU_DURATION, size="xlarge")
+def generate_step(prompt: dict, generated: list, max_new_tokens: int, sampling: dict):
+    """Generate up to one reservation's worth of tokens, yielding id lists as they arrive."""
+    started = time.monotonic()
+    inputs = {k: v.to("cuda") if torch.is_tensor(v) else v
+              for k, v in extend_inputs(prompt, generated).items()}
+    streamer = IdStreamer()
+    failure = []
+
+    def run():
+        try:
+            model.generate(**inputs, **sampling, streamer=streamer,
+                           max_new_tokens=max_new_tokens,
+                           max_time=GPU_DURATION - GPU_MARGIN - (time.monotonic() - started))
+        except Exception as exc:  # surfaced below instead of hanging the stream
+            failure.append(exc)
+            streamer.end()
+
+    thread = Thread(target=run)
+    thread.start()
+    n = 0
+    for ids in streamer:
+        n += len(ids)
+        yield ids
+    thread.join()
+    elapsed = time.monotonic() - started
+    print(f"[step] context={inputs['input_ids'].shape[1]} new={n} "
+          f"{elapsed:.1f}s {n / max(elapsed, 1e-6):.1f} tok/s", flush=True)
+    if failure:
+        raise failure[0]
+
+
+def respond(message: dict, history: list, reasoning_effort: str = "xhigh",
+            max_new_tokens: int = 8192, temperature: float = 1.0, top_p: float = 0.95,
+            top_k: int = 20):
+    """Chat with ThinkingCap-Qwen3.8-27B, a token-efficient reasoning model. Streams the reply.
+
+    Args:
+        message: The user turn: {"text": str, "files": [image or text file paths]}.
+        history: Prior conversation turns (Gradio messages format).
+        reasoning_effort: Thinking budget: "xhigh" (recommended), "medium", "low" or "off".
+        max_new_tokens: Cap on reasoning + answer tokens for this reply.
+        temperature: Sampling temperature; 0 means greedy decoding.
+        top_p: Nucleus sampling cutoff.
+        top_k: Top-k sampling cutoff.
+    """
+    try:
+        messages = to_messages(message, history)
+    except (ValueError, OSError) as exc:
+        raise gr.Error(str(exc))
+
+    thinking = reasoning_effort != "off"
+    template_kwargs = ({"reasoning_effort": reasoning_effort} if thinking
+                       else {"enable_thinking": False})
+    prompt = dict(processor.apply_chat_template(
+        messages, add_generation_prompt=True, tokenize=True, return_dict=True,
+        return_tensors="pt", preserve_thinking=False, **template_kwargs))
+    sampling = (dict(do_sample=True, temperature=temperature, top_p=top_p, top_k=int(top_k))
+                if temperature > 0 else dict(do_sample=False))
+
+    generated, started, n_reasoning = [], time.monotonic(), 0
+
+    def render(done):
+        nonlocal n_reasoning
+        if not thinking:
+            return render_reply(None, tokenizer.decode(generated, skip_special_tokens=True),
+                                0, 0, done)
+        reasoning_ids, answer_ids = split_ids(generated, THINK_END_ID)
+        n_reasoning = len(reasoning_ids)
+        answer = (None if answer_ids is None
+                  else tokenizer.decode(answer_ids, skip_special_tokens=True))
+        return render_reply(tokenizer.decode(reasoning_ids, skip_special_tokens=True),
+                            answer, n_reasoning, time.monotonic() - started, done)
+
+    stopped_by = None
+    while len(generated) < max_new_tokens:
+        before = len(generated)
+        try:
+            for ids in generate_step(prompt, list(generated), max_new_tokens - before, sampling):
+                generated.extend(ids)
+                yield render(done=False)
+        except Exception as exc:
+            if not generated:
+                raise
+            stopped_by = str(exc) or type(exc).__name__  # e.g. visitor's GPU quota ran out
+            break
+        if len(generated) == before or generated[-1] in EOS_IDS:
+            break
+
+    out = render(done=True)
+    if stopped_by:
+        out.append(notice("⚠️ Stopped early",
+                          f"{stopped_by}\n\nSign in to Hugging Face for a larger daily GPU "
+                          "quota, or lower **Reasoning effort**."))
+    elif generated and generated[-1] not in EOS_IDS:
+        where = " while still reasoning" if thinking and len(out) == 1 else ""
+        out.append(notice("⚠️ Reply was cut off",
+                          f"Hit the {max_new_tokens:,}-token limit{where}. Raise **Max new "
+                          "tokens** or lower **Reasoning effort** under Settings."))
+    yield out
+
+
+EXAMPLES = [[{"text": t, "files": []}] for t in (
+    "Find all real x such that √(x + 3) = x − 3.",
+    "A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. "
+    "How much does the ball cost?",
+    "Write a Python function that returns the longest palindromic substring of a string "
+    "in O(n²) time, with a short explanation.",
+    "Explain the difference between TCP and UDP to a new backend engineer, with one "
+    "example where each is the right choice.",
+)]
+
+DESCRIPTION = f"""\
+# 🧢 ThinkingCap · Qwen3.8-27B
+[**{MODEL_ID}**](https://huggingface.co/{MODEL_ID}) keeps Qwen3.8-27B's accuracy on hard and
+agentic tasks while thinking **37% fewer reasoning tokens** on average. Each reply shows how many
+tokens the model spent reasoning. Text and image input.
+[Blog post](https://bottlecapai.com/post/thinkingcap-qwen3-8-27b/) ·
+[FP8](https://huggingface.co/{MODEL_ID}-FP8) · [NVFP4](https://huggingface.co/{MODEL_ID}-NVFP4) ·
+[GGUF](https://huggingface.co/{MODEL_ID}-GGUF) · by [BottleCap AI](https://www.bottlecapai.com/)
+
+<sub>Runs bf16 on ZeroGPU and uses your daily Hugging Face GPU quota. Sign in for more.</sub>
+"""
+
+with gr.Blocks(title="ThinkingCap Qwen3.8-27B", fill_height=True) as demo:
+    gr.Markdown(DESCRIPTION)
+    with gr.Accordion("Settings", open=False):
+        effort = gr.Dropdown(EFFORT_CHOICES, value="xhigh", label="Reasoning effort")
+        max_tokens = gr.Slider(256, 16384, value=8192, step=256, label="Max new tokens")
+        temperature = gr.Slider(0.0, 1.5, value=1.0, step=0.05, label="Temperature")
+        top_p = gr.Slider(0.05, 1.0, value=0.95, step=0.01, label="Top-p")
+        top_k = gr.Slider(1, 100, value=20, step=1, label="Top-k")
+    gr.ChatInterface(
+        fn=respond,
+        multimodal=True,
+        chatbot=gr.Chatbot(height="65vh", label="ThinkingCap", buttons=["copy"]),
+        textbox=gr.MultimodalTextbox(placeholder="Ask something hard, or attach an image…",
+                                     file_types=["image", "text"], show_label=False,
+                                     max_plain_text_length=100_000),
+        additional_inputs=[effort, max_tokens, temperature, top_p, top_k],
+        examples=EXAMPLES,
+        cache_examples=False,
+        concurrency_limit=8,
+    )
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True, theme=gr.themes.Soft())

chat_utils.py

@@ -0,0 +1,135 @@
+"""Gradio chat history <-> Qwen chat-template messages, and reply rendering.
+
+Pure Python (no torch / gradio imports) so it is unit-testable without the Space runtime.
+Rendered replies are plain dicts, which Gradio's Chatbot accepts as messages.
+"""
+import os
+
+IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff"}
+TEXT_SUFFIXES = {".txt", ".md", ".rst", ".log", ".csv", ".tsv", ".json", ".jsonl", ".yaml",
+                 ".yml", ".toml", ".xml", ".html", ".css", ".js", ".ts", ".py", ".c", ".h",
+                 ".cpp", ".java", ".go", ".rs", ".rb", ".sh", ".sql", ".tex"}
+MAX_FILE_CHARS = 30_000
+
+
+def _file_path(item):
+    """Path of an attachment in any of the shapes Gradio uses, else None."""
+    if isinstance(item, (list, tuple)):
+        return item[0] if item else None
+    if isinstance(item, dict):
+        nested = item.get("file")
+        if isinstance(nested, dict):
+            return nested.get("path") or nested.get("url")
+        return item.get("path") or item.get("url")
+    return None
+
+
+def file_part(path):
+    """Turn an attachment into an image or inlined-text part; raise ValueError otherwise."""
+    name = os.path.basename(path)
+    suffix = os.path.splitext(path)[1].lower()
+    if suffix in IMAGE_SUFFIXES:
+        return {"type": "image", "image": path}
+    if suffix in TEXT_SUFFIXES:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            body = fh.read(MAX_FILE_CHARS + 1)
+        if len(body) > MAX_FILE_CHARS:
+            body = body[:MAX_FILE_CHARS] + "\n[... truncated]"
+        # Gradio turns a long paste into pasted_text.txt; that is the message itself.
+        if name == "pasted_text.txt":
+            return {"type": "text", "text": body}
+        return {"type": "text", "text": f"--- {name} ---\n{body}\n--- end of {name} ---"}
+    raise ValueError(f"Unsupported attachment '{name}'. Attach an image or a text file.")
+
+
+def content_parts(content):
+    """Normalise one turn's content (str, tuple, list of str/dict) into chat-template parts."""
+    # A tuple is Gradio's file-only shape: ("/tmp/a.png",).
+    if isinstance(content, tuple):
+        return [file_part(p) for p in content if p]
+    items = content if isinstance(content, list) else [content]
+    parts = []
+    for item in items:
+        if isinstance(item, str):
+            if item.strip():
+                parts.append({"type": "text", "text": item})
+        elif isinstance(item, dict) and item.get("type") == "text":
+            if (item.get("text") or "").strip():
+                parts.append({"type": "text", "text": item["text"]})
+        elif isinstance(item, dict) and item.get("type") == "image" and "image" in item:
+            parts.append(item)
+        elif (path := _file_path(item)):
+            parts.append(file_part(path))
+    return parts
+
+
+def to_messages(message, history):
+    """Build chat-template messages from a MultimodalTextbox value and Chatbot history.
+
+    Assistant turns carrying a metadata title (reasoning bubbles, notices) are dropped:
+    Qwen's template expects earlier turns without their thinking.
+    """
+    messages = []
+    for turn in history or []:
+        role = turn.get("role") if isinstance(turn, dict) else None
+        if role not in ("user", "assistant"):
+            continue
+        if role == "assistant" and (turn.get("metadata") or {}).get("title"):
+            continue
+        parts = content_parts(turn.get("content"))
+        if not parts:
+            continue
+        # Consecutive same-role entries (Gradio stores a file and its caption separately)
+        # are one turn for the model.
+        if messages and messages[-1]["role"] == role:
+            messages[-1]["content"].extend(parts)
+        else:
+            messages.append({"role": role, "content": parts})
+
+    if isinstance(message, str):
+        message = {"text": message, "files": []}
+    parts = [file_part(p) for p in (_file_path(f) or f for f in message.get("files") or []) if p]
+    if (message.get("text") or "").strip():
+        parts.append({"type": "text", "text": message["text"]})
+    if not parts:
+        raise ValueError("Nothing to send: type a message or attach a file.")
+    if messages and messages[-1]["role"] == "user":
+        messages[-1]["content"].extend(parts)
+    else:
+        messages.append({"role": "user", "content": parts})
+    return messages
+
+
+def split_ids(ids, think_end_id):
+    """Split generated ids at the first </think>: (reasoning_ids, answer_ids or None).
+
+    The prompt already opens <think>, so the model only ever emits the closing tag;
+    answer_ids is None while the model is still reasoning.
+    """
+    if think_end_id in ids:
+        i = ids.index(think_end_id)
+        return ids[:i], ids[i + 1:]
+    return ids, None
+
+
+def render_reply(reasoning, answer, n_reasoning, seconds, done):
+    """Chatbot messages for one reply: a collapsible reasoning bubble, then the answer.
+
+    Pass reasoning=None when thinking is off.
+    """
+    out = []
+    if reasoning is not None:
+        still_thinking = answer is None and not done
+        title = f"Reasoning · {n_reasoning:,} tokens"
+        out.append({"role": "assistant", "content": reasoning.strip() or "…",
+                    "metadata": {"title": title,
+                                 "status": "pending" if still_thinking else "done",
+                                 "duration": round(seconds, 1)}})
+    if answer is not None and answer.strip():
+        out.append({"role": "assistant", "content": answer.strip()})
+    return out
+
+
+def notice(title, text):
+    """A titled assistant message; to_messages drops it from later turns' context."""
+    return {"role": "assistant", "content": text, "metadata": {"title": title}}

requirements.txt

@@ -0,0 +1,9 @@
+# torchvision must match the torch ZeroGPU preinstalls; unpinned, pip pulls a newer
+# torchvision that drags torch outside ZeroGPU's supported set.
+torch==2.11.0
+torchvision==0.26.0
+transformers==5.17.0
+accelerate==1.15.0
+# Triton kernels for the Gated-DeltaNet layers (48 of 64); without them transformers
+# falls back to a reference PyTorch path that is an order of magnitude slower.
+flash-linear-attention==0.5.2