Livrare LOT 1 - Didi

ai_platform/modules/didi_brain/scripts/01_sanity_full.py

@@ -0,0 +1,433 @@
+"""Critical-gate sanity check for the entire upstream stack.
+
+Runs every check that must pass before we trust the brain to do real work:
+
+  1. LLM router reachable, models loaded
+  2. LLM JSON extraction (claim extraction primitive) on a disinfo topic
+  3. LLM NLI single-label classification (SUPPORT/CONTRADICT/NEUTRAL)
+  4. LLM Romanian native generation
+  5. LLM cross-lingual semantic understanding (RO claim ↔ EN claim)
+  6. Embeddings reachable, model name correct, vectors 1024-dim
+  7. Embedding cross-lingual cosine RO↔EN > 0.80 on disinfo test pairs
+  8. Embedding discriminates unrelated topics (cosine < 0.60)
+  9. Reranker reachable, top-1 is the most relevant document with > 100x gap
+
+Idempotent. Safe to rerun any time. Exit code 0 = all passed, 1 = any failed.
+A JSON report is written to reports/sanity_<timestamp>.json for diffing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+# Allow running as `python scripts/01_sanity_full.py` from project root
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.config import LlmRole, settings  # noqa: E402
+from shared.embedding_client import EmbeddingClient, cosine  # noqa: E402
+from shared.llm_client import LlmClient  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+
+@dataclass(slots=True)
+class CheckResult:
+    name: str
+    passed: bool
+    duration_ms: int
+    detail: str = ""
+    metrics: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class Report:
+    started_at: str
+    finished_at: str = ""
+    all_passed: bool = False
+    checks: list[CheckResult] = field(default_factory=list)
+
+
+# ============================================================ individual checks
+
+
+async def check_llm_models(llm: LlmClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        models = await llm.list_models()
+        names = [m.get("id", "?") for m in models]
+        wanted = settings.model_reasoning
+        passed = wanted in names
+        return CheckResult(
+            name="LLM router /v1/models",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"loaded={names}",
+            metrics={"models": names, "wanted": wanted},
+            error=None if passed else f"required model {wanted!r} not in router",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="LLM router /v1/models",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_llm_json_extraction(llm: LlmClient) -> CheckResult:
+    t0 = time.perf_counter()
+    system = (
+        "You extract verifiable factual claims from text. "
+        "Respond ONLY with valid JSON, no commentary."
+    )
+    user = (
+        "Extract claims from: \"WHO data shows that the Pfizer vaccine caused "
+        "1,200 myocarditis cases in 2023.\" "
+        'Return JSON: {"claims": [{"claim": "...", "quote": "..."}]}'
+    )
+    try:
+        result, usage = await llm.chat_json(
+            role=LlmRole.REASONING, system=system, user=user, max_tokens=300
+        )
+        claims = result.get("claims", []) if isinstance(result, dict) else []
+        passed = len(claims) >= 1 and all("claim" in c and "quote" in c for c in claims)
+        return CheckResult(
+            name="LLM JSON claim extraction",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"extracted {len(claims)} claim(s) — backend={usage.get('backend')}",
+            metrics={"claim_count": len(claims), "usage": usage, "first": claims[0] if claims else None},
+            error=None if passed else "no valid claim objects returned",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="LLM JSON claim extraction",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_llm_nli(llm: LlmClient) -> CheckResult:
+    t0 = time.perf_counter()
+    system = "Reply with exactly one word: SUPPORT, CONTRADICT, or NEUTRAL."
+    user = (
+        "CLAIM: Childhood vaccines cause autism.\n"
+        "EVIDENCE: A 2019 Danish cohort study of 657,461 children found no "
+        "association between MMR vaccination and autism.\n\nStance:"
+    )
+    try:
+        label, usage = await llm.chat_label(
+            role=LlmRole.REASONING,
+            system=system,
+            user=user,
+            allowed=["SUPPORT", "CONTRADICT", "NEUTRAL"],
+            max_tokens=20,
+        )
+        passed = label == "CONTRADICT"
+        return CheckResult(
+            name="LLM NLI stance classification",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"label={label} — backend={usage.get('backend')}",
+            metrics={"label": label, "usage": usage},
+            error=None if passed else f"expected CONTRADICT, got {label}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="LLM NLI stance classification",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_llm_romanian(llm: LlmClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        text, usage = await llm.chat_text(
+            role=LlmRole.REASONING,
+            system="Respond only in Romanian, in one short sentence.",
+            user="Care este capitala Romaniei si ce populatie are aproximativ?",
+            max_tokens=100,
+        )
+        # Heuristic: response must contain Romanian-specific characters or words
+        ro_markers = ["București", "Bucuresti", "milion", "România", "Romania", "este"]
+        hits = [m for m in ro_markers if m.lower() in text.lower()]
+        passed = len(hits) >= 2
+        return CheckResult(
+            name="LLM Romanian native generation",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=text.strip()[:200],
+            metrics={"hits": hits, "usage": usage},
+            error=None if passed else f"text does not look Romanian: {text[:100]}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="LLM Romanian native generation",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_llm_crosslingual(llm: LlmClient) -> CheckResult:
+    """Sanity that the LLM understands RO↔EN claim equivalence (separate from BGE)."""
+    t0 = time.perf_counter()
+    system = (
+        "You analyze whether two statements express the same factual claim, "
+        "possibly in different languages. Respond ONLY with JSON."
+    )
+    user = (
+        'A: "Vaccinurile pediatrice cauzeaza autism la copii"\n'
+        'B: "Childhood vaccines are linked to autism"\n\n'
+        'Return: {"same_claim": true/false, "confidence": 0-100}'
+    )
+    try:
+        result, usage = await llm.chat_json(
+            role=LlmRole.REASONING, system=system, user=user, max_tokens=200
+        )
+        same = result.get("same_claim") if isinstance(result, dict) else None
+        conf = result.get("confidence", 0) if isinstance(result, dict) else 0
+        passed = same is True and conf >= 70
+        return CheckResult(
+            name="LLM cross-lingual claim equivalence",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"same={same} confidence={conf} — backend={usage.get('backend')}",
+            metrics={"result": result, "usage": usage},
+            error=None if passed else f"got same={same} conf={conf}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="LLM cross-lingual claim equivalence",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_embedding_basic(embed: EmbeddingClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        vec = await embed.embed_one("hello world")
+        passed = len(vec) == settings.embedding_dim
+        return CheckResult(
+            name="Embedding basic call",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"dim={len(vec)} (wanted {settings.embedding_dim})",
+            metrics={"dim": len(vec)},
+            error=None if passed else f"dim mismatch: {len(vec)}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="Embedding basic call",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+# Test pairs for cross-lingual gate. Must hit cosine >= CROSSLINGUAL_THRESHOLD.
+CROSSLINGUAL_PAIRS: list[tuple[str, str, str]] = [
+    ("vax", "vaccinurile pediatrice cauzeaza autism", "childhood vaccines cause autism"),
+    ("covid", "covidul a fost o pandemie globala", "COVID was a global pandemic"),
+    ("elect", "alegerile prezidentiale din Romania 2024", "Romanian presidential elections 2024"),
+    ("war", "razboiul din Ucraina a inceput in 2022", "the war in Ukraine started in 2022"),
+]
+UNRELATED_PAIRS: list[tuple[str, str, str]] = [
+    ("vax-weather", "vaccinurile pediatrice cauzeaza autism", "the weather is nice today"),
+    ("vax-elect", "childhood vaccines cause autism", "Romanian presidential elections 2024"),
+]
+CROSSLINGUAL_THRESHOLD = 0.80
+UNRELATED_MAX = 0.60
+
+
+async def check_embedding_crosslingual(embed: EmbeddingClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        all_texts: list[str] = []
+        for _, ro, en in CROSSLINGUAL_PAIRS:
+            all_texts.extend([ro, en])
+        for _, a, b in UNRELATED_PAIRS:
+            all_texts.extend([a, b])
+        vecs = await embed.embed(all_texts)
+
+        idx = 0
+        cross_scores: dict[str, float] = {}
+        for label, _, _ in CROSSLINGUAL_PAIRS:
+            cross_scores[label] = cosine(vecs[idx], vecs[idx + 1])
+            idx += 2
+        unrelated_scores: dict[str, float] = {}
+        for label, _, _ in UNRELATED_PAIRS:
+            unrelated_scores[label] = cosine(vecs[idx], vecs[idx + 1])
+            idx += 2
+
+        cross_ok = all(s >= CROSSLINGUAL_THRESHOLD for s in cross_scores.values())
+        unrelated_ok = all(s <= UNRELATED_MAX for s in unrelated_scores.values())
+        passed = cross_ok and unrelated_ok
+
+        detail_lines = [f"{k}={v:.3f}" for k, v in cross_scores.items()]
+        detail_lines += [f"!{k}={v:.3f}" for k, v in unrelated_scores.items()]
+
+        return CheckResult(
+            name="Embedding cross-lingual cosine",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=" ".join(detail_lines),
+            metrics={
+                "crosslingual": cross_scores,
+                "unrelated": unrelated_scores,
+                "threshold_cross": CROSSLINGUAL_THRESHOLD,
+                "threshold_unrelated": UNRELATED_MAX,
+            },
+            error=None
+            if passed
+            else f"cross_ok={cross_ok} unrelated_ok={unrelated_ok}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="Embedding cross-lingual cosine",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_reranker(embed: EmbeddingClient) -> CheckResult:
+    t0 = time.perf_counter()
+    query = "Does the Pfizer vaccine cause myocarditis?"
+    docs = [
+        "A 2022 study found rare myocarditis cases in young men after mRNA vaccination, mostly mild.",
+        "The Kremlin announced new sanctions on European imports yesterday.",
+        "Pfizer reported strong Q3 2023 earnings driven by COVID antiviral sales.",
+        "Danish cohort study of 657,461 children found no link between MMR vaccine and autism.",
+        "A case report described acute pericarditis 4 days after second Pfizer-BioNTech dose in a 17-year-old male.",
+    ]
+    relevant_indices = {0, 4}  # the two docs that actually answer the query
+    try:
+        results = await embed.rerank(query, docs)
+        top2 = {r.index for r in results[:2]}
+        # Top-1 must be relevant; top-2 should both be relevant; gap to #3 must be large
+        top1_relevant = results[0].index in relevant_indices
+        top2_relevant = top2 == relevant_indices
+        # ratio of top-1 score to score of best irrelevant
+        irrelevant = [r for r in results if r.index not in relevant_indices]
+        gap = (results[0].score / irrelevant[0].score) if irrelevant and irrelevant[0].score > 0 else float("inf")
+        passed = top1_relevant and top2_relevant and gap >= 100
+        return CheckResult(
+            name="Reranker top-K precision",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"top1={results[0].index}({results[0].score:.4f}) top2_set={top2} gap≈{gap:.0f}x",
+            metrics={
+                "ranked": [{"index": r.index, "score": r.score} for r in results],
+                "gap_top1_vs_best_irrelevant": gap,
+            },
+            error=None
+            if passed
+            else f"top1_relevant={top1_relevant} top2_relevant={top2_relevant} gap={gap:.1f}",
+        )
+    except Exception as e:
+        return CheckResult(
+            name="Reranker top-K precision",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+# =============================================================== orchestration
+
+
+async def run_all() -> Report:
+    report = Report(started_at=datetime.now(timezone.utc).isoformat())
+
+    async with LlmClient() as llm, EmbeddingClient() as embed:
+        # Run independent checks concurrently. Each check is self-contained.
+        coros = [
+            check_llm_models(llm),
+            check_llm_json_extraction(llm),
+            check_llm_nli(llm),
+            check_llm_romanian(llm),
+            check_llm_crosslingual(llm),
+            check_embedding_basic(embed),
+            check_embedding_crosslingual(embed),
+            check_reranker(embed),
+        ]
+        results = await asyncio.gather(*coros, return_exceptions=False)
+        report.checks.extend(results)
+
+    report.finished_at = datetime.now(timezone.utc).isoformat()
+    report.all_passed = all(c.passed for c in report.checks)
+    return report
+
+
+def render(report: Report) -> None:
+    table = Table(title="DidiBrain — Sanity Check", show_lines=False)
+    table.add_column("Check", style="bold")
+    table.add_column("Status", justify="center")
+    table.add_column("Time", justify="right")
+    table.add_column("Detail", overflow="fold")
+
+    for c in report.checks:
+        status = "[green]PASS[/green]" if c.passed else "[red]FAIL[/red]"
+        detail = c.detail or (c.error or "")
+        table.add_row(c.name, status, f"{c.duration_ms} ms", detail)
+
+    console.print(table)
+    if report.all_passed:
+        console.print("\n[bold green]ALL CHECKS PASSED[/bold green] — go ahead.\n")
+    else:
+        console.print("\n[bold red]GATE FAILED[/bold red] — fix before continuing.\n")
+        for c in report.checks:
+            if not c.passed and c.error:
+                console.print(f"  • [red]{c.name}[/red]: {c.error}")
+
+
+def save_report(report: Report) -> Path:
+    reports_dir = Path(__file__).resolve().parent.parent / "reports"
+    reports_dir.mkdir(exist_ok=True)
+    ts = report.started_at.replace(":", "").replace("-", "")[:15]
+    path = reports_dir / f"sanity_{ts}.json"
+    path.write_text(
+        json.dumps(asdict(report), indent=2, ensure_ascii=False, default=str),
+        encoding="utf-8",
+    )
+    return path
+
+
+def main() -> int:
+    setup_logging()
+    console.print(f"[dim]Router: {settings.llm_router_url}[/dim]")
+    console.print(f"[dim]Embed:  {settings.embedding_url} ({settings.embedding_model})[/dim]")
+    console.print(f"[dim]Rerank: {settings.reranker_url} ({settings.reranker_model})[/dim]\n")
+
+    try:
+        report = asyncio.run(run_all())
+    except KeyboardInterrupt:
+        console.print("\n[yellow]interrupted[/yellow]")
+        return 130
+
+    render(report)
+    path = save_report(report)
+    console.print(f"[dim]report → {path}[/dim]")
+    return 0 if report.all_passed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/02_bootstrap_atomic.py

@@ -0,0 +1,209 @@
+"""Bootstrap a fresh atomic-server instance for DidiBrain.
+
+Run this ONCE after `docker compose up -d` against a fresh stack. It will:
+
+  1. Wait for atomic-server to be reachable on /health
+  2. Check setup status; if no token exists yet, claim the instance to create one
+     - If a token already exists locally in .env, skip claiming and just verify
+  3. Configure provider settings to use BGE-M3 (via openai_compat) for embeddings
+  4. Disable auto_tagging (we control tagging from extractor service)
+  5. Sanity-poke /api/settings to confirm everything stuck
+  6. Patch the local .env file with ATOMIC_TOKEN if it changed
+
+Idempotent. Safe to rerun. Will not overwrite an existing valid token.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+
+from shared.atomic_api import AtomicApiError, AtomicClient  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+ENV_FILE = Path(__file__).resolve().parent.parent / ".env"
+
+# Provider settings to push into Atomic. These tell atomic-core to use the
+# vLLM-served BGE-M3 endpoint via the OpenAI-compatible interface.
+PROVIDER_SETTINGS: dict[str, str] = {
+    "provider": "openai_compat",
+    "openai_compat_base_url": settings.embedding_url,
+    "openai_compat_embedding_model": settings.embedding_model,
+    "openai_compat_embedding_dimension": str(settings.embedding_dim),
+    "openai_compat_context_length": str(settings.embedding_max_tokens),
+    "openai_compat_llm_model": settings.model_reasoning,  # placeholder, not called
+    "openai_compat_timeout_secs": "300",
+    "openai_compat_api_key": settings.embedding_api_key,
+    # We do tagging ourselves with explicit taxonomy in extractor — disable
+    # the built-in LLM auto-tagging to avoid Atomic trying to call the BGE
+    # endpoint as if it were an LLM (which would fail).
+    "auto_tagging_enabled": "false",
+}
+
+
+async def wait_for_atomic(timeout_s: int = 60) -> None:
+    """Poll /health until 200 OK or timeout."""
+    deadline = time.monotonic() + timeout_s
+    last_err: str = ""
+    async with AtomicClient(token="") as client:  # no token needed for health
+        while time.monotonic() < deadline:
+            try:
+                await client.health()
+                console.print("[green]✓[/green] atomic-server /health responding")
+                return
+            except Exception as e:  # noqa: BLE001
+                last_err = f"{type(e).__name__}: {e}"
+                await asyncio.sleep(2)
+    raise RuntimeError(f"atomic-server not healthy after {timeout_s}s — {last_err}")
+
+
+async def ensure_token() -> str:
+    """Return a valid API token, creating one via /api/setup/claim if needed.
+
+    Tries (in order):
+    1. The token already in .env (verify it works by calling /api/settings)
+    2. /api/setup/status — if needs_setup, /api/setup/claim to mint one
+    3. Fall back to advising the user to run `docker exec ... token create`
+    """
+    # 1. Existing token works?
+    if settings.atomic_token:
+        async with AtomicClient(token=settings.atomic_token) as c:
+            try:
+                await c.get_settings()
+                console.print("[green]✓[/green] existing ATOMIC_TOKEN is valid")
+                return settings.atomic_token
+            except AtomicApiError as e:
+                if e.status == 401:
+                    console.print(
+                        "[yellow]![/yellow] existing ATOMIC_TOKEN rejected (401), reclaiming"
+                    )
+                else:
+                    raise
+
+    # 2. Need to claim
+    async with AtomicClient(token="") as c:
+        status = await c.setup_status()
+        needs_setup = status.get("needs_setup", True)
+        if needs_setup:
+            console.print(
+                "[cyan]→[/cyan] instance not yet claimed, calling /api/setup/claim"
+            )
+            result = await c._request(  # type: ignore[attr-defined]
+                "POST",
+                "/api/setup/claim",
+                json={"name": "didibrain-bootstrap"},
+            )
+            token = result.get("token") or result.get("api_token")
+            if not token:
+                raise RuntimeError(f"claim succeeded but no token in response: {result}")
+            console.print("[green]✓[/green] new token claimed")
+            return token
+
+    # 3. Already claimed but we have no token — user must mint one manually
+    raise RuntimeError(
+        "Instance is already claimed but ATOMIC_TOKEN is empty in .env.\n"
+        "Run: docker exec didibrain-atomic atomic-server "
+        "--data-dir /data token create --name didibrain\n"
+        "Then paste the token into .env as ATOMIC_TOKEN=..."
+    )
+
+
+def patch_env_token(new_token: str) -> None:
+    """Update ATOMIC_TOKEN= line in .env in place. Creates the line if missing."""
+    if not ENV_FILE.exists():
+        console.print(f"[red]![/red] .env not found at {ENV_FILE}, skipping write")
+        return
+    text = ENV_FILE.read_text(encoding="utf-8")
+    pattern = re.compile(r"^ATOMIC_TOKEN=.*$", re.MULTILINE)
+    if pattern.search(text):
+        new_text = pattern.sub(f"ATOMIC_TOKEN={new_token}", text)
+    else:
+        new_text = text.rstrip() + f"\nATOMIC_TOKEN={new_token}\n"
+    if new_text != text:
+        ENV_FILE.write_text(new_text, encoding="utf-8")
+        console.print(f"[green]✓[/green] wrote ATOMIC_TOKEN to {ENV_FILE.name}")
+    else:
+        console.print("[dim]·[/dim] ATOMIC_TOKEN already current in .env")
+
+
+async def configure_provider(token: str) -> None:
+    async with AtomicClient(token=token) as c:
+        before = await c.get_settings()
+        before_provider = before.get("provider", "?")
+        console.print(f"[dim]current provider:[/dim] {before_provider}")
+
+        # Push our settings
+        applied: list[str] = []
+        for key, value in PROVIDER_SETTINGS.items():
+            current = before.get(key)
+            if current == value:
+                continue
+            await c.set_setting(key, value)
+            applied.append(key)
+            console.print(f"  [cyan]·[/cyan] {key} = {value}")
+
+        if not applied:
+            console.print("[dim]· all provider settings already current[/dim]")
+        else:
+            console.print(f"[green]✓[/green] applied {len(applied)} setting(s)")
+
+        # Verify
+        after = await c.get_settings()
+        if after.get("provider") != "openai_compat":
+            raise RuntimeError(
+                f"provider did not stick: got {after.get('provider')!r}"
+            )
+        if after.get("openai_compat_base_url") != settings.embedding_url:
+            raise RuntimeError(
+                f"base_url did not stick: got {after.get('openai_compat_base_url')!r}"
+            )
+        console.print("[green]✓[/green] settings verified post-write")
+
+
+async def main_async() -> int:
+    setup_logging()
+    console.print(f"[bold]Bootstrap atomic-server[/bold] @ {settings.atomic_url}")
+    console.print(f"[dim]target embedding endpoint:[/dim] {settings.embedding_url}\n")
+
+    try:
+        await wait_for_atomic(timeout_s=120)
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]✗ atomic not reachable:[/red] {e}")
+        return 2
+
+    try:
+        token = await ensure_token()
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]✗ token bootstrap failed:[/red] {e}")
+        return 3
+
+    patch_env_token(token)
+
+    try:
+        await configure_provider(token)
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]✗ provider config failed:[/red] {e}")
+        return 4
+
+    console.print("\n[bold green]bootstrap complete[/bold green] — ready for sanity check")
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async())
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/03_sanity_atomic.py

@@ -0,0 +1,386 @@
+"""End-to-end brain sanity check.
+
+Validates that atomic-server + Postgres + BGE-M3 work together by:
+
+  1. Hitting /health
+  2. Reading /api/settings and confirming provider is openai_compat → BGE-M3
+  3. Creating a known test atom with a unique source_url marker
+  4. Polling /api/atoms/{id}/embedding-status until 'completed' (or fail at timeout)
+  5. Issuing a semantic search for a paraphrase that should match the test atom
+  6. Verifying the test atom appears in the top results with similarity above threshold
+  7. Cleaning up the test atom (delete) so reruns are clean
+
+Idempotent. Cleans up on success AND on most failure paths.
+Exit 0 = brain works. Exit non-zero = something's broken.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import time
+import uuid
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.atomic_api import AtomicApiError, AtomicClient  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+# Test atom content. The text is bilingual on purpose: it lets us prove that
+# BGE-M3 cross-lingual works end-to-end through the brain (not just at the
+# embedding endpoint level we already validated in 01_sanity_full).
+TEST_MARKER = f"didibrain-sanity-{uuid.uuid4().hex[:8]}"
+TEST_SOURCE_URL = f"https://didibrain.test/sanity/{TEST_MARKER}"
+
+TEST_ATOM_CONTENT = """# Sanity test atom
+
+This is a synthetic atom created by the DidiBrain sanity script.
+
+The 2019 Danish cohort study of 657,461 children found no association
+between MMR vaccination and autism. The Pfizer-BioNTech mRNA vaccine has
+been linked in rare cases to mild myocarditis in young men, but the
+benefits clearly outweigh the risks for most populations.
+
+In Romanian: Studiul danez din 2019 nu a gasit nicio legatura intre
+vaccinul MMR si autism la copii.
+"""
+
+# Cross-lingual paraphrase used to query the brain. If BGE-M3 is wired up
+# correctly, this RO query should retrieve the test atom (which mixes EN+RO).
+TEST_QUERY = "studii care arata ca vaccinurile nu cauzeaza autism"
+SIMILARITY_MIN = 0.4  # generous; chunked match should easily exceed this
+
+
+@dataclass(slots=True)
+class CheckResult:
+    name: str
+    passed: bool
+    duration_ms: int
+    detail: str = ""
+    metrics: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class Report:
+    started_at: str
+    finished_at: str = ""
+    all_passed: bool = False
+    checks: list[CheckResult] = field(default_factory=list)
+    test_atom_id: str | None = None
+
+
+# ============================================================ orchestration
+
+
+async def check_health(client: AtomicClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        h = await client.health()
+        return CheckResult(
+            name="atomic-server /health",
+            passed=True,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=str(h)[:120],
+        )
+    except Exception as e:  # noqa: BLE001
+        return CheckResult(
+            name="atomic-server /health",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_provider_settings(client: AtomicClient) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        s = await client.get_settings()
+        provider = s.get("provider")
+        base_url = s.get("openai_compat_base_url")
+        emb_model = s.get("openai_compat_embedding_model")
+        emb_dim = s.get("openai_compat_embedding_dimension")
+
+        ok = (
+            provider == "openai_compat"
+            and base_url == settings.embedding_url
+            and emb_model == settings.embedding_model
+            and str(emb_dim) == str(settings.embedding_dim)
+        )
+        return CheckResult(
+            name="provider settings (BGE-M3 wired)",
+            passed=ok,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"provider={provider} base_url={base_url} model={emb_model} dim={emb_dim}",
+            metrics={
+                "provider": provider,
+                "base_url": base_url,
+                "embedding_model": emb_model,
+                "embedding_dimension": emb_dim,
+            },
+            error=None if ok else "settings do not match expected BGE-M3 config",
+        )
+    except Exception as e:  # noqa: BLE001
+        return CheckResult(
+            name="provider settings (BGE-M3 wired)",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_create_atom(
+    client: AtomicClient, report: Report
+) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        # First, defensive cleanup: if a previous run left an orphan, drop it.
+        existing = await client.get_atom_by_source_url(TEST_SOURCE_URL)
+        if existing:
+            try:
+                await client.delete_atom(existing["id"])
+            except Exception:  # noqa: BLE001, S110
+                pass
+
+        atom = await client.create_atom(
+            content=TEST_ATOM_CONTENT,
+            source_url=TEST_SOURCE_URL,
+        )
+        atom_id = atom.get("id") or atom.get("atom_id")
+        if not atom_id:
+            return CheckResult(
+                name="create test atom",
+                passed=False,
+                duration_ms=int((time.perf_counter() - t0) * 1000),
+                error=f"no id in response: {str(atom)[:300]}",
+            )
+        report.test_atom_id = atom_id
+        return CheckResult(
+            name="create test atom",
+            passed=True,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"id={atom_id}",
+            metrics={"atom_id": atom_id},
+        )
+    except Exception as e:  # noqa: BLE001
+        return CheckResult(
+            name="create test atom",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_embedding_pipeline(
+    client: AtomicClient, atom_id: str, *, timeout_s: int = 60
+) -> CheckResult:
+    t0 = time.perf_counter()
+    deadline = time.monotonic() + timeout_s
+    last_status: str = "?"
+    try:
+        while time.monotonic() < deadline:
+            status = await client.get_embedding_status(atom_id)
+            last_status = (
+                status.get("embedding_status")
+                or status.get("status")
+                or "unknown"
+            )
+            # Atomic uses 'complete' (not 'completed'); accept both defensively.
+            if last_status in ("complete", "completed"):
+                return CheckResult(
+                    name="embedding pipeline (BGE-M3 → atom)",
+                    passed=True,
+                    duration_ms=int((time.perf_counter() - t0) * 1000),
+                    detail=f"{last_status} in {int(time.perf_counter() - t0)}s",
+                    metrics={"final_status": last_status},
+                )
+            if last_status == "failed":
+                return CheckResult(
+                    name="embedding pipeline (BGE-M3 → atom)",
+                    passed=False,
+                    duration_ms=int((time.perf_counter() - t0) * 1000),
+                    error=f"pipeline reported failed: {status}",
+                )
+            await asyncio.sleep(1.5)
+        return CheckResult(
+            name="embedding pipeline (BGE-M3 → atom)",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"timed out after {timeout_s}s — last status={last_status}",
+        )
+    except Exception as e:  # noqa: BLE001
+        return CheckResult(
+            name="embedding pipeline (BGE-M3 → atom)",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def check_semantic_search(
+    client: AtomicClient, atom_id: str
+) -> CheckResult:
+    t0 = time.perf_counter()
+    try:
+        hits = await client.search(TEST_QUERY, mode="semantic", limit=20)
+        if not hits:
+            return CheckResult(
+                name="semantic search retrieves test atom",
+                passed=False,
+                duration_ms=int((time.perf_counter() - t0) * 1000),
+                error="search returned 0 hits",
+            )
+        # Find our test atom in the hits
+        match = next((h for h in hits if h.atom_id == atom_id), None)
+        if not match:
+            top_ids = [h.atom_id for h in hits[:5]]
+            return CheckResult(
+                name="semantic search retrieves test atom",
+                passed=False,
+                duration_ms=int((time.perf_counter() - t0) * 1000),
+                error=f"test atom {atom_id} not in top {len(hits)} hits, top5={top_ids}",
+            )
+        passed = match.similarity >= SIMILARITY_MIN
+        return CheckResult(
+            name="semantic search retrieves test atom",
+            passed=passed,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            detail=f"sim={match.similarity:.3f} (≥{SIMILARITY_MIN}) of {len(hits)} hits",
+            metrics={
+                "test_atom_similarity": match.similarity,
+                "total_hits": len(hits),
+            },
+            error=None
+            if passed
+            else f"similarity {match.similarity:.3f} below {SIMILARITY_MIN}",
+        )
+    except Exception as e:  # noqa: BLE001
+        return CheckResult(
+            name="semantic search retrieves test atom",
+            passed=False,
+            duration_ms=int((time.perf_counter() - t0) * 1000),
+            error=f"{type(e).__name__}: {e}",
+        )
+
+
+async def cleanup_test_atom(client: AtomicClient, atom_id: str | None) -> None:
+    if not atom_id:
+        return
+    try:
+        await client.delete_atom(atom_id)
+        console.print(f"[dim]· cleaned up test atom {atom_id}[/dim]")
+    except AtomicApiError as e:
+        console.print(f"[yellow]! cleanup failed:[/yellow] {e}")
+
+
+async def run_all() -> Report:
+    report = Report(started_at=datetime.now(timezone.utc).isoformat())
+
+    if not settings.atomic_token:
+        console.print(
+            "[red]ATOMIC_TOKEN is empty in .env — run 02_bootstrap_atomic.py first[/red]"
+        )
+        report.checks.append(
+            CheckResult(name="precondition", passed=False, duration_ms=0,
+                        error="ATOMIC_TOKEN missing")
+        )
+        report.finished_at = datetime.now(timezone.utc).isoformat()
+        return report
+
+    async with AtomicClient() as client:
+        report.checks.append(await check_health(client))
+        if not report.checks[-1].passed:
+            report.finished_at = datetime.now(timezone.utc).isoformat()
+            return report
+
+        report.checks.append(await check_provider_settings(client))
+        if not report.checks[-1].passed:
+            report.finished_at = datetime.now(timezone.utc).isoformat()
+            return report
+
+        report.checks.append(await check_create_atom(client, report))
+        if not report.checks[-1].passed or not report.test_atom_id:
+            report.finished_at = datetime.now(timezone.utc).isoformat()
+            return report
+
+        try:
+            report.checks.append(
+                await check_embedding_pipeline(client, report.test_atom_id)
+            )
+            if not report.checks[-1].passed:
+                return report
+            report.checks.append(
+                await check_semantic_search(client, report.test_atom_id)
+            )
+        finally:
+            await cleanup_test_atom(client, report.test_atom_id)
+            report.finished_at = datetime.now(timezone.utc).isoformat()
+
+    report.all_passed = all(c.passed for c in report.checks)
+    return report
+
+
+# ====================================================================== render
+
+
+def render(report: Report) -> None:
+    table = Table(title="DidiBrain — Atomic Sanity", show_lines=False)
+    table.add_column("Check", style="bold")
+    table.add_column("Status", justify="center")
+    table.add_column("Time", justify="right")
+    table.add_column("Detail", overflow="fold")
+
+    for c in report.checks:
+        status = "[green]PASS[/green]" if c.passed else "[red]FAIL[/red]"
+        detail = c.detail or (c.error or "")
+        table.add_row(c.name, status, f"{c.duration_ms} ms", detail)
+
+    console.print(table)
+    if report.all_passed:
+        console.print("\n[bold green]BRAIN OK[/bold green] — atomic + Postgres + BGE-M3 fully wired.\n")
+    else:
+        console.print("\n[bold red]BRAIN FAILED[/bold red]\n")
+        for c in report.checks:
+            if not c.passed and c.error:
+                console.print(f"  • [red]{c.name}[/red]: {c.error}")
+
+
+def save_report(report: Report) -> Path:
+    import json
+    reports_dir = Path(__file__).resolve().parent.parent / "reports"
+    reports_dir.mkdir(exist_ok=True)
+    ts = report.started_at.replace(":", "").replace("-", "")[:15]
+    path = reports_dir / f"sanity_atomic_{ts}.json"
+    path.write_text(
+        json.dumps(asdict(report), indent=2, ensure_ascii=False, default=str),
+        encoding="utf-8",
+    )
+    return path
+
+
+def main() -> int:
+    setup_logging()
+    console.print(f"[dim]Atomic: {settings.atomic_url}[/dim]")
+    console.print(f"[dim]Token:  {'set' if settings.atomic_token else 'NOT SET'}[/dim]\n")
+    try:
+        report = asyncio.run(run_all())
+    except KeyboardInterrupt:
+        return 130
+    render(report)
+    path = save_report(report)
+    console.print(f"[dim]report → {path}[/dim]")
+    return 0 if report.all_passed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/04_seed_taxonomy.py

@@ -0,0 +1,125 @@
+"""Seed the canonical tag taxonomy into Atomic.
+
+Reads `shared.taxonomy.TAXONOMY` and creates any missing tags in Atomic,
+preserving the parent-child structure. Reuses Atomic's default root tags
+(Topics, People, Locations, Organizations, Events) where they already exist.
+
+After seeding, writes the complete `path → tag_id` map to
+`shared/_tag_ids.json` for use by all downstream scripts.
+
+Idempotent: re-running it will only create tags that don't yet exist by
+(name, parent_id). Safe to invoke at any time, e.g. after pulling a newer
+TAXONOMY definition.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.atomic_api import AtomicClient  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+from shared.taxonomy import (  # noqa: E402
+    TAXONOMY,
+    TagResolver,
+    build_path_map_from_tags,
+    walk,
+)
+
+console = Console()
+
+
+async def seed() -> dict[str, int]:
+    """Walk TAXONOMY and create missing tags. Return summary counts."""
+    if not settings.atomic_token:
+        raise RuntimeError("ATOMIC_TOKEN missing — run 02_bootstrap_atomic.py first")
+
+    counts = {"created": 0, "existing": 0, "errors": 0}
+
+    async with AtomicClient() as client:
+        # 1. Pull current state
+        existing_tags = await client.list_tags(min_count=0)
+        path_map = build_path_map_from_tags(existing_tags)
+        console.print(
+            f"[dim]Atomic currently has [bold]{len(path_map)}[/bold] tags "
+            f"({len([p for p in path_map if '/' not in p])} roots)[/dim]"
+        )
+
+        # 2. Walk taxonomy depth-first; create what's missing
+        for path, parent_path, name in walk(TAXONOMY):
+            if path in path_map:
+                counts["existing"] += 1
+                continue
+            parent_id = path_map.get(parent_path) if parent_path else None
+            if parent_path and not parent_id:
+                console.print(
+                    f"[red]✗[/red] cannot create {path!r}: parent {parent_path!r} "
+                    f"missing — should have been created earlier"
+                )
+                counts["errors"] += 1
+                continue
+            try:
+                created = await client.create_tag(name=name, parent_id=parent_id)
+                new_id = created.get("id")
+                if not new_id:
+                    counts["errors"] += 1
+                    console.print(f"[red]✗[/red] {path}: no id in response: {created}")
+                    continue
+                path_map[path] = new_id
+                counts["created"] += 1
+                console.print(f"[green]+[/green] {path}")
+            except Exception as e:  # noqa: BLE001
+                counts["errors"] += 1
+                console.print(f"[red]✗[/red] {path}: {type(e).__name__}: {e}")
+
+        # 3. Re-read full state to make sure path_map is fresh and complete
+        final_tags = await client.list_tags(min_count=0)
+        final_map = build_path_map_from_tags(final_tags)
+
+    # 4. Persist the map
+    resolver = TagResolver()
+    resolver.save(final_map)
+    console.print(
+        f"\n[dim]wrote [bold]{len(final_map)}[/bold] entries to "
+        f"shared/_tag_ids.json[/dim]"
+    )
+    return counts
+
+
+def render_summary(counts: dict[str, int]) -> None:
+    table = Table(title="Taxonomy seeding")
+    table.add_column("Status", style="bold")
+    table.add_column("Count", justify="right")
+    style = {"created": "green", "existing": "dim", "errors": "red"}
+    for k, v in counts.items():
+        table.add_row(f"[{style[k]}]{k}[/{style[k]}]", str(v))
+    console.print(table)
+
+
+def main() -> int:
+    setup_logging()
+    console.print(f"[dim]Atomic: {settings.atomic_url}[/dim]\n")
+    try:
+        counts = asyncio.run(seed())
+    except KeyboardInterrupt:
+        return 130
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]seeding failed:[/red] {e}")
+        return 1
+    render_summary(counts)
+    if counts["errors"]:
+        console.print("\n[red]some tags failed[/red]")
+        return 1
+    console.print("\n[bold green]taxonomy seeded[/bold green]")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/05_import_wikipedia_seed.py

@@ -0,0 +1,332 @@
+"""Import a curated seed of Wikipedia articles about vaccines into the brain.
+
+This is the FIRST real ingestion script. Goal: get ~20-30 high-quality
+articles in EN and RO about vaccines / vaccine misinformation into Atomic,
+properly tagged, so we can run real Didi-style queries against the brain.
+
+Why a curated seed and not category-crawl?
+- Quality > quantity for first validation
+- Avoids legal gray areas of mass scraping
+- Wikipedia categories are noisy (stub pages, redirects, lists)
+- 20-30 well-chosen articles cover all the disinfo claims we want to test
+
+Pipeline per article:
+  1. MediaWiki action API: action=query&prop=extracts (plain text, full)
+  2. Build clean markdown: "# Title\n\n{extract}"
+  3. Dedup by source_url against existing atoms
+  4. POST to Atomic with explicit taxonomy tags
+  5. (Atomic embeds it in background; we don't wait)
+
+Idempotent: re-running skips articles already present (by canonical URL).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.atomic_api import AtomicApiError, AtomicClient  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+from shared.taxonomy import TagResolver  # noqa: E402
+
+console = Console()
+
+# Per-language seed lists. Keys are MediaWiki page titles. Order = priority.
+# These are chosen to span the full disinfo landscape on vaccines:
+#   - factual baseline (Vaccine, Vaccination)
+#   - the canonical false claim (MMR vaccine and autism)
+#   - the perpetrator (Andrew Wakefield)
+#   - movements (Anti-vaccinationism, Vaccine hesitancy)
+#   - COVID-era specifics (Pfizer-BioNTech vaccine, COVID-19 vaccine misinformation)
+#   - relevant figures and incidents
+SEED_EN: list[str] = [
+    "Vaccine",
+    "Vaccination",
+    "Vaccine hesitancy",
+    "MMR vaccine and autism",
+    "Andrew Wakefield",
+    "Anti-vaccinationism",
+    "Vaccine controversies",
+    "Vaccine injury",
+    "Pfizer–BioNTech COVID-19 vaccine",
+    "Moderna COVID-19 vaccine",
+    "COVID-19 vaccine misinformation",
+    "Robert F. Kennedy Jr.",
+    "Plandemic",
+    "Children's Health Defense",
+]
+SEED_RO: list[str] = [
+    "Vaccin",
+    "Vaccinare",
+    "Mișcarea antivaccinare",
+    "Pandemia de COVID-19 în România",
+    "Vaccin împotriva COVID-19",
+    "Vaccinare împotriva COVID-19 în România",
+    "Andrew Wakefield",
+    "Vaccin ROR",
+    "Tiomersal",
+    "Variolă",
+]
+
+# Wikimedia rejects vague User-Agents and Mozilla-like impersonations.
+# Compliant format per their policy:
+#   https://meta.wikimedia.org/wiki/User-Agent_policy
+# "AppName/Version (URL or email contact) optional-libraries"
+USER_AGENT = (
+    "DidiBrain/0.1 (https://github.com/didibrain; didibrain@local.test) httpx/0.28"
+)
+RATE_LIMIT_SECONDS = 0.5  # be a good Wikipedia citizen
+
+
+@dataclass(slots=True)
+class WikiArticle:
+    title: str
+    extract: str
+    canonical_url: str
+    page_id: int
+    language: str  # "EN" / "RO"
+
+
+@dataclass(slots=True)
+class ImportStats:
+    fetched: int = 0
+    skipped_missing: int = 0
+    skipped_duplicate: int = 0
+    created: int = 0
+    errors: int = 0
+
+
+# ============================================================ wikipedia client
+
+
+async def fetch_article(
+    http: httpx.AsyncClient, lang: str, title: str
+) -> WikiArticle | None:
+    """Fetch one Wikipedia page via the MediaWiki action API.
+
+    Returns None if the page doesn't exist (missing) or has no extract.
+    """
+    base = f"https://{lang.lower()}.wikipedia.org/w/api.php"
+    params = {
+        "action": "query",
+        "format": "json",
+        "prop": "extracts|info",
+        "explaintext": "1",
+        "exsectionformat": "plain",
+        "exlimit": "1",
+        "inprop": "url",
+        "redirects": "1",
+        "titles": title,
+        "formatversion": "2",
+    }
+    try:
+        resp = await http.get(base, params=params)
+        resp.raise_for_status()
+    except httpx.HTTPError as e:
+        console.print(f"[red]✗[/red] HTTP {lang}/{title}: {e}")
+        return None
+
+    data = resp.json()
+    pages = (data.get("query") or {}).get("pages") or []
+    if not pages:
+        return None
+    page = pages[0]
+    if page.get("missing"):
+        return None
+    extract = (page.get("extract") or "").strip()
+    if not extract or len(extract) < 200:
+        # too short to be useful
+        return None
+    return WikiArticle(
+        title=page.get("title", title),
+        extract=extract,
+        canonical_url=page.get("canonicalurl") or page.get("fullurl") or "",
+        page_id=int(page.get("pageid", 0)),
+        language=lang.upper(),
+    )
+
+
+def article_to_markdown(article: WikiArticle) -> str:
+    """Convert a Wikipedia plain-text extract to Atomic-friendly markdown.
+
+    The action API extract uses plain text section breaks like:
+        Title
+
+        First paragraph.
+
+        Section name
+
+        Section content.
+
+    We can't reliably distinguish section headers from short paragraphs from
+    the plain text alone. So we just preserve the structure with the page
+    title as a single H1, then the body verbatim. Atomic's chunker is smart
+    enough to chunk on paragraph boundaries; we don't need section markers.
+    """
+    return f"# {article.title}\n\n{article.extract}\n"
+
+
+# ============================================================== atomic push
+
+
+def tag_ids_for_language(resolver: TagResolver, language: str) -> list[str]:
+    """Return the canonical tag-id list for a Wikipedia article in a given lang."""
+    return resolver.ids_for(
+        [
+            "Topics/Health/Vaccines",
+            "SourceType/Wikipedia",
+            "Credibility/Tier2",
+            f"Language/{language}",
+            "Type/Document",
+            "Country/Global",
+        ]
+    )
+
+
+async def import_one(
+    client: AtomicClient,
+    http: httpx.AsyncClient,
+    resolver: TagResolver,
+    lang: str,
+    title: str,
+    stats: ImportStats,
+) -> None:
+    article = await fetch_article(http, lang, title)
+    if not article:
+        stats.skipped_missing += 1
+        console.print(f"  [dim]·[/dim] {lang}/{title}: missing/empty")
+        return
+    stats.fetched += 1
+
+    # Dedup
+    existing = await client.get_atom_by_source_url(article.canonical_url)
+    if existing:
+        stats.skipped_duplicate += 1
+        console.print(
+            f"  [dim]·[/dim] {lang}/{article.title}: already in brain"
+        )
+        return
+
+    # Push
+    md = article_to_markdown(article)
+    tag_ids = tag_ids_for_language(resolver, article.language)
+    try:
+        atom = await client.create_atom(
+            content=md,
+            source_url=article.canonical_url,
+            tag_ids=tag_ids,
+        )
+        stats.created += 1
+        size_kb = len(md) / 1024
+        console.print(
+            f"  [green]+[/green] {lang}/{article.title} "
+            f"[dim]({size_kb:.1f} KB → {atom.get('id', '?')[:8]}...)[/dim]"
+        )
+    except AtomicApiError as e:
+        stats.errors += 1
+        console.print(
+            f"  [red]✗[/red] {lang}/{article.title}: {e.status} {e.body[:200]}"
+        )
+
+
+# ================================================================== main flow
+
+
+async def import_seeds(
+    client: AtomicClient, resolver: TagResolver
+) -> ImportStats:
+    stats = ImportStats()
+
+    headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
+    async with httpx.AsyncClient(timeout=30.0, headers=headers) as http:
+        for lang, seeds in (("EN", SEED_EN), ("RO", SEED_RO)):
+            console.print(f"\n[bold]{lang}[/bold] — {len(seeds)} seed articles")
+            for title in seeds:
+                await import_one(client, http, resolver, lang, title, stats)
+                await asyncio.sleep(RATE_LIMIT_SECONDS)
+    return stats
+
+
+async def main_async() -> int:
+    setup_logging()
+
+    if not settings.atomic_token:
+        console.print(
+            "[red]ATOMIC_TOKEN missing — run 02_bootstrap_atomic.py first[/red]"
+        )
+        return 2
+
+    resolver = TagResolver()
+    if not resolver.all:
+        console.print(
+            "[red]tag id cache empty — run 04_seed_taxonomy.py first[/red]"
+        )
+        return 3
+
+    # Sanity: required tags exist
+    required = [
+        "Topics/Health/Vaccines",
+        "SourceType/Wikipedia",
+        "Credibility/Tier2",
+        "Language/EN",
+        "Language/RO",
+        "Type/Document",
+        "Country/Global",
+    ]
+    missing = [p for p in required if p not in resolver]
+    if missing:
+        console.print(f"[red]missing required tags: {missing}[/red]")
+        return 4
+
+    console.print(
+        f"[dim]Atomic: {settings.atomic_url}  "
+        f"|  taxonomy: {len(resolver.all)} tags loaded[/dim]"
+    )
+
+    started = time.perf_counter()
+    async with AtomicClient() as client:
+        stats = await import_seeds(client, resolver)
+    elapsed = time.perf_counter() - started
+
+    table = Table(title=f"Wikipedia seed import (took {elapsed:.1f}s)")
+    table.add_column("Status", style="bold")
+    table.add_column("Count", justify="right")
+    table.add_row("[green]created[/green]", str(stats.created))
+    table.add_row("[dim]duplicate (skipped)[/dim]", str(stats.skipped_duplicate))
+    table.add_row("[dim]missing on Wikipedia[/dim]", str(stats.skipped_missing))
+    table.add_row("[red]errors[/red]", str(stats.errors))
+    table.add_row("fetched total", str(stats.fetched))
+    console.print(table)
+
+    if stats.errors:
+        return 1
+    if stats.created == 0 and stats.skipped_duplicate == 0:
+        console.print("\n[yellow]nothing imported — Wikipedia returned nothing[/yellow]")
+        return 5
+    console.print(
+        f"\n[bold green]ok[/bold green] — brain now has {stats.created} new "
+        f"atom(s) (embedding processes async, check status with sanity)"
+    )
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async())
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/06_validate_queries.py

@@ -0,0 +1,175 @@
+"""Run real Didi-style queries against the brain and show retrieval quality.
+
+This is the FIRST end-to-end validation on real corpus content. It runs
+three claim-style queries (mix of EN and RO) and shows:
+
+  - Atomic semantic search (top 10 hits with similarity)
+  - BGE reranker rescoring (top 5 with cross-encoder scores)
+  - Title + URL of each hit so you can eyeball relevance
+
+This is how Didi's retrieval layer will work in production: first-stage
+embedding retrieval, then cross-encoder rerank for precision.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.atomic_api import AtomicClient, SearchHit  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.embedding_client import EmbeddingClient  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+
+@dataclass(slots=True, frozen=True)
+class QuerySpec:
+    label: str
+    query: str
+    expect_titles: list[str]  # substrings we hope to see at the top
+
+
+QUERIES: list[QuerySpec] = [
+    QuerySpec(
+        label="EN claim: MMR autism",
+        query="Does the MMR vaccine cause autism in children?",
+        expect_titles=["MMR", "autism", "Wakefield"],
+    ),
+    QuerySpec(
+        label="RO query → expects EN+RO hits",
+        query="Cine este Andrew Wakefield si ce a facut cu studiul despre vaccinul MMR?",
+        expect_titles=["Wakefield", "MMR"],
+    ),
+    QuerySpec(
+        label="RO claim: COVID vax danger",
+        query="Vaccinurile COVID-19 sunt periculoase pentru tineri si cauzeaza miocardita?",
+        expect_titles=["Pfizer", "COVID", "BioNTech", "misinformation"],
+    ),
+    QuerySpec(
+        label="EN claim: RFK Jr disinfo",
+        query="Robert F Kennedy Jr anti-vaccine claims",
+        expect_titles=["Kennedy", "Children's Health"],
+    ),
+]
+
+
+async def query_with_rerank(
+    atomic: AtomicClient,
+    embed: EmbeddingClient,
+    spec: QuerySpec,
+) -> None:
+    console.print(f"\n[bold cyan]── {spec.label} ──[/bold cyan]")
+    console.print(f"[dim]query:[/dim] {spec.query}")
+
+    # Stage 1: semantic search via Atomic (which uses BGE-M3 internally)
+    hits = await atomic.search(spec.query, mode="semantic", limit=20, threshold=0.2)
+    if not hits:
+        console.print("[yellow]no hits[/yellow]")
+        return
+
+    # Stage 2: rerank with BGE-reranker-v2-m3 cross-encoder
+    docs = [_doc_for_rerank(h) for h in hits]
+    rerank_results = await embed.rerank(spec.query, docs, top_n=5)
+
+    # Build display table
+    table = Table(show_lines=False, expand=False)
+    table.add_column("#", justify="right", width=3)
+    table.add_column("emb_sim", justify="right", width=8)
+    table.add_column("rerank", justify="right", width=10)
+    table.add_column("source")
+    table.add_column("preview", overflow="fold")
+
+    # Map back: rerank result.index → original hit
+    rank_position: dict[int, int] = {r.index: i for i, r in enumerate(rerank_results)}
+
+    for i, hit in enumerate(hits[:10]):
+        rerank_idx = rank_position.get(i)
+        rerank_str = (
+            f"#{rerank_idx + 1}: {rerank_results[rerank_idx].score:.3f}"
+            if rerank_idx is not None
+            else "[dim]-[/dim]"
+        )
+        title = _extract_title(hit)
+        preview = (hit.matching_chunk_content or hit.snippet or "")[:120]
+        table.add_row(
+            str(i + 1),
+            f"{hit.similarity:.3f}",
+            rerank_str,
+            title,
+            preview.replace("\n", " "),
+        )
+    console.print(table)
+
+    # Did we get what we expected?
+    matched = [
+        e
+        for e in spec.expect_titles
+        if any(e.lower() in _extract_title(h).lower() for h in hits[:5])
+    ]
+    if len(matched) == len(spec.expect_titles):
+        console.print(
+            f"[green]✓ all expected hit substrings present in top 5: {matched}[/green]"
+        )
+    else:
+        missing = [e for e in spec.expect_titles if e not in matched]
+        console.print(
+            f"[yellow]partial: matched {matched}  missing {missing}[/yellow]"
+        )
+
+
+def _doc_for_rerank(hit: SearchHit) -> str:
+    """Build the text payload to send to the reranker for one hit."""
+    title = _extract_title(hit)
+    body = hit.matching_chunk_content or hit.snippet or ""
+    return f"{title}\n\n{body}"[:2000]  # cap to keep reranker fast
+
+
+def _extract_title(hit: SearchHit) -> str:
+    """Best-effort title from source URL or chunk content."""
+    if hit.source_url:
+        # https://en.wikipedia.org/wiki/Andrew_Wakefield → Andrew Wakefield
+        from urllib.parse import unquote
+
+        last = hit.source_url.rstrip("/").rsplit("/", 1)[-1]
+        return unquote(last).replace("_", " ")
+    return (hit.matching_chunk_content or "")[:60]
+
+
+async def main_async() -> int:
+    setup_logging()
+    if not settings.atomic_token:
+        console.print("[red]ATOMIC_TOKEN missing[/red]")
+        return 2
+
+    console.print(
+        f"[dim]Atomic: {settings.atomic_url}  |  Reranker: {settings.reranker_url}[/dim]"
+    )
+
+    async with AtomicClient() as atomic, EmbeddingClient() as embed:
+        for spec in QUERIES:
+            try:
+                await query_with_rerank(atomic, embed, spec)
+            except Exception as e:  # noqa: BLE001
+                console.print(f"[red]× error on {spec.label}: {e}[/red]")
+    console.print("\n[bold]done[/bold]")
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async())
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/07_run_extraction.py

@@ -0,0 +1,94 @@
+"""Run claim extraction over all Type/Document atoms in the brain.
+
+Reads from extractor.batch.run_batch() and renders a summary table.
+Idempotent: documents already processed at the current prompt version are
+skipped automatically (state file at extractor/_extracted.json).
+
+Usage:
+  python scripts/07_run_extraction.py              # process everything new
+  python scripts/07_run_extraction.py --limit 5    # cap docs (smoke test)
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from extractor.batch import run_batch  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Run claim extraction batch")
+    p.add_argument("--limit", type=int, default=None, help="cap number of docs")
+    return p.parse_args()
+
+
+async def main_async(args: argparse.Namespace) -> int:
+    setup_logging()
+    console.print(
+        f"[dim]Atomic: {settings.atomic_url}  "
+        f"|  LLM router: {settings.llm_router_url}  "
+        f"|  Reasoning model: {settings.model_reasoning}[/dim]\n"
+    )
+
+    started = time.perf_counter()
+    try:
+        stats = await run_batch(limit=args.limit)
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]batch failed:[/red] {e}")
+        return 1
+    elapsed = time.perf_counter() - started
+
+    table = Table(title=f"Claim extraction (took {elapsed:.1f}s)", show_lines=False)
+    table.add_column("Metric", style="bold")
+    table.add_column("Count", justify="right")
+    table.add_row("documents seen", str(stats.docs_seen))
+    table.add_row("[dim]skipped (already done)[/dim]", str(stats.docs_skipped_already_done))
+    table.add_row("[green]processed[/green]", str(stats.docs_processed))
+    table.add_row("[red]failed[/red]", str(stats.docs_failed))
+    table.add_row("", "")
+    table.add_row("LLM raw claims", str(stats.claims_raw))
+    table.add_row("[green]valid claims[/green]", str(stats.claims_valid))
+    table.add_row("[green]created in brain[/green]", str(stats.claims_created))
+    table.add_row("[dim]duplicate (skipped)[/dim]", str(stats.claims_duplicate))
+    table.add_row("[red]push errors[/red]", str(stats.claims_error))
+    console.print(table)
+
+    if stats.rejected_reasons:
+        rt = Table(title="Validation rejections", show_lines=False)
+        rt.add_column("Reason")
+        rt.add_column("Count", justify="right")
+        for k, v in sorted(stats.rejected_reasons.items(), key=lambda x: -x[1]):
+            rt.add_row(k, str(v))
+        console.print(rt)
+
+    if stats.docs_failed > 0 and stats.docs_processed == 0:
+        return 2
+    if stats.docs_processed == 0 and stats.docs_skipped_already_done == 0:
+        console.print("[yellow]no documents found to process[/yellow]")
+        return 3
+    console.print("\n[bold green]extraction complete[/bold green]")
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async(parse_args()))
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/08_validate_claims.py

@@ -0,0 +1,248 @@
+"""Demonstrate claim-level retrieval — the payoff of script 07.
+
+For each test claim, this script:
+
+  1. Queries Atomic semantic search (no filter — gets BOTH docs and claims)
+  2. Splits results into Type/Document hits and Type/Claim hits
+  3. For the claim hits: shows the actual claim text, stance, and source language
+  4. Reranks the claim hits with BGE-reranker-v2-m3 for precision
+  5. Aggregates by stance (asserts/reports/refutes) and language
+
+This is the verdict-packet primitive that Didi will call. After D, the brain
+returns ATOMIC FACTUAL CLAIMS, not paragraphs of articles to read.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import sys
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.atomic_api import AtomicClient, SearchHit  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.embedding_client import EmbeddingClient  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+
+@dataclass(slots=True, frozen=True)
+class ClaimQuery:
+    label: str
+    query: str
+
+
+QUERIES: list[ClaimQuery] = [
+    ClaimQuery(
+        label="EN — Wakefield fraud",
+        query="Andrew Wakefield falsified medical records in his 1998 study",
+    ),
+    ClaimQuery(
+        label="RO — vaccinurile cauzeaza autism",
+        query="vaccinurile pediatrice cauzeaza autism la copii",
+    ),
+    ClaimQuery(
+        label="EN — RFK Jr COVID lies",
+        query="Robert F Kennedy Jr promoted COVID-19 vaccine misinformation",
+    ),
+    ClaimQuery(
+        label="RO — Pfizer myocarditis",
+        query="vaccinul Pfizer cauzeaza miocardita la tineri",
+    ),
+]
+
+
+# ============================================================ rendering
+
+
+# Match the canonical claim text from a Type/Claim atom's markdown body
+_CLAIM_BODY_RE = re.compile(
+    r"^# Claim\s*\n+(.+?)\n+##",
+    re.DOTALL | re.MULTILINE,
+)
+# Match the stance line in the markdown body
+_STANCE_RE = re.compile(r"Stance in source:\s*(\w+)")
+
+
+def parse_claim_atom(content: str) -> tuple[str, str | None]:
+    """Pull (claim_text, stance) out of a claim atom's markdown body."""
+    m = _CLAIM_BODY_RE.search(content)
+    claim_text = m.group(1).strip() if m else content[:200]
+    s = _STANCE_RE.search(content)
+    stance = s.group(1) if s else None
+    return claim_text, stance
+
+
+def is_claim_hit(hit: SearchHit) -> bool:
+    """A hit is a claim if it has the Type/Claim tag."""
+    return any(t.get("name") == "Claim" for t in hit.tags)
+
+
+def language_of(hit: SearchHit) -> str | None:
+    for t in hit.tags:
+        if t.get("name") in {"RO", "EN", "RU", "UA", "FR", "DE"}:
+            return t["name"]
+    return None
+
+
+def credibility_of(hit: SearchHit) -> str | None:
+    for t in hit.tags:
+        n = t.get("name", "")
+        if n.startswith("Tier") or n in {"StateAffiliated", "KnownDisinfo"}:
+            return n
+    return None
+
+
+def parent_url_of(hit: SearchHit) -> str:
+    if not hit.source_url:
+        return ""
+    return hit.source_url.split("#", 1)[0]
+
+
+# ====================================================== orchestration per query
+
+
+async def evaluate_query(
+    spec: ClaimQuery,
+    atomic: AtomicClient,
+    embed: EmbeddingClient,
+) -> None:
+    console.print(f"\n[bold cyan]── {spec.label} ──[/bold cyan]")
+    console.print(f"[dim]query:[/dim] {spec.query}\n")
+
+    # Pull a wide net of semantic hits — we'll split docs vs claims after
+    raw_hits = await atomic.search(spec.query, mode="semantic", limit=50, threshold=0.2)
+    if not raw_hits:
+        console.print("[yellow]no hits[/yellow]")
+        return
+
+    # Fetch full content for the claim hits so we can extract claim text + stance
+    claim_hits = [h for h in raw_hits if is_claim_hit(h)]
+    doc_hits = [h for h in raw_hits if not is_claim_hit(h)]
+
+    # Resolve full atom content for top-15 claim hits in one async batch
+    top_claim_hits = claim_hits[:15]
+    full_atoms = await asyncio.gather(
+        *(atomic.get_atom(h.atom_id) for h in top_claim_hits)
+    )
+
+    # Build (claim_text, stance, hit, parent_url) tuples
+    parsed: list[tuple[SearchHit, str, str | None, str]] = []
+    for hit, full in zip(top_claim_hits, full_atoms, strict=True):
+        text, stance = parse_claim_atom(full.get("content") or "")
+        parent = parent_url_of(hit)
+        parsed.append((hit, text, stance, parent))
+
+    if not parsed:
+        console.print("[yellow]no claim hits — only documents matched[/yellow]")
+        _render_doc_table(doc_hits[:5])
+        return
+
+    # Rerank the claims with cross-encoder for precision
+    docs_for_rerank = [text for _, text, _, _ in parsed]
+    reranked = await embed.rerank(spec.query, docs_for_rerank, top_n=10)
+    rank_position = {r.index: (i, r.score) for i, r in enumerate(reranked)}
+
+    # Pretty print top claims
+    table = Table(show_lines=False)
+    table.add_column("#", width=3, justify="right")
+    table.add_column("emb", width=6, justify="right")
+    table.add_column("rerank", width=8, justify="right")
+    table.add_column("stance", width=10)
+    table.add_column("lang", width=4)
+    table.add_column("claim", overflow="fold")
+
+    # Show top 8 by reranker order
+    sorted_by_rerank = sorted(
+        enumerate(parsed),
+        key=lambda x: rank_position.get(x[0], (999, 0))[0],
+    )
+    for display_i, (orig_idx, (hit, text, stance, parent)) in enumerate(sorted_by_rerank[:8], 1):
+        rerank_info = rank_position.get(orig_idx)
+        rerank_str = f"{rerank_info[1]:.3f}" if rerank_info else "-"
+        lang = language_of(hit) or "?"
+        stance_color = {
+            "ASSERTS": "green",
+            "REPORTS": "blue",
+            "REFUTES": "red",
+            "QUESTIONS": "yellow",
+            "NEUTRAL": "dim",
+        }.get(stance or "", "dim")
+        table.add_row(
+            str(display_i),
+            f"{hit.similarity:.3f}",
+            rerank_str,
+            f"[{stance_color}]{stance or '?'}[/{stance_color}]",
+            lang,
+            text[:200],
+        )
+    console.print(table)
+
+    # Aggregations: stance, language, parent doc count, credibility
+    stance_counts = Counter(stance for _, _, stance, _ in parsed if stance)
+    lang_counts = Counter(language_of(h) for h, _, _, _ in parsed if language_of(h))
+    parent_docs = {parent for _, _, _, parent in parsed if parent}
+
+    summary = Table(show_header=False, show_lines=False, padding=(0, 2))
+    summary.add_column(style="bold dim")
+    summary.add_column()
+    summary.add_row("total claim hits", str(len(claim_hits)))
+    summary.add_row("distinct parent docs", str(len(parent_docs)))
+    summary.add_row("by stance (top 15)", " ".join(f"{k}={v}" for k, v in stance_counts.most_common()))
+    summary.add_row("by language (top 15)", " ".join(f"{k}={v}" for k, v in lang_counts.most_common()))
+    summary.add_row("doc-level matches also", str(len(doc_hits)))
+    console.print(summary)
+
+
+def _render_doc_table(hits: list[SearchHit]) -> None:
+    if not hits:
+        return
+    t = Table(title="Document fallback", show_lines=False)
+    t.add_column("#", width=3, justify="right")
+    t.add_column("sim", width=6, justify="right")
+    t.add_column("doc")
+    t.add_column("preview", overflow="fold")
+    for i, h in enumerate(hits, 1):
+        from urllib.parse import unquote
+        slug = unquote((h.source_url or "").rsplit("/", 1)[-1])
+        t.add_row(str(i), f"{h.similarity:.3f}", slug, (h.matching_chunk_content or "")[:120])
+    console.print(t)
+
+
+async def main_async() -> int:
+    setup_logging()
+    if not settings.atomic_token:
+        console.print("[red]ATOMIC_TOKEN missing[/red]")
+        return 2
+
+    console.print(
+        f"[dim]Atomic: {settings.atomic_url}  |  Reranker: {settings.reranker_url}[/dim]"
+    )
+
+    async with AtomicClient() as atomic, EmbeddingClient() as embed:
+        for spec in QUERIES:
+            try:
+                await evaluate_query(spec, atomic, embed)
+            except Exception as e:  # noqa: BLE001
+                console.print(f"[red]× error on {spec.label}: {type(e).__name__}: {e}[/red]")
+    console.print("\n[bold]done[/bold]")
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async())
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/09_brain_api_demo.py

@@ -0,0 +1,296 @@
+"""End-to-end demo of the brain_api HTTP service.
+
+Assumes the API is already running (`python -m brain_api.run` in another
+terminal, or via the helper flag --spawn here).
+
+Hits all five endpoints in sequence with realistic inputs and prints the
+key fields so you can eyeball both schema compliance and retrieval quality.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import httpx  # noqa: E402
+from rich.console import Console  # noqa: E402
+from rich.panel import Panel  # noqa: E402
+from rich.table import Table  # noqa: E402
+
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+BASE = "http://127.0.0.1:8090"
+TIMEOUT = 120.0
+
+
+GATHER_CLAIMS = [
+    "vaccinurile pediatrice cauzeaza autism la copii",
+    "Andrew Wakefield falsified medical records in his 1998 study",
+    "Robert F Kennedy Jr promoted COVID-19 vaccine misinformation",
+    "HPV vaccines cause infertility in teenage girls",  # expected MISS / weak
+]
+
+SEARCH_QUERIES = [
+    "MMR vaccine autism controversy",
+    "Pfizer BioNTech myocarditis",
+]
+
+FETCH_URLS_KNOWN = [
+    "https://en.wikipedia.org/wiki/Andrew_Wakefield",
+    "https://en.wikipedia.org/wiki/Vaccine_hesitancy",
+    "https://en.wikipedia.org/wiki/MMR_vaccine_and_autism",
+]
+FETCH_URLS_UNKNOWN = [
+    "https://example.com/totally-not-in-brain",
+]
+
+
+async def wait_for_api(client: httpx.AsyncClient, timeout: float = 30.0) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            r = await client.get(f"{BASE}/health")
+            if r.status_code == 200:
+                return True
+        except httpx.HTTPError:
+            pass
+        await asyncio.sleep(0.5)
+    return False
+
+
+async def demo_gather(client: httpx.AsyncClient) -> None:
+    console.print(Panel.fit("[bold cyan]POST /v1/gather[/bold cyan]"))
+    for claim in GATHER_CLAIMS:
+        t0 = time.perf_counter()
+        r = await client.post(
+            f"{BASE}/v1/gather",
+            json={
+                "claim": claim,
+                "max_evidence": 8,
+                "include_full_text": False,
+                "summarize": True,
+                "score_relevance": True,
+            },
+        )
+        elapsed = (time.perf_counter() - t0) * 1000
+        if r.status_code != 200:
+            console.print(f"[red]HTTP {r.status_code}[/red]: {r.text[:200]}")
+            continue
+
+        data = r.json()
+        brain_meta = data.get("brain_meta") or {}
+        cache = brain_meta.get("cache_status", "?")
+        ev_count = data.get("total_evidence_items", 0)
+        sources = brain_meta.get("evidence_sources", 0)
+
+        color = {
+            "HIT": "green",
+            "PARTIAL": "yellow",
+            "MISS": "dim",
+        }.get(cache, "white")
+
+        console.print(
+            f"\n[bold]{claim}[/bold]\n"
+            f"  [{color}]cache={cache}[/{color}]  "
+            f"evidence={ev_count}  "
+            f"sources={sources}  "
+            f"time={elapsed:.0f}ms  "
+            f"server_reported={data.get('execution_time_ms', 0):.0f}ms"
+        )
+
+        if not data.get("evidence"):
+            console.print("  [dim]· no evidence[/dim]")
+            continue
+
+        t = Table(show_header=True, header_style="bold", show_lines=False, padding=(0, 1))
+        t.add_column("#", width=3)
+        t.add_column("rel", width=6, justify="right")
+        t.add_column("cred", width=6, justify="right")
+        t.add_column("publisher", width=20)
+        t.add_column("summary (best claim)", overflow="fold")
+        for i, ev in enumerate(data["evidence"][:5], 1):
+            summary = (ev.get("summary") or ev.get("snippet") or "")[:180]
+            t.add_row(
+                str(i),
+                f"{ev.get('relevance_score', 0):.3f}",
+                f"{ev.get('credibility_score', 0):.2f}",
+                (ev.get("publisher") or "")[:20],
+                summary,
+            )
+        console.print(t)
+
+        # Show the stages timing
+        stages = data.get("stages") or []
+        stage_line = "  ".join(
+            f"{s['stage']}={s.get('duration_ms', 0):.0f}ms"
+            for s in stages
+        )
+        console.print(f"  [dim]stages: {stage_line}[/dim]")
+
+
+async def demo_search(client: httpx.AsyncClient) -> None:
+    console.print(Panel.fit("[bold cyan]POST /v1/search[/bold cyan]"))
+    r = await client.post(
+        f"{BASE}/v1/search",
+        json={"queries": SEARCH_QUERIES, "max_results": 5},
+    )
+    if r.status_code != 200:
+        console.print(f"[red]HTTP {r.status_code}[/red]: {r.text[:200]}")
+        return
+    data = r.json()
+    console.print(
+        f"total={data['total_results']}  "
+        f"queries_processed={data['queries_processed']}  "
+        f"time={data['execution_time_ms']:.0f}ms"
+    )
+    t = Table(show_header=True)
+    t.add_column("rank", width=4)
+    t.add_column("query", width=20)
+    t.add_column("site", width=20)
+    t.add_column("title", overflow="fold")
+    for res in data.get("results", []):
+        t.add_row(
+            str(res.get("rank", 0)),
+            (res.get("query") or "")[:18],
+            (res.get("site") or "")[:18],
+            (res.get("title") or "")[:60],
+        )
+    console.print(t)
+
+
+async def demo_fetch(client: httpx.AsyncClient) -> None:
+    console.print(Panel.fit("[bold cyan]POST /v1/fetch[/bold cyan]"))
+    urls = FETCH_URLS_KNOWN + FETCH_URLS_UNKNOWN
+    r = await client.post(f"{BASE}/v1/fetch", json={"urls": urls})
+    if r.status_code != 200:
+        console.print(f"[red]HTTP {r.status_code}[/red]: {r.text[:200]}")
+        return
+    data = r.json()
+    console.print(
+        f"total_fetched={data['total_fetched']}  "
+        f"total_failed={data['total_failed']}  "
+        f"time={data['execution_time_ms']:.0f}ms"
+    )
+    for p in data.get("pages", []):
+        console.print(
+            f"  [green]HIT[/green] {p.get('title', '?')[:60]}  "
+            f"({len(p.get('text') or '')} chars)"
+        )
+    for f in data.get("failed_urls", []):
+        console.print(f"  [dim]MISS[/dim] {f['url']}  ({f['error']})")
+
+
+async def demo_image_search(client: httpx.AsyncClient) -> None:
+    console.print(Panel.fit("[bold cyan]POST /v1/image-search[/bold cyan]"))
+    r = await client.post(
+        f"{BASE}/v1/image-search",
+        json={"queries": ["Andrew Wakefield"], "max_results": 10},
+    )
+    console.print(
+        f"status={r.status_code}  "
+        f"body={r.json() if r.status_code == 200 else r.text[:200]}"
+    )
+
+
+async def demo_ingest(client: httpx.AsyncClient) -> None:
+    console.print(Panel.fit("[bold cyan]POST /v1/ingest  (dry smoke)[/bold cyan]"))
+    body = {
+        "claim": "Sample unit-test claim for brain_api ingest smoke",
+        "default_tags": [
+            "Topics/Health/Vaccines",
+            "Language/EN",
+        ],
+        "run_extraction": False,
+        "evidence": [
+            {
+                "url": "https://example.test/brain-ingest-smoke-1",
+                "title": "Brain ingest smoke test 1",
+                "publisher": "example.test",
+                "retrieved_at": "2026-04-11T00:00:00+00:00",
+                "full_text": (
+                    "# Brain ingest smoke 1\n\n"
+                    "This is a synthetic document created by the brain_api "
+                    "demo script to verify /v1/ingest. It does not represent "
+                    "any real factual claim."
+                ),
+                "relevance_score": 0.75,
+                "credibility_score": 0.70,
+            }
+        ],
+    }
+    r = await client.post(f"{BASE}/v1/ingest", json=body)
+    if r.status_code != 200:
+        console.print(f"[red]HTTP {r.status_code}[/red]: {r.text[:300]}")
+        return
+    data = r.json()
+    console.print(
+        f"accepted={data['accepted']}  "
+        f"skipped_duplicate={data['skipped_duplicate']}  "
+        f"errors={data['errors']}  "
+        f"ids={data.get('created_atom_ids', [])}"
+    )
+    if data.get("warnings"):
+        for w in data["warnings"]:
+            console.print(f"  [yellow]![/yellow] {w}")
+
+
+async def main_async(args: argparse.Namespace) -> int:
+    setup_logging()
+
+    spawned: subprocess.Popen | None = None
+    if args.spawn:
+        console.print("[dim]spawning brain_api in background...[/dim]")
+        spawned = subprocess.Popen(
+            [sys.executable, "-m", "brain_api.run"],
+            cwd=str(Path(__file__).resolve().parent.parent),
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+    try:
+        async with httpx.AsyncClient(timeout=TIMEOUT) as client:
+            if not await wait_for_api(client):
+                console.print("[red]brain_api did not come up within 30s[/red]")
+                return 1
+            console.print("[green]✓[/green] brain_api /health OK\n")
+
+            await demo_gather(client)
+            console.print()
+            await demo_search(client)
+            console.print()
+            await demo_fetch(client)
+            console.print()
+            await demo_image_search(client)
+            console.print()
+            if args.ingest:
+                await demo_ingest(client)
+
+        console.print("\n[bold green]demo complete[/bold green]")
+        return 0
+    finally:
+        if spawned is not None:
+            spawned.terminate()
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--spawn", action="store_true", help="start uvicorn ourselves")
+    p.add_argument("--ingest", action="store_true", help="also exercise /v1/ingest (writes to brain)")
+    args = p.parse_args()
+    try:
+        return asyncio.run(main_async(args))
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/10_run_lint.py

@@ -0,0 +1,85 @@
+"""Run the Lint pass over the current claim atoms in the brain.
+
+    python scripts/10_run_lint.py              # full corpus
+    python scripts/10_run_lint.py --limit 30   # smoke test
+    python scripts/10_run_lint.py --force      # re-evaluate cached pairs
+
+The state file at lint/_contradictions.json is always updated idempotently.
+Interrupting the run with Ctrl-C will still save whatever has been classified
+so far, and a subsequent run will pick up from there.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+
+from lint._state import LintState  # noqa: E402
+from lint.reporter import (  # noqa: E402
+    render_contradictions,
+    render_equivalents,
+    render_stats,
+)
+from lint.runner import run_lint_pass  # noqa: E402
+from shared.config import settings  # noqa: E402
+from shared.logging import setup_logging  # noqa: E402
+
+console = Console()
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Run cross-corpus contradiction detection")
+    p.add_argument("--limit", type=int, default=None, help="cap number of source claim atoms")
+    p.add_argument("--force", action="store_true", help="re-evaluate cached pairs")
+    p.add_argument(
+        "--show-equivalents",
+        action="store_true",
+        help="also print paraphrase clusters (not just contradictions)",
+    )
+    return p.parse_args()
+
+
+async def main_async(args: argparse.Namespace) -> int:
+    setup_logging()
+    console.print(
+        f"[dim]Atomic: {settings.atomic_url}  |  "
+        f"LLM: {settings.llm_router_url}  |  "
+        f"model: {settings.model_reasoning}[/dim]\n"
+    )
+
+    try:
+        stats = await run_lint_pass(limit_atoms=args.limit, force=args.force)
+    except KeyboardInterrupt:
+        console.print("\n[yellow]interrupted — state file preserved[/yellow]")
+        return 130
+    except Exception as e:  # noqa: BLE001
+        console.print(f"[red]lint failed:[/red] {type(e).__name__}: {e}")
+        return 1
+
+    console.print()
+    render_stats(stats)
+
+    state = LintState()
+    render_contradictions(state, top_n=10, min_confidence=0.7)
+    if args.show_equivalents:
+        render_equivalents(state, top_n=10, min_confidence=0.85)
+
+    console.print(f"\n[dim]ledger → {state.path}[/dim]")
+    return 0
+
+
+def main() -> int:
+    try:
+        return asyncio.run(main_async(parse_args()))
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/11_show_contradictions.py

@@ -0,0 +1,61 @@
+"""Read the Lint pass state file and render detected contradictions.
+
+Does NOT run any classification — it's a read-only view over whatever the
+last run of scripts/10_run_lint.py produced.
+
+    python scripts/11_show_contradictions.py                  # top 20, >= 0.7 conf
+    python scripts/11_show_contradictions.py --min 0.85       # stricter
+    python scripts/11_show_contradictions.py --top 50         # wider net
+    python scripts/11_show_contradictions.py --equivalents    # also paraphrases
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from rich.console import Console  # noqa: E402
+
+from lint._state import LintState  # noqa: E402
+from lint.reporter import render_contradictions, render_equivalents  # noqa: E402
+
+console = Console()
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--min", type=float, default=0.7, help="minimum confidence")
+    p.add_argument("--top", type=int, default=20, help="top N to display")
+    p.add_argument("--equivalents", action="store_true", help="also show paraphrases")
+    args = p.parse_args()
+
+    state = LintState()
+    if not state.all_verdicts:
+        console.print(
+            f"[yellow]no verdicts in state file at {state.path}[/yellow]\n"
+            "Run scripts/10_run_lint.py first."
+        )
+        return 2
+
+    total = len(state.all_verdicts)
+    contras_total = len(state.all_contradictions)
+    equivs_total = len(state.all_equivalents)
+    console.print(
+        f"[dim]state file: {state.path}[/dim]\n"
+        f"[dim]total pairs: {total}  |  "
+        f"contradictory: {contras_total}  |  "
+        f"equivalent: {equivs_total}  |  "
+        f"incomparable: {total - contras_total - equivs_total}[/dim]\n"
+    )
+
+    render_contradictions(state, top_n=args.top, min_confidence=args.min)
+    if args.equivalents:
+        render_equivalents(state, top_n=args.top, min_confidence=max(args.min, 0.85))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ai_platform/modules/didi_brain/scripts/bootstrap_deploy.sh

@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+# =============================================================================
+#  DidiBrain — fresh-server bootstrap
+# =============================================================================
+#
+#  End-to-end deploy of the DidiBrain stack on a clean Linux server:
+#
+#     1. Preflight (docker, compose, python3, curl)
+#     2. .env validation
+#     3. docker compose build + up
+#     4. Wait for all three containers to become healthy
+#     5. Create a local Python venv for operator scripts
+#     6. Install operator dependencies (httpx, pydantic, etc.)
+#     7. Claim the Atomic instance + configure the BGE-M3 provider
+#     8. Seed the canonical tag taxonomy
+#     9. (Optional) Import the seed Wikipedia corpus
+#    10. (Optional) Run the claim extraction batch
+#    11. Final smoke test against /v1/gather
+#
+#  Every step is idempotent — you can re-run this script after a crash or
+#  after editing .env, and it will only do what still needs doing. No step
+#  is destructive (no `down -v`, no volume deletions).
+#
+#  Environment flags you can set before running:
+#
+#     BRAIN_IMPORT_CORPUS   1 to import the Wikipedia seed (default 1)
+#     BRAIN_RUN_EXTRACTION  1 to run claim extraction after import (default 1)
+#     BRAIN_SKIP_VENV       1 to skip venv creation / reuse existing .venv
+#     BRAIN_SKIP_SANITY     1 to skip the final smoke test (save a couple sec)
+#
+#  Usage:
+#     cd ~/didibrain
+#     cp .env.example .env   # then edit LLM_ROUTER_URL / EMBEDDING_URL / ...
+#     ./scripts/bootstrap_deploy.sh
+#
+# =============================================================================
+
+set -euo pipefail
+
+# ----------------------------------------------------------------- paths
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+INFRA_DIR="${PROJECT_ROOT}/infra"
+COMPOSE_FILE="${INFRA_DIR}/docker-compose.yml"
+ENV_FILE="${PROJECT_ROOT}/.env"
+ENV_EXAMPLE="${PROJECT_ROOT}/.env.example"
+VENV_DIR="${PROJECT_ROOT}/.venv"
+
+cd "${PROJECT_ROOT}"
+
+# ----------------------------------------------------------------- colors
+
+RED=$'\033[31m'
+GREEN=$'\033[32m'
+YELLOW=$'\033[33m'
+BLUE=$'\033[34m'
+DIM=$'\033[2m'
+BOLD=$'\033[1m'
+RESET=$'\033[0m'
+
+step()    { echo "${BLUE}${BOLD}== $* ==${RESET}"; }
+info()    { echo "${DIM}  · $*${RESET}"; }
+ok()      { echo "${GREEN}  ✓ $*${RESET}"; }
+warn()    { echo "${YELLOW}  ! $*${RESET}"; }
+fail()    { echo "${RED}${BOLD}  ✗ $*${RESET}" >&2; exit 1; }
+
+# ------------------------------------------------------------- step 1 preflight
+
+step "1. Preflight checks"
+
+require_cmd() {
+    command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1"
+    ok "$1 found"
+}
+
+require_cmd docker
+require_cmd python3
+require_cmd curl
+
+# Docker Compose can be `docker compose` (v2) or `docker-compose` (v1). Prefer v2.
+if docker compose version >/dev/null 2>&1; then
+    DC="docker compose"
+    ok "docker compose (v2) found"
+elif command -v docker-compose >/dev/null 2>&1; then
+    DC="docker-compose"
+    warn "using legacy docker-compose (v1) — v2 recommended"
+else
+    fail "neither 'docker compose' (v2) nor 'docker-compose' (v1) found"
+fi
+
+# Docker daemon reachable?
+docker info >/dev/null 2>&1 || fail "docker daemon not reachable — is Docker running and your user in the docker group?"
+ok "docker daemon reachable"
+
+# ------------------------------------------------------------- step 2 .env
+
+step "2. Environment file"
+
+if [[ ! -f "${ENV_FILE}" ]]; then
+    if [[ -f "${ENV_EXAMPLE}" ]]; then
+        warn ".env missing — copying from .env.example"
+        warn "REVIEW IT AND FILL IN LLM_ROUTER_URL / EMBEDDING_URL / RERANKER_URL"
+        cp "${ENV_EXAMPLE}" "${ENV_FILE}"
+        fail ".env was just created from template. Edit it, then rerun this script."
+    else
+        fail "no .env and no .env.example in ${PROJECT_ROOT}"
+    fi
+fi
+ok ".env present at ${ENV_FILE}"
+
+# Minimal sanity on required variables
+check_env_var() {
+    local key="$1"
+    if ! grep -E "^${key}=" "${ENV_FILE}" >/dev/null 2>&1; then
+        fail "${key} is missing from .env"
+    fi
+    local value
+    value="$(grep -E "^${key}=" "${ENV_FILE}" | head -1 | cut -d= -f2-)"
+    if [[ -z "${value}" ]]; then
+        warn "${key} is empty in .env (may be filled by bootstrap — continuing)"
+    fi
+}
+
+check_env_var LLM_ROUTER_URL
+check_env_var EMBEDDING_URL
+check_env_var RERANKER_URL
+check_env_var POSTGRES_USER
+check_env_var POSTGRES_PASSWORD
+ok ".env keys look structurally correct"
+
+# ------------------------------------------------------ step 3 docker compose
+
+step "3. Build and start containers"
+
+info "building brain-api image (cached layers reused where possible)..."
+${DC} -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" build brain-api
+
+info "starting the full stack (postgres, atomic-server, brain-api)..."
+${DC} -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d
+
+ok "containers started"
+
+# ------------------------------------------------------ step 4 wait for healthy
+
+step "4. Wait for all three containers to become healthy"
+
+wait_healthy() {
+    local name="$1"
+    local deadline=$(( $(date +%s) + 180 ))
+    while (( $(date +%s) < deadline )); do
+        local status
+        status=$(docker inspect --format='{{.State.Health.Status}}' "${name}" 2>/dev/null || echo "missing")
+        case "${status}" in
+            healthy)
+                ok "${name} healthy"
+                return 0
+                ;;
+            unhealthy)
+                fail "${name} reports unhealthy — check 'docker logs ${name}'"
+                ;;
+            starting)
+                info "${name} starting..."
+                ;;
+            missing)
+                info "${name} not yet visible to docker..."
+                ;;
+            *)
+                info "${name} status: ${status}"
+                ;;
+        esac
+        sleep 3
+    done
+    fail "${name} did not reach healthy within 180 seconds"
+}
+
+wait_healthy didibrain-postgres
+wait_healthy didibrain-atomic
+wait_healthy didibrain-api
+
+# -------------------------------------------------------------- step 5 venv
+
+step "5. Python operator venv"
+
+if [[ "${BRAIN_SKIP_VENV:-0}" == "1" ]]; then
+    warn "BRAIN_SKIP_VENV=1 — skipping venv creation"
+elif [[ -d "${VENV_DIR}" ]]; then
+    ok "venv already exists at ${VENV_DIR}"
+else
+    info "creating venv at ${VENV_DIR}..."
+    python3 -m venv "${VENV_DIR}"
+    ok "venv created"
+fi
+
+if [[ "${BRAIN_SKIP_VENV:-0}" != "1" ]]; then
+    info "installing operator dependencies..."
+    "${VENV_DIR}/bin/pip" install --quiet --upgrade pip
+    "${VENV_DIR}/bin/pip" install --quiet \
+        "httpx>=0.28,<0.30" \
+        "pydantic>=2.12,<3.0" \
+        "pydantic-settings>=2.13,<3.0" \
+        "structlog>=25.5,<26.0" \
+        "python-dotenv>=1.2,<2.0" \
+        "tenacity>=9.1,<10.0" \
+        "rich>=14.3,<15.0"
+    ok "operator dependencies installed"
+fi
+
+PY="${VENV_DIR}/bin/python"
+export PYTHONIOENCODING=utf-8
+
+# -------------------------------------------------------------- step 6 bootstrap
+
+step "6. Atomic bootstrap (claim instance + provider config)"
+
+info "running scripts/02_bootstrap_atomic.py (idempotent)..."
+"${PY}" "${PROJECT_ROOT}/scripts/02_bootstrap_atomic.py"
+ok "atomic bootstrapped"
+
+# -------------------------------------------------------------- step 7 taxonomy
+
+step "7. Seed canonical taxonomy"
+
+info "running scripts/04_seed_taxonomy.py (idempotent)..."
+"${PY}" "${PROJECT_ROOT}/scripts/04_seed_taxonomy.py"
+ok "taxonomy seeded"
+
+# -------------------------------------------------------- step 8 corpus import
+
+if [[ "${BRAIN_IMPORT_CORPUS:-1}" == "1" ]]; then
+    step "8. Import the Wikipedia seed corpus"
+    info "running scripts/05_import_wikipedia_seed.py..."
+    "${PY}" "${PROJECT_ROOT}/scripts/05_import_wikipedia_seed.py"
+    ok "seed corpus imported"
+else
+    warn "BRAIN_IMPORT_CORPUS=0 — skipping Wikipedia seed import"
+fi
+
+# -------------------------------------------------------- step 9 extraction
+
+if [[ "${BRAIN_RUN_EXTRACTION:-1}" == "1" && "${BRAIN_IMPORT_CORPUS:-1}" == "1" ]]; then
+    step "9. Run claim extraction (may take ~15-20 min for the seed)"
+    info "running scripts/07_run_extraction.py..."
+    "${PY}" "${PROJECT_ROOT}/scripts/07_run_extraction.py"
+    ok "claim extraction complete"
+else
+    warn "skipping claim extraction"
+fi
+
+# ---------------------------------------------------------- step 10 smoke test
+
+if [[ "${BRAIN_SKIP_SANITY:-0}" == "1" ]]; then
+    warn "BRAIN_SKIP_SANITY=1 — skipping final smoke test"
+else
+    step "10. Final smoke test — /v1/gather against brain_api"
+
+    HEALTH_JSON="$(curl -fsS http://localhost:8090/health)"
+    info "brain_api /health → ${HEALTH_JSON}"
+
+    GATHER_JSON="$(curl -fsS -X POST http://localhost:8090/v1/gather \
+        -H 'Content-Type: application/json' \
+        -d '{"claim":"vaccines cause autism","max_evidence":3,"include_full_text":false,"run_nli":false}')"
+
+    CACHE_STATUS=$(echo "${GATHER_JSON}" | "${PY}" -c \
+        "import sys,json; print(json.load(sys.stdin).get('brain_meta',{}).get('cache_status','?'))")
+    ITEM_COUNT=$(echo "${GATHER_JSON}" | "${PY}" -c \
+        "import sys,json; print(json.load(sys.stdin).get('total_evidence_items',0))")
+
+    info "gather: cache_status=${CACHE_STATUS}  evidence_items=${ITEM_COUNT}"
+
+    if [[ "${CACHE_STATUS}" == "HIT" ]]; then
+        ok "brain returned HIT with ${ITEM_COUNT} items — end-to-end working"
+    elif [[ "${CACHE_STATUS}" == "PARTIAL" ]]; then
+        ok "brain returned PARTIAL (${ITEM_COUNT} items) — end-to-end working, partial coverage"
+    elif [[ "${CACHE_STATUS}" == "MISS" ]]; then
+        warn "brain returned MISS — this is expected if the corpus was not imported"
+        warn "(re-run with BRAIN_IMPORT_CORPUS=1 BRAIN_RUN_EXTRACTION=1 to populate)"
+    else
+        fail "unexpected cache_status: ${CACHE_STATUS}"
+    fi
+fi
+
+# ---------------------------------------------------------------- done
+
+echo
+step "DONE"
+echo
+echo "  brain_api:           http://localhost:8090"
+echo "  Swagger UI:          http://localhost:8090/docs"
+echo "  ReDoc:               http://localhost:8090/redoc"
+echo "  OpenAPI spec:        http://localhost:8090/openapi.json"
+echo "  atomic-server API:   http://localhost:8088"
+echo "  atomic API docs:     http://localhost:8088/api/docs"
+echo "  postgres:            localhost:5434"
+echo
+echo "${DIM}Next steps:${RESET}"
+echo "  · Point Didi backend at http://<this-host>:8090/v1/gather"
+echo "  · Run ${BOLD}${PY} scripts/10_run_lint.py${RESET} overnight for the first contradiction audit"
+echo "  · When corpus needs to grow, extend scripts/05 seed lists or feed"
+echo "    web-module output back via ${BOLD}POST /v1/ingest${RESET}"
+echo