Livrare LOT 1 - Didi

ai_platform/modules/didi_brain/shared/__init__.py

@@ -0,0 +1 @@
+"""Shared infrastructure: config, LLM client, embedding client, Atomic client."""

ai_platform/modules/didi_brain/shared/_tag_ids.json

@@ -0,0 +1,81 @@
+{
+  "ClaimStatus": "a4bdf09e-52f2-4b85-8a56-e22c3b05f835",
+  "ClaimStatus/Confirmed": "fd25a0b8-bfb1-4cc5-a5c1-225070f49d5c",
+  "ClaimStatus/Debunked": "50b9e623-144c-4093-8a3c-0ded59d533e8",
+  "ClaimStatus/Disputed": "3f0e90a5-f64a-41a6-b430-e08bfb326d90",
+  "ClaimStatus/PartiallyTrue": "554a7f3f-63f3-4de5-bb26-83d9e5eb8495",
+  "ClaimStatus/Unverified": "fb5ecbf5-7237-4ed7-b694-2a871c1c3038",
+  "Country": "9b22cf38-0175-4cc4-952c-318ae735ecec",
+  "Country/France": "b7ae6946-b2ae-474d-906e-f25c2b5f08ce",
+  "Country/Germany": "986f37e0-be60-4e1c-b966-0b2273358f57",
+  "Country/Global": "ef8938f9-8866-4ebf-8412-1603b4220c5e",
+  "Country/Italy": "fb724b1f-7b66-4567-912a-bf7b17013921",
+  "Country/Moldova": "35e44ad8-31a2-43be-8fdc-149a1d48fc1c",
+  "Country/Poland": "57d3023b-165c-4d3c-bdc6-8203d938ab94",
+  "Country/Romania": "6e5657a5-47d8-4602-82ee-b9ef27e05b6f",
+  "Country/Russia": "b045c4cd-b346-4dba-815f-8bdb86ac5aff",
+  "Country/Spain": "9037debf-6108-4634-9955-6e3ab7803c24",
+  "Country/UK": "52200de8-841a-46f1-be12-39ee7d5df275",
+  "Country/USA": "a86f71dd-ab0c-4b26-8f0c-e6370f41ab14",
+  "Country/Ukraine": "f4d6b5ef-3114-43ad-9349-a2215765babf",
+  "Credibility": "e1d6214c-4e8d-4f55-81ff-c51130638108",
+  "Credibility/KnownDisinfo": "bbcbbacd-cbd8-4da8-b08e-1fa420bffe8a",
+  "Credibility/StateAffiliated": "75bdaecb-61e3-4f52-b459-fd9d35584a88",
+  "Credibility/Tier1": "a6fd68db-7d24-4155-b038-a354d7aa90a7",
+  "Credibility/Tier2": "ca4b2bad-7f9b-44f4-86e1-9fa6cc460807",
+  "Credibility/Tier3": "b180c0ed-0488-4098-8b36-5bc9eddb095e",
+  "Credibility/Unknown": "e3122bf0-de42-473e-acc4-06b29cfd58b0",
+  "Events": "acf1d4c9-6e7b-4dc0-8d4e-f0db324e038d",
+  "Language": "d7a7b7fa-d7e1-465d-95f4-fe242afa1da4",
+  "Language/DE": "5fbcb199-e5b2-409f-86a4-c267a64609e8",
+  "Language/EN": "1f901309-ed11-4235-aa7b-39953cbe74da",
+  "Language/ES": "209c6c0d-efac-481b-bb3f-69345c1f0f8d",
+  "Language/FR": "6e0ac309-9b3d-4ed0-b4f8-d26f89c7a940",
+  "Language/IT": "b9ddebbf-db73-4d08-a82c-90c726d2b61b",
+  "Language/PL": "8375f071-fa08-4b09-a135-74c288a8d293",
+  "Language/RO": "5b8e7fa9-42ae-40f0-ab02-8959e3703700",
+  "Language/RU": "69d1b480-2bac-447c-9cd0-845a6dc04f24",
+  "Language/UA": "8b976668-cd2f-4374-b251-dcf35bb40e92",
+  "Locations": "b6383171-7b06-4761-ae9d-95e2a6c4de31",
+  "Organizations": "1b0eca4b-1e21-425c-9813-07337ae4af83",
+  "People": "f8864b48-8ecc-482a-846f-6f2adfa67b82",
+  "SourceType": "de854162-f5e3-4430-8af1-77727344617b",
+  "SourceType/Blog": "bfdc7f83-e900-4972-b728-06a53fb4c40d",
+  "SourceType/FactCheck": "2a2c40cf-d4db-4a26-a347-2377ad11acaf",
+  "SourceType/Forum": "052fa147-ba51-4fde-89da-be9a88170a2d",
+  "SourceType/Government": "03dbd479-939d-4e8a-8d3f-8e9a54e9eb4b",
+  "SourceType/MainstreamMedia": "783ca275-165d-4669-ad09-2ee98d0f85ce",
+  "SourceType/ScientificJournal": "0b5d65a0-fcc5-46f6-8e29-9fb5564e728b",
+  "SourceType/SocialMedia": "46e6b7a6-d8fd-48f9-a5c2-db0fe6f149ec",
+  "SourceType/StateMedia": "3b95b904-c240-4841-9c59-5abe699f983d",
+  "SourceType/TabloidMedia": "1eff4e9c-e4f5-4439-b10c-d0feb189d25e",
+  "SourceType/Wikipedia": "ea109ab3-b709-4199-9833-47283b862ba0",
+  "Stance": "615ddf04-28eb-42a4-9ea1-8ce0c1c5f346",
+  "Stance/Asserts": "45f71063-e0dc-48ab-b80b-5b26cf68cc3d",
+  "Stance/Neutral": "e8d52343-0062-472c-ab4e-c5b473f267db",
+  "Stance/Questions": "d6eb0f8e-e633-41b1-ba4e-483451fe9e75",
+  "Stance/Refutes": "590afc4c-beff-4973-b8c3-c2f93fe5be13",
+  "Stance/Reports": "e09f46d5-2d84-4934-916a-9bdb9e1fbb1d",
+  "Topics": "8efc2a64-3ccb-4e7e-a100-a6d83de031f0",
+  "Topics/Climate": "047490e0-c848-4bc9-bedb-2a4e7c5d2f3d",
+  "Topics/Economy": "7bb131b0-99b8-4c3a-bf72-25c781f0b4f4",
+  "Topics/Health": "3c4151fa-8fcc-48d3-a367-277c204f23e2",
+  "Topics/Health/COVID": "a14e59fa-a7c3-4cec-b35a-67e1455335bf",
+  "Topics/Health/Disease": "4187f778-a1cf-4baa-9686-ce48e556f841",
+  "Topics/Health/Medicine": "fd011a07-e373-4a85-ab2a-ec6da89945b7",
+  "Topics/Health/PublicHealth": "773c7f9d-b9e8-478f-821e-6daf59e53e9a",
+  "Topics/Health/Vaccines": "3e0c54b9-1554-4b2e-8030-15417c47c04d",
+  "Topics/Politics": "f714e9ce-78d6-4739-be5f-b273a7ab7445",
+  "Topics/Politics/Diplomacy": "0d946f6e-0b9a-4dc2-8244-cdb7fbf741e6",
+  "Topics/Politics/Elections": "e3d4764b-49cb-47bb-ae5a-24a75f6ea845",
+  "Topics/Politics/Government": "da76c072-682f-4e1e-b481-2f805bdb8d8a",
+  "Topics/Politics/War": "f3dd2d86-054d-4cc2-9757-0337af2e90c5",
+  "Topics/Society": "af362549-35e0-4a3e-9b53-485f46376a52",
+  "Topics/Technology": "5d2b4444-2b4f-4494-a673-0904bbf801f2",
+  "Type": "656dac2b-c77c-4e21-b779-78a029b797f7",
+  "Type/Annotation": "af20bc10-8142-44ad-a826-b9214755fe8d",
+  "Type/Claim": "a56f9dce-0ba5-4504-9ff7-38ee005da084",
+  "Type/Document": "c80cbebd-4fae-4395-8992-b510335872ab",
+  "Type/Quote": "f4fdae65-293e-4d9a-8336-9b8838894f4d",
+  "Type/Summary": "6899e2ed-62f7-4336-9c60-2d1a8d1e0418"
+}

ai_platform/modules/didi_brain/shared/atomic_api.py

@@ -0,0 +1,307 @@
+"""Typed async client for the Atomic REST API.
+
+Atomic is the brain — this client is how every other DidiBrain component
+(bootstrap, scraper, extractor, lint, didi_client) talks to it.
+
+Design notes:
+- One AtomicClient instance per long-running process. Reusable httpx.AsyncClient.
+- Auth: Bearer token loaded from `settings.atomic_token`. If empty, public
+  endpoints (/health, setup) still work.
+- Errors: any non-2xx is wrapped in AtomicApiError with the response body.
+- Pagination: helpers like `iter_atoms()` (added later) yield pages.
+
+Only the endpoints we actually need are wrapped here. Add more as required;
+do not pre-emptively wrap the full ~78-route surface.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from shared.config import settings
+from shared.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class AtomicApiError(Exception):
+    """Raised on any non-2xx response from atomic-server."""
+
+    def __init__(self, message: str, *, status: int, body: str = "", url: str = ""):
+        super().__init__(message)
+        self.status = status
+        self.body = body
+        self.url = url
+
+
+@dataclass(slots=True)
+class AtomSummary:
+    """Minimal atom view returned by list/search endpoints."""
+
+    id: str
+    content: str
+    source_url: str | None
+    embedding_status: str
+    tagging_status: str
+    created_at: str
+    updated_at: str
+    tags: list[dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> AtomSummary:
+        return cls(
+            id=d["id"],
+            content=d.get("content", ""),
+            source_url=d.get("source_url"),
+            embedding_status=d.get("embedding_status", "unknown"),
+            tagging_status=d.get("tagging_status", "unknown"),
+            created_at=d.get("created_at", ""),
+            updated_at=d.get("updated_at", ""),
+            tags=d.get("tags", []),
+        )
+
+
+@dataclass(slots=True)
+class SearchHit:
+    """One result from POST /api/search."""
+
+    atom_id: str
+    similarity: float
+    matching_chunk_content: str | None
+    snippet: str | None
+    source_url: str | None
+    tags: list[dict[str, Any]]
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> SearchHit:
+        return cls(
+            atom_id=d.get("id") or d.get("atom_id", ""),
+            similarity=float(d.get("similarity_score") or d.get("similarity", 0.0)),
+            matching_chunk_content=d.get("matching_chunk_content"),
+            snippet=d.get("snippet"),
+            source_url=d.get("source_url"),
+            tags=d.get("tags", []),
+        )
+
+
+class AtomicClient:
+    """Async REST client for atomic-server. Use as `async with AtomicClient() as a:`."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str | None = None,
+        token: str | None = None,
+        timeout: float = 60.0,
+        max_attempts: int = 3,
+    ):
+        self._base_url = (base_url or settings.atomic_url).rstrip("/")
+        self._token = token if token is not None else settings.atomic_token
+        self._max_attempts = max_attempts
+
+        headers = {"Content-Type": "application/json"}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+
+        self._http = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers=headers,
+            timeout=httpx.Timeout(timeout, connect=10.0),
+        )
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> AtomicClient:
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        await self.aclose()
+
+    @property
+    def base_url(self) -> str:
+        return self._base_url
+
+    @property
+    def has_token(self) -> bool:
+        return bool(self._token)
+
+    # ---------------------------------------------------------------- internals
+    async def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        json: Any = None,
+        params: dict[str, Any] | None = None,
+        retry_on_5xx: bool = True,
+    ) -> Any:
+        async for attempt in AsyncRetrying(
+            stop=stop_after_attempt(self._max_attempts),
+            wait=wait_exponential(multiplier=1, min=1, max=10),
+            retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)),
+            reraise=True,
+        ):
+            with attempt:
+                resp = await self._http.request(method, path, json=json, params=params)
+                if resp.status_code >= 500 and retry_on_5xx:
+                    raise httpx.HTTPError(f"5xx from {path}: {resp.status_code}")
+                if resp.status_code >= 400:
+                    raise AtomicApiError(
+                        f"HTTP {resp.status_code} {method} {path}",
+                        status=resp.status_code,
+                        body=resp.text[:1000],
+                        url=str(resp.url),
+                    )
+                if resp.status_code == 204 or not resp.content:
+                    return None
+                try:
+                    return resp.json()
+                except ValueError:
+                    return resp.text
+        raise AtomicApiError("retry loop exited without result", status=0)  # unreachable
+
+    # ----------------------------------------------------------------- health
+    async def health(self) -> dict[str, Any]:
+        return await self._request("GET", "/health")
+
+    async def setup_status(self) -> dict[str, Any]:
+        """Whether the instance still needs initial token claim."""
+        return await self._request("GET", "/api/setup/status")
+
+    # ----------------------------------------------------------------- settings
+    async def get_settings(self) -> dict[str, Any]:
+        return await self._request("GET", "/api/settings")
+
+    async def set_setting(self, key: str, value: str) -> Any:
+        """Set a single setting. Use set_settings() for bulk."""
+        return await self._request(
+            "PUT", f"/api/settings/{key}", json={"value": value}
+        )
+
+    async def set_settings(self, items: dict[str, str]) -> dict[str, Any]:
+        """Apply many settings sequentially. Returns map of key → response."""
+        out: dict[str, Any] = {}
+        for k, v in items.items():
+            out[k] = await self.set_setting(k, v)
+        return out
+
+    # -------------------------------------------------------------------- tags
+    async def list_tags(self, *, min_count: int = 0) -> list[dict[str, Any]]:
+        return await self._request("GET", "/api/tags", params={"min_count": min_count})
+
+    async def create_tag(
+        self, name: str, *, parent_id: str | None = None
+    ) -> dict[str, Any]:
+        body: dict[str, Any] = {"name": name}
+        if parent_id:
+            body["parent_id"] = parent_id
+        return await self._request("POST", "/api/tags", json=body)
+
+    # ------------------------------------------------------------------- atoms
+    async def create_atom(
+        self,
+        *,
+        content: str,
+        source_url: str | None = None,
+        tag_ids: Sequence[str] | None = None,
+        published_at: str | None = None,
+    ) -> dict[str, Any]:
+        body: dict[str, Any] = {"content": content, "tag_ids": list(tag_ids or [])}
+        if source_url:
+            body["source_url"] = source_url
+        if published_at:
+            body["published_at"] = published_at
+        return await self._request("POST", "/api/atoms", json=body)
+
+    async def get_atom(self, atom_id: str) -> dict[str, Any]:
+        return await self._request("GET", f"/api/atoms/{atom_id}")
+
+    async def delete_atom(self, atom_id: str) -> None:
+        await self._request("DELETE", f"/api/atoms/{atom_id}")
+
+    async def get_atom_by_source_url(self, source_url: str) -> dict[str, Any] | None:
+        try:
+            # Atomic's GetAtomBySourceUrlQuery uses `url` not `source_url`.
+            return await self._request(
+                "GET", "/api/atoms/by-source-url", params={"url": source_url}
+            )
+        except AtomicApiError as e:
+            if e.status == 404:
+                return None
+            raise
+
+    async def get_embedding_status(self, atom_id: str) -> dict[str, Any]:
+        return await self._request("GET", f"/api/atoms/{atom_id}/embedding-status")
+
+    async def list_atoms(
+        self,
+        *,
+        limit: int = 50,
+        offset: int = 0,
+        tag_id: str | None = None,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit, "offset": offset}
+        if tag_id:
+            params["tag_id"] = tag_id
+        return await self._request("GET", "/api/atoms", params=params)
+
+    # ------------------------------------------------------------------ search
+    async def search(
+        self,
+        query: str,
+        *,
+        mode: str = "semantic",
+        limit: int = 20,
+        threshold: float | None = None,
+    ) -> list[SearchHit]:
+        body: dict[str, Any] = {"query": query, "mode": mode, "limit": limit}
+        if threshold is not None:
+            body["threshold"] = threshold
+        result = await self._request("POST", "/api/search", json=body)
+        # Atomic returns either a list directly or {"results": [...]}
+        if isinstance(result, dict):
+            items = result.get("results") or result.get("data") or []
+        else:
+            items = result or []
+        return [SearchHit.from_dict(item) for item in items]
+
+    async def find_similar(
+        self, atom_id: str, *, threshold: float = 0.5, limit: int = 20
+    ) -> list[SearchHit]:
+        params = {"threshold": threshold, "limit": limit}
+        result = await self._request(
+            "GET", f"/api/atoms/{atom_id}/similar", params=params
+        )
+        items = result if isinstance(result, list) else result.get("results", [])
+        return [SearchHit.from_dict(item) for item in items]
+
+    # ------------------------------------------------------------- embeddings
+    async def get_pipeline_status(self) -> dict[str, Any]:
+        return await self._request("GET", "/api/embeddings/status")
+
+    async def process_pending_embeddings(self) -> dict[str, Any]:
+        return await self._request("POST", "/api/embeddings/process-pending")
+
+
+@asynccontextmanager
+async def atomic_client(
+    *, token: str | None = None
+) -> AsyncIterator[AtomicClient]:
+    """`async with atomic_client() as a:` for short-lived scripts."""
+    client = AtomicClient(token=token)
+    try:
+        yield client
+    finally:
+        await client.aclose()

ai_platform/modules/didi_brain/shared/config.py

@@ -0,0 +1,162 @@
+"""Centralized configuration via Pydantic Settings.
+
+All env vars are loaded from .env once at import time and validated.
+Import the singleton `settings` everywhere — never read os.environ directly.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import Field, HttpUrl, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class LlmRole(str, Enum):
+    """Logical role a caller asks for. Routing decides which model serves it."""
+
+    REASONING = "reasoning"  # critical: extraction, NLI, verdict, wiki
+    FAST = "fast"  # mass processing (currently disabled)
+    VISION = "vision"  # multimodal (currently disabled)
+
+
+class Settings(BaseSettings):
+    """Top-level config. Validated at startup, immutable thereafter."""
+
+    model_config = SettingsConfigDict(
+        env_file=Path(__file__).parent.parent / ".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # ---- LLM router ---------------------------------------------------------
+    llm_router_url: str = Field(default="http://localhost:14011")
+    llm_router_api_key: str = Field(default="")
+    llm_vllm_url: str = Field(default="http://localhost:14001")
+    llm_llamacpp_urls: str = Field(default="")  # comma-separated
+
+    # ---- Models -------------------------------------------------------------
+    model_reasoning: str = Field(default="Qwen3.5-397B-A17B")
+    model_reasoning_backend: str = Field(default="llamacpp")
+
+    model_fast: str = Field(default="qwen3.5")
+    model_fast_backend: str = Field(default="vllm")
+    model_fast_enabled: bool = Field(default=False)
+
+    model_vision: str = Field(default="gemma-3-27b-it")
+    model_vision_url: str = Field(default="")
+    model_vision_enabled: bool = Field(default=False)
+
+    # ---- Embeddings ---------------------------------------------------------
+    embedding_url: str = Field(default="http://10.11.10.15:8200")
+    embedding_api_key: str = Field(default="")
+    embedding_model: str = Field(default="BAAI/bge-m3")
+    embedding_dim: int = Field(default=1024)
+    embedding_max_tokens: int = Field(default=8192)
+
+    # ---- Reranker -----------------------------------------------------------
+    reranker_url: str = Field(default="http://10.11.10.15:8100")
+    reranker_api_key: str = Field(default="")
+    reranker_model: str = Field(default="BAAI/bge-reranker-v2-m3")
+
+    # ---- Atomic -------------------------------------------------------------
+    atomic_url: str = Field(default="http://localhost:8080")
+    atomic_token: str = Field(default="")
+
+    # ---- Postgres -----------------------------------------------------------
+    postgres_user: str = Field(default="atomic")
+    postgres_password: str = Field(default="atomic_dev_changeme")
+    postgres_db: str = Field(default="atomic")
+    postgres_port: int = Field(default=5434)
+    postgres_host: str = Field(
+        default="postgres",
+        description="Hostname for direct PG connection (Docker: 'postgres', host: 'localhost')",
+    )
+    postgres_internal_port: int = Field(
+        default=5432,
+        description="Port inside the Docker network (external is postgres_port)",
+    )
+
+    # ---- Verification cache --------------------------------------------------
+    verification_cache_ttl_days: int = Field(
+        default=30,
+        description="How long cached verification entries live before auto-expiry",
+    )
+    verification_cache_max_payload_kb: int = Field(
+        default=64,
+        description="Reject POST /v1/verification_cache with payloads above this cap",
+    )
+
+    # ---- Analysis atom tier policy ------------------------------------------
+    # These are read live from the AI platform dashboard via RuntimeConfigClient
+    # (keys: brain.atom.silver_ttl_days, brain.atom.bronze_ttl_days,
+    # brain.atom.confidence_silver_threshold). The values below are fallbacks
+    # used at startup until the first dashboard poll completes (~30s).
+    atom_silver_ttl_days: int = Field(
+        default=90,
+        description="TTL for LLM-cached analysis atoms (silver tier)",
+    )
+    atom_bronze_ttl_days: int = Field(
+        default=30,
+        description="TTL for low-confidence atoms (never served, kept for audit)",
+    )
+    atom_confidence_silver_threshold: float = Field(
+        default=60.0,
+        description="LLM confidence ≥ this stores atom as silver, else bronze",
+    )
+
+    # ---- Logging ------------------------------------------------------------
+    log_level: str = Field(default="INFO")
+
+    # ---- Runtime config (live polling from AI platform dashboard) -----------
+    dashboard_url: str | None = Field(
+        default=None,
+        description=(
+            "Optional dashboard base URL (e.g. http://didiAI-dashboard:51300). "
+            "When set, RuntimeConfigClient polls /api/config every 30s for live "
+            "overrides on atom_* and log_level."
+        ),
+    )
+
+    # ---- Computed -----------------------------------------------------------
+    @property
+    def postgres_dsn(self) -> str:
+        """Async-compatible DSN for direct asyncpg connections."""
+        return (
+            f"postgresql://{self.postgres_user}:{self.postgres_password}"
+            f"@{self.postgres_host}:{self.postgres_internal_port}/{self.postgres_db}"
+        )
+
+    @property
+    def llamacpp_urls_list(self) -> list[str]:
+        return [u.strip() for u in self.llm_llamacpp_urls.split(",") if u.strip()]
+
+    def model_for(self, role: LlmRole) -> tuple[str, str] | None:
+        """Return (model_id, backend_hint) for a logical role, or None if disabled."""
+        if role == LlmRole.REASONING:
+            return (self.model_reasoning, self.model_reasoning_backend)
+        if role == LlmRole.FAST and self.model_fast_enabled:
+            return (self.model_fast, self.model_fast_backend)
+        if role == LlmRole.VISION and self.model_vision_enabled:
+            return (self.model_vision, "external")
+        return None
+
+    @field_validator("log_level")
+    @classmethod
+    def _validate_log_level(cls, v: str) -> str:
+        v = v.upper()
+        if v not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}:
+            raise ValueError(f"invalid log_level: {v}")
+        return v
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Singleton accessor. Cached so .env is parsed only once per process."""
+    return Settings()
+
+
+# Convenience: most code can `from shared.config import settings`
+settings = get_settings()

ai_platform/modules/didi_brain/shared/embedding_client.py

@@ -0,0 +1,203 @@
+"""Embedding + reranking client.
+
+- Embeddings: BAAI/bge-m3 via vLLM OpenAI-compat endpoint, 1024-dim, 8K context.
+- Reranker: BAAI/bge-reranker-v2-m3 cross-encoder via vllm-rerank-api endpoint.
+
+Both run on the same host (10.11.10.15) on different ports (8200 / 8100).
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from shared.config import settings
+from shared.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class EmbeddingError(Exception):
+    pass
+
+
+@dataclass(slots=True, frozen=True)
+class RerankResult:
+    """One reranked document, sorted by score descending."""
+
+    index: int  # original position in the input documents list
+    score: float  # cross-encoder score (higher = more relevant)
+    document: str  # the text of that document
+
+
+class EmbeddingClient:
+    """Async client for BGE-M3 embeddings + BGE-reranker-v2-m3 reranking."""
+
+    def __init__(
+        self,
+        *,
+        embed_url: str | None = None,
+        rerank_url: str | None = None,
+        timeout: float = 60.0,
+        max_attempts: int = 3,
+    ):
+        self._embed_base = (embed_url or settings.embedding_url).rstrip("/")
+        self._rerank_base = (rerank_url or settings.reranker_url).rstrip("/")
+        self._max_attempts = max_attempts
+
+        embed_headers = {"Content-Type": "application/json"}
+        if settings.embedding_api_key:
+            embed_headers["Authorization"] = f"Bearer {settings.embedding_api_key}"
+        rerank_headers = {"Content-Type": "application/json"}
+        if settings.reranker_api_key:
+            rerank_headers["Authorization"] = f"Bearer {settings.reranker_api_key}"
+
+        self._embed_http = httpx.AsyncClient(
+            base_url=self._embed_base,
+            headers=embed_headers,
+            timeout=httpx.Timeout(timeout, connect=10.0),
+        )
+        self._rerank_http = httpx.AsyncClient(
+            base_url=self._rerank_base,
+            headers=rerank_headers,
+            timeout=httpx.Timeout(timeout, connect=10.0),
+        )
+
+    async def aclose(self) -> None:
+        await self._embed_http.aclose()
+        await self._rerank_http.aclose()
+
+    async def __aenter__(self) -> EmbeddingClient:
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        await self.aclose()
+
+    # ------------------------------------------------------------------ embed
+    async def embed(self, texts: Sequence[str]) -> list[list[float]]:
+        """Embed a batch of texts. Returns 1024-dim vectors in input order.
+
+        BGE-M3 max input is 8192 tokens; we don't truncate here — caller is
+        responsible for chunking long inputs (Atomic does this server-side).
+        Empty input returns empty list.
+        """
+        if not texts:
+            return []
+
+        payload = {"model": settings.embedding_model, "input": list(texts)}
+
+        async for attempt in AsyncRetrying(
+            stop=stop_after_attempt(self._max_attempts),
+            wait=wait_exponential(multiplier=1, min=1, max=10),
+            retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)),
+            reraise=True,
+        ):
+            with attempt:
+                resp = await self._embed_http.post("/v1/embeddings", json=payload)
+                if resp.status_code >= 400:
+                    raise EmbeddingError(
+                        f"HTTP {resp.status_code} from embedding endpoint: "
+                        f"{resp.text[:500]}"
+                    )
+                data = resp.json()
+                items = data.get("data")
+                if not items or len(items) != len(texts):
+                    raise EmbeddingError(
+                        f"Embedding count mismatch: got {len(items or [])} for "
+                        f"{len(texts)} inputs"
+                    )
+                vecs = [item["embedding"] for item in items]
+                if vecs and len(vecs[0]) != settings.embedding_dim:
+                    raise EmbeddingError(
+                        f"Embedding dim mismatch: got {len(vecs[0])}, "
+                        f"expected {settings.embedding_dim}"
+                    )
+                return vecs
+        raise EmbeddingError("retry loop exited without result")  # unreachable
+
+    async def embed_one(self, text: str) -> list[float]:
+        """Convenience for single-text embedding."""
+        result = await self.embed([text])
+        return result[0]
+
+    # ----------------------------------------------------------------- rerank
+    async def rerank(
+        self,
+        query: str,
+        documents: Sequence[str],
+        *,
+        top_n: int | None = None,
+    ) -> list[RerankResult]:
+        """Cross-encoder rerank query × documents. Returns sorted by score desc.
+
+        Use this AFTER first-stage embedding retrieval to dramatically boost
+        precision on the top-N candidates (typical pattern: embed retrieves 50,
+        rerank picks 10).
+        """
+        if not documents:
+            return []
+
+        payload: dict[str, Any] = {
+            "model": settings.reranker_model,
+            "query": query,
+            "documents": list(documents),
+        }
+        if top_n is not None:
+            payload["top_n"] = top_n
+
+        async for attempt in AsyncRetrying(
+            stop=stop_after_attempt(self._max_attempts),
+            wait=wait_exponential(multiplier=1, min=1, max=10),
+            retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)),
+            reraise=True,
+        ):
+            with attempt:
+                resp = await self._rerank_http.post("/v1/rerank", json=payload)
+                if resp.status_code >= 400:
+                    raise EmbeddingError(
+                        f"HTTP {resp.status_code} from reranker: {resp.text[:500]}"
+                    )
+                data = resp.json()
+                results = data.get("results", [])
+                return [
+                    RerankResult(
+                        index=int(r["index"]),
+                        score=float(r["score"]),
+                        document=r.get("document") or documents[int(r["index"])],
+                    )
+                    for r in results
+                ]
+        raise EmbeddingError("retry loop exited without result")  # unreachable
+
+
+# --------------------------------------------------------------------- math util
+def cosine(a: Sequence[float], b: Sequence[float]) -> float:
+    """Cosine similarity. BGE-M3 vectors are NOT pre-normalized; we compute it."""
+    if len(a) != len(b):
+        raise ValueError(f"vector dim mismatch: {len(a)} vs {len(b)}")
+    dot = sum(x * y for x, y in zip(a, b, strict=True))
+    na = math.sqrt(sum(x * x for x in a))
+    nb = math.sqrt(sum(x * x for x in b))
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+@asynccontextmanager
+async def embedding_client() -> AsyncIterator[EmbeddingClient]:
+    client = EmbeddingClient()
+    try:
+        yield client
+    finally:
+        await client.aclose()

ai_platform/modules/didi_brain/shared/llm_client.py

@@ -0,0 +1,260 @@
+"""LLM client — async wrapper over the OpenAI-compat router on :14011.
+
+Design:
+- One LlmClient instance per long-running process (reusable httpx.AsyncClient).
+- Roles (REASONING / FAST / VISION) decide model + backend automatically.
+- Hard timeouts and retries are enforced; no LLM call hangs forever.
+- Helpers for the two patterns we use most: structured JSON and single-token NLI.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+import httpx
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from shared.config import LlmRole, settings
+from shared.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class LlmError(Exception):
+    """Wraps any failure from the LLM stack with enough context to debug."""
+
+    def __init__(self, message: str, *, status: int | None = None, body: str | None = None):
+        super().__init__(message)
+        self.status = status
+        self.body = body
+
+
+class RoleNotAvailable(LlmError):
+    """The requested role has no enabled model behind it."""
+
+
+class LlmClient:
+    """Async OpenAI-compat client targeting the unified router."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str | None = None,
+        timeout: float = 120.0,
+        max_attempts: int = 3,
+    ):
+        self._base_url = (base_url or settings.llm_router_url).rstrip("/")
+        self._timeout = timeout
+        self._max_attempts = max_attempts
+        headers = {"Content-Type": "application/json"}
+        if settings.llm_router_api_key:
+            headers["Authorization"] = f"Bearer {settings.llm_router_api_key}"
+        self._http = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers=headers,
+            timeout=httpx.Timeout(timeout, connect=10.0),
+        )
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> LlmClient:
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        await self.aclose()
+
+    # ------------------------------------------------------------------ core
+    async def chat(
+        self,
+        *,
+        role: LlmRole = LlmRole.REASONING,
+        messages: list[dict[str, str]],
+        temperature: float = 0.1,
+        max_tokens: int = 1024,
+        stop: list[str] | None = None,
+        extra: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Raw chat completion. Returns the parsed response dict.
+
+        The response shape is OpenAI-compatible:
+            {"choices": [{"message": {"content": "..."}}], "usage": {...}, "backend": "..."}
+        """
+        model_info = settings.model_for(role)
+        if not model_info:
+            raise RoleNotAvailable(f"Role {role.value} has no enabled model")
+        model_id, backend = model_info
+
+        payload: dict[str, Any] = {
+            "model": model_id,
+            "backend": backend,
+            "messages": messages,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+        }
+        if stop:
+            payload["stop"] = stop
+        if extra:
+            payload.update(extra)
+
+        async for attempt in AsyncRetrying(
+            stop=stop_after_attempt(self._max_attempts),
+            wait=wait_exponential(multiplier=1, min=1, max=10),
+            retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)),
+            reraise=True,
+        ):
+            with attempt:
+                resp = await self._http.post("/v1/chat/completions", json=payload)
+                if resp.status_code >= 400:
+                    raise LlmError(
+                        f"HTTP {resp.status_code} from LLM router",
+                        status=resp.status_code,
+                        body=resp.text[:1000],
+                    )
+                data = resp.json()
+                if "choices" not in data:
+                    raise LlmError(
+                        "LLM response missing 'choices'",
+                        body=json.dumps(data)[:1000],
+                    )
+                return data
+        raise LlmError("retry loop exited without result")  # unreachable
+
+    # ------------------------------------------------------------------ helpers
+    async def chat_text(
+        self,
+        *,
+        role: LlmRole = LlmRole.REASONING,
+        system: str | None = None,
+        user: str,
+        temperature: float = 0.1,
+        max_tokens: int = 1024,
+        stop: list[str] | None = None,
+    ) -> tuple[str, dict[str, Any]]:
+        """Convenience: send (system?, user) → return (text, usage_dict)."""
+        msgs: list[dict[str, str]] = []
+        if system:
+            msgs.append({"role": "system", "content": system})
+        msgs.append({"role": "user", "content": user})
+        data = await self.chat(
+            role=role,
+            messages=msgs,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            stop=stop,
+        )
+        text = data["choices"][0]["message"]["content"]
+        usage = data.get("usage", {}) | {"backend": data.get("backend", "")}
+        return text, usage
+
+    async def chat_json(
+        self,
+        *,
+        role: LlmRole = LlmRole.REASONING,
+        system: str | None = None,
+        user: str,
+        max_tokens: int = 2048,
+        temperature: float = 0.0,
+    ) -> tuple[dict[str, Any] | list[Any], dict[str, Any]]:
+        """Send a request expected to return JSON. Strips fences if present.
+
+        Raises LlmError if the response is not parseable as JSON.
+        """
+        text, usage = await self.chat_text(
+            role=role,
+            system=system,
+            user=user,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        cleaned = _extract_json(text)
+        try:
+            return json.loads(cleaned), usage
+        except json.JSONDecodeError as e:
+            raise LlmError(
+                f"LLM did not return valid JSON: {e}",
+                body=text[:1000],
+            ) from e
+
+    async def chat_label(
+        self,
+        *,
+        role: LlmRole = LlmRole.REASONING,
+        system: str,
+        user: str,
+        allowed: list[str],
+        max_tokens: int = 16,
+    ) -> tuple[str, dict[str, Any]]:
+        """Single-label classification. Returns the matched label uppercased.
+
+        Useful for NLI (SUPPORT/CONTRADICT/NEUTRAL), credibility tiers, etc.
+        Raises LlmError if no allowed label is found in the response.
+        """
+        text, usage = await self.chat_text(
+            role=role,
+            system=system,
+            user=user,
+            temperature=0.0,
+            max_tokens=max_tokens,
+        )
+        upper = text.upper()
+        for label in allowed:
+            if label.upper() in upper:
+                return label.upper(), usage
+        raise LlmError(
+            f"LLM response did not contain any allowed label {allowed}",
+            body=text[:500],
+        )
+
+    # ------------------------------------------------------------------ health
+    async def list_models(self) -> list[dict[str, Any]]:
+        resp = await self._http.get("/v1/models")
+        resp.raise_for_status()
+        data = resp.json()
+        return data.get("data", [])
+
+    async def list_backends(self) -> list[str]:
+        try:
+            resp = await self._http.get("/v1/backends")
+            resp.raise_for_status()
+            return resp.json().get("backends", [])
+        except httpx.HTTPError:
+            return []
+
+
+# --------------------------------------------------------------------- utilities
+_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL)
+
+
+def _extract_json(text: str) -> str:
+    """Strip markdown code fences and isolate the JSON object/array if needed."""
+    text = text.strip()
+    m = _FENCE_RE.search(text)
+    if m:
+        return m.group(1).strip()
+    # Find first { or [ and last matching close
+    starts = [text.find("{"), text.find("[")]
+    starts = [s for s in starts if s >= 0]
+    if not starts:
+        return text
+    start = min(starts)
+    return text[start:].strip()
+
+
+@asynccontextmanager
+async def llm_client() -> AsyncIterator[LlmClient]:
+    """`async with llm_client() as llm:` for short-lived scripts."""
+    client = LlmClient()
+    try:
+        yield client
+    finally:
+        await client.aclose()

ai_platform/modules/didi_brain/shared/logging.py

@@ -0,0 +1,50 @@
+"""Structured logging via structlog. Single setup function called from main scripts."""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+import structlog
+
+from shared.config import settings
+
+
+def setup_logging(level: str | None = None) -> structlog.stdlib.BoundLogger:
+    """Configure structlog + stdlib logging once. Returns a base logger."""
+    log_level = (level or settings.log_level).upper()
+
+    # Force UTF-8 stdout on Windows so Romanian/Cyrillic/etc. don't crash rich.
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")  # type: ignore[union-attr]
+        sys.stderr.reconfigure(encoding="utf-8")  # type: ignore[union-attr]
+    except (AttributeError, OSError):
+        pass
+
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=getattr(logging, log_level),
+    )
+
+    # Silence overly chatty third-party loggers (httpx prints every request).
+    for noisy in ("httpx", "httpcore", "urllib3"):
+        logging.getLogger(noisy).setLevel(logging.WARNING)
+
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso", utc=True),
+            structlog.dev.ConsoleRenderer(colors=False),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)),
+        cache_logger_on_first_use=True,
+    )
+
+    return structlog.get_logger()
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    """Get a logger; setup_logging() must have been called once first."""
+    return structlog.get_logger(name) if name else structlog.get_logger()

ai_platform/modules/didi_brain/shared/taxonomy.py

@@ -0,0 +1,262 @@
+"""Canonical tag taxonomy for DidiBrain.
+
+This module is the SINGLE SOURCE OF TRUTH for what tags exist in the brain.
+All scrapers, extractors, and importers reference these names — never make up
+ad-hoc tags.
+
+The structure is a nested dict where each key is either:
+  - a leaf (value is None), or
+  - a subtree (value is another dict)
+
+The seeder script (scripts/04_seed_taxonomy.py) walks this tree and creates
+any missing tags in Atomic. It is idempotent.
+
+Atomic creates 5 default root tags at first boot: Topics, People, Locations,
+Organizations, Events. We REUSE the "Topics" root by extending it with our
+own children — the rest stay as-is for compatibility with Atomic's optional
+auto-tagging (which we keep disabled but won't fight).
+
+After seeding, scripts/04 writes a flat map of "Path/Like/This → tag_uuid"
+to shared/_tag_ids.json, which downstream code reads via TagResolver.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+# A node is either a dict (subtree) or None (leaf).
+TaxonomyTree = dict[str, "TaxonomyTree | None"]
+
+# ---------------------------------------------------------------- the spec
+# Order in this dict is the order tags will be created in Atomic.
+TAXONOMY: TaxonomyTree = {
+    # Reuse Atomic's existing "Topics" root and extend it with our hierarchy.
+    "Topics": {
+        "Health": {
+            "Vaccines": None,
+            "COVID": None,
+            "Disease": None,
+            "Medicine": None,
+            "PublicHealth": None,
+        },
+        "Politics": {
+            "Elections": None,
+            "Diplomacy": None,
+            "War": None,
+            "Government": None,
+        },
+        "Climate": None,
+        "Technology": None,
+        "Economy": None,
+        "Society": None,
+    },
+
+    # Country of origin for the source / event.
+    "Country": {
+        "Romania": None,
+        "USA": None,
+        "Russia": None,
+        "Ukraine": None,
+        "UK": None,
+        "France": None,
+        "Germany": None,
+        "Spain": None,
+        "Italy": None,
+        "Poland": None,
+        "Moldova": None,
+        "Global": None,  # for transnational / multi-country items
+    },
+
+    # What kind of source the atom came from.
+    "SourceType": {
+        "Wikipedia": None,
+        "MainstreamMedia": None,
+        "StateMedia": None,        # state-affiliated outlets (TASS, RT, Sputnik, Xinhua...)
+        "TabloidMedia": None,
+        "FactCheck": None,         # Snopes, PolitiFact, AFP FC, Veridica, Funky...
+        "Government": None,        # gov.ro, whitehouse.gov, who.int...
+        "ScientificJournal": None, # peer-reviewed
+        "SocialMedia": None,
+        "Blog": None,
+        "Forum": None,
+    },
+
+    # Editorial credibility tier — applied by scraper from a static registry.
+    "Credibility": {
+        "Tier1": None,           # Reuters/AP/BBC class
+        "Tier2": None,           # major mainstream
+        "Tier3": None,           # weaker mainstream / tabloid
+        "StateAffiliated": None,
+        "KnownDisinfo": None,    # known disinfo outlets (we do still ingest these)
+        "Unknown": None,
+    },
+
+    # Primary language of the atom content.
+    "Language": {
+        "RO": None,
+        "EN": None,
+        "RU": None,
+        "UA": None,
+        "FR": None,
+        "DE": None,
+        "ES": None,
+        "IT": None,
+        "PL": None,
+    },
+
+    # What KIND of atom this is (vs "what topic" — that's Topics).
+    "Type": {
+        "Document": None,       # full scraped article
+        "Claim": None,          # extracted atomic claim
+        "Quote": None,          # verbatim quote/excerpt
+        "Summary": None,        # synthesized summary
+        "Annotation": None,     # human/AI annotation about another atom
+    },
+
+    # Stance the source itself takes toward the central claim of the document.
+    "Stance": {
+        "Asserts": None,        # source presents it as fact
+        "Reports": None,        # source describes it as someone else's claim
+        "Refutes": None,        # source disagrees / debunks
+        "Questions": None,      # source raises doubts but doesn't refute
+        "Neutral": None,        # purely informational, no stance
+    },
+
+    # Verification status of a Claim atom — populated by Didi after analysis,
+    # not at ingest. Documents stay un-tagged here.
+    "ClaimStatus": {
+        "Confirmed": None,
+        "Disputed": None,
+        "Debunked": None,
+        "Unverified": None,
+        "PartiallyTrue": None,
+    },
+}
+
+# ============================================================ flatten helpers
+
+
+def walk(tree: TaxonomyTree, parent_path: str = "") -> Iterator[tuple[str, str | None, str]]:
+    """Yield (full_path, parent_path_or_None, name) for every node, depth-first.
+
+    Example output for {"A": {"B": None}}:
+        ("A", None, "A")
+        ("A/B", "A", "B")
+    """
+    for name, children in tree.items():
+        path = f"{parent_path}/{name}" if parent_path else name
+        yield (path, parent_path or None, name)
+        if children:
+            yield from walk(children, path)
+
+
+def all_paths(tree: TaxonomyTree | None = None) -> list[str]:
+    """All canonical paths in the taxonomy, in creation order."""
+    return [p for p, _, _ in walk(tree if tree is not None else TAXONOMY)]
+
+
+# ============================================================ tag id resolver
+
+_DEFAULT_CACHE = Path(__file__).resolve().parent / "_tag_ids.json"
+
+
+class TagResolver:
+    """Resolves canonical tag paths to Atomic UUIDs.
+
+    Loads from a JSON cache file written by the seeder. If the file is
+    missing or stale, callers should re-run scripts/04_seed_taxonomy.py.
+    """
+
+    def __init__(self, cache_path: Path | None = None):
+        self._cache_path = cache_path or _DEFAULT_CACHE
+        self._map: dict[str, str] = {}
+        if self._cache_path.exists():
+            self._map = json.loads(self._cache_path.read_text(encoding="utf-8"))
+
+    def __contains__(self, path: str) -> bool:
+        return path in self._map
+
+    def get(self, path: str) -> str | None:
+        return self._map.get(path)
+
+    def require(self, path: str) -> str:
+        v = self._map.get(path)
+        if not v:
+            raise KeyError(
+                f"Tag path {path!r} not in resolver cache at {self._cache_path}. "
+                f"Run scripts/04_seed_taxonomy.py."
+            )
+        return v
+
+    def ids_for(self, paths: list[str], *, ignore_missing: bool = False) -> list[str]:
+        ids: list[str] = []
+        missing: list[str] = []
+        for p in paths:
+            v = self._map.get(p)
+            if v:
+                ids.append(v)
+            else:
+                missing.append(p)
+        if missing and not ignore_missing:
+            raise KeyError(f"Missing tag paths: {missing}")
+        return ids
+
+    @property
+    def all(self) -> dict[str, str]:
+        return dict(self._map)
+
+    def save(self, mapping: dict[str, str]) -> None:
+        self._cache_path.write_text(
+            json.dumps(mapping, indent=2, ensure_ascii=False, sort_keys=True),
+            encoding="utf-8",
+        )
+        self._map = mapping
+
+    def load_from_mapping(self, mapping: dict[str, str]) -> None:
+        """Replace the in-memory map without touching disk.
+
+        Used by long-running services (e.g. containerized brain_api) that
+        refresh the resolver from Atomic at startup, so they don't need
+        the _tag_ids.json file baked into the image.
+        """
+        self._map = dict(mapping)
+
+
+def build_path_map_from_tags(tags: list[dict[str, Any]]) -> dict[str, str]:
+    """Convert Atomic's flat tag list (each with parent_id) into path → id map.
+
+    Atomic /api/tags returns each tag with id, name, parent_id, and a nested
+    children list. We don't trust the children list (depth may be limited)
+    and instead walk parent_id chains ourselves.
+    """
+    by_id: dict[str, dict[str, Any]] = {}
+
+    def collect(items: list[dict[str, Any]]) -> None:
+        for t in items:
+            tid = t.get("id")
+            if not tid:
+                continue
+            by_id[tid] = t
+            kids = t.get("children") or []
+            if kids:
+                collect(kids)
+
+    collect(tags)
+
+    def path_for(tid: str) -> str:
+        parts: list[str] = []
+        cur: str | None = tid
+        seen: set[str] = set()
+        while cur and cur not in seen:
+            seen.add(cur)
+            t = by_id.get(cur)
+            if not t:
+                break
+            parts.append(t.get("name", "?"))
+            cur = t.get("parent_id")
+        return "/".join(reversed(parts))
+
+    return {path_for(tid): tid for tid in by_id}