Livrare LOT 1 - Didi
This commit is contained in:
commit
5380c3fc63
990 changed files with 133308 additions and 0 deletions
262
ai_platform/modules/didi_brain/shared/taxonomy.py
Normal file
262
ai_platform/modules/didi_brain/shared/taxonomy.py
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
"""Canonical tag taxonomy for DidiBrain.
|
||||
|
||||
This module is the SINGLE SOURCE OF TRUTH for what tags exist in the brain.
|
||||
All scrapers, extractors, and importers reference these names — never make up
|
||||
ad-hoc tags.
|
||||
|
||||
The structure is a nested dict where each key is either:
|
||||
- a leaf (value is None), or
|
||||
- a subtree (value is another dict)
|
||||
|
||||
The seeder script (scripts/04_seed_taxonomy.py) walks this tree and creates
|
||||
any missing tags in Atomic. It is idempotent.
|
||||
|
||||
Atomic creates 5 default root tags at first boot: Topics, People, Locations,
|
||||
Organizations, Events. We REUSE the "Topics" root by extending it with our
|
||||
own children — the rest stay as-is for compatibility with Atomic's optional
|
||||
auto-tagging (which we keep disabled but won't fight).
|
||||
|
||||
After seeding, scripts/04 writes a flat map of "Path/Like/This → tag_uuid"
|
||||
to shared/_tag_ids.json, which downstream code reads via TagResolver.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# A node is either a dict (subtree) or None (leaf).
|
||||
TaxonomyTree = dict[str, "TaxonomyTree | None"]
|
||||
|
||||
# ---------------------------------------------------------------- the spec
|
||||
# Order in this dict is the order tags will be created in Atomic.
|
||||
TAXONOMY: TaxonomyTree = {
|
||||
# Reuse Atomic's existing "Topics" root and extend it with our hierarchy.
|
||||
"Topics": {
|
||||
"Health": {
|
||||
"Vaccines": None,
|
||||
"COVID": None,
|
||||
"Disease": None,
|
||||
"Medicine": None,
|
||||
"PublicHealth": None,
|
||||
},
|
||||
"Politics": {
|
||||
"Elections": None,
|
||||
"Diplomacy": None,
|
||||
"War": None,
|
||||
"Government": None,
|
||||
},
|
||||
"Climate": None,
|
||||
"Technology": None,
|
||||
"Economy": None,
|
||||
"Society": None,
|
||||
},
|
||||
|
||||
# Country of origin for the source / event.
|
||||
"Country": {
|
||||
"Romania": None,
|
||||
"USA": None,
|
||||
"Russia": None,
|
||||
"Ukraine": None,
|
||||
"UK": None,
|
||||
"France": None,
|
||||
"Germany": None,
|
||||
"Spain": None,
|
||||
"Italy": None,
|
||||
"Poland": None,
|
||||
"Moldova": None,
|
||||
"Global": None, # for transnational / multi-country items
|
||||
},
|
||||
|
||||
# What kind of source the atom came from.
|
||||
"SourceType": {
|
||||
"Wikipedia": None,
|
||||
"MainstreamMedia": None,
|
||||
"StateMedia": None, # state-affiliated outlets (TASS, RT, Sputnik, Xinhua...)
|
||||
"TabloidMedia": None,
|
||||
"FactCheck": None, # Snopes, PolitiFact, AFP FC, Veridica, Funky...
|
||||
"Government": None, # gov.ro, whitehouse.gov, who.int...
|
||||
"ScientificJournal": None, # peer-reviewed
|
||||
"SocialMedia": None,
|
||||
"Blog": None,
|
||||
"Forum": None,
|
||||
},
|
||||
|
||||
# Editorial credibility tier — applied by scraper from a static registry.
|
||||
"Credibility": {
|
||||
"Tier1": None, # Reuters/AP/BBC class
|
||||
"Tier2": None, # major mainstream
|
||||
"Tier3": None, # weaker mainstream / tabloid
|
||||
"StateAffiliated": None,
|
||||
"KnownDisinfo": None, # known disinfo outlets (we do still ingest these)
|
||||
"Unknown": None,
|
||||
},
|
||||
|
||||
# Primary language of the atom content.
|
||||
"Language": {
|
||||
"RO": None,
|
||||
"EN": None,
|
||||
"RU": None,
|
||||
"UA": None,
|
||||
"FR": None,
|
||||
"DE": None,
|
||||
"ES": None,
|
||||
"IT": None,
|
||||
"PL": None,
|
||||
},
|
||||
|
||||
# What KIND of atom this is (vs "what topic" — that's Topics).
|
||||
"Type": {
|
||||
"Document": None, # full scraped article
|
||||
"Claim": None, # extracted atomic claim
|
||||
"Quote": None, # verbatim quote/excerpt
|
||||
"Summary": None, # synthesized summary
|
||||
"Annotation": None, # human/AI annotation about another atom
|
||||
},
|
||||
|
||||
# Stance the source itself takes toward the central claim of the document.
|
||||
"Stance": {
|
||||
"Asserts": None, # source presents it as fact
|
||||
"Reports": None, # source describes it as someone else's claim
|
||||
"Refutes": None, # source disagrees / debunks
|
||||
"Questions": None, # source raises doubts but doesn't refute
|
||||
"Neutral": None, # purely informational, no stance
|
||||
},
|
||||
|
||||
# Verification status of a Claim atom — populated by Didi after analysis,
|
||||
# not at ingest. Documents stay un-tagged here.
|
||||
"ClaimStatus": {
|
||||
"Confirmed": None,
|
||||
"Disputed": None,
|
||||
"Debunked": None,
|
||||
"Unverified": None,
|
||||
"PartiallyTrue": None,
|
||||
},
|
||||
}
|
||||
|
||||
# ============================================================ flatten helpers
|
||||
|
||||
|
||||
def walk(tree: TaxonomyTree, parent_path: str = "") -> Iterator[tuple[str, str | None, str]]:
|
||||
"""Yield (full_path, parent_path_or_None, name) for every node, depth-first.
|
||||
|
||||
Example output for {"A": {"B": None}}:
|
||||
("A", None, "A")
|
||||
("A/B", "A", "B")
|
||||
"""
|
||||
for name, children in tree.items():
|
||||
path = f"{parent_path}/{name}" if parent_path else name
|
||||
yield (path, parent_path or None, name)
|
||||
if children:
|
||||
yield from walk(children, path)
|
||||
|
||||
|
||||
def all_paths(tree: TaxonomyTree | None = None) -> list[str]:
|
||||
"""All canonical paths in the taxonomy, in creation order."""
|
||||
return [p for p, _, _ in walk(tree if tree is not None else TAXONOMY)]
|
||||
|
||||
|
||||
# ============================================================ tag id resolver
|
||||
|
||||
_DEFAULT_CACHE = Path(__file__).resolve().parent / "_tag_ids.json"
|
||||
|
||||
|
||||
class TagResolver:
|
||||
"""Resolves canonical tag paths to Atomic UUIDs.
|
||||
|
||||
Loads from a JSON cache file written by the seeder. If the file is
|
||||
missing or stale, callers should re-run scripts/04_seed_taxonomy.py.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_path: Path | None = None):
|
||||
self._cache_path = cache_path or _DEFAULT_CACHE
|
||||
self._map: dict[str, str] = {}
|
||||
if self._cache_path.exists():
|
||||
self._map = json.loads(self._cache_path.read_text(encoding="utf-8"))
|
||||
|
||||
def __contains__(self, path: str) -> bool:
|
||||
return path in self._map
|
||||
|
||||
def get(self, path: str) -> str | None:
|
||||
return self._map.get(path)
|
||||
|
||||
def require(self, path: str) -> str:
|
||||
v = self._map.get(path)
|
||||
if not v:
|
||||
raise KeyError(
|
||||
f"Tag path {path!r} not in resolver cache at {self._cache_path}. "
|
||||
f"Run scripts/04_seed_taxonomy.py."
|
||||
)
|
||||
return v
|
||||
|
||||
def ids_for(self, paths: list[str], *, ignore_missing: bool = False) -> list[str]:
|
||||
ids: list[str] = []
|
||||
missing: list[str] = []
|
||||
for p in paths:
|
||||
v = self._map.get(p)
|
||||
if v:
|
||||
ids.append(v)
|
||||
else:
|
||||
missing.append(p)
|
||||
if missing and not ignore_missing:
|
||||
raise KeyError(f"Missing tag paths: {missing}")
|
||||
return ids
|
||||
|
||||
@property
|
||||
def all(self) -> dict[str, str]:
|
||||
return dict(self._map)
|
||||
|
||||
def save(self, mapping: dict[str, str]) -> None:
|
||||
self._cache_path.write_text(
|
||||
json.dumps(mapping, indent=2, ensure_ascii=False, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
self._map = mapping
|
||||
|
||||
def load_from_mapping(self, mapping: dict[str, str]) -> None:
|
||||
"""Replace the in-memory map without touching disk.
|
||||
|
||||
Used by long-running services (e.g. containerized brain_api) that
|
||||
refresh the resolver from Atomic at startup, so they don't need
|
||||
the _tag_ids.json file baked into the image.
|
||||
"""
|
||||
self._map = dict(mapping)
|
||||
|
||||
|
||||
def build_path_map_from_tags(tags: list[dict[str, Any]]) -> dict[str, str]:
|
||||
"""Convert Atomic's flat tag list (each with parent_id) into path → id map.
|
||||
|
||||
Atomic /api/tags returns each tag with id, name, parent_id, and a nested
|
||||
children list. We don't trust the children list (depth may be limited)
|
||||
and instead walk parent_id chains ourselves.
|
||||
"""
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def collect(items: list[dict[str, Any]]) -> None:
|
||||
for t in items:
|
||||
tid = t.get("id")
|
||||
if not tid:
|
||||
continue
|
||||
by_id[tid] = t
|
||||
kids = t.get("children") or []
|
||||
if kids:
|
||||
collect(kids)
|
||||
|
||||
collect(tags)
|
||||
|
||||
def path_for(tid: str) -> str:
|
||||
parts: list[str] = []
|
||||
cur: str | None = tid
|
||||
seen: set[str] = set()
|
||||
while cur and cur not in seen:
|
||||
seen.add(cur)
|
||||
t = by_id.get(cur)
|
||||
if not t:
|
||||
break
|
||||
parts.append(t.get("name", "?"))
|
||||
cur = t.get("parent_id")
|
||||
return "/".join(reversed(parts))
|
||||
|
||||
return {path_for(tid): tid for tid in by_id}
|
||||
Loading…
Add table
Add a link
Reference in a new issue