clossers/didi-lot1-ai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Livrare LOT 1 - Didi

Browse source
This commit is contained in:
Dezvoltari Evotech 2026-06-25 14:13:25 -07:00
commit 5380c3fc63
990 changed files with 133308 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

49
ai_platform/modules/didi_brain/scheduler/Dockerfile Normal file
View file

@ -0,0 +1,49 @@
# syntax=docker/dockerfile:1.6
# =============================================================================
# didibrain-scheduler — feeder + auditor + watcher in a single container.
#
# Imports brain_api.* directly (same Python package) so the auditor can use
# DB pool + cache_judge + apply_judge_verdict without re-implementing them.
# Build context is the didibrain/ project root (one level up).
#
# docker build -t didibrain-scheduler -f scheduler/Dockerfile .
# =============================================================================
FROM python:3.12-slim-bookworm AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONIOENCODING=utf-8 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_ROOT_USER_ACTION=ignore
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY scheduler/requirements.txt /tmp/requirements.txt
RUN pip install --upgrade pip \
&& pip install -r /tmp/requirements.txt \
&& rm /tmp/requirements.txt
# Brain modules (auditor imports them directly).
COPY shared /app/shared
COPY extractor /app/extractor
COPY brain_api /app/brain_api
COPY scheduler /app/scheduler
RUN useradd --system --create-home --shell /bin/false brain \
&& chown -R brain:brain /app
USER brain
# Healthcheck: main.py touches /tmp/scheduler.healthy every 30s. If the
# file is older than 5 min, the container is considered unhealthy.
HEALTHCHECK --interval=60s --timeout=10s --start-period=60s --retries=3 \
CMD test -f /tmp/scheduler.healthy && \
test "$(( $(date +%s) - $(stat -c %Y /tmp/scheduler.healthy) ))" -lt 300 \
|| exit 1
CMD ["python", "-m", "scheduler.main"]

21
ai_platform/modules/didi_brain/scheduler/__init__.py Normal file
View file

@ -0,0 +1,21 @@
"""didibrain-scheduler — Phase C orchestrator.
Single container running three independent asyncio tasks for cache freshness
defense:
- **feeder** (Pilon 4) — pulls RSS feeds at volatility-aware intervals
(15min for volatile topics, 6h for evolving, 24h for stable) and POSTs
new articles to /v1/ingest so the brain corpus stays current.
- **auditor** (Pilon 5) — daily sweep of gold + silver atoms whose
last_audited_at is older than the audit interval, judging each against
fresh evidence and applying KEEP/INVALIDATE decisions.
- **watcher** (Pilon 9) — fast RSS pull on a small breaking-news feed set,
LLM-classifies each item to identify affected topics/entities, and
triggers /v1/cache/invalidate when the news affects cached verdicts.
Each task is independent and self-restarting — a failure in one does not
stop the others. All HTTP calls go to the brain-api container by Docker DNS
(``BRAIN_API_URL``, default ``http://brain-api:8090``).
"""
__version__ = "0.1.0"

311
ai_platform/modules/didi_brain/scheduler/auditor.py Normal file
View file

@ -0,0 +1,311 @@
"""Daily auditor — Pilon 5 of the cache freshness defense.
Sweeps non-stable cache rows whose ``last_audited_at`` is older than the
audit interval, runs the cache_judge against fresh evidence from a /v1/gather
call, and applies KEEP/INVALIDATE decisions. Atoms that survive consecutive
audits accumulate ``consecutive_audit_passes`` so the judge becomes harder
to flip them — trusted gold/silver gain inertia over time.
Pipeline per atom:
1. SELECT candidate atom from brain_analysis_atom
2. POST /v1/gather to brain to get fresh top-3 evidence for the claim
3. Extract cached_truth from result_processed (TRUE / FALSE / MIXED / UV)
4. cache_judge.judge_cache_validity(claim, cached_truth, evidence, ...)
5. analysis_atom.apply_judge_verdict(atom_id, verdict)
— INVALIDATE → expires_at = now()
— KEEP_CACHE → bump consecutive_audit_passes (rate-limited)
— NEEDS_FULL_RECHECK → no DB change, just logged
This task imports brain_api directly (same Python code) for DB pool access
and helper functions; HTTP is used only for gather (which orchestrates
search + rerank + NLI inside brain itself).
"""
from __future__ import annotations
import asyncio
import json
from datetime import datetime, timezone
from typing import Any
from brain_api.db import db
from brain_api.services.analysis_atom import (
apply_judge_verdict,
is_effectively_fresh,
)
from brain_api.services.cache_judge import (
EvidenceSnippet,
JudgeVerdict,
judge_cache_validity,
)
from scheduler.brain_client import BrainClient
from scheduler.config import settings
from shared.llm_client import LlmClient
from shared.logging import get_logger
log = get_logger(__name__)
def _extract_cached_truth(processed: dict | None) -> str:
"""Pull a cached_truth label from result_processed.
DIDI v1 schema for the claims component uses status: TRUE/FALSE/UV/OP/MIXED.
For techniques + ai_tampered the result is a structured score object
rather than a truth direction — those components don't benefit from the
NLI judge anyway (their result depends on the input text, not the world),
so we return UNVERIFIED to make the judge degrade to NEEDS_FULL_RECHECK
or KEEP_CACHE depending on evidence.
"""
if not isinstance(processed, dict):
return "UNVERIFIED"
raw = processed.get("status") or processed.get("verdict")
if not isinstance(raw, str):
return "UNVERIFIED"
s = raw.strip().upper()
if s in ("TRUE", "VT", "VERIFIED_TRUE"):
return "TRUE"
if s in ("FALSE", "VF", "VERIFIED_FALSE"):
return "FALSE"
if s in ("MIXED",):
return "MIXED"
return "UNVERIFIED"
async def _select_candidates(limit: int) -> list[dict[str, Any]]:
"""Pick atoms eligible for audit, oldest-first.
Eligibility:
- cache_tier IN ('gold','silver')
- volatility IS NOT NULL AND volatility != 'stable'
- last_audited_at IS NULL OR < now() - auditor_interval
- created_at < now() - min_age_hours (let new writes settle)
- expires_at IS NULL OR > now() (don't audit dead rows)
"""
if not db.pool:
raise RuntimeError("brain_db not connected")
sql = """
SELECT atom_id, content_hash, content_preview, component, tier,
cache_tier, volatility, topic_codes, llm_confidence,
result_processed, created_at, last_audited_at,
consecutive_audit_passes
FROM brain_analysis_atom
WHERE cache_tier IN ('gold', 'silver')
AND volatility IS NOT NULL
AND volatility <> 'stable'
AND (expires_at IS NULL OR expires_at > now())
AND created_at < now() - ($1 || ' hours')::interval
AND (
last_audited_at IS NULL
OR last_audited_at < now() - ($2 || ' seconds')::interval
)
ORDER BY last_audited_at NULLS FIRST, created_at ASC
LIMIT $3
"""
async with db.pool.acquire() as conn:
rows = await conn.fetch(
sql,
str(settings.auditor_min_age_hours),
str(settings.auditor_interval_s),
limit,
)
out: list[dict[str, Any]] = []
for r in rows:
result_processed = r["result_processed"]
if isinstance(result_processed, str):
result_processed = json.loads(result_processed)
out.append({
"atom_id": r["atom_id"],
"content_hash": r["content_hash"],
"content_preview": r["content_preview"] or "",
"component": r["component"],
"tier": r["tier"],
"cache_tier": r["cache_tier"],
"volatility": r["volatility"],
"topic_codes": list(r["topic_codes"]) if r["topic_codes"] else [],
"llm_confidence": (
float(r["llm_confidence"])
if r["llm_confidence"] is not None
else None
),
"result_processed": result_processed or {},
"created_at": r["created_at"],
"last_audited_at": r["last_audited_at"],
"consecutive_audit_passes": r["consecutive_audit_passes"],
})
return out
def _evidence_from_gather_response(resp: dict | None) -> list[EvidenceSnippet]:
"""Convert a brain /v1/gather response payload into EvidenceSnippet list."""
if not resp:
return []
items = resp.get("evidence") or []
out: list[EvidenceSnippet] = []
for it in items[:3]: # judge uses top 3 anyway
url = (it.get("url") or "").strip()
text = (
it.get("full_text")
or it.get("summary")
or it.get("snippet")
or ""
)
published = it.get("published_at")
if url and text:
out.append(EvidenceSnippet(url=url, text=text, published_at=published))
return out
async def _gather_fresh_evidence(
brain: BrainClient, *, claim: str, volatility: str
) -> list[EvidenceSnippet]:
"""Call /v1/gather over HTTP to get fresh evidence for the claim.
Disables NLI on this internal call (we run our own NLI via cache_judge
afterwards, so doing it twice would be wasteful).
"""
body = {
"claim": claim[:1500],
"max_evidence": 5,
"include_full_text": True,
"run_nli": False,
"volatility_hint": volatility,
}
try:
resp = await brain._http.post("/v1/gather", json=body)
if resp.status_code >= 400:
log.debug(
"auditor_gather_http_error",
status=resp.status_code,
)
return []
return _evidence_from_gather_response(resp.json())
except Exception as e: # noqa: BLE001
log.debug("auditor_gather_failed", error=f"{type(e).__name__}:{e}")
return []
async def _audit_one(
*,
candidate: dict[str, Any],
brain: BrainClient,
llm: LlmClient,
) -> str:
"""Run a single audit. Returns the decision label for telemetry."""
atom_id = candidate["atom_id"]
claim = candidate["content_preview"]
volatility = candidate["volatility"] or "evolving"
if not claim:
# Can't judge without a claim text — touch last_audited_at to defer
# this row and move on.
log.debug("auditor_no_claim_text", atom_id=atom_id)
return "SKIPPED"
cached_truth = _extract_cached_truth(candidate["result_processed"])
age_hours = (
datetime.now(tz=timezone.utc) - candidate["created_at"]
).total_seconds() / 3600.0
# Cheap path: if effective confidence is still high we don't even need
# to gather. Bumps consecutive_audit_passes via apply_judge_verdict.
eff_fresh = is_effectively_fresh(
base_confidence=candidate["llm_confidence"],
volatility=volatility,
age_hours=age_hours,
consecutive_audit_passes=candidate["consecutive_audit_passes"],
)
if not eff_fresh:
# Confidence has decayed below floor — auto-invalidate without
# spending an LLM call.
verdict = JudgeVerdict(
decision="INVALIDATE",
nli_skipped=True,
reasoning=(
f"effective confidence below floor "
f"(volatility={volatility}, age={age_hours:.0f}h)"
),
evaluated_at=datetime.now(tz=timezone.utc).isoformat(),
)
await apply_judge_verdict(atom_id, verdict)
log.info(
"auditor_invalidated_decay",
atom_id=atom_id,
volatility=volatility,
age_hours=round(age_hours, 1),
)
return "INVALIDATE_DECAY"
# Active path: gather + judge.
evidence = await _gather_fresh_evidence(
brain, claim=claim, volatility=volatility
)
verdict = await judge_cache_validity(
llm,
claim=claim,
cached_truth=cached_truth,
current_evidence=evidence,
volatility=volatility,
age_hours=age_hours,
consecutive_audit_passes=candidate["consecutive_audit_passes"],
)
await apply_judge_verdict(atom_id, verdict)
log.info(
"auditor_decision",
atom_id=atom_id,
decision=verdict.decision,
volatility=volatility,
cached_truth=cached_truth,
)
return verdict.decision
async def _run_one_cycle(brain: BrainClient, llm: LlmClient) -> dict[str, int]:
"""Pull candidates and audit them sequentially.
Sequential rather than parallel because each judge call hits the LLM
router — running 200 in parallel would saturate it. The local Qwen
handles ~2-4 concurrent requests well, so we could batch with a
semaphore later if cycle duration becomes an issue.
"""
candidates = await _select_candidates(settings.auditor_batch_limit)
if not candidates:
return {"audited": 0, "candidates": 0}
counts: dict[str, int] = {}
for c in candidates:
try:
decision = await _audit_one(candidate=c, brain=brain, llm=llm)
counts[decision] = counts.get(decision, 0) + 1
except Exception as e: # noqa: BLE001
log.warning(
"auditor_one_failed",
atom_id=c.get("atom_id"),
error=f"{type(e).__name__}:{e}",
)
counts["ERROR"] = counts.get("ERROR", 0) + 1
log.info(
"auditor_cycle_done",
audited=len(candidates),
breakdown=counts,
)
return {"audited": len(candidates), **counts}
async def run_auditor(brain: BrainClient, llm: LlmClient) -> None:
"""Long-running task — sweeps daily by default."""
if not settings.auditor_enabled:
log.info("auditor_disabled")
return
log.info(
"auditor_loop_start",
interval_s=settings.auditor_interval_s,
batch_limit=settings.auditor_batch_limit,
)
while True:
try:
await _run_one_cycle(brain, llm)
except Exception: # noqa: BLE001
log.exception("auditor_cycle_error")
await asyncio.sleep(settings.auditor_interval_s)

145
ai_platform/modules/didi_brain/scheduler/brain_client.py Normal file
View file

@ -0,0 +1,145 @@
"""HTTP client for the brain API.
Thin wrapper that the three scheduler tasks (feeder/auditor/watcher) use to
talk to didibrain-api. All calls have generous timeouts (some operations
internally trigger LLM calls + DB queries) and return None on failure so
the caller can decide whether to retry or just skip.
"""
from __future__ import annotations
from typing import Any
import httpx
from scheduler.config import settings
from shared.logging import get_logger
log = get_logger(__name__)
class BrainClient:
"""Async client for the brain HTTP API.
Reuse one instance per task — httpx.AsyncClient pools connections.
"""
def __init__(self, *, base_url: str | None = None) -> None:
self._base = (base_url or settings.brain_api_url).rstrip("/")
self._http = httpx.AsyncClient(
base_url=self._base,
timeout=httpx.Timeout(
settings.brain_api_timeout_s, connect=10.0
),
headers={"Content-Type": "application/json"},
)
async def aclose(self) -> None:
await self._http.aclose()
async def __aenter__(self) -> BrainClient:
return self
async def __aexit__(self, *args: Any) -> None:
await self.aclose()
# --------------------------------------------------------------- ingest
async def ingest(
self,
*,
claim: str | None,
evidence: list[dict],
default_tags: list[str] | None = None,
run_extraction: bool = True,
) -> dict | None:
"""POST /v1/ingest. Returns response dict or None on error.
Each evidence item should have: url, title, summary, full_text (opt),
publisher (opt), published_at (ISO str, opt).
"""
body = {
"claim": claim,
"evidence": evidence,
"default_tags": default_tags or [],
"run_extraction": run_extraction,
}
try:
resp = await self._http.post("/v1/ingest", json=body)
if resp.status_code >= 400:
log.warning(
"brain_ingest_http_error",
status=resp.status_code,
body=resp.text[:200],
)
return None
return resp.json()
except httpx.HTTPError as e:
log.warning("brain_ingest_failed", error=str(e))
return None
# ----------------------------------------------------------- invalidate
async def invalidate(
self,
*,
topic_codes: list[str] | None = None,
entity_canonicals: list[str] | None = None,
claim_pattern: str | None = None,
since_iso: str | None = None,
invalidate_gold: bool = False,
dry_run: bool = False,
actor: str = "scheduler",
reason: str | None = None,
) -> dict | None:
"""POST /v1/cache/invalidate. Returns counts dict or None on error."""
body: dict[str, Any] = {
"invalidate_gold": invalidate_gold,
"dry_run": dry_run,
"actor": actor,
}
if topic_codes:
body["topic_codes"] = topic_codes
if entity_canonicals:
body["entity_canonicals"] = entity_canonicals
if claim_pattern:
body["claim_pattern"] = claim_pattern
if since_iso:
body["since"] = since_iso
if reason:
body["reason"] = reason
try:
resp = await self._http.post("/v1/cache/invalidate", json=body)
if resp.status_code >= 400:
log.warning(
"brain_invalidate_http_error",
status=resp.status_code,
body=resp.text[:200],
)
return None
return resp.json()
except httpx.HTTPError as e:
log.warning("brain_invalidate_failed", error=str(e))
return None
# ------------------------------------------------------------ canonicalize
async def canonicalize(
self, *, claim: str, current_date: str | None = None
) -> dict | None:
"""POST /v1/canonicalize. Returns response dict or None on error."""
body: dict[str, Any] = {"claim": claim}
if current_date:
body["current_date"] = current_date
try:
resp = await self._http.post("/v1/canonicalize", json=body)
if resp.status_code >= 400:
return None
return resp.json()
except httpx.HTTPError:
return None
# ------------------------------------------------------------------ misc
async def health(self) -> bool:
try:
resp = await self._http.get("/health")
return resp.status_code == 200
except httpx.HTTPError:
return False

124
ai_platform/modules/didi_brain/scheduler/config.py Normal file
View file

@ -0,0 +1,124 @@
"""Configuration for the scheduler container.
All settings flow from env vars (prefix ``SCHED_``) so deployment can tune
intervals + feed lists without rebuilds. Reasonable defaults for a typical
DIDI deployment are baked in, but ``brain_api_url`` is required to fail-fast
if misconfigured.
"""
from __future__ import annotations
from functools import lru_cache
from pathlib import Path
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class SchedulerSettings(BaseSettings):
"""Top-level scheduler config — validated at startup."""
model_config = SettingsConfigDict(
env_file=Path(__file__).parent.parent / ".env",
env_file_encoding="utf-8",
extra="ignore",
env_prefix="SCHED_",
)
# ---- Brain API target -------------------------------------------------
brain_api_url: str = Field(
default="http://brain-api:8090",
description="Base URL of the brain HTTP API. Docker DNS by default.",
)
brain_api_timeout_s: float = Field(default=120.0)
# ---- Feeder (Pilon 4) -------------------------------------------------
feeder_enabled: bool = Field(default=True)
# Intervals (seconds) per volatility — controls how often we poll RSS
# feeds in each tier.
feeder_volatile_interval_s: int = Field(default=900) # 15 min
feeder_evolving_interval_s: int = Field(default=21600) # 6 h
feeder_stable_interval_s: int = Field(default=86400) # 24 h
feeder_max_items_per_run: int = Field(default=20)
feeder_min_published_age_s: int = Field(default=3600) # skip <1h items
feeder_max_published_age_s: int = Field(default=259200) # skip >3d items
# ---- Auditor (Pilon 5) ------------------------------------------------
auditor_enabled: bool = Field(default=True)
auditor_interval_s: int = Field(default=86400) # 24h sweep
auditor_batch_limit: int = Field(default=200)
auditor_min_age_hours: float = Field(
default=24.0,
description=(
"Don't audit atoms younger than this — they were just written, "
"judging them yields no new signal."
),
)
# ---- Breaking-news watcher (Pilon 9) ----------------------------------
watcher_enabled: bool = Field(default=True)
watcher_poll_interval_s: int = Field(default=300) # 5 min
watcher_max_items_per_run: int = Field(default=10)
watcher_min_published_age_s: int = Field(default=60) # >1min old
watcher_max_published_age_s: int = Field(default=3600) # <1h old
# ---- Logging ----------------------------------------------------------
log_level: str = Field(default="INFO")
log_json: bool = Field(default=False)
@lru_cache(maxsize=1)
def get_settings() -> SchedulerSettings:
return SchedulerSettings()
settings = get_settings()
# ----------------------------------------------------------------------------
# Default feed lists per volatility — overridable via env (FEED_VOLATILE_URLS,
# FEED_EVOLVING_URLS, FEED_STABLE_URLS, FEED_BREAKING_URLS as
# comma-separated strings) for ops flexibility.
# ----------------------------------------------------------------------------
# Volatile: fast-moving news (war, breaking events, daily politics).
# Note: Reuters retired their public RSS feeds. Operators with paid Reuters
# access can add their feed URLs via the FEED_VOLATILE_URLS env override.
DEFAULT_VOLATILE_FEEDS: list[str] = [
"https://feeds.bbci.co.uk/news/world/rss.xml",
"https://www.aljazeera.com/xml/rss/all.xml",
"https://feeds.npr.org/1004/rss.xml",
"https://www.theguardian.com/world/rss",
# Romanian
"https://www.digi24.ro/rss",
"https://www.hotnews.ro/rss",
"https://www.g4media.ro/feed",
]
# Evolving: weekly-stable topics (economy, climate, science debates).
DEFAULT_EVOLVING_FEEDS: list[str] = [
"https://feeds.bbci.co.uk/news/business/rss.xml",
"https://feeds.bbci.co.uk/news/health/rss.xml",
"https://rss.nytimes.com/services/xml/rss/nyt/Science.xml",
]
# Stable: long-cycle topics (basic science, history, settled facts).
DEFAULT_STABLE_FEEDS: list[str] = [
"https://feeds.bbci.co.uk/news/science_and_environment/rss.xml",
]
# Breaking: small set polled fast (5 min) — only the highest-credibility
# real-time wires. Each item runs through the LLM classifier to decide
# which topics/entities to invalidate caches for.
DEFAULT_BREAKING_FEEDS: list[str] = [
"https://www.theguardian.com/world/rss",
"https://feeds.bbci.co.uk/news/world/rss.xml",
"https://www.aljazeera.com/xml/rss/all.xml",
]
def parse_csv(value: str | None, default: list[str]) -> list[str]:
"""Parse a comma-separated env var into a list, falling back to default."""
if not value:
return list(default)
return [v.strip() for v in value.split(",") if v.strip()]

197
ai_platform/modules/didi_brain/scheduler/feed_parser.py Normal file
View file

@ -0,0 +1,197 @@
"""RSS feed parsing + normalization.
Thin wrapper over feedparser that returns a clean list of FeedItem records.
We deliberately keep field extraction conservative — every downstream task
works with the same minimal shape.
"""
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any
import feedparser
import httpx
from shared.logging import get_logger
log = get_logger(__name__)
FETCH_TIMEOUT_S = 30.0
USER_AGENT = "didibrain-scheduler/0.1 (+https://didi365.eu)"
@dataclass(slots=True, frozen=True)
class FeedItem:
"""One article retrieved from an RSS feed.
Attributes:
title: Article title (cleaned).
url: Canonical URL (link tag).
summary: Short summary or description, may be empty.
full_text: Full article body if the feed exposes it (RSS rarely does;
most feeds only have summaries — caller may fetch the URL
separately to enrich).
publisher: Hostname of the source URL.
published_at: When the article was published. UTC.
feed_url: Source feed URL (for traceability).
"""
title: str
url: str
summary: str
full_text: str
publisher: str
published_at: datetime
feed_url: str
def _parse_published(entry: Any) -> datetime | None:
"""Best-effort parser for feedparser's various date fields.
Falls back to None if the entry has no usable date.
"""
for field in ("published_parsed", "updated_parsed", "created_parsed"):
struct = getattr(entry, field, None) or entry.get(field)
if struct:
try:
# struct_time is naive; treat as UTC (most feeds are).
return datetime(*struct[:6], tzinfo=timezone.utc)
except (TypeError, ValueError):
continue
return None
def _publisher_of(url: str) -> str:
"""Extract host from URL — used as the EvidenceItem.publisher field."""
try:
from urllib.parse import urlparse
host = urlparse(url).hostname or ""
return host.lower().lstrip("www.")
except Exception: # noqa: BLE001
return ""
async def fetch_feed(feed_url: str) -> list[FeedItem]:
"""Fetch and parse a single RSS feed.
Returns an empty list on any error (logged) — the caller iterates over
many feeds and shouldn't be derailed by one bad source.
"""
try:
async with httpx.AsyncClient(
timeout=FETCH_TIMEOUT_S,
headers={"User-Agent": USER_AGENT},
follow_redirects=True,
) as client:
resp = await client.get(feed_url)
if resp.status_code >= 400:
log.warning(
"feed_fetch_http_error",
feed=feed_url,
status=resp.status_code,
)
return []
body = resp.text
except httpx.HTTPError as e:
log.warning("feed_fetch_failed", feed=feed_url, error=str(e))
return []
except Exception as e: # noqa: BLE001
log.warning(
"feed_fetch_unexpected",
feed=feed_url,
error=f"{type(e).__name__}:{e}",
)
return []
# feedparser is synchronous + CPU-bound on parse; offload to a thread so
# we don't block the event loop.
parsed = await asyncio.to_thread(feedparser.parse, body)
if parsed.bozo and not parsed.entries:
log.debug(
"feed_bozo",
feed=feed_url,
error=str(parsed.bozo_exception)[:120],
)
return []
items: list[FeedItem] = []
for entry in parsed.entries:
url = (entry.get("link") or "").strip()
if not url:
continue
title = (entry.get("title") or "").strip()
summary = (entry.get("summary") or entry.get("description") or "").strip()
# full_text rarely present — feedparser exposes 'content' on some
# feeds. Take the first content block when available.
full_text = ""
contents = entry.get("content") or []
if contents and isinstance(contents, list):
first = contents[0]
if isinstance(first, dict):
full_text = (first.get("value") or "").strip()
published_at = _parse_published(entry) or datetime.now(tz=timezone.utc)
items.append(
FeedItem(
title=title,
url=url,
summary=summary,
full_text=full_text,
publisher=_publisher_of(url),
published_at=published_at,
feed_url=feed_url,
)
)
return items
async def fetch_feeds(feed_urls: list[str]) -> list[FeedItem]:
"""Fetch many feeds in parallel, flatten results."""
if not feed_urls:
return []
tasks = [fetch_feed(u) for u in feed_urls]
results = await asyncio.gather(*tasks, return_exceptions=True)
flat: list[FeedItem] = []
for r in results:
if isinstance(r, list):
flat.extend(r)
return flat
def filter_by_age(
items: list[FeedItem],
*,
min_age_s: int,
max_age_s: int,
) -> list[FeedItem]:
"""Keep items whose age is within ``[min_age_s, max_age_s]``.
The min bound exists because some feeds publish before the article
body is fully crawlable; we'd rather wait a bit. The max bound prevents
re-ingesting old items that were already in the corpus.
"""
now = datetime.now(tz=timezone.utc)
out: list[FeedItem] = []
for item in items:
age = (now - item.published_at).total_seconds()
if age < min_age_s or age > max_age_s:
continue
out.append(item)
return out
def dedup_by_url(items: list[FeedItem]) -> list[FeedItem]:
"""Drop duplicates within a batch (same URL across multiple feeds)."""
seen: set[str] = set()
out: list[FeedItem] = []
for item in items:
key = item.url.split("?")[0].rstrip("/").lower()
if key in seen:
continue
seen.add(key)
out.append(item)
return out

204
ai_platform/modules/didi_brain/scheduler/feeder.py Normal file
View file

@ -0,0 +1,204 @@
"""Fresh feeder — Pilon 4 of the cache freshness defense.
Polls RSS feeds at volatility-aware intervals and POSTs new articles to
``/v1/ingest`` so the brain corpus stays current. The intervals are:
- volatile feeds → every 15 min (war, breaking, daily politics)
- evolving feeds → every 6 h (economy, climate, science debates)
- stable feeds → every 24 h (settled science, history)
Each cycle:
1. Fetch all feeds in the tier (parallel httpx).
2. Drop items outside the freshness window (default: ≥1h old, ≤3 days old).
3. Dedup by URL within the batch.
4. POST to /v1/ingest with run_extraction=true so brain extracts claim
atoms in the background.
Brain handles deduplication against existing atoms via canonical URL match
(see brain_api/services/ingest.py), so resending the same article is
idempotent — already-ingested items are skipped silently.
"""
from __future__ import annotations
import asyncio
import os
from datetime import datetime, timezone # noqa: F401 (timezone used below)
from scheduler.brain_client import BrainClient
from scheduler.config import (
DEFAULT_EVOLVING_FEEDS,
DEFAULT_STABLE_FEEDS,
DEFAULT_VOLATILE_FEEDS,
parse_csv,
settings,
)
from scheduler.feed_parser import (
FeedItem,
dedup_by_url,
fetch_feeds,
filter_by_age,
)
from shared.logging import get_logger
log = get_logger(__name__)
def _feed_item_to_evidence(item: FeedItem) -> dict:
"""Convert a FeedItem to the EvidenceItem shape that /v1/ingest expects.
EvidenceItem requires ``retrieved_at`` (when we fetched it) plus optional
``published_at`` (the source's publish date). Ingest.evidence_to_markdown
assembles the Document atom body from these fields.
"""
body = item.full_text or item.summary or ""
return {
"url": item.url,
"title": item.title,
"summary": item.summary,
"full_text": body,
"publisher": item.publisher,
"published_at": item.published_at.isoformat(),
"retrieved_at": datetime.now(tz=timezone.utc).isoformat(),
"credibility_score": _publisher_credibility_score(item.publisher),
"relevance_score": 0.5, # neutral; real ranking happens at gather time
}
# Heuristic credibility tiers — keeps brain_api/services/ingest.py's
# credibility_score_to_tag_path mapping coherent.
_TIER_1_PUBLISHERS = {
"reuters.com", "ap.org", "apnews.com", "bbc.co.uk", "bbc.com",
"afp.com", "npr.org", "aljazeera.com", "scientificamerican.com",
}
_TIER_2_PUBLISHERS = {
"digi24.ro", "hotnews.ro", "g4media.ro",
}
def _publisher_credibility_score(publisher: str) -> float:
"""Map publisher hostname to a coarse credibility score in [0,1].
Used by brain's credibility_score_to_tag_path to assign Credibility/Tier
tags to ingested Document atoms. Conservative defaults — anything we
don't recognize gets the middle tier.
"""
p = publisher.lower().lstrip("www.")
if p in _TIER_1_PUBLISHERS:
return 0.90
if p in _TIER_2_PUBLISHERS:
return 0.70
return 0.50
def _feeds_for_volatility(volatility: str) -> list[str]:
"""Resolve env override → default for a volatility tier."""
env_var = f"FEED_{volatility.upper()}_URLS"
defaults = {
"volatile": DEFAULT_VOLATILE_FEEDS,
"evolving": DEFAULT_EVOLVING_FEEDS,
"stable": DEFAULT_STABLE_FEEDS,
}[volatility]
return parse_csv(os.environ.get(env_var), defaults)
async def _run_one_cycle(
*, brain: BrainClient, volatility: str, max_items: int
) -> tuple[int, int]:
"""Pull feeds for one volatility tier and ingest fresh items.
Returns ``(fetched, ingested)`` — fetched is items that passed the age
filter, ingested is what brain accepted (could be lower if some were
duplicates of existing atoms).
"""
feeds = _feeds_for_volatility(volatility)
if not feeds:
return 0, 0
items = await fetch_feeds(feeds)
items = filter_by_age(
items,
min_age_s=settings.feeder_min_published_age_s,
max_age_s=settings.feeder_max_published_age_s,
)
items = dedup_by_url(items)
# Newest first, cap at max_items per cycle so a single feed flood doesn't
# overwhelm the LLM-backed extraction pipeline downstream.
items.sort(key=lambda i: i.published_at, reverse=True)
items = items[:max_items]
if not items:
return 0, 0
evidence = [_feed_item_to_evidence(i) for i in items]
# Tag each batch with its volatility so brain's classifier has a hint
# already and the resulting Document atoms can be invalidated by
# topic + volatility later.
response = await brain.ingest(
claim=None, # this is corpus refresh, not a specific user claim
evidence=evidence,
default_tags=[
f"Volatility/{volatility.capitalize()}",
],
run_extraction=True,
)
if response is None:
log.warning(
"feeder_ingest_failed",
volatility=volatility,
attempted=len(items),
)
return len(items), 0
accepted = int(response.get("accepted") or 0)
skipped = int(response.get("skipped_duplicate") or 0)
log.info(
"feeder_cycle_done",
volatility=volatility,
feeds=len(feeds),
fetched=len(items),
accepted=accepted,
skipped_dup=skipped,
errors=int(response.get("errors") or 0),
)
return len(items), accepted
async def feeder_loop(brain: BrainClient, volatility: str) -> None:
"""Long-running task — one per volatility tier.
Runs forever; each cycle catches its own exceptions so a single failure
doesn't kill the loop.
"""
interval = {
"volatile": settings.feeder_volatile_interval_s,
"evolving": settings.feeder_evolving_interval_s,
"stable": settings.feeder_stable_interval_s,
}[volatility]
log.info(
"feeder_loop_start",
volatility=volatility,
interval_s=interval,
)
while True:
try:
await _run_one_cycle(
brain=brain,
volatility=volatility,
max_items=settings.feeder_max_items_per_run,
)
except Exception as e: # noqa: BLE001
log.exception("feeder_cycle_error", volatility=volatility)
await asyncio.sleep(interval)
async def run_feeder(brain: BrainClient) -> None:
"""Launch one loop per volatility tier — runs forever."""
if not settings.feeder_enabled:
log.info("feeder_disabled")
return
await asyncio.gather(
feeder_loop(brain, "volatile"),
feeder_loop(brain, "evolving"),
feeder_loop(brain, "stable"),
)

146
ai_platform/modules/didi_brain/scheduler/main.py Normal file
View file

@ -0,0 +1,146 @@
"""Scheduler entry point — orchestrates feeder + auditor + watcher.
Runs as a single container with three independent asyncio tasks. Each task
is supervisor-style: catches its own exceptions and continues, so a failure
in one (e.g., RSS feed temporarily down) doesn't kill the others.
Health check: writes ``/tmp/scheduler.healthy`` periodically. Docker
healthcheck probes the file's mtime to detect a stuck loop.
"""
from __future__ import annotations
import asyncio
import signal
import time
from pathlib import Path
from brain_api.db import db
from scheduler.auditor import run_auditor
from scheduler.brain_client import BrainClient
from scheduler.config import settings
from scheduler.feeder import run_feeder
from scheduler.watcher import run_watcher
from shared.llm_client import LlmClient
from shared.logging import setup_logging, get_logger
log = get_logger(__name__)
HEALTH_FILE = Path("/tmp/scheduler.healthy")
HEALTH_INTERVAL_S = 30.0
async def _heartbeat() -> None:
"""Periodically touch the health file so healthcheck sees activity."""
while True:
try:
HEALTH_FILE.write_text(str(time.time()))
except Exception: # noqa: BLE001
pass
await asyncio.sleep(HEALTH_INTERVAL_S)
async def _wait_brain_ready(brain: BrainClient, *, max_attempts: int = 60) -> None:
"""Spin until brain-api answers /health, with bounded retries.
Compose dependencies don't always order brain-api before scheduler at
runtime; we'd rather wait than crash on first call.
"""
for attempt in range(max_attempts):
if await brain.health():
log.info("brain_api_reachable", attempts=attempt + 1)
return
await asyncio.sleep(2.0)
log.warning("brain_api_unreachable_after_retries", attempts=max_attempts)
async def _supervised(name: str, coro_factory) -> None:
"""Wrap a long-running task so a crash inside doesn't unwind the rest.
The task never returns under normal operation; if it raises, we log and
restart after a short backoff.
"""
backoff = 5.0
while True:
try:
await coro_factory()
log.warning("supervised_task_returned", name=name)
except asyncio.CancelledError:
log.info("supervised_task_cancelled", name=name)
raise
except Exception: # noqa: BLE001
log.exception("supervised_task_crashed", name=name)
await asyncio.sleep(backoff)
backoff = min(backoff * 2, 300.0) # cap at 5 min
async def main() -> None:
setup_logging()
log.info(
"scheduler_starting",
feeder=settings.feeder_enabled,
auditor=settings.auditor_enabled,
watcher=settings.watcher_enabled,
brain_url=settings.brain_api_url,
)
# Connect to PG (auditor needs direct DB access for SELECT/UPDATE).
try:
await db.connect()
except Exception: # noqa: BLE001
log.exception("scheduler_db_connect_failed")
# Without DB the auditor can't run, but feeder + watcher only need
# HTTP — degrade gracefully rather than exit.
pass
brain = BrainClient()
llm = LlmClient()
await _wait_brain_ready(brain)
# Graceful shutdown: cancel all tasks on SIGTERM/SIGINT.
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
def _trigger_stop() -> None:
stop_event.set()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, _trigger_stop)
except NotImplementedError:
# Windows / restricted env — ignore.
pass
tasks = [
asyncio.create_task(_heartbeat(), name="heartbeat"),
asyncio.create_task(
_supervised("feeder", lambda: run_feeder(brain)),
name="feeder",
),
asyncio.create_task(
_supervised("auditor", lambda: run_auditor(brain, llm)),
name="auditor",
),
asyncio.create_task(
_supervised("watcher", lambda: run_watcher(brain, llm)),
name="watcher",
),
]
try:
await stop_event.wait()
finally:
log.info("scheduler_stopping")
for t in tasks:
t.cancel()
# Give tasks a moment to clean up.
await asyncio.gather(*tasks, return_exceptions=True)
await brain.aclose()
await llm.aclose()
await db.close()
log.info("scheduler_stopped")
if __name__ == "__main__":
asyncio.run(main())

18
ai_platform/modules/didi_brain/scheduler/requirements.txt Normal file
View file

@ -0,0 +1,18 @@
# Scheduler container — feeder + auditor + watcher.
# Inherits brain_api dependencies (we import brain_api.* directly) plus
# RSS parsing.
# Inherit the brain_api stack
fastapi>=0.135,<0.140
uvicorn>=0.44,<0.50
httpx>=0.28,<0.30
pydantic>=2.12,<3.0
pydantic-settings>=2.13,<3.0
python-dotenv>=1.2,<2.0
tenacity>=9.1,<10.0
structlog>=25.5,<26.0
rich>=14.3,<15.0
asyncpg>=0.30,<0.32
# RSS feed parsing (the only scheduler-specific dep)
feedparser>=6.0,<7.0

228
ai_platform/modules/didi_brain/scheduler/watcher.py Normal file
View file

@ -0,0 +1,228 @@
"""Breaking-news watcher — Pilon 9 of the cache freshness defense.
Polls a small set of high-credibility breaking-news RSS feeds every 5 min.
For each new item, runs the volatility classifier (LLM) to identify the
affected topics + entities, then triggers ``/v1/cache/invalidate`` on those
topics so cached verdicts that depend on the just-changed reality are
flushed within minutes of the breaking story going live.
End-to-end latency target: under 30s from RSS publish to cache invalidation
(network + LLM ≈ 5-15s in practice with local Qwen).
Memory: keeps an in-process LRU of recently-seen URLs to avoid re-classifying
the same article on every poll. Restart-safe: brain's ingest dedup also
skips already-known URLs.
"""
from __future__ import annotations
import asyncio
import os
from collections import deque
from datetime import datetime, timezone
from brain_api.services.classifier import (
ClaimVolatility,
classify_claim_volatility,
)
from brain_api.services.fact_status import canonicalize_triple
from scheduler.brain_client import BrainClient
from scheduler.config import (
DEFAULT_BREAKING_FEEDS,
parse_csv,
settings,
)
from scheduler.feed_parser import (
FeedItem,
dedup_by_url,
fetch_feeds,
filter_by_age,
)
from shared.llm_client import LlmClient
from shared.logging import get_logger
log = get_logger(__name__)
# In-process LRU of seen URLs — capped so restart doesn't accumulate forever.
SEEN_LRU_MAX = 2000
_seen_urls: deque[str] = deque(maxlen=SEEN_LRU_MAX)
_seen_set: set[str] = set()
def _mark_seen(url: str) -> None:
"""Track URL as seen, evicting oldest if at capacity."""
if url in _seen_set:
return
if len(_seen_urls) == SEEN_LRU_MAX:
# deque.append at maxlen drops the oldest; reflect in the set.
oldest = _seen_urls[0]
_seen_set.discard(oldest)
_seen_urls.append(url)
_seen_set.add(url)
def _is_seen(url: str) -> bool:
return url in _seen_set
def _build_invalidation_targets(
classification: ClaimVolatility,
) -> tuple[list[str], list[str]]:
"""Decide what to invalidate based on classifier output.
Only volatile and evolving classifications trigger invalidation —
stable items (background pieces, historical recap) shouldn't flush
anything.
Returns ``(topic_codes, entity_canonicals)``:
- topic_codes: pass through directly to /v1/cache/invalidate
- entity_canonicals: canonicalize each binding so PG can match
against cached entity_bindings JSONB
"""
if classification.volatility not in ("volatile", "evolving"):
return [], []
topics = list(classification.topic_codes or [])
canonicals: list[str] = []
for b in classification.entity_bindings or []:
if b.confidence < 0.6:
# Low-confidence extractions are noisy — skip to avoid
# accidental mass invalidation.
continue
canonicals.append(
canonicalize_triple(b.subject, b.predicate, b.obj)
)
return topics, canonicals
async def _process_one_item(
*,
item: FeedItem,
brain: BrainClient,
llm: LlmClient,
) -> str:
"""Classify one breaking item and trigger invalidation if applicable.
Returns a short label for telemetry: 'classified_no_action' /
'invalidated' / 'ignored_low_confidence' / 'classifier_failed'.
"""
# Title + summary is what we feed the classifier — full article body
# would be expensive and the headline carries the signal we need.
text = (item.title or "")
if item.summary:
text = f"{text}. {item.summary}"
if not text.strip():
return "no_text"
classification = await classify_claim_volatility(llm, claim=text)
if classification.degraded:
return "classifier_failed"
topics, canonicals = _build_invalidation_targets(classification)
if not topics and not canonicals:
return "classified_no_action"
# Dry-run first to count, then real invalidate. We tolerate partial
# successes — if a network blip kills the real call, the next watcher
# cycle will catch the same item again.
result = await brain.invalidate(
topic_codes=topics or None,
entity_canonicals=canonicals or None,
# Only invalidate verdicts written before this breaking story —
# avoids racing with concurrent writes that may have used fresh
# information already.
since_iso=None,
invalidate_gold=False,
actor=f"breaking_watcher:{item.publisher}",
reason=(
f"breaking story: {item.title[:120]} "
f"(vol={classification.volatility})"
),
)
if result is None:
return "invalidate_http_error"
log.info(
"watcher_invalidated",
title=item.title[:80],
publisher=item.publisher,
volatility=classification.volatility,
topics=topics,
entities=len(canonicals),
atoms=result.get("invalidated_atoms"),
vcache=result.get("invalidated_vcache"),
)
return "invalidated"
async def _run_one_cycle(brain: BrainClient, llm: LlmClient) -> dict[str, int]:
"""Pull breaking feeds, classify novel items, invalidate as needed."""
feed_urls = parse_csv(
os.environ.get("FEED_BREAKING_URLS"), DEFAULT_BREAKING_FEEDS
)
if not feed_urls:
return {"checked": 0, "novel": 0}
items = await fetch_feeds(feed_urls)
items = filter_by_age(
items,
min_age_s=settings.watcher_min_published_age_s,
max_age_s=settings.watcher_max_published_age_s,
)
items = dedup_by_url(items)
# Drop items already processed in a previous cycle.
novel = [i for i in items if not _is_seen(i.url)]
novel.sort(key=lambda i: i.published_at, reverse=True)
novel = novel[: settings.watcher_max_items_per_run]
if not novel:
return {"checked": len(items), "novel": 0}
results: dict[str, int] = {}
for item in novel:
try:
label = await _process_one_item(
item=item, brain=brain, llm=llm
)
results[label] = results.get(label, 0) + 1
except Exception as e: # noqa: BLE001
log.warning(
"watcher_item_failed",
url=item.url,
error=f"{type(e).__name__}:{e}",
)
results["item_error"] = results.get("item_error", 0) + 1
finally:
_mark_seen(item.url)
log.info(
"watcher_cycle_done",
feeds=len(feed_urls),
items_total=len(items),
novel=len(novel),
breakdown=results,
)
return {"checked": len(items), "novel": len(novel), **results}
async def run_watcher(brain: BrainClient, llm: LlmClient) -> None:
"""Long-running task — polls breaking-news feeds at watcher_poll_interval_s."""
if not settings.watcher_enabled:
log.info("watcher_disabled")
return
log.info(
"watcher_loop_start",
interval_s=settings.watcher_poll_interval_s,
feeds=len(
parse_csv(
os.environ.get("FEED_BREAKING_URLS"), DEFAULT_BREAKING_FEEDS
)
),
)
while True:
try:
await _run_one_cycle(brain, llm)
except Exception: # noqa: BLE001
log.exception("watcher_cycle_error")
await asyncio.sleep(settings.watcher_poll_interval_s)