clossers/didi-lot1-ai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Livrare LOT 1 - Didi

Browse source
This commit is contained in:
Dezvoltari Evotech 2026-06-25 14:13:25 -07:00
commit 5380c3fc63
990 changed files with 133308 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
ai_platform/modules/web/.dockerignore Normal file
View file

@ -0,0 +1,15 @@
.venv/
__pycache__/
*.pyc
.pytest_cache/
.ruff_cache/
.mypy_cache/
tests/
benchmark/
deploy/
.env
.env.*
*.md
!README.md
.git/
.gitignore

146
ai_platform/modules/web/.env.example Normal file
View file

@ -0,0 +1,146 @@
# =============================================================================
# Web Module — Environment Configuration
# =============================================================================
# Copy this file to .env and fill in the values marked with CHANGE_ME.
# All variables use the WEB_ prefix.
# =============================================================================
# -----------------------------------------------------------------------------
# REQUIRED — Search (SearXNG metasearch)
# -----------------------------------------------------------------------------
# SearXNG base URL (internal Docker hostname or LAN IP).
# If you already have a SearXNG running elsewhere, point to it here.
WEB_SEARXNG_BASE_URL=http://didiAI-web-searxng:8080
# -----------------------------------------------------------------------------
# REQUIRED — External URL for OpenAPI spec (Swagger / catalog)
# -----------------------------------------------------------------------------
# The publicly reachable URL where this web-api is exposed.
WEB_EXTERNAL_URL=http://localhost:51100
# -----------------------------------------------------------------------------
# REQUIRED — Remote LLM inference (runs on the GPU machine)
# -----------------------------------------------------------------------------
# URL of your llm-inference router (Qwen3.5-35B vLLM or similar).
# Used by the free tier for context detection + evidence extraction.
WEB_LLM_BASE_URL=CHANGE_ME_http://your-gpu-host:14011
# Text model name served by the LLM endpoint
WEB_TEXT_MODEL=qwen3.5
# Vision model — usually same endpoint + model (Qwen3.5 is multimodal)
WEB_VISION_BASE_URL=CHANGE_ME_http://your-gpu-host:14011
WEB_VISION_MODEL=qwen3.5
# Optional API key if your LLM endpoint enforces auth
# WEB_LLM_API_KEY=
# -----------------------------------------------------------------------------
# OPTIONAL — Paid search providers (used by premium tier)
# -----------------------------------------------------------------------------
# Leave blank if you don't have a subscription. If all are blank,
# premium tier returns no search results.
# WEB_SERPAPI_API_KEY=CHANGE_ME_or_leave_blank
# WEB_TAVILY_API_KEY=CHANGE_ME_or_leave_blank
# WEB_BRAVE_API_KEY=CHANGE_ME_or_leave_blank
# WEB_LINKUP_API_KEY=CHANGE_ME_or_leave_blank
# WEB_EXA_API_KEY=CHANGE_ME_or_leave_blank
# -----------------------------------------------------------------------------
# OPTIONAL — OpenRouter (premium tier LLM)
# -----------------------------------------------------------------------------
# When set, premium tier routes all context + evidence LLM calls to
# OpenRouter instead of your local Qwen. Model is runtime-configurable
# via the dashboard.
# WEB_OPENROUTER_API_KEY=CHANGE_ME_sk-or-v1-...
WEB_OPENROUTER_MODEL=google/gemini-3.1-flash-lite-preview
# -----------------------------------------------------------------------------
# OPTIONAL — Dashboard event sink
# -----------------------------------------------------------------------------
# When set, web-api forwards every request event (tier, duration, cost,
# results) to the dashboard and polls /api/config every 30s for runtime
# overrides. Leave blank to run without the dashboard.
WEB_DASHBOARD_URL=http://didiAI-dashboard:51300
# WEB_DASHBOARD_TOKEN= # optional if dashboard ingest requires auth
# -----------------------------------------------------------------------------
# OPTIONAL — External LLM fallback (used when local is down)
# -----------------------------------------------------------------------------
# WEB_OPENAI_API_KEY=
# WEB_ANTHROPIC_API_KEY=
# -----------------------------------------------------------------------------
# OPTIONAL — Server settings
# -----------------------------------------------------------------------------
# WEB_HOST=0.0.0.0
# WEB_PORT=51100
# -----------------------------------------------------------------------------
# OPTIONAL — Search defaults
# -----------------------------------------------------------------------------
# WEB_SEARCH_DEFAULT_MAX_RESULTS=10
# WEB_SEARCH_DEFAULT_LANGUAGE=en
# WEB_SEARCH_DEFAULT_COUNTRY=US
# -----------------------------------------------------------------------------
# OPTIONAL — Fetch & Browse tuning
# -----------------------------------------------------------------------------
# WEB_FETCH_TIMEOUT=30.0
# WEB_FETCH_MIN_TEXT_LENGTH=200
# WEB_BROWSE_TIMEOUT=30000
# WEB_BROWSE_VIEWPORT_WIDTH=1280
# WEB_BROWSE_VIEWPORT_HEIGHT=720
# WEB_BROWSE_BLOCK_RESOURCES=true
# WEB_BROWSE_EXTRA_WAIT_MS=500
# -----------------------------------------------------------------------------
# OPTIONAL — Vision / Evidence tuning
# -----------------------------------------------------------------------------
# WEB_VISION_MAX_TOKENS=2000
# WEB_VISION_SCREENSHOT_QUALITY=80
# WEB_EVIDENCE_MAX_ITEMS=30
# WEB_EVIDENCE_MAX_SNIPPET_LENGTH=500
# WEB_EVIDENCE_DEDUPE_THRESHOLD=0.9
# -----------------------------------------------------------------------------
# OPTIONAL — Rate limiting & concurrency
# -----------------------------------------------------------------------------
# WEB_RATE_LIMIT_RPS=10.0
# WEB_RATE_LIMIT_BURST=20
# WEB_MAX_CONCURRENT_REQUESTS=10
# -----------------------------------------------------------------------------
# OPTIONAL — Timeouts & retries
# -----------------------------------------------------------------------------
# WEB_REQUEST_TIMEOUT=30.0
# WEB_CONNECT_TIMEOUT=10.0
# WEB_MAX_RETRIES=3
# -----------------------------------------------------------------------------
# OPTIONAL — Logging
# -----------------------------------------------------------------------------
# WEB_LOG_LEVEL=INFO
# WEB_LOG_JSON=false
# -----------------------------------------------------------------------------
# OPTIONAL — Authentication (comma-separated Bearer tokens)
# -----------------------------------------------------------------------------
# If set, all /v1/* endpoints require Authorization: Bearer <token>.
# Leave blank to run open (fine for VPN-internal deployments).
# WEB_API_TOKENS=token1,token2

478
ai_platform/modules/web/API.md Normal file
View file

@ -0,0 +1,478 @@
# Web API Documentation
## Base URL
```
{BASE_URL}
```
- **Local development:** `http://localhost:51100`
- **Docker (internal):** `http://web-api:51100`
- **Production:** Use your configured hostname
## Authentication
Authentication is optional. If `WEB_API_TOKENS` is set, Bearer token authentication is required.
```
Authorization: Bearer <token>
```
If authentication is disabled (default), no Authorization header is needed.
## Endpoints
### Gather (Main Endpoint)
Execute the full evidence-gathering pipeline: search → fetch → evidence.
```
POST /v1/gather
```
#### Request Body
```json
{
"claim": "The claim to verify",
"search_queries": ["optional custom queries"],
"max_search_results": 20,
"site_allowlist": ["reuters.com"],
"site_blocklist": ["spam-site.com"],
"fetch_method": "auto",
"auto_fallback": true,
"extract_snippets": true,
"max_evidence_items": 15,
"dedupe": true,
"timeout_seconds": 60.0
}
```
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
| `claim` | string | Yes | - | Claim to gather evidence for (10-1000 chars) |
| `search_queries` | array[string] | No | null | Custom search queries (auto-generated if not provided) |
| `max_search_results` | integer | No | 20 | Max search results (5-50) |
| `site_allowlist` | array[string] | No | null | Only search these domains |
| `site_blocklist` | array[string] | No | null | Exclude these domains |
| `fetch_method` | string | No | "auto" | "auto", "http", "browse", "vision" |
| `auto_fallback` | boolean | No | true | Escalate on fetch failure |
| `extract_snippets` | boolean | No | true | Use LLM for snippet extraction |
| `max_evidence_items` | integer | No | 15 | Max items in final pack (1-50) |
| `dedupe` | boolean | No | true | Deduplicate evidence |
| `parallel_fetches` | integer | No | 5 | Concurrent fetch operations (1-10) |
| `timeout_seconds` | number | No | 60.0 | Total pipeline timeout (10-300) |
#### Response
```json
{
"request_id": "uuid",
"claim": "The claim",
"evidence": [
{
"url": "https://example.com/article",
"title": "Article Title",
"publisher": "example.com",
"snippet": "Relevant excerpt...",
"relevance_score": 0.8,
"credibility_score": null,
"published_at": "2024-01-15",
"retrieved_at": "2025-01-20T10:00:00Z",
"full_text_hash": "sha256...",
"provenance": {
"extraction_method": "http",
"fallback_chain": []
}
}
],
"evidence_stats": {
"input_items": 10,
"after_dedup": 8,
"output_items": 8,
"duplicates_removed": 2,
"tokens_used": 1500
},
"stages": [
{"stage": "search", "success": true, "items_processed": 10, "items_failed": 0, "duration_ms": 500.0},
{"stage": "fetch", "success": true, "items_processed": 8, "items_failed": 2, "duration_ms": 5000.0},
{"stage": "evidence", "success": true, "items_processed": 8, "items_failed": 0, "duration_ms": 3000.0}
],
"total_urls_found": 10,
"total_pages_fetched": 8,
"total_evidence_items": 8,
"execution_time_ms": 8500.0
}
```
#### Example
```bash
curl -X POST http://localhost:51100/v1/gather \
-H "Content-Type: application/json" \
-d '{
"claim": "Romania had the highest economic growth in the EU in 2024",
"max_search_results": 5,
"extract_snippets": true,
"max_evidence_items": 5
}'
```
---
### Search
Execute web search queries.
```
POST /v1/search
```
#### Request Body
```json
{
"queries": ["string"],
"max_results": 10,
"site_allowlist": ["string"],
"site_blocklist": ["string"],
"language": "en",
"country": "US",
"freshness": "month",
"safe_search": "moderate"
}
```
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
| `queries` | array[string] | Yes | - | Search queries (1-10) |
| `max_results` | integer | No | 10 | Results per query (1-100) |
| `site_allowlist` | array[string] | No | null | Only include these domains |
| `site_blocklist` | array[string] | No | null | Exclude these domains |
| `language` | string | No | "en" | Search language (ISO 639-1) |
| `country` | string | No | "US" | Search country (ISO 3166-1) |
| `freshness` | string | No | null | Filter by age: day, week, month, year |
| `safe_search` | string | No | "moderate" | off, moderate, strict |
#### Response
```json
{
"request_id": "uuid",
"results": [
{
"query": "string",
"url": "string",
"title": "string",
"snippet": "string",
"rank": 1,
"site": "string",
"published_at": "string|null"
}
],
"total_results": 10,
"execution_time_ms": 150.5,
"queries_processed": 1
}
```
#### Example
```bash
curl -X POST http://localhost:51100/v1/search \
-H "Content-Type: application/json" \
-d '{
"queries": ["climate change effects", "renewable energy"],
"max_results": 5,
"site_allowlist": ["reuters.com", "bbc.com"],
"freshness": "month"
}'
```
---
### Image Search
Search for images using SearXNG image search.
```
POST /v1/image-search
```
#### Request Body
```json
{
"queries": ["solar eclipse"],
"max_results": 50,
"language": "en",
"country": "US",
"safe_search": "strict",
"spellcheck": true
}
```
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
| `queries` | array[string] | Yes | - | Search queries (1-10) |
| `max_results` | integer | No | 50 | Results per query (1-200) |
| `language` | string | No | "en" | Search language (ISO 639-1) |
| `country` | string | No | "US" | Search country (ISO 3166-1) |
| `safe_search` | string | No | "strict" | off or strict |
| `spellcheck` | boolean | No | true | Enable spellcheck |
#### Response
```json
{
"request_id": "uuid",
"results": [
{
"query": "solar eclipse",
"image_url": "https://example.com/eclipse.jpg",
"thumbnail_url": "https://example.com/eclipse_thumb.jpg",
"source_url": "https://example.com/article",
"title": "Solar Eclipse Photo",
"description": "A total solar eclipse captured in 2024",
"width": 1920,
"height": 1080,
"publisher": "example.com",
"rank": 1
}
],
"total_results": 50,
"execution_time_ms": 200.5,
"queries_processed": 1
}
```
#### Example
```bash
curl -X POST http://localhost:51100/v1/image-search \
-H "Content-Type: application/json" \
-d '{
"queries": ["solar eclipse", "northern lights"],
"max_results": 5,
"safe_search": "strict"
}'
```
---
### Fetch
Fetch and extract content from URLs with automatic fallback (HTTP → Playwright → Vision LLM).
```
POST /v1/fetch
```
#### Request Body
```json
{
"urls": ["https://example.com/article"],
"extract_text": true,
"include_html": false,
"extract_metadata": true,
"auto_fallback": true,
"method": "auto",
"timeout_seconds": 30.0,
"min_text_length": 200,
"parallel_fetches": 5
}
```
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
| `urls` | array[string] | Yes | - | URLs to fetch (1-50) |
| `extract_text` | boolean | No | true | Extract main text content |
| `include_html` | boolean | No | false | Include raw HTML |
| `extract_metadata` | boolean | No | true | Extract metadata |
| `auto_fallback` | boolean | No | true | Auto-escalate on failure |
| `method` | string | No | "auto" | "auto", "http", "browse", "vision" |
| `timeout_seconds` | number | No | 30.0 | Timeout per URL (1-120) |
| `min_text_length` | integer | No | 200 | Min text before fallback |
| `parallel_fetches` | integer | No | 5 | Concurrent fetches (1-20) |
#### Response
```json
{
"request_id": "uuid",
"pages": [
{
"url": "https://example.com/article",
"title": "Article Title",
"text": "Extracted text content...",
"text_hash": "sha256...",
"extraction_method": "http",
"fallback_chain": [],
"retrieved_at": "2025-01-20T10:00:00Z",
"extraction_time_ms": 500.0,
"status_code": 200,
"content_type": "text/html"
}
],
"total_fetched": 1,
"total_failed": 0,
"execution_time_ms": 600.0,
"failed_urls": []
}
```
#### Example
```bash
curl -X POST http://localhost:51100/v1/fetch \
-H "Content-Type: application/json" \
-d '{
"urls": ["https://en.wikipedia.org/wiki/Climate_change"],
"method": "auto",
"auto_fallback": true
}'
```
---
### Info
Get component information for service catalog.
```
GET /v1/info
```
#### Response
Returns complete metadata about this service including resource information, available functions, and their schemas. Used by the catalog-api for service discovery.
#### Example
```bash
curl http://localhost:51100/v1/info
```
---
### Health Check
Check service health and provider status.
```
GET /health
```
#### Response
```json
{
"status": "healthy",
"providers": [
{
"name": "searxng",
"healthy": true,
"message": null
}
]
}
```
| Status | Description |
|--------|-------------|
| `healthy` | All providers operational |
| `degraded` | Some providers unavailable |
| `unhealthy` | All providers down |
---
### Readiness Probe
Check if service is ready to accept requests.
```
GET /ready
```
#### Response
```json
{
"ready": true
}
```
---
## Error Responses
### Error Format
```json
{
"detail": {
"error": "error_code",
"message": "Human readable message",
"provider": "searxng"
}
}
```
### HTTP Status Codes
| Code | Error | Description |
|------|-------|-------------|
| 400 | `validation_error` | Invalid request body |
| 401 | `authentication_required` | Missing or invalid auth token |
| 422 | `validation_error` | Request validation failed |
| 429 | `rate_limit_exceeded` | Too many requests |
| 502 | `provider_error` | Search provider error |
| 503 | `service_unavailable` | Concurrency limit reached |
| 504 | `timeout` | Request timeout |
---
## Request Headers
| Header | Required | Description |
|--------|----------|-------------|
| `Content-Type` | Yes | Must be `application/json` |
| `Authorization` | Conditional | Bearer token (if auth enabled) |
| `X-Request-ID` | No | Custom request ID for tracing |
---
## Rate Limiting
The API implements token bucket rate limiting:
- **Default:** 10 requests/second with burst of 20
- **429 Response:** Includes `Retry-After` header
**Note:** Rate limiting is per-process. In multi-replica deployments, use external rate limiting.
---
## SDK Example (Python)
```python
import httpx
async def search(query: str, base_url: str = "http://localhost:51100"):
async with httpx.AsyncClient() as client:
response = await client.post(
f"{base_url}/v1/search",
json={
"queries": [query],
"max_results": 10,
},
)
response.raise_for_status()
return response.json()
# Usage
results = await search("climate change")
for r in results["results"]:
print(f"{r['title']}: {r['url']}")
```

193
ai_platform/modules/web/BRAIN_INTEGRATION.md Normal file
View file

@ -0,0 +1,193 @@
# Web-API ↔ didi-brain integration
Web-api folosește didi-brain ca **layer de cache cu 2 fețe**:
1. **Cache read** (ambele tiers — free și premium) — înainte să cheme SearXNG
sau paid providers, web-api întreabă brain. Dacă brain are evidence
relevante cached, le servește direct; clientul primește răspuns în ~1-2s
în loc de 10-30s.
2. **Cache write** (doar tier premium, quality-gated) — după o cerere premium
care a trecut pragurile de calitate, web-api pompează evidence în brain
fire-and-forget. Brain face extraction async. Viitoarele cereri pe claim-uri
similare vor hit-ui cache-ul.
Ideea de bază: **free users primesc quality-ul plătit de premium users**, dar
doar premium plătește pentru popularea cache-ului.
---
## Flow-ul per request
```
POST /v1/gather { claim, tier }
│
▼
┌─────────────────────────────────────────┐
│ 1. Cache READ (brain /v1/gather) │
│ (ambele tiers, dacă brain configured│
│ și brain_cache_read_enabled=true) │
└────────┬────────────────────────────────┘
│
├─ brain HIT cu quality OK
│ → returnează direct răspunsul (skip orchestrator)
│
└─ brain MISS sau quality slab
│
▼
┌────────────────────────────────┐
│ 2. Orchestrator normal │
│ free → SearXNG + Qwen │
│ premium → Paid + OpenRouter│
└────────┬───────────────────────┘
│
▼
┌────────────────────────────────┐
│ 3. Cache WRITE (doar premium) │
│ dacă quality_ok_for_cache │
│ fire-and-forget ingest │
│ (nu blochează response) │
└────────────────────────────────┘
│
▼
Return to client
```
---
## Componente
### `src/web/brain/client.py` — `BrainClient`
Client HTTP minimal pentru brain:
- `gather(claim, max_evidence, run_nli, ...)` — cache read, returnează JSON sau None pe eroare
- `ingest(payload)` — cache write, fire-and-forget via sink de obicei
Timeout-uri separate (gather e rapid, ingest poate fi mai lent).
### `src/web/brain/sink.py` — `BrainIngestSink`
Queue bounded (1000 items max) processat de un single worker task.
Overflow → drop oldest. Eșecuri → log DEBUG, swallowed. Zero impact
pe latența web-api.
### `src/web/brain/quality.py` — Quality gates
- `quality_ok_for_cache(response)` — decide dacă un gather result e worth-caching
(3+ evidence, cel puțin 1 sursă credibilă dacă sunt scored, toate stages OK,
execution > 2s)
- `brain_hit_acceptable(brain_raw)` — decide dacă un brain HIT e suficient
de bun (HIT = da; PARTIAL = da doar dacă 3+ items cu relevance ≥ 0.7)
### `src/web/brain/adapter.py` — Schema conversion
- `brain_response_to_web(brain_raw, request_id)` — convertește response-ul
brain în `GatherResponse` web (datetime → ISO string, provenance dict,
`brain_meta` flatten în provenance per evidence item)
- `web_response_to_ingest_payload(response, claim)` — convertește
`GatherResponse` în `IngestRequest` pentru brain (ISO dates, safe defaults
pentru field-urile required brain-side: title/retrieved_at/score defaults)
---
## Configurare
În `deploy/.env`:
```bash
WEB_BRAIN_URL=http://didibrain-api:8090
# Feature toggles (default true dacă brain_url setat)
WEB_BRAIN_CACHE_READ_ENABLED=true
WEB_BRAIN_INGEST_ENABLED=true
# Quality thresholds pentru ingest
WEB_BRAIN_INGEST_MIN_EVIDENCE=3
WEB_BRAIN_INGEST_MIN_CREDIBILITY=0.7
WEB_BRAIN_INGEST_MIN_EXECUTION_MS=2000
# Acceptance thresholds pentru PARTIAL cache hit
WEB_BRAIN_HIT_MIN_EVIDENCE=3
WEB_BRAIN_HIT_MIN_RELEVANCE=0.7
# Timeouts
WEB_BRAIN_GATHER_TIMEOUT=8.0
WEB_BRAIN_INGEST_TIMEOUT=15.0
```
Toate sunt optional. Dacă `WEB_BRAIN_URL` e gol, integrarea e dezactivată
complet (fallback la orchestrator normal pentru tot).
---
## Networking Docker
Web-api trăiește pe rețeaua `deploy_default` (cu dashboard, video, audio,
searxng). Brain trăiește pe rețeaua proprie `didibrain`.
Web-api e atașat la **ambele** rețele în `deploy/docker-compose.yml`:
```yaml
networks:
deploy_default:
external: true
didibrain:
external: true
services:
web-api:
networks:
- deploy_default
- didibrain
```
Asta îi permite să rezolve `didibrain-api` prin DNS-ul Docker.
---
## Fail-open
Brain down ≠ web-api down. Toate apelurile spre brain sunt wrapped cu
try/except generos:
- Gather cache read eșuează → continuă la orchestrator normal
- Ingest eșuează → log și drop event (queue-ul va reumple)
Clientul nu vede niciodată o eroare datorită brain.
---
## Cum monitorizezi HIT rate
Log-urile web-api emit `Brain cache HIT [tier=X, N items]: <claim>` pe fiecare
hit. Pentru producție, integrarea cu dashboard face count-uri în tabela
`request_history` (câmp `raw_response.stages` conține `retrieval`/`rerank` când
e din brain vs `search`/`fetch` când e direct).
Query exemplu (post-deploy):
```sql
-- HIT rate per tier pe ultimele 24h
SELECT
tier,
COUNT(*) FILTER (
WHERE raw_response::jsonb -> 'stages' @> '[{"stage":"retrieval"}]'
) AS brain_hits,
COUNT(*) AS total_gathers,
ROUND(100.0 * COUNT(*) FILTER (
WHERE raw_response::jsonb -> 'stages' @> '[{"stage":"retrieval"}]'
) / NULLIF(COUNT(*), 0), 1) AS hit_rate_pct
FROM request_history
WHERE endpoint = '/v1/gather'
AND created_at > now() - interval '24 hours'
GROUP BY tier;
```
---
## Ce NU face integrarea
- Nu scrie la brain pe tier `free` (doar citește) — tip premium gold standard
- Nu face fallback la premium când brain MISS pe free (menține tier isolation)
- Nu retry — un singur request la brain, orice eșec → fallback silent
Pentru detaliile contractului verification_cache (backend ↔ brain),
vezi `modules/didi_brain/CONTRACT_VERIFICATION_CACHE.md`.

156
ai_platform/modules/web/INDEX.md Normal file
View file

@ -0,0 +1,156 @@
# Web Module
Web search service for DIDI claim verification. Routes between free (SearXNG meta-search) and premium providers (Brave, Tavily, SerpAPI, Linkup, Exa) based on the `X-Search-Tier` header. Provides a unified gather pipeline (search -> fetch -> evidence extraction) used by agent-v3 as the fallback path when the brain knowledge base returns no evidence for a claim.
- **Stack:** Python 3.10+, FastAPI, Pydantic v2, httpx (HTTP/2), Uvicorn
- **URL:** `http://10.11.10.12:51100` (Dev) / `http://10.11.10.13:51100` (Prod-style reference per agent-v3 default)
- **Container:** `didiAI-web-api`
- **SearXNG cluster:** 3 replicas (`didiAI-web-searxng-1/2/3`) behind nginx LB (`didiAI-web-searxng`, port 55100)
- **SearXNG cache:** 3 Valkey/Redis replicas (`didiAI-web-searxng-redis-1/2/3`)
- **Anonymity proxy:** `didiAI-web-tor` (Tor SOCKS5 shared by all SearXNG instances)
## Ce face
The module powers the fact-checking pipeline:
- **Free search** via SearXNG metasearch (3 round-robin instances, each with a dedicated Valkey for isolated cache/state). Local Qwen LLM is used for context detection + evidence snippet extraction.
- **Premium search** via paid APIs — Brave, Tavily, SerpAPI, Linkup, Exa — selected by `PaidSearchClient` rotation (read keys from `WEB_*_API_KEY` envs). Uses OpenRouter (configurable model) for LLM steps.
- **URL fetching** with readability-lxml extraction; auto-fallback chain `HTTP -> Playwright (browse) -> Vision LLM screenshot OCR` for JS-heavy or protected pages. PDF URLs are skipped.
- **Tor proxy** for sensitive/anonymized SearXNG queries.
- **Brain cache** integration: both tiers READ from the brain cache on `/v1/gather`; only premium WRITES quality results back via the `BrainIngestSink` (fire-and-forget). Free users effectively get the paid knowledge base for free.
- **Dashboard event sink:** every request emits a structured event (tier, provider, duration, status, results_count, raw response snapshot for `/v1/gather`) to the configured dashboard.
- **Runtime config:** `RuntimeConfigClient` polls the dashboard for tier overrides (e.g. `web.tier.free.max_search_results`).
## API endpoints
All `/v1/*` routes are protected by Bearer auth (`verify_bearer_token`) and concurrency-limited. Tier-aware routes read the `X-Search-Tier: free | premium` header (default `free`).
| Method | Path | Body schema | Description |
|--------|------|-------------|-------------|
| `POST` | `/v1/gather` | `GatherRequest` | **Main endpoint** — full pipeline (cache check -> search -> fetch -> evidence) returning `GatherResponse`. Tier-aware (free/premium orchestrators). Premium populates the brain cache when quality threshold is met. |
| `POST` | `/v1/search` | `SearchRequest` | Multi-query web search. Tier-aware (`searxng` for free, `paid-rotation` for premium). |
| `POST` | `/v1/image-search` | `ImageSearchRequest` | Image search (tier-aware, same routing as text search). |
| `POST` | `/v1/fetch` | `FetchRequest` | Fetch URL list with readability extraction. |
| `GET` | `/v1/info` | — | Service catalog metadata (resource + functions, JSON schemas) for the catalog-api. |
| `GET` | `/health` | — | Liveness + provider health (`searxng` reachability). |
| `GET` | `/ready` | — | Readiness probe (verifies `search_client_free` + `orchestrator_free` are wired). |
Errors map to standard HTTP codes via `make_error_detail()`: 429 rate-limit, 502 connection/provider, 504 timeout, 500 generic.
## How didi-backend uses it
- **agent-v3** calls this service via `M17_WEB_API_URL` (default `http://10.11.10.13:51100` per agent-v3 `docker-compose.yml`).
- The brain client in agent-v3 first queries the DIDI brain knowledge base; on a MISS (or low-quality hit), agent-v3 falls through to `POST /v1/gather` here.
- Agent-v3 sets the `X-Search-Tier` header to choose providers: `free` for normal sessions, `premium` for paid/priority queries. Premium runs are what populate the shared brain cache for everyone.
## Structura fisiere
```
src/web/
├── __init__.py
├── cli.py # `web` CLI entry point
├── config.py # WebSettings (pydantic-settings, WEB_* prefix) + SettingsCache
├── exceptions.py # WebError hierarchy (Provider/RateLimit/Timeout/Connection/Search)
├── logging.py # Structured logging + request_id contextvar
├── orchestrator.py # Orchestrator: search -> fetch -> evidence pack pipeline
├── runtime_config.py # RuntimeConfigClient (polls dashboard /api/config)
├── validation.py # Schema validators
│
├── api/
│ ├── app.py # FastAPI factory, lifespan (clients + free/premium orchestrators)
│ ├── dependencies.py # Bearer auth, concurrency limiter, tier resolver
│ ├── middleware.py # CombinedMiddleware (request_id + rate limit)
│ └── routes/
│ ├── gather.py # POST /v1/gather (main pipeline + brain cache read/write)
│ ├── search.py # POST /v1/search
│ ├── image_search.py # POST /v1/image-search
│ ├── fetch.py # POST /v1/fetch
│ ├── health.py # GET /health, /ready
│ └── info.py # GET /v1/info (catalog metadata)
│
├── schemas/ # Pydantic request/response models
│ ├── common.py # PageContent, ProviderHealth, error helpers
│ ├── search.py # SearchRequest/Response
│ ├── image_search.py # ImageSearchRequest/Response
│ ├── fetch.py # FetchRequest/Response
│ ├── browse.py # BrowseRequest/Response
│ ├── vision.py # VisionExtractRequest/Response
│ ├── evidence.py # EvidencePackRequest/Response
│ ├── context.py # Context detection schemas
│ └── gather.py # GatherRequest/Response (unified)
│
├── search/ # Provider implementations
│ ├── protocol.py # SearchProvider Protocol
│ ├── multi.py # MultiSearchClient (default, backwards-compat)
│ ├── paid.py # PaidSearchClient (rotation across paid providers)
│ ├── brave.py # Brave Search API
│ ├── tavily.py # Tavily API
│ ├── serpapi.py # SerpAPI
│ ├── linkup.py # Linkup API
│ └── exa.py # Exa API
│
├── metasearch/
│ └── client.py # SearXNGClient (free tier)
│
├── fetch/ # HTTP + readability extraction
├── browse/ # Playwright browser automation
├── vision/ # Screenshot + Vision LLM extraction
├── evidence/ # EvidencePacker (dedupe, snippets, scoring)
├── llm/ # LLMProviderChain (local -> OpenRouter -> OpenAI -> Anthropic)
├── context/ # Claim context detection helpers
├── events/
│ └── sink.py # DashboardEventSink (fire-and-forget telemetry)
└── brain/
├── client.py # BrainClient (gather + ingest HTTP client)
├── adapter.py # brain <-> web schema converters
├── quality.py # quality_ok_for_cache, brain_hit_acceptable
└── sink.py # BrainIngestSink (premium-only cache writer)
```
## SearXNG cluster
Located at `deploy/metasearch/`:
- 3 SearXNG instances (`docker.io/searxng/searxng:latest`) round-robin behind an nginx LB (`searxng-lb` -> port 55100).
- Each SearXNG instance has its own dedicated Valkey 8 cache (`searxng-redis-data-{1,2,3}`) for isolated state.
- Shared Tor SOCKS5 proxy (`dperson/torproxy`) for queries needing anonymity.
- Per-instance config under `deploy/metasearch/searxng-{1,2,3}/`.
- Caddyfile + reset script (`searxng-reset.sh`) included for ops.
- Article extraction in the API uses readability-lxml; JS-heavy pages fall through to Playwright (chromium) and finally to a vision-LLM screenshot pass.
## Configuration
All envs use the `WEB_` prefix (loaded via pydantic-settings). Highlights:
| Variable | Purpose |
|----------|---------|
| `WEB_SEARXNG_BASE_URL` | SearXNG LB endpoint (default `http://didiAI-web-searxng:8080`) — REQUIRED |
| `WEB_LLM_BASE_URL` / `WEB_VISION_BASE_URL` | Local LLM router (Qwen) for free tier |
| `WEB_TEXT_MODEL` / `WEB_VISION_MODEL` | Model names served by the LLM endpoint |
| `WEB_LLM_API_KEY` | Optional auth for the LLM endpoint |
| `WEB_BRAVE_API_KEY` / `WEB_TAVILY_API_KEY` / `WEB_SERPAPI_API_KEY` / `WEB_LINKUP_API_KEY` / `WEB_EXA_API_KEY` | Premium-tier provider keys (any subset) |
| `WEB_OPENROUTER_API_KEY` / `WEB_OPENROUTER_MODEL` | Premium-tier LLM provider |
| `WEB_OPENAI_API_KEY` / `WEB_ANTHROPIC_API_KEY` | Optional fallback LLM providers |
| `WEB_DASHBOARD_URL` / `WEB_DASHBOARD_TOKEN` | Dashboard event sink + runtime config polling |
| `WEB_EXTERNAL_URL` | Public URL advertised in OpenAPI/catalog |
| `WEB_HOST` / `WEB_PORT` | Bind (defaults `0.0.0.0:51100`) |
| `WEB_RATE_LIMIT_RPS` / `WEB_RATE_LIMIT_BURST` / `WEB_MAX_CONCURRENT_REQUESTS` | Throttling |
| `WEB_API_TOKENS` | Comma-separated bearer tokens for `/v1/*`; blank = open (VPN deploys) |
| `WEB_FETCH_*` / `WEB_BROWSE_*` / `WEB_VISION_*` / `WEB_EVIDENCE_*` | Tuning knobs (timeouts, viewport, dedupe threshold, snippet length) |
Full reference: `.env.example`.
## Deployment
- **API:** `cd deploy/ && docker compose --profile api up -d` (builds `didiai-web-api` from `deploy/Dockerfile`, joins networks `didi-network` + `didibrain`, exposes `51100:51100`).
- **SearXNG cluster:** `cd deploy/metasearch/ && docker compose up -d` (LB + 3x SearXNG + 3x Valkey + Tor).
- **Healthcheck:** `curl http://localhost:51100/health`.
- Compose files: `deploy/docker-compose.yml`, `deploy/metasearch/docker-compose.yaml`.
## Related docs
- Module README: `README.md` (quick-start, gather request schema, fallback chain diagram).
- API reference: `API.md`.
- Brain cache integration: `BRAIN_INTEGRATION.md`.
- AI platform CLAUDE.md (parent module conventions).
- Backend integration: agent-v3 brain client (`backend/services/orchestration-layer/agent-v3`) falls through to `POST /v1/gather` here when the brain returns a MISS or low-quality hit.

317
ai_platform/modules/web/README.md Normal file
View file

@ -0,0 +1,317 @@
# Web Module
Unified web module for search, fetch, browse, vision, and evidence gathering. Provides REST APIs for web content extraction with automatic fallback between methods.
## What It Does
This module is designed for **fact-checking pipelines**. Given a claim, it:
1. **Searches** the web for relevant sources (SearXNG metasearch)
2. **Fetches** page content using the best method available
3. **Extracts** relevant evidence snippets using LLM
4. Returns structured evidence for verification
## Prerequisites
**Required:**
- All global prerequisites (see main [README.md](../../README.md))
- SearXNG instance (deploy with `cd deploy/metasearch && docker compose up -d`)
- LLM Inference server (llm-inference module at port 14011)
**Optional:**
- OpenAI API key (for vision/LLM fallback)
- Anthropic API key (for vision/LLM fallback)
## Features
| Component | Description |
|-----------|-------------|
| **Search** | SearXNG metasearch client with site filtering, freshness, language |
| **Fetch** | HTTP content extraction with readability-lxml |
| **Browse** | Playwright-based JavaScript rendering for dynamic pages |
| **Vision** | Screenshot + Vision LLM for complex/protected pages |
| **Evidence** | Deduplicate + LLM snippet extraction + relevance scoring |
| **Orchestrator** | Auto-fallback chain: HTTP → Playwright → Vision LLM |
### Smart Features
- **PDF Filtering**: Automatically skips direct PDF URLs (can't extract text)
- **Thinking Mode Handling**: Strips `<think>` tags from reasoning models (Qwen3)
- **Auto-Fallback**: Escalates to more powerful methods when content is poor
- **Deduplication**: Removes duplicate content based on text similarity
## Installation
```bash
cd modules/web
# Install base dependencies
uv sync
# Install with all extras (fetch, browse, vision)
uv sync --all-extras
# Install Playwright browsers
uv run playwright install chromium
# Install with dev dependencies
uv sync --extra dev --all-extras
```
## Quick Start
### Docker Deployment (Recommended)
```bash
cd deploy/
# Configure environment
cat > .env << 'EOF'
WEB_SEARXNG_BASE_URL=http://localhost:55100
WEB_LLM_BASE_URL=http://didiAI-llm-api:14011
WEB_LLM_API_KEY=your-llm-api-key
WEB_VISION_MODEL=qwen-vl
WEB_TEXT_MODEL=qwen3-235b
WEB_LOG_LEVEL=INFO
EOF
# Start the server
docker compose --profile api up -d
# Check health
curl http://localhost:51100/health
```
### Example: Fact-Check a Claim
```bash
curl -X POST http://localhost:51100/v1/gather \
-H "Content-Type: application/json" \
-d '{
"claim": "Romania had the highest economic growth in the EU in 2024",
"search_queries": ["Romania GDP growth 2024 EU"],
"max_search_results": 5,
"extract_snippets": true,
"max_evidence_items": 5
}'
```
**Response:**
```json
{
"claim": "Romania had the highest economic growth in the EU in 2024",
"evidence": [
{
"url": "https://en.wikipedia.org/wiki/Economy_of_Romania",
"title": "Economy of Romania - Wikipedia",
"snippet": "Romania's nominal GDP reached approximately $423 billion in 2024, reflecting real growth of 0.9% that year...",
"relevance_score": 0.6
}
],
"stages": [
{"stage": "search", "success": true, "items_processed": 5},
{"stage": "fetch", "success": true, "items_processed": 4},
{"stage": "evidence", "success": true, "items_processed": 3}
],
"execution_time_ms": 40643.68
}
```
## API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/v1/gather` | POST | **Main endpoint** - unified pipeline |
| `/v1/search` | POST | Execute web search only |
| `/v1/image-search` | POST | Search for images |
| `/v1/fetch` | POST | Fetch URLs (HTTP + readability) |
| `/health` | GET | Health check |
| `/ready` | GET | Readiness probe |
### Gather Request Schema
```json
{
"claim": "The claim to verify",
"search_queries": ["optional", "custom", "queries"],
"max_search_results": 20,
"site_allowlist": ["reuters.com", "bbc.com"],
"site_blocklist": ["spam-site.com"],
"fetch_method": "auto",
"auto_fallback": true,
"extract_snippets": true,
"max_evidence_items": 15,
"dedupe": true
}
```
| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `claim` | string | required | The claim to gather evidence for |
| `search_queries` | array | null | Custom search queries (auto-generated if not provided) |
| `max_search_results` | int | 20 | Max search results (5-50) |
| `site_allowlist` | array | null | Only search these domains |
| `site_blocklist` | array | null | Exclude these domains |
| `fetch_method` | string | "auto" | "auto", "http", "browse", "vision" |
| `auto_fallback` | bool | true | Escalate on fetch failure |
| `extract_snippets` | bool | true | Use LLM for snippet extraction |
| `max_evidence_items` | int | 15 | Max items in final pack |
| `dedupe` | bool | true | Deduplicate evidence |
## Configuration
Configure via environment variables (prefix: `WEB_`):
| Variable | Default | Description |
|----------|---------|-------------|
| `WEB_SEARXNG_BASE_URL` | - | **Required** SearXNG instance URL |
| `WEB_LLM_BASE_URL` | - | **Required** LLM inference server URL |
| `WEB_LLM_API_KEY` | - | LLM API authentication token |
| `WEB_VISION_MODEL` | `qwen-vl` | Vision model for screenshots |
| `WEB_TEXT_MODEL` | `qwen3-235b` | Text model for snippets |
| `WEB_PORT` | `51100` | API server port |
| `WEB_OPENAI_API_KEY` | - | OpenAI fallback (optional) |
| `WEB_ANTHROPIC_API_KEY` | - | Anthropic fallback (optional) |
See `.env.example` for full configuration options.
## Auto-Fallback Logic
The module uses intelligent fallback to get the best content:
```
┌─────────────────────────────────────────────────────────┐
│ URL Input │
└─────────────────┬───────────────────────────────────────┘
▼
┌─────────────────────────────────────────────────────────┐
│ 1. PDF Check - Skip .pdf URLs (can't extract text) │
└─────────────────┬───────────────────────────────────────┘
▼
┌─────────────────────────────────────────────────────────┐
│ 2. HTTP Fetch (fastest, cheapest) │
│ - Uses readability-lxml for content extraction │
│ - Detects JS-heavy pages and insufficient content │
└─────────────────┬───────────────────────────────────────┘
▼ if text < min_length or JS detected
┌─────────────────────────────────────────────────────────┐
│ 3. Playwright Browse (renders JavaScript) │
│ - Waits for networkidle │
│ - Extracts rendered content │
└─────────────────┬───────────────────────────────────────┘
▼ if content still poor quality
┌─────────────────────────────────────────────────────────┐
│ 4. Vision LLM (screenshot → extract) │
│ - Takes full-page screenshot │
│ - Uses vision model to extract text │
└─────────────────────────────────────────────────────────┘
```
Enable with `fetch_method: "auto"` and `auto_fallback: true`.
## Architecture
```
src/web/
├── __init__.py # Package exports
├── config.py # Unified WebSettings
├── exceptions.py # Custom exceptions
├── logging.py # Structured logging
├── orchestrator.py # Auto-fallback pipeline orchestration
├── cli.py # CLI entry point
│
├── schemas/ # All request/response schemas
│ ├── common.py # PageContent, FailedUrl, PageImage, shared types
│ ├── search.py # SearchRequest/Response
│ ├── fetch.py # FetchRequest/Response
│ ├── browse.py # BrowseRequest/Response
│ ├── vision.py # VisionExtractRequest/Response
│ ├── evidence.py # EvidencePackRequest/Response
│ └── gather.py # GatherRequest/Response (unified)
│
├── llm/ # LLM provider abstraction
│ ├── __init__.py
│ └── provider.py # LLMProviderChain (local→OpenAI→Anthropic)
│
├── search/ # Search providers
│ ├── __init__.py
│ ├── protocol.py # SearchProvider Protocol
│ └── searxng.py # SearXNGClient
│
├── fetch/ # HTTP + readability extraction
│ └── client.py # FetchClient
│
├── browse/ # Playwright browser automation
│ └── client.py # BrowseClient
│
├── vision/ # Screenshot + Vision LLM
│ └── client.py # VisionClient (local + OpenAI + Anthropic)
│
├── evidence/ # Evidence processing
│ └── packer.py # EvidencePacker (dedupe, snippets, scoring)
│
└── api/
├── app.py # FastAPI app factory
├── dependencies.py # DI (auth, rate limiter)
├── middleware.py # RequestId, RateLimit
└── routes/
├── search.py # POST /v1/search
├── fetch.py # POST /v1/fetch
├── gather.py # POST /v1/gather (main endpoint)
└── health.py # /health, /ready
```
## Deployment
```bash
cd deploy/
# Configure environment (REQUIRED)
# Edit .env with your API keys
# Start server
docker compose --profile api up -d
# View logs
docker compose --profile api logs -f
# Stop server
docker compose --profile api down
```
### Port Allocation
| Port | Service |
|------|---------|
| 51100 | Web API |
### Network
The container joins the `deploy_default` network to communicate with:
- `didiAI-llm-api:14011` - LLM inference server
## Development
```bash
# Install dev dependencies
uv sync --extra dev --all-extras
# Run tests
uv run pytest
# Run tests with coverage
uv run pytest --cov=src/web --cov-report=term-missing
# Lint and format
uv run ruff check .
uv run ruff format .
```
## Dependencies on Other Modules
| Module | Purpose | Required |
|--------|---------|----------|
| llm-inference | LLM for snippet extraction & relevance scoring | Yes (for extract_snippets) |
## License
MIT

405
ai_platform/modules/web/benchmark/RESULTS.md Normal file
View file

@ -0,0 +1,405 @@
# Web API Benchmark Results
**Date:** 2026-02-08
**Base URL:** http://localhost:51100
**Iterations:** 3 (+ 1 warmup)
## Summary
| Endpoint | Scenarios | Avg Latency | Status |
|----------|-----------|-------------|--------|
| Health | 1 | ~10ms | OK |
| Search (`/v1/search`) | 8 | 1181ms | OK |
| Fetch (`/v1/fetch`) | 7 | 939ms | OK |
| Image Search (`/v1/image-search`) | 2 | 1469ms | 3 errors |
| Gather (`/v1/gather`) | 9 | 6165ms | OK |
---
## Health
### `health`
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| curl_ttfb_s | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 |
| curl_total_s | 0.01 | 0.01 | 0.01 | 0.01 | 0.01 |
---
## Search (`/v1/search`)
### `search-single`
**Config:** 1 query, max_results=10
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 574 | 661 | 650 | 749 | 760 |
| curl_ttfb_s | 0.58 | 0.67 | 0.66 | 0.75 | 0.76 |
| curl_total_s | 0.58 | 0.67 | 0.66 | 0.75 | 0.76 |
**Results:** 10 items returned
### `search-multi-query`
**Config:** 5 queries, max_results=10
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 1775 | 1936 | 1972 | 2053 | 2062 |
| curl_ttfb_s | 1.78 | 1.94 | 1.98 | 2.06 | 2.07 |
| curl_total_s | 1.78 | 1.94 | 1.98 | 2.06 | 2.07 |
**Results:** 50 items returned
### `search-max-queries`
**Config:** 10 queries, max_results=5
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 2776 | 2945 | 3012 | 3045 | 3049 |
| curl_ttfb_s | 2.78 | 2.95 | 3.02 | 3.05 | 3.05 |
| curl_total_s | 2.78 | 2.95 | 3.02 | 3.05 | 3.05 |
**Results:** 50 items returned
### `search-large-results`
**Config:** 2 queries, max_results=50
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 930 | 1087 | 1105 | 1213 | 1225 |
| curl_ttfb_s | 0.94 | 1.09 | 1.11 | 1.22 | 1.23 |
| curl_total_s | 0.94 | 1.09 | 1.11 | 1.22 | 1.23 |
**Results:** 37 items returned
### `search-freshness-day`
**Config:** 1 query, max_results=10, freshness=day
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 662 | 690 | 704 | 705 | 706 |
| curl_ttfb_s | 0.67 | 0.70 | 0.71 | 0.71 | 0.71 |
| curl_total_s | 0.67 | 0.70 | 0.71 | 0.71 | 0.71 |
**Results:** 10 items returned
### `search-freshness-week`
**Config:** 1 query, max_results=10, freshness=week
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 573 | 623 | 614 | 675 | 681 |
| curl_ttfb_s | 0.58 | 0.63 | 0.62 | 0.68 | 0.69 |
| curl_total_s | 0.58 | 0.63 | 0.62 | 0.68 | 0.69 |
**Results:** 10 items returned
### `search-site-filter`
**Config:** 1 query, max_results=20, site_allowlist=[wikipedia.org, python.org, realpython.com]
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 586 | 701 | 612 | 876 | 905 |
| curl_ttfb_s | 0.59 | 0.71 | 0.62 | 0.88 | 0.91 |
| curl_total_s | 0.59 | 0.71 | 0.62 | 0.88 | 0.91 |
**Results:** 9 items returned
### `search-non-english`
**Config:** 1 query (French), max_results=10, language=fr, country=FR
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 570 | 805 | 779 | 1036 | 1065 |
| curl_ttfb_s | 0.58 | 0.81 | 0.78 | 1.04 | 1.07 |
| curl_total_s | 0.58 | 0.81 | 0.79 | 1.04 | 1.07 |
**Results:** 10 items returned
---
## Fetch (`/v1/fetch`)
### `fetch-single`
**Config:** 1 URL
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 138 | 142 | 144 | 145 | 146 |
| curl_ttfb_s | 0.14 | 0.15 | 0.15 | 0.15 | 0.15 |
| curl_total_s | 0.14 | 0.15 | 0.15 | 0.15 | 0.15 |
**Results:** 1 item returned
### `fetch-multi`
**Config:** 5 URLs, parallel_fetches=5
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 481 | 512 | 496 | 552 | 558 |
| curl_ttfb_s | 0.49 | 0.52 | 0.50 | 0.56 | 0.57 |
| curl_total_s | 0.49 | 0.52 | 0.51 | 0.56 | 0.57 |
**Results:** 5 items returned
### `fetch-large`
**Config:** 10 URLs, parallel_fetches=10
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 3342 | 3521 | 3552 | 3658 | 3670 |
| curl_ttfb_s | 3.36 | 3.54 | 3.57 | 3.68 | 3.69 |
| curl_total_s | 3.36 | 3.54 | 3.57 | 3.68 | 3.69 |
**Results:** 9 items (1 failed — reuters.com timeout)
### `fetch-serial`
**Config:** 5 URLs, parallel_fetches=1
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 791 | 895 | 894 | 990 | 1001 |
| curl_ttfb_s | 0.80 | 0.90 | 0.90 | 1.00 | 1.01 |
| curl_total_s | 0.80 | 0.90 | 0.90 | 1.00 | 1.01 |
**Results:** 5 items returned
### `fetch-no-text`
**Config:** 3 URLs, extract_text=false, include_html=true
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 492 | 501 | 502 | 508 | 509 |
| curl_ttfb_s | 0.52 | 0.53 | 0.53 | 0.53 | 0.53 |
| curl_total_s | 0.52 | 0.53 | 0.53 | 0.54 | 0.54 |
**Results:** 3 items returned
### `fetch-no-fallback`
**Config:** 3 URLs, auto_fallback=false, method=http
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 497 | 501 | 501 | 505 | 505 |
| curl_ttfb_s | 0.51 | 0.51 | 0.51 | 0.51 | 0.51 |
| curl_total_s | 0.51 | 0.51 | 0.51 | 0.51 | 0.52 |
**Results:** 3 items returned
### `fetch-short-timeout`
**Config:** 3 URLs, timeout_seconds=5
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 485 | 498 | 496 | 511 | 512 |
| curl_ttfb_s | 0.49 | 0.51 | 0.50 | 0.52 | 0.52 |
| curl_total_s | 0.50 | 0.51 | 0.51 | 0.52 | 0.52 |
**Results:** 3 items returned
---
## Image Search (`/v1/image-search`)
### `image-search-small`
**Config:** 1 query, max_results=5
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 1365 | 1469 | 1487 | 1549 | 1555 |
| curl_ttfb_s | 1.37 | 1.47 | 1.49 | 1.55 | 1.56 |
| curl_total_s | 1.37 | 1.47 | 1.49 | 1.55 | 1.56 |
**Results:** 5 items returned
### `image-search-large`
**Config:** 2 queries, max_results=100
**Status:** 3/3 FAILED
**Error:** `internal_error: An unexpected error occurred`
> **Note:** Large image search result counts (100+) trigger internal errors. Needs investigation.
---
## Gather (`/v1/gather`)
### `gather-minimal`
**Config:** max_search_results=5, max_evidence_items=3
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 2499 | 2843 | 2641 | 3315 | 3390 |
| search_ms | 855 | 1172 | 1008 | 1588 | 1653 |
| fetch_ms | 1537 | 1575 | 1553 | 1626 | 1635 |
| evidence_ms | 84 | 84 | 84 | 85 | 85 |
**Results:** 3 evidence items
### `gather-default`
**Config:** max_search_results=10, max_evidence_items=8
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 4604 | 6402 | 5055 | 9099 | 9548 |
| search_ms | 901 | 1493 | 1120 | 2324 | 2458 |
| fetch_ms | 3183 | 4516 | 3762 | 6319 | 6603 |
| evidence_ms | 206 | 208 | 208 | 211 | 211 |
**Results:** 8 evidence items
### `gather-snippets`
**Config:** max_search_results=5, max_evidence_items=5, extract_snippets=true
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 8000 | 8074 | 8087 | 8130 | 8134 |
| search_ms | 595 | 908 | 1024 | 1098 | 1106 |
| fetch_ms | 1742 | 1764 | 1773 | 1777 | 1777 |
| evidence_ms | 5107 | 5370 | 5355 | 5619 | 5648 |
**Results:** 5 evidence items
> **Note:** LLM snippet extraction adds ~5s to evidence processing.
### `gather-high-parallel`
**Config:** max_search_results=20, max_evidence_items=10, parallel_fetches=10
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 3043 | 3704 | 3663 | 4333 | 4407 |
| search_ms | 797 | 1227 | 1389 | 1485 | 1496 |
| fetch_ms | 2223 | 2357 | 2260 | 2555 | 2588 |
**Results:** 0 evidence items (no matching content)
### `gather-serial-fetch`
**Config:** max_search_results=10, max_evidence_items=5, parallel_fetches=1
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 6100 | 9994 | 10926 | 12754 | 12957 |
| search_ms | 775 | 1478 | 982 | 2509 | 2678 |
| fetch_ms | 4209 | 5771 | 5171 | 7656 | 7932 |
**Results:** 0 evidence items
> **Note:** Serial fetching is 2-3x slower than parallel.
### `gather-no-fallback`
**Config:** max_search_results=10, max_evidence_items=5, fetch_method=http, auto_fallback=false
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 2475 | 3091 | 3339 | 3446 | 3458 |
| search_ms | 460 | 1055 | 895 | 1718 | 1809 |
| fetch_ms | 1061 | 1467 | 1593 | 1732 | 1747 |
| evidence_ms | 30 | 79 | 37 | 158 | 171 |
**Results:** 5 evidence items
### `gather-site-restricted`
**Config:** max_search_results=15, max_evidence_items=8, site_allowlist=[wikipedia.org, bbc.com, reuters.com]
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 5941 | 6925 | 6357 | 8266 | 8478 |
| search_ms | 3411 | 4141 | 3767 | 5098 | 5246 |
| fetch_ms | 2338 | 2530 | 2372 | 2828 | 2879 |
| evidence_ms | 184 | 206 | 209 | 223 | 224 |
**Results:** 8 evidence items
> **Note:** Site filtering increases search latency (SearXNG site: prefix).
### `gather-max-evidence`
**Config:** max_search_results=30, max_evidence_items=25
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 5780 | 6238 | 6123 | 6743 | 6812 |
| search_ms | 875 | 1005 | 921 | 1191 | 1221 |
| fetch_ms | 4796 | 5199 | 4885 | 5812 | 5916 |
**Results:** 0 evidence items
### `gather-large`
**Config:** max_search_results=20, max_evidence_items=15
| Metric | Min | Mean | Median | P95 | Max |
|--------|-----|------|--------|-----|-----|
| total_ms | 6053 | 8215 | 8421 | 9996 | 10171 |
| search_ms | 792 | 987 | 1047 | 1115 | 1122 |
| fetch_ms | 4603 | 6648 | 6789 | 8376 | 8552 |
| evidence_ms | 134 | 167 | 148 | 213 | 220 |
**Results:** 15 evidence items
---
## Key Findings
### Performance Rankings
**Fastest scenarios:**
1. `fetch-single` — 142ms
2. `fetch-short-timeout` — 498ms
3. `fetch-no-text` — 501ms
4. `fetch-no-fallback` — 501ms
5. `fetch-multi` — 512ms
**Slowest scenarios:**
1. `gather-serial-fetch` — 9994ms
2. `gather-large` — 8215ms
3. `gather-snippets` — 8074ms
4. `gather-site-restricted` — 6925ms
5. `gather-default` — 6402ms
### Issues Detected
- `image-search-large`: Internal error when max_results=100 — needs investigation
### Observations
| Finding | Impact |
|---------|--------|
| **Search scaling** | ~300ms per additional query (linear) |
| **Fetch parallelism** | parallel_fetches=5-10 is 2x faster than serial |
| **LLM snippets** | extract_snippets=true adds ~5s to evidence stage |
| **Site filtering** | site_allowlist adds 2-3s to search (SearXNG overhead) |
| **Image search limits** | max_results > ~50 causes internal errors |
### Recommendations
1. **Use parallel fetches** — Default of 5 is good, 10 for large workloads
2. **Avoid snippets for speed** — Only enable when LLM extraction is needed
3. **Site filters are expensive** — Use sparingly, prefer post-fetch filtering
4. **Image search cap** — Keep max_results ≤ 50 until bug is fixed

967
ai_platform/modules/web/benchmark/bench.py Normal file
View file

@ -0,0 +1,967 @@
#!/usr/bin/env python3
"""Benchmark runner for the web API.
Usage:
uv run python bench.py # basic run (default scenario)
uv run python bench.py -n 5 # 5 iterations with stats
uv run python bench.py --all -n 3 # all scenarios
uv run python bench.py --group search -n 3 # all search scenarios
uv run python bench.py -c 3 -n 5 # concurrent load test
uv run python bench.py -o out.json --tag "before-refactor" # save with label
"""
from __future__ import annotations
import argparse
import json
import math
import subprocess
import sys
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
# ---------------------------------------------------------------------------
# Benchmark URLs (reliable, fast, publicly available)
# ---------------------------------------------------------------------------
BENCH_URLS = [
"https://www.python.org/",
"https://en.wikipedia.org/wiki/Python_(programming_language)",
"https://docs.python.org/3/tutorial/index.html",
"https://httpbin.org/html",
"https://example.com",
"https://www.reuters.com/",
"https://en.wikipedia.org/wiki/Machine_learning",
"https://en.wikipedia.org/wiki/Artificial_intelligence",
"https://www.bbc.com/news",
"https://docs.python.org/3/library/asyncio.html",
]
# ---------------------------------------------------------------------------
# Scenarios
# ---------------------------------------------------------------------------
SCENARIOS: dict[str, dict] = {
# -------------------------------------------------------------------------
# Health baseline
# -------------------------------------------------------------------------
"health": {
"endpoint": "/health",
"method": "GET",
"body": None,
},
# -------------------------------------------------------------------------
# Search endpoint variations
# -------------------------------------------------------------------------
"search-single": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["Python programming language popularity"],
"max_results": 10,
},
},
"search-multi-query": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": [
"Python popularity 2025",
"JavaScript frameworks comparison",
"Rust programming adoption",
"machine learning trends",
"cloud computing market",
],
"max_results": 10,
},
},
"search-max-queries": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": [
"Python web frameworks",
"JavaScript runtime performance",
"Rust memory safety",
"Go concurrency patterns",
"TypeScript adoption rate",
"Kotlin multiplatform",
"Swift server side",
"C++ modern standards",
"Java virtual threads",
"Ruby on Rails 2025",
],
"max_results": 5,
},
},
"search-large-results": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["artificial intelligence news", "climate change research"],
"max_results": 50,
},
},
"search-freshness-day": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["breaking news today"],
"max_results": 10,
"freshness": "day",
},
},
"search-freshness-week": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["technology announcements"],
"max_results": 10,
"freshness": "week",
},
},
"search-site-filter": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["Python programming"],
"max_results": 20,
"site_allowlist": ["wikipedia.org", "python.org", "realpython.com"],
},
},
"search-non-english": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["intelligence artificielle actualités"],
"max_results": 10,
"language": "fr",
"country": "FR",
},
},
# -------------------------------------------------------------------------
# Fetch endpoint variations
# -------------------------------------------------------------------------
"fetch-single": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": [BENCH_URLS[0]],
"timeout_seconds": 30,
},
},
"fetch-multi": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:5],
"parallel_fetches": 5,
"timeout_seconds": 30,
},
},
"fetch-large": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS,
"parallel_fetches": 10,
"timeout_seconds": 45,
},
},
"fetch-serial": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:5],
"parallel_fetches": 1,
"timeout_seconds": 60,
},
},
"fetch-no-text": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:3],
"extract_text": False,
"include_html": True,
"timeout_seconds": 30,
},
},
"fetch-no-fallback": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:3],
"auto_fallback": False,
"method": "http",
"timeout_seconds": 30,
},
},
"fetch-short-timeout": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:3],
"timeout_seconds": 5,
},
},
# -------------------------------------------------------------------------
# Image search endpoint variations
# -------------------------------------------------------------------------
"image-search-small": {
"endpoint": "/v1/image-search",
"method": "POST",
"body": {
"queries": ["cute cats"],
"max_results": 5,
},
},
"image-search-large": {
"endpoint": "/v1/image-search",
"method": "POST",
"body": {
"queries": ["nature photography", "city skyline"],
"max_results": 100,
},
},
# -------------------------------------------------------------------------
# Gather endpoint variations (unified pipeline)
# -------------------------------------------------------------------------
"gather-minimal": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is the most popular programming language in 2025",
"search_queries": ["Python popularity 2025"],
"max_search_results": 5,
"max_evidence_items": 3,
"extract_snippets": False,
"timeout_seconds": 60,
},
},
"gather-default": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is the most popular programming language in 2025",
"search_queries": [
"Python popularity 2025",
"TIOBE index programming languages",
],
"max_search_results": 10,
"max_evidence_items": 8,
"extract_snippets": False,
"timeout_seconds": 90,
},
},
"gather-snippets": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is the most popular programming language in 2025",
"search_queries": ["Python popularity 2025"],
"max_search_results": 5,
"max_evidence_items": 5,
"extract_snippets": True,
"timeout_seconds": 120,
},
},
"gather-high-parallel": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Artificial intelligence is transforming healthcare",
"search_queries": [
"AI healthcare applications",
"machine learning medical diagnosis",
],
"max_search_results": 20,
"max_evidence_items": 10,
"parallel_fetches": 10,
"extract_snippets": False,
"timeout_seconds": 90,
},
},
"gather-serial-fetch": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Electric vehicles are becoming mainstream",
"search_queries": ["electric vehicle adoption 2025"],
"max_search_results": 10,
"max_evidence_items": 5,
"parallel_fetches": 1,
"extract_snippets": False,
"timeout_seconds": 120,
},
},
"gather-no-fallback": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Renewable energy costs are declining",
"search_queries": ["renewable energy cost trends"],
"max_search_results": 10,
"max_evidence_items": 5,
"fetch_method": "http",
"auto_fallback": False,
"extract_snippets": False,
"timeout_seconds": 60,
},
},
"gather-site-restricted": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is widely used in data science",
"search_queries": ["Python data science"],
"max_search_results": 15,
"max_evidence_items": 8,
"site_allowlist": ["wikipedia.org", "bbc.com", "reuters.com"],
"extract_snippets": False,
"timeout_seconds": 90,
},
},
"gather-max-evidence": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Climate change is accelerating",
"search_queries": [
"climate change scientific evidence",
"global warming data 2025",
],
"max_search_results": 30,
"max_evidence_items": 25,
"extract_snippets": False,
"timeout_seconds": 180,
},
},
# Legacy aliases for backward compatibility
"default": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is the most popular programming language in 2025",
"search_queries": [
"Python popularity 2025",
"TIOBE index programming languages",
],
"max_search_results": 5,
"max_evidence_items": 5,
"extract_snippets": True,
"timeout_seconds": 90,
},
},
"gather-large": {
"endpoint": "/v1/gather",
"method": "POST",
"body": {
"claim": "Python is the most popular programming language in 2025",
"search_queries": [
"Python popularity 2025",
"TIOBE index programming languages",
],
"max_search_results": 20,
"max_evidence_items": 15,
"extract_snippets": False,
"timeout_seconds": 120,
},
},
"search-only": {
"endpoint": "/v1/search",
"method": "POST",
"body": {
"queries": ["Python popularity 2025", "TIOBE index 2025"],
"max_results": 10,
},
},
"image-search": {
"endpoint": "/v1/image-search",
"method": "POST",
"body": {
"queries": [
"Python programming language",
"machine learning visualization",
],
"max_results": 10,
},
},
"fetch-only": {
"endpoint": "/v1/fetch",
"method": "POST",
"body": {
"urls": BENCH_URLS[:3],
"extract_text": True,
"timeout_seconds": 30,
},
},
}
# ---------------------------------------------------------------------------
# Scenario Groups
# ---------------------------------------------------------------------------
SCENARIO_GROUPS: dict[str, list[str]] = {
"search": [
"search-single",
"search-multi-query",
"search-max-queries",
"search-large-results",
"search-freshness-day",
"search-freshness-week",
"search-site-filter",
"search-non-english",
],
"fetch": [
"fetch-single",
"fetch-multi",
"fetch-large",
"fetch-serial",
"fetch-no-text",
"fetch-no-fallback",
"fetch-short-timeout",
],
"image": [
"image-search-small",
"image-search-large",
],
"gather": [
"gather-minimal",
"gather-default",
"gather-snippets",
"gather-high-parallel",
"gather-serial-fetch",
"gather-no-fallback",
"gather-site-restricted",
"gather-max-evidence",
],
"quick": [
"health",
"search-single",
"fetch-single",
"gather-minimal",
],
}
# curl -w format string for transport-level timing
CURL_WRITE_OUT = json.dumps(
{
"status_code": "%{http_code}",
"time_total": "%{time_total}",
"time_connect": "%{time_connect}",
"time_starttransfer": "%{time_starttransfer}",
"size_download": "%{size_download}",
}
)
# ---------------------------------------------------------------------------
# RunResult
# ---------------------------------------------------------------------------
@dataclass
class RunResult:
scenario: str
iteration: int
status_code: int = 0
curl_time_total: float = 0.0
curl_time_connect: float = 0.0
curl_time_starttransfer: float = 0.0
api_time_ms: float | None = None
stages: dict[str, float] = field(default_factory=dict)
evidence_items: int | None = None
failed_items: int | None = None
queries_processed: int | None = None
tokens_used: int | None = None
error: str | None = None
# ---------------------------------------------------------------------------
# Runner
# ---------------------------------------------------------------------------
def run_single(base_url: str, scenario_name: str, timeout: int) -> RunResult:
"""Execute a single benchmark request via curl."""
scenario = SCENARIOS[scenario_name]
url = f"{base_url}{scenario['endpoint']}"
result = RunResult(scenario=scenario_name, iteration=0)
with tempfile.NamedTemporaryFile(suffix=".json", delete=True) as body_file:
cmd = [
"curl",
"-s",
"--max-time",
str(timeout),
"-w",
CURL_WRITE_OUT,
"-o",
body_file.name,
]
if scenario["method"] == "POST" and scenario["body"] is not None:
cmd += [
"-X",
"POST",
"-H",
"Content-Type: application/json",
"-d",
json.dumps(scenario["body"]),
]
cmd.append(url)
try:
proc = subprocess.run(
cmd, capture_output=True, text=True, timeout=timeout + 10
)
except subprocess.TimeoutExpired:
result.error = "subprocess timeout"
return result
except Exception as exc:
result.error = str(exc)
return result
# Parse curl write-out (appended to stdout)
try:
curl_stats = json.loads(proc.stdout)
result.status_code = int(curl_stats["status_code"])
result.curl_time_total = float(curl_stats["time_total"])
result.curl_time_connect = float(curl_stats["time_connect"])
result.curl_time_starttransfer = float(curl_stats["time_starttransfer"])
except (json.JSONDecodeError, KeyError, ValueError):
result.error = f"failed to parse curl output: {proc.stdout[:200]}"
return result
if result.status_code == 0:
stderr_snippet = proc.stderr[:200] if proc.stderr else "no stderr"
result.error = f"curl failed (code 0): {stderr_snippet}"
return result
# Parse response body
try:
with open(body_file.name) as f:
body = json.load(f)
except (json.JSONDecodeError, OSError):
body = None
if body and isinstance(body, dict):
result.api_time_ms = body.get("execution_time_ms")
# Gather-specific fields (stages, evidence stats)
for stage in body.get("stages", []):
if isinstance(stage, dict) and "stage" in stage:
result.stages[stage["stage"]] = stage.get("duration_ms", 0.0)
stats = body.get("evidence_stats")
if isinstance(stats, dict):
result.evidence_items = stats.get("output_items")
result.tokens_used = stats.get("tokens_used")
# Search / image-search specific
if "total_results" in body:
result.evidence_items = body["total_results"]
if "queries_processed" in body:
result.queries_processed = body["queries_processed"]
# Fetch-specific
if "total_fetched" in body:
result.evidence_items = body["total_fetched"]
if "total_failed" in body:
result.failed_items = body["total_failed"]
# Error in response body
if result.status_code >= 400:
detail = body.get("detail", body.get("error", ""))
if detail:
result.error = str(detail)[:200]
return result
# ---------------------------------------------------------------------------
# Stats
# ---------------------------------------------------------------------------
def _percentile(sorted_vals: list[float], p: float) -> float:
"""Compute the p-th percentile from a sorted list."""
if not sorted_vals:
return 0.0
k = (len(sorted_vals) - 1) * (p / 100.0)
f = math.floor(k)
c = math.ceil(k)
if f == c:
return sorted_vals[int(k)]
return sorted_vals[f] * (c - k) + sorted_vals[c] * (k - f)
def compute_stats(values: list[float]) -> dict[str, float]:
"""Compute min/mean/median/p95/max for a list of values."""
if not values:
return {"min": 0, "mean": 0, "median": 0, "p95": 0, "max": 0}
s = sorted(values)
return {
"min": round(s[0], 2),
"mean": round(sum(s) / len(s), 2),
"median": round(_percentile(s, 50), 2),
"p95": round(_percentile(s, 95), 2),
"max": round(s[-1], 2),
}
def compute_all_stats(results: list[RunResult]) -> dict[str, dict[str, float]]:
"""Compute stats across all metric dimensions."""
out: dict[str, dict[str, float]] = {}
api_times = [r.api_time_ms for r in results if r.api_time_ms is not None]
if api_times:
out["total_ms"] = compute_stats(api_times)
# Collect all stage names
stage_names: set[str] = set()
for r in results:
stage_names.update(r.stages.keys())
for name in sorted(stage_names):
vals = [r.stages[name] for r in results if name in r.stages]
if vals:
out[f"{name}_ms"] = compute_stats(vals)
ttfb = [r.curl_time_starttransfer for r in results if r.curl_time_starttransfer > 0]
if ttfb:
out["curl_ttfb_s"] = compute_stats(ttfb)
totals = [r.curl_time_total for r in results if r.curl_time_total > 0]
if totals:
out["curl_total_s"] = compute_stats(totals)
return out
# ---------------------------------------------------------------------------
# Reporter
# ---------------------------------------------------------------------------
def _fmt_stages(r: RunResult) -> str:
if not r.stages:
return ""
parts = [f"{k}:{v:.0f}" for k, v in r.stages.items()]
return f" [{' '.join(parts)}]"
def _fmt_items(r: RunResult) -> str:
parts = []
if r.evidence_items is not None:
parts.append(f"{r.evidence_items} items")
if r.failed_items is not None and r.failed_items > 0:
parts.append(f"{r.failed_items} failed")
if r.queries_processed is not None:
parts.append(f"{r.queries_processed} queries")
return f" ({', '.join(parts)})" if parts else ""
def print_run(r: RunResult, quiet: bool) -> None:
"""Print a single run result line."""
if quiet:
return
if r.error:
print(f" #{r.iteration} ERROR: {r.error}")
return
time_str = (
f"{r.api_time_ms:.0f}ms"
if r.api_time_ms is not None
else f"{r.curl_time_total:.2f}s"
)
print(f" #{r.iteration} {time_str}{_fmt_stages(r)}{_fmt_items(r)}")
def print_stats_table(stats: dict[str, dict[str, float]]) -> None:
"""Print the statistics summary table."""
if not stats:
return
header = (
f" {'Metric':<16} {'min':>8} {'mean':>8} {'median':>8} {'p95':>8} {'max':>8}"
)
print()
print(header)
print(f" {'─' * 56}")
for metric, vals in stats.items():
fmt = ".0f" if metric.endswith("_ms") else ".2f"
print(
f" {metric:<16}"
f" {vals['min']:>8{fmt}}"
f" {vals['mean']:>8{fmt}}"
f" {vals['median']:>8{fmt}}"
f" {vals['p95']:>8{fmt}}"
f" {vals['max']:>8{fmt}}"
)
print()
def print_docker_logs(container: str) -> None:
"""Print recent docker logs."""
print(f"=== Recent logs ({container}) ===")
try:
proc = subprocess.run(
["docker", "logs", container, "--tail", "10"],
capture_output=True,
text=True,
timeout=5,
)
output = proc.stdout or proc.stderr or "(no output)"
print(output.rstrip())
except (subprocess.TimeoutExpired, FileNotFoundError):
print(" (docker logs unavailable)")
print()
# ---------------------------------------------------------------------------
# Main execution
# ---------------------------------------------------------------------------
def run_scenario(
base_url: str,
scenario_name: str,
iterations: int,
warmup: int,
concurrency: int,
timeout: int,
quiet: bool,
) -> list[RunResult]:
"""Run a scenario for the given number of iterations."""
scenario = SCENARIOS[scenario_name]
method = scenario["method"]
endpoint = scenario["endpoint"]
print(f"=== {scenario_name} ({method} {endpoint}) ===")
# Warmup
if warmup > 0:
sys.stdout.write(" warmup...")
sys.stdout.flush()
t0 = time.monotonic()
for _ in range(warmup):
run_single(base_url, scenario_name, timeout)
elapsed = time.monotonic() - t0
print(f"done ({elapsed:.1f}s)")
results: list[RunResult] = []
if concurrency <= 1:
# Sequential mode
for i in range(1, iterations + 1):
r = run_single(base_url, scenario_name, timeout)
r.iteration = i
results.append(r)
print_run(r, quiet)
else:
# Concurrent mode
for i in range(1, iterations + 1):
batch: list[RunResult] = []
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = {
pool.submit(run_single, base_url, scenario_name, timeout): c
for c in range(concurrency)
}
for fut in as_completed(futures):
r = fut.result()
r.iteration = i
batch.append(r)
results.extend(batch)
if not quiet:
ok = sum(1 for r in batch if r.error is None)
times = [r.curl_time_total for r in batch if r.error is None]
avg = sum(times) / len(times) if times else 0
print(
f" #{i} {concurrency} reqs {ok}/{concurrency} ok avg {avg:.2f}s"
)
# Stats
successful = [r for r in results if r.error is None]
if len(successful) >= 2:
stats = compute_all_stats(successful)
print_stats_table(stats)
elif successful:
print()
else:
print(" No successful runs.\n")
# Concurrency throughput summary
if concurrency > 1 and successful:
total_time = sum(r.curl_time_total for r in successful)
wall_time = total_time / concurrency
rps = len(successful) / wall_time if wall_time > 0 else 0
print(
f" throughput: ~{rps:.1f} req/s ({len(successful)} reqs, {concurrency} concurrent)"
)
print()
return results
def build_json_output(
base_url: str,
all_results: dict[str, list[RunResult]],
tag: str | None = None,
) -> dict:
"""Build the JSON output structure."""
scenarios_out = {}
for name, results in all_results.items():
successful = [r for r in results if r.error is None]
stats = compute_all_stats(successful) if len(successful) >= 2 else {}
scenarios_out[name] = {
"config": SCENARIOS[name],
"runs": [asdict(r) for r in results],
"stats": stats,
}
output: dict = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"base_url": base_url,
"scenarios": scenarios_out,
}
if tag:
output["tag"] = tag
return output
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="web API benchmark runner",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=(
f"Available scenarios: {', '.join(SCENARIOS)}\n"
f"Available groups: {', '.join(SCENARIO_GROUPS)}"
),
)
parser.add_argument(
"--base-url",
default="http://localhost:51100",
help="API base URL (default: http://localhost:51100)",
)
parser.add_argument(
"-n",
"--iterations",
type=int,
default=3,
help="runs per scenario (default: 3)",
)
parser.add_argument(
"--warmup",
type=int,
default=1,
help="warmup runs excluded from stats (default: 1)",
)
parser.add_argument(
"-c",
"--concurrency",
type=int,
default=1,
help="parallel requests per iteration (default: 1)",
)
parser.add_argument(
"-s",
"--scenario",
default=None,
choices=list(SCENARIOS.keys()),
help="single scenario to run",
)
parser.add_argument(
"--group",
default=None,
choices=list(SCENARIO_GROUPS.keys()),
help="run all scenarios in a group",
)
parser.add_argument(
"--all",
action="store_true",
help="run all scenarios",
)
parser.add_argument(
"--timeout",
type=int,
default=120,
help="curl --max-time in seconds (default: 120)",
)
parser.add_argument(
"-o",
"--output",
default=None,
help="write JSON results to file",
)
parser.add_argument(
"--tag",
default=None,
help="label for this benchmark run (saved in JSON output)",
)
parser.add_argument(
"--docker-logs",
default="didiAI-web-api",
help="container name for log tail (empty to skip, default: didiAI-web-api)",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
help="suppress per-run output",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> None:
args = parse_args(argv)
# Determine which scenarios to run
if args.all:
# All scenarios except legacy aliases
legacy = {"default", "search-only", "image-search", "fetch-only"}
scenario_names = [s for s in SCENARIOS if s not in legacy]
elif args.group:
scenario_names = SCENARIO_GROUPS[args.group]
elif args.scenario:
scenario_names = [args.scenario]
else:
scenario_names = ["gather-default"]
print("bench.py — web API benchmark")
print(f"base_url: {args.base_url}")
if args.tag:
print(f"tag: {args.tag}")
print(f"scenarios: {', '.join(scenario_names)}")
warmup_note = f" (+ {args.warmup} warmup)" if args.warmup else ""
conc_note = f", concurrency {args.concurrency}" if args.concurrency > 1 else ""
print(f"iterations: {args.iterations}{warmup_note}{conc_note}")
print()
all_results: dict[str, list[RunResult]] = {}
for name in scenario_names:
results = run_scenario(
base_url=args.base_url,
scenario_name=name,
iterations=args.iterations,
warmup=args.warmup,
concurrency=args.concurrency,
timeout=args.timeout,
quiet=args.quiet,
)
all_results[name] = results
# Docker logs
if args.docker_logs:
print_docker_logs(args.docker_logs)
# JSON output
if args.output:
output = build_json_output(args.base_url, all_results, args.tag)
with open(args.output, "w") as f:
json.dump(output, f, indent=2)
print(f"Results written to {args.output}")
if __name__ == "__main__":
main()

71
ai_platform/modules/web/deploy/Dockerfile Normal file
View file

@ -0,0 +1,71 @@
# Web API Server
# Multi-stage build for smaller final image
FROM python:3.11.12-slim AS builder
# Install uv
COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /usr/local/bin/uv
WORKDIR /app
# Layer 1: Install dependencies only (cached unless pyproject.toml changes)
COPY pyproject.toml README.md ./
RUN uv sync --no-dev --all-extras --no-install-project
# Layer 2: Install Playwright browsers (cached with dependencies)
RUN uv run playwright install chromium --with-deps
# Layer 3: Copy source and install project (fast — deps already cached)
COPY src/ ./src/
RUN uv sync --no-dev --all-extras
# Production image
FROM python:3.11.12-slim
WORKDIR /app
# Install Playwright dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
libnss3 libnspr4 libdbus-1-3 libatk1.0-0 libatk-bridge2.0-0 \
libcups2 libdrm2 libxkbcommon0 libatspi2.0-0 libxcomposite1 \
libxdamage1 libxfixes3 libxrandr2 libgbm1 libasound2 \
libcairo2 libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf-2.0-0 \
libgtk-3-0 libx11-6 libx11-xcb1 libxcb1 libxext6 libxrender1 \
fonts-liberation fonts-noto-color-emoji libopus0 libharfbuzz0b \
libglib2.0-0 libxshmfence1 \
&& rm -rf /var/lib/apt/lists/*
# Create non-root user
RUN groupadd -r webuser && useradd -r -g webuser -d /home/webuser -m webuser
# Copy virtual environment from builder
COPY --from=builder /app/.venv /app/.venv
# Copy Playwright browsers to non-root user's home
COPY --from=builder /root/.cache/ms-playwright /home/webuser/.cache/ms-playwright
# Copy source code
COPY src/ ./src/
# Set ownership
RUN chown -R webuser:webuser /app /home/webuser
# Set environment variables
ENV PATH="/app/.venv/bin:$PATH"
ENV PYTHONUNBUFFERED=1
ENV WEB_HOST=0.0.0.0
ENV WEB_PORT=51100
ENV PLAYWRIGHT_BROWSERS_PATH=/home/webuser/.cache/ms-playwright
# Switch to non-root user
USER webuser
# Health check
HEALTHCHECK --interval=30s --timeout=10s --retries=3 \
CMD python -c "import urllib.request; import os; urllib.request.urlopen(f'http://localhost:{os.environ.get(\"WEB_PORT\", 51100)}/health')" || exit 1
# Expose port
EXPOSE 51100
# Run the server
CMD ["python", "-m", "web.cli"]

105
ai_platform/modules/web/deploy/deploy.sh Normal file
View file

@ -0,0 +1,105 @@
#!/usr/bin/env bash
#
# Docker Compose Startup Script for Web Module
#
# Usage: ./deploy/deploy.sh [OPTIONS]
#
# Options:
# --profile <api> Docker compose profile (default: api)
# --detach, -d Run in detached mode
# --down Stop and remove containers
# --logs Show logs
# --help, -h Show this help message
#
# Required: Set environment variables in .env file or export them before running.
# See .env.example for the full list of required variables.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Load .env file if it exists
ENV_FILE=""
if [[ -f "$SCRIPT_DIR/.env" ]]; then
ENV_FILE="$SCRIPT_DIR/.env"
elif [[ -f "$SCRIPT_DIR/../.env" ]]; then
ENV_FILE="$SCRIPT_DIR/../.env"
fi
if [[ -n "$ENV_FILE" ]]; then
echo "Loading environment from: $ENV_FILE"
set -a
source "$ENV_FILE"
set +a
fi
PROFILE="api"
DETACH=""
ACTION="up"
show_help() {
sed -n '2,15p' "$0" | sed 's/^# //' | sed 's/^#//'
exit 0
}
check_required_var() {
local var_name="$1"
if [[ -z "${!var_name:-}" ]]; then
echo "ERROR: Required environment variable $var_name is not set"
echo "Set it in .env file or export it before running this script"
exit 1
fi
}
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--profile)
PROFILE="$2"
shift 2
;;
--detach|-d)
DETACH="-d"
shift
;;
--down)
ACTION="down"
shift
;;
--logs)
ACTION="logs"
shift
;;
--help|-h)
show_help
;;
*)
echo "Unknown option: $1"
echo "Use --help for usage information"
exit 1
;;
esac
done
# Check required variables
check_required_var "WEB_LLM_BASE_URL"
check_required_var "WEB_EXTERNAL_URL"
check_required_var "WEB_SEARXNG_BASE_URL"
cd "$SCRIPT_DIR"
case $ACTION in
up)
echo "Starting Web module with profile: $PROFILE"
echo ""
# shellcheck disable=SC2086
exec docker compose --profile "$PROFILE" up $DETACH
;;
down)
echo "Stopping Web module containers..."
exec docker compose --profile "$PROFILE" down
;;
logs)
exec docker compose --profile "$PROFILE" logs -f
;;
esac

53
ai_platform/modules/web/deploy/docker-compose.yml Normal file
View file

@ -0,0 +1,53 @@
# Web Module - Docker Compose Configuration
#
# Port Allocation (Dev API Gateway: 51100):
# 51100 - Web API (fact-checking service)
#
# Naming Convention: didiAI-{module}-{service}
#
# Profiles:
# api - Web API server
networks:
didi-network:
external: true # single shared network for all DIDI + AI platform stacks
services:
# ==========================================================================
# Web API Server
# ==========================================================================
web-api:
container_name: didiAI-web-api
image: didiai-web-api
build:
context: ..
dockerfile: deploy/Dockerfile
ports:
- "51100:51100"
shm_size: '2gb'
networks:
- didi-network
env_file:
- .env
environment:
# Container-internal overrides (always set regardless of .env)
- WEB_HOST=0.0.0.0
- WEB_PORT=51100
# Tier-3 search fallback (cloak stealth-browser scraping service).
# Default OFF; flip to true once cloak service is verified healthy.
- WEB_CLOAK_ENABLED=${WEB_CLOAK_ENABLED:-false}
- WEB_CLOAK_URL=${WEB_CLOAK_URL:-http://didiAI-cloak:8770}
- WEB_CLOAK_FALLBACK_THRESHOLD=${WEB_CLOAK_FALLBACK_THRESHOLD:-5}
- WEB_CLOAK_TIMEOUT_SEC=${WEB_CLOAK_TIMEOUT_SEC:-20}
# OTel — traces to Jaeger via OTel Collector
- OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://didi-otel-collector:4317}
- OTEL_SERVICE_NAME=didiAI-web-api
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:51100/health')"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
profiles:
- api

91
ai_platform/modules/web/deploy/metasearch/Caddyfile Normal file
View file

@ -0,0 +1,91 @@
{
admin off
log {
output stderr
format filter {
# Preserves first 8 bits from IPv4 and 32 bits from IPv6
request>remote_ip ip_mask 8 32
request>client_ip ip_mask 8 32
# Remove identificable information
request>remote_port delete
request>headers delete
request>uri query {
delete url
delete h
delete q
}
}
}
servers {
client_ip_headers X-Forwarded-For X-Real-IP
# Allow the following IP to passthrough the "X-Forwarded-*" headers to SearXNG
# https://caddyserver.com/docs/caddyfile/options#trusted-proxies
trusted_proxies static private_ranges
trusted_proxies_strict
}
}
{$SEARXNG_HOSTNAME}
tls {$SEARXNG_TLS}
encode zstd gzip
@api {
path /config
path /healthz
path /stats/errors
path /stats/checker
}
@static {
path /static/*
}
@imageproxy {
path /image_proxy
}
header {
# CSP (https://content-security-policy.com)
Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self' https:; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self'; img-src * data:; frame-src https:;"
# Disable browser features
Permissions-Policy "accelerometer=(),camera=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),payment=(),usb=()"
# Only allow same-origin requests
Referrer-Policy "same-origin"
# Prevent MIME type sniffing from the declared Content-Type
X-Content-Type-Options "nosniff"
# Comment header to allow indexing by search engines
X-Robots-Tag "noindex, nofollow, noarchive, nositelinkssearchbox, nosnippet, notranslate, noimageindex"
# enable HSTS
# WARNING: Once this value is set, the site must continue to support HTTPS until the expiry time is reached.
# Strict-Transport-Security max-age=15768000;
# Remove "Server" header
-Server
}
header @api {
Access-Control-Allow-Methods "GET, OPTIONS"
Access-Control-Allow-Origin "*"
}
route {
# Cache policy
header Cache-Control "no-cache"
header @static Cache-Control "public, max-age=30, stale-while-revalidate=60"
header @imageproxy Cache-Control "public, max-age=3600"
}
# SearXNG
reverse_proxy localhost:8080

661
ai_platform/modules/web/deploy/metasearch/LICENSE Normal file
View file

@ -0,0 +1,661 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<https://www.gnu.org/licenses/>.

119
ai_platform/modules/web/deploy/metasearch/README.upstream.md Normal file
View file

@ -0,0 +1,119 @@
# searxng-docker
Create a new SearXNG instance in five minutes using Docker
## What is included?
| Name | Description | Docker image | Dockerfile |
|-----------------------------------------------|----------------------------------------------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [Caddy](https://github.com/caddyserver/caddy) | Reverse proxy (create a LetsEncrypt certificate automatically) | [docker.io/library/caddy:2-alpine](https://hub.docker.com/_/caddy) | [Dockerfile](https://github.com/caddyserver/caddy-docker/blob/master/Dockerfile.tmpl) |
| [SearXNG](https://github.com/searxng/searxng) | SearXNG by itself | [docker.io/searxng/searxng:latest](https://hub.docker.com/r/searxng/searxng) | [builder.dockerfile](https://github.com/searxng/searxng/blob/master/container/builder.dockerfile) [dist.dockerfile](https://github.com/searxng/searxng/blob/master/container/dist.dockerfile) |
| [Valkey](https://github.com/valkey-io/valkey) | In-memory database | [docker.io/valkey/valkey:8-alpine](https://hub.docker.com/r/valkey/valkey) | [Dockerfile](https://github.com/valkey-io/valkey-container/blob/mainline/Dockerfile.template) |
## How to use it
There are two ways to host SearXNG. The first one doesn't require any prior knowledge about self-hosting and thus is
recommended for beginners. It includes caddy as a reverse proxy and automatically deals with the TLS certificates for
you. The second one is recommended for more advanced users that already have their own reverse proxy (e.g. Nginx,
HAProxy, ...) and probably some other services running on their machine. The first few steps are the same for both
installation methods however.
1. [Install docker](https://docs.docker.com/install/)
2. Get searxng-docker
```shell
cd /usr/local
git clone https://github.com/searxng/searxng-docker.git
cd searxng-docker
```
3. Edit the [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file to set the hostname and an email
4. Generate the secret key `sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" searxng/settings.yml`
On a Mac: `sed -i '' "s|ultrasecretkey|$(openssl rand -hex 32)|g" searxng/settings.yml`
5. Edit [searxng/settings.yml](https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml) according to
your needs
> [!NOTE]
> Windows users can use the following powershell script to generate the secret key:
> ```powershell
> $randomBytes = New-Object byte[] 32
> (New-Object Security.Cryptography.RNGCryptoServiceProvider).GetBytes($randomBytes)
> $secretKey = -join ($randomBytes | ForEach-Object { "{0:x2}" -f $_ })
> (Get-Content searxng/settings.yml) -replace 'ultrasecretkey', $secretKey | Set-Content searxng/settings.yml
> ```
### Method 1: With Caddy included (recommended for beginners)
6. Run SearXNG in the background: `docker compose up -d`
### Method 2: Bring your own reverse proxy (experienced users)
6. Remove the caddy related parts in `docker-compose.yaml` such as the caddy service and its volumes.
7. Point your reverse proxy to the port set for the `searxng` service in `docker-compose.yml` (8080 by default).
8. Generate and configure the required TLS certificates with the reverse proxy of your choice.
9. Run SearXNG in the background: `docker compose up -d`
> [!NOTE]
> You can change the port `searxng` listens on inside the docker container (e.g. if you want to operate in `host`
> network mode) with the `BIND_ADDRESS` environment variable (defaults to `[::]:8080`). The environment variable can be
> set directly inside `docker-compose.yaml`.
## Troubleshooting - How to access the logs
To access the logs from all the containers use: `docker compose logs -f`.
To access the logs of one specific container:
- Caddy: `docker compose logs -f caddy`
- SearXNG: `docker compose logs -f searxng`
- Valkey: `docker compose logs -f redis`
### Start SearXNG with systemd
You can skip this step if you don't use systemd.
1. Copy the service template file:
```sh
cp searxng-docker.service.template searxng-docker.service
```
2. Edit the content of ```WorkingDirectory``` in the ```searxng-docker.service``` file (only if the installation path is
different from ```/usr/local/searxng-docker```)
3. Enable the service:
```sh
systemctl enable $(pwd)/searxng-docker.service
```
4. Start the service:
```sh
systemctl start searxng-docker.service
```
**Note:** Ensure the service file path matches your installation directory before enabling it.
## Multi Architecture Docker images
Supported architecture:
- amd64
- arm64
- arm/v7
## How to update ?
To update the SearXNG stack:
```sh
git pull
docker compose pull
docker compose up -d
```
Or the old way (with the old docker-compose version):
```sh
git pull
docker-compose pull
docker-compose up -d
```

201
ai_platform/modules/web/deploy/metasearch/docker-compose.yaml Normal file
View file

@ -0,0 +1,201 @@
# Metasearch Module - Docker Compose Configuration
#
# Architecture:
# nginx (load balancer) -> 3x SearXNG instances + dedicated Redis each
# Round-robin distributes requests to avoid CAPTCHA/rate-limits
#
# Port Allocation (x55xx = Backend Services / Search):
# 55100 - nginx load balancer (entry point)
#
# Naming Convention: didiAI-{module}-{service}
#
# Network:
# Uses deploy_default network (shared with other modules)
networks:
didi-network:
external: true # single shared network for all DIDI + AI platform stacks
services:
# ==========================================================================
# Nginx Load Balancer (entry point)
# ==========================================================================
searxng-lb:
container_name: didiAI-web-searxng
image: nginx:1.27-alpine
ports:
- "${SEARXNG_PORT:-55100}:8080"
networks:
- didi-network
volumes:
- ./nginx-lb.conf:/etc/nginx/nginx.conf:ro
depends_on:
searxng-1:
condition: service_healthy
searxng-2:
condition: service_healthy
searxng-3:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/healthz"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
restart: unless-stopped
# ==========================================================================
# SearXNG Instance 1
# ==========================================================================
searxng-1:
container_name: didiAI-web-searxng-1
image: docker.io/searxng/searxng:latest
networks:
- didi-network
volumes:
- ./searxng-1:/etc/searxng:rw
- searxng-data-1:/var/cache/searxng:rw
environment:
- SEARXNG_BASE_URL=http://localhost:55100
depends_on:
redis-1:
condition: service_healthy
tor:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
# ==========================================================================
# SearXNG Instance 2
# ==========================================================================
searxng-2:
container_name: didiAI-web-searxng-2
image: docker.io/searxng/searxng:latest
networks:
- didi-network
volumes:
- ./searxng-2:/etc/searxng:rw
- searxng-data-2:/var/cache/searxng:rw
environment:
- SEARXNG_BASE_URL=http://localhost:55100
depends_on:
redis-2:
condition: service_healthy
tor:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
# ==========================================================================
# SearXNG Instance 3
# ==========================================================================
searxng-3:
container_name: didiAI-web-searxng-3
image: docker.io/searxng/searxng:latest
networks:
- didi-network
volumes:
- ./searxng-3:/etc/searxng:rw
- searxng-data-3:/var/cache/searxng:rw
environment:
- SEARXNG_BASE_URL=http://localhost:55100
depends_on:
redis-3:
condition: service_healthy
tor:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
# ==========================================================================
# Tor SOCKS5 proxy (shared by all SearXNG instances)
# ==========================================================================
tor:
container_name: didiAI-web-tor
image: dperson/torproxy:latest
networks:
- didi-network
environment:
- TOR_MaxCircuitDirtiness=300
- TOR_NewCircuitPeriod=60
healthcheck:
test: ["CMD", "curl", "-sf", "--socks5-hostname", "127.0.0.1:9050", "https://check.torproject.org/api/ip"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
restart: unless-stopped
# ==========================================================================
# Redis instances (one per SearXNG for isolated cache/state)
# ==========================================================================
redis-1:
container_name: didiAI-web-searxng-redis-1
image: docker.io/valkey/valkey:8-alpine
command: valkey-server --save 30 1 --loglevel warning
networks:
- didi-network
volumes:
- searxng-redis-data-1:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
restart: unless-stopped
redis-2:
container_name: didiAI-web-searxng-redis-2
image: docker.io/valkey/valkey:8-alpine
command: valkey-server --save 30 1 --loglevel warning
networks:
- didi-network
volumes:
- searxng-redis-data-2:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
restart: unless-stopped
redis-3:
container_name: didiAI-web-searxng-redis-3
image: docker.io/valkey/valkey:8-alpine
command: valkey-server --save 30 1 --loglevel warning
networks:
- didi-network
volumes:
- searxng-redis-data-3:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
restart: unless-stopped
volumes:
searxng-data-1:
searxng-data-2:
searxng-data-3:
searxng-redis-data-1:
searxng-redis-data-2:
searxng-redis-data-3:

43
ai_platform/modules/web/deploy/metasearch/load-tests/baseline_test.js Normal file
View file

@ -0,0 +1,43 @@
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';
// Custom metrics
const errorRate = new Rate('errors');
const searchDuration = new Trend('search_duration');
// Simple baseline test - 1 user, 2 minutes
export const options = {
vus: 1, // 1 virtual user
duration: '2m', // 2 minutes
thresholds: {
http_req_duration: ['p(95)<3000'], // 95% under 3s
errors: ['rate<0.05'], // Error rate < 5%
},
};
const queries = [
'python tutorial',
'machine learning',
'linux commands',
'docker compose',
];
export default function () {
const query = queries[Math.floor(Math.random() * queries.length)];
const url = `http://localhost:8080/search?q=${encodeURIComponent(query)}&format=html`;
const res = http.get(url);
searchDuration.add(res.timings.duration);
errorRate.add(res.status !== 200);
check(res, {
'status is 200': (r) => r.status === 200,
'response has results': (r) => r.body.includes('result'),
'response time < 3s': (r) => r.timings.duration < 3000,
});
sleep(2); // 2 seconds between requests
}

144
ai_platform/modules/web/deploy/metasearch/load-tests/factcheck_test.js Normal file
View file

@ -0,0 +1,144 @@
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';
// Custom metrics
const errorRate = new Rate('errors');
const searchDuration = new Trend('search_duration');
// Test configuration - 50 users, similar to previous test
export const options = {
stages: [
{ duration: '2m', target: 10 }, // Ramp up to 10 users
{ duration: '5m', target: 10 }, // Stay at 10 users
{ duration: '2m', target: 50 }, // Ramp up to 50 users
{ duration: '5m', target: 50 }, // Stay at 50 users
{ duration: '2m', target: 0 }, // Ramp down
],
thresholds: {
http_req_duration: ['p(95)<10000'], // 95% under 10s (longer for complex queries)
errors: ['rate<0.1'],
},
};
// =============================================================================
// REAL FACT-CHECKING CLAIMS - All TRUE and VERIFIABLE
// These simulate what users would search on a disinformation verification platform
// =============================================================================
const factCheckClaims = [
// --- US POLITICS ---
"Joe Biden signed Infrastructure Investment and Jobs Act 2021",
"Donald Trump impeached twice by House of Representatives",
"Barack Obama born in Hawaii August 4 1961",
"January 6 2021 Capitol building attack Washington DC",
"Supreme Court overturned Roe v Wade June 2022",
"Biden student loan forgiveness plan blocked by Supreme Court",
"Trump classified documents Mar-a-Lago FBI search 2022",
"Ketanji Brown Jackson first Black woman Supreme Court",
// --- EUROPEAN POLITICS ---
"Russia invaded Ukraine February 24 2022",
"UK Brexit officially completed January 31 2020",
"Emmanuel Macron reelected French president 2022",
"Olaf Scholz became German Chancellor December 2021",
"Boris Johnson resigned as UK Prime Minister July 2022",
"Sweden Finland applied NATO membership 2022",
"Italy Giorgia Meloni first female prime minister 2022",
"European Union sanctions Russia Ukraine invasion",
// --- COVID-19 FACTS ---
"COVID-19 pandemic declared by WHO March 11 2020",
"Pfizer BioNTech vaccine FDA emergency authorization December 2020",
"COVID-19 originated Wuhan China December 2019",
"WHO declared COVID-19 emergency over May 2023",
"mRNA vaccines developed Moderna Pfizer coronavirus",
"COVID-19 global deaths exceeded 6 million 2022",
// --- CLIMATE & SCIENCE ---
"Paris Agreement climate change signed 2015",
"IPCC report 1.5 degrees celsius warming target",
"NASA confirms 2023 hottest year on record",
"Arctic sea ice minimum record low 2012",
"UN climate summit COP28 Dubai 2023",
"Greenhouse gas emissions carbon dioxide methane",
// --- ELECTIONS & DEMOCRACY ---
"2020 US presidential election Biden 306 electoral votes",
"Brazil Lula da Silva elected president October 2022",
"India Narendra Modi BJP won 2024 general election",
"Mexico Claudia Sheinbaum elected first female president 2024",
"Argentina Javier Milei elected president November 2023",
"Poland Donald Tusk returned as prime minister 2023",
// --- INTERNATIONAL EVENTS ---
"Hamas attack Israel October 7 2023",
"Nord Stream pipeline explosions September 2022",
"Queen Elizabeth II died September 8 2022",
"Evan Gershkovich Wall Street Journal detained Russia",
"Wagner Group rebellion Prigozhin June 2023",
"China spy balloon shot down US February 2023",
// --- ECONOMIC FACTS ---
"US Federal Reserve raised interest rates 2022 2023",
"Silicon Valley Bank collapsed March 2023",
"US debt ceiling crisis June 2023",
"Inflation rate US peaked 9.1 percent June 2022",
"Twitter acquired by Elon Musk October 2022 renamed X",
"OpenAI released ChatGPT November 2022",
// --- MISINFORMATION TOPICS (TRUE FACTS TO COUNTER) ---
"vaccines do not cause autism scientific consensus",
"earth is approximately 4.5 billion years old",
"climate change caused by human activity scientific consensus",
"2020 US election no widespread voter fraud evidence",
"COVID-19 lab leak theory investigated WHO",
"5G technology does not spread coronavirus",
// --- POLITICAL QUOTES (VERIFIABLE) ---
"Putin called Ukraine invasion special military operation",
"Trump said inject disinfectant COVID briefing April 2020",
"Biden said democracy is on the ballot 2024",
"Zelensky addressed US Congress December 2022",
"Xi Jinping third term Chinese president 2023",
// --- LEGAL & INVESTIGATIONS ---
"Trump indicted Manhattan DA March 2023",
"Trump indicted classified documents June 2023",
"Trump indicted January 6 investigation August 2023",
"Hunter Biden convicted gun charges June 2024",
"Alex Jones ordered pay Sandy Hook families damages",
"Sam Bankman-Fried FTX fraud conviction 2023",
// --- TECHNOLOGY & AI ---
"artificial intelligence ChatGPT GPT-4 release 2023",
"European Union AI Act regulation passed 2024",
"Google Bard AI chatbot released 2023",
"Elon Musk founded xAI company 2023",
"deepfake technology election misinformation concerns",
"social media algorithms spread misinformation studies",
];
export default function () {
// Random claim from list
const claim = factCheckClaims[Math.floor(Math.random() * factCheckClaims.length)];
// Make search request
const url = `http://localhost:8080/search?q=${encodeURIComponent(claim)}&format=html`;
const res = http.get(url);
// Record metrics
searchDuration.add(res.timings.duration);
errorRate.add(res.status !== 200);
// Validate response
check(res, {
'status is 200': (r) => r.status === 200,
'response has results': (r) => r.body.includes('result'),
'response time < 10s': (r) => r.timings.duration < 10000,
});
// Think time (user reads results, 2-5 seconds)
sleep(Math.random() * 3 + 2);
}

60
ai_platform/modules/web/deploy/metasearch/load-tests/searxng_test.js Normal file
View file

@ -0,0 +1,60 @@
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';
// Custom metrics
const errorRate = new Rate('errors');
const searchDuration = new Trend('search_duration');
// Test configuration
export const options = {
// Test stages (ramp up, steady, ramp down)
stages: [
{ duration: '2m', target: 10 }, // Ramp up to 10 users
{ duration: '5m', target: 10 }, // Stay at 10 users
{ duration: '2m', target: 50 }, // Ramp up to 50 users
{ duration: '5m', target: 50 }, // Stay at 50 users
{ duration: '2m', target: 0 }, // Ramp down
],
// Pass/fail thresholds
thresholds: {
http_req_duration: ['p(95)<5000'], // 95% under 5s
errors: ['rate<0.1'], // Error rate < 10%
},
};
// Sample search queries (vary these!)
const queries = [
'python tutorial',
'machine learning',
'linux commands',
'javascript async await',
'docker compose',
'rust programming',
'kubernetes deployment',
'postgresql optimization',
];
export default function () {
// Random query from list
const query = queries[Math.floor(Math.random() * queries.length)];
// Make search request
const url = `http://localhost:8080/search?q=${encodeURIComponent(query)}&format=html`;
const res = http.get(url);
// Record custom metrics
searchDuration.add(res.timings.duration);
errorRate.add(res.status !== 200);
// Validate response
check(res, {
'status is 200': (r) => r.status === 200,
'response has results': (r) => r.body.includes('result'),
'response time < 5s': (r) => r.timings.duration < 5000,
});
// Think time (simulates real user)
sleep(Math.random() * 3 + 1); // 1-4 seconds between requests
}

69
ai_platform/modules/web/deploy/metasearch/load-tests/stress_test.js Normal file
View file

@ -0,0 +1,69 @@
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend, Counter } from 'k6/metrics';
const errorRate = new Rate('errors');
const searchDuration = new Trend('search_duration');
const requestCount = new Counter('total_requests');
// =============================================================================
// STRESS TEST - Push to 200 users, find breaking point
// Duration: ~13 minutes
// =============================================================================
export const options = {
stages: [
{ duration: '1m', target: 50 }, // Warm up to 50 users
{ duration: '2m', target: 50 }, // Hold - baseline
{ duration: '1m', target: 100 }, // Double to 100 users
{ duration: '2m', target: 100 }, // Hold - watch for degradation
{ duration: '1m', target: 150 }, // Push to 150 users
{ duration: '2m', target: 150 }, // Hold - likely slowing down
{ duration: '1m', target: 200 }, // Push to 200 users
{ duration: '2m', target: 200 }, // Hold - stress zone
{ duration: '1m', target: 0 }, // Ramp down - watch recovery
],
thresholds: {
errors: ['rate<0.5'], // Allow up to 50% errors
http_req_duration: ['p(50)<30000'], // Median under 30s (very lenient)
},
};
// Fact-checking claims for realistic testing
const factCheckClaims = [
"Joe Biden signed Infrastructure Investment and Jobs Act 2021",
"Russia invaded Ukraine February 24 2022",
"COVID-19 pandemic declared by WHO March 11 2020",
"Supreme Court overturned Roe v Wade June 2022",
"Twitter acquired by Elon Musk October 2022",
"Hamas attack Israel October 7 2023",
"Trump indicted classified documents June 2023",
"ChatGPT released by OpenAI November 2022",
"Queen Elizabeth II died September 8 2022",
"2020 US election Biden 306 electoral votes",
"Brexit UK left European Union January 2020",
"Zelensky addressed US Congress December 2022",
"Silicon Valley Bank collapsed March 2023",
"Wagner Group rebellion Prigozhin June 2023",
"OpenAI Sam Altman fired rehired November 2023",
];
export default function () {
const claim = factCheckClaims[Math.floor(Math.random() * factCheckClaims.length)];
const url = `http://localhost:8080/search?q=${encodeURIComponent(claim)}&format=html`;
const res = http.get(url, { timeout: '60s' }); // Longer timeout for stress
searchDuration.add(res.timings.duration);
errorRate.add(res.status !== 200);
requestCount.add(1);
check(res, {
'status is 200': (r) => r.status === 200,
'response has results': (r) => r.body && r.body.includes('result'),
'no timeout': (r) => r.timings.duration < 60000,
});
// Minimal sleep = maximum pressure (0.5-1.5 seconds)
sleep(Math.random() + 0.5);
}

29
ai_platform/modules/web/deploy/metasearch/nginx-lb.conf Normal file
View file

@ -0,0 +1,29 @@
worker_processes auto;
events {
worker_connections 256;
}
http {
# Round-robin upstream across 3 SearXNG instances
upstream searxng {
server didiAI-web-searxng-1:8080;
server didiAI-web-searxng-2:8080;
server didiAI-web-searxng-3:8080;
}
server {
listen 8080;
location / {
proxy_pass http://searxng;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 10s;
proxy_read_timeout 30s;
proxy_next_upstream error timeout http_502 http_503;
proxy_next_upstream_tries 3;
}
}
}

7
ai_platform/modules/web/deploy/metasearch/searxng-1/limiter.toml Normal file
View file

@ -0,0 +1,7 @@
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate advanced bot protection
# enable this when running the instance for a public usage on the internet
link_token = false

140
ai_platform/modules/web/deploy/metasearch/searxng-1/settings.yml Normal file
View file

@ -0,0 +1,140 @@
# SearXNG Configuration for didiAI Web Module
# https://docs.searxng.org/admin/settings/settings.html
use_default_settings: true
general:
instance_name: "didiAI-searxng"
enable_metrics: false
debug: false
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable
secret_key: "instance1-2d596738d2092ebd"
bind_address: "0.0.0.0:8080"
limiter: false
image_proxy: true
search:
formats:
- html
- json
default_lang: "auto"
safe_search: 1
autocomplete: ""
suspended_times:
SearxEngineAccessDenied: 5
SearxEngineCaptcha: 10
SearxEngineTooManyRequests: 5
cf_SearxEngineAccessDenied: 5
cf_SearxEngineCaptcha: 10
cf_SearxEngineTooManyRequests: 5
outgoing:
request_timeout: 10.0
max_request_timeout: 15.0
useragent_suffix: ""
pool_connections: 50
pool_maxsize: 8
proxies:
all://:
- http://smart-ouowetdhvcnw:ctAFve4tpQkTyLtB@proxy.smartproxy.net:3120
redis:
url: redis://didiAI-web-searxng-redis-1:6379/0
# =============================================================================
# Engine Configuration
# =============================================================================
# Primary engines (general web) + specialized engines for comprehensive coverage
#
# Weight affects result ranking (higher = more prominent)
# Timeout is per-engine maximum wait time
engines:
# ===========================================================================
# Primary - General Web Search
# ===========================================================================
- name: google
disabled: false
timeout: 6.0
weight: 1.2
- name: bing
disabled: false
timeout: 4.0
weight: 1.0
- name: duckduckgo
disabled: false
timeout: 4.0
weight: 1.0
- name: brave
disabled: false
timeout: 4.0
weight: 1.2
api_key: BSAuTXcB4TopHanUatjFxNuOixFYV3j
- name: qwant
disabled: false
timeout: 4.0
weight: 0.8
# ===========================================================================
# Secondary - Specialized Sources
# ===========================================================================
- name: wikipedia
disabled: false
timeout: 3.0
weight: 0.9
- name: google news
disabled: false
timeout: 4.0
weight: 0.8
- name: google scholar
disabled: false
timeout: 5.0
weight: 0.8
- name: reddit
disabled: false
timeout: 4.0
weight: 0.6
- name: archive.org
disabled: false
timeout: 5.0
weight: 0.5
# ===========================================================================
# Disabled - Not needed for fact-checking
# ===========================================================================
- name: tineye
disabled: true
- name: youtube
disabled: true
- name: dailymotion
disabled: true
- name: vimeo
disabled: true
- name: soundcloud
disabled: true
- name: bandcamp
disabled: true
- name: piratebay
disabled: true
- name: 1337x
disabled: true
- name: nyaa
disabled: true

2771
ai_platform/modules/web/deploy/metasearch/searxng-1/settings.yml.new Normal file
View file

File diff suppressed because it is too large Load diff

7
ai_platform/modules/web/deploy/metasearch/searxng-2/limiter.toml Normal file
View file

@ -0,0 +1,7 @@
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate advanced bot protection
# enable this when running the instance for a public usage on the internet
link_token = false

140
ai_platform/modules/web/deploy/metasearch/searxng-2/settings.yml Normal file
View file

@ -0,0 +1,140 @@
# SearXNG Configuration for didiAI Web Module
# https://docs.searxng.org/admin/settings/settings.html
use_default_settings: true
general:
instance_name: "didiAI-searxng"
enable_metrics: false
debug: false
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable
secret_key: "instance2-2d596738d2092ebd"
bind_address: "0.0.0.0:8080"
limiter: false
image_proxy: true
search:
formats:
- html
- json
default_lang: "auto"
safe_search: 1
autocomplete: ""
suspended_times:
SearxEngineAccessDenied: 5
SearxEngineCaptcha: 10
SearxEngineTooManyRequests: 5
cf_SearxEngineAccessDenied: 5
cf_SearxEngineCaptcha: 10
cf_SearxEngineTooManyRequests: 5
outgoing:
request_timeout: 10.0
max_request_timeout: 15.0
useragent_suffix: ""
pool_connections: 50
pool_maxsize: 8
proxies:
all://:
- http://smart-ouowetdhvcnw:ctAFve4tpQkTyLtB@proxy.smartproxy.net:3120
redis:
url: redis://didiAI-web-searxng-redis-2:6379/0
# =============================================================================
# Engine Configuration
# =============================================================================
# Primary engines (general web) + specialized engines for comprehensive coverage
#
# Weight affects result ranking (higher = more prominent)
# Timeout is per-engine maximum wait time
engines:
# ===========================================================================
# Primary - General Web Search
# ===========================================================================
- name: google
disabled: false
timeout: 6.0
weight: 1.2
- name: bing
disabled: false
timeout: 4.0
weight: 1.0
- name: duckduckgo
disabled: false
timeout: 4.0
weight: 1.0
- name: brave
disabled: false
timeout: 4.0
weight: 1.2
api_key: BSAuTXcB4TopHanUatjFxNuOixFYV3j
- name: qwant
disabled: false
timeout: 4.0
weight: 0.8
# ===========================================================================
# Secondary - Specialized Sources
# ===========================================================================
- name: wikipedia
disabled: false
timeout: 3.0
weight: 0.9
- name: google news
disabled: false
timeout: 4.0
weight: 0.8
- name: google scholar
disabled: false
timeout: 5.0
weight: 0.8
- name: reddit
disabled: false
timeout: 4.0
weight: 0.6
- name: archive.org
disabled: false
timeout: 5.0
weight: 0.5
# ===========================================================================
# Disabled - Not needed for fact-checking
# ===========================================================================
- name: tineye
disabled: true
- name: youtube
disabled: true
- name: dailymotion
disabled: true
- name: vimeo
disabled: true
- name: soundcloud
disabled: true
- name: bandcamp
disabled: true
- name: piratebay
disabled: true
- name: 1337x
disabled: true
- name: nyaa
disabled: true

2771
ai_platform/modules/web/deploy/metasearch/searxng-2/settings.yml.new Normal file
View file

File diff suppressed because it is too large Load diff

7
ai_platform/modules/web/deploy/metasearch/searxng-3/limiter.toml Normal file
View file

@ -0,0 +1,7 @@
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate advanced bot protection
# enable this when running the instance for a public usage on the internet
link_token = false

140
ai_platform/modules/web/deploy/metasearch/searxng-3/settings.yml Normal file
View file

@ -0,0 +1,140 @@
# SearXNG Configuration for didiAI Web Module
# https://docs.searxng.org/admin/settings/settings.html
use_default_settings: true
general:
instance_name: "didiAI-searxng"
enable_metrics: false
debug: false
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable
secret_key: "instance3-2d596738d2092ebd"
bind_address: "0.0.0.0:8080"
limiter: false
image_proxy: true
search:
formats:
- html
- json
default_lang: "auto"
safe_search: 1
autocomplete: ""
suspended_times:
SearxEngineAccessDenied: 5
SearxEngineCaptcha: 10
SearxEngineTooManyRequests: 5
cf_SearxEngineAccessDenied: 5
cf_SearxEngineCaptcha: 10
cf_SearxEngineTooManyRequests: 5
outgoing:
request_timeout: 10.0
max_request_timeout: 15.0
useragent_suffix: ""
pool_connections: 50
pool_maxsize: 8
proxies:
all://:
- http://smart-ouowetdhvcnw:ctAFve4tpQkTyLtB@proxy.smartproxy.net:3120
redis:
url: redis://didiAI-web-searxng-redis-3:6379/0
# =============================================================================
# Engine Configuration
# =============================================================================
# Primary engines (general web) + specialized engines for comprehensive coverage
#
# Weight affects result ranking (higher = more prominent)
# Timeout is per-engine maximum wait time
engines:
# ===========================================================================
# Primary - General Web Search
# ===========================================================================
- name: google
disabled: false
timeout: 6.0
weight: 1.2
- name: bing
disabled: false
timeout: 4.0
weight: 1.0
- name: duckduckgo
disabled: false
timeout: 4.0
weight: 1.0
- name: brave
disabled: false
timeout: 4.0
weight: 1.2
api_key: BSAuTXcB4TopHanUatjFxNuOixFYV3j
- name: qwant
disabled: false
timeout: 4.0
weight: 0.8
# ===========================================================================
# Secondary - Specialized Sources
# ===========================================================================
- name: wikipedia
disabled: false
timeout: 3.0
weight: 0.9
- name: google news
disabled: false
timeout: 4.0
weight: 0.8
- name: google scholar
disabled: false
timeout: 5.0
weight: 0.8
- name: reddit
disabled: false
timeout: 4.0
weight: 0.6
- name: archive.org
disabled: false
timeout: 5.0
weight: 0.5
# ===========================================================================
# Disabled - Not needed for fact-checking
# ===========================================================================
- name: tineye
disabled: true
- name: youtube
disabled: true
- name: dailymotion
disabled: true
- name: vimeo
disabled: true
- name: soundcloud
disabled: true
- name: bandcamp
disabled: true
- name: piratebay
disabled: true
- name: 1337x
disabled: true
- name: nyaa
disabled: true

2771
ai_platform/modules/web/deploy/metasearch/searxng-3/settings.yml.new Normal file
View file

File diff suppressed because it is too large Load diff

162
ai_platform/modules/web/deploy/metasearch/searxng-reset.sh Normal file
View file

@ -0,0 +1,162 @@
#!/usr/bin/env bash
#
# SearXNG Reset Script (3-instance cluster)
# ==========================================
# Clears CAPTCHA/rate-limit state and restarts SearXNG instances.
#
# Usage:
# ./searxng-reset.sh # Full reset (flush all + restart all)
# ./searxng-reset.sh --status # Show engine error counts
# ./searxng-reset.sh --check # Quick health check
# ./searxng-reset.sh --one 2 # Reset only instance 2
#
set -euo pipefail
INSTANCES=(1 2 3)
LB_CONTAINER="didiAI-web-searxng"
SEARXNG_URL="http://localhost:55100"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
check_health() {
echo "=== Health Check ==="
echo ""
# LB
local lb_status
lb_status=$(curl -s --max-time 5 "${SEARXNG_URL}/healthz" 2>/dev/null) || true
if [[ "$lb_status" == "OK" ]]; then
echo -e "Load balancer (55100): ${GREEN}UP${NC}"
else
echo -e "Load balancer (55100): ${RED}DOWN${NC}"
fi
# Individual instances
for i in "${INSTANCES[@]}"; do
local container="didiAI-web-searxng-${i}"
local status
status=$(docker inspect --format '{{.State.Health.Status}}' "$container" 2>/dev/null) || status="not found"
if [[ "$status" == "healthy" ]]; then
echo -e " Instance ${i}: ${GREEN}${status}${NC}"
else
echo -e " Instance ${i}: ${RED}${status}${NC}"
fi
done
# Test search
echo ""
local results
results=$(curl -s --max-time 10 "${SEARXNG_URL}/search?q=test&format=json" 2>/dev/null \
| python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('results',[])))" 2>/dev/null) || results=0
if [[ "$results" -gt 0 ]]; then
echo -e "Search test: ${GREEN}OK ($results results)${NC}"
else
echo -e "Search test: ${RED}FAILED (0 results)${NC}"
fi
}
show_status() {
echo "=== Engine Errors (all instances) ==="
echo ""
for i in "${INSTANCES[@]}"; do
local container="didiAI-web-searxng-${i}"
echo -e "${YELLOW}Instance ${i} ($container):${NC}"
local captcha_count
captcha_count=$(docker logs "$container" 2>&1 | grep -ci 'CAPTCHA' 2>/dev/null) || captcha_count=0
local toomany_count
toomany_count=$(docker logs "$container" 2>&1 | grep -ci 'TooManyRequests' 2>/dev/null) || toomany_count=0
if [[ $captcha_count -gt 0 || $toomany_count -gt 0 ]]; then
echo " CAPTCHA: $captcha_count | TooManyRequests: $toomany_count"
docker logs "$container" 2>&1 \
| grep -iE 'CAPTCHA|TooMany' \
| grep -oP 'engines\.\K[^:]+' \
| sort | uniq -c | sort -rn \
| sed 's/^/ /' || true
else
echo -e " ${GREEN}Clean (no errors)${NC}"
fi
echo ""
done
}
reset_instance() {
local i=$1
local container="didiAI-web-searxng-${i}"
local redis="didiAI-web-searxng-redis-${i}"
echo -n " [${i}] Flush Redis... "
if docker exec "$redis" valkey-cli FLUSHALL > /dev/null 2>&1; then
echo -e "${GREEN}OK${NC}"
else
echo -e "${YELLOW}WARN${NC}"
fi
echo -n " [${i}] Restart SearXNG... "
docker restart "$container" > /dev/null 2>&1
echo -e "${GREEN}OK${NC}"
}
do_reset() {
local targets=("${@}")
if [[ ${#targets[@]} -eq 0 ]]; then
targets=("${INSTANCES[@]}")
fi
echo "=== SearXNG Reset ==="
echo "Resetting instances: ${targets[*]}"
echo ""
for i in "${targets[@]}"; do
reset_instance "$i"
done
# Wait for health
echo ""
echo -n "Waiting for startup... "
local attempts=0
while [[ $attempts -lt 15 ]]; do
sleep 2
local healthy=0
for i in "${targets[@]}"; do
local status
status=$(docker inspect --format '{{.State.Health.Status}}' "didiAI-web-searxng-${i}" 2>/dev/null) || status="none"
[[ "$status" == "healthy" ]] && healthy=$((healthy + 1))
done
if [[ $healthy -eq ${#targets[@]} ]]; then
echo -e "${GREEN}All ${healthy} instances UP${NC}"
echo ""
check_health
return 0
fi
attempts=$((attempts + 1))
done
echo -e "${YELLOW}Some instances still starting (check with --check)${NC}"
}
# Parse arguments
case "${1:-}" in
--status|-s)
show_status
;;
--check|-c)
check_health
;;
--one|-o)
[[ -z "${2:-}" ]] && echo "Usage: $0 --one <1|2|3>" && exit 1
do_reset "$2"
;;
--help|-h)
sed -n '2,12p' "$0" | sed 's/^# //' | sed 's/^#//'
;;
*)
do_reset
;;
esac

7
ai_platform/modules/web/deploy/metasearch/searxng/limiter.toml Normal file
View file

@ -0,0 +1,7 @@
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate advanced bot protection
# enable this when running the instance for a public usage on the internet
link_token = false

139
ai_platform/modules/web/deploy/metasearch/searxng/settings.yml Normal file
View file

@ -0,0 +1,139 @@
# SearXNG Configuration for didiAI Web Module
# https://docs.searxng.org/admin/settings/settings.html
use_default_settings: true
general:
instance_name: "didiAI-searxng"
enable_metrics: false
debug: false
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable
secret_key: "2d596738d2092ebd"
bind_address: "0.0.0.0:8080"
limiter: false
image_proxy: true
search:
formats:
- html
- json
default_lang: "auto"
safe_search: 1
autocomplete: ""
suspended_times:
SearxEngineAccessDenied: 5
SearxEngineCaptcha: 10
SearxEngineTooManyRequests: 5
cf_SearxEngineAccessDenied: 5
cf_SearxEngineCaptcha: 10
cf_SearxEngineTooManyRequests: 5
outgoing:
request_timeout: 10.0
max_request_timeout: 15.0
useragent_suffix: "didiAI-metasearch"
pool_connections: 100
pool_maxsize: 20
proxies:
all://:
- http://smart-ouowetdhvcnw:ctAFve4tpQkTyLtB@proxy.smartproxy.net:3120
redis:
url: redis://didiAI-web-searxng-redis:6379/0
# =============================================================================
# Engine Configuration
# =============================================================================
# Primary engines (general web) + specialized engines for comprehensive coverage
#
# Weight affects result ranking (higher = more prominent)
# Timeout is per-engine maximum wait time
engines:
# ===========================================================================
# Primary - General Web Search
# ===========================================================================
- name: google
disabled: false
timeout: 6.0
weight: 1.2
- name: bing
disabled: true
timeout: 4.0
weight: 1.0
- name: duckduckgo
disabled: false
timeout: 4.0
weight: 1.0
- name: brave
disabled: false
timeout: 4.0
weight: 1.0
- name: qwant
disabled: false
timeout: 4.0
weight: 0.8
# ===========================================================================
# Secondary - Specialized Sources
# ===========================================================================
- name: wikipedia
disabled: false
timeout: 3.0
weight: 0.9
- name: google news
disabled: false
timeout: 4.0
weight: 0.8
- name: google scholar
disabled: false
timeout: 5.0
weight: 0.8
- name: reddit
disabled: false
timeout: 4.0
weight: 0.6
- name: archive.org
disabled: false
timeout: 5.0
weight: 0.5
# ===========================================================================
# Disabled - Not needed for fact-checking
# ===========================================================================
- name: tineye
disabled: true
- name: youtube
disabled: true
- name: dailymotion
disabled: true
- name: vimeo
disabled: true
- name: soundcloud
disabled: true
- name: bandcamp
disabled: true
- name: piratebay
disabled: true
- name: 1337x
disabled: true
- name: nyaa
disabled: true

51
ai_platform/modules/web/deploy/nginx.conf Normal file
View file

@ -0,0 +1,51 @@
upstream webapi {
server didiAI-web-api:51100;
}
server {
listen 80;
server_name _;
# Request body limit
client_max_body_size 1m;
# Timeouts
proxy_connect_timeout 60s;
proxy_send_timeout 120s;
proxy_read_timeout 120s;
# Gzip compression
gzip on;
gzip_types application/json text/plain;
gzip_min_length 256;
gzip_vary on;
# Proxy buffering
proxy_buffering on;
proxy_buffer_size 4k;
proxy_buffers 8 8k;
# Proxy settings
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Request-ID $request_id;
# Health check endpoint
location /health {
proxy_pass http://webapi/health;
add_header Cache-Control "no-cache";
}
location /ready {
proxy_pass http://webapi/ready;
add_header Cache-Control "no-cache";
}
# API endpoints
location / {
proxy_pass http://webapi;
}
}

63
ai_platform/modules/web/gather_results.md Normal file
View file

@ -0,0 +1,63 @@
# Web API Gather Results - NER & Fact-Checking Research
Generated: 2026-01-30
---
## Query 1: MultiNERD Entity Types
**Claim:** `MultiNERD entity types list NER categories person organization location date time money percentage fact-checking`
**Execution Time:** 5203ms
### URLs Accessed
- [Named-entity recognition - Wikipedia](https://en.wikipedia.org/wiki/Named-entity_recognition)
- [Gold standard, multi-genre dataset for named entity recognition and linking | Scientific Data](https://www.nature.com/articles/s41597-025-05274-4)
- [Named Entity Recognition (NER): Ultimate Guide | Encord](https://encord.com/blog/named-entity-recognition/)
- [A Comprehensive Guide to Named Entity Recognition](https://www.turing.com/kb/a-comprehensive-guide-to-named-entity-recognition)
- [Entity categories recognized by Named Entity Recognition in Azure](https://learn.microsoft.com/en-us/azure/ai-services/language-service/named-entity-recognition/concepts/named-entity-categories)
---
## Query 2: Fact-Checking Methodology
**Claim:** `fact-checking entity extraction named entity recognition claim verification ClaimBuster Full Fact methodology 2024`
**Execution Time:** 9698ms
### URLs Accessed
- [arxiv.org PDF 1809.08193](https://arxiv.org/pdf/1809.08193)
- [Claim Extraction for Fact-Checking: Data, Models, and Automated Metrics](https://arxiv.org/html/2502.04955v1)
- [Facilitating automated fact-checking: a machine learning based weighted ensemble technique for claim detection](https://link.springer.com/article/10.1007/s42452-024-06444-6)
- [Document-level Claim Extraction and Decontextualisation for Fact-Checking](https://arxiv.org/html/2406.03239v2)
- [An Entity-based Claim Extraction Pipeline for Real-world Biomedical Fact-checking](https://arxiv.org/abs/2304.05268)
---
## Query 3: NER Query Generation for Fact-Checking
**Claim:** `named entity recognition query generation fact-checking entity types person organization location date event claim verification search query templates`
**Execution Time:** 3485ms
### URLs Accessed
- [Tools for Named Entity Recognition | CLARIN ERIC](https://www.clarin.eu/resource-families/tools-named-entity-recognition)
- [Named-entity recognition - Wikipedia](https://en.wikipedia.org/wiki/Named-entity_recognition)
- [Named Entity Recognition - GeeksforGeeks](https://www.geeksforgeeks.org/nlp/named-entity-recognition/)
- [Named Entity Recognition | Yext](https://www.yext.com/platform/features/named-entity-recognition)
- [Named Entity Recognition on Search Engine Queries with Python - Stack Overflow](https://stackoverflow.com/questions/78151241/named-entity-recognition-on-search-engine-queries-with-python)
---
## Summary
| Query | Evidence URLs | Execution Time |
|-------|---------------|----------------|
| 1 - MultiNERD Entity Types | 5 | 5.2s |
| 2 - Fact-Checking Methodology | 5 | 9.7s |
| 3 - NER Query Generation | 5 | 3.5s |
**Total URLs:** 15

67
ai_platform/modules/web/pyproject.toml Normal file
View file

@ -0,0 +1,67 @@
[project]
name = "web"
version = "0.1.0"
description = "Web module - search, fetch, browse, vision, evidence for ml-projects"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"fastapi>=0.115.0,<1.0",
"uvicorn[standard]>=0.32.0",
"pydantic>=2.0,<3.0",
"pydantic-settings>=2.0,<3.0",
"httpx[http2]>=0.27.0,<1.0",
"prometheus-fastapi-instrumentator>=7.0.0",
"opentelemetry-instrumentation-fastapi>=0.50b0",
"opentelemetry-instrumentation-httpx>=0.50b0",
"opentelemetry-exporter-otlp-proto-grpc>=1.30.0",
]
[project.optional-dependencies]
dev = [
"pytest>=8.0",
"pytest-cov>=4.0",
"pytest-asyncio>=0.24.0",
"ruff>=0.8",
"respx>=0.21.0",
]
browse = [
"playwright>=1.40.0",
"readability-lxml>=0.8.0",
"lxml>=5.0.0",
]
vision = [
"openai>=1.0.0",
"anthropic>=0.25.0",
]
fetch = [
"readability-lxml>=0.8.0",
"beautifulsoup4>=4.12.0",
"lxml>=5.0.0",
]
all = [
"playwright>=1.40.0",
"openai>=1.0.0",
"anthropic>=0.25.0",
"readability-lxml>=0.8.0",
"beautifulsoup4>=4.12.0",
"lxml>=5.0.0",
]
[project.scripts]
web = "web.cli:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/web"]
[tool.ruff]
extend = "../../ruff.toml"
[tool.pytest.ini_options]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
testpaths = ["tests"]
addopts = "-v --tb=short"

32
ai_platform/modules/web/src/web/__init__.py Normal file
View file

@ -0,0 +1,32 @@
"""Web module - search, fetch, browse, vision, evidence."""
__version__ = "0.1.0"
from web.browse.client import BrowseClient
from web.evidence.packer import EvidencePacker
from web.fetch.client import FetchClient
from web.orchestrator import Orchestrator
from web.schemas.image_search import (
ImageSearchRequest,
ImageSearchResponse,
ImageSearchResult,
)
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
from web.search.protocol import SearchProvider
from web.vision.client import VisionClient
__all__ = [
"BrowseClient",
"EvidencePacker",
"FetchClient",
"ImageSearchRequest",
"ImageSearchResponse",
"ImageSearchResult",
"Orchestrator",
"SearchProvider",
"SearchRequest",
"SearchResponse",
"SearchResult",
"VisionClient",
"__version__",
]

1
ai_platform/modules/web/src/web/api/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Web Search API package."""

172
ai_platform/modules/web/src/web/api/app.py Normal file
View file

@ -0,0 +1,172 @@
"""FastAPI application factory."""
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from fastapi import FastAPI
from web.api.dependencies import init_concurrency_limiter
from web.api.middleware import CombinedMiddleware
from web.api.routes import fetch, gather, health, image_search, info, search
from web.brain.client import BrainClient
from web.brain.sink import BrainIngestSink
from web.config import SettingsCache
from web.events.sink import DashboardEventSink
from web.fetch.client import FetchClient
from web.logging import configure_logging, get_logger
from web.metasearch.client import SearXNGClient
from web.orchestrator import Orchestrator
from web.runtime_config import RuntimeConfigClient
from web.search.multi import MultiSearchClient
from web.search.paid import PaidSearchClient
logger = get_logger("app")
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Application lifespan manager.
Initializes clients, settings, logging, and concurrency limiter
on startup, and performs cleanup on shutdown.
"""
settings = SettingsCache.get()
configure_logging(settings.log_level, settings.log_json)
logger.info("Starting Web API")
logger.info("Port: %d", settings.port)
init_concurrency_limiter(settings.max_concurrent_requests)
app.state.settings = settings
# Runtime config client — fetched from dashboard
app.state.runtime_config = RuntimeConfigClient(
dashboard_url=settings.dashboard_url,
)
await app.state.runtime_config.start()
# Shared fetch client (used by both tiers)
app.state.fetch_client = FetchClient(settings)
# FREE tier: SearXNG only + local Qwen LLM
app.state.search_client_free = SearXNGClient(settings)
app.state.orchestrator_free = Orchestrator(
settings,
search_client=app.state.search_client_free,
fetch_client=app.state.fetch_client,
llm_provider="local",
runtime_config=app.state.runtime_config,
)
# PREMIUM tier: paid engines only + OpenRouter LLM
app.state.search_client_premium = PaidSearchClient(
settings, runtime_config=app.state.runtime_config
)
app.state.orchestrator_premium = Orchestrator(
settings,
search_client=app.state.search_client_premium,
fetch_client=app.state.fetch_client,
llm_provider="openrouter",
runtime_config=app.state.runtime_config,
)
# Default search client (backwards compat — used by /v1/search, /v1/image-search)
app.state.search_client = MultiSearchClient(settings)
# Dashboard event sink (fire-and-forget, optional)
app.state.event_sink = DashboardEventSink(
dashboard_url=settings.dashboard_url,
token=settings.dashboard_token,
)
await app.state.event_sink.start()
# Brain cache client + ingest sink (premium tier only; fire-and-forget)
app.state.brain_client = BrainClient(
base_url=settings.brain_url,
gather_timeout=settings.brain_gather_timeout,
ingest_timeout=settings.brain_ingest_timeout,
)
await app.state.brain_client.start()
app.state.brain_sink = BrainIngestSink(app.state.brain_client)
await app.state.brain_sink.start()
logger.info(
"Web API started successfully (free + premium tiers, brain=%s)",
"on" if app.state.brain_client.enabled else "off",
)
yield
logger.info("Shutting down Web API")
await app.state.brain_sink.stop()
await app.state.brain_client.close()
await app.state.event_sink.stop()
await app.state.runtime_config.stop()
await app.state.search_client.close()
await app.state.search_client_free.close()
await app.state.search_client_premium.close()
await app.state.fetch_client.close()
await app.state.orchestrator_free.close()
await app.state.orchestrator_premium.close()
def create_app() -> FastAPI:
"""Create and configure the FastAPI application.
Returns:
FastAPI: Configured FastAPI application.
"""
settings = SettingsCache.get()
app = FastAPI(
title="Web API",
description="Web module - search, fetch, browse, vision, evidence",
version="0.1.0",
lifespan=lifespan,
servers=[{"url": settings.external_url, "description": "Web API"}],
)
app.add_middleware(
CombinedMiddleware,
rate=settings.rate_limit_rps,
burst=settings.rate_limit_burst,
exclude_paths=["/health", "/ready"],
)
# Prometheus /metrics + OTel tracing (no-op if deps missing or OTEL endpoint unset)
try:
from prometheus_fastapi_instrumentator import Instrumentator # type: ignore
Instrumentator(should_group_status_codes=True).instrument(app).expose(
app, endpoint="/metrics", include_in_schema=False
)
except ImportError:
pass
import os as _os
_otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
if _otel_ep:
try:
from opentelemetry import trace as _trace # type: ignore
from opentelemetry.sdk.resources import Resource as _R # type: ignore
from opentelemetry.sdk.trace import TracerProvider as _TP # type: ignore
from opentelemetry.sdk.trace.export import BatchSpanProcessor as _BSP # type: ignore
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as _Exp # type: ignore
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor as _FInst # type: ignore
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor as _HXInst # type: ignore
_provider = _TP(resource=_R.create({"service.name": _os.environ.get("OTEL_SERVICE_NAME", "didiAI-web-api")}))
_provider.add_span_processor(_BSP(_Exp(endpoint=_otel_ep, insecure=True)))
_trace.set_tracer_provider(_provider)
_FInst.instrument_app(app)
_HXInst().instrument()
print(f"[otel] didiAI-web-api instrumented → {_otel_ep}")
except ImportError as _e:
print(f"[otel] skip: {_e}")
app.include_router(health.router, tags=["Health"])
app.include_router(search.router, prefix="/v1", tags=["Search"])
app.include_router(image_search.router, prefix="/v1", tags=["Image Search"])
app.include_router(fetch.router, prefix="/v1", tags=["Fetch"])
app.include_router(gather.router, prefix="/v1", tags=["Gather"])
app.include_router(info.router, tags=["Catalog"])
return app

177
ai_platform/modules/web/src/web/api/dependencies.py Normal file
View file

@ -0,0 +1,177 @@
"""FastAPI dependencies for Web module."""
import asyncio
import hmac
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from fastapi import Header, HTTPException, Request
from web.config import WebSettings
from web.logging import get_logger
from web.search.protocol import SearchProvider
logger = get_logger("dependencies")
def get_search_tier(
request: Request,
x_search_tier: str | None = Header(default=None, alias="X-Search-Tier"),
) -> str:
"""Extract search tier from X-Search-Tier header.
Returns:
"free" or "premium". Defaults to "free".
"""
if x_search_tier and x_search_tier.lower() == "premium":
return "premium"
return "free"
def get_search_client(request: Request) -> SearchProvider:
"""Get the Search client from application state (default/legacy)."""
return request.app.state.search_client
def get_search_client_for_tier(
request: Request,
tier: str,
) -> SearchProvider:
"""Get tier-specific search client.
Args:
request: FastAPI request.
tier: "free" or "premium".
Returns:
SearchProvider for the given tier.
"""
if tier == "premium":
return request.app.state.search_client_premium
return request.app.state.search_client_free
def get_settings(request: Request) -> WebSettings:
"""Get settings from application state."""
return request.app.state.settings
def verify_bearer_token(
request: Request,
authorization: str | None = Header(default=None, alias="Authorization"),
) -> str | None:
"""Verify Bearer token authentication.
If authentication is disabled (no tokens configured), returns None.
If authentication is enabled, validates the token and returns it.
"""
settings = request.app.state.settings
if not settings.auth_enabled:
return None
if not authorization:
raise HTTPException(
status_code=401,
detail={
"error": "Authentication required",
"message": "Missing Authorization header",
},
headers={"WWW-Authenticate": "Bearer"},
)
parts = authorization.split()
if len(parts) != 2 or parts[0].lower() != "bearer":
raise HTTPException(
status_code=401,
detail={
"error": "Authentication required",
"message": "Invalid Authorization header format. Use: Bearer <token>",
},
headers={"WWW-Authenticate": "Bearer"},
)
token = parts[1]
is_valid = any(
hmac.compare_digest(token.encode(), valid_token.encode())
for valid_token in settings.api_tokens
)
if not is_valid:
raise HTTPException(
status_code=401,
detail={
"error": "Authentication failed",
"message": "Invalid API token",
},
headers={"WWW-Authenticate": "Bearer"},
)
return token
class ConcurrencyLimiter:
"""Limits concurrent searches to prevent resource exhaustion."""
def __init__(self, max_concurrent: int) -> None:
"""Initialize concurrency limiter."""
self.max_concurrent = max_concurrent
self._semaphore = asyncio.Semaphore(max_concurrent)
self._current = 0
@property
def current_count(self) -> int:
"""Get current number of active requests."""
return self._current
@property
def available(self) -> int:
"""Get number of available slots."""
return self.max_concurrent - self._current
@asynccontextmanager
async def acquire(self, blocking: bool = False) -> AsyncIterator[None]:
"""Acquire a search slot."""
if not blocking and self._semaphore.locked():
logger.warning(
"Concurrency limit reached: %d/%d active",
self._current,
self.max_concurrent,
)
raise HTTPException(
status_code=503,
detail={
"error": "Service temporarily unavailable",
"reason": "Too many concurrent requests",
"max_concurrent": self.max_concurrent,
"retry_after": 5,
},
headers={"Retry-After": "5"},
)
await self._semaphore.acquire()
self._current += 1
try:
yield
finally:
self._current -= 1
self._semaphore.release()
_concurrency_limiter: ConcurrencyLimiter | None = None
def init_concurrency_limiter(max_concurrent: int) -> ConcurrencyLimiter:
"""Initialize the global concurrency limiter."""
global _concurrency_limiter
_concurrency_limiter = ConcurrencyLimiter(max_concurrent)
logger.info("Concurrency limiter initialized: max_concurrent=%d", max_concurrent)
return _concurrency_limiter
def get_concurrency_limiter() -> ConcurrencyLimiter:
"""Get the global concurrency limiter."""
if _concurrency_limiter is None:
raise RuntimeError("Concurrency limiter not initialized")
return _concurrency_limiter

164
ai_platform/modules/web/src/web/api/middleware.py Normal file
View file

@ -0,0 +1,164 @@
"""Pure ASGI middleware for request handling and rate limiting."""
import asyncio
import math
import re
import time
import uuid
from typing import Any
from starlette.types import ASGIApp, Receive, Scope, Send
from web.logging import get_logger, set_request_id
logger = get_logger("middleware")
_REQUEST_ID_RE = re.compile(r"^[a-zA-Z0-9\-_.]{1,128}$")
class TokenBucket:
"""Token bucket for rate limiting (async-safe)."""
def __init__(self, rate: float, burst: int) -> None:
"""Initialize token bucket.
Args:
rate: Tokens added per second.
burst: Maximum tokens (bucket capacity).
"""
self.rate = rate
self.burst = burst
self.tokens = float(burst)
self.last_update = time.monotonic()
self._lock = asyncio.Lock()
async def acquire(self) -> tuple[bool, float]:
"""Try to acquire a token (async-safe).
Returns:
tuple: (success, retry_after_seconds)
"""
async with self._lock:
now = time.monotonic()
elapsed = now - self.last_update
self.last_update = now
self.tokens = min(self.burst, self.tokens + elapsed * self.rate)
if self.tokens >= 1:
self.tokens -= 1
return True, 0.0
tokens_needed = 1 - self.tokens
return False, tokens_needed / self.rate
class CombinedMiddleware:
"""Combined request-id + rate-limiting pure ASGI middleware.
WARNING: Rate limiting is per-process. In multi-replica deployments,
each replica maintains its own independent rate limit.
TODO: For multi-replica deployments, replace TokenBucket with a
Redis-based rate limiter (e.g., sliding window counter in Redis)
to enforce global rate limits across all replicas. Add a
``rate_limit_backend`` config option (``memory`` | ``redis``)
to WebSettings to select the backend at startup.
"""
def __init__(
self,
app: ASGIApp,
rate: float = 10.0,
burst: int = 20,
exclude_paths: list[str] | None = None,
) -> None:
"""Initialize combined middleware.
Args:
app: The ASGI application.
rate: Tokens added per second for rate limiting.
burst: Maximum tokens (bucket capacity).
exclude_paths: Paths to exclude from rate limiting.
"""
self.app = app
self.bucket = TokenBucket(rate, burst)
self.exclude_paths = set(exclude_paths or ["/health", "/ready"])
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
"""Process ASGI request."""
if scope["type"] != "http":
await self.app(scope, receive, send)
return
# Extract/generate request ID from headers
headers_list: list[tuple[bytes, bytes]] = scope.get("headers", [])
request_id = ""
for name, value in headers_list:
if name == b"x-request-id":
request_id = value.decode()
break
if not request_id or not _REQUEST_ID_RE.match(request_id):
request_id = str(uuid.uuid4())
set_request_id(request_id)
path = scope.get("path", "")
# Rate limiting (skip excluded paths)
if path not in self.exclude_paths:
allowed, retry_after = await self.bucket.acquire()
if not allowed:
logger.warning(
"Rate limit exceeded for %s, retry_after=%.2f",
path,
retry_after,
)
await self._send_429(send, request_id, retry_after)
set_request_id(None)
return
# Inject response header
async def send_with_request_id(message: dict[str, Any]) -> None:
if message["type"] == "http.response.start":
headers = list(message.get("headers", []))
headers.append((b"x-request-id", request_id.encode()))
message["headers"] = headers
await send(message)
try:
await self.app(scope, receive, send_with_request_id)
finally:
set_request_id(None)
async def _send_429(self, send: Send, request_id: str, retry_after: float) -> None:
"""Send a 429 Too Many Requests response."""
import json
from web.schemas.common import make_error_detail
body = json.dumps(
make_error_detail(
"rate_limit",
"Too many requests",
request_id=request_id,
retry_after=retry_after,
)
).encode()
await send(
{
"type": "http.response.start",
"status": 429,
"headers": [
(b"content-type", b"application/json"),
(b"retry-after", str(math.ceil(retry_after)).encode()),
(b"x-request-id", request_id.encode()),
],
}
)
await send(
{
"type": "http.response.body",
"body": body,
}
)

1
ai_platform/modules/web/src/web/api/routes/__init__.py Normal file
View file

@ -0,0 +1 @@
"""API routes package."""

88
ai_platform/modules/web/src/web/api/routes/fetch.py Normal file
View file

@ -0,0 +1,88 @@
"""Fetch endpoint."""
from fastapi import APIRouter, Depends, HTTPException, Request
from web.api.dependencies import (
get_concurrency_limiter,
verify_bearer_token,
)
from web.exceptions import (
WebConnectionError,
WebError,
WebTimeoutError,
)
from web.fetch.client import FetchClient
from web.logging import get_logger, get_request_id
from web.schemas.common import make_error_detail
from web.schemas.fetch import FetchRequest, FetchResponse
logger = get_logger("routes.fetch")
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
def get_fetch_client(request: Request) -> FetchClient:
"""Get the fetch client from application state."""
return request.app.state.fetch_client
@router.post("/fetch", response_model=FetchResponse)
async def fetch(
request: Request,
fetch_request: FetchRequest,
client: FetchClient = Depends(get_fetch_client),
) -> FetchResponse:
"""Fetch content from URLs.
Args:
request: FastAPI request object.
fetch_request: Fetch parameters.
client: Fetch client.
Returns:
FetchResponse: Fetched content.
Raises:
HTTPException: On fetch errors.
"""
limiter = get_concurrency_limiter()
request_id = get_request_id()
async with limiter.acquire():
try:
result = await client.fetch(fetch_request, request_id=request_id)
return result
except WebTimeoutError as e:
logger.error("Fetch timeout: %s", e)
raise HTTPException(
status_code=504,
detail=make_error_detail("timeout", str(e), request_id=request_id),
) from None
except WebConnectionError as e:
logger.error("Connection error: %s", e)
raise HTTPException(
status_code=502,
detail=make_error_detail(
"connection_error", str(e), request_id=request_id
),
) from None
except WebError as e:
logger.error("Fetch error: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail("fetch_error", str(e), request_id=request_id),
) from None
except Exception as e:
logger.exception("Unexpected error during fetch: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail(
"internal_error",
f"Unexpected error: {type(e).__name__}: {e}",
request_id=request_id,
),
) from e

265
ai_platform/modules/web/src/web/api/routes/gather.py Normal file
View file

@ -0,0 +1,265 @@
"""Gather endpoint - unified pipeline."""
import time
from fastapi import APIRouter, Depends, HTTPException, Request
from web.api.dependencies import (
get_concurrency_limiter,
get_search_tier,
verify_bearer_token,
)
from web.brain.adapter import brain_response_to_web, web_response_to_ingest_payload
from web.brain.quality import brain_hit_acceptable, quality_ok_for_cache
from web.exceptions import WebError
from web.logging import get_logger, get_request_id
from web.orchestrator import Orchestrator
from web.schemas.common import make_error_detail
from web.schemas.gather import GatherRequest, GatherResponse
logger = get_logger("routes.gather")
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
def get_orchestrator_for_tier(request: Request, tier: str) -> Orchestrator:
"""Get tier-specific orchestrator from application state."""
if tier == "premium":
return request.app.state.orchestrator_premium
return request.app.state.orchestrator_free
def _provider_for_tier(tier: str) -> str:
return "paid-rotation" if tier == "premium" else "searxng"
_MAX_RAW_RESPONSE_BYTES = 200_000
def _trim_response_for_storage(response: GatherResponse) -> dict:
"""Build a persistable snapshot of the GatherResponse.
Drops full_text on evidence items beyond the first few to keep
row size reasonable. The trimmed snapshot is still enough to
promote a gather into the archive.
"""
data = response.model_dump(mode="json")
# Keep full_text on the first 3 items only
evidence = data.get("evidence") or []
for i, item in enumerate(evidence):
if i >= 3 and item.get("full_text"):
item["full_text"] = item["full_text"][:500] + "…"
# Truncate search_results snippets
for sr in data.get("search_results") or []:
if sr.get("snippet") and len(sr["snippet"]) > 400:
sr["snippet"] = sr["snippet"][:400] + "…"
return data
def _emit(
request: Request,
request_id: str,
tier: str,
start: float,
status_code: int,
response: GatherResponse | None = None,
error: str | None = None,
claim: str | None = None,
) -> None:
sink = getattr(request.app.state, "event_sink", None)
if sink is None or not sink.enabled:
return
duration_ms = int((time.perf_counter() - start) * 1000)
event: dict = {
"request_id": request_id,
"tier": tier,
"endpoint": "/v1/gather",
"provider": _provider_for_tier(tier),
"claim": claim,
"duration_ms": duration_ms,
"status_code": status_code,
"error": error,
}
if response is not None:
event["results_count"] = response.total_urls_found
event["evidence_count"] = response.total_evidence_items
event["stages"] = [s.model_dump() for s in response.stages]
# Persist the full response for archive promotion — trimmed to avoid
# blowing up the history table. Drop entirely if over the cap.
try:
snapshot = _trim_response_for_storage(response)
import json
serialized = json.dumps(snapshot, default=str)
if len(serialized.encode()) <= _MAX_RAW_RESPONSE_BYTES:
event["raw_response"] = snapshot
except Exception:
pass
sink.emit(event)
@router.post("/gather", response_model=GatherResponse)
async def gather(
request: Request,
gather_request: GatherRequest,
tier: str = Depends(get_search_tier),
) -> GatherResponse:
"""Execute the full gather pipeline: search -> fetch -> evidence.
Tier selection via X-Search-Tier header:
- "free" (default): SearXNG + local Qwen LLM
- "premium": Paid search engines + OpenRouter LLM
Args:
request: FastAPI request object.
gather_request: Gather parameters.
tier: Search tier from X-Search-Tier header.
Returns:
GatherResponse: Gathered evidence.
Raises:
HTTPException: On pipeline errors.
"""
orchestrator = get_orchestrator_for_tier(request, tier)
limiter = get_concurrency_limiter()
request_id = get_request_id()
start = time.perf_counter()
# Apply runtime config override for max_search_results if backend
# didn't explicitly pass a value (comparing against the default is
# fragile but workable: if tier override is set, apply it).
runtime_cfg = getattr(request.app.state, "runtime_config", None)
if runtime_cfg is not None:
override_key = f"web.tier.{tier}.max_search_results"
override_val = runtime_cfg.get_int(override_key, 0)
if override_val and gather_request.max_search_results == 10:
gather_request = gather_request.model_copy(
update={"max_search_results": override_val}
)
logger.info(
"Gather request [tier=%s max_results=%d]: %.100s",
tier,
gather_request.max_search_results,
gather_request.claim,
)
settings = request.app.state.settings
brain_client = getattr(request.app.state, "brain_client", None)
brain_sink = getattr(request.app.state, "brain_sink", None)
# --- Cache read: both tiers benefit from the premium-quality cache.
# Free users get the paid-for knowledge base for free; only premium
# populates it (see cache-write below).
if (
brain_client is not None
and brain_client.enabled
and settings.brain_cache_read_enabled
):
brain_raw = await brain_client.gather(
claim=gather_request.claim,
max_evidence=max(gather_request.max_evidence_items, 5),
run_nli=False,
include_full_text=gather_request.include_full_text,
)
if brain_hit_acceptable(
brain_raw,
min_evidence=settings.brain_hit_min_evidence,
min_relevance=settings.brain_hit_min_relevance,
):
cache_status = (
(brain_raw or {}).get("brain_meta") or {}
).get("cache_status", "?")
logger.info(
"Brain cache %s [tier=%s, %s items]: %.80s",
cache_status,
tier,
len((brain_raw or {}).get("evidence") or []),
gather_request.claim,
)
web_result = brain_response_to_web(brain_raw or {}, request_id)
_emit(
request,
request_id,
tier,
start,
status_code=200,
response=web_result,
claim=gather_request.claim,
)
return web_result
async with limiter.acquire():
try:
result = await orchestrator.gather(gather_request, request_id=request_id)
# --- Cache write: only for successful premium runs meeting quality bar
if (
tier == "premium"
and brain_sink is not None
and brain_sink.enabled
and settings.brain_ingest_enabled
and quality_ok_for_cache(
result,
min_evidence=settings.brain_ingest_min_evidence,
min_credibility=settings.brain_ingest_min_credibility,
min_execution_ms=settings.brain_ingest_min_execution_ms,
)
):
payload = web_response_to_ingest_payload(
result, claim=gather_request.claim
)
brain_sink.emit(payload)
_emit(
request,
request_id,
tier,
start,
status_code=200,
response=result,
claim=gather_request.claim,
)
return result
except WebError as e:
logger.error("Gather error: %s", e)
_emit(
request,
request_id,
tier,
start,
status_code=500,
error=str(e),
claim=gather_request.claim,
)
raise HTTPException(
status_code=500,
detail=make_error_detail("gather_error", str(e), request_id=request_id),
) from None
except Exception as e:
logger.exception("Unexpected error during gather: %s", e)
_emit(
request,
request_id,
tier,
start,
status_code=500,
error=f"{type(e).__name__}: {e}",
claim=gather_request.claim,
)
raise HTTPException(
status_code=500,
detail=make_error_detail(
"internal_error",
f"Unexpected error: {type(e).__name__}: {e}",
request_id=request_id,
),
) from e

57
ai_platform/modules/web/src/web/api/routes/health.py Normal file
View file

@ -0,0 +1,57 @@
"""Health check endpoints."""
from fastapi import APIRouter, Depends, Request
from web.api.dependencies import get_search_client
from web.schemas.common import HealthResponse, ProviderHealth, ReadinessResponse
from web.search.protocol import SearchProvider
router = APIRouter()
@router.get("/health", response_model=HealthResponse)
async def health_check(
client: SearchProvider = Depends(get_search_client),
) -> HealthResponse:
"""Check the health of the service and its providers.
Returns:
HealthResponse: Health status of the service.
"""
search_healthy = await client.health_check()
# Determine provider name from client attribute or default
provider_name = getattr(client, "provider_name", "searxng")
providers = [
ProviderHealth(
name=provider_name,
healthy=search_healthy,
message=None if search_healthy else f"{provider_name} search unavailable",
)
]
all_healthy = all(p.healthy for p in providers)
any_healthy = any(p.healthy for p in providers)
if all_healthy:
status = "healthy"
elif any_healthy:
status = "degraded"
else:
status = "unhealthy"
return HealthResponse(status=status, providers=providers)
@router.get("/ready", response_model=ReadinessResponse)
async def readiness_check(request: Request) -> ReadinessResponse:
"""Check if the service is ready to accept requests.
Returns:
ReadinessResponse: Readiness status.
"""
ready = hasattr(request.app.state, "search_client_free") and hasattr(
request.app.state, "orchestrator_free"
)
return ReadinessResponse(ready=ready)

115
ai_platform/modules/web/src/web/api/routes/image_search.py Normal file
View file

@ -0,0 +1,115 @@
"""Image search endpoint."""
from fastapi import APIRouter, Depends, HTTPException, Request
from web.api.dependencies import (
get_concurrency_limiter,
get_search_client_for_tier,
get_search_tier,
verify_bearer_token,
)
from web.exceptions import (
ProviderError,
RateLimitError,
SearchError,
WebConnectionError,
WebTimeoutError,
)
from web.logging import get_logger, get_request_id
from web.schemas.common import make_error_detail
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
logger = get_logger("routes.image_search")
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
@router.post("/image-search", response_model=ImageSearchResponse)
async def image_search(
request: Request,
search_request: ImageSearchRequest,
tier: str = Depends(get_search_tier),
) -> ImageSearchResponse:
"""Execute image search queries.
Args:
request: FastAPI request object.
search_request: Image search parameters.
tier: Search tier from X-Search-Tier header.
Returns:
ImageSearchResponse: Image search results.
Raises:
HTTPException: On search errors.
"""
client = get_search_client_for_tier(request, tier)
limiter = get_concurrency_limiter()
request_id = get_request_id()
async with limiter.acquire():
try:
result = await client.image_search(search_request, request_id=request_id)
return result
except RateLimitError as e:
logger.warning("Rate limit exceeded: %s", e)
raise HTTPException(
status_code=429,
detail=make_error_detail(
"rate_limit_exceeded",
str(e),
request_id=request_id,
retry_after=e.retry_after,
),
headers={"Retry-After": str(int(e.retry_after or 60))},
) from None
except WebTimeoutError as e:
logger.error("Image search timeout: %s", e)
raise HTTPException(
status_code=504,
detail=make_error_detail("timeout", str(e), request_id=request_id),
) from None
except WebConnectionError as e:
logger.error("Connection error: %s", e)
raise HTTPException(
status_code=502,
detail=make_error_detail(
"connection_error",
str(e),
request_id=request_id,
provider=e.provider,
),
) from None
except ProviderError as e:
logger.error("Provider error: %s", e)
raise HTTPException(
status_code=502,
detail=make_error_detail(
"provider_error",
str(e),
request_id=request_id,
provider=e.provider,
),
) from None
except SearchError as e:
logger.error("Image search error: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail("search_error", str(e), request_id=request_id),
) from None
except Exception as e:
logger.exception("Unexpected error during image search: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail(
"internal_error",
f"Unexpected error: {type(e).__name__}: {e}",
request_id=request_id,
),
) from e

169
ai_platform/modules/web/src/web/api/routes/info.py Normal file
View file

@ -0,0 +1,169 @@
"""Component information endpoint for service catalog."""
from fastapi import APIRouter
from web.config import get_settings
from web.schemas.fetch import FetchRequest, FetchResponse
from web.schemas.gather import GatherRequest, GatherResponse
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse
router = APIRouter()
@router.get("/v1/info")
def get_component_info():
"""Get component information for service catalog.
Returns complete metadata about this service including:
- Resource information (component metadata)
- Available functions (API endpoints)
This endpoint is used by the catalog-api to aggregate service information
and by backend systems to populate the catalog database.
Returns:
dict: Component information matching catalog.resources and catalog.functions schemas.
"""
settings = get_settings()
# Build resource information (maps to catalog.resources)
resource = {
"name": "Web Fact-checking Service",
"slug": "web-factcheck",
"resource_type": "api_service",
"provider": "internal",
"base_url": f"http://web-api:{settings.port}",
"configuration": {
"version": "0.1.0",
"port": settings.port,
"external_url": f"http://localhost:{settings.port}",
"llm_base_url": settings.llm_base_url,
"vision_model": settings.vision_model,
"text_model": settings.text_model,
},
"authentication": {
"type": "bearer" if settings.api_tokens else "none",
"required": bool(settings.api_tokens),
"env_var": "WEB_API_TOKENS",
},
"headers": {
"Content-Type": "application/json",
"Accept": "application/json",
},
"rate_limits": {
"requests_per_second": settings.rate_limit_rps,
"burst": settings.rate_limit_burst,
"concurrent": settings.max_concurrent_requests,
},
"cost_tracking": {
"enabled": False,
},
"tags": ["web", "fact-checking", "search", "evidence", "searxng"],
"is_active": True,
"metadata": {
"category": "web",
"gpu_required": False,
"status": "healthy",
"brave_api_required": False,
"llm_integration": bool(settings.llm_base_url),
},
}
# No models for web module (it's an orchestrator)
models = []
# Define available functions (maps to catalog.functions)
functions = [
{
"name": "Fact-checking Evidence Gathering",
"slug": "web-gather-evidence",
"category": "fact-checking",
"description": (
"Complete fact-checking pipeline: search web via SearXNG metasearch, "
"fetch URLs with fallback, extract evidence snippets using LLM, "
"deduplicate and rank by relevance."
),
"input_schema": GatherRequest.model_json_schema(),
"output_schema": GatherResponse.model_json_schema(),
"implementation": {
"method": "POST",
"path": "/v1/gather",
"content_type": "application/json",
"timeout": 120,
},
"endpoint": f"http://localhost:{settings.port}/v1/gather",
"tags": ["fact-checking", "evidence", "web-search"],
"is_active": True,
"metadata": {
"average_latency_s": 40,
"requires_brave_api": False,
"requires_llm": True,
},
},
{
"name": "Web Search",
"slug": "web-search",
"category": "search",
"description": "Search the web using SearXNG metasearch engine",
"input_schema": SearchRequest.model_json_schema(),
"output_schema": SearchResponse.model_json_schema(),
"implementation": {
"method": "POST",
"path": "/v1/search",
"content_type": "application/json",
"timeout": 10,
},
"endpoint": f"http://localhost:{settings.port}/v1/search",
"tags": ["search", "searxng"],
"is_active": True,
"metadata": {},
},
{
"name": "Image Search",
"slug": "web-image-search",
"category": "search",
"description": "Search for images using SearXNG metasearch engine",
"input_schema": ImageSearchRequest.model_json_schema(),
"output_schema": ImageSearchResponse.model_json_schema(),
"implementation": {
"method": "POST",
"path": "/v1/image-search",
"content_type": "application/json",
"timeout": 10,
},
"endpoint": f"http://localhost:{settings.port}/v1/image-search",
"tags": ["search", "images", "searxng"],
"is_active": True,
"metadata": {},
},
{
"name": "Fetch URLs",
"slug": "web-fetch",
"category": "fetch",
"description": (
"Fetch and extract content from URLs with automatic fallback "
"(HTTP \u2192 Playwright \u2192 Vision LLM)"
),
"input_schema": FetchRequest.model_json_schema(),
"output_schema": FetchResponse.model_json_schema(),
"implementation": {
"method": "POST",
"path": "/v1/fetch",
"content_type": "application/json",
"timeout": 60,
},
"endpoint": f"http://localhost:{settings.port}/v1/fetch",
"tags": ["fetch", "scraping"],
"is_active": True,
"metadata": {
"supports_fallback": True,
},
},
]
return {
"resource": resource,
"models": models,
"functions": functions,
}

181
ai_platform/modules/web/src/web/api/routes/search.py Normal file
View file

@ -0,0 +1,181 @@
"""Search endpoint."""
import time
from fastapi import APIRouter, Depends, HTTPException, Request
from web.api.dependencies import (
get_concurrency_limiter,
get_search_client_for_tier,
get_search_tier,
verify_bearer_token,
)
from web.exceptions import (
ProviderError,
RateLimitError,
SearchError,
WebConnectionError,
WebTimeoutError,
)
from web.logging import get_logger, get_request_id
from web.schemas.common import make_error_detail
from web.schemas.search import SearchRequest, SearchResponse
logger = get_logger("routes.search")
router = APIRouter(dependencies=[Depends(verify_bearer_token)])
def _provider_for_tier(tier: str) -> str:
return "paid-rotation" if tier == "premium" else "searxng"
def _emit(
request: Request,
request_id: str,
tier: str,
start: float,
status_code: int,
response: SearchResponse | None = None,
error: str | None = None,
query: str | None = None,
) -> None:
sink = getattr(request.app.state, "event_sink", None)
if sink is None or not sink.enabled:
return
duration_ms = int((time.perf_counter() - start) * 1000)
event = {
"request_id": request_id,
"tier": tier,
"endpoint": "/v1/search",
"provider": _provider_for_tier(tier),
"query": query,
"duration_ms": duration_ms,
"status_code": status_code,
"error": error,
}
if response is not None:
event["results_count"] = response.total_results
sink.emit(event)
@router.post("/search", response_model=SearchResponse)
async def search(
request: Request,
search_request: SearchRequest,
tier: str = Depends(get_search_tier),
) -> SearchResponse:
"""Execute web search queries.
Tier selection via X-Search-Tier header:
- "free" (default): SearXNG only
- "premium": Paid search engines only
Args:
request: FastAPI request object.
search_request: Search parameters.
tier: Search tier from X-Search-Tier header.
Returns:
SearchResponse: Search results.
Raises:
HTTPException: On search errors.
"""
client = get_search_client_for_tier(request, tier)
limiter = get_concurrency_limiter()
request_id = get_request_id()
start = time.perf_counter()
query_summary = " | ".join(search_request.queries)[:500]
logger.info("Search request [tier=%s]", tier)
async with limiter.acquire():
try:
result = await client.search(search_request, request_id=request_id)
_emit(
request,
request_id,
tier,
start,
status_code=200,
response=result,
query=query_summary,
)
return result
except RateLimitError as e:
_emit(request, request_id, tier, start, 429, error=str(e), query=query_summary)
logger.warning("Rate limit exceeded: %s", e)
raise HTTPException(
status_code=429,
detail=make_error_detail(
"rate_limit_exceeded",
str(e),
request_id=request_id,
retry_after=e.retry_after,
),
headers={"Retry-After": str(int(e.retry_after or 60))},
) from None
except WebTimeoutError as e:
_emit(request, request_id, tier, start, 504, error=str(e), query=query_summary)
logger.error("Search timeout: %s", e)
raise HTTPException(
status_code=504,
detail=make_error_detail("timeout", str(e), request_id=request_id),
) from None
except WebConnectionError as e:
_emit(request, request_id, tier, start, 502, error=str(e), query=query_summary)
logger.error("Connection error: %s", e)
raise HTTPException(
status_code=502,
detail=make_error_detail(
"connection_error",
str(e),
request_id=request_id,
provider=e.provider,
),
) from None
except ProviderError as e:
_emit(request, request_id, tier, start, 502, error=str(e), query=query_summary)
logger.error("Provider error: %s", e)
raise HTTPException(
status_code=502,
detail=make_error_detail(
"provider_error",
str(e),
request_id=request_id,
provider=e.provider,
),
) from None
except SearchError as e:
_emit(request, request_id, tier, start, 500, error=str(e), query=query_summary)
logger.error("Search error: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail("search_error", str(e), request_id=request_id),
) from None
except Exception as e:
_emit(
request,
request_id,
tier,
start,
500,
error=f"{type(e).__name__}: {e}",
query=query_summary,
)
logger.exception("Unexpected error during search: %s", e)
raise HTTPException(
status_code=500,
detail=make_error_detail(
"internal_error",
f"Unexpected error: {type(e).__name__}: {e}",
request_id=request_id,
),
) from e

14
ai_platform/modules/web/src/web/brain/__init__.py Normal file
View file

@ -0,0 +1,14 @@
"""Brain integration — premium results → brain cache + cache-read."""
from web.brain.adapter import brain_response_to_web
from web.brain.client import BrainClient
from web.brain.quality import quality_ok_for_cache, brain_hit_acceptable
from web.brain.sink import BrainIngestSink
__all__ = [
"BrainClient",
"BrainIngestSink",
"brain_response_to_web",
"quality_ok_for_cache",
"brain_hit_acceptable",
]

211
ai_platform/modules/web/src/web/brain/adapter.py Normal file
View file

@ -0,0 +1,211 @@
"""Map brain-api responses to web-api GatherResponse shape.
Brain and web use slightly different schemas (datetime vs ISO string,
nested Provenance with brain_meta vs plain dict, etc). This adapter
normalizes brain responses so web clients see a consistent shape.
"""
from __future__ import annotations
import hashlib
from typing import Any
from web.schemas.context import EntitySet, SearchContext
from web.schemas.evidence import EvidenceItem, EvidenceStats
from web.schemas.gather import GatherResponse, GatherStageResult
from web.schemas.search import SearchResult
def _iso(value: Any) -> str | None:
"""Normalize a datetime-like value to ISO string (brain emits datetime)."""
if value is None:
return None
if isinstance(value, str):
return value
# datetime has isoformat(); pydantic will usually have already serialized
try:
return value.isoformat() # type: ignore[no-any-return]
except Exception:
return str(value)
def _hash_if_missing(text: str | None, existing: str | None) -> str:
"""Compute a sha256 hex digest when brain didn't supply one."""
if existing:
return existing
if not text:
return ""
return hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
def _map_evidence(item: dict[str, Any]) -> EvidenceItem:
provenance_src = item.get("provenance") or {}
extraction_method = (
provenance_src.get("extraction_method")
if isinstance(provenance_src, dict)
else None
) or "brain"
provenance = {
"extraction_method": extraction_method,
"fallback_chain": (
provenance_src.get("fallback_chain")
if isinstance(provenance_src, dict)
else None
) or [],
"source": "brain",
}
# Preserve brain's NLI metadata inside provenance (non-breaking addition)
if isinstance(provenance_src, dict) and provenance_src.get("brain_meta"):
provenance["brain_meta"] = provenance_src["brain_meta"]
return EvidenceItem(
url=item.get("url") or "",
canonical_url=item.get("canonical_url"),
title=item.get("title") or None,
publisher=item.get("publisher") or None,
published_at=_iso(item.get("published_at")),
retrieved_at=_iso(item.get("retrieved_at")) or "",
snippet=item.get("snippet"),
summary=item.get("summary"),
full_text=item.get("full_text"),
full_text_hash=_hash_if_missing(
item.get("full_text"), item.get("full_text_hash")
),
provenance=provenance,
relevance_score=item.get("relevance_score"),
credibility_score=item.get("credibility_score"),
)
def _map_search_result(item: dict[str, Any]) -> SearchResult:
return SearchResult(
query=item.get("query") or "",
url=item.get("url") or "",
title=item.get("title") or "",
snippet=item.get("snippet") or "",
rank=item.get("rank") or 1,
site=item.get("site") or "",
published_at=_iso(item.get("published_at")),
)
def _map_stage(stage: dict[str, Any]) -> GatherStageResult:
return GatherStageResult(
stage=stage.get("stage") or "unknown",
success=bool(stage.get("success", False)),
items_processed=stage.get("items_processed") or 0,
items_failed=stage.get("items_failed") or 0,
duration_ms=float(stage.get("duration_ms") or 0.0),
error=stage.get("error"),
)
def _map_context(ctx: dict[str, Any] | None) -> SearchContext | None:
if not ctx:
return None
entities = ctx.get("entities") or {}
return SearchContext(
primary_country=ctx.get("primary_country"),
secondary_countries=ctx.get("secondary_countries") or [],
entities=EntitySet(
persons=entities.get("persons") or [],
institutions=entities.get("institutions") or [],
locations=entities.get("locations") or [],
),
detected_language=ctx.get("detected_language") or "en",
search_queries=ctx.get("search_queries") or [],
)
def _map_stats(stats: dict[str, Any] | None) -> EvidenceStats:
s = stats or {}
return EvidenceStats(
input_items=s.get("input_items") or 0,
after_dedup=s.get("after_dedup") or 0,
output_items=s.get("output_items") or 0,
duplicates_removed=s.get("duplicates_removed") or 0,
tokens_used=s.get("tokens_used") or 0,
)
def brain_response_to_web(
brain: dict[str, Any],
request_id: str,
) -> GatherResponse:
"""Convert a brain /v1/gather JSON response into a web GatherResponse."""
evidence_items = [_map_evidence(e) for e in (brain.get("evidence") or [])]
search_results = [_map_search_result(s) for s in (brain.get("search_results") or [])]
stages = [_map_stage(s) for s in (brain.get("stages") or [])]
return GatherResponse(
request_id=request_id,
claim=brain.get("claim") or "",
evidence=evidence_items,
evidence_stats=_map_stats(brain.get("evidence_stats")),
search_context=_map_context(brain.get("search_context")),
search_results=search_results,
stages=stages,
total_urls_found=brain.get("total_urls_found") or len(search_results),
total_pages_fetched=brain.get("total_pages_fetched") or len(evidence_items),
total_evidence_items=brain.get("total_evidence_items") or len(evidence_items),
execution_time_ms=float(brain.get("execution_time_ms") or 0.0),
)
def web_response_to_ingest_payload(
response: GatherResponse,
claim: str,
default_tags: list[str] | None = None,
) -> dict[str, Any]:
"""Build a brain /v1/ingest payload from a web GatherResponse.
Brain's IngestRequest schema accepts EvidenceItem with the same field
names as web; only datetime fields get interpreted by pydantic on the
brain side, so sending ISO strings is fine.
"""
evidence_payload: list[dict[str, Any]] = []
for ev in response.evidence:
provenance = ev.provenance if isinstance(ev.provenance, dict) else {}
# Inherit extraction method; stamp source as web premium so we can
# audit/regret later if premium-sourced results turn out to be noisy.
provenance = {
**provenance,
"source": provenance.get("source") or "web-premium",
}
# Brain requires title/retrieved_at non-null and uses float defaults
# for relevance/credibility. Map None → safe defaults so ingest validates.
evidence_payload.append(
{
"url": ev.url,
"canonical_url": ev.canonical_url,
"title": ev.title or "",
"publisher": ev.publisher or "",
"published_at": ev.published_at,
"retrieved_at": ev.retrieved_at,
"snippet": ev.snippet,
"summary": ev.summary,
"full_text": ev.full_text,
"full_text_hash": ev.full_text_hash or "",
"provenance": provenance,
"relevance_score": (
ev.relevance_score if ev.relevance_score is not None else 0.0
),
"credibility_score": (
ev.credibility_score if ev.credibility_score is not None else 0.5
),
}
)
tags = list(default_tags or [])
ctx = response.search_context
if ctx and ctx.primary_country and ctx.primary_country.lower() != "global":
tags.append(f"Country/{ctx.primary_country}")
if ctx and ctx.detected_language:
tags.append(f"Language/{ctx.detected_language.upper()}")
return {
"claim": claim,
"evidence": evidence_payload,
"default_tags": tags,
"run_extraction": True,
}

103
ai_platform/modules/web/src/web/brain/client.py Normal file
View file

@ -0,0 +1,103 @@
"""Async HTTP client for didi-brain (cache read + ingest write)."""
from typing import Any
import httpx
from web.logging import get_logger
logger = get_logger("brain.client")
class BrainClient:
"""Minimal client that talks to brain_api /v1/gather and /v1/ingest."""
def __init__(
self,
base_url: str | None,
gather_timeout: float = 8.0,
ingest_timeout: float = 15.0,
) -> None:
self.base_url = base_url.rstrip("/") if base_url else None
self._client: httpx.AsyncClient | None = None
self._gather_timeout = gather_timeout
self._ingest_timeout = ingest_timeout
self._enabled = bool(base_url)
@property
def enabled(self) -> bool:
return self._enabled
async def start(self) -> None:
if not self._enabled:
logger.info("BrainClient disabled (no brain URL)")
return
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=2.0,
read=max(self._gather_timeout, self._ingest_timeout),
write=5.0,
pool=5.0,
),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
)
logger.info("BrainClient started → %s", self.base_url)
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def gather(
self,
claim: str,
max_evidence: int = 15,
run_nli: bool = False,
include_full_text: bool = True,
) -> dict[str, Any] | None:
"""Ask brain for cached evidence. Returns None if brain unavailable."""
if not self._enabled or self._client is None or self.base_url is None:
return None
try:
resp = await self._client.post(
f"{self.base_url}/v1/gather",
json={
"claim": claim,
"max_evidence": max_evidence,
"run_nli": run_nli,
"include_full_text": include_full_text,
"summarize": False,
"score_relevance": True,
},
timeout=self._gather_timeout,
)
if resp.status_code != 200:
logger.debug("Brain gather non-200: %s", resp.status_code)
return None
return resp.json()
except Exception as e:
logger.debug("Brain gather failed (%s): %s", type(e).__name__, e)
return None
async def ingest(self, payload: dict[str, Any]) -> bool:
"""Push evidence to brain. Returns True on success, False otherwise."""
if not self._enabled or self._client is None or self.base_url is None:
return False
try:
resp = await self._client.post(
f"{self.base_url}/v1/ingest",
json=payload,
timeout=self._ingest_timeout,
)
resp.raise_for_status()
body = resp.json()
logger.info(
"Brain ingest OK: accepted=%d skipped_dup=%d errors=%d",
body.get("accepted", 0),
body.get("skipped_duplicate", 0),
body.get("errors", 0),
)
return True
except Exception as e:
logger.warning("Brain ingest failed (%s): %s", type(e).__name__, e)
return False

70
ai_platform/modules/web/src/web/brain/quality.py Normal file
View file

@ -0,0 +1,70 @@
"""Quality gates for brain cache read/write decisions."""
from typing import Any
from web.schemas.gather import GatherResponse
def quality_ok_for_cache(
response: GatherResponse,
min_evidence: int = 3,
min_credibility: float = 0.7,
min_execution_ms: float = 2000.0,
require_all_stages_ok: bool = True,
) -> bool:
"""Decide whether a web gather response is worth caching into brain.
Applied only to premium results (caller enforces tier). Conservative by
default — we prefer undercaching to polluting the brain with junk.
"""
if response.total_evidence_items < min_evidence:
return False
# Credibility check: if any item has a score, at least one must clear the bar.
# If no item was scored (scoring disabled), we trust premium by default.
scored = [e for e in response.evidence if e.credibility_score is not None]
if scored and not any(
(e.credibility_score or 0.0) >= min_credibility for e in scored
):
return False
if require_all_stages_ok and any(not s.success for s in response.stages):
return False
if response.execution_time_ms < min_execution_ms:
return False
return True
def brain_hit_acceptable(
brain_response: dict[str, Any] | None,
min_evidence: int = 3,
min_relevance: float = 0.7,
) -> bool:
"""Decide whether a brain gather response is good enough to serve.
HIT = always acceptable (brain already has strong confidence).
PARTIAL = require ≥ min_evidence items with relevance ≥ min_relevance.
MISS = never acceptable.
"""
if not brain_response:
return False
meta = brain_response.get("brain_meta") or {}
cache_status = meta.get("cache_status", "MISS")
if cache_status == "MISS":
return False
evidence = brain_response.get("evidence") or []
if cache_status == "HIT":
return len(evidence) >= 1 # HIT with any evidence is fine
# PARTIAL — require some strong items
strong = [
e
for e in evidence
if (e.get("relevance_score") or 0.0) >= min_relevance
]
return len(strong) >= min_evidence

69
ai_platform/modules/web/src/web/brain/sink.py Normal file
View file

@ -0,0 +1,69 @@
"""Fire-and-forget async sink that pushes premium web results to brain.
Mirror of DashboardEventSink: bounded queue, single worker, drop-oldest
on overflow, failures swallowed so brain outages don't affect web latency.
"""
import asyncio
import contextlib
from typing import Any
from web.brain.client import BrainClient
from web.logging import get_logger
logger = get_logger("brain.sink")
class BrainIngestSink:
def __init__(
self,
client: BrainClient,
queue_size: int = 1000,
) -> None:
self._client = client
self._queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=queue_size)
self._worker_task: asyncio.Task[None] | None = None
@property
def enabled(self) -> bool:
return self._client.enabled
async def start(self) -> None:
if not self._client.enabled:
logger.info("BrainIngestSink disabled (no brain URL)")
return
self._worker_task = asyncio.create_task(self._worker())
logger.info("BrainIngestSink started")
async def stop(self) -> None:
if self._worker_task is not None:
self._worker_task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await self._worker_task
self._worker_task = None
def emit(self, payload: dict[str, Any]) -> None:
"""Enqueue a brain /v1/ingest payload. Never raises."""
if not self.enabled:
return
try:
self._queue.put_nowait(payload)
except asyncio.QueueFull:
try:
_ = self._queue.get_nowait()
self._queue.put_nowait(payload)
except Exception:
pass
async def _worker(self) -> None:
while True:
try:
payload = await self._queue.get()
except asyncio.CancelledError:
raise
try:
await self._client.ingest(payload)
except Exception as e:
logger.debug("Brain ingest dropped (%s): %s", type(e).__name__, e)
finally:
self._queue.task_done()

5
ai_platform/modules/web/src/web/browse/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Browse submodule - Playwright browser automation."""
from web.browse.client import BrowseClient
__all__ = ["BrowseClient"]

701
ai_platform/modules/web/src/web/browse/client.py Normal file
View file

@ -0,0 +1,701 @@
"""Playwright-based browser client for JavaScript-heavy pages."""
import asyncio
import hashlib
import time
from datetime import datetime, timezone
from typing import Any
from web.config import WebSettings
from web.exceptions import (
WebError,
WebTimeoutError,
)
from web.logging import get_logger
from web.schemas.browse import BrowsePageResult, BrowseRequest, BrowseResponse
from web.schemas.common import FailedUrl, PageContent
logger = get_logger("browse.client")
# ---------------------------------------------------------------------------
# Stealth: realistic Chrome UA and headers to avoid bot detection
# ---------------------------------------------------------------------------
_CHROME_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
_STEALTH_HEADERS = {
"Accept": (
"text/html,application/xhtml+xml,application/xml;"
"q=0.9,image/avif,image/webp,*/*;q=0.8"
),
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
"Upgrade-Insecure-Requests": "1",
}
# Hide navigator.webdriver from page JS (survives navigations within context)
_STEALTH_INIT_SCRIPT = (
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
)
# ---------------------------------------------------------------------------
# Resource types to block in Playwright (images, fonts, media, stylesheets)
# ---------------------------------------------------------------------------
_BLOCK_RESOURCE_TYPES = frozenset({"image", "media", "font", "stylesheet"})
# ---------------------------------------------------------------------------
# CMP-aware consent dismissal (runs in browser JS with 2s self-timeout)
# ---------------------------------------------------------------------------
_DISMISS_CONSENT_JS = """
() => Promise.race([
(async () => {
// Tier 1: Call major CMP platform APIs directly
if (window.OneTrust && window.OneTrust.AllowAll) {
window.OneTrust.AllowAll();
return 'onetrust-api';
}
if (window.Cookiebot && window.Cookiebot.submitCustomConsent) {
window.Cookiebot.submitCustomConsent(true, true, true);
return 'cookiebot-api';
}
if (typeof window.__tcfapi === 'function') {
window.__tcfapi('setConsent', 2, () => {}, {purpose: {consents: {}}});
}
if (window.CookieControl && window.CookieControl.acceptAll) {
window.CookieControl.acceptAll();
return 'cookiecontrol-api';
}
// Tier 2: DOM selector fallback (major CMPs + generic)
const SELECTORS = [
'#onetrust-accept-btn-handler',
'#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
'.trustarc-agree-btn',
'.qc-cmp2-summary-buttons button:last-child',
'[data-testid="uc-accept-all-button"]',
'[data-cookiebanner="accept_button"]',
'[data-testid="cookie-policy-manage-dialog-accept-button"]',
'button[id*="cookie-accept"]',
'button[id*="cookieAccept"]',
'button[id*="accept"]',
'button[class*="accept"]',
'[aria-label*="Accept all" i]',
'[aria-label*="Agree" i]',
'button#tiktok-accept-row',
'[data-testid="xMigrationBottomBar"] button',
];
// Multilingual text patterns
const TEXT_PATTERNS = [
/^accept all$/i, /^accept cookies$/i, /^i accept$/i,
/^agree$/i, /^agree all$/i, /^got it$/i, /^ok$/i,
/^allow all$/i, /^allow cookies$/i, /^i agree$/i,
/^accept all cookies$/i,
/^akzeptieren$/i,
/^tout accepter$/i,
/^aceptar todo$/i,
/^accetta tutto$/i,
/^accept\\u0103 tot$/i,
/^accept\\u0103$/i,
/^de acord$/i,
];
for (const sel of SELECTORS) {
const el = document.querySelector(sel);
if (el && el.offsetParent !== null) { el.click(); return 'selector:' + sel; }
}
for (const btn of document.querySelectorAll('button, [role="button"]')) {
if (btn.offsetParent !== null && TEXT_PATTERNS.some(p => p.test(btn.innerText.trim()))) {
btn.click(); return 'text:' + btn.innerText.trim();
}
}
// Tier 3: generic dialog close buttons
const closeBtn = document.querySelector(
'div[role="dialog"] button[aria-label="Close"],' +
'div[role="dialog"] button[aria-label="Dismiss"],' +
'div[role="dialog"] div[aria-label="Close"],' +
'div[role="dialog"] div[aria-label="Închide"]'
);
if (closeBtn && closeBtn.offsetParent !== null) {
closeBtn.click(); return 'dialog-close';
}
return null;
})(),
new Promise(r => setTimeout(() => r(null), 2000))
])
"""
# ---------------------------------------------------------------------------
# Auth-wall / paywall detection (runs in browser JS with 5s self-timeout)
# ---------------------------------------------------------------------------
_DETECT_AUTH_WALL_JS = """
() => Promise.race([
Promise.resolve({
hasPasswordField: !!document.querySelector('input[type="password"]'),
hasPaywall: !!(document.querySelector(
'[class*="paywall"],[id*="paywall"],[class*="subscribe"],[class*="premium"]'
)),
hasCFChallenge: !!(
document.querySelector('script[src*="challenge-platform"]') ||
document.querySelector('iframe[src*="turnstile"]')
),
currentUrl: window.location.href,
}),
new Promise(r => setTimeout(() => r({__timeout: true}), 5000))
])
"""
# ---------------------------------------------------------------------------
# Content extraction via readability (Python-side) with JS metadata fallback
# ---------------------------------------------------------------------------
_EXTRACT_METADATA_JS = """
() => Promise.race([
Promise.resolve({
canonical: (() => {
const l = document.querySelector('link[rel="canonical"]');
return l ? l.href : null;
})(),
publishedAt: (() => {
const sels = [
'time[datetime]', '[itemprop="datePublished"]',
'meta[property="article:published_time"]', '.date', '.published'
];
for (const s of sels) {
const el = document.querySelector(s);
if (el) return el.getAttribute('datetime') || el.getAttribute('content') || el.innerText;
}
return null;
})(),
}),
new Promise(r => setTimeout(() => r({canonical: null, publishedAt: null}), 3000))
])
"""
# Fallback JS text extraction with browser-side timeout
_EXTRACT_TEXT_JS = """
() => Promise.race([
Promise.resolve((() => {
const main = document.querySelector('main, article, [role="main"], .content, #content');
return (main || document.body).innerText;
})()),
new Promise(r => setTimeout(() => r(''), 5000))
])
"""
# Optional imports
try:
from playwright.async_api import (
Browser,
Page,
Route,
TimeoutError as PlaywrightTimeout,
async_playwright,
)
HAS_PLAYWRIGHT = True
except ImportError:
HAS_PLAYWRIGHT = False
logger.warning("playwright not installed, browse functionality unavailable")
# Optional readability import (used for content extraction)
try:
from readability import Document as ReadabilityDocument
HAS_READABILITY = True
except ImportError:
HAS_READABILITY = False
try:
from lxml.html import fromstring as lxml_parse
HAS_LXML = True
except ImportError:
HAS_LXML = False
# Global lock for readability-lxml parsing (lxml is not thread-safe,
# concurrent parsing causes "double free or corruption" crashes)
_readability_lock = asyncio.Lock()
async def _block_resources(route: Route) -> None:
"""Abort image/media/font/stylesheet requests to speed up browsing."""
if route.request.resource_type in _BLOCK_RESOURCE_TYPES:
await route.abort()
else:
await route.continue_()
class BrowseClient:
"""Client for browser-based content extraction using Playwright."""
def __init__(
self,
settings: WebSettings,
browser: Browser | None = None,
) -> None:
"""Initialize the client.
Args:
settings: Application settings.
browser: Optional shared browser instance. If provided, the client
will not launch its own browser and will not close this one.
"""
if not HAS_PLAYWRIGHT:
raise ImportError(
"playwright is required for BrowseClient. "
"Install with: uv add playwright && uv run playwright install chromium"
)
self.settings = settings
self._playwright = None
self._browser: Browser | None = browser
self._owns_browser = browser is None
self._browser_lock = asyncio.Lock()
async def _get_browser(self) -> Browser:
"""Get or create the browser instance."""
async with self._browser_lock:
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-sandbox",
"--disable-setuid-sandbox",
],
)
return self._browser
@property
def shared_browser(self) -> Browser | None:
"""Get browser instance for sharing (e.g., with VisionClient)."""
if self._browser and self._browser.is_connected():
return self._browser
return None
async def __aenter__(self) -> "BrowseClient":
return self
async def __aexit__(self, *exc: object) -> None:
await self.close()
async def close(self) -> None:
"""Close the browser (only if this client owns it)."""
if self._owns_browser:
if self._browser is not None:
await self._browser.close()
self._browser = None
if self._playwright is not None:
await self._playwright.stop()
self._playwright = None
async def browse(
self,
request: BrowseRequest,
request_id: str | None = None,
) -> BrowseResponse:
"""Browse URLs and extract content.
Args:
request: Browse request with URLs and parameters.
request_id: Optional request ID for tracing.
Returns:
BrowseResponse: Browsed page content.
"""
start_time = time.perf_counter()
pages: list[BrowsePageResult] = []
failed: list[FailedUrl] = []
browser = await self._get_browser()
# Process URLs with concurrency limit
semaphore = asyncio.Semaphore(request.parallel_browses)
async def browse_with_semaphore(url: str) -> BrowsePageResult | None:
async with semaphore:
try:
return await self._browse_single(browser, url, request)
except Exception as e:
logger.error("Failed to browse %s: %s", url, e)
failed.append(FailedUrl(url=url, error=str(e)))
return None
# Browse all URLs concurrently
tasks = [browse_with_semaphore(url) for url in request.urls]
results = await asyncio.gather(*tasks)
for result in results:
if result is not None:
pages.append(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return BrowseResponse(
request_id=request_id or "-",
pages=pages,
total_browsed=len(pages),
total_failed=len(failed),
failed_urls=failed,
execution_time_ms=round(execution_time_ms, 2),
)
async def _browse_single(
self,
browser: Browser,
url: str,
request: BrowseRequest,
) -> BrowsePageResult:
"""Browse a single URL with retry on transient timeout.
Args:
browser: Playwright browser instance.
url: URL to browse.
request: Browse request with parameters.
Returns:
BrowsePageResult: Browsed page content.
"""
last_exc: Exception | None = None
max_attempts = max(1, self.settings.max_retries)
for attempt in range(max_attempts):
try:
return await self._attempt_browse(browser, url, request)
except WebTimeoutError as e:
last_exc = e
if attempt < max_attempts - 1:
wait = min(
self.settings.retry_min_wait * (2**attempt),
self.settings.retry_max_wait,
)
logger.warning(
"Browse timeout for %s (attempt %d/%d), retrying in %.1fs",
url,
attempt + 1,
max_attempts,
wait,
)
await asyncio.sleep(wait)
except WebError:
raise # Non-transient — no retry
raise last_exc # type: ignore[misc]
async def _attempt_browse(
self,
browser: Browser,
url: str,
request: BrowseRequest,
) -> BrowsePageResult:
"""Execute a single browse attempt.
Args:
browser: Playwright browser instance.
url: URL to browse.
request: Browse request with parameters.
Returns:
BrowsePageResult: Browsed page content.
"""
start_time = time.perf_counter()
warnings: list[str] = []
context = await browser.new_context(
viewport={
"width": self.settings.browse_viewport_width,
"height": self.settings.browse_viewport_height,
},
user_agent=_CHROME_UA,
extra_http_headers=_STEALTH_HEADERS,
)
# Hide navigator.webdriver
await context.add_init_script(_STEALTH_INIT_SCRIPT)
# Block heavy resources to speed up browsing
if self.settings.browse_block_resources:
await context.route("**/*", _block_resources)
page = await context.new_page()
# Auth-wall redirect listener
redirect_to_auth: list[str] = []
_login_path_patterns = (
"/login", "/signin", "/sign-in", "/auth", "/subscribe",
)
def _on_response(resp: Any) -> None:
if (
hasattr(resp, "status")
and resp.status in (301, 302, 303, 307, 308)
and any(p in resp.url for p in _login_path_patterns)
):
redirect_to_auth.append(resp.url)
page.on("response", _on_response)
try:
# Navigate
try:
response = await page.goto(
url,
wait_until=request.wait_until,
timeout=request.timeout_ms,
)
except PlaywrightTimeout as e:
raise WebTimeoutError(
f"Timeout loading {url}",
timeout=request.timeout_ms / 1000,
) from e
# Check HTTP status from page.goto() response
if response is None or not response.ok:
status = response.status if response else 0
raise WebError(f"HTTP {status} browsing {url}")
# Auth redirect detected
if redirect_to_auth:
warnings.append(
f"Redirect to auth wall observed: {redirect_to_auth[0]}"
)
# Wait for selector if requested
if request.wait_for_selector:
try:
await page.wait_for_selector(
request.wait_for_selector,
timeout=5000,
)
except PlaywrightTimeout:
logger.warning(
"Selector %s not found on %s",
request.wait_for_selector,
url,
)
# Dismiss consent banners (CMP-aware, browser-side 2s timeout)
await self._dismiss_consent_banners(page)
# Additional wait for dynamic content (SPA rendering)
if request.extra_wait_ms > 0:
await asyncio.sleep(request.extra_wait_ms / 1000)
# Detect auth-wall / paywall (browser-side 5s timeout)
auth_warning = await self._detect_auth_wall(page, url)
if auth_warning:
warnings.append(auth_warning)
# Extract content (readability + JS metadata)
extracted = await self._extract_content(page, request)
warnings.extend(extracted.get("warnings", []))
# Take screenshot if requested
screenshot_base64 = None
if request.screenshot:
screenshot_bytes = await page.screenshot(
type="jpeg",
quality=self.settings.vision_screenshot_quality,
full_page=request.full_page_screenshot,
)
import base64
screenshot_base64 = base64.b64encode(screenshot_bytes).decode()
extraction_time_ms = (time.perf_counter() - start_time) * 1000
retrieved_at = datetime.now(timezone.utc).isoformat()
return BrowsePageResult(
url=url,
final_url=page.url,
canonical_url=extracted.get("canonical_url"),
title=extracted.get("title"),
text=extracted["text"],
text_hash=hashlib.sha256(extracted["text"].encode()).hexdigest(),
html=extracted.get("html") if request.include_html else None,
extraction_method="browse",
fallback_chain=[],
published_at=extracted.get("published_at"),
retrieved_at=retrieved_at,
extraction_time_ms=round(extraction_time_ms, 2),
warnings=warnings,
screenshot_base64=screenshot_base64,
viewport_width=self.settings.browse_viewport_width,
viewport_height=self.settings.browse_viewport_height,
)
finally:
await context.close()
async def _dismiss_consent_banners(self, page: Page) -> None:
"""Dismiss cookie/consent banners using CMP APIs + DOM selectors.
Uses browser-side Promise.race with 2s timeout so it never blocks
longer than that regardless of page state.
Args:
page: Playwright page instance.
"""
try:
method = await page.evaluate(_DISMISS_CONSENT_JS)
if method:
logger.debug("Dismissed consent banner via: %s", method)
await asyncio.sleep(0.3)
except Exception as e:
logger.debug("Consent dismissal failed (non-fatal): %s", e)
async def _detect_auth_wall(self, page: Page, original_url: str) -> str | None:
"""Detect login walls, paywalls, and Cloudflare challenges.
Uses browser-side Promise.race with 5s timeout.
Args:
page: Playwright page instance.
original_url: The URL we intended to visit.
Returns:
Warning string if auth wall detected, None otherwise.
"""
try:
signals = await page.evaluate(_DETECT_AUTH_WALL_JS)
if isinstance(signals, dict) and signals.get("__timeout"):
logger.warning("Auth-wall detection timed out for %s", original_url)
return None
final_url = signals.get("currentUrl", "") if isinstance(signals, dict) else ""
login_keywords = (
"/login", "/signin", "/sign-in", "/auth/",
"/account/login", "/subscribe",
)
if any(kw in final_url for kw in login_keywords):
return f"Redirected to auth wall: {final_url}"
if isinstance(signals, dict):
if signals.get("hasCFChallenge"):
return "Cloudflare challenge page detected"
if signals.get("hasPasswordField"):
return "Login form detected on page"
if signals.get("hasPaywall"):
return "Paywall detected on page"
except Exception as e:
logger.debug("Auth-wall detection failed (non-fatal): %s", e)
return None
async def _extract_content(
self,
page: Page,
request: BrowseRequest,
) -> dict[str, Any]:
"""Extract content using Python readability on rendered HTML.
Falls back to JS innerText if readability is not available or fails.
Metadata extracted via a single JS evaluate with browser-side timeout.
Args:
page: Playwright page instance.
request: Browse request with parameters.
Returns:
dict: Extracted content.
"""
result: dict[str, Any] = {
"text": "",
"title": None,
"canonical_url": None,
"published_at": None,
"html": None,
"warnings": [],
}
# Title from Playwright API (governed by page default timeout)
result["title"] = await page.title()
# Get full rendered HTML
html = await page.content()
if request.include_html:
result["html"] = html
# Primary: Python readability-lxml on rendered HTML
# Lock required: lxml is not thread-safe, concurrent parsing crashes
if HAS_READABILITY and html:
try:
async with _readability_lock:
doc = ReadabilityDocument(html)
result["title"] = result["title"] or doc.title()
summary_html = doc.summary()
if HAS_LXML:
tree = lxml_parse(summary_html)
result["text"] = tree.text_content().strip()
else:
import re
text = re.sub(r"<[^>]+>", " ", summary_html)
result["text"] = re.sub(r"\s+", " ", text).strip()
except Exception as e:
logger.warning(
"Readability extraction failed for %s: %s", page.url, e
)
# Fallback: JS body text (browser-side 5s timeout)
if not result["text"]:
try:
text = await page.evaluate(_EXTRACT_TEXT_JS)
result["text"] = text.strip() if text else ""
except Exception as e:
logger.warning("JS text extraction failed: %s", e)
# Metadata: single JS evaluate, browser-side 3s timeout
try:
meta = await page.evaluate(_EXTRACT_METADATA_JS)
if isinstance(meta, dict):
result["canonical_url"] = meta.get("canonical")
result["published_at"] = meta.get("publishedAt")
except Exception as e:
logger.warning("Metadata extraction failed: %s", e)
return result
async def browse_to_page_content(
self,
url: str,
request: BrowseRequest | None = None,
) -> PageContent:
"""Convenience method to browse a single URL and return PageContent.
Args:
url: URL to browse.
request: Optional browse request parameters.
Returns:
PageContent: Extracted page content.
"""
if request is None:
request = BrowseRequest(urls=[url])
else:
request = BrowseRequest(
urls=[url],
wait_until=request.wait_until,
timeout_ms=request.timeout_ms,
screenshot=request.screenshot,
)
response = await self.browse(request)
if not response.pages:
raise WebError(f"Failed to browse {url}")
return response.pages[0].to_page_content()

123
ai_platform/modules/web/src/web/cli.py Normal file
View file

@ -0,0 +1,123 @@
"""CLI entry point for the Web server."""
import argparse
import signal
import sys
from typing import Any
import uvicorn
from web.config import SettingsCache
from web.logging import configure_logging, get_logger
class GracefulShutdown:
"""Handles graceful shutdown on signals."""
def __init__(self) -> None:
"""Initialize shutdown handler."""
self._server: uvicorn.Server | None = None
self._shutdown_requested = False
self._logger = get_logger("cli.shutdown")
def register_signals(self) -> None:
"""Register signal handlers for graceful shutdown."""
signal.signal(signal.SIGTERM, self._signal_handler)
signal.signal(signal.SIGINT, self._signal_handler)
def _signal_handler(self, signum: int, frame: Any) -> None:
"""Handle shutdown signals."""
if self._shutdown_requested:
self._logger.warning("Forced shutdown requested")
sys.exit(1)
self._shutdown_requested = True
signal_name = signal.Signals(signum).name
self._logger.info("Received %s, initiating graceful shutdown...", signal_name)
if self._server:
self._server.should_exit = True
def set_server(self, server: uvicorn.Server) -> None:
"""Set the uvicorn server for shutdown control."""
self._server = server
def main() -> None:
"""Run the Web server with graceful shutdown support."""
parser = argparse.ArgumentParser(
description="Web API Server",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--host",
type=str,
default=None,
help="Host to bind to (default: from WEB_HOST env or 0.0.0.0)",
)
parser.add_argument(
"--port",
type=int,
default=None,
help="Port to bind to (default: from WEB_PORT env or 8110)",
)
parser.add_argument(
"--workers",
type=int,
default=1,
help="Number of worker processes",
)
parser.add_argument(
"--reload",
action="store_true",
help="Enable auto-reload for development",
)
parser.add_argument(
"--graceful-timeout",
type=int,
default=30,
help="Timeout in seconds for graceful shutdown",
)
args = parser.parse_args()
settings = SettingsCache.get()
configure_logging(settings.log_level, settings.log_json)
logger = get_logger("cli")
host = args.host or settings.host
port = args.port or settings.port
logger.info("Starting Web API Server on %s:%d", host, port)
if args.reload:
logger.info("Development mode - auto-reload enabled")
uvicorn.run(
"web.api.app:create_app",
factory=True,
host=host,
port=port,
workers=1,
reload=True,
)
else:
shutdown_handler = GracefulShutdown()
shutdown_handler.register_signals()
config = uvicorn.Config(
"web.api.app:create_app",
factory=True,
host=host,
port=port,
workers=args.workers,
timeout_graceful_shutdown=args.graceful_timeout,
)
server = uvicorn.Server(config)
shutdown_handler.set_server(server)
server.run()
if __name__ == "__main__":
main()

562
ai_platform/modules/web/src/web/config.py Normal file
View file

@ -0,0 +1,562 @@
"""Unified configuration for Web module."""
import threading
from typing import Literal
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class WebSettings(BaseSettings):
"""Unified Web module configuration.
All settings can be configured via environment variables with the WEB_ prefix.
Example:
WEB_SEARXNG_BASE_URL=http://localhost:55100
WEB_LLM_BASE_URL=http://localhost:14011
WEB_PORT=51100
"""
model_config = SettingsConfigDict(
env_prefix="WEB_",
env_file=".env",
env_file_encoding="utf-8",
extra="forbid",
env_ignore_empty=True,
)
# ==========================================================================
# REQUIRED - Search (SearXNG) - Primary
# ==========================================================================
searxng_base_url: str = Field(
description="SearXNG base URL (REQUIRED). E.g., http://localhost:55100",
)
# ==========================================================================
# OPTIONAL - Additional Search Providers
# ==========================================================================
serpapi_api_key: str | None = Field(
default=None,
description="SerpAPI key for Google results (optional, runs parallel to SearXNG)",
)
tavily_api_key: str | None = Field(
default=None,
description="Tavily API key for AI-optimized search (optional, runs parallel to SearXNG)",
)
exa_api_key: str | None = Field(
default=None,
description="Exa API key for neural search (optional, runs parallel to SearXNG)",
)
linkup_api_key: str | None = Field(
default=None,
description="LinkUp API key for web content search (optional, runs parallel to SearXNG)",
)
brave_api_key: str | None = Field(
default=None,
description="Brave Search API key (optional, premium tier)",
)
# ==========================================================================
# OPTIONAL - Tier 3 fallback: Cloak stealth-browser scraping service
# ==========================================================================
# Invoked only when SearXNG + paid rotation return fewer results than the
# threshold. Scrapes Google/Bing/DDG SERPs through CloakBrowser. Service
# lives in ai_platform/modules/cloak.
cloak_enabled: bool = Field(
default=False,
description="Enable cloak tier-3 fallback. Default OFF.",
)
cloak_url: str = Field(
default="http://didiAI-cloak:8770",
description="Cloak service base URL inside the cluster.",
)
cloak_fallback_threshold: int = Field(
default=5,
description="Invoke cloak when dedup'd primary results are below this count.",
)
cloak_timeout_sec: int = Field(
default=20,
description="Hard cap on the cloak call. Errors return empty (non-fatal).",
)
cloak_auth_token: str | None = Field(
default=None,
description="Optional bearer token if cloak service is configured with auth.",
)
# ==========================================================================
# REQUIRED - LLM Integration (for vision + evidence)
# ==========================================================================
llm_base_url: str = Field(
description="LLM Inference API base URL (for text model). REQUIRED.",
)
llm_api_key: str | None = Field(
default=None,
description="LLM API key (if auth enabled)",
)
vision_base_url: str | None = Field(
default=None,
description="Vision model base URL (if different from llm_base_url)",
)
# Vision model
vision_model: str = Field(
default="qwen-vl",
description="Default vision model for screenshot extraction",
)
# Text model (for evidence snippets)
text_model: str = Field(
default="qwen3-235b",
description="Default text model for snippet extraction",
)
# ==========================================================================
# OPTIONAL - External LLM Fallback
# ==========================================================================
openai_api_key: str | None = Field(
default=None,
description="OpenAI API key for vision fallback",
)
anthropic_api_key: str | None = Field(
default=None,
description="Anthropic API key for vision fallback",
)
openrouter_api_key: str | None = Field(
default=None,
description="OpenRouter API key for premium tier LLM",
)
openrouter_base_url: str = Field(
default="https://openrouter.ai/api/v1",
description="OpenRouter API base URL",
)
openrouter_model: str = Field(
default="google/gemini-2.5-flash-preview",
description="OpenRouter model for premium tier",
)
# ==========================================================================
# API Server Settings
# ==========================================================================
host: str = Field(
default="0.0.0.0",
description="Host to bind the server to",
)
port: int = Field(
default=51100,
description="Port to bind the server to (Dev API Gateway: 51100)",
)
external_url: str = Field(
description="External URL for OpenAPI spec (e.g., http://10.11.10.42:51100). REQUIRED.",
)
# ==========================================================================
# Search Defaults
# ==========================================================================
search_default_max_results: int = Field(
default=10,
ge=1,
le=100,
description="Default search results per query",
)
search_default_language: str = Field(
default="en",
description="Default search language",
)
search_default_country: str = Field(
default="US",
description="Default search country",
)
# ==========================================================================
# Fetch Settings
# ==========================================================================
fetch_timeout: float = Field(
default=30.0,
ge=1.0,
description="HTTP fetch timeout in seconds",
)
fetch_min_text_length: int = Field(
default=200,
ge=0,
description="Minimum text length before triggering fallback",
)
fetch_user_agent: str = Field(
default="Mozilla/5.0 (compatible; WebBot/1.0)",
description="User agent for HTTP requests",
)
# ==========================================================================
# Browse (Playwright) Settings
# ==========================================================================
browse_timeout: int = Field(
default=30000,
ge=5000,
le=120000,
description="Playwright page load timeout in milliseconds",
)
browse_viewport_width: int = Field(default=1280, ge=320, le=1920)
browse_viewport_height: int = Field(default=720, ge=240, le=1080)
browse_block_resources: bool = Field(
default=True,
description="Block images/media/fonts/stylesheets in Playwright to speed up browsing",
)
browse_extra_wait_ms: int = Field(
default=500,
ge=0,
le=5000,
description="Extra ms to wait after page load for JS-rendered content (SPAs)",
)
# ==========================================================================
# Vision Settings
# ==========================================================================
vision_max_tokens: int = Field(
default=2000,
ge=100,
le=8000,
description="Max tokens for vision extraction",
)
vision_screenshot_quality: int = Field(
default=80,
ge=10,
le=100,
description="Screenshot JPEG quality",
)
vision_max_concurrent: int = Field(
default=3,
ge=1,
le=10,
description="Max concurrent vision extractions",
)
# ==========================================================================
# Evidence Settings
# ==========================================================================
evidence_max_items: int = Field(
default=30,
ge=1,
le=100,
description="Max items in evidence pack",
)
evidence_max_snippet_length: int = Field(
default=500,
ge=100,
le=2000,
description="Max characters per snippet",
)
evidence_dedupe_threshold: float = Field(
default=0.9,
ge=0.0,
le=1.0,
description="Similarity threshold for deduplication",
)
evidence_max_concurrent: int = Field(
default=5,
ge=1,
le=20,
description="Max concurrent evidence item builds",
)
summary_max_length: int = Field(
default=800,
ge=200,
le=2000,
description="Max chars for LLM-generated summary",
)
llm_max_context_chars: int = Field(
default=8000,
ge=1000,
le=32000,
description="Max chars of page text sent to LLM per snippet extraction",
)
llm_batch_context_chars: int = Field(
default=3000,
ge=500,
le=16000,
description="Max chars per page in batched LLM calls",
)
# ==========================================================================
# Gather Pipeline Settings
# ==========================================================================
gather_max_vision_urls: int = Field(
default=5,
ge=1,
le=20,
description="Max URLs to send to vision fallback in gather pipeline",
)
gather_max_search_queries: int = Field(
default=3,
ge=1,
le=10,
description="Max search queries per gather request",
)
# ==========================================================================
# Context Detection (Stage 0)
# ==========================================================================
context_detection_timeout: float = Field(
default=10.0,
ge=1.0,
le=30.0,
description="Timeout for Stage 0 context detection in seconds",
)
context_detection_model: str = Field(
default="qwen3.5",
description="Model to use for context detection (should be fast)",
)
# ==========================================================================
# External LLM Model Names
# ==========================================================================
openai_model: str = Field(
default="gpt-4o",
description="OpenAI model for vision/evidence fallback",
)
anthropic_model: str = Field(
default="claude-3-5-sonnet-20241022",
description="Anthropic model for vision/evidence fallback",
)
# ==========================================================================
# Timeouts and Retries
# ==========================================================================
request_timeout: float = Field(
default=30.0,
ge=1.0,
description="Default request timeout in seconds",
)
connect_timeout: float = Field(
default=5.0,
ge=1.0,
description="Connection timeout in seconds",
)
max_retries: int = Field(
default=3,
ge=0,
description="Maximum retries for transient failures",
)
retry_min_wait: float = Field(
default=1.0,
ge=0.1,
description="Minimum wait between retries",
)
retry_max_wait: float = Field(
default=10.0,
ge=1.0,
description="Maximum wait between retries",
)
# ==========================================================================
# HTTP Connection Pool Sizing
# ==========================================================================
searxng_rate_min_interval: float = Field(
default=1.0,
ge=0.1,
le=10.0,
description="Minimum seconds between SearXNG requests to avoid upstream rate limits",
)
search_pool_max_connections: int = Field(
default=20,
ge=1,
description="Max HTTP connections for search client (talks to one API)",
)
search_pool_max_keepalive: int = Field(
default=5,
ge=0,
description="Max keepalive connections for search client",
)
fetch_pool_max_connections: int = Field(
default=50,
ge=1,
description="Max HTTP connections for fetch client (talks to many websites)",
)
fetch_pool_max_keepalive: int = Field(
default=10,
ge=0,
description="Max keepalive connections for fetch client",
)
llm_pool_max_connections: int = Field(
default=10,
ge=1,
description="Max HTTP connections for LLM clients (vision + evidence packer)",
)
llm_pool_max_keepalive: int = Field(
default=3,
ge=0,
description="Max keepalive connections for LLM clients",
)
# ==========================================================================
# Observability
# ==========================================================================
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(
default="INFO",
description="Logging level",
)
log_json: bool = Field(
default=False,
description="Output logs in JSON format",
)
# ==========================================================================
# Rate Limiting and Concurrency
# ==========================================================================
rate_limit_rps: float = Field(
default=10.0,
ge=0.1,
description="Rate limit: requests per second",
)
rate_limit_burst: int = Field(
default=20,
ge=1,
description="Rate limit: burst capacity",
)
max_concurrent_requests: int = Field(
default=10,
ge=1,
description="Maximum concurrent requests",
)
# ==========================================================================
# Authentication
# ==========================================================================
api_tokens: frozenset[str] | None = Field(
default=None,
description="API tokens for Bearer auth (comma-separated)",
)
# ==========================================================================
# Dashboard event sink (optional)
# ==========================================================================
dashboard_url: str | None = Field(
default=None,
description="Dashboard base URL for event ingestion (e.g., http://didiAI-dashboard:51300)",
)
dashboard_token: str | None = Field(
default=None,
description="Bearer token for dashboard /api/ingest/event",
)
# ==========================================================================
# Brain cache integration (premium tier only)
# ==========================================================================
brain_url: str | None = Field(
default=None,
description="didi-brain base URL (e.g., http://didibrain-api:8090). Unset disables brain integration.",
)
brain_cache_read_enabled: bool = Field(
default=True,
description="If brain_url is set, try brain cache before calling premium providers.",
)
brain_ingest_enabled: bool = Field(
default=True,
description="If brain_url is set, push successful premium gathers back to brain.",
)
brain_ingest_min_evidence: int = Field(
default=3,
ge=1,
description="Minimum evidence items for a premium gather to qualify for ingest.",
)
brain_ingest_min_credibility: float = Field(
default=0.7,
ge=0.0,
le=1.0,
description="At least one evidence item must have credibility ≥ this to qualify for ingest.",
)
brain_ingest_min_execution_ms: float = Field(
default=2000.0,
ge=0.0,
description="Skip ingest if gather completed faster than this (likely a nothing-found fast path).",
)
brain_hit_min_evidence: int = Field(
default=3,
ge=1,
description="Minimum evidence items a PARTIAL brain hit must contain to be served instead of calling premium.",
)
brain_hit_min_relevance: float = Field(
default=0.7,
ge=0.0,
le=1.0,
description="For PARTIAL brain hits, this many items must exceed this relevance score.",
)
brain_gather_timeout: float = Field(
default=8.0,
ge=1.0,
description="Timeout for brain cache-read calls.",
)
brain_ingest_timeout: float = Field(
default=15.0,
ge=1.0,
description="Timeout for brain ingest calls.",
)
@field_validator("api_tokens", mode="before")
@classmethod
def parse_api_tokens(cls, v: str | list[str] | None) -> frozenset[str] | None:
"""Parse comma-separated tokens into a frozenset."""
if v is None or v == "":
return None
if isinstance(v, str):
tokens = [t.strip() for t in v.split(",") if t.strip()]
return frozenset(tokens) if tokens else None
return frozenset(v) if v else None
@property
def auth_enabled(self) -> bool:
"""Check if authentication is enabled."""
return bool(self.api_tokens)
class SettingsCache:
"""Thread-safe settings cache."""
_instance: WebSettings | None = None
_lock: threading.Lock = threading.Lock()
@classmethod
def get(cls) -> WebSettings:
"""Get or create the settings instance."""
with cls._lock:
if cls._instance is None:
cls._instance = WebSettings()
return cls._instance
@classmethod
def clear(cls) -> None:
"""Clear the cached settings instance."""
with cls._lock:
cls._instance = None
@classmethod
def set(cls, settings: WebSettings) -> None:
"""Set a specific settings instance."""
with cls._lock:
cls._instance = settings
def get_settings() -> WebSettings:
"""Get cached settings instance."""
return SettingsCache.get()

0
ai_platform/modules/web/src/web/context/__init__.py Normal file
View file

251
ai_platform/modules/web/src/web/context/detector.py Normal file
View file

@ -0,0 +1,251 @@
"""Stage 0: Context detection from claim text using local LLM.
Analyzes the claim before search to extract entities, detect
the primary country, and generate optimized search queries.
"""
import json
import re
import time
import httpx
from web.config import WebSettings
from web.llm.provider import LLMProviderChain
from web.logging import get_logger
from web.schemas.context import EntitySet, SearchContext
logger = get_logger("context.detector")
class ContextDetector:
"""Detects search context (country, entities, language) from claim text.
Uses the local LLM to analyze the claim before search begins.
Falls back to empty context on any failure so the pipeline
continues with default (current) behavior.
"""
_LLM_PROBE_CACHE_SECS = 60.0
def __init__(self, settings: WebSettings, llm_provider: str = "local") -> None:
"""Initialize the detector.
Args:
settings: Application settings.
llm_provider: LLM provider to use ("local" or "openrouter").
"""
self.settings = settings
self.llm_provider = llm_provider
self._http_client: httpx.AsyncClient | None = None
self._llm_available: bool | None = None
self._llm_check_time: float = 0.0
self._llm_chain = LLMProviderChain(
settings, self._get_http_client, domain="context"
)
async def _get_http_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client with tight timeouts."""
if self._http_client is None or self._http_client.is_closed:
self._http_client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=2.0,
read=15.0,
write=5.0,
pool=15.0,
),
limits=httpx.Limits(
max_connections=self.settings.llm_pool_max_connections,
max_keepalive_connections=self.settings.llm_pool_max_keepalive,
),
)
return self._http_client
async def _is_llm_available(self) -> bool:
"""Check if the local LLM is reachable (cached with negative TTL).
Returns:
True if the LLM responded within the timeout window.
"""
now = time.perf_counter()
if self._llm_available is not None:
if self._llm_available:
return True
if now - self._llm_check_time < self._LLM_PROBE_CACHE_SECS:
return False
try:
headers: dict[str, str] = {}
if self.settings.llm_api_key:
headers["Authorization"] = f"Bearer {self.settings.llm_api_key}"
async with httpx.AsyncClient(
timeout=httpx.Timeout(connect=2.0, read=5.0, write=5.0, pool=5.0),
) as probe:
resp = await probe.get(
f"{self.settings.llm_base_url}/v1/models",
headers=headers,
)
resp.raise_for_status()
self._llm_available = True
except Exception:
self._llm_available = False
self._llm_check_time = now
logger.info(
"Local LLM at %s unreachable — skipping context detection",
self.settings.llm_base_url,
)
return self._llm_available
async def detect(self, claim: str) -> SearchContext:
"""Analyze claim text and return search context.
Args:
claim: The claim text to analyze.
Returns:
SearchContext with detected country, entities, language.
Returns empty SearchContext on any failure.
"""
if not await self._is_llm_available():
return SearchContext()
try:
return await self._detect_with_llm(claim)
except Exception as e:
logger.warning("Context detection failed, using defaults: %s", e)
return SearchContext()
async def _detect_with_llm(self, claim: str) -> SearchContext:
"""Call LLM for context extraction.
Args:
claim: The claim text.
Returns:
Parsed SearchContext.
"""
prompt = _build_prompt(claim)
messages = [{"role": "user", "content": prompt}]
text, _tokens, _provider, _model = await self._llm_chain.call_chat(
messages=messages,
provider=self.llm_provider,
max_tokens=2000,
temperature=0.1,
model=self.settings.context_detection_model,
)
text = _strip_thinking_tags(text)
return _parse_response(text)
async def close(self) -> None:
"""Close resources."""
if self._http_client is not None and not self._http_client.is_closed:
await self._http_client.aclose()
self._http_client = None
def _build_prompt(claim: str) -> str:
"""Build the LLM prompt for context extraction.
Args:
claim: The claim text.
Returns:
Formatted prompt string.
"""
return (
"Analyze this claim and extract structured context. "
"Respond ONLY with a JSON object, no other text.\n\n"
f"CLAIM: {claim}\n\n"
"Return this exact JSON structure:\n"
"{\n"
' "primary_country": "<ISO 3166-1 alpha-2 code or null>",\n'
' "secondary_countries": ["<codes>"],\n'
' "entities": {\n'
' "persons": ["<full names>"],\n'
' "institutions": ["<organizations, parties, companies>"],\n'
' "locations": ["<cities, regions, countries>"]\n'
" },\n"
' "detected_language": "<ISO 639-1 code>",\n'
' "search_queries": ["<2-3 optimized fact-check queries>"]\n'
"}\n\n"
"Rules:\n"
"- primary_country: the country this claim is PRIMARILY ABOUT based on its content, NOT the language it is written in\n"
"- The language of the text must NEVER influence primary_country\n"
"- detected_language: language the claim is written in\n"
"- search_queries: 2-3 queries that would help verify or debunk this claim. "
"ALWAYS write search queries in ENGLISH regardless of the claim language.\n"
"- Output ONLY valid JSON. No markdown fences. No explanation.\n"
"- Do NOT use <think> tags."
)
def _parse_response(text: str) -> SearchContext:
"""Parse LLM JSON response into SearchContext.
Args:
text: Raw LLM output (should be JSON).
Returns:
Parsed SearchContext.
Raises:
json.JSONDecodeError: If JSON parsing fails.
KeyError: If required fields are missing.
"""
text = text.strip()
# Handle markdown code fences
if text.startswith("```"):
text = re.sub(r"```(?:json)?\n?", "", text)
text = text.rstrip("`").strip()
# Try to find JSON object in response
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
text = text[start : end + 1]
data = json.loads(text)
entities_data = data.get("entities", {})
entities = EntitySet(
persons=entities_data.get("persons", []),
institutions=entities_data.get("institutions", []),
locations=entities_data.get("locations", []),
)
primary = data.get("primary_country")
if isinstance(primary, str):
primary = primary.upper() if primary else None
secondary = data.get("secondary_countries", [])
if isinstance(secondary, list):
secondary = [c.upper() for c in secondary if isinstance(c, str) and c]
return SearchContext(
primary_country=primary,
secondary_countries=secondary,
entities=entities,
detected_language=data.get("detected_language", "en"),
search_queries=data.get("search_queries", []),
)
def _strip_thinking_tags(text: str) -> str:
"""Strip <think>...</think> tags from LLM output.
Qwen3 models emit reasoning in <think> blocks by default.
Args:
text: Raw LLM output.
Returns:
Text with thinking tags removed.
"""
cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE)
cleaned = re.sub(r"<think>.*$", "", cleaned, flags=re.DOTALL | re.IGNORECASE)
return cleaned.strip()

92
ai_platform/modules/web/src/web/context/sources.py Normal file
View file

@ -0,0 +1,92 @@
"""Country-specific source mappings for search prioritization.
Used by the multi-round search strategy to prioritize local and
fact-check sources based on the detected country.
"""
from dataclasses import dataclass, field
@dataclass(frozen=True)
class CountrySources:
"""Source domains for a specific country."""
official: list[str] = field(default_factory=list)
media: list[str] = field(default_factory=list)
COUNTRY_SOURCES: dict[str, CountrySources] = {
"RO": CountrySources(
official=[
"gov.ro",
"cdep.ro",
"senat.ro",
"presidency.ro",
"mae.ro",
"edu.ro",
"ms.ro",
"insse.ro",
],
media=[
"digi24.ro",
"hotnews.ro",
"g4media.ro",
"mediafax.ro",
"agerpres.ro",
"adevarul.ro",
"romania-insider.com",
"stiripesurse.ro",
],
),
"US": CountrySources(
official=[
"whitehouse.gov",
"congress.gov",
"senate.gov",
"house.gov",
"state.gov",
"cdc.gov",
"nih.gov",
"fbi.gov",
],
media=[
"nytimes.com",
"washingtonpost.com",
"apnews.com",
"reuters.com",
"npr.org",
"pbs.org",
"politico.com",
"thehill.com",
],
),
}
FACTCHECK_SOURCES: list[str] = [
"reuters.com",
"snopes.com",
"politifact.com",
"factcheck.org",
"fullfact.org",
"apnews.com",
"bbc.com",
"afp.com",
"leadstories.com",
"veridica.ro",
"factual.ro",
]
def get_country_allowlist(country_code: str) -> list[str]:
"""Get combined official + media domains for a country.
Args:
country_code: ISO 3166-1 alpha-2 code (uppercase).
Returns:
List of domain strings, or empty list if country unknown.
"""
sources = COUNTRY_SOURCES.get(country_code.upper())
if sources is None:
return []
return sources.official + sources.media

1
ai_platform/modules/web/src/web/events/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Dashboard event sink — sends request metadata to the dashboard module."""

112
ai_platform/modules/web/src/web/events/sink.py Normal file
View file

@ -0,0 +1,112 @@
"""Async event sink that forwards request events to the dashboard module.
Fire-and-forget — dashboard outages must never affect web-api latency
or availability. Events are dropped silently on failure.
"""
import asyncio
import contextlib
from typing import Any
import httpx
from web.logging import get_logger
logger = get_logger("events.sink")
class DashboardEventSink:
"""POSTs request events to the dashboard ingest endpoint, fire-and-forget.
The sink runs all writes through a bounded queue processed by a single
worker task. Overflow drops oldest. Failures are logged but swallowed.
"""
def __init__(
self,
dashboard_url: str | None,
token: str | None = None,
queue_size: int = 1000,
request_timeout: float = 5.0,
) -> None:
self.dashboard_url = dashboard_url.rstrip("/") if dashboard_url else None
self.token = token
self._queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=queue_size)
self._client: httpx.AsyncClient | None = None
self._worker_task: asyncio.Task[None] | None = None
self._request_timeout = request_timeout
self._enabled = bool(dashboard_url)
@property
def enabled(self) -> bool:
return self._enabled
async def start(self) -> None:
"""Start the background worker."""
if not self._enabled:
logger.info("DashboardEventSink disabled (no dashboard URL)")
return
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=2.0, read=self._request_timeout, write=2.0, pool=5.0
),
limits=httpx.Limits(max_connections=5, max_keepalive_connections=2),
)
self._worker_task = asyncio.create_task(self._worker())
logger.info("DashboardEventSink started → %s", self.dashboard_url)
async def stop(self) -> None:
"""Stop worker and drain remaining events (best-effort)."""
if self._worker_task is not None:
self._worker_task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await self._worker_task
self._worker_task = None
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
def emit(self, event: dict[str, Any]) -> None:
"""Enqueue an event for async send. Never raises.
Args:
event: Event dict matching IngestEvent schema.
"""
if not self._enabled:
return
try:
self._queue.put_nowait(event)
except asyncio.QueueFull:
# Drop the oldest event to make room for the new one
try:
_ = self._queue.get_nowait()
self._queue.put_nowait(event)
except Exception:
pass
async def _worker(self) -> None:
"""Background loop that drains the queue and POSTs events."""
while True:
try:
event = await self._queue.get()
except asyncio.CancelledError:
raise
try:
await self._send(event)
except Exception as e:
logger.debug("Event dropped (%s): %s", type(e).__name__, e)
finally:
self._queue.task_done()
async def _send(self, event: dict[str, Any]) -> None:
if self._client is None or self.dashboard_url is None:
return
headers = {"Content-Type": "application/json"}
if self.token:
headers["Authorization"] = f"Bearer {self.token}"
resp = await self._client.post(
f"{self.dashboard_url}/api/ingest/event",
json=event,
headers=headers,
)
resp.raise_for_status()

5
ai_platform/modules/web/src/web/evidence/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Evidence submodule - Evidence pack builder with deduplication."""
from web.evidence.packer import EvidencePacker
__all__ = ["EvidencePacker"]

1129
ai_platform/modules/web/src/web/evidence/packer.py Normal file
View file

File diff suppressed because it is too large Load diff

89
ai_platform/modules/web/src/web/exceptions.py Normal file
View file

@ -0,0 +1,89 @@
"""Custom exceptions for Web module."""
class WebError(Exception):
"""Base exception for Web module errors."""
class AuthenticationError(WebError):
"""Raised when authentication fails."""
def __init__(self, message: str = "Authentication required") -> None:
super().__init__(message)
class ProviderError(WebError):
"""Raised when a search provider returns an error."""
def __init__(self, provider: str, message: str) -> None:
self.provider = provider
super().__init__(f"{provider}: {message}")
class ProviderNotAvailableError(WebError):
"""Raised when a search provider is not available."""
def __init__(self, provider: str, reason: str | None = None) -> None:
self.provider = provider
self.reason = reason
message = f"Provider '{provider}' is not available"
if reason:
message += f": {reason}"
super().__init__(message)
class SearchError(WebError):
"""Raised when a search request fails."""
def __init__(
self,
query: str,
reason: str,
provider: str | None = None,
) -> None:
self.query = query
self.reason = reason
self.provider = provider
message = f"Search failed for '{query}': {reason}"
if provider:
message += f" (provider: {provider})"
super().__init__(message)
class RateLimitError(WebError):
"""Raised when rate limit is exceeded."""
def __init__(
self,
message: str,
retry_after: float | None = None,
) -> None:
self.retry_after = retry_after
full_message = message
if retry_after:
full_message += f" (retry after {retry_after}s)"
super().__init__(full_message)
class WebTimeoutError(WebError):
"""Raised when a request times out."""
def __init__(
self,
message: str,
timeout: float | None = None,
) -> None:
self.timeout = timeout
super().__init__(message)
class WebConnectionError(WebError):
"""Raised when connection to provider fails."""
def __init__(self, provider: str, reason: str | None = None) -> None:
self.provider = provider
self.reason = reason
message = f"Connection to provider '{provider}' failed"
if reason:
message += f": {reason}"
super().__init__(message)

5
ai_platform/modules/web/src/web/fetch/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Fetch submodule - HTTP fetch with text extraction."""
from web.fetch.client import FetchClient
__all__ = ["FetchClient"]

460
ai_platform/modules/web/src/web/fetch/client.py Normal file
View file

@ -0,0 +1,460 @@
"""HTTP fetch client with readability extraction."""
import asyncio
import hashlib
import re
import threading
import time
from datetime import datetime, timezone
from typing import Any
import httpx
from web.config import WebSettings
from web.exceptions import (
WebConnectionError,
WebError,
WebTimeoutError,
)
from web.logging import get_logger
from web.schemas.common import FailedUrl, PageContent
from web.schemas.fetch import FetchPageResult, FetchRequest, FetchResponse
from web.validation import validate_urls_async
logger = get_logger("fetch.client")
# Optional imports for content extraction
try:
from readability import Document
HAS_READABILITY = True
except ImportError:
HAS_READABILITY = False
logger.warning("readability-lxml not installed, text extraction will be limited")
try:
from bs4 import BeautifulSoup
HAS_BS4 = True
except ImportError:
HAS_BS4 = False
logger.warning("beautifulsoup4 not installed, HTML parsing will be limited")
# Pre-compiled regex for noscript JS detection
_NOSCRIPT_JS_RE = re.compile(
r"<noscript[^>]*>([^<]*javascript[^<]*)</noscript>",
re.IGNORECASE | re.DOTALL,
)
# Thread lock for readability-lxml/lxml parsing (not thread-safe,
# concurrent parsing causes "double free or corruption" crashes)
_lxml_lock = threading.Lock()
class FetchClient:
"""Client for HTTP content fetching with readability extraction."""
MAX_PAGE_SIZE = 5 * 1024 * 1024 # 5MB
def __init__(self, settings: WebSettings) -> None:
"""Initialize the client.
Args:
settings: Application settings.
"""
self.settings = settings
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client."""
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
http2=True,
timeout=httpx.Timeout(
connect=self.settings.connect_timeout,
read=self.settings.fetch_timeout,
write=self.settings.fetch_timeout,
pool=self.settings.fetch_timeout,
),
limits=httpx.Limits(
max_connections=self.settings.fetch_pool_max_connections,
max_keepalive_connections=self.settings.fetch_pool_max_keepalive,
),
headers={
"User-Agent": self.settings.fetch_user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate",
},
follow_redirects=True,
max_redirects=5,
)
return self._client
async def __aenter__(self) -> "FetchClient":
return self
async def __aexit__(self, *exc: object) -> None:
await self.close()
async def close(self) -> None:
"""Close the HTTP client."""
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def fetch(
self,
request: FetchRequest,
request_id: str | None = None,
) -> FetchResponse:
"""Fetch content from URLs.
Args:
request: Fetch request with URLs and parameters.
request_id: Optional request ID for tracing.
Returns:
FetchResponse: Fetched page content.
"""
start_time = time.perf_counter()
pages: list[FetchPageResult] = []
failed: list[FailedUrl] = []
# Async SSRF DNS check before fetching (skipped when orchestrator already validated)
if not request.skip_validation:
request.urls, dns_failed = await validate_urls_async(request.urls)
failed.extend(dns_failed)
# Process URLs with concurrency limit
semaphore = asyncio.Semaphore(request.parallel_fetches)
async def fetch_with_semaphore(url: str) -> FetchPageResult | None:
async with semaphore:
try:
return await self._fetch_single(url, request)
except Exception as e:
logger.error("Failed to fetch %s: %s", url, e)
failed.append(FailedUrl(url=url, error=str(e)))
return None
# Fetch all URLs concurrently
tasks = [fetch_with_semaphore(url) for url in request.urls]
results = await asyncio.gather(*tasks)
for result in results:
if result is not None:
pages.append(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return FetchResponse(
request_id=request_id or "-",
pages=pages,
total_fetched=len(pages),
total_failed=len(failed),
failed_urls=failed,
execution_time_ms=round(execution_time_ms, 2),
)
async def _fetch_single(
self,
url: str,
request: FetchRequest,
) -> FetchPageResult:
"""Fetch content from a single URL.
Args:
url: URL to fetch.
request: Fetch request with parameters.
Returns:
FetchPageResult: Fetched page content.
Raises:
WebError: If fetch fails.
"""
start_time = time.perf_counter()
client = await self._get_client()
try:
async with client.stream("GET", url) as response:
response.raise_for_status()
status_code = response.status_code
content_type = response.headers.get("content-type", "")
chunks: list[str] = []
total = 0
async for chunk in response.aiter_text(chunk_size=8192):
chunks.append(chunk)
total += len(chunk)
if total > self.MAX_PAGE_SIZE:
break
html = "".join(chunks)
except httpx.TimeoutException as e:
raise WebTimeoutError(
f"Timeout fetching {url}",
timeout=self.settings.fetch_timeout,
) from e
except httpx.ConnectError as e:
raise WebConnectionError(url, str(e)) from e
except httpx.HTTPStatusError as e:
raise WebError(f"HTTP {e.response.status_code} for {url}") from e
except Exception as e:
raise WebError(f"Failed to fetch {url}: {e}") from e
# Extract content + JS detection (run in thread to avoid blocking event loop)
loop = asyncio.get_running_loop()
extracted = await loop.run_in_executor(
None, self._extract_and_analyze, html, url, request
)
extraction_time_ms = (time.perf_counter() - start_time) * 1000
retrieved_at = datetime.now(timezone.utc).isoformat()
js_detected = extracted.pop("js_detected", False)
needs_fallback = (
request.auto_fallback and len(extracted["text"]) < request.min_text_length
)
if js_detected and request.auto_fallback:
needs_fallback = True
warnings = []
if needs_fallback:
warnings.append(
f"Content may be incomplete (length: {len(extracted['text'])})"
)
if js_detected:
warnings.append("JavaScript-heavy page detected")
return FetchPageResult(
url=url,
canonical_url=extracted.get("canonical_url"),
title=extracted.get("title"),
text=extracted["text"],
text_hash=hashlib.sha256(extracted["text"].encode()).hexdigest(),
html=html if request.include_html else None,
extraction_method="http",
fallback_chain=[],
published_at=extracted.get("published_at"),
retrieved_at=retrieved_at,
extraction_time_ms=round(extraction_time_ms, 2),
warnings=warnings,
needs_fallback=needs_fallback,
status_code=status_code,
content_type=content_type,
)
def _extract_content(
self,
html: str,
url: str,
request: FetchRequest,
) -> dict[str, Any]:
"""Extract readable content from HTML.
Args:
html: Raw HTML content.
url: Source URL (for relative link resolution).
request: Fetch request with parameters.
Returns:
dict: Extracted content with title, text, etc.
"""
result: dict[str, Any] = {
"text": "",
"title": None,
"canonical_url": None,
"published_at": None,
}
# Try readability first (best quality)
# Lock required: lxml is not thread-safe, concurrent run_in_executor
# calls cause "double free or corruption" crashes
if HAS_READABILITY:
try:
with _lxml_lock:
doc = Document(html)
result["title"] = doc.title()
# Get summary (cleaned HTML)
summary_html = doc.summary()
# Use lxml directly (readability already depends on it)
from lxml.html import fromstring as lxml_parse
tree = lxml_parse(summary_html)
result["text"] = tree.text_content().strip()
except Exception as e:
logger.warning("Readability extraction failed for %s: %s", url, e)
# Fallback to BS4 if readability returned nothing or too little.
# Use a higher threshold (500) than fetch_min_text_length to catch cases
# where readability extracts only a fragment of the article.
_bs4_threshold = max(self.settings.fetch_min_text_length, 500)
if len(result["text"]) < _bs4_threshold and HAS_BS4:
try:
soup = BeautifulSoup(html, "lxml")
# Remove script, style, nav, footer, etc.
for tag in soup(
["script", "style", "nav", "footer", "header", "aside"]
):
tag.decompose()
# Extract title
if soup.title:
result["title"] = soup.title.get_text(strip=True)
# Try to find main content - keep the longer extraction
main = soup.find("main") or soup.find("article") or soup.find("body")
if main:
bs4_text = main.get_text(separator="\n", strip=True)
if len(bs4_text) > len(result["text"]):
result["text"] = bs4_text
# Extract canonical URL
canonical = soup.find("link", rel="canonical")
if canonical and canonical.get("href"):
result["canonical_url"] = canonical["href"]
# Try to find publication date
for selector in ["time", "[datetime]", ".date", ".published"]:
elem = soup.select_one(selector)
if elem:
dt = elem.get("datetime") or elem.get_text(strip=True)
if dt:
result["published_at"] = dt
break
except Exception as e:
logger.warning("BS4 extraction failed for %s: %s", url, e)
# Last resort: basic regex extraction
if not result["text"]:
# Remove script and style tags
text = re.sub(
r"<(script|style)[^>]*>.*?</\1>",
"",
html,
flags=re.DOTALL | re.IGNORECASE,
)
# Remove HTML tags
text = re.sub(r"<[^>]+>", " ", text)
# Clean up whitespace
text = re.sub(r"\s+", " ", text).strip()
result["text"] = text
# Try to extract title
title_match = re.search(r"<title[^>]*>([^<]+)</title>", html, re.IGNORECASE)
if title_match:
result["title"] = title_match.group(1).strip()
return result
def _extract_and_analyze(
self,
html: str,
url: str,
request: FetchRequest,
) -> dict[str, Any]:
"""Extract content and detect JS requirement in a single executor call.
Args:
html: Raw HTML content.
url: Source URL.
request: Fetch request with parameters.
Returns:
dict: Extracted content with an added 'js_detected' key.
"""
result = self._extract_content(html, url, request)
result["js_detected"] = self._detect_javascript_required(html, result["text"])
return result
def _detect_javascript_required(self, html: str, extracted_text: str) -> bool:
"""Detect if page requires JavaScript for content.
Only flags JS-required when extracted text is also short, to avoid
false positives on SSR frameworks (Next.js, Nuxt/Vue SSR) that
render full HTML server-side.
Args:
html: Raw HTML content.
extracted_text: Text extracted from page.
Returns:
bool: True if JavaScript is likely required.
"""
# Check for "Please enable JavaScript" messages (always a strong signal)
js_required_phrases = [
"enable javascript",
"javascript required",
"javascript is required",
"please turn on javascript",
"this app requires javascript",
]
text_lower = extracted_text.lower()
for phrase in js_required_phrases:
if phrase in text_lower:
return True
# Check for noscript warning
html_lower = html.lower()
if (
"<noscript>" in html_lower
and "javascript" in html_lower
and _NOSCRIPT_JS_RE.search(html)
):
return True
# SPA indicators only matter when extracted text is short
# (SSR frameworks like Next.js and Nuxt render full HTML)
if len(extracted_text) >= self.settings.fetch_min_text_length:
return False
spa_indicators = [
"react-root",
"ng-app",
"ng-view",
"ember-view",
"data-reactroot",
]
return any(indicator in html_lower for indicator in spa_indicators)
async def fetch_to_page_content(
self,
url: str,
request: FetchRequest | None = None,
) -> PageContent:
"""Convenience method to fetch a single URL and return PageContent.
Args:
url: URL to fetch.
request: Optional fetch request parameters.
Returns:
PageContent: Extracted page content.
"""
if request is None:
request = FetchRequest(urls=[url])
else:
request = FetchRequest(
urls=[url],
timeout_seconds=request.timeout_seconds,
min_text_length=request.min_text_length,
auto_fallback=request.auto_fallback,
include_html=request.include_html,
)
response = await self.fetch(request)
if not response.pages:
raise WebError(f"Failed to fetch {url}")
return response.pages[0].to_page_content()

5
ai_platform/modules/web/src/web/llm/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""LLM provider utilities."""
from web.llm.provider import LLMProviderChain
__all__ = ["LLMProviderChain"]

362
ai_platform/modules/web/src/web/llm/provider.py Normal file
View file

@ -0,0 +1,362 @@
"""Shared LLM provider chain with auto-fallback."""
from typing import Any
import httpx
from web.config import WebSettings
from web.exceptions import ProviderError
from web.logging import get_logger
logger = get_logger("llm.provider")
class LLMProviderChain:
"""Routes LLM calls through local -> OpenAI -> Anthropic fallback.
Consolidates the duplicated provider selection logic from VisionClient
and EvidencePacker into a single reusable class.
"""
def __init__(
self,
settings: WebSettings,
get_http_client: Any,
domain: str = "llm",
local_base_url: str | None = None,
runtime_config: object | None = None,
) -> None:
"""Initialize the provider chain.
Args:
settings: Application settings.
get_http_client: Async callable that returns an httpx.AsyncClient.
domain: Label for error messages (e.g. "vision", "text_llm").
local_base_url: Override base URL for local LLM calls.
Defaults to settings.llm_base_url.
runtime_config: Optional RuntimeConfigClient for model overrides.
"""
self.settings = settings
self._get_http_client = get_http_client
self._domain = domain
self._local_base_url = local_base_url
self._runtime_config = runtime_config
async def call_chat(
self,
messages: list[dict[str, Any]],
provider: str,
max_tokens: int,
temperature: float = 0.1,
model: str | None = None,
) -> tuple[str, int, str, str]:
"""Send a chat completion request with provider fallback.
Args:
messages: Chat messages in OpenAI format.
provider: Provider selection ("auto", "local", "openai", "anthropic").
max_tokens: Max response tokens.
temperature: Sampling temperature.
model: Model override for local provider (defaults from settings).
Returns:
Tuple of (response_text, tokens_used, provider_used, model_used).
Raises:
ProviderError: If all providers fail or none are available.
"""
if provider == "auto":
providers = ["local", "openai", "anthropic"]
elif provider == "openrouter":
providers = ["openrouter"]
else:
providers = [provider]
last_error = None
for p in providers:
try:
if p == "local":
return await self._call_local(
messages, max_tokens, temperature, model
)
elif p == "openrouter":
if not self.settings.openrouter_api_key:
continue
return await self._call_openrouter(
messages, max_tokens, temperature
)
elif p == "openai":
if not self.settings.openai_api_key:
continue
return await self._call_openai(messages, max_tokens, temperature)
elif p == "anthropic":
if not self.settings.anthropic_api_key:
continue
return await self._call_anthropic(messages, max_tokens, temperature)
except Exception as e:
logger.warning("Provider %s failed: %s", p, e)
last_error = e
continue
if last_error:
raise ProviderError(self._domain, str(last_error))
raise ProviderError(self._domain, f"No {self._domain} provider available")
async def _call_local(
self,
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
model: str | None,
) -> tuple[str, int, str, str]:
"""Call local vLLM-compatible endpoint.
Args:
messages: Chat messages.
max_tokens: Max response tokens.
temperature: Sampling temperature.
model: Model override.
Returns:
Tuple of (text, tokens, "local", model_name).
"""
client = await self._get_http_client()
model_name = model or self.settings.text_model
headers: dict[str, str] = {"Content-Type": "application/json"}
if self.settings.llm_api_key:
headers["Authorization"] = f"Bearer {self.settings.llm_api_key}"
base_url = self._local_base_url or self.settings.llm_base_url
try:
payload: dict[str, Any] = {
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
"chat_template_kwargs": {"enable_thinking": False},
}
response = await client.post(
f"{base_url}/v1/chat/completions",
headers=headers,
json=payload,
)
response.raise_for_status()
data = response.json()
text = data["choices"][0]["message"]["content"]
tokens = data.get("usage", {}).get("total_tokens", 0)
return text, tokens, "local", model_name
except httpx.HTTPStatusError as e:
raise ProviderError("local", f"HTTP {e.response.status_code}") from e
except Exception as e:
raise ProviderError("local", str(e)) from e
async def _call_openrouter(
self,
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
) -> tuple[str, int, str, str]:
"""Call OpenRouter chat completions API (OpenAI-compatible).
Args:
messages: Chat messages.
max_tokens: Max response tokens.
temperature: Sampling temperature.
Returns:
Tuple of (text, tokens, "openrouter", model_name).
"""
client = await self._get_http_client()
# Prefer runtime override, fall back to env-based setting
model_name = self.settings.openrouter_model
if self._runtime_config is not None:
override = self._runtime_config.get_str(
"web.openrouter.model", default=model_name
)
if override:
model_name = override
try:
response = await client.post(
f"{self.settings.openrouter_base_url}/chat/completions",
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {self.settings.openrouter_api_key}",
},
json={
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
},
)
response.raise_for_status()
data = response.json()
text = data["choices"][0]["message"]["content"]
tokens = data.get("usage", {}).get("total_tokens", 0)
return text, tokens, "openrouter", model_name
except httpx.HTTPStatusError as e:
raise ProviderError("openrouter", f"HTTP {e.response.status_code}") from e
except Exception as e:
raise ProviderError("openrouter", str(e)) from e
async def _call_openai(
self,
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
) -> tuple[str, int, str, str]:
"""Call OpenAI chat completions API.
Args:
messages: Chat messages.
max_tokens: Max response tokens.
temperature: Sampling temperature.
Returns:
Tuple of (text, tokens, "openai", model_name).
"""
client = await self._get_http_client()
model_name = self.settings.openai_model
try:
response = await client.post(
"https://api.openai.com/v1/chat/completions",
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {self.settings.openai_api_key}",
},
json={
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
},
)
response.raise_for_status()
data = response.json()
text = data["choices"][0]["message"]["content"]
tokens = data.get("usage", {}).get("total_tokens", 0)
return text, tokens, "openai", model_name
except httpx.HTTPStatusError as e:
raise ProviderError("openai", f"HTTP {e.response.status_code}") from e
except Exception as e:
raise ProviderError("openai", str(e)) from e
async def _call_anthropic(
self,
messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
) -> tuple[str, int, str, str]:
"""Call Anthropic messages API.
Converts OpenAI-format messages to Anthropic format.
Args:
messages: Chat messages in OpenAI format.
max_tokens: Max response tokens.
temperature: Sampling temperature.
Returns:
Tuple of (text, tokens, "anthropic", model_name).
"""
client = await self._get_http_client()
model_name = self.settings.anthropic_model
# Convert OpenAI message format to Anthropic format
anthropic_messages = _convert_to_anthropic_messages(messages)
try:
response = await client.post(
"https://api.anthropic.com/v1/messages",
headers={
"Content-Type": "application/json",
"x-api-key": self.settings.anthropic_api_key,
"anthropic-version": "2023-06-01",
},
json={
"model": model_name,
"messages": anthropic_messages,
"max_tokens": max_tokens,
"temperature": temperature,
},
)
response.raise_for_status()
data = response.json()
text = data["content"][0]["text"]
tokens = data.get("usage", {}).get("input_tokens", 0) + data.get(
"usage", {}
).get("output_tokens", 0)
return text, tokens, "anthropic", model_name
except httpx.HTTPStatusError as e:
raise ProviderError("anthropic", f"HTTP {e.response.status_code}") from e
except Exception as e:
raise ProviderError("anthropic", str(e)) from e
def _convert_to_anthropic_messages(
messages: list[dict[str, Any]],
) -> list[dict[str, Any]]:
"""Convert OpenAI-format messages to Anthropic format.
Handles both text-only and multimodal (image) messages.
Args:
messages: Messages in OpenAI chat format.
Returns:
Messages in Anthropic format.
"""
result = []
for msg in messages:
content = msg.get("content")
if isinstance(content, str):
result.append({"role": msg["role"], "content": content})
elif isinstance(content, list):
anthropic_content = []
for part in content:
if part.get("type") == "text":
anthropic_content.append({"type": "text", "text": part["text"]})
elif part.get("type") == "image_url":
url = part["image_url"]["url"]
if url.startswith("data:"):
# Parse data URI: data:image/jpeg;base64,<data>
header, data = url.split(",", 1)
media_type = header.split(":")[1].split(";")[0]
anthropic_content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
},
}
)
else:
anthropic_content.append(
{
"type": "image",
"source": {"type": "url", "url": url},
}
)
result.append({"role": msg["role"], "content": anthropic_content})
else:
result.append(msg)
return result

102
ai_platform/modules/web/src/web/logging.py Normal file
View file

@ -0,0 +1,102 @@
"""Structured logging with request_id context propagation.
This module provides:
- ContextVar-based request_id tracking
- Structured JSON logging option
- Request ID injection into all log records
"""
import json
import logging
import sys
from contextvars import ContextVar
from typing import Any
# Context variable for request ID propagation
request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
class RequestIdFilter(logging.Filter):
"""Logging filter that adds request_id to log records."""
def filter(self, record: logging.LogRecord) -> bool:
"""Add request_id to the log record."""
record.request_id = request_id_ctx.get() or "-"
return True
class JsonFormatter(logging.Formatter):
"""JSON log formatter for structured logging."""
def format(self, record: logging.LogRecord) -> str:
"""Format log record as JSON."""
log_data: dict[str, Any] = {
"timestamp": self.formatTime(record, self.datefmt),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
"request_id": getattr(record, "request_id", "-"),
}
if record.exc_info:
log_data["exception"] = self.formatException(record.exc_info)
if hasattr(record, "extra_data"):
log_data.update(record.extra_data)
return json.dumps(log_data)
def configure_logging(level: str = "INFO", json_format: bool = False) -> None:
"""Configure logging for the application.
Args:
level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL).
json_format: If True, use JSON formatting; otherwise use text.
"""
root_logger = logging.getLogger("web")
root_logger.setLevel(getattr(logging, level.upper(), logging.INFO))
for handler in root_logger.handlers[:]:
root_logger.removeHandler(handler)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
handler.addFilter(RequestIdFilter())
if json_format:
handler.setFormatter(JsonFormatter())
else:
handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] [%(request_id)s] %(name)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
)
root_logger.addHandler(handler)
root_logger.propagate = False
def get_logger(name: str) -> logging.Logger:
"""Get a logger with the web prefix.
Args:
name: Logger name (will be prefixed with 'web.').
Returns:
logging.Logger: Configured logger instance.
"""
if name.startswith("web."):
return logging.getLogger(name)
return logging.getLogger(f"web.{name}")
def set_request_id(request_id: str | None) -> None:
"""Set the request ID for the current context."""
request_id_ctx.set(request_id)
def get_request_id() -> str | None:
"""Get the request ID for the current context."""
return request_id_ctx.get()

5
ai_platform/modules/web/src/web/metasearch/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Metasearch module - SearXNG client for web search."""
from web.metasearch.client import SearXNGClient
__all__ = ["SearXNGClient"]

537
ai_platform/modules/web/src/web/metasearch/client.py Normal file
View file

@ -0,0 +1,537 @@
"""SearXNG metasearch client."""
import asyncio
import random
import time
from typing import Any
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.exceptions import (
ProviderError,
SearchError,
WebConnectionError,
WebTimeoutError,
)
from web.logging import get_logger
from web.schemas.image_search import (
ImageSearchRequest,
ImageSearchResponse,
ImageSearchResult,
)
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("metasearch")
class SearXNGClient:
"""Client for SearXNG metasearch engine.
SearXNG aggregates results from multiple search engines (Google, Bing,
DuckDuckGo, etc.) and returns unified results via JSON API.
"""
provider_name: str = "searxng"
def __init__(self, settings: WebSettings) -> None:
"""Initialize the client.
Args:
settings: Application settings.
"""
self.settings = settings
self._client: httpx.AsyncClient | None = None
self._rate_lock = asyncio.Lock()
self._last_request_time: float = 0.0
self._min_interval: float = settings.searxng_rate_min_interval
async def _get_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client."""
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=self.settings.connect_timeout,
read=self.settings.request_timeout,
write=self.settings.request_timeout,
pool=self.settings.request_timeout,
),
limits=httpx.Limits(
max_connections=self.settings.search_pool_max_connections,
max_keepalive_connections=self.settings.search_pool_max_keepalive,
),
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip",
},
)
return self._client
async def __aenter__(self) -> "SearXNGClient":
return self
async def __aexit__(self, *exc: object) -> None:
await self.close()
async def close(self) -> None:
"""Close the HTTP client."""
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
"""Check if SearXNG is reachable.
Returns:
bool: True if healthy, False otherwise.
"""
try:
client = await self._get_client()
response = await client.get(
f"{self.settings.searxng_base_url}/healthz",
timeout=5.0,
)
return response.status_code == 200
except Exception as e:
logger.warning("SearXNG health check failed: %s", e)
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
"""Execute search queries via SearXNG.
Args:
request: Search request with queries and parameters.
request_id: Optional request ID for tracing.
Returns:
SearchResponse: Search results.
Raises:
SearchError: If search fails.
WebTimeoutError: If request times out.
WebConnectionError: If connection fails.
"""
start_time = time.perf_counter()
# Run all queries in parallel
tasks = [self._search_single(query, request) for query in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
first_error: Exception | None = None
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.error(
"SearXNG search failed for query '%s': %s",
request.queries[i],
result,
)
if first_error is None:
first_error = result
else:
all_results.extend(result)
# If ALL queries failed, raise the first error
if not all_results and first_error is not None:
raise first_error
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self,
query: str,
request: SearchRequest,
) -> list[SearchResult]:
"""Execute a single search query.
Enforces a minimum interval between requests to SearXNG,
preventing upstream engines from rate-limiting our IP.
Args:
query: The search query.
request: Search request with parameters.
Returns:
list[SearchResult]: Search results for this query.
"""
async with self._rate_lock:
now = time.perf_counter()
wait = self._min_interval - (now - self._last_request_time)
if wait > 0:
self._last_request_time = now + wait
else:
self._last_request_time = now
wait = 0
if wait > 0:
await asyncio.sleep(wait)
# Build queries (may split site allowlist into chunks)
effective_queries = self._build_queries(query, request)
# Base parameters (shared across chunked queries)
language = request.language if request.language not in ("auto", "") else "all"
base_params: dict[str, Any] = {
"format": "json",
"language": language,
}
if request.country and request.country.lower() not in ("auto", ""):
base_params["country"] = request.country
safesearch_map = {
"off": 0,
"moderate": 1,
"strict": 2,
}
base_params["safesearch"] = safesearch_map.get(request.safe_search, 1)
if request.freshness:
base_params["time_range"] = request.freshness
# Execute all chunked queries and merge results
all_results: list[SearchResult] = []
seen_urls: set[str] = set()
for eq in effective_queries:
params = {**base_params, "q": eq}
response_data = await self._execute_request(params)
for result in self._parse_results(query, response_data, request):
if result.url not in seen_urls:
seen_urls.add(result.url)
all_results.append(result)
return all_results
# Max site: filters per query to avoid bot detection by search engines.
# Google/Brave flag queries with 10+ site: operators as automated traffic.
_max_sites_per_query: int = 5
def _build_queries(self, query: str, request: SearchRequest) -> list[str]:
"""Build one or more queries with site filters.
Long site allowlists are split into chunks to produce shorter,
more natural-looking queries that avoid bot detection.
Args:
query: Original search query.
request: Search request with site filters.
Returns:
list[str]: One or more query strings.
"""
blocklist_part = ""
if request.site_blocklist:
blocklist_part = " ".join(f"-site:{s}" for s in request.site_blocklist)
if not request.site_allowlist:
parts = [query]
if blocklist_part:
parts.append(blocklist_part)
return [" ".join(parts)]
# Split site allowlist into chunks
sites = list(request.site_allowlist)
queries: list[str] = []
for i in range(0, len(sites), self._max_sites_per_query):
chunk = sites[i : i + self._max_sites_per_query]
site_filter = " OR ".join(f"site:{s}" for s in chunk)
parts = [query, f"({site_filter})"]
if blocklist_part:
parts.append(blocklist_part)
queries.append(" ".join(parts))
return queries
async def _execute_request(
self,
params: dict[str, Any],
) -> dict[str, Any]:
"""Execute HTTP request with retries.
Args:
params: Query parameters.
Returns:
dict: Response JSON data.
Raises:
SearchError: If request fails after retries.
WebTimeoutError: If request times out.
WebConnectionError: If connection fails.
"""
client = await self._get_client()
url = f"{self.settings.searxng_base_url}/search"
last_error: Exception | None = None
for attempt in range(self.settings.max_retries + 1):
try:
response = await client.get(url, params=params)
if response.status_code == 200:
return response.json()
if response.status_code == 429:
raise ProviderError("searxng", "Rate limit exceeded")
if response.status_code >= 500:
raise ProviderError(
"searxng", f"Server error: HTTP {response.status_code}"
)
# Other errors
error_msg = f"HTTP {response.status_code}"
try:
error_data = response.json()
if "error" in error_data:
error_msg = error_data["error"]
except Exception:
pass
raise ProviderError("searxng", error_msg)
except httpx.TimeoutException as e:
last_error = WebTimeoutError(
f"Request timed out after {self.settings.request_timeout}s",
timeout=self.settings.request_timeout,
)
logger.warning(
"SearXNG request timeout (attempt %d/%d): %s",
attempt + 1,
self.settings.max_retries + 1,
e,
)
except httpx.ConnectError as e:
last_error = WebConnectionError("searxng", str(e))
logger.warning(
"SearXNG connection error (attempt %d/%d): %s",
attempt + 1,
self.settings.max_retries + 1,
e,
)
except ProviderError as e:
if "Server error" in str(e):
last_error = e
logger.warning(
"SearXNG server error (attempt %d/%d): %s",
attempt + 1,
self.settings.max_retries + 1,
e,
)
else:
raise
except Exception as e:
last_error = SearchError(
params.get("q", "unknown"),
str(e),
provider="searxng",
)
logger.warning(
"SearXNG request failed (attempt %d/%d): %s",
attempt + 1,
self.settings.max_retries + 1,
e,
)
# Wait before retry (exponential backoff with jitter)
if attempt < self.settings.max_retries:
wait_time = min(
self.settings.retry_min_wait * (2**attempt),
self.settings.retry_max_wait,
)
wait_time *= 0.5 + random.random() # jitter: 50%-150% of base
await asyncio.sleep(wait_time)
# All retries exhausted
if last_error:
raise last_error
raise SearchError(
params.get("q", "unknown"), "Unknown error", provider="searxng"
)
def _parse_results(
self,
query: str,
response_data: dict[str, Any],
request: SearchRequest,
) -> list[SearchResult]:
"""Parse SearXNG API response.
Args:
query: The original query.
response_data: Raw API response.
request: Search request with filters.
Returns:
list[SearchResult]: Parsed search results.
"""
results: list[SearchResult] = []
searxng_results = response_data.get("results", [])
# Limit to max_results
searxng_results = searxng_results[: request.max_results]
for rank, item in enumerate(searxng_results, start=1):
url = item.get("url", "")
parsed_url = urlparse(url)
site = parsed_url.netloc
# Apply blocklist filter (in case SearXNG didn't fully filter)
if request.site_blocklist and site in request.site_blocklist:
continue
result = SearchResult(
query=query,
url=url,
title=item.get("title") or "",
snippet=item.get("content") or "",
rank=rank,
site=site,
published_at=item.get("publishedDate"),
)
results.append(result)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
"""Execute image search queries via SearXNG.
Args:
request: Image search request with queries and parameters.
request_id: Optional request ID for tracing.
Returns:
ImageSearchResponse: Image search results.
Raises:
SearchError: If search fails.
WebTimeoutError: If request times out.
WebConnectionError: If connection fails.
"""
start_time = time.perf_counter()
tasks = [self._image_search_single(query, request) for query in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[ImageSearchResult] = []
first_error: Exception | None = None
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.error(
"SearXNG image search failed for query '%s': %s",
request.queries[i],
result,
)
if first_error is None:
first_error = result
else:
all_results.extend(result)
if not all_results and first_error is not None:
raise first_error
execution_time_ms = (time.perf_counter() - start_time) * 1000
return ImageSearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _image_search_single(
self,
query: str,
request: ImageSearchRequest,
) -> list[ImageSearchResult]:
"""Execute a single image search query.
Args:
query: The search query.
request: Image search request with parameters.
Returns:
list[ImageSearchResult]: Image search results for this query.
"""
params: dict[str, Any] = {
"q": query,
"format": "json",
"categories": "images",
"language": request.language,
}
# SearXNG uses 'safesearch' with values: 0 (off), 1 (moderate), 2 (strict)
safesearch_map = {
"off": 0,
"strict": 2,
}
params["safesearch"] = safesearch_map.get(request.safe_search, 2)
response_data = await self._execute_request(params)
return self._parse_image_results(query, response_data, request)
def _parse_image_results(
self,
query: str,
response_data: dict[str, Any],
request: ImageSearchRequest,
) -> list[ImageSearchResult]:
"""Parse SearXNG image search response.
Args:
query: The original query.
response_data: Raw API response.
request: Image search request.
Returns:
list[ImageSearchResult]: Parsed image search results.
"""
results: list[ImageSearchResult] = []
searxng_results = response_data.get("results", [])
# Limit to max_results
searxng_results = searxng_results[: request.max_results]
for rank, item in enumerate(searxng_results, start=1):
# Parse resolution string (e.g., "1920x1080" or "1920 x 1080")
width, height = None, None
resolution = item.get("resolution", "")
if resolution:
parts = resolution.replace(" ", "").split("x")
if len(parts) == 2:
width = int(parts[0]) if parts[0].isdigit() else None
height = int(parts[1]) if parts[1].isdigit() else None
result = ImageSearchResult(
query=query,
image_url=item.get("img_src") or item.get("url") or "",
thumbnail_url=item.get("thumbnail_src") or item.get("thumbnail") or "",
source_url=item.get("url") or "",
title=item.get("title") or "",
description=item.get("content") or "",
width=width,
height=height,
publisher=item.get("source") or "",
rank=rank,
)
results.append(result)
return results

972
ai_platform/modules/web/src/web/orchestrator.py Normal file
View file

@ -0,0 +1,972 @@
"""Orchestrator for unified content gathering with auto-fallback.
Implements the fallback chain: HTTP Fetch → Playwright Browse → Vision LLM
"""
import asyncio
import hashlib
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from urllib.parse import parse_qs, urlparse
from web.browse.client import BrowseClient
from web.config import WebSettings, get_settings
from web.context.sources import (
COUNTRY_SOURCES,
FACTCHECK_SOURCES,
get_country_allowlist,
)
from web.evidence.packer import EvidencePacker
from web.fetch.client import FetchClient
from web.logging import get_logger
from web.schemas.browse import BrowseRequest
from web.schemas.common import PageContent
from web.schemas.context import SearchContext
from web.schemas.evidence import EvidenceItem, EvidencePackRequest, EvidenceStats
from web.schemas.fetch import FetchRequest
from web.schemas.gather import GatherRequest, GatherResponse, GatherStageResult
from web.schemas.search import SearchRequest, SearchResult
from web.search.protocol import SearchProvider
from web.validation import validate_urls_async
from web.vision.client import VisionClient
logger = get_logger("orchestrator")
@dataclass
class _StageData:
"""Internal container for inter-stage results."""
context: SearchContext | None = None
results: list[SearchResult] = field(default_factory=list)
pages: list[PageContent] = field(default_factory=list)
evidence: list[EvidenceItem] = field(default_factory=list)
stats: EvidenceStats | None = None
def _is_pdf_url(url: str) -> bool:
"""Check if URL points directly to a PDF document.
Detects:
- URLs ending with .pdf
- URLs with filename=*.pdf in query string
- URLs with .pdf before query string
Args:
url: URL to check.
Returns:
bool: True if URL is a direct PDF link.
"""
try:
parsed = urlparse(url)
path = parsed.path.lower()
# Check path ends with .pdf
if path.endswith(".pdf"):
return True
# Check query string for filename=*.pdf
if parsed.query:
query_params = parse_qs(parsed.query)
for key, values in query_params.items():
if key.lower() == "filename":
for val in values:
if val.lower().endswith(".pdf"):
return True
return False
except Exception:
return False
def _search_result_to_page(sr: "SearchResult") -> PageContent:
"""Convert a SearchResult into a PageContent using its snippet."""
text = f"{sr.title}\n\n{sr.snippet}" if sr.title else sr.snippet
return PageContent(
url=sr.url,
title=sr.title or None,
text=text,
text_hash=hashlib.sha256(text.encode()).hexdigest(),
extraction_method="snippet",
retrieved_at=datetime.now(timezone.utc).isoformat(),
extraction_time_ms=0.0,
published_at=sr.published_at,
warnings=["Content from search snippet only (full page not accessible)"],
)
class Orchestrator:
"""Orchestrates the full gather pipeline: search → fetch → evidence."""
def __init__(
self,
settings: WebSettings | None = None,
search_client: SearchProvider | None = None,
fetch_client: FetchClient | None = None,
llm_provider: str = "local",
runtime_config: object | None = None,
) -> None:
"""Initialize the orchestrator.
Args:
settings: Application settings.
search_client: Optional shared search client.
fetch_client: Optional shared fetch client.
llm_provider: Default LLM provider for evidence/context/vision.
"local" for free tier (Qwen), "openrouter" for premium tier.
runtime_config: Optional RuntimeConfigClient for live overrides.
"""
self.settings = settings or get_settings()
self.llm_provider = llm_provider
self.runtime_config = runtime_config
# Use shared clients when provided, otherwise lazy-create
self._search_client: SearchProvider | None = search_client
self._fetch_client: FetchClient | None = fetch_client
self._owns_search = search_client is None
self._owns_fetch = fetch_client is None
self._browse_client: BrowseClient | None = None
self._vision_client: VisionClient | None = None
self._evidence_packer: EvidencePacker | None = None
self._context_detector = None
@property
def search_client(self) -> SearchProvider:
"""Get or create search client."""
if self._search_client is None:
from web.metasearch.client import SearXNGClient
self._search_client = SearXNGClient(self.settings)
return self._search_client
@property
def fetch_client(self) -> FetchClient:
"""Get or create fetch client."""
if self._fetch_client is None:
self._fetch_client = FetchClient(self.settings)
return self._fetch_client
@property
def browse_client(self) -> BrowseClient:
"""Get or create browse client."""
if self._browse_client is None:
self._browse_client = BrowseClient(self.settings)
return self._browse_client
@property
def vision_client(self) -> VisionClient:
"""Get or create vision client, sharing browser from browse client."""
if self._vision_client is None:
# Share browser if browse client already has one
browser = None
if self._browse_client:
browser = self._browse_client.shared_browser
self._vision_client = VisionClient(self.settings, browser=browser)
return self._vision_client
@property
def evidence_packer(self) -> EvidencePacker:
"""Get or create evidence packer."""
if self._evidence_packer is None:
self._evidence_packer = EvidencePacker(self.settings)
if self.runtime_config is not None:
# Inject runtime config into the packer's LLM chain
self._evidence_packer._llm_chain._runtime_config = self.runtime_config # type: ignore[attr-defined]
return self._evidence_packer
@property
def context_detector(self):
"""Get or create context detector."""
if self._context_detector is None:
from web.context.detector import ContextDetector
self._context_detector = ContextDetector(
self.settings, llm_provider=self.llm_provider
)
if self.runtime_config is not None:
self._context_detector._llm_chain._runtime_config = ( # type: ignore[attr-defined]
self.runtime_config
)
return self._context_detector
async def close(self) -> None:
"""Close only self-owned clients (not shared ones passed via __init__)."""
if self._search_client and self._owns_search:
await self._search_client.close()
if self._fetch_client and self._owns_fetch:
await self._fetch_client.close()
if self._browse_client:
await self._browse_client.close()
if self._vision_client:
await self._vision_client.close()
if self._evidence_packer:
await self._evidence_packer.close()
if self._context_detector:
await self._context_detector.close()
async def gather(
self,
request: GatherRequest,
request_id: str | None = None,
) -> GatherResponse:
"""Execute the full gather pipeline.
Args:
request: Gather request.
request_id: Optional request ID for tracing.
Returns:
GatherResponse: Gathered evidence.
"""
start_time = time.perf_counter()
try:
return await asyncio.wait_for(
self._run_pipeline(request, request_id, start_time),
timeout=request.timeout_seconds,
)
except asyncio.TimeoutError:
logger.warning(
"Gather pipeline timed out after %.1fs for claim: %.100s",
request.timeout_seconds,
request.claim,
)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return GatherResponse(
request_id=request_id or "-",
claim=request.claim,
evidence=[],
evidence_stats=EvidenceStats(
input_items=0,
after_dedup=0,
output_items=0,
duplicates_removed=0,
tokens_used=0,
),
search_results=[],
stages=[
GatherStageResult(
stage="pipeline",
success=False,
items_processed=0,
items_failed=0,
duration_ms=round(execution_time_ms, 2),
error=f"Pipeline timed out after {request.timeout_seconds}s",
)
],
total_urls_found=0,
total_pages_fetched=0,
total_evidence_items=0,
execution_time_ms=round(execution_time_ms, 2),
)
async def _run_pipeline(
self,
request: GatherRequest,
request_id: str | None,
start_time: float,
) -> GatherResponse:
"""Run the full gather pipeline (called within timeout).
Args:
request: Gather request.
request_id: Optional request ID for tracing.
start_time: Pipeline start time.
Returns:
GatherResponse: Gathered evidence.
"""
stages: list[GatherStageResult] = []
data = _StageData()
# Stage 0: Context Detection (optional)
context: SearchContext | None = None
if request.country:
# Caller provided explicit country — build minimal context
context = SearchContext(primary_country=request.country.upper())
data.context = context
elif request.enable_context_detection:
ctx_stage, context = await self._run_context_stage(request)
stages.append(ctx_stage)
data.context = context
# Stage 1: Search (with multi-round if context available)
search_stage, search_data = await self._run_search_stage(request, context)
stages.append(search_stage)
if search_stage.success:
data.results = search_data
# Stage 2: Fetch/Browse/Vision
if data.results:
urls = list(
dict.fromkeys(r.url for r in data.results[: request.max_search_results])
)
# Pre-validate URLs (SSRF DNS check) at pipeline level so fetch can skip it
urls, dns_failed = await validate_urls_async(urls)
dns_failed_urls = {f.url for f in dns_failed}
fetch_stage, fetch_data = await self._run_fetch_stage(request, urls)
stages.append(fetch_stage)
if fetch_stage.success:
data.pages = fetch_data
# Snippet fallback: if fetch returned few pages, fill from
# search result snippets so evidence stage has something to work with.
# Skip URLs that failed DNS validation (gibberish/fake domains).
fetched_urls = {p.url for p in data.pages}
for sr in data.results[: request.max_search_results]:
if sr.url in dns_failed_urls:
continue
if sr.url not in fetched_urls and sr.snippet:
data.pages.append(
_search_result_to_page(sr)
)
fetched_urls.add(sr.url)
# Stage 3: Evidence Pack
evidence_items: list[EvidenceItem] = []
evidence_stats = EvidenceStats(
input_items=0,
after_dedup=0,
output_items=0,
duplicates_removed=0,
tokens_used=0,
)
if data.pages:
evidence_stage, evidence_data, stats = await self._run_evidence_stage(
request, data.pages
)
stages.append(evidence_stage)
if evidence_stage.success:
evidence_items = evidence_data
evidence_stats = stats
execution_time_ms = (time.perf_counter() - start_time) * 1000
return GatherResponse(
request_id=request_id or "-",
claim=request.claim,
search_context=data.context,
evidence=evidence_items,
evidence_stats=evidence_stats,
search_results=data.results,
stages=stages,
total_urls_found=len(data.results),
total_pages_fetched=len(data.pages),
total_evidence_items=len(evidence_items),
execution_time_ms=round(execution_time_ms, 2),
)
async def _run_context_stage(
self,
request: GatherRequest,
) -> tuple[GatherStageResult, SearchContext | None]:
"""Run Stage 0: Context detection from claim text.
Analyzes the claim using the local LLM to extract entities,
detect country, and generate optimized search queries.
Args:
request: Gather request.
Returns:
Tuple of (stage result, search context or None).
"""
start_time = time.perf_counter()
try:
context = await asyncio.wait_for(
self.context_detector.detect(request.claim),
timeout=self.settings.context_detection_timeout,
)
duration_ms = (time.perf_counter() - start_time) * 1000
logger.info(
"Context detected: country=%s, language=%s, queries=%d (%.0fms)",
context.primary_country,
context.detected_language,
len(context.search_queries),
duration_ms,
)
result = GatherStageResult(
stage="context",
success=True,
items_processed=1,
items_failed=0,
duration_ms=round(duration_ms, 2),
)
return result, context
except asyncio.TimeoutError:
duration_ms = (time.perf_counter() - start_time) * 1000
logger.warning(
"Context detection timed out after %.1fs", duration_ms / 1000
)
result = GatherStageResult(
stage="context",
success=False,
items_processed=0,
items_failed=1,
duration_ms=round(duration_ms, 2),
error="Context detection timed out",
)
return result, None
except Exception as e:
duration_ms = (time.perf_counter() - start_time) * 1000
logger.warning("Context detection failed: %s", e)
result = GatherStageResult(
stage="context",
success=False,
items_processed=0,
items_failed=1,
duration_ms=round(duration_ms, 2),
error=str(e),
)
return result, None
async def _run_search_stage(
self,
request: GatherRequest,
context: SearchContext | None = None,
) -> tuple[GatherStageResult, list[SearchResult]]:
"""Run the search stage with optional multi-round strategy.
When context is provided with a primary_country, runs up to three
search rounds in parallel:
Round 1: Country-specific official + media sources
Round 2: International fact-check sources
Round 3: Normal unrestricted search (current behavior)
Args:
request: Gather request.
context: Optional search context from Stage 0.
Returns:
Tuple of (stage result, deduplicated search results).
"""
start_time = time.perf_counter()
try:
# Determine queries: explicit > context-generated > raw claim
if request.search_queries:
queries = request.search_queries
elif context and context.search_queries:
queries = context.search_queries
else:
queries = [request.claim]
queries = queries[: self.settings.gather_max_search_queries]
# Determine search country for SearXNG regional results.
# If the detected country has no sources in our mapping, fall back
# to "US" so SearXNG returns useful international results instead
# of regional junk (e.g. zhihu.com for country=IR).
detected_country = (
request.country
or (context.primary_country if context else None)
or "US"
)
country = (
detected_country
if detected_country in COUNTRY_SOURCES
else "US"
)
# Determine language for SearXNG.
# Context-generated queries are always in English, so use "en"
# when the queries come from context detection. Only use the
# detected language when the caller provided explicit queries.
language = request.language
if language == "auto":
if request.search_queries:
# Caller-provided queries: use detected language
language = (
context.detected_language
if context and context.detected_language
else "en"
)
else:
# Context-generated queries are in English
language = "en"
# Build search tasks (multi-round when context available)
tasks = []
# Round 1: Country-specific sources (only when the detected
# country actually has sources defined — never use the fallback)
country_sites = (
get_country_allowlist(detected_country)
if context and context.primary_country
else []
)
if country_sites and not request.site_allowlist:
r1_request = SearchRequest(
queries=queries,
max_results=request.max_search_results,
site_allowlist=country_sites,
site_blocklist=request.site_blocklist,
language=language,
country=country,
)
tasks.append(self.search_client.search(r1_request))
# Round 2: Fact-check sources
if context and context.primary_country and not request.site_allowlist:
r2_request = SearchRequest(
queries=queries,
max_results=10,
site_allowlist=FACTCHECK_SOURCES,
site_blocklist=request.site_blocklist,
language=language,
country=country,
)
tasks.append(self.search_client.search(r2_request))
# Round 3: Normal unrestricted search (always runs)
r3_request = SearchRequest(
queries=queries,
max_results=request.max_search_results,
site_allowlist=request.site_allowlist,
site_blocklist=request.site_blocklist,
language=language,
country=country,
)
tasks.append(self.search_client.search(r3_request))
# Run all rounds in parallel
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
# Combine and deduplicate by URL (first seen wins)
all_results: list[SearchResult] = []
seen_urls: set[str] = set()
failed_rounds = 0
for result in results_or_errors:
if isinstance(result, Exception):
logger.warning("Search round failed: %s", result)
failed_rounds += 1
continue
for sr in result.results:
if sr.url not in seen_urls:
seen_urls.add(sr.url)
all_results.append(sr)
# ─── Tier 3 fallback: cloak stealth scraping ───────────────────
# Only invoked when:
# 1. CLOAK_ENABLED is true
# 2. Primary tiers (SearXNG + paid rotation) returned thin results
# Failures are non-fatal (we keep whatever we already have).
if (
getattr(self.settings, "cloak_enabled", False)
and len(all_results) < self.settings.cloak_fallback_threshold
):
try:
from web.search.cloak import CloakHTTPClient
if not hasattr(self, "_cloak_client") or self._cloak_client is None:
self._cloak_client = CloakHTTPClient(self.settings)
cloak_req = SearchRequest(
queries=queries,
max_results=10,
site_allowlist=request.site_allowlist,
site_blocklist=request.site_blocklist,
language=language if language != "auto" else None,
country=country,
)
cloak_resp = await self._cloak_client.search(cloak_req)
before = len(all_results)
for sr in cloak_resp.results:
if sr.url not in seen_urls:
seen_urls.add(sr.url)
all_results.append(sr)
logger.info(
"Tier-3 cloak top-up: +%d new results (was %d, now %d) in %dms",
len(all_results) - before, before, len(all_results),
int(cloak_resp.execution_time_ms),
)
except Exception as cloak_err:
logger.warning("Tier-3 cloak fallback failed (non-fatal): %s", cloak_err)
duration_ms = (time.perf_counter() - start_time) * 1000
stage_result = GatherStageResult(
stage="search",
success=len(all_results) > 0,
items_processed=len(all_results),
items_failed=failed_rounds,
duration_ms=round(duration_ms, 2),
)
return stage_result, all_results
except Exception as e:
duration_ms = (time.perf_counter() - start_time) * 1000
logger.error("Search stage failed: %s", e)
result = GatherStageResult(
stage="search",
success=False,
items_processed=0,
items_failed=1,
duration_ms=round(duration_ms, 2),
error=str(e),
)
return result, []
async def _run_fetch_stage(
self,
request: GatherRequest,
urls: list[str],
) -> tuple[GatherStageResult, list[PageContent]]:
"""Run the fetch stage with fallback.
Args:
request: Gather request.
urls: URLs to fetch.
Returns:
Tuple of (stage result, fetched pages).
"""
start_time = time.perf_counter()
pages: list[PageContent] = []
failed_count = 0
# Filter out direct PDF URLs (we can't extract text from PDFs via HTTP)
fetchable_urls = []
skipped_pdfs = []
for url in urls:
if _is_pdf_url(url):
skipped_pdfs.append(url)
logger.info("Skipping PDF URL: %s", url)
else:
fetchable_urls.append(url)
if skipped_pdfs:
logger.info("Skipped %d PDF URLs", len(skipped_pdfs))
# SSRF DNS validation already done at pipeline level (_run_pipeline)
try:
# Determine fetch method
if request.fetch_method == "auto":
# Use auto-fallback logic
pages, failed_count = await self._fetch_with_fallback(
fetchable_urls,
request.auto_fallback,
request.parallel_fetches,
)
elif request.fetch_method == "http":
pages, failed_count = await self._fetch_http_only(
fetchable_urls, request.parallel_fetches
)
elif request.fetch_method == "browse":
pages, failed_count = await self._fetch_browse_only(
fetchable_urls, request.parallel_fetches
)
elif request.fetch_method == "vision":
pages, failed_count = await self._fetch_vision_only(fetchable_urls)
duration_ms = (time.perf_counter() - start_time) * 1000
result = GatherStageResult(
stage="fetch",
success=True,
items_processed=len(pages),
items_failed=failed_count,
duration_ms=round(duration_ms, 2),
)
return result, pages
except Exception as e:
duration_ms = (time.perf_counter() - start_time) * 1000
logger.error("Fetch stage failed: %s", e)
result = GatherStageResult(
stage="fetch",
success=False,
items_processed=0,
items_failed=len(fetchable_urls),
duration_ms=round(duration_ms, 2),
error=str(e),
)
return result, []
async def _fetch_with_fallback(
self,
urls: list[str],
auto_fallback: bool,
parallel_fetches: int,
) -> tuple[list[PageContent], int]:
"""Fetch URLs with auto-fallback: HTTP → Browse → Vision.
Args:
urls: URLs to fetch.
auto_fallback: Whether to auto-fallback.
parallel_fetches: Number of parallel fetches.
Returns:
tuple: (pages, failed_count)
"""
pages: list[PageContent] = []
urls_needing_browse: list[str] = []
urls_needing_vision: list[str] = []
# Step 1: Try HTTP fetch first (skip_validation=True: orchestrator already validated)
fetch_request = FetchRequest(
urls=urls,
auto_fallback=auto_fallback,
parallel_fetches=parallel_fetches,
skip_validation=True,
)
fetch_response = await self.fetch_client.fetch(fetch_request)
for page_result in fetch_response.pages:
if page_result.needs_fallback and auto_fallback:
urls_needing_browse.append(page_result.url)
else:
pages.append(page_result.to_page_content())
# Add failed URLs to browse queue if auto_fallback
if auto_fallback:
for failed in fetch_response.failed_urls:
urls_needing_browse.append(failed.url)
# Step 2: Try Playwright browse for failed/incomplete pages
if urls_needing_browse:
logger.info(
"Browse fallback: %d URLs to retry with Playwright",
len(urls_needing_browse),
)
try:
browse_request = BrowseRequest(
urls=urls_needing_browse,
parallel_browses=min(3, parallel_fetches),
extra_wait_ms=self.settings.browse_extra_wait_ms,
)
browse_response = await self.browse_client.browse(browse_request)
for page_result in browse_response.pages:
# Check if content is still insufficient
if (
len(page_result.text) < self.settings.fetch_min_text_length
and auto_fallback
):
urls_needing_vision.append(page_result.url)
else:
pages.append(
page_result.to_page_content(fallback_chain=["http"])
)
# Add browse failures to vision queue
if auto_fallback:
for failed in browse_response.failed_urls:
urls_needing_vision.append(failed.url)
except ImportError:
# Playwright not installed, add to vision queue
logger.warning("Playwright not available, trying vision")
urls_needing_vision.extend(urls_needing_browse)
except Exception as e:
logger.warning("Browse failed: %s", e)
urls_needing_vision.extend(urls_needing_browse)
# Step 3: Try vision extraction as last resort (parallel, semaphore in VisionClient)
if urls_needing_vision and auto_fallback:
try:
# Vision is expensive, limit to first few
vision_urls = urls_needing_vision[
: self.settings.gather_max_vision_urls
]
tasks = [
self.vision_client.extract_to_page_content(url)
for url in vision_urls
]
results = await asyncio.gather(*tasks, return_exceptions=True)
for i, result in enumerate(results):
if isinstance(result, Exception):
logger.warning(
"Vision extraction failed for %s: %s",
vision_urls[i],
result,
)
else:
pages.append(result)
except ImportError:
logger.warning("Vision not available")
except Exception as e:
logger.warning("Vision stage failed: %s", e)
# Count failures once: URLs that failed HTTP and were never recovered
all_failed = {f.url for f in fetch_response.failed_urls}
all_recovered = {p.url for p in pages}
failed_count = len(all_failed - all_recovered)
return pages, failed_count
async def _fetch_http_only(
self,
urls: list[str],
parallel_fetches: int,
) -> tuple[list[PageContent], int]:
"""Fetch URLs using HTTP only.
Args:
urls: URLs to fetch.
parallel_fetches: Number of parallel fetches.
Returns:
tuple: (pages, failed_count)
"""
fetch_request = FetchRequest(
urls=urls,
auto_fallback=False,
parallel_fetches=parallel_fetches,
skip_validation=True,
)
response = await self.fetch_client.fetch(fetch_request)
pages = [page_result.to_page_content() for page_result in response.pages]
return pages, len(response.failed_urls)
async def _fetch_browse_only(
self,
urls: list[str],
parallel_fetches: int,
) -> tuple[list[PageContent], int]:
"""Fetch URLs using Playwright browse only.
Args:
urls: URLs to fetch.
parallel_fetches: Number of parallel fetches.
Returns:
tuple: (pages, failed_count)
"""
browse_request = BrowseRequest(
urls=urls,
parallel_browses=min(3, parallel_fetches),
)
response = await self.browse_client.browse(browse_request)
pages = [page_result.to_page_content() for page_result in response.pages]
return pages, len(response.failed_urls)
async def _fetch_vision_only(
self,
urls: list[str],
) -> tuple[list[PageContent], int]:
"""Fetch URLs using vision extraction only (parallel).
Args:
urls: URLs to fetch.
Returns:
tuple: (pages, failed_count)
"""
pages: list[PageContent] = []
failed_count = 0
vision_urls = urls[: self.settings.gather_max_vision_urls * 2]
tasks = [self.vision_client.extract_to_page_content(url) for url in vision_urls]
results = await asyncio.gather(*tasks, return_exceptions=True)
for i, result in enumerate(results):
if isinstance(result, Exception):
logger.warning(
"Vision extraction failed for %s: %s", vision_urls[i], result
)
failed_count += 1
else:
pages.append(result)
return pages, failed_count
async def _run_evidence_stage(
self,
request: GatherRequest,
pages: list[PageContent],
) -> tuple[GatherStageResult, list[EvidenceItem], EvidenceStats]:
"""Run the evidence packing stage.
Args:
request: Gather request.
pages: Fetched pages.
Returns:
Tuple of (stage result, evidence items, evidence stats).
"""
start_time = time.perf_counter()
try:
# Derive score_relevance: explicit request value wins,
# otherwise only score when LLM is already in use for snippets
score_rel = request.score_relevance
if score_rel is None:
score_rel = request.extract_snippets
evidence_request = EvidencePackRequest(
pages=pages,
claim=request.claim,
dedupe=request.dedupe,
max_items=request.max_evidence_items,
include_full_text=request.include_full_text,
extract_snippets=request.extract_snippets,
summarize=request.summarize,
score_relevance=score_rel,
llm_provider=self.llm_provider,
)
response = await self.evidence_packer.pack(evidence_request)
duration_ms = (time.perf_counter() - start_time) * 1000
result = GatherStageResult(
stage="evidence",
success=True,
items_processed=response.stats.output_items,
items_failed=0,
duration_ms=round(duration_ms, 2),
)
return result, response.evidence, response.stats
except Exception as e:
duration_ms = (time.perf_counter() - start_time) * 1000
logger.error("Evidence stage failed: %s", e)
empty_stats = EvidenceStats(
input_items=len(pages),
after_dedup=0,
output_items=0,
duplicates_removed=0,
tokens_used=0,
)
result = GatherStageResult(
stage="evidence",
success=False,
items_processed=0,
items_failed=len(pages),
duration_ms=round(duration_ms, 2),
error=str(e),
)
return result, [], empty_stats

131
ai_platform/modules/web/src/web/runtime_config.py Normal file
View file

@ -0,0 +1,131 @@
"""Runtime config client — fetches config overrides from dashboard.
Polls the dashboard /api/config endpoint periodically and caches values
in memory. All accessors fall back to defaults if the dashboard is
unreachable or the key is missing.
"""
import asyncio
import contextlib
from typing import Any
import httpx
from web.logging import get_logger
logger = get_logger("runtime_config")
# Default values — matches dashboard KNOWN_KEYS for safety
_DEFAULTS: dict[str, Any] = {
"web.providers.serpapi.enabled": True,
"web.providers.tavily.enabled": True,
"web.providers.brave.enabled": True,
"web.providers.linkup.enabled": True,
"web.providers.exa.enabled": True,
"web.premium.strategy": "round-robin",
"web.premium.priority_order": "serpapi,brave,tavily,linkup",
"web.openrouter.model": "google/gemini-3.1-flash-lite-preview",
"web.tier.free.max_search_results": 10,
"web.tier.premium.max_search_results": 10,
}
class RuntimeConfigClient:
"""Polls dashboard /api/config and caches values in-process."""
def __init__(
self,
dashboard_url: str | None,
poll_interval_seconds: int = 30,
request_timeout: float = 5.0,
) -> None:
self.dashboard_url = dashboard_url.rstrip("/") if dashboard_url else None
self.poll_interval = poll_interval_seconds
self._timeout = request_timeout
self._cache: dict[str, Any] = dict(_DEFAULTS)
self._client: httpx.AsyncClient | None = None
self._task: asyncio.Task[None] | None = None
self._enabled = bool(dashboard_url)
@property
def enabled(self) -> bool:
return self._enabled
def get(self, key: str, default: Any = None) -> Any:
"""Return cached config value (or default)."""
if key in self._cache:
return self._cache[key]
if default is not None:
return default
return _DEFAULTS.get(key)
def get_bool(self, key: str, default: bool = True) -> bool:
v = self.get(key, default)
return bool(v)
def get_int(self, key: str, default: int = 0) -> int:
v = self.get(key, default)
try:
return int(v)
except (TypeError, ValueError):
return default
def get_str(self, key: str, default: str = "") -> str:
v = self.get(key, default)
return str(v) if v is not None else default
async def start(self) -> None:
"""Start the background poller."""
if not self._enabled:
logger.info("RuntimeConfigClient disabled (no dashboard URL) — using defaults")
return
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=2.0, read=self._timeout, write=2.0, pool=5.0
)
)
# Fetch once synchronously so first requests already have overrides
await self._refresh()
self._task = asyncio.create_task(self._loop())
logger.info(
"RuntimeConfigClient started (polling %s every %ds)",
self.dashboard_url,
self.poll_interval,
)
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await self._task
self._task = None
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def _loop(self) -> None:
while True:
try:
await asyncio.sleep(self.poll_interval)
await self._refresh()
except asyncio.CancelledError:
raise
except Exception as e:
logger.debug("Config poll failed: %s", e)
async def _refresh(self) -> None:
if self._client is None or self.dashboard_url is None:
return
try:
resp = await self._client.get(f"{self.dashboard_url}/api/config")
resp.raise_for_status()
data = resp.json()
except Exception as e:
logger.debug("Config refresh failed: %s", e)
return
items = data.get("items", {})
new_cache: dict[str, Any] = dict(_DEFAULTS)
for key, entry in items.items():
new_cache[key] = entry.get("value")
self._cache = new_cache

64
ai_platform/modules/web/src/web/schemas/__init__.py Normal file
View file

@ -0,0 +1,64 @@
"""Schemas package - All request/response schemas."""
from web.schemas.browse import BrowsePageResult, BrowseRequest, BrowseResponse
from web.schemas.common import (
ErrorDetail,
ErrorResponse,
HealthResponse,
PageContent,
ProviderHealth,
ReadinessResponse,
)
from web.schemas.evidence import EvidenceItem, EvidencePackRequest, EvidencePackResponse
from web.schemas.fetch import FetchPageResult, FetchRequest, FetchResponse
from web.schemas.gather import GatherRequest, GatherResponse
from web.schemas.image_search import (
ImageSearchRequest,
ImageSearchResponse,
ImageSearchResult,
)
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
from web.schemas.vision import (
ImageContext,
VisionExtractRequest,
VisionExtractResponse,
VisionPageResult,
)
__all__ = [
"BrowsePageResult",
# Browse
"BrowseRequest",
"BrowseResponse",
# Common
"ErrorDetail",
"ErrorResponse",
"EvidenceItem",
# Evidence
"EvidencePackRequest",
"EvidencePackResponse",
"FetchPageResult",
# Fetch
"FetchRequest",
"FetchResponse",
# Gather
"GatherRequest",
"GatherResponse",
"HealthResponse",
"ImageContext",
# Image Search
"ImageSearchRequest",
"ImageSearchResponse",
"ImageSearchResult",
"PageContent",
"ProviderHealth",
"ReadinessResponse",
# Search
"SearchRequest",
"SearchResponse",
"SearchResult",
# Vision
"VisionExtractRequest",
"VisionExtractResponse",
"VisionPageResult",
]

167
ai_platform/modules/web/src/web/schemas/browse.py Normal file
View file

@ -0,0 +1,167 @@
"""Browse schemas - Playwright browser automation."""
from typing import TYPE_CHECKING, Literal
if TYPE_CHECKING:
from web.schemas.common import PageContent
from pydantic import BaseModel, ConfigDict, Field, field_validator
from web.schemas.common import FailedUrl
from web.validation import validate_url
class BrowseRequest(BaseModel):
"""Request schema for browser-based fetch."""
model_config = ConfigDict(extra="forbid")
urls: list[str] = Field(
min_length=1,
max_length=20,
description="URLs to browse (1-20, limited due to browser overhead)",
)
@field_validator("urls")
@classmethod
def validate_urls(cls, v: list[str]) -> list[str]:
"""Validate all URLs for SSRF protection."""
for url in v:
validate_url(url)
return v
# Browser behavior
wait_until: Literal["load", "domcontentloaded", "networkidle"] = Field(
default="load",
description="Wait condition before extraction",
)
timeout_ms: int = Field(
default=30000,
ge=1000,
le=60000,
description="Max wait time for page load",
)
extra_wait_ms: int = Field(
default=0,
ge=0,
le=10000,
description="Additional wait after page load for dynamic content",
)
wait_for_selector: str | None = Field(
default=None,
description="CSS selector to wait for before extraction",
)
# Screenshot options
screenshot: bool = Field(
default=False,
description="Capture screenshot",
)
full_page_screenshot: bool = Field(
default=True,
description="Capture full page (scroll) or viewport only",
)
# Extraction
extract_text: bool = Field(default=True, description="Extract rendered text")
include_html: bool = Field(default=False, description="Include rendered HTML")
# Concurrency
parallel_browses: int = Field(
default=3,
ge=1,
le=5,
description="Number of parallel browser contexts",
)
class BrowsePageResult(BaseModel):
"""Result for a single browsed page."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Original URL")
final_url: str | None = Field(default=None, description="Final URL after redirects")
canonical_url: str | None = Field(
default=None, description="Canonical URL if found"
)
title: str | None = Field(default=None, description="Page title")
text: str = Field(description="Extracted text content")
text_hash: str = Field(description="SHA256 hash of text content")
html: str | None = Field(default=None, description="Rendered HTML (if requested)")
extraction_method: Literal["http", "browse", "vision"] = Field(
default="browse",
description="Method used to extract content",
)
fallback_chain: list[str] = Field(
default_factory=list,
description="Methods attempted before success",
)
published_at: str | None = Field(
default=None, description="Publication date if found"
)
retrieved_at: str = Field(description="ISO timestamp of retrieval")
extraction_time_ms: float = Field(description="Time taken to extract")
warnings: list[str] = Field(
default_factory=list,
description="Warnings during extraction",
)
# Browse-specific fields
screenshot_base64: str | None = Field(
default=None, description="Screenshot as base64"
)
viewport_width: int | None = Field(
default=None, description="Browser viewport width"
)
viewport_height: int | None = Field(
default=None, description="Browser viewport height"
)
def to_page_content(self, fallback_chain: list[str] | None = None) -> "PageContent":
"""Convert to PageContent for orchestrator pipeline.
Args:
fallback_chain: Override fallback chain (defaults to self.fallback_chain).
Returns:
PageContent with fields mapped from this result.
"""
from web.schemas.common import PageContent
return PageContent(
url=self.url,
canonical_url=self.canonical_url,
title=self.title,
text=self.text,
text_hash=self.text_hash,
html=self.html,
extraction_method="browse",
fallback_chain=fallback_chain
if fallback_chain is not None
else self.fallback_chain,
published_at=self.published_at,
retrieved_at=self.retrieved_at,
extraction_time_ms=self.extraction_time_ms,
warnings=self.warnings,
)
class BrowseResponse(BaseModel):
"""Response schema for browser-based fetch."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Request ID for tracing")
pages: list[BrowsePageResult] = Field(description="Browsed pages")
total_browsed: int = Field(description="Pages successfully browsed")
total_failed: int = Field(description="Pages that failed")
execution_time_ms: float = Field(description="Total execution time")
failed_urls: list[FailedUrl] = Field(
default_factory=list,
description="URLs that failed",
)

123
ai_platform/modules/web/src/web/schemas/common.py Normal file
View file

@ -0,0 +1,123 @@
"""Common schemas shared across all submodules."""
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
class FailedUrl(BaseModel):
"""A URL that failed during processing."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="URL that failed")
error: str = Field(description="Error message")
class PageImage(BaseModel):
"""An image found on a page."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Image URL")
alt: str | None = Field(default=None, description="Alt text")
def make_error_detail(
code: str,
message: str,
request_id: str | None = None,
**extra: Any,
) -> dict[str, Any]:
"""Build a standardized error detail dict.
Args:
code: Machine-readable error code.
message: Human-readable error message.
request_id: Optional request ID for tracing.
**extra: Additional key-value pairs to include.
Returns:
Standardized error detail dict.
"""
result: dict[str, Any] = {"error": code, "message": message}
if request_id:
result["request_id"] = request_id
result.update(extra)
return result
class PageContent(BaseModel):
"""Extracted content from a web page."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Original URL")
canonical_url: str | None = Field(
default=None, description="Canonical URL if found"
)
title: str | None = Field(default=None, description="Page title")
text: str = Field(description="Extracted text content")
text_hash: str = Field(description="SHA256 hash of text content")
html: str | None = Field(default=None, description="Raw HTML (if requested)")
extraction_method: Literal["http", "browse", "vision", "snippet"] = Field(
description="Method used to extract content"
)
fallback_chain: list[str] = Field(
default_factory=list,
description="Methods attempted before success",
)
published_at: str | None = Field(
default=None, description="Publication date if found"
)
retrieved_at: str = Field(description="ISO timestamp of retrieval")
extraction_time_ms: float = Field(description="Time taken to extract")
images: list[PageImage] | None = Field(
default=None,
description="Images with extracted context (from vision)",
)
warnings: list[str] = Field(
default_factory=list,
description="Warnings during extraction",
)
class ProviderHealth(BaseModel):
"""Health status for a provider."""
name: str = Field(description="Provider name")
healthy: bool = Field(description="Whether the provider is healthy")
message: str | None = Field(default=None, description="Status message")
class HealthResponse(BaseModel):
"""Health check response."""
status: Literal["healthy", "degraded", "unhealthy"] = Field(
description="Overall status"
)
providers: list[ProviderHealth] = Field(description="Per-provider health")
class ReadinessResponse(BaseModel):
"""Readiness probe response."""
ready: bool = Field(default=True)
class ErrorDetail(BaseModel):
"""Error detail."""
code: str = Field(description="Error code")
message: str = Field(description="Error message")
details: dict | None = Field(default=None, description="Additional details")
class ErrorResponse(BaseModel):
"""Standard error response."""
request_id: str | None = Field(default=None, description="Request ID")
error: ErrorDetail = Field(description="Error information")

47
ai_platform/modules/web/src/web/schemas/context.py Normal file
View file

@ -0,0 +1,47 @@
"""Context detection schemas for Stage 0 of the gather pipeline."""
from pydantic import BaseModel, ConfigDict, Field
class EntitySet(BaseModel):
"""Extracted named entities from claim text."""
model_config = ConfigDict(extra="forbid")
persons: list[str] = Field(default_factory=list, description="Person names")
institutions: list[str] = Field(
default_factory=list, description="Organizations, parties, companies"
)
locations: list[str] = Field(
default_factory=list, description="Cities, regions, countries mentioned"
)
class SearchContext(BaseModel):
"""Output of Stage 0: context detected from claim text.
Used by the search stage to prioritize country-specific sources.
"""
model_config = ConfigDict(extra="forbid")
primary_country: str | None = Field(
default=None,
description="ISO 3166-1 alpha-2 code of primary country (e.g. 'RO', 'US')",
)
secondary_countries: list[str] = Field(
default_factory=list,
description="Other countries involved",
)
entities: EntitySet = Field(
default_factory=EntitySet,
description="Extracted named entities",
)
detected_language: str = Field(
default="en",
description="ISO 639-1 language code detected in claim",
)
search_queries: list[str] = Field(
default_factory=list,
description="LLM-generated search queries tailored to the claim",
)

157
ai_platform/modules/web/src/web/schemas/evidence.py Normal file
View file

@ -0,0 +1,157 @@
"""Evidence schemas - Evidence pack builder."""
from pydantic import BaseModel, ConfigDict, Field
from web.schemas.common import PageContent
class EvidenceItem(BaseModel):
"""A single evidence item in the pack."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Source URL")
canonical_url: str | None = Field(default=None)
title: str | None = Field(default=None, description="Page title")
publisher: str | None = Field(default=None, description="Publisher/domain")
published_at: str | None = Field(default=None, description="Publication date")
retrieved_at: str = Field(description="When retrieved")
# Content
snippet: str | None = Field(
default=None,
description="LLM-extracted relevant snippet (deprecated: use full_text)",
)
summary: str | None = Field(
default=None,
description="LLM-generated summary relevant to the claim",
)
full_text: str | None = Field(
default=None,
description="Full page text",
)
full_text_hash: str = Field(description="Hash of full text for dedup")
# Provenance
provenance: dict = Field(
description="How this evidence was found",
default_factory=dict,
)
# Scoring
relevance_score: float | None = Field(
default=None,
ge=0.0,
le=1.0,
description="Relevance to claim (0-1)",
)
credibility_score: float | None = Field(
default=None,
ge=0.0,
le=1.0,
description="Source credibility (0-1)",
)
class EvidencePackRequest(BaseModel):
"""Request schema for evidence pack building."""
model_config = ConfigDict(extra="forbid")
pages: list[PageContent] = Field(
min_length=1,
description="Pages to pack (from fetch/browse/vision)",
)
# Context for snippet extraction
claim: str = Field(
description="The claim being verified - used for relevance scoring",
)
# Deduplication
dedupe: bool = Field(
default=True,
description="Remove duplicate content",
)
dedupe_threshold: float = Field(
default=0.9,
ge=0.0,
le=1.0,
description="Similarity threshold for dedup (0-1)",
)
# Limits
max_items: int = Field(
default=30,
ge=1,
le=100,
description="Maximum items in pack",
)
max_snippet_length: int = Field(
default=500,
ge=100,
le=2000,
description="Max characters per snippet",
)
# Full text inclusion
include_full_text: bool = Field(
default=True,
description="Include full page text in evidence items",
)
# Snippet extraction
extract_snippets: bool = Field(
default=False,
description="Use LLM to extract relevant snippets (deprecated: use full_text)",
)
# Summarization
summarize: bool = Field(
default=False,
description="Use LLM to summarize page content",
)
max_summary_length: int = Field(
default=800,
ge=200,
le=2000,
description="Max summary length in chars",
)
# Scoring
score_relevance: bool = Field(
default=False,
description="Score relevance to claim (requires LLM)",
)
score_credibility: bool = Field(
default=False,
description="Score source credibility",
)
# LLM settings for snippet extraction
llm_provider: str = Field(
default="auto",
description="LLM provider for snippet extraction",
)
class EvidenceStats(BaseModel):
"""Statistics about evidence pack building."""
input_items: int = Field(description="Items received")
after_dedup: int = Field(description="Items after deduplication")
output_items: int = Field(description="Items in final pack")
duplicates_removed: int = Field(description="Duplicates removed")
tokens_used: int = Field(description="LLM tokens used")
class EvidencePackResponse(BaseModel):
"""Response schema for evidence pack."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Request ID for tracing")
claim: str = Field(description="Original claim")
evidence: list[EvidenceItem] = Field(description="Evidence items")
stats: EvidenceStats = Field(description="Pack statistics")
execution_time_ms: float = Field(description="Total execution time")

164
ai_platform/modules/web/src/web/schemas/fetch.py Normal file
View file

@ -0,0 +1,164 @@
"""Fetch schemas - HTTP fetch with text extraction."""
from typing import TYPE_CHECKING, Literal
if TYPE_CHECKING:
from web.schemas.common import PageContent
from pydantic import BaseModel, ConfigDict, Field, field_validator
from web.schemas.common import FailedUrl
from web.validation import validate_url
class FetchRequest(BaseModel):
"""Request schema for web fetch."""
model_config = ConfigDict(extra="forbid")
urls: list[str] = Field(
min_length=1,
max_length=50,
description="URLs to fetch (1-50)",
)
@field_validator("urls")
@classmethod
def validate_urls(cls, v: list[str]) -> list[str]:
"""Validate all URLs for SSRF protection."""
for url in v:
validate_url(url)
return v
# Extraction options
extract_text: bool = Field(default=True, description="Extract main text content")
include_html: bool = Field(default=False, description="Include raw HTML")
extract_metadata: bool = Field(default=True, description="Extract metadata")
# Auto-fallback behavior
auto_fallback: bool = Field(
default=True,
description="Auto-escalate: HTTP → Browse → Vision on failure",
)
method: Literal["http", "browse", "vision", "auto"] = Field(
default="auto",
description="Extraction method (auto = try HTTP first)",
)
# Timeouts
timeout_seconds: float = Field(
default=30.0,
ge=1.0,
le=120.0,
description="Timeout per URL",
)
# Quality thresholds for auto-fallback
min_text_length: int = Field(
default=200,
ge=0,
description="Minimum text length before fallback",
)
# Concurrency
parallel_fetches: int = Field(
default=5,
ge=1,
le=20,
description="Number of parallel fetches",
)
# Internal flag: skip async SSRF validation when already done by orchestrator
skip_validation: bool = Field(
default=False,
description="Skip async SSRF DNS validation (set by orchestrator after pre-validation)",
exclude=True,
)
class FetchPageResult(BaseModel):
"""Result for a single fetched page."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Original URL")
canonical_url: str | None = Field(
default=None, description="Canonical URL if found"
)
title: str | None = Field(default=None, description="Page title")
text: str = Field(description="Extracted text content")
text_hash: str = Field(description="SHA256 hash of text content")
html: str | None = Field(default=None, description="Raw HTML (if requested)")
extraction_method: Literal["http", "browse", "vision"] = Field(
default="http",
description="Method used to extract content",
)
fallback_chain: list[str] = Field(
default_factory=list,
description="Methods attempted before success",
)
published_at: str | None = Field(
default=None, description="Publication date if found"
)
retrieved_at: str = Field(description="ISO timestamp of retrieval")
extraction_time_ms: float = Field(description="Time taken to extract")
warnings: list[str] = Field(
default_factory=list,
description="Warnings during extraction",
)
# HTTP-specific fields
needs_fallback: bool = Field(
default=False,
description="Whether content needs fallback to browse/vision",
)
status_code: int | None = Field(default=None, description="HTTP status code")
content_type: str | None = Field(default=None, description="Content-Type header")
def to_page_content(self, fallback_chain: list[str] | None = None) -> "PageContent":
"""Convert to PageContent for orchestrator pipeline.
Args:
fallback_chain: Override fallback chain (defaults to self.fallback_chain).
Returns:
PageContent with fields mapped from this result.
"""
from web.schemas.common import PageContent
return PageContent(
url=self.url,
canonical_url=self.canonical_url,
title=self.title,
text=self.text,
text_hash=self.text_hash,
html=self.html,
extraction_method="http",
fallback_chain=fallback_chain
if fallback_chain is not None
else self.fallback_chain,
published_at=self.published_at,
retrieved_at=self.retrieved_at,
extraction_time_ms=self.extraction_time_ms,
warnings=self.warnings,
)
class FetchResponse(BaseModel):
"""Response schema for web fetch."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Request ID for tracing")
pages: list[FetchPageResult] = Field(description="Fetched pages")
total_fetched: int = Field(description="Number of pages successfully fetched")
total_failed: int = Field(description="Number of pages that failed")
execution_time_ms: float = Field(description="Total execution time")
failed_urls: list[FailedUrl] = Field(
default_factory=list,
description="URLs that failed with error details",
)

161
ai_platform/modules/web/src/web/schemas/gather.py Normal file
View file

@ -0,0 +1,161 @@
"""Gather schemas - Unified endpoint that orchestrates everything."""
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from web.schemas.context import SearchContext
from web.schemas.evidence import EvidenceItem, EvidenceStats
from web.schemas.search import SearchResult
class GatherRequest(BaseModel):
"""Request schema for unified gather endpoint.
This endpoint orchestrates: search → fetch → evidence pack
"""
model_config = ConfigDict(extra="forbid")
# What to gather evidence for
claim: str = Field(
min_length=10,
max_length=1000,
description="The claim to gather evidence for",
)
# Search options
search_queries: list[str] | None = Field(
default=None,
description="Custom search queries (auto-generated if not provided)",
)
max_search_results: int = Field(
default=10,
ge=5,
le=50,
description="Max search results to fetch",
)
site_allowlist: list[str] | None = Field(
default=None,
description="Only search these domains",
)
site_blocklist: list[str] | None = Field(
default=None,
description="Exclude these domains",
)
language: str = Field(
default="auto",
description="Search language (ISO 639-1 code, or 'auto' for auto-detect)",
)
# Fetch options
fetch_method: Literal["auto", "http", "browse", "vision"] = Field(
default="auto",
description="How to fetch pages",
)
auto_fallback: bool = Field(
default=True,
description="Auto-escalate on fetch failure",
)
# Evidence options
max_evidence_items: int = Field(
default=15,
ge=1,
le=50,
description="Max items in final evidence pack",
)
include_full_text: bool = Field(
default=True,
description="Include full page text in evidence items",
)
extract_snippets: bool = Field(
default=False,
description="Extract LLM snippets (deprecated: use full_text)",
)
summarize: bool = Field(
default=False,
description="Generate LLM summaries of page content relative to the claim",
)
dedupe: bool = Field(
default=True,
description="Deduplicate evidence",
)
score_relevance: bool | None = Field(
default=None,
description="Score relevance via LLM. Defaults to True when extract_snippets=True.",
)
# Context detection options
enable_context_detection: bool = Field(
default=True,
description="Run Stage 0 context detection before search",
)
country: str | None = Field(
default=None,
description=(
"Override country for search (ISO 3166-1 alpha-2). "
"Skips auto-detection when set."
),
)
# Performance options
parallel_fetches: int = Field(
default=5,
ge=1,
le=10,
description="Concurrent fetch operations",
)
timeout_seconds: float = Field(
default=90.0,
ge=10.0,
le=300.0,
description="Total timeout for gather",
)
class GatherStageResult(BaseModel):
"""Result from a single stage of gathering."""
stage: str = Field(description="Stage name")
success: bool = Field(description="Stage completed successfully")
items_processed: int = Field(description="Items processed")
items_failed: int = Field(description="Items that failed")
duration_ms: float = Field(description="Stage duration")
error: str | None = Field(default=None, description="Error if failed")
class GatherResponse(BaseModel):
"""Response schema for unified gather endpoint."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Request ID for tracing")
claim: str = Field(description="Original claim")
# Final evidence
evidence: list[EvidenceItem] = Field(description="Gathered evidence")
evidence_stats: EvidenceStats = Field(description="Evidence statistics")
# Context detection output (Stage 0)
search_context: SearchContext | None = Field(
default=None,
description="Detected context from claim (Stage 0 output)",
)
# Search results (intermediate)
search_results: list[SearchResult] = Field(
default_factory=list,
description="Raw search results",
)
# Stage results for debugging
stages: list[GatherStageResult] = Field(
description="Results from each stage",
)
# Overall stats
total_urls_found: int = Field(description="URLs from search")
total_pages_fetched: int = Field(description="Pages successfully fetched")
total_evidence_items: int = Field(description="Final evidence count")
execution_time_ms: float = Field(description="Total execution time")

81
ai_platform/modules/web/src/web/schemas/image_search.py Normal file
View file

@ -0,0 +1,81 @@
"""Image search schemas for image search endpoints."""
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
class ImageSearchResult(BaseModel):
"""A single image search result."""
model_config = ConfigDict(extra="forbid")
query: str = Field(description="The query that produced this result")
image_url: str = Field(description="Direct URL to the image")
thumbnail_url: str = Field(default="", description="URL to the thumbnail")
source_url: str = Field(description="URL of the page containing the image")
title: str = Field(description="Title of the image or page")
description: str = Field(default="", description="Description of the image")
width: int | None = Field(default=None, description="Image width in pixels")
height: int | None = Field(default=None, description="Image height in pixels")
publisher: str = Field(default="", description="Publisher or source domain")
rank: int = Field(ge=1, description="Result rank (1-indexed)")
class ImageSearchRequest(BaseModel):
"""Request schema for image search."""
model_config = ConfigDict(extra="forbid")
queries: list[str] = Field(
min_length=1,
max_length=10,
description="List of search queries (1-10 queries)",
)
max_results: int = Field(
default=50,
ge=1,
le=200,
description="Maximum results per query",
)
language: str = Field(
default="en",
description="Search language (ISO 639-1 code)",
)
country: str = Field(
default="US",
description="Search country (ISO 3166-1 alpha-2 code)",
)
safe_search: Literal["off", "strict"] = Field(
default="strict",
description="Safe search filter level (off or strict)",
)
spellcheck: bool = Field(
default=True,
description="Enable spellcheck for queries",
)
@field_validator("queries")
@classmethod
def validate_queries(cls, v: list[str]) -> list[str]:
"""Validate that queries are not empty."""
for i, query in enumerate(v):
if not query or not query.strip():
raise ValueError(f"Query at index {i} cannot be empty")
if len(query) > 500:
raise ValueError(
f"Query at index {i} exceeds maximum length of 500 characters"
)
return [q.strip() for q in v]
class ImageSearchResponse(BaseModel):
"""Response schema for image search."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Unique request ID for tracing")
results: list[ImageSearchResult] = Field(description="Image search results")
total_results: int = Field(description="Total number of results returned")
execution_time_ms: float = Field(description="Execution time in milliseconds")
queries_processed: int = Field(description="Number of queries processed")

106
ai_platform/modules/web/src/web/schemas/search.py Normal file
View file

@ -0,0 +1,106 @@
"""Search schemas for web search endpoints."""
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
class SearchResult(BaseModel):
"""A single search result."""
model_config = ConfigDict(extra="forbid")
query: str = Field(description="The query that produced this result")
url: str = Field(description="URL of the result")
title: str = Field(description="Title of the result")
snippet: str = Field(description="Text snippet from the result")
rank: int = Field(ge=1, description="Result rank (1-indexed)")
site: str = Field(description="Domain of the result")
published_at: str | None = Field(
default=None, description="Publication date if available"
)
class SearchRequest(BaseModel):
"""Request schema for web search."""
model_config = ConfigDict(extra="forbid")
queries: list[str] = Field(
min_length=1,
max_length=10,
description="List of search queries (1-10 queries)",
)
max_results: int = Field(
default=10,
ge=1,
le=100,
description="Maximum results per query",
)
site_allowlist: list[str] | None = Field(
default=None,
description="Only include results from these domains",
)
site_blocklist: list[str] | None = Field(
default=None,
description="Exclude results from these domains",
)
language: str = Field(
default="auto",
description="Search language (ISO 639-1 code, or 'auto' for auto-detect)",
)
country: str = Field(
default="US",
description="Search country (ISO 3166-1 alpha-2 code)",
)
freshness: Literal["day", "week", "month", "year"] | None = Field(
default=None,
description="Filter results by freshness",
)
safe_search: Literal["off", "moderate", "strict"] = Field(
default="moderate",
description="Safe search filter level",
)
@field_validator("queries")
@classmethod
def validate_queries(cls, v: list[str]) -> list[str]:
"""Validate that queries are not empty."""
for i, query in enumerate(v):
if not query or not query.strip():
raise ValueError(f"Query at index {i} cannot be empty")
if len(query) > 500:
raise ValueError(
f"Query at index {i} exceeds maximum length of 500 characters"
)
return [q.strip() for q in v]
@field_validator("site_allowlist", "site_blocklist")
@classmethod
def validate_site_list(cls, v: list[str] | None) -> list[str] | None:
"""Validate and clean site lists."""
if v is None:
return None
cleaned = []
for site in v:
site = site.strip().lower()
if site.startswith("http://"):
site = site[7:]
elif site.startswith("https://"):
site = site[8:]
site = site.rstrip("/")
if site:
cleaned.append(site)
return cleaned if cleaned else None
class SearchResponse(BaseModel):
"""Response schema for web search."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Unique request ID for tracing")
results: list[SearchResult] = Field(description="Search results")
total_results: int = Field(description="Total number of results returned")
execution_time_ms: float = Field(description="Execution time in milliseconds")
queries_processed: int = Field(description="Number of queries processed")

128
ai_platform/modules/web/src/web/schemas/vision.py Normal file
View file

@ -0,0 +1,128 @@
"""Vision schemas - Screenshot + Vision LLM extraction."""
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
from web.schemas.common import FailedUrl
from web.validation import validate_url
class ImageContext(BaseModel):
"""Extracted context from an image."""
model_config = ConfigDict(extra="forbid")
image_url: str | None = Field(default=None, description="Image source URL")
alt_text: str | None = Field(default=None, description="Alt text if available")
description: str = Field(description="LLM-generated description")
extracted_text: str | None = Field(default=None, description="Text found in image")
relevance_score: float | None = Field(
default=None, description="Relevance to context"
)
class VisionExtractRequest(BaseModel):
"""Request schema for vision-based extraction."""
model_config = ConfigDict(extra="forbid")
urls: list[str] = Field(
min_length=1,
max_length=10,
description="URLs to process (1-10, vision is expensive)",
)
@field_validator("urls")
@classmethod
def validate_urls(cls, v: list[str]) -> list[str]:
"""Validate all URLs for SSRF protection."""
for url in v:
validate_url(url)
return v
# What to extract
extract_page_text: bool = Field(
default=True,
description="Extract main text from page screenshot",
)
extract_images: bool = Field(
default=True,
description="Analyze images found on page",
)
# Context for relevance scoring
context_query: str | None = Field(
default=None,
description="Query/claim for relevance scoring",
)
# Vision model settings
model: str | None = Field(
default=None,
description="Vision model to use (default from config)",
)
max_tokens: int = Field(
default=2000,
ge=100,
le=8000,
description="Max tokens for vision response",
)
# Screenshot options
full_page: bool = Field(
default=True,
description="Screenshot full page or viewport",
)
max_screenshots: int = Field(
default=3,
ge=1,
le=10,
description="Max screenshots per page (for long pages)",
)
# LLM provider fallback
llm_provider: Literal["local", "openai", "anthropic", "auto"] = Field(
default="auto",
description="LLM provider (auto = local first, fallback to external)",
)
class VisionPageResult(BaseModel):
"""Vision extraction result for a single page."""
model_config = ConfigDict(extra="forbid")
url: str = Field(description="Page URL")
title: str | None = Field(default=None, description="Page title")
extracted_text: str = Field(description="Text extracted via vision")
images: list[ImageContext] = Field(
default_factory=list,
description="Analyzed images from page",
)
extraction_method: Literal["vision"] = Field(default="vision")
model_used: str = Field(description="Vision model used")
llm_provider: str = Field(description="LLM provider used")
screenshots_processed: int = Field(description="Number of screenshots processed")
tokens_used: int = Field(description="Total tokens used")
extraction_time_ms: float = Field(description="Extraction time")
class VisionExtractResponse(BaseModel):
"""Response schema for vision extraction."""
model_config = ConfigDict(extra="forbid")
request_id: str = Field(description="Request ID for tracing")
pages: list[VisionPageResult] = Field(description="Extracted pages")
total_processed: int = Field(description="Pages processed")
total_failed: int = Field(description="Pages failed")
total_tokens_used: int = Field(description="Total tokens across all pages")
execution_time_ms: float = Field(description="Total execution time")
failed_urls: list[FailedUrl] = Field(
default_factory=list,
description="URLs that failed",
)

5
ai_platform/modules/web/src/web/search/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Search submodule — search provider protocol."""
from web.search.protocol import SearchProvider
__all__ = ["SearchProvider"]

169
ai_platform/modules/web/src/web/search/brave.py Normal file
View file

@ -0,0 +1,169 @@
"""Brave Search API client."""
import asyncio
import time
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.brave")
_BRAVE_BASE = "https://api.search.brave.com/res/v1/web/search"
class BraveSearchClient:
"""Search client using Brave Search API."""
provider_name: str = "brave"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._api_key = settings.brave_api_key
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=15.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip",
"X-Subscription-Token": self._api_key or "",
},
)
return self._client
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
if not self._api_key:
return False
try:
client = await self._get_client()
resp = await client.get(
_BRAVE_BASE,
params={"q": "test", "count": 1},
timeout=5.0,
)
return resp.status_code == 200
except Exception:
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
if not self._api_key:
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)
start_time = time.perf_counter()
tasks = [self._search_single(q, request) for q in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.warning(
"Brave search failed for '%s': %s", request.queries[i], result
)
else:
all_results.extend(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self, query: str, request: SearchRequest
) -> list[SearchResult]:
client = await self._get_client()
# Build query with site filters
effective_query = query
if request.site_allowlist:
site_filter = " OR ".join(f"site:{s}" for s in request.site_allowlist)
effective_query = f"{query} ({site_filter})"
if request.site_blocklist:
for site in request.site_blocklist:
effective_query += f" -site:{site}"
params = {
"q": effective_query,
"count": min(request.max_results, 20),
}
# Country
if request.country and request.country.lower() not in ("auto", ""):
params["country"] = request.country.lower()
# Safe search
safesearch_map = {"off": "off", "moderate": "moderate", "strict": "strict"}
params["safesearch"] = safesearch_map.get(request.safe_search, "moderate")
# Freshness
if request.freshness:
freshness_map = {
"day": "pd",
"week": "pw",
"month": "pm",
"year": "py",
}
if request.freshness in freshness_map:
params["freshness"] = freshness_map[request.freshness]
resp = await client.get(_BRAVE_BASE, params=params)
resp.raise_for_status()
data = resp.json()
results: list[SearchResult] = []
web_results = data.get("web", {}).get("results", [])
for rank, item in enumerate(web_results, start=1):
url = item.get("url", "")
parsed = urlparse(url)
results.append(
SearchResult(
query=query,
url=url,
title=item.get("title", ""),
snippet=item.get("description", ""),
rank=rank,
site=parsed.netloc,
published_at=item.get("page_age"),
)
)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)

139
ai_platform/modules/web/src/web/search/cloak.py Normal file
View file

@ -0,0 +1,139 @@
"""Cloak HTTP client — invokes the `didiAI-cloak` standalone scraping service.
The `cloak` service (ai_platform/modules/cloak) wraps CloakBrowser (patched
stealth Chromium) behind a FastAPI endpoint. It scrapes Google/Bing/DDG SERPs
and returns parsed organic results. We invoke it as a tier-3 search fallback
in `orchestrator._run_search_stage` when SearXNG + paid rotation return thin
results (often the case on very niche or recent queries).
This client matches the SearchProvider protocol so it could in theory be
slotted into MultiSearchClient — but the intended use is tier-3 fallback only,
not part of the per-call rotation.
"""
from __future__ import annotations
import time
import uuid
from typing import Any
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.cloak")
class CloakHTTPClient:
"""Thin HTTP wrapper around the cloak scraping service."""
provider_name: str = "cloak"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._url = (settings.cloak_url or "http://didiAI-cloak:8770").rstrip("/")
self._timeout = settings.cloak_timeout_sec or 20
self._token = settings.cloak_auth_token or ""
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
headers = {"Accept": "application/json"}
if self._token:
headers["Authorization"] = f"Bearer {self._token}"
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=self._timeout, write=5.0, pool=self._timeout),
headers=headers,
)
return self._client
async def close(self) -> None:
if self._client and not self._client.is_closed:
await self._client.aclose()
async def health(self) -> bool:
"""Quick readiness probe — returns True if cloak is up & has browsers ready."""
try:
client = await self._get_client()
r = await client.get(f"{self._url}/health", timeout=3.0)
if r.status_code != 200:
return False
j = r.json()
return j.get("status") in ("healthy", "degraded")
except Exception as e:
logger.debug("cloak health check failed: %s", e)
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
"""Run a search through cloak service.
Maps web.SearchRequest → cloak's POST /v1/search body and converts the
response back to web.SearchResponse so it composes with other providers.
"""
client = await self._get_client()
body: dict[str, Any] = {
"queries": list(request.queries[:5]), # service caps at 5 anyway
"engines": ["google", "bing", "ddg"],
"max_results_per_engine": min(request.max_results or 10, 30),
}
if request.language and request.language != "auto":
body["language"] = request.language
t0 = time.perf_counter()
try:
r = await client.post(f"{self._url}/v1/search", json=body)
r.raise_for_status()
data = r.json()
except Exception as e:
elapsed = (time.perf_counter() - t0) * 1000
logger.warning("cloak search failed: %s", e)
return SearchResponse(
request_id=request_id or str(uuid.uuid4()),
results=[],
total_results=0,
execution_time_ms=elapsed,
queries_processed=len(body["queries"]),
)
# Map cloak results into web.SearchResult shape (extra='forbid' strict)
results: list[SearchResult] = []
rank_counter: dict[str, int] = {}
for r in data.get("results", []):
try:
q = r.get("query", body["queries"][0])
rank_counter[q] = rank_counter.get(q, 0) + 1
url = r["url"]
host = urlparse(url).hostname or ""
results.append(SearchResult(
query=q,
url=url,
title=(r.get("title") or "")[:300],
snippet=(r.get("snippet") or "")[:500],
rank=rank_counter[q],
site=host,
))
except Exception:
continue
elapsed_ms = (time.perf_counter() - t0) * 1000
stats = data.get("stats", [])
blocked = sum(1 for s in stats if s.get("blocked"))
captcha = sum(1 for s in stats if s.get("captcha"))
logger.info(
"cloak[req=%s] q=%d -> %d results in %dms (blocked=%d, captcha=%d)",
request_id or "-", len(body["queries"]), len(results), int(elapsed_ms), blocked, captcha,
)
return SearchResponse(
request_id=request_id or str(uuid.uuid4()),
results=results,
total_results=len(results),
execution_time_ms=elapsed_ms,
queries_processed=len(body["queries"]),
)

158
ai_platform/modules/web/src/web/search/exa.py Normal file
View file

@ -0,0 +1,158 @@
"""Exa search client — neural search API."""
import asyncio
import time
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.exa")
_EXA_BASE = "https://api.exa.ai/search"
class ExaClient:
"""Search client using Exa neural search API."""
provider_name: str = "exa"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._api_key = settings.exa_api_key
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=15.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
headers={
"x-api-key": self._api_key or "",
"Content-Type": "application/json",
},
)
return self._client
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
if not self._api_key:
return False
try:
client = await self._get_client()
resp = await client.post(
_EXA_BASE,
json={"query": "test", "type": "auto", "numResults": 1},
timeout=5.0,
)
return resp.status_code == 200
except Exception:
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
if not self._api_key:
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)
start_time = time.perf_counter()
tasks = [self._search_single(q, request) for q in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.warning(
"Exa search failed for '%s': %s", request.queries[i], result
)
else:
all_results.extend(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self, query: str, request: SearchRequest
) -> list[SearchResult]:
client = await self._get_client()
payload = {
"query": query,
"type": "auto",
"numResults": min(request.max_results, 10),
"contents": {
"highlights": {"maxCharacters": 4000},
},
}
# Domain filtering
if request.site_allowlist:
payload["includeDomains"] = list(request.site_allowlist)
if request.site_blocklist:
payload["excludeDomains"] = list(request.site_blocklist)
# Category for news freshness
if request.freshness in ("day", "week"):
payload["category"] = "news"
resp = await client.post(_EXA_BASE, json=payload)
resp.raise_for_status()
data = resp.json()
results: list[SearchResult] = []
for rank, item in enumerate(data.get("results", []), start=1):
url = item.get("url", "")
parsed = urlparse(url)
# Build snippet from highlights or text
highlights = item.get("highlights", [])
snippet = " ".join(highlights) if highlights else item.get("text", "")[:500]
results.append(
SearchResult(
query=query,
url=url,
title=item.get("title", ""),
snippet=snippet,
rank=rank,
site=parsed.netloc,
published_at=item.get("publishedDate"),
)
)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)

150
ai_platform/modules/web/src/web/search/linkup.py Normal file
View file

@ -0,0 +1,150 @@
"""LinkUp search client — web content API."""
import asyncio
import time
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.linkup")
_LINKUP_BASE = "https://api.linkup.so/v1/search"
class LinkUpClient:
"""Search client using LinkUp API."""
provider_name: str = "linkup"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._api_key = settings.linkup_api_key
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=15.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
headers={
"Authorization": f"Bearer {self._api_key}",
"Content-Type": "application/json",
},
)
return self._client
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
if not self._api_key:
return False
try:
client = await self._get_client()
resp = await client.post(
_LINKUP_BASE,
json={"q": "test", "depth": "standard", "outputType": "searchResults"},
timeout=5.0,
)
return resp.status_code == 200
except Exception:
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
if not self._api_key:
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)
start_time = time.perf_counter()
tasks = [self._search_single(q, request) for q in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.warning(
"LinkUp search failed for '%s': %s", request.queries[i], result
)
else:
all_results.extend(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self, query: str, request: SearchRequest
) -> list[SearchResult]:
client = await self._get_client()
# Build query with site filters
effective_query = query
if request.site_allowlist:
site_filter = " OR ".join(f"site:{s}" for s in request.site_allowlist)
effective_query = f"{query} ({site_filter})"
if request.site_blocklist:
for site in request.site_blocklist:
effective_query += f" -site:{site}"
payload = {
"q": effective_query,
"depth": "standard",
"outputType": "searchResults",
"includeImages": False,
}
resp = await client.post(_LINKUP_BASE, json=payload)
resp.raise_for_status()
data = resp.json()
results: list[SearchResult] = []
for rank, item in enumerate(data.get("results", []), start=1):
url = item.get("url", "")
parsed = urlparse(url)
results.append(
SearchResult(
query=query,
url=url,
title=item.get("name", ""),
snippet=item.get("content", ""),
rank=rank,
site=parsed.netloc,
published_at=None,
)
)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)

178
ai_platform/modules/web/src/web/search/multi.py Normal file
View file

@ -0,0 +1,178 @@
"""Multi-provider search client — SearXNG always + one paid provider rotated."""
import asyncio
import time
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.multi")
class MultiSearchClient:
"""Combines SearXNG (always) with paid providers on round-robin rotation.
Each search() call uses SearXNG + ONE paid provider. The paid provider
rotates on every call to spread API credits evenly.
Satisfies the SearchProvider protocol.
"""
provider_name: str = "multi"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
# Primary provider — always runs (free / self-hosted)
from web.metasearch.client import SearXNGClient
self._primary = SearXNGClient(settings)
# Paid providers — rotated round-robin, one per search() call
self._paid: list = []
self._paid_index: int = 0
if settings.serpapi_api_key:
from web.search.serpapi import SerpAPIClient
self._paid.append(SerpAPIClient(settings))
logger.info("SerpAPI provider enabled (paid, rotated)")
if settings.tavily_api_key:
from web.search.tavily import TavilyClient
self._paid.append(TavilyClient(settings))
logger.info("Tavily provider enabled (paid, rotated)")
if settings.exa_api_key:
from web.search.exa import ExaClient
self._paid.append(ExaClient(settings))
logger.info("Exa provider enabled (paid, rotated)")
if settings.linkup_api_key:
from web.search.linkup import LinkUpClient
self._paid.append(LinkUpClient(settings))
logger.info("LinkUp provider enabled (paid, rotated)")
if settings.brave_api_key:
from web.search.brave import BraveSearchClient
self._paid.append(BraveSearchClient(settings))
logger.info("Brave Search provider enabled (paid, rotated)")
logger.info(
"MultiSearchClient: primary=searxng, paid=%d providers [%s], rotation=round-robin",
len(self._paid),
", ".join(p.provider_name for p in self._paid),
)
def _next_paid(self) -> object | None:
"""Get next paid provider in round-robin rotation."""
if not self._paid:
return None
provider = self._paid[self._paid_index % len(self._paid)]
self._paid_index += 1
return provider
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
"""Run SearXNG + one rotated paid provider in parallel, deduplicate.
Args:
request: Search request.
request_id: Optional request ID.
Returns:
Combined, deduplicated SearchResponse.
"""
start_time = time.perf_counter()
# Always run primary (SearXNG)
tasks = [self._primary.search(request, request_id=request_id)]
providers_used = ["searxng"]
# Add one paid provider (round-robin)
paid = self._next_paid()
if paid is not None:
tasks.append(paid.search(request, request_id=request_id))
providers_used.append(paid.provider_name)
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
# Combine and deduplicate by URL (first seen wins)
all_results: list[SearchResult] = []
seen_urls: set[str] = set()
for i, result in enumerate(results_or_errors):
name = providers_used[i]
if isinstance(result, Exception):
logger.warning("Provider %s failed: %s", name, result)
continue
count = 0
for sr in result.results:
if sr.url not in seen_urls:
seen_urls.add(sr.url)
all_results.append(sr)
count += 1
logger.debug(
"Provider %s: %d results (%d new)", name, result.total_results, count
)
execution_time_ms = (time.perf_counter() - start_time) * 1000
logger.info(
"Search [%s]: %d results in %.0fms",
"+".join(providers_used),
len(all_results),
execution_time_ms,
)
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
"""Image search — delegates to SearXNG (primary)."""
try:
return await self._primary.image_search(request, request_id=request_id)
except Exception as e:
logger.warning("Image search failed: %s", e)
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=len(request.queries),
)
async def health_check(self) -> bool:
"""Healthy if primary (SearXNG) is healthy."""
try:
return await self._primary.health_check()
except Exception:
return False
async def close(self) -> None:
await self._primary.close()
for provider in self._paid:
try:
await provider.close()
except Exception as e:
logger.warning("Error closing %s: %s", provider.provider_name, e)

306
ai_platform/modules/web/src/web/search/paid.py Normal file
View file

@ -0,0 +1,306 @@
"""Paid-only search client — rotates SerpAPI, Tavily, Exa, LinkUp, Brave.
Supports three rotation strategies, configurable at runtime:
round-robin: one provider per request, rotated
parallel: all enabled providers in parallel, results deduped
priority: try providers in configured order, fallback on empty/error
"""
import asyncio
import time
from typing import TYPE_CHECKING
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
if TYPE_CHECKING:
from web.runtime_config import RuntimeConfigClient
logger = get_logger("search.paid")
class PaidSearchClient:
"""Premium-tier search client with runtime-configurable rotation."""
provider_name: str = "paid"
def __init__(
self,
settings: WebSettings,
runtime_config: "RuntimeConfigClient | None" = None,
) -> None:
self.settings = settings
self.runtime_config = runtime_config
self._index: int = 0
# Instantiate every provider that has a key — enable/disable is runtime
self._providers: dict[str, object] = {}
if settings.serpapi_api_key:
from web.search.serpapi import SerpAPIClient
self._providers["serpapi"] = SerpAPIClient(settings)
if settings.tavily_api_key:
from web.search.tavily import TavilyClient
self._providers["tavily"] = TavilyClient(settings)
if settings.exa_api_key:
from web.search.exa import ExaClient
self._providers["exa"] = ExaClient(settings)
if settings.linkup_api_key:
from web.search.linkup import LinkUpClient
self._providers["linkup"] = LinkUpClient(settings)
if settings.brave_api_key:
from web.search.brave import BraveSearchClient
self._providers["brave"] = BraveSearchClient(settings)
logger.info(
"PaidSearchClient: %d providers registered [%s]",
len(self._providers),
", ".join(self._providers.keys()),
)
# ------------------------------------------------------------------
# Runtime config helpers
# ------------------------------------------------------------------
def _enabled_providers(self) -> list[tuple[str, object]]:
"""Return list of (name, client) for providers currently enabled."""
if self.runtime_config is None:
return list(self._providers.items())
out = []
for name, client in self._providers.items():
if self.runtime_config.get_bool(
f"web.providers.{name}.enabled", default=True
):
out.append((name, client))
return out
def _strategy(self) -> str:
if self.runtime_config is None:
return "round-robin"
return self.runtime_config.get_str(
"web.premium.strategy", default="round-robin"
)
def _priority_order(self) -> list[str]:
if self.runtime_config is None:
return ["serpapi", "brave", "tavily", "linkup", "exa"]
raw = self.runtime_config.get_str(
"web.premium.priority_order",
default="serpapi,brave,tavily,linkup",
)
return [p.strip() for p in raw.split(",") if p.strip()]
# ------------------------------------------------------------------
# Search dispatch
# ------------------------------------------------------------------
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
enabled = self._enabled_providers()
if not enabled:
logger.warning("PaidSearchClient: no providers enabled")
return self._empty(request, request_id)
strategy = self._strategy()
if strategy == "parallel":
return await self._search_parallel(enabled, request, request_id)
if strategy == "priority":
return await self._search_priority(enabled, request, request_id)
return await self._search_round_robin(enabled, request, request_id)
async def _search_round_robin(
self,
enabled: list[tuple[str, object]],
request: SearchRequest,
request_id: str | None,
) -> SearchResponse:
"""Pick one provider per request, rotate, fallback on empty."""
start_time = time.perf_counter()
start_idx = self._index % len(enabled)
self._index = (self._index + 1) % max(1, len(enabled))
last_error: Exception | None = None
for offset in range(len(enabled)):
idx = (start_idx + offset) % len(enabled)
name, provider = enabled[idx]
try:
result: SearchResponse = await provider.search( # type: ignore[attr-defined]
request, request_id=request_id
)
if result.results:
duration = (time.perf_counter() - start_time) * 1000
logger.info(
"Search [%s/round-robin]: %d results in %.0fms",
name,
len(result.results),
duration,
)
return SearchResponse(
request_id=request_id or "-",
results=result.results,
total_results=result.total_results,
execution_time_ms=round(duration, 2),
queries_processed=result.queries_processed,
)
logger.warning("Provider %s returned 0 results, trying next", name)
except Exception as e:
logger.warning("Provider %s failed: %s", name, e)
last_error = e
if last_error:
logger.error("All paid providers failed. Last: %s", last_error)
return self._empty(request, request_id, start_time)
async def _search_parallel(
self,
enabled: list[tuple[str, object]],
request: SearchRequest,
request_id: str | None,
) -> SearchResponse:
"""Hit all enabled providers at once, dedupe by URL."""
start_time = time.perf_counter()
tasks = [
provider.search(request, request_id=request_id) # type: ignore[attr-defined]
for _, provider in enabled
]
results = await asyncio.gather(*tasks, return_exceptions=True)
seen: set[str] = set()
combined: list[SearchResult] = []
providers_hit: list[str] = []
for (name, _), result in zip(enabled, results, strict=False):
if isinstance(result, BaseException):
logger.warning("Provider %s failed: %s", name, result)
continue
response: SearchResponse = result # type: ignore[assignment]
new_count = 0
for r in response.results:
if r.url not in seen:
seen.add(r.url)
combined.append(r)
new_count += 1
providers_hit.append(f"{name}:{new_count}")
duration = (time.perf_counter() - start_time) * 1000
logger.info(
"Search [parallel: %s]: %d unique results in %.0fms",
", ".join(providers_hit),
len(combined),
duration,
)
return SearchResponse(
request_id=request_id or "-",
results=combined,
total_results=len(combined),
execution_time_ms=round(duration, 2),
queries_processed=len(request.queries),
)
async def _search_priority(
self,
enabled: list[tuple[str, object]],
request: SearchRequest,
request_id: str | None,
) -> SearchResponse:
"""Try providers in configured priority order, fallback on empty/error."""
start_time = time.perf_counter()
enabled_map = dict(enabled)
ordered = [name for name in self._priority_order() if name in enabled_map]
# Append any enabled providers not in the priority list
for name in enabled_map:
if name not in ordered:
ordered.append(name)
last_error: Exception | None = None
for name in ordered:
provider = enabled_map[name]
try:
result: SearchResponse = await provider.search( # type: ignore[attr-defined]
request, request_id=request_id
)
if result.results:
duration = (time.perf_counter() - start_time) * 1000
logger.info(
"Search [%s/priority]: %d results in %.0fms",
name,
len(result.results),
duration,
)
return SearchResponse(
request_id=request_id or "-",
results=result.results,
total_results=result.total_results,
execution_time_ms=round(duration, 2),
queries_processed=result.queries_processed,
)
except Exception as e:
logger.warning("Provider %s failed: %s", name, e)
last_error = e
if last_error:
logger.error("All paid providers failed. Last: %s", last_error)
return self._empty(request, request_id, start_time)
# ------------------------------------------------------------------
# Utilities
# ------------------------------------------------------------------
def _empty(
self,
request: SearchRequest,
request_id: str | None,
start_time: float | None = None,
) -> SearchResponse:
duration = (time.perf_counter() - start_time) * 1000 if start_time else 0.0
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=round(duration, 2),
queries_processed=len(request.queries),
)
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=len(request.queries),
)
async def health_check(self) -> bool:
if not self._providers:
return False
checks = await asyncio.gather(
*[p.health_check() for p in self._providers.values()], # type: ignore[attr-defined]
return_exceptions=True,
)
return any(c is True for c in checks)
async def close(self) -> None:
for name, provider in self._providers.items():
try:
await provider.close() # type: ignore[attr-defined]
except Exception as e:
logger.warning("Error closing %s: %s", name, e)

23
ai_platform/modules/web/src/web/search/protocol.py Normal file
View file

@ -0,0 +1,23 @@
"""Search provider protocol — contract shared by all search backends."""
from typing import Protocol, runtime_checkable
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse
@runtime_checkable
class SearchProvider(Protocol):
"""Protocol that all search backends must satisfy."""
async def search(
self, request: SearchRequest, *, request_id: str | None = None
) -> SearchResponse: ...
async def image_search(
self, request: ImageSearchRequest, *, request_id: str | None = None
) -> ImageSearchResponse: ...
async def health_check(self) -> bool: ...
async def close(self) -> None: ...

164
ai_platform/modules/web/src/web/search/serpapi.py Normal file
View file

@ -0,0 +1,164 @@
"""SerpAPI search client — wraps Google/Bing via SerpAPI."""
import asyncio
import time
from typing import Any
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import (
ImageSearchRequest,
ImageSearchResponse,
)
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.serpapi")
_SERPAPI_BASE = "https://serpapi.com/search"
class SerpAPIClient:
"""Search client using SerpAPI (Google results without CAPTCHA)."""
provider_name: str = "serpapi"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._api_key = settings.serpapi_api_key
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=15.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
)
return self._client
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
if not self._api_key:
return False
try:
client = await self._get_client()
resp = await client.get(
_SERPAPI_BASE,
params={"q": "test", "api_key": self._api_key, "engine": "google", "num": 1},
timeout=5.0,
)
return resp.status_code == 200
except Exception:
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
if not self._api_key:
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)
start_time = time.perf_counter()
tasks = [self._search_single(q, request) for q in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.warning("SerpAPI search failed for '%s': %s", request.queries[i], result)
else:
all_results.extend(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self, query: str, request: SearchRequest
) -> list[SearchResult]:
client = await self._get_client()
# Build query with site filters
effective_query = query
if request.site_allowlist:
site_filter = " OR ".join(f"site:{s}" for s in request.site_allowlist)
effective_query = f"{query} ({site_filter})"
if request.site_blocklist:
for site in request.site_blocklist:
effective_query += f" -site:{site}"
params: dict[str, Any] = {
"q": effective_query,
"api_key": self._api_key,
"engine": "google",
"num": min(request.max_results, 20),
}
# Language / country
lang = request.language if request.language not in ("auto", "") else "en"
params["hl"] = lang
if request.country and request.country.lower() not in ("auto", ""):
params["gl"] = request.country.lower()
# Safe search
if request.safe_search == "strict":
params["safe"] = "active"
# Freshness
freshness_map = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m", "year": "qdr:y"}
if request.freshness and request.freshness in freshness_map:
params["tbs"] = freshness_map[request.freshness]
resp = await client.get(_SERPAPI_BASE, params=params)
resp.raise_for_status()
data = resp.json()
results: list[SearchResult] = []
for rank, item in enumerate(data.get("organic_results", []), start=1):
url = item.get("link", "")
parsed = urlparse(url)
results.append(
SearchResult(
query=query,
url=url,
title=item.get("title", ""),
snippet=item.get("snippet", ""),
rank=rank,
site=parsed.netloc,
published_at=item.get("date"),
)
)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
# Not implemented — SerpAPI image search costs extra
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)

156
ai_platform/modules/web/src/web/search/tavily.py Normal file
View file

@ -0,0 +1,156 @@
"""Tavily search client — AI-optimized search API."""
import asyncio
import time
from urllib.parse import urlparse
import httpx
from web.config import WebSettings
from web.logging import get_logger
from web.schemas.image_search import ImageSearchRequest, ImageSearchResponse
from web.schemas.search import SearchRequest, SearchResponse, SearchResult
logger = get_logger("search.tavily")
_TAVILY_BASE = "https://api.tavily.com/search"
class TavilyClient:
"""Search client using Tavily API (optimized for AI/fact-checking)."""
provider_name: str = "tavily"
def __init__(self, settings: WebSettings) -> None:
self.settings = settings
self._api_key = settings.tavily_api_key
self._client: httpx.AsyncClient | None = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(connect=5.0, read=15.0, write=10.0, pool=15.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=3),
)
return self._client
async def close(self) -> None:
if self._client is not None and not self._client.is_closed:
await self._client.aclose()
self._client = None
async def health_check(self) -> bool:
if not self._api_key:
return False
try:
client = await self._get_client()
resp = await client.post(
_TAVILY_BASE,
json={"query": "test", "api_key": self._api_key, "max_results": 1},
timeout=5.0,
)
return resp.status_code == 200
except Exception:
return False
async def search(
self,
request: SearchRequest,
request_id: str | None = None,
) -> SearchResponse:
if not self._api_key:
return SearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)
start_time = time.perf_counter()
tasks = [self._search_single(q, request) for q in request.queries]
results_or_errors = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for i, result in enumerate(results_or_errors):
if isinstance(result, Exception):
logger.warning(
"Tavily search failed for '%s': %s", request.queries[i], result
)
else:
all_results.extend(result)
execution_time_ms = (time.perf_counter() - start_time) * 1000
return SearchResponse(
request_id=request_id or "-",
results=all_results,
total_results=len(all_results),
execution_time_ms=round(execution_time_ms, 2),
queries_processed=len(request.queries),
)
async def _search_single(
self, query: str, request: SearchRequest
) -> list[SearchResult]:
client = await self._get_client()
# Build query with site filters
effective_query = query
if request.site_allowlist:
site_filter = " OR ".join(f"site:{s}" for s in request.site_allowlist)
effective_query = f"{query} ({site_filter})"
if request.site_blocklist:
for site in request.site_blocklist:
effective_query += f" -site:{site}"
payload = {
"query": effective_query,
"api_key": self._api_key,
"max_results": min(request.max_results, 20),
"search_depth": "basic",
"include_answer": False,
}
# Freshness mapping
if request.freshness:
freshness_map = {"day": "day", "week": "week", "month": "month", "year": "year"}
if request.freshness in freshness_map:
payload["days"] = {"day": 1, "week": 7, "month": 30, "year": 365}[
request.freshness
]
resp = await client.post(_TAVILY_BASE, json=payload)
resp.raise_for_status()
data = resp.json()
results: list[SearchResult] = []
for rank, item in enumerate(data.get("results", []), start=1):
url = item.get("url", "")
parsed = urlparse(url)
results.append(
SearchResult(
query=query,
url=url,
title=item.get("title", ""),
snippet=item.get("content", ""),
rank=rank,
site=parsed.netloc,
published_at=item.get("published_date"),
)
)
return results
async def image_search(
self,
request: ImageSearchRequest,
request_id: str | None = None,
) -> ImageSearchResponse:
# Tavily doesn't have image search
return ImageSearchResponse(
request_id=request_id or "-",
results=[],
total_results=0,
execution_time_ms=0,
queries_processed=0,
)

154
ai_platform/modules/web/src/web/validation.py Normal file
View file

@ -0,0 +1,154 @@
"""URL validation utilities for SSRF protection."""
import asyncio
import ipaddress
import socket
from urllib.parse import urlparse
from web.logging import get_logger
from web.schemas.common import FailedUrl
logger = get_logger("validation")
# Allowed URL schemes
_ALLOWED_SCHEMES = {"http", "https"}
# Blocked hostnames
_BLOCKED_HOSTNAMES = {
"localhost",
"localhost.localdomain",
"metadata.google.internal",
"169.254.169.254",
}
def _is_private_ip(hostname: str) -> bool:
"""Check if a hostname resolves to a private/reserved IP.
Args:
hostname: Hostname or IP address string.
Returns:
True if the address is private or reserved.
"""
try:
addr = ipaddress.ip_address(hostname)
return addr.is_private or addr.is_reserved or addr.is_loopback
except ValueError:
# Not a raw IP - will be checked via DNS later
return False
def validate_url(url: str) -> str:
"""Validate a URL for safe fetching (cheap synchronous checks only).
Performs fast SSRF checks: scheme, hostname format, blocklist, raw IP.
Does NOT do DNS resolution (use validate_url_dns for that).
Args:
url: The URL to validate.
Returns:
The validated URL (unchanged).
Raises:
ValueError: If the URL is not safe to fetch.
"""
try:
parsed = urlparse(url)
except Exception as e:
raise ValueError(f"Invalid URL: {e}") from e
# Check scheme
if parsed.scheme not in _ALLOWED_SCHEMES:
raise ValueError(
f"URL scheme '{parsed.scheme}' is not allowed. "
f"Only {_ALLOWED_SCHEMES} are permitted."
)
# Check hostname exists
hostname = parsed.hostname
if not hostname:
raise ValueError("URL must include a hostname.")
hostname_lower = hostname.lower()
# Check blocked hostnames
if hostname_lower in _BLOCKED_HOSTNAMES:
raise ValueError(f"URL hostname '{hostname}' is not allowed.")
# Check for private/reserved IPs (raw IP literals only)
if _is_private_ip(hostname):
raise ValueError(f"URL points to a private/reserved address: {hostname}")
return url
async def validate_url_dns(url: str) -> str:
"""Async DNS resolution check for SSRF protection.
Resolves the URL's hostname and checks that it does not point to
a private/reserved IP address. Should be called after validate_url().
Args:
url: The URL to check (must already pass validate_url).
Returns:
The validated URL (unchanged).
Raises:
ValueError: If the hostname resolves to a private/reserved address.
"""
hostname = urlparse(url).hostname
if not hostname:
return url # Already validated by validate_url
loop = asyncio.get_running_loop()
try:
resolved = await loop.getaddrinfo(
hostname, None, family=socket.AF_UNSPEC, type=socket.SOCK_STREAM
)
for _family, _, _, _, sockaddr in resolved:
ip_str = sockaddr[0]
try:
addr = ipaddress.ip_address(ip_str)
if addr.is_private or addr.is_reserved or addr.is_loopback:
raise ValueError(
f"URL hostname '{hostname}' resolves to "
f"private/reserved address {ip_str}"
)
except ValueError:
raise
except socket.gaierror as e:
logger.warning("DNS resolution failed for '%s': %s — blocking URL", hostname, e)
raise ValueError(f"URL hostname '{hostname}' could not be resolved") from e
return url
async def validate_urls_async(
urls: list[str],
) -> tuple[list[str], list[FailedUrl]]:
"""Validate a list of URLs with async DNS resolution.
Runs DNS-based SSRF checks for all URLs in parallel.
Returns both validated URLs and failed URLs with error details.
Args:
urls: URLs to validate (must already pass validate_url).
Returns:
Tuple of (validated_urls, failed_urls).
"""
results = await asyncio.gather(
*(validate_url_dns(url) for url in urls), return_exceptions=True
)
validated: list[str] = []
dns_failed: list[FailedUrl] = []
for url, result in zip(urls, results):
if isinstance(result, Exception):
logger.warning("Dropping URL %s: %s", url, result)
dns_failed.append(FailedUrl(url=url, error=str(result)))
else:
validated.append(url)
return validated, dns_failed

5
ai_platform/modules/web/src/web/vision/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""Vision submodule - Screenshot + Vision LLM extraction."""
from web.vision.client import VisionClient
__all__ = ["VisionClient"]

437
ai_platform/modules/web/src/web/vision/client.py Normal file
View file

@ -0,0 +1,437 @@
"""Vision-based content extraction using Screenshot + Vision LLM."""
import asyncio
import base64
import hashlib
import time
from datetime import datetime, timezone
from typing import Any
import httpx
from web.config import WebSettings
from web.exceptions import (
WebError,
WebTimeoutError,
)
from web.llm.provider import LLMProviderChain
from web.logging import get_logger
from web.schemas.common import FailedUrl, PageContent, PageImage
from web.schemas.vision import (
ImageContext,
VisionExtractRequest,
VisionExtractResponse,
VisionPageResult,
)
logger = get_logger("vision.client")
# Optional Playwright import
try:
from playwright.async_api import (
Browser,
TimeoutError as PlaywrightTimeout,
async_playwright,
)
HAS_PLAYWRIGHT = True
except ImportError:
HAS_PLAYWRIGHT = False
logger.warning("playwright not installed, vision functionality unavailable")
class VisionClient:
"""Client for vision-based content extraction using Screenshot + LLM."""
def __init__(
self,
settings: WebSettings,
browser: Browser | None = None,
) -> None:
"""Initialize the client.
Args:
settings: Application settings.
browser: Optional shared browser instance. If provided, the client
will not launch its own browser and will not close this one.
"""
if not HAS_PLAYWRIGHT:
raise ImportError(
"playwright is required for VisionClient. "
"Install with: uv add playwright && uv run playwright install chromium"
)
self.settings = settings
self._playwright = None
self._browser: Browser | None = browser
self._owns_browser = browser is None
self._browser_lock = asyncio.Lock()
self._http_client: httpx.AsyncClient | None = None
self._llm_chain = LLMProviderChain(
settings,
self._get_http_client,
domain="vision",
local_base_url=settings.vision_base_url or settings.llm_base_url,
)
async def _get_browser(self) -> Browser:
"""Get or create the browser instance."""
async with self._browser_lock:
if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
headless=True,
args=[
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-sandbox",
],
)
return self._browser
async def _get_http_client(self) -> httpx.AsyncClient:
"""Get or create the HTTP client for LLM API calls."""
if self._http_client is None or self._http_client.is_closed:
self._http_client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=10.0,
read=120.0, # Vision LLM can be slow
write=30.0,
pool=60.0,
),
limits=httpx.Limits(
max_connections=self.settings.llm_pool_max_connections,
max_keepalive_connections=self.settings.llm_pool_max_keepalive,
),
)
return self._http_client
async def __aenter__(self) -> "VisionClient":
return self
async def __aexit__(self, *exc: object) -> None:
await self.close()
async def close(self) -> None:
"""Close all resources."""
if self._owns_browser:
if self._browser is not None:
await self._browser.close()
self._browser = None
if self._playwright is not None:
await self._playwright.stop()
self._playwright = None
if self._http_client is not None and not self._http_client.is_closed:
await self._http_client.aclose()
self._http_client = None
async def extract(
self,
request: VisionExtractRequest,
request_id: str | None = None,
) -> VisionExtractResponse:
"""Extract content from URLs using vision.
Args:
request: Vision extraction request.
request_id: Optional request ID for tracing.
Returns:
VisionExtractResponse: Extracted content.
"""
start_time = time.perf_counter()
pages: list[VisionPageResult] = []
failed: list[FailedUrl] = []
total_tokens = 0
browser = await self._get_browser()
sem = asyncio.Semaphore(self.settings.vision_max_concurrent)
async def extract_with_sem(url: str) -> VisionPageResult | None:
async with sem:
try:
return await self._extract_single(browser, url, request)
except Exception as e:
logger.error("Failed to extract %s: %s", url, e)
failed.append(FailedUrl(url=url, error=str(e)))
return None
results = await asyncio.gather(*[extract_with_sem(url) for url in request.urls])
for result in results:
if result is not None:
pages.append(result)
total_tokens += result.tokens_used
execution_time_ms = (time.perf_counter() - start_time) * 1000
return VisionExtractResponse(
request_id=request_id or "-",
pages=pages,
total_processed=len(pages),
total_failed=len(failed),
total_tokens_used=total_tokens,
execution_time_ms=round(execution_time_ms, 2),
failed_urls=failed,
)
async def _extract_single(
self,
browser: Browser,
url: str,
request: VisionExtractRequest,
) -> VisionPageResult:
"""Extract content from a single URL using vision.
Args:
browser: Playwright browser instance.
url: URL to extract.
request: Vision extraction request.
Returns:
VisionPageResult: Extracted content.
"""
start_time = time.perf_counter()
context = await browser.new_context(
viewport={
"width": self.settings.browse_viewport_width,
"height": self.settings.browse_viewport_height,
},
user_agent=self.settings.fetch_user_agent,
)
page = await context.new_page()
try:
# Navigate to URL
try:
await page.goto(
url,
wait_until="networkidle",
timeout=self.settings.browse_timeout,
)
except PlaywrightTimeout as e:
raise WebTimeoutError(
f"Timeout loading {url}",
timeout=self.settings.browse_timeout / 1000,
) from e
# Get page title
title = await page.title()
# Take screenshot(s)
screenshots_base64 = []
if request.full_page:
# Take full page screenshot
screenshot_bytes = await page.screenshot(
type="jpeg",
quality=self.settings.vision_screenshot_quality,
full_page=True,
)
screenshots_base64.append(base64.b64encode(screenshot_bytes).decode())
else:
# Take viewport screenshot only
screenshot_bytes = await page.screenshot(
type="jpeg",
quality=self.settings.vision_screenshot_quality,
full_page=False,
)
screenshots_base64.append(base64.b64encode(screenshot_bytes).decode())
# Extract content via Vision LLM
(
extracted_text,
tokens_used,
llm_provider,
model_used,
) = await self._call_vision_llm(
screenshots_base64,
request.context_query,
request.llm_provider,
request.model,
)
# Extract images if requested
images: list[ImageContext] = []
if request.extract_images:
images = await self._extract_images(page, request.context_query)
extraction_time_ms = (time.perf_counter() - start_time) * 1000
return VisionPageResult(
url=url,
title=title,
extracted_text=extracted_text,
images=images,
extraction_method="vision",
model_used=model_used,
llm_provider=llm_provider,
screenshots_processed=len(screenshots_base64),
tokens_used=tokens_used,
extraction_time_ms=round(extraction_time_ms, 2),
)
finally:
await context.close()
async def _call_vision_llm(
self,
screenshots_base64: list[str],
context_query: str | None,
llm_provider: str,
model: str | None,
) -> tuple[str, int, str, str]:
"""Call Vision LLM to extract text from screenshots.
Args:
screenshots_base64: Screenshots as base64 strings.
context_query: Optional context for extraction.
llm_provider: LLM provider to use.
model: Model to use.
Returns:
tuple: (extracted_text, tokens_used, provider_used, model_used)
"""
# Build prompt
context_line = (
f"\nFocus on content relevant to: {context_query}\n"
if context_query
else ""
)
prompt = f"""Extract all text content from this webpage screenshot.
{context_line}
Return the extracted text in a clear, readable format. Include:
- Main article/page content
- Important headings and subheadings
- Key facts and data
- Publication date if visible
Do NOT include:
- Navigation menus
- Advertisements
- Cookie banners
- Social media buttons"""
# Build OpenAI-format multimodal messages
content: list[dict[str, Any]] = []
for img_b64 in screenshots_base64:
content.append(
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{img_b64}",
},
}
)
content.append({"type": "text", "text": prompt})
messages = [{"role": "user", "content": content}]
return await self._llm_chain.call_chat(
messages=messages,
provider=llm_provider,
max_tokens=self.settings.vision_max_tokens,
model=model or self.settings.vision_model,
)
async def _extract_images(
self,
page: Any,
context_query: str | None,
) -> list[ImageContext]:
"""Extract and analyze images from the page.
Args:
page: Playwright page instance.
context_query: Optional context for relevance scoring.
Returns:
list[ImageContext]: Analyzed images.
"""
# Get all images from page
images_data = await page.evaluate("""
() => {
const images = [];
document.querySelectorAll('img').forEach(img => {
if (img.naturalWidth > 100 && img.naturalHeight > 100) {
images.push({
src: img.src,
alt: img.alt,
width: img.naturalWidth,
height: img.naturalHeight,
});
}
});
return images.slice(0, 5); // Limit to 5 images
}
""")
images: list[ImageContext] = []
for img in images_data:
images.append(
ImageContext(
image_url=img.get("src"),
alt_text=img.get("alt"),
description=img.get("alt") or "Image found on page",
extracted_text=None,
relevance_score=None,
)
)
return images
async def extract_to_page_content(
self,
url: str,
request: VisionExtractRequest | None = None,
) -> PageContent:
"""Convenience method to extract a single URL and return PageContent.
Args:
url: URL to extract.
request: Optional vision extraction request.
Returns:
PageContent: Extracted page content.
"""
if request is None:
request = VisionExtractRequest(urls=[url])
else:
request = VisionExtractRequest(
urls=[url],
extract_page_text=request.extract_page_text,
extract_images=request.extract_images,
context_query=request.context_query,
llm_provider=request.llm_provider,
)
response = await self.extract(request)
if not response.pages:
raise WebError(f"Failed to extract {url}")
page = response.pages[0]
retrieved_at = datetime.now(timezone.utc).isoformat()
return PageContent(
url=page.url,
canonical_url=None,
title=page.title,
text=page.extracted_text,
text_hash=hashlib.sha256(page.extracted_text.encode()).hexdigest(),
html=None,
extraction_method="vision",
fallback_chain=["http", "browse"],
published_at=None,
retrieved_at=retrieved_at,
extraction_time_ms=page.extraction_time_ms,
images=[
PageImage(url=img.image_url or "", alt=img.alt_text)
for img in page.images
]
if page.images
else None,
warnings=[],
)

1
ai_platform/modules/web/tests/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Tests package."""

115
ai_platform/modules/web/tests/conftest.py Normal file
View file

@ -0,0 +1,115 @@
"""Pytest fixtures for Web module tests."""
from unittest.mock import AsyncMock
import pytest
from fastapi.testclient import TestClient
from web.api.app import create_app
from web.api.dependencies import init_concurrency_limiter
from web.config import SettingsCache, WebSettings
from web.fetch.client import FetchClient
from web.metasearch.client import SearXNGClient
from web.orchestrator import Orchestrator
from web.schemas.search import SearchResponse, SearchResult
@pytest.fixture(autouse=True)
def clear_settings_cache() -> None:
"""Clear settings cache before each test."""
SettingsCache.clear()
@pytest.fixture
def test_settings() -> WebSettings:
"""Create test settings with mocked values."""
return WebSettings(
searxng_base_url="http://localhost:55100",
llm_base_url="http://localhost:14011",
external_url="http://localhost:51100",
host="127.0.0.1",
port=51100,
api_tokens=None,
)
@pytest.fixture
def mock_search_response() -> SearchResponse:
"""Create a mock search response."""
return SearchResponse(
request_id="test-request-id",
results=[
SearchResult(
query="test query",
url="https://example.com/article",
title="Test Article Title",
snippet="This is a test snippet from the search result.",
rank=1,
site="example.com",
published_at="2024-01-15",
),
SearchResult(
query="test query",
url="https://news.example.com/story",
title="Another Test Result",
snippet="Another snippet for testing purposes.",
rank=2,
site="news.example.com",
published_at=None,
),
],
total_results=2,
execution_time_ms=150.5,
queries_processed=1,
)
@pytest.fixture
def mock_searxng_api_response() -> dict:
"""Create a mock SearXNG API response."""
return {
"results": [
{
"url": "https://example.com/article",
"title": "Test Article Title",
"content": "This is a test snippet from the search result.",
"publishedDate": "2024-01-15",
},
{
"url": "https://news.example.com/story",
"title": "Another Test Result",
"content": "Another snippet for testing purposes.",
"publishedDate": None,
},
]
}
@pytest.fixture
def client_with_mock(
test_settings: WebSettings, mock_search_response: SearchResponse
) -> SearXNGClient:
"""Create a SearXNGClient with mocked search method."""
SettingsCache.set(test_settings)
client = SearXNGClient(settings=test_settings)
client.search = AsyncMock(return_value=mock_search_response)
return client
@pytest.fixture
def app_client(test_settings: WebSettings) -> TestClient:
"""Create a FastAPI TestClient with test settings."""
SettingsCache.set(test_settings)
init_concurrency_limiter(test_settings.max_concurrent_requests)
app = create_app()
app.state.settings = test_settings
app.state.search_client = SearXNGClient(settings=test_settings)
app.state.fetch_client = FetchClient(settings=test_settings)
app.state.orchestrator = Orchestrator(settings=test_settings)
return TestClient(app)

391
ai_platform/modules/web/tests/test_api.py Normal file
View file

@ -0,0 +1,391 @@
"""Tests for API endpoints."""
from unittest.mock import AsyncMock, patch
import pytest
from fastapi.testclient import TestClient
from web.api.app import create_app
from web.api.dependencies import init_concurrency_limiter
from web.config import SettingsCache, WebSettings
from web.exceptions import (
ProviderError,
RateLimitError,
WebConnectionError,
WebTimeoutError,
)
from web.fetch.client import FetchClient
from web.metasearch.client import SearXNGClient
from web.orchestrator import Orchestrator
from web.schemas.search import SearchResponse, SearchResult
from web.search.paid import PaidSearchClient
@pytest.fixture
def mock_response() -> SearchResponse:
"""Create a mock search response."""
return SearchResponse(
request_id="test-id",
results=[
SearchResult(
query="test",
url="https://example.com",
title="Test",
snippet="Test snippet",
rank=1,
site="example.com",
published_at=None,
)
],
total_results=1,
execution_time_ms=100.0,
queries_processed=1,
)
@pytest.fixture
def test_client(test_settings: WebSettings) -> TestClient:
"""Create test client with mocked settings."""
SettingsCache.set(test_settings)
init_concurrency_limiter(test_settings.max_concurrent_requests)
app = create_app()
app.state.settings = test_settings
app.state.search_client = SearXNGClient(settings=test_settings)
app.state.search_client_free = SearXNGClient(settings=test_settings)
app.state.search_client_premium = PaidSearchClient(settings=test_settings)
app.state.fetch_client = FetchClient(settings=test_settings)
app.state.orchestrator_free = Orchestrator(settings=test_settings)
app.state.orchestrator_premium = Orchestrator(
settings=test_settings, llm_provider="openrouter"
)
return TestClient(app)
class TestHealthEndpoints:
"""Tests for health check endpoints."""
def test_ready_endpoint(self, test_client: TestClient) -> None:
"""Test /ready endpoint."""
response = test_client.get("/ready")
assert response.status_code == 200
assert response.json() == {"ready": True}
def test_health_endpoint(self, test_client: TestClient) -> None:
"""Test /health endpoint."""
with patch.object(
SearXNGClient, "health_check", new_callable=AsyncMock
) as mock_health:
mock_health.return_value = True
response = test_client.get("/health")
assert response.status_code == 200
data = response.json()
assert data["status"] == "healthy"
assert len(data["providers"]) == 1
assert data["providers"][0]["name"] == "searxng"
class TestSearchEndpoint:
"""Tests for /v1/search endpoint."""
def test_search_success(
self, test_client: TestClient, mock_response: SearchResponse
) -> None:
"""Test successful search."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.return_value = mock_response
response = test_client.post(
"/v1/search",
json={"queries": ["test query"], "max_results": 5},
)
assert response.status_code == 200
data = response.json()
assert data["total_results"] == 1
assert len(data["results"]) == 1
assert data["results"][0]["url"] == "https://example.com"
def test_search_validation_empty_query(self, test_client: TestClient) -> None:
"""Test that empty query is rejected."""
response = test_client.post(
"/v1/search",
json={"queries": [""], "max_results": 5},
)
assert response.status_code == 422
def test_search_validation_too_many_queries(self, test_client: TestClient) -> None:
"""Test that too many queries are rejected."""
response = test_client.post(
"/v1/search",
json={"queries": [f"query{i}" for i in range(15)], "max_results": 5},
)
assert response.status_code == 422
def test_search_with_site_allowlist(
self, test_client: TestClient, mock_response: SearchResponse
) -> None:
"""Test search with site allowlist."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.return_value = mock_response
response = test_client.post(
"/v1/search",
json={
"queries": ["test"],
"site_allowlist": ["example.com", "test.com"],
},
)
assert response.status_code == 200
def test_request_id_header(
self, test_client: TestClient, mock_response: SearchResponse
) -> None:
"""Test that X-Request-ID header is returned."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.return_value = mock_response
response = test_client.post(
"/v1/search",
json={"queries": ["test"]},
headers={"X-Request-ID": "custom-request-id"},
)
assert response.status_code == 200
assert "X-Request-ID" in response.headers
assert response.headers["X-Request-ID"] == "custom-request-id"
class TestFetchEndpoint:
"""Tests for /v1/fetch endpoint."""
def test_fetch_rejects_private_urls(self, test_client: TestClient) -> None:
"""Test that private/localhost URLs are rejected (SSRF protection)."""
response = test_client.post(
"/v1/fetch",
json={"urls": ["http://localhost/secret"]},
)
assert response.status_code == 422
def test_fetch_rejects_non_http_schemes(self, test_client: TestClient) -> None:
"""Test that non-HTTP(S) schemes are rejected."""
response = test_client.post(
"/v1/fetch",
json={"urls": ["ftp://example.com/file"]},
)
assert response.status_code == 422
def test_fetch_validation_empty_urls(self, test_client: TestClient) -> None:
"""Test that empty URL list is rejected."""
response = test_client.post(
"/v1/fetch",
json={"urls": []},
)
assert response.status_code == 422
class TestInfoEndpoint:
"""Tests for /v1/info endpoint."""
def test_info_returns_resource(self, test_client: TestClient) -> None:
"""Test that /v1/info returns resource information."""
response = test_client.get("/v1/info")
assert response.status_code == 200
data = response.json()
assert "resource" in data
assert data["resource"]["slug"] == "web-factcheck"
def test_info_returns_functions(self, test_client: TestClient) -> None:
"""Test that /v1/info returns function definitions."""
response = test_client.get("/v1/info")
assert response.status_code == 200
data = response.json()
assert "functions" in data
assert len(data["functions"]) >= 2
slugs = [f["slug"] for f in data["functions"]]
assert "web-gather-evidence" in slugs
assert "web-search" in slugs
assert "web-fetch" in slugs
class TestSearchErrorResponses:
"""Tests for error response codes on /v1/search."""
def test_search_returns_429_on_rate_limit(self, test_client: TestClient) -> None:
"""Test that RateLimitError returns 429."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.side_effect = RateLimitError(
"Rate limit exceeded", retry_after=5.0
)
response = test_client.post(
"/v1/search",
json={"queries": ["test"]},
)
assert response.status_code == 429
assert "Retry-After" in response.headers
def test_search_returns_502_on_connection_error(
self, test_client: TestClient
) -> None:
"""Test that WebConnectionError returns 502."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.side_effect = WebConnectionError("searxng", "refused")
response = test_client.post(
"/v1/search",
json={"queries": ["test"]},
)
assert response.status_code == 502
data = response.json()
assert data["detail"]["error"] == "connection_error"
def test_search_returns_502_on_provider_error(
self, test_client: TestClient
) -> None:
"""Test that ProviderError returns 502."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.side_effect = ProviderError("searxng", "Server error")
response = test_client.post(
"/v1/search",
json={"queries": ["test"]},
)
assert response.status_code == 502
data = response.json()
assert data["detail"]["error"] == "provider_error"
def test_search_returns_504_on_timeout(self, test_client: TestClient) -> None:
"""Test that WebTimeoutError returns 504."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.side_effect = WebTimeoutError("Timeout", timeout=30.0)
response = test_client.post(
"/v1/search",
json={"queries": ["test"]},
)
assert response.status_code == 504
data = response.json()
assert data["detail"]["error"] == "timeout"
class TestAuthFlow:
"""Tests for Bearer token authentication."""
@pytest.fixture
def auth_settings(self) -> WebSettings:
"""Create settings with authentication enabled."""
return WebSettings(
searxng_base_url="http://localhost:55100",
llm_base_url="http://localhost:14011",
external_url="http://localhost:51100",
api_tokens="valid-token-123,valid-token-456",
)
@pytest.fixture
def auth_client(self, auth_settings: WebSettings) -> TestClient:
"""Create test client with auth enabled."""
SettingsCache.set(auth_settings)
init_concurrency_limiter(auth_settings.max_concurrent_requests)
app = create_app()
app.state.settings = auth_settings
app.state.search_client = SearXNGClient(settings=auth_settings)
app.state.search_client_free = SearXNGClient(settings=auth_settings)
app.state.search_client_premium = PaidSearchClient(settings=auth_settings)
app.state.fetch_client = FetchClient(settings=auth_settings)
app.state.orchestrator_free = Orchestrator(settings=auth_settings)
app.state.orchestrator_premium = Orchestrator(
settings=auth_settings, llm_provider="openrouter"
)
return TestClient(app)
def test_auth_required_returns_401_without_token(
self, auth_client: TestClient
) -> None:
"""Test that missing auth header returns 401."""
response = auth_client.post(
"/v1/search",
json={"queries": ["test"]},
)
assert response.status_code == 401
def test_auth_required_returns_401_with_invalid_token(
self, auth_client: TestClient
) -> None:
"""Test that invalid token returns 401."""
response = auth_client.post(
"/v1/search",
json={"queries": ["test"]},
headers={"Authorization": "Bearer wrong-token"},
)
assert response.status_code == 401
def test_auth_succeeds_with_valid_token(self, auth_client: TestClient) -> None:
"""Test that valid token allows access."""
with patch.object(
SearXNGClient, "search", new_callable=AsyncMock
) as mock_search:
mock_search.return_value = SearchResponse(
request_id="test",
results=[],
total_results=0,
execution_time_ms=10.0,
queries_processed=1,
)
response = auth_client.post(
"/v1/search",
json={"queries": ["test"]},
headers={"Authorization": "Bearer valid-token-123"},
)
assert response.status_code == 200
def test_auth_invalid_format_returns_401(self, auth_client: TestClient) -> None:
"""Test that non-Bearer auth format returns 401."""
response = auth_client.post(
"/v1/search",
json={"queries": ["test"]},
headers={"Authorization": "Basic dXNlcjpwYXNz"},
)
assert response.status_code == 401
def test_health_endpoints_skip_auth(self, auth_client: TestClient) -> None:
"""Test that health endpoints don't require auth."""
response = auth_client.get("/ready")
assert response.status_code == 200

198
ai_platform/modules/web/tests/test_browse_client.py Normal file
View file

@ -0,0 +1,198 @@
"""Unit tests for BrowseClient with mocked Playwright."""
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from web.config import WebSettings
@pytest.fixture
def settings() -> WebSettings:
"""Create test settings."""
return WebSettings(
searxng_base_url="http://localhost:55100",
llm_base_url="http://localhost:14011",
external_url="http://localhost:51100",
)
class TestBrowseClientImport:
"""Tests for BrowseClient import handling."""
def test_raises_import_error_without_playwright(
self, settings: WebSettings
) -> None:
"""Test that BrowseClient raises ImportError without playwright."""
with patch("web.browse.client.HAS_PLAYWRIGHT", False):
from web.browse.client import BrowseClient
with pytest.raises(ImportError, match="playwright"):
BrowseClient(settings)
class TestBrowseClientBrowse:
"""Tests for BrowseClient.browse with mocked browser."""
def _make_mock_page(self) -> MagicMock:
"""Create a fresh mock Playwright page."""
page = AsyncMock()
page.goto = AsyncMock()
page.title = AsyncMock(return_value="Test Page Title")
page.url = "https://example.com/page"
page.content = AsyncMock(return_value="<html><body>Content</body></html>")
page.evaluate = AsyncMock(
side_effect=[
None, # remove unwanted elements
"Main content of the page with sufficient text for testing.",
"https://example.com/canonical", # canonical URL
"2024-01-15T00:00:00Z", # publication date
]
)
page.screenshot = AsyncMock(return_value=b"\x89PNG\r\n")
return page
def _make_mock_context(self, mock_page: MagicMock | None = None) -> MagicMock:
"""Create a mock browser context."""
if mock_page is None:
mock_page = self._make_mock_page()
context = AsyncMock()
context.new_page = AsyncMock(return_value=mock_page)
context.close = AsyncMock()
return context
def _make_mock_browser(self, mock_context: MagicMock | None = None) -> MagicMock:
"""Create a mock browser that creates fresh contexts each time."""
if mock_context is not None:
browser = MagicMock()
browser.is_connected = MagicMock(return_value=True)
browser.new_context = AsyncMock(return_value=mock_context)
browser.close = AsyncMock()
return browser
# Create a browser that returns fresh page mocks each time
browser = MagicMock()
browser.is_connected = MagicMock(return_value=True)
async def _new_context(**kwargs):
return self._make_mock_context()
browser.new_context = AsyncMock(side_effect=_new_context)
browser.close = AsyncMock()
return browser
@pytest.mark.asyncio
async def test_browse_single_page(self, settings: WebSettings) -> None:
"""Test browsing a single page returns content."""
with patch("web.browse.client.HAS_PLAYWRIGHT", True):
from web.browse.client import BrowseClient
from web.schemas.browse import BrowseRequest
mock_browser = self._make_mock_browser()
client = BrowseClient(settings, browser=mock_browser)
request = BrowseRequest(urls=["https://example.com/page"])
response = await client.browse(request)
assert response.total_browsed == 1
assert response.pages[0].title == "Test Page Title"
assert "Main content" in response.pages[0].text
@pytest.mark.asyncio
async def test_browse_handles_timeout(self, settings: WebSettings) -> None:
"""Test that Playwright timeout is properly wrapped."""
class MockPlaywrightTimeout(Exception):
pass
with (
patch("web.browse.client.HAS_PLAYWRIGHT", True),
patch(
"web.browse.client.PlaywrightTimeout",
MockPlaywrightTimeout,
create=True,
),
):
from web.browse.client import BrowseClient
from web.schemas.browse import BrowseRequest
mock_page = AsyncMock()
mock_page.goto = AsyncMock(side_effect=MockPlaywrightTimeout("Timeout"))
mock_context = self._make_mock_context(mock_page)
mock_browser = self._make_mock_browser(mock_context)
client = BrowseClient(settings, browser=mock_browser)
request = BrowseRequest(urls=["https://slow.example.com"])
response = await client.browse(request)
assert response.total_failed == 1
assert response.total_browsed == 0
@pytest.mark.asyncio
async def test_browse_multiple_urls(self, settings: WebSettings) -> None:
"""Test browsing multiple URLs concurrently."""
with patch("web.browse.client.HAS_PLAYWRIGHT", True):
from web.browse.client import BrowseClient
from web.schemas.browse import BrowseRequest
# Use browser that creates fresh contexts/pages
mock_browser = self._make_mock_browser()
client = BrowseClient(settings, browser=mock_browser)
request = BrowseRequest(
urls=["https://example.com/a", "https://example.com/b"],
parallel_browses=2,
)
response = await client.browse(request)
assert response.total_browsed == 2
@pytest.mark.asyncio
async def test_close_only_closes_owned_browser(self, settings: WebSettings) -> None:
"""Test that close() only closes browser if client owns it."""
with patch("web.browse.client.HAS_PLAYWRIGHT", True):
from web.browse.client import BrowseClient
mock_browser = self._make_mock_browser()
client = BrowseClient(settings, browser=mock_browser)
await client.close()
mock_browser.close.assert_not_called()
@pytest.mark.asyncio
async def test_concurrent_get_browser_single_instance(
self, settings: WebSettings
) -> None:
"""Test that concurrent _get_browser calls produce a single browser."""
with patch("web.browse.client.HAS_PLAYWRIGHT", True):
from web.browse.client import BrowseClient
mock_browser = self._make_mock_browser()
launch_count = 0
async def mock_launch(**kwargs):
nonlocal launch_count
launch_count += 1
await asyncio.sleep(0.05) # Simulate async work
return mock_browser
mock_pw = AsyncMock()
mock_pw.chromium.launch = mock_launch
mock_pw_ctx = AsyncMock()
mock_pw_ctx.start = AsyncMock(return_value=mock_pw)
client = BrowseClient(settings)
with patch("web.browse.client.async_playwright", return_value=mock_pw_ctx):
# Launch many concurrent _get_browser calls
results = await asyncio.gather(
*[client._get_browser() for _ in range(5)]
)
# Lock ensures only one browser was launched
assert launch_count == 1
# All results should be the same instance
assert all(r is results[0] for r in results)

Some files were not shown because too many files have changed in this diff Show more