livrare lot 2
This commit is contained in:
commit
8ecc78e729
763 changed files with 164593 additions and 0 deletions
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Helpers for building + persisting a minimal AnalysisSession when a single
|
||||
* component runs standalone (i.e. /techniques/analyze, not /pipeline/analyze).
|
||||
*
|
||||
* Replaces old saveAndSyncComponent + syncToPostgres flow.
|
||||
*/
|
||||
import type { AnalysisSession, InputType } from '../../shared/types/analysis-session';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getPersistServiceInstance } from '../_init';
|
||||
|
||||
interface BuildOpts {
|
||||
sessionId: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType: InputType;
|
||||
inputText?: string;
|
||||
inputUrl?: string;
|
||||
mediaUrl?: string;
|
||||
result: any;
|
||||
/** 'techniques' | 'ai_tampered' | 'claims' | 'source_assessment'. Defaults to 'techniques'. */
|
||||
component?: string;
|
||||
durationMs: number;
|
||||
llmUsage?: any[];
|
||||
}
|
||||
|
||||
export function buildStandaloneSession(opts: BuildOpts): AnalysisSession {
|
||||
const now = new Date().toISOString();
|
||||
const comp = opts.component || 'techniques';
|
||||
let llm_usage = null;
|
||||
if (opts.llmUsage && opts.llmUsage.length > 0) {
|
||||
const summary = { calls: opts.llmUsage.length, prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
|
||||
for (const e of opts.llmUsage) {
|
||||
summary.prompt_tokens += e.prompt_tokens;
|
||||
summary.completion_tokens += e.completion_tokens;
|
||||
summary.total_tokens += e.total_tokens;
|
||||
}
|
||||
llm_usage = { total: summary, by_component: { [comp]: summary } };
|
||||
}
|
||||
return {
|
||||
session_id: opts.sessionId,
|
||||
user_id: opts.userId || null,
|
||||
user_email: opts.userEmail || null,
|
||||
input_type: opts.inputType,
|
||||
input_text: opts.inputText || null,
|
||||
input_url: opts.inputUrl || null,
|
||||
input_media_url: opts.mediaUrl || null,
|
||||
input_hash: null,
|
||||
status: 'completed',
|
||||
components_run: [comp],
|
||||
components_skipped: ['ai_tampered', 'claims', 'domain', 'verdict'].filter(c => c !== comp),
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: new Date(Date.now() - opts.durationMs).toISOString(),
|
||||
completed_at: now,
|
||||
total_duration_ms: opts.durationMs,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'web',
|
||||
api_version: 'v3',
|
||||
created_at: now,
|
||||
techniques: comp === 'techniques' ? opts.result : null,
|
||||
ai_tampered: comp === 'ai_tampered' ? opts.result : null,
|
||||
claims: comp === 'claims' ? opts.result : null,
|
||||
domain: null,
|
||||
source_assessment: comp === 'source_assessment' ? opts.result : null,
|
||||
verdict: null,
|
||||
llm_usage,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget persist for standalone component runs.
|
||||
* Uses PersistService (Redis + PG). Logs but does not throw on failure —
|
||||
* caller has already returned a response by then.
|
||||
*/
|
||||
export function persistStandaloneResult(opts: BuildOpts, logPrefix = 'StandaloneSession'): void {
|
||||
if (!opts.userId) return;
|
||||
const session = buildStandaloneSession(opts);
|
||||
getPersistServiceInstance().persist(session)
|
||||
.catch(err => log.error(`[${logPrefix}] Persist error:`, (err as Error).message));
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Shared lazy-init singletons + multer config used by route modules under
|
||||
* src/api/. Extracted from the original routes.ts so techniques/media/domain
|
||||
* sub-routers can reuse them without each duplicating connection setup.
|
||||
*/
|
||||
import multer from 'multer';
|
||||
import { lazyRedis } from '../shared/redis/connection';
|
||||
import { MediaService } from '../shared/media/media-service';
|
||||
import { PersistService, PgSessionAdapter, getPgPool } from '../shared/persistence';
|
||||
import { SessionStore } from '../shared/redis/session-store';
|
||||
|
||||
/** Per-module Redis singleton (label visible in connection metadata). */
|
||||
export const getRedis = lazyRedis('routes');
|
||||
|
||||
/** Multer upload — memory storage, 50MB max. Used by /media/upload + /techniques/analyze-media. */
|
||||
export const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
limits: { fileSize: 50 * 1024 * 1024 },
|
||||
});
|
||||
|
||||
let _mediaService: MediaService | null = null;
|
||||
export function getMediaService(): MediaService {
|
||||
if (!_mediaService) _mediaService = new MediaService();
|
||||
return _mediaService;
|
||||
}
|
||||
|
||||
let _persistService: PersistService | null = null;
|
||||
export function getPersistServiceInstance(): PersistService {
|
||||
if (!_persistService) {
|
||||
const r = getRedis();
|
||||
_persistService = new PersistService(
|
||||
new SessionStore(r),
|
||||
new PgSessionAdapter(getPgPool()),
|
||||
);
|
||||
}
|
||||
return _persistService;
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Re-export of the new ai-tampered barrel. Kept at this path so src/index.ts
|
||||
* (which imports `./api/ai-tampered-routes`) continues to work unchanged after
|
||||
* the 764-LOC → 9-file split. See ./ai-tampered/index.ts for the routing map.
|
||||
*/
|
||||
export { default } from './ai-tampered';
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
/**
|
||||
* Image AI-detection via the shared vision cascade (Qwen Vision Local → Gemini → GPT-4o).
|
||||
*
|
||||
* Used by /analyze-image (the lightweight image-only entry point — separate from
|
||||
* the full /analyze-media flow which goes through dispatcher + worker).
|
||||
*
|
||||
* The 10-indicator prompt is intentionally inline — it's the exact rubric we
|
||||
* train against and changing it requires deliberate review (not a hot-reload).
|
||||
*/
|
||||
import { callVision } from '../../shared/media/vision';
|
||||
import { analyzeForensic, formatForensicForVisionPrompt } from '../../shared/media/forensic';
|
||||
import { analyzeMetadata, formatMetadataForPrompt, analyzeOcr, analyzeDetect, formatFeaturesForPrompt } from '../../shared/media/extractors';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
export interface ImageAnalysisResult {
|
||||
ai_generated_probability: number;
|
||||
indicators: string[];
|
||||
evidence: string;
|
||||
model_used: string;
|
||||
forensic_score?: number | null;
|
||||
forensic_label?: string;
|
||||
}
|
||||
|
||||
/** Download image bytes pentru apel forensic. Fail-open. */
|
||||
async function fetchImageBuffer(imageUrl: string): Promise<{ buffer: Buffer; filename: string } | null> {
|
||||
try {
|
||||
const r = await fetch(imageUrl, { signal: AbortSignal.timeout(20000) });
|
||||
if (!r.ok) return null;
|
||||
const buffer = Buffer.from(await r.arrayBuffer());
|
||||
const filename = imageUrl.split('/').pop()?.split('?')[0] || 'image.jpg';
|
||||
return { buffer, filename };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getImageVerdict(probability: number): 'LIKELY_AI' | 'POSSIBLY_AI' | 'MIXED' | 'LIKELY_HUMAN' {
|
||||
if (probability >= 70) return 'LIKELY_AI';
|
||||
if (probability >= 50) return 'POSSIBLY_AI';
|
||||
if (probability >= 30) return 'MIXED';
|
||||
return 'LIKELY_HUMAN';
|
||||
}
|
||||
|
||||
export async function analyzeImageForAI(
|
||||
imageUrl: string,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<ImageAnalysisResult> {
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// FORENSIC FEATURES — rulează în paralel cu pregătirea apel Vision.
|
||||
// Tier-aware: free user primește doar m27 (AI detector) + m28 (heatmap)
|
||||
// ca să economisim CPU; premium primește toate 4 (skip m26 audio fără
|
||||
// sens pe imagine).
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
const forensicEnabled = process.env.FORENSIC_ENABLED !== 'false';
|
||||
const extractorsEnabled = process.env.EXTRACTORS_ENABLED !== 'false';
|
||||
// Descarcă imaginea O SINGURĂ dată şi partajează buffer-ul între forensic
|
||||
// (m25-m29) şi metadata/integrity (EXIF/ELA/C2PA). Ambele fail-open.
|
||||
const analysisPromise = (forensicEnabled || extractorsEnabled)
|
||||
? fetchImageBuffer(imageUrl).then(async (img) => {
|
||||
if (!img) return { forensic: null, metadata: null, ocr: null, detect: null };
|
||||
const [forensic, metadata, ocr, detect] = await Promise.all([
|
||||
forensicEnabled ? analyzeForensic(img.buffer, img.filename, {
|
||||
modules: tier === 'premium'
|
||||
? ['m25', 'm27', 'm28', 'm29'] // toate vizual-aplicabile pe imagine
|
||||
: ['m27', 'm28'], // core pe free
|
||||
encodeImages: true,
|
||||
timeoutMs: 60000, // imagini sunt rapide vs video
|
||||
}).catch(() => null) : Promise.resolve(null),
|
||||
extractorsEnabled ? analyzeMetadata(img.buffer, img.filename, { timeoutMs: 30000 }).catch(() => null) : Promise.resolve(null),
|
||||
extractorsEnabled ? analyzeOcr(img.buffer, img.filename).catch(() => null) : Promise.resolve(null),
|
||||
extractorsEnabled ? analyzeDetect(img.buffer, img.filename).catch(() => null) : Promise.resolve(null),
|
||||
]);
|
||||
return { forensic, metadata, ocr, detect };
|
||||
}).catch(() => ({ forensic: null, metadata: null, ocr: null, detect: null }))
|
||||
: Promise.resolve({ forensic: null, metadata: null, ocr: null, detect: null });
|
||||
|
||||
const basePrompt = `Analyze this image to determine if it was AI-generated (by DALL-E, Midjourney, Stable Diffusion, etc.) or is a real photograph/human-created image.
|
||||
|
||||
Look for these AI generation indicators:
|
||||
1. **Anatomical errors**: Extra fingers, merged hands, distorted faces, asymmetric eyes
|
||||
2. **Texture anomalies**: Overly smooth skin, plastic-like appearance, inconsistent textures
|
||||
3. **Background artifacts**: Blurred or nonsensical backgrounds, floating objects
|
||||
4. **Lighting inconsistencies**: Shadows going different directions, incorrect reflections
|
||||
5. **Text/writing errors**: Garbled text, nonsensical letters
|
||||
6. **Repetitive patterns**: Unnatural repetition in textures or elements
|
||||
7. **Watermarks/signatures**: AI tool watermarks (Midjourney, DALL-E signatures)
|
||||
8. **Style indicators**: Characteristic AI art styles, over-processed look
|
||||
9. **Edge artifacts**: Unnatural edges, halos around objects
|
||||
10. **Composition issues**: Unnatural object placement, perspective errors
|
||||
|
||||
Return JSON only:
|
||||
{
|
||||
"ai_generated_probability": 75,
|
||||
"indicators": ["extra fingers visible", "plastic skin texture", "background artifacts"],
|
||||
"evidence": "Detailed explanation of what you observed"
|
||||
}`;
|
||||
|
||||
// Aşteaptă forensic (sau null) — apoi compune prompt + messages
|
||||
const { forensic, metadata, ocr, detect } = await analysisPromise;
|
||||
|
||||
// Construieşte prompt-ul augmented: dacă avem forensic, append evidence_text
|
||||
// (LLM Vision primește masurători + ghidare cum să le folosească).
|
||||
let prompt = basePrompt;
|
||||
if (forensic) prompt += '\n\n' + formatForensicForVisionPrompt(forensic);
|
||||
if (metadata) prompt += formatMetadataForPrompt(metadata);
|
||||
const contentSeg = formatFeaturesForPrompt([ocr, detect], 'IMAGE CONTENT EXTRACTORS — OCR text + detected objects (YOLO)');
|
||||
if (contentSeg) prompt += contentSeg;
|
||||
|
||||
// Construieşte content multimodal: imagine originală + heatmap-uri forensic
|
||||
// (LLM vede vizual unde să se uite — m28 forgery heatmap, m29 lighting, etc.)
|
||||
const content: Array<{ type: string; text?: string; image_url?: { url: string } }> = [
|
||||
{ type: 'text', text: prompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
];
|
||||
if (forensic?.images?.length) {
|
||||
for (const img of forensic.images) {
|
||||
if (img.data_url) {
|
||||
content.push({
|
||||
type: 'image_url',
|
||||
image_url: { url: img.data_url },
|
||||
});
|
||||
}
|
||||
}
|
||||
if (forensic.images.some(i => i.data_url)) {
|
||||
// Marker text pt LLM să ştie ce sunt imaginile adiționale
|
||||
content.splice(2, 0, {
|
||||
type: 'text',
|
||||
text: `Below are ${forensic.images.filter(i => i.data_url).length} additional forensic visualizations (heatmaps, signal plots) produced by the m25-m29 detectors. Use them alongside the original image to localize suspicious regions:`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await callVision(getRedis(), [{
|
||||
role: 'user',
|
||||
content,
|
||||
}], { max_tokens: 1500, temperature: 0.2 }, tier);
|
||||
|
||||
log.info(`[AI-Tampered] Image analysis via ${result.provider} (tier: ${tier})${forensic ? ` + forensic ${forensic.summary.overall_label} (${forensic.summary.overall_score})` : ''}`);
|
||||
|
||||
const jsonMatch = result.content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
return {
|
||||
ai_generated_probability: parsed.ai_generated_probability || 50,
|
||||
indicators: parsed.indicators || [],
|
||||
evidence: parsed.evidence || '',
|
||||
model_used: result.provider,
|
||||
forensic_score: forensic?.summary.overall_score,
|
||||
forensic_label: forensic?.summary.overall_label,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
log.error(`[AI-Tampered] All vision models failed: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
ai_generated_probability: 50,
|
||||
indicators: ['Analysis failed - using neutral score'],
|
||||
evidence: 'Could not analyze image with vision models',
|
||||
model_used: 'none',
|
||||
forensic_score: forensic?.summary.overall_score,
|
||||
forensic_label: forensic?.summary.overall_label,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Shared lazy singletons + constants for ai-tampered routes.
|
||||
*/
|
||||
import { lazyRedis } from '../../shared/redis/connection';
|
||||
import { PersistService, PgSessionAdapter, getPgPool } from '../../shared/persistence';
|
||||
import { SessionStore } from '../../shared/redis/session-store';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
|
||||
export const REDIS_PREFIX = ConfigKeys.aiTamperedPrefix;
|
||||
|
||||
export const getRedis = lazyRedis('ai-tampered-routes');
|
||||
|
||||
let _persistService: PersistService | null = null;
|
||||
|
||||
export function getPersistServiceInstance(): PersistService {
|
||||
if (!_persistService) {
|
||||
const r = getRedis();
|
||||
_persistService = new PersistService(
|
||||
new SessionStore(r),
|
||||
new PgSessionAdapter(getPgPool()),
|
||||
);
|
||||
}
|
||||
return _persistService;
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Inline LLM client for the sync-fallback path + /test-model endpoint.
|
||||
*
|
||||
* Two layers:
|
||||
* - callLLM(model, prompt, options?) — low-level: builds headers per provider's
|
||||
* auth_type (bearer / x-api-key / x-goog-api-key), calls /chat/completions,
|
||||
* parses choices/usage, optionally pushes a usage entry into a tracker array.
|
||||
* - createLLMClient() — adapts callLLM to the (prompt, systemPrompt, options)
|
||||
* shape expected by ComponentRunner / AITamperedExecutor.
|
||||
*/
|
||||
import { callVision } from '../../shared/media/vision';
|
||||
import { getRedis, REDIS_PREFIX } from './_init';
|
||||
|
||||
void callVision;
|
||||
|
||||
export interface LLMUsageEntry {
|
||||
model: string;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
}
|
||||
|
||||
export interface LLMCallOptions {
|
||||
provider_routing?: { order?: string[]; allow_fallbacks?: boolean };
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
/** Optional per-call usage tracker — callers push token counts here. */
|
||||
_usage_tracker?: LLMUsageEntry[];
|
||||
}
|
||||
|
||||
export async function callLLM(model: any, prompt: string, options?: LLMCallOptions): Promise<string> {
|
||||
const { provider, provider_config, model_code } = model;
|
||||
|
||||
const apiKeyEnvName = `${provider.toUpperCase()}_API_KEY`;
|
||||
const apiKey = process.env[apiKeyEnvName] || process.env.OPENROUTER_API_KEY;
|
||||
|
||||
if (!apiKey && provider_config.auth_type !== 'none') {
|
||||
throw new Error(`API key not found for provider ${provider}. Set ${apiKeyEnvName}`);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (provider_config.auth_type === 'bearer') headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
else if (provider_config.auth_type === 'x-api-key') headers['x-api-key'] = apiKey!;
|
||||
else if (provider_config.auth_type === 'api_key') headers['x-goog-api-key'] = apiKey!;
|
||||
|
||||
if (provider === 'openrouter') {
|
||||
headers['HTTP-Referer'] = 'https://didi.ai';
|
||||
headers['X-Title'] = 'DIDI Agent V3 - AI Tampered';
|
||||
}
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model: model_code,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: options?.max_tokens || 500,
|
||||
temperature: options?.temperature || 0.3,
|
||||
};
|
||||
|
||||
if (provider === 'openrouter' && options?.provider_routing) {
|
||||
body.provider = {
|
||||
order: options.provider_routing.order || [],
|
||||
allow_fallbacks: options.provider_routing.allow_fallbacks ?? true,
|
||||
};
|
||||
}
|
||||
|
||||
const response = await fetch(`${provider_config.base_url}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`LLM API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number };
|
||||
};
|
||||
|
||||
if (options?._usage_tracker && data.usage) {
|
||||
options._usage_tracker.push({
|
||||
model: model_code,
|
||||
prompt_tokens: data.usage.prompt_tokens || 0,
|
||||
completion_tokens: data.usage.completion_tokens || 0,
|
||||
total_tokens: data.usage.total_tokens || (data.usage.prompt_tokens || 0) + (data.usage.completion_tokens || 0),
|
||||
});
|
||||
}
|
||||
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
|
||||
export function createLLMClient() {
|
||||
return {
|
||||
async call(prompt: string, systemPrompt: string, options: any): Promise<string> {
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
if (!modelsData) throw new Error('Models not configured');
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === options.model_key);
|
||||
if (!model) throw new Error(`Model ${options.model_key} not found`);
|
||||
|
||||
const fullPrompt = systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt;
|
||||
const providerRouting = options.provider_routing || model.provider_routing;
|
||||
const llmOptions: LLMCallOptions = {
|
||||
temperature: options.temperature,
|
||||
max_tokens: options.max_tokens,
|
||||
};
|
||||
|
||||
if (providerRouting && providerRouting.length > 0) {
|
||||
llmOptions.provider_routing = { order: providerRouting, allow_fallbacks: true };
|
||||
}
|
||||
|
||||
if (options._usage_tracker) {
|
||||
llmOptions._usage_tracker = options._usage_tracker;
|
||||
}
|
||||
|
||||
return callLLM(model, fullPrompt, llmOptions);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/**
|
||||
* Build + persist a minimal AnalysisSession for standalone ai_tampered runs.
|
||||
*
|
||||
* Used by the sync-fallback path when RabbitMQ dispatch reports !async — keeps
|
||||
* persistence behavior identical to the queue path so admin dashboards / history
|
||||
* see one consistent session shape regardless of which path served the result.
|
||||
*
|
||||
* Persist is fire-and-forget: failures are logged, never propagated to the response.
|
||||
*/
|
||||
import type { AnalysisSession, InputType } from '../../shared/types/analysis-session';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getPersistServiceInstance } from './_init';
|
||||
|
||||
export function buildStandaloneSession(opts: {
|
||||
sessionId: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType: InputType;
|
||||
inputText?: string;
|
||||
mediaUrl?: string;
|
||||
result: any;
|
||||
durationMs: number;
|
||||
llmUsage?: any[];
|
||||
}): AnalysisSession {
|
||||
const now = new Date().toISOString();
|
||||
let llm_usage = null;
|
||||
if (opts.llmUsage && opts.llmUsage.length > 0) {
|
||||
const summary = { calls: opts.llmUsage.length, prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
|
||||
for (const e of opts.llmUsage) {
|
||||
summary.prompt_tokens += e.prompt_tokens;
|
||||
summary.completion_tokens += e.completion_tokens;
|
||||
summary.total_tokens += e.total_tokens;
|
||||
}
|
||||
llm_usage = { total: summary, by_component: { ai_tampered: summary } };
|
||||
}
|
||||
return {
|
||||
session_id: opts.sessionId,
|
||||
user_id: opts.userId || null,
|
||||
user_email: opts.userEmail || null,
|
||||
input_type: opts.inputType,
|
||||
input_text: opts.inputText || null,
|
||||
input_url: null,
|
||||
input_media_url: opts.mediaUrl || null,
|
||||
input_hash: null,
|
||||
status: 'completed',
|
||||
components_run: ['ai_tampered'],
|
||||
components_skipped: ['techniques', 'claims', 'domain', 'verdict'],
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: new Date(Date.now() - opts.durationMs).toISOString(),
|
||||
completed_at: now,
|
||||
total_duration_ms: opts.durationMs,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'web',
|
||||
api_version: 'v3',
|
||||
created_at: now,
|
||||
techniques: null,
|
||||
ai_tampered: opts.result,
|
||||
claims: null,
|
||||
domain: null,
|
||||
source_assessment: null,
|
||||
verdict: null,
|
||||
llm_usage,
|
||||
};
|
||||
}
|
||||
|
||||
export function persistStandaloneResult(opts: {
|
||||
sessionId: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType: InputType;
|
||||
inputText?: string;
|
||||
mediaUrl?: string;
|
||||
result: any;
|
||||
durationMs: number;
|
||||
llmUsage?: any[];
|
||||
}): void {
|
||||
if (!opts.userId) return;
|
||||
const session = buildStandaloneSession(opts);
|
||||
getPersistServiceInstance().persist(session)
|
||||
.catch(err => log.error('[AI-Tampered] Persist error:', (err as Error).message));
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* POST /analyze-image — Lightweight image-only AI-detection (sync, no queue).
|
||||
*
|
||||
* This is intentionally separate from /analyze-media (which always queues): for
|
||||
* a single image we have a fast vision-cascade path that returns in seconds.
|
||||
* Premium users get the higher-tier vision model; free users hit the cheaper one.
|
||||
*
|
||||
* Credit deduction happens BEFORE responding 200 so a credit-service failure
|
||||
* can't be hidden behind a successful analysis.
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { checkCredits, deductCredits, getSearchTier } from '../../shared/credits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { analyzeImageForAI, getImageVerdict } from './_image-analyzer';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/analyze-image', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { image_url, user_id: body_uid } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
|
||||
if (!image_url) {
|
||||
return res.status(400).json({ success: false, error: 'image_url is required' });
|
||||
}
|
||||
|
||||
let tier: 'free' | 'premium' = 'free';
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, 'image');
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
tier = getSearchTier(creditCheck.planType);
|
||||
}
|
||||
|
||||
const sessionId = randomUUID();
|
||||
const result = await analyzeImageForAI(image_url, tier);
|
||||
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, 'image', sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=image`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=image err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
ai_probability: result.ai_generated_probability,
|
||||
verdict: getImageVerdict(result.ai_generated_probability),
|
||||
indicators: result.indicators,
|
||||
evidence: result.evidence,
|
||||
model_used: result.model_used,
|
||||
forensic: result.forensic_score !== undefined
|
||||
? { score: result.forensic_score, label: result.forensic_label }
|
||||
: null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_analyze_image');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
/**
|
||||
* Async ai-tampered dispatch — backs /analyze, /analyze-media, /analyze-async.
|
||||
*
|
||||
* Same pattern as claims:
|
||||
* 1) Validate input + media_type.
|
||||
* 2) Credit check (skipped if no user_id).
|
||||
* 3) URL inputs without text: inline fetch + HTML strip up to MAX_TEXT_LENGTH
|
||||
* (we don't go through the M17 helper here — ai-tampered analyzes the
|
||||
* surface text, not the structured article body).
|
||||
* 4) Validate text/url content against analysisLimits.
|
||||
* 5) Dispatch to RabbitMQ; on !async, fall back to inline ComponentRunner.
|
||||
* 6) On async: deduct credits BEFORE responding 202 (no lost-credit risk).
|
||||
*
|
||||
* One handler wired to all three POST routes — the prefix is purely cosmetic.
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { checkCredits, deductCredits } from '../../shared/credits';
|
||||
import { validateExternalUrl } from '../../shared/helpers/validate-url';
|
||||
import { MAX_TEXT_LENGTH, validateTextInput } from '../../config/analysisLimits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import type { InputType } from '../../shared/types/analysis-session';
|
||||
import { getRedis } from './_init';
|
||||
import { createLLMClient } from './_llm-client';
|
||||
import { persistStandaloneResult } from './_standalone-session';
|
||||
|
||||
const router = Router();
|
||||
|
||||
async function dispatchAiTamperedAsync(req: Request, res: Response) {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_uid, user_email: body_email, plan_type = 1 } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
const validTypes = ['text', 'image', 'audio', 'video', 'url'];
|
||||
const inputType = media_type || (text ? 'text' : url ? 'url' : null);
|
||||
if (!inputType || !validTypes.includes(inputType)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, inputType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sessionId = randomUUID();
|
||||
log.info(`[${sessionId}] AI-Tampered analysis (async), type: ${inputType}`);
|
||||
|
||||
let content = text || '';
|
||||
if (inputType === 'url' && (url || media_url) && !text) {
|
||||
try {
|
||||
const fetchUrl = url || media_url;
|
||||
validateExternalUrl(fetchUrl);
|
||||
const response = await fetch(fetchUrl, { headers: { 'User-Agent': 'Mozilla/5.0 DIDI-Bot' } });
|
||||
const html = await response.text();
|
||||
content = html
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.substring(0, MAX_TEXT_LENGTH);
|
||||
} catch (e) {
|
||||
log.warn(`[${sessionId}] Failed to fetch URL: ${(e as Error).message}`);
|
||||
content = `URL: ${url || media_url}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (['text', 'url'].includes(inputType) && content && content.trim().length > 0) {
|
||||
const textValidation = validateTextInput(content);
|
||||
if (!textValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false, error: textValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: textValidation.error_code, stats: textValidation.stats },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const { dispatch } = await import('../../queue/dispatcher');
|
||||
type PlanType = 1 | 2 | 3 | 4 | 5 | 6;
|
||||
const planTypeNum = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
|
||||
const result = await dispatch(
|
||||
sessionId,
|
||||
{ content, url, mediaPath: media_url, userId: user_id, userEmail: user_email, inputType },
|
||||
planTypeNum,
|
||||
['ai_tampered'],
|
||||
);
|
||||
|
||||
if (!result.async) {
|
||||
const { ComponentRunner } = await import('../../components/component-runner');
|
||||
const r = getRedis();
|
||||
const llmClient = createLLMClient();
|
||||
const runner = new ComponentRunner(r, llmClient);
|
||||
const syncResult = await runner.runAiTampered({ text: content || undefined, media_url, media_type: inputType, sessionId });
|
||||
|
||||
persistStandaloneResult({
|
||||
sessionId, userId: user_id, userEmail: user_email,
|
||||
inputType: inputType as InputType,
|
||||
inputText: inputType === 'text' ? content : undefined,
|
||||
mediaUrl: media_url, result: syncResult, durationMs: 0,
|
||||
llmUsage: runner.getLastUsageTracker(),
|
||||
});
|
||||
|
||||
return res.json({ success: true, async: false, data: { session_id: sessionId, media_type: inputType, result: syncResult } });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, inputType, sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType}`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType} err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
media_type: inputType,
|
||||
queued_components: result.queued,
|
||||
plan_type: planTypeNum,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_dispatch_async');
|
||||
}
|
||||
}
|
||||
|
||||
router.post('/analyze-async', dispatchAiTamperedAsync);
|
||||
router.post('/analyze', dispatchAiTamperedAsync);
|
||||
router.post('/analyze-media', dispatchAiTamperedAsync);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
/**
|
||||
* AI-tampered config + introspection endpoints.
|
||||
*
|
||||
* GET /health
|
||||
* GET /config — full config snapshot (manifest, models, prompts, schemas, scoring, categories, vision)
|
||||
* GET /models — available models
|
||||
* GET /stage-assignments — per-tier stage→model mapping
|
||||
* PUT /stage-assignments — zod-validated update
|
||||
* GET /categories — categories_compact + indicators_hierarchy
|
||||
* POST /quick — pattern-match-only quickAnalyze (no LLM)
|
||||
* POST /test-model — connectivity probe (returns upstream error in
|
||||
* data field intentionally for operator debugging)
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { TierStageAssignmentsSchema } from '../../shared/helpers/config-schemas';
|
||||
import { getRedis, REDIS_PREFIX } from './_init';
|
||||
import { callLLM } from './_llm-client';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/health', (_req: Request, res: Response) => {
|
||||
res.json({
|
||||
success: true,
|
||||
service: 'ai-tampered-v1',
|
||||
version: '1.0.0',
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/config', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
|
||||
const [manifest, availableModels, stageAssignments, prompts, schemas, scoringConfig, categoriesCompact, indicatorsHierarchy, visionModels] = await Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:manifest`),
|
||||
r.get(`${REDIS_PREFIX}:available_models`),
|
||||
r.get(`${REDIS_PREFIX}:stage_assignments`),
|
||||
Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:prompts:screening`),
|
||||
r.get(`${REDIS_PREFIX}:prompts:deep_analysis`),
|
||||
]),
|
||||
Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:schemas:screening`),
|
||||
r.get(`${REDIS_PREFIX}:schemas:complete`),
|
||||
]),
|
||||
r.get(`${REDIS_PREFIX}:scoring_config`),
|
||||
r.get(`${REDIS_PREFIX}:categories_compact`),
|
||||
r.get(`${REDIS_PREFIX}:indicators_hierarchy`),
|
||||
r.get(ConfigKeys.visionModels),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
manifest: manifest ? JSON.parse(manifest) : null,
|
||||
available_models: availableModels ? JSON.parse(availableModels) : null,
|
||||
stage_assignments: stageAssignments ? JSON.parse(stageAssignments) : null,
|
||||
prompts: {
|
||||
screening: prompts[0] ? JSON.parse(prompts[0]) : null,
|
||||
deep_analysis: prompts[1] ? JSON.parse(prompts[1]) : null,
|
||||
},
|
||||
schemas: {
|
||||
screening: schemas[0] ? JSON.parse(schemas[0]) : null,
|
||||
complete: schemas[1] ? JSON.parse(schemas[1]) : null,
|
||||
},
|
||||
scoring_config: scoringConfig ? JSON.parse(scoringConfig) : null,
|
||||
categories_compact: categoriesCompact ? JSON.parse(categoriesCompact) : null,
|
||||
indicators_hierarchy: indicatorsHierarchy ? JSON.parse(indicatorsHierarchy) : null,
|
||||
vision_models: visionModels ? JSON.parse(visionModels) : null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_config');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/models', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
|
||||
if (!data) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Models not configured. Load AI Tampered config to Redis.',
|
||||
});
|
||||
}
|
||||
|
||||
const { models } = JSON.parse(data);
|
||||
res.json({ success: true, data: { models } });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_models');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/stage-assignments', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${REDIS_PREFIX}:stage_assignments`);
|
||||
if (!data) {
|
||||
return res.status(404).json({ success: false, error: 'Stage assignments not configured.' });
|
||||
}
|
||||
res.json({ success: true, data: JSON.parse(data) });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_get_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/stage-assignments', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const parsed = TierStageAssignmentsSchema.safeParse(req.body?.stage_assignments);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid stage_assignments payload',
|
||||
details: parsed.error.issues,
|
||||
});
|
||||
}
|
||||
await r.set(`${REDIS_PREFIX}:stage_assignments`, JSON.stringify(parsed.data));
|
||||
res.json({ success: true, message: 'Stage assignments updated', data: parsed.data });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_put_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/categories', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const [categoriesCompact, indicatorsHierarchy] = await Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:categories_compact`),
|
||||
r.get(`${REDIS_PREFIX}:indicators_hierarchy`),
|
||||
]);
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
categories_compact: categoriesCompact ? JSON.parse(categoriesCompact) : null,
|
||||
indicators_hierarchy: indicatorsHierarchy ? JSON.parse(indicatorsHierarchy) : null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_categories');
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/quick', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text } = req.body;
|
||||
if (!text) {
|
||||
return res.status(400).json({ success: false, error: 'text is required' });
|
||||
}
|
||||
|
||||
const { AITamperedExecutor } = await import('../../components/ai-tampered/executor');
|
||||
const r = getRedis();
|
||||
const mockLLMClient = {
|
||||
async call(): Promise<string> { throw new Error('Quick analysis does not use LLM'); }
|
||||
};
|
||||
const executor = new AITamperedExecutor(r, mockLLMClient);
|
||||
const result = await executor.quickAnalyze(text);
|
||||
res.json({ success: true, data: result });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_quick');
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/test-model', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { model_key, test_prompt } = req.body;
|
||||
if (!model_key) {
|
||||
return res.status(400).json({ success: false, error: 'model_key is required' });
|
||||
}
|
||||
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
if (!modelsData) {
|
||||
return res.status(404).json({ success: false, error: 'Models not configured' });
|
||||
}
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === model_key);
|
||||
if (!model) {
|
||||
return res.status(404).json({ success: false, error: `Model ${model_key} not found` });
|
||||
}
|
||||
|
||||
const prompt = test_prompt || 'Respond with JSON: {"status": "ok", "model": "your_model_name"}';
|
||||
const testStart = Date.now();
|
||||
|
||||
try {
|
||||
const response = await callLLM(model, prompt);
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
model_key,
|
||||
model_name: model.model_name,
|
||||
provider: model.provider,
|
||||
response_time_ms: Date.now() - testStart,
|
||||
response: response.substring(0, 500),
|
||||
status: 'connected',
|
||||
},
|
||||
});
|
||||
} catch (llmError) {
|
||||
res.json({
|
||||
success: false,
|
||||
data: {
|
||||
model_key,
|
||||
model_name: model.model_name,
|
||||
provider: model.provider,
|
||||
status: 'error',
|
||||
error: (llmError as Error).message,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_test_model');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* AGENT V3 — AI-TAMPERED API barrel router.
|
||||
*
|
||||
* Original 764-line ai-tampered-routes.ts split into:
|
||||
* _init.ts — lazyRedis + persistService + REDIS_PREFIX
|
||||
* _standalone-session.ts — buildStandaloneSession + persistStandaloneResult (sync fallback persist)
|
||||
* _llm-client.ts — callLLM + createLLMClient (sync fallback / test-model)
|
||||
* _image-analyzer.ts — analyzeImageForAI + getImageVerdict (vision-cascade rubric)
|
||||
* config.ts — health, config, models, stage-assignments, categories, quick, test-model
|
||||
* analyze.ts — POST /analyze, /analyze-media, /analyze-async (shared dispatcher)
|
||||
* analyze-image.ts — POST /analyze-image (sync image-only fast path)
|
||||
* results.ts — GET /results/:sessionId
|
||||
*
|
||||
* Mounted at /api/v3/ai-tampered in src/index.ts.
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import configRouter from './config';
|
||||
import analyzeRouter from './analyze';
|
||||
import analyzeImageRouter from './analyze-image';
|
||||
import resultsRouter from './results';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(configRouter);
|
||||
router.use(analyzeRouter);
|
||||
router.use(analyzeImageRouter);
|
||||
router.use(resultsRouter);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* GET /results/:sessionId — fetch per-stage ai-tampered results via SCAN
|
||||
* (cursor-based, never KEYS — keys.ts pattern uses dashed component name).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { AgentKeys } from '../../shared/redis/keys';
|
||||
import { scanKeys } from '../../shared/redis/scan';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/results/:sessionId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const r = getRedis();
|
||||
|
||||
const keys = await scanKeys(r, AgentKeys.componentPattern(sessionId, 'ai-tampered'));
|
||||
const results: Record<string, any> = {};
|
||||
|
||||
for (const key of keys) {
|
||||
const stage = key.split(':').pop()!;
|
||||
const data = await r.get(key);
|
||||
results[stage] = data ? JSON.parse(data) : null;
|
||||
}
|
||||
|
||||
if (Object.keys(results).length === 0) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: `No results found for session ${sessionId}`,
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ success: true, data: { session_id: sessionId, results } });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'ai_tampered_results');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Re-export of the new claims barrel. Kept at this path so src/index.ts
|
||||
* (which imports `./api/claims-routes`) continues to work unchanged after
|
||||
* the 668-LOC → 8-file split. See ./claims/index.ts for the routing map.
|
||||
*/
|
||||
export { default } from './claims';
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Shared lazy singletons + constants for claims routes.
|
||||
*/
|
||||
import { lazyRedis } from '../../shared/redis/connection';
|
||||
import { PersistService, PgSessionAdapter, getPgPool } from '../../shared/persistence';
|
||||
import { SessionStore } from '../../shared/redis/session-store';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
|
||||
export const REDIS_PREFIX = ConfigKeys.claimsPrefix;
|
||||
|
||||
export const getRedis = lazyRedis('claims-routes');
|
||||
|
||||
let _persistService: PersistService | null = null;
|
||||
|
||||
export function getPersistServiceInstance(): PersistService {
|
||||
if (!_persistService) {
|
||||
const r = getRedis();
|
||||
_persistService = new PersistService(
|
||||
new SessionStore(r),
|
||||
new PgSessionAdapter(getPgPool()),
|
||||
);
|
||||
}
|
||||
return _persistService;
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Inline LLM client used by the sync fallback path (when RabbitMQ is down and
|
||||
* we have to run ClaimsExecutor inline). Picks a model from Redis by model_key,
|
||||
* sends a chat-completions request, and tracks usage if a tracker array is
|
||||
* passed in `options._usage_tracker`.
|
||||
*
|
||||
* The dispatcher path uses ClaimsExecutor's own LLM client through the worker;
|
||||
* this is a separate, smaller client for the single-call sync fallback.
|
||||
*/
|
||||
import { getRedis, REDIS_PREFIX } from './_init';
|
||||
|
||||
export function createLLMClient() {
|
||||
return {
|
||||
async call(prompt: string, systemPrompt: string, options: any): Promise<string> {
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
if (!modelsData) throw new Error('Models not configured');
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === options.model_key);
|
||||
if (!model) throw new Error(`Model ${options.model_key} not found`);
|
||||
|
||||
const baseUrl = model.provider_config?.base_url || 'https://openrouter.ai/api/v1';
|
||||
const authType = model.provider_config?.auth_type || 'bearer';
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
|
||||
if (authType === 'bearer') {
|
||||
const apiKey = process.env[`${model.provider?.toUpperCase()}_API_KEY`] || process.env.OPENROUTER_API_KEY;
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
} else if (authType === 'x-api-key') {
|
||||
headers['x-api-key'] = process.env[`${model.provider?.toUpperCase()}_API_KEY`] || '';
|
||||
}
|
||||
|
||||
if (model.provider === 'openrouter' || baseUrl.includes('openrouter.ai')) {
|
||||
headers['HTTP-Referer'] = 'https://didi.ai';
|
||||
headers['X-Title'] = 'DIDI Agent V3 - Claims';
|
||||
}
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model: model.model_code,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: prompt },
|
||||
],
|
||||
max_tokens: options.max_tokens || 2000,
|
||||
temperature: options.temperature || 0.2,
|
||||
};
|
||||
|
||||
if (model.provider_routing?.length > 0) {
|
||||
body.provider = { order: model.provider_routing, allow_fallbacks: true };
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(options.timeout_ms || 30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`LLM API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number };
|
||||
};
|
||||
|
||||
if (options._usage_tracker && Array.isArray(options._usage_tracker) && data.usage) {
|
||||
options._usage_tracker.push({
|
||||
model: model.model_code,
|
||||
prompt_tokens: data.usage.prompt_tokens || 0,
|
||||
completion_tokens: data.usage.completion_tokens || 0,
|
||||
total_tokens: data.usage.total_tokens || (data.usage.prompt_tokens || 0) + (data.usage.completion_tokens || 0),
|
||||
});
|
||||
}
|
||||
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
/**
|
||||
* Media-to-text extraction for claims:
|
||||
* - Image → vision OCR (Redis-configured prompt with safe default)
|
||||
* - Audio → Whisper transcription
|
||||
* - Video → transcript + visual analysis (merged)
|
||||
* - URL → M17 Web API with direct-fetch fallback (HTML strip)
|
||||
*
|
||||
* extractTextContent dispatches by inputType. Used by the async dispatcher to
|
||||
* pre-extract content for the URL path so the worker doesn't have to refetch.
|
||||
*/
|
||||
import { callVision } from '../../shared/media/vision';
|
||||
import { transcribe } from '../../shared/media/transcription';
|
||||
import { processVideoUrl } from '../../shared/media/video-processor';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import { validateExternalUrl } from '../../shared/helpers/validate-url';
|
||||
import { MAX_TEXT_LENGTH } from '../../config/analysisLimits';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
export async function extractTextFromImage(imageUrl: string, tier: 'free' | 'premium' = 'free'): Promise<string> {
|
||||
const redis = getRedis();
|
||||
|
||||
let systemPrompt = '';
|
||||
let userPrompt = 'Extract the main text content from this image. Return ONLY the actual message, post, article text, or statement visible in the image. Do NOT describe UI elements. If there is no meaningful text, respond with NO_TEXT_FOUND.';
|
||||
|
||||
try {
|
||||
const promptData = await redis.get(ConfigKeys.visionPromptExtraction);
|
||||
if (promptData) {
|
||||
const parsed = JSON.parse(promptData);
|
||||
if (parsed.system) systemPrompt = parsed.system;
|
||||
if (parsed.user_template) userPrompt = parsed.user_template;
|
||||
}
|
||||
} catch {
|
||||
log.warn('[Claims] Failed to load vision prompt from Redis, using fallback');
|
||||
}
|
||||
|
||||
const messages: any[] = [];
|
||||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: userPrompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await callVision(redis, messages, { max_tokens: 1500 }, tier);
|
||||
log.info(`[Claims] Image OCR via ${result.provider} (tier: ${tier})`);
|
||||
return result.content;
|
||||
}
|
||||
|
||||
export async function extractVideoContent(
|
||||
videoUrl: string,
|
||||
sessionId: string,
|
||||
): Promise<{ transcript: string; visual_text: string }> {
|
||||
const videoResult = await processVideoUrl(videoUrl, sessionId, {
|
||||
redis: getRedis(),
|
||||
maxFrames: 3,
|
||||
logPrefix: `${sessionId} Claims`,
|
||||
});
|
||||
|
||||
return {
|
||||
transcript: videoResult.transcript,
|
||||
visual_text: videoResult.visual_analysis,
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchUrlContent(url: string): Promise<string> {
|
||||
validateExternalUrl(url);
|
||||
const M17_WEB_API = process.env.M17_WEB_API_URL;
|
||||
if (!M17_WEB_API) throw new Error('M17_WEB_API_URL env var is not set');
|
||||
|
||||
try {
|
||||
const response = await fetch(`${M17_WEB_API}/v1/fetch`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ urls: [url], extract_text: true }),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json() as { pages?: { title?: string; text: string }[] };
|
||||
if (data.pages?.[0]?.text) {
|
||||
const page = data.pages[0];
|
||||
return (page.title ? `Title: ${page.title}\n\n` : '') + page.text;
|
||||
}
|
||||
}
|
||||
} catch { /* fallback */ }
|
||||
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DIDI-Bot/1.0)' },
|
||||
});
|
||||
|
||||
if (!response.ok) throw new Error(`Failed to fetch URL: ${response.status}`);
|
||||
|
||||
const html = await response.text();
|
||||
return html
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.substring(0, MAX_TEXT_LENGTH);
|
||||
}
|
||||
|
||||
export async function extractTextContent(
|
||||
inputType: string,
|
||||
opts: { text?: string; media_url?: string; url?: string; sessionId: string },
|
||||
): Promise<{ text: string; metadata: any }> {
|
||||
let contentToAnalyze = opts.text || '';
|
||||
let metadata: any = null;
|
||||
|
||||
if (inputType === 'image' && opts.media_url) {
|
||||
log.info(`[${opts.sessionId}] Extracting text from image...`);
|
||||
try {
|
||||
contentToAnalyze = await extractTextFromImage(opts.media_url);
|
||||
} catch (visionError) {
|
||||
log.warn(`[${opts.sessionId}] Vision extraction failed: ${(visionError as Error).message}`);
|
||||
contentToAnalyze = '';
|
||||
}
|
||||
|
||||
} else if (inputType === 'audio' && opts.media_url) {
|
||||
log.info(`[${opts.sessionId}] Transcribing audio...`);
|
||||
const transcription = await transcribe(opts.media_url, { logPrefix: `${opts.sessionId} Claims` });
|
||||
if (!transcription.success) {
|
||||
throw new Error(transcription.error || 'Transcription failed');
|
||||
}
|
||||
contentToAnalyze = transcription.text;
|
||||
metadata = { provider: transcription.provider, duration_ms: transcription.duration_ms };
|
||||
|
||||
} else if (inputType === 'video' && opts.media_url) {
|
||||
log.info(`[${opts.sessionId}] Processing video...`);
|
||||
const videoContent = await extractVideoContent(opts.media_url, opts.sessionId);
|
||||
const parts: string[] = [];
|
||||
if (videoContent.transcript) parts.push(`[TRANSCRIPT]\n${videoContent.transcript}`);
|
||||
if (videoContent.visual_text) parts.push(`[VISUAL TEXT]\n${videoContent.visual_text}`);
|
||||
contentToAnalyze = parts.join('\n\n');
|
||||
metadata = {
|
||||
has_transcript: !!videoContent.transcript,
|
||||
has_visual_text: !!videoContent.visual_text,
|
||||
};
|
||||
|
||||
} else if (inputType === 'url' && (opts.url || opts.media_url)) {
|
||||
log.info(`[${opts.sessionId}] Fetching URL content...`);
|
||||
contentToAnalyze = await fetchUrlContent(opts.url || opts.media_url!);
|
||||
}
|
||||
|
||||
return { text: contentToAnalyze, metadata };
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Build + persist a standalone (single-component) AnalysisSession for the claims
|
||||
* sync fallback path. Mirrors the pipeline session shape but with only the claims
|
||||
* slot populated — used when RabbitMQ is unavailable and we run the executor inline.
|
||||
*
|
||||
* Persist is fire-and-forget (logs on failure, never throws to the caller).
|
||||
*/
|
||||
import type { AnalysisSession, InputType } from '../../shared/types/analysis-session';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getPersistServiceInstance } from './_init';
|
||||
|
||||
export function buildStandaloneSession(opts: {
|
||||
sessionId: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType: InputType;
|
||||
inputText?: string;
|
||||
mediaUrl?: string;
|
||||
result: any;
|
||||
durationMs: number;
|
||||
llmUsage?: any[];
|
||||
}): AnalysisSession {
|
||||
const now = new Date().toISOString();
|
||||
let llm_usage = null;
|
||||
if (opts.llmUsage && opts.llmUsage.length > 0) {
|
||||
const summary = { calls: opts.llmUsage.length, prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
|
||||
for (const e of opts.llmUsage) {
|
||||
summary.prompt_tokens += e.prompt_tokens;
|
||||
summary.completion_tokens += e.completion_tokens;
|
||||
summary.total_tokens += e.total_tokens;
|
||||
}
|
||||
llm_usage = { total: summary, by_component: { claims: summary } };
|
||||
}
|
||||
return {
|
||||
session_id: opts.sessionId,
|
||||
user_id: opts.userId || null,
|
||||
user_email: opts.userEmail || null,
|
||||
input_type: opts.inputType,
|
||||
input_text: opts.inputText || null,
|
||||
input_url: null,
|
||||
input_media_url: opts.mediaUrl || null,
|
||||
input_hash: null,
|
||||
status: 'completed',
|
||||
components_run: ['claims'],
|
||||
components_skipped: ['techniques', 'ai_tampered', 'domain', 'verdict'],
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: new Date(Date.now() - opts.durationMs).toISOString(),
|
||||
completed_at: now,
|
||||
total_duration_ms: opts.durationMs,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'web',
|
||||
api_version: 'v3',
|
||||
created_at: now,
|
||||
techniques: null,
|
||||
ai_tampered: null,
|
||||
claims: opts.result,
|
||||
domain: null,
|
||||
source_assessment: null,
|
||||
verdict: null,
|
||||
llm_usage,
|
||||
};
|
||||
}
|
||||
|
||||
export function persistStandaloneResult(opts: {
|
||||
sessionId: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType: InputType;
|
||||
inputText?: string;
|
||||
mediaUrl?: string;
|
||||
result: any;
|
||||
durationMs: number;
|
||||
llmUsage?: any[];
|
||||
}): void {
|
||||
if (!opts.userId) return;
|
||||
const session = buildStandaloneSession(opts);
|
||||
getPersistServiceInstance().persist(session)
|
||||
.catch(err => log.error('[Claims] Persist error:', (err as Error).message));
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
/**
|
||||
* Async claims dispatch — used by all three analyze endpoints (text/media/url).
|
||||
*
|
||||
* Flow:
|
||||
* 1) Validate input shape (one of text/media_url/url) + media_type.
|
||||
* 2) Credit check via shared credits service (skipped if no user_id).
|
||||
* 3) For URL inputs without text: pre-fetch via M17/direct-fetch fallback.
|
||||
* 4) Validate extracted/provided text against analysisLimits (text/url only).
|
||||
* 5) Dispatch to RabbitMQ; if dispatcher returns !async, fall back to inline
|
||||
* ClaimsExecutor and persist a standalone session.
|
||||
* 6) On async success: deduct credits BEFORE responding 202 so a credit
|
||||
* deduction failure can't be lost behind a successful 202.
|
||||
*
|
||||
* Same handler is wired to /analyze, /analyze-media, and /analyze-async — they
|
||||
* all behave identically (the route prefix is purely for client clarity).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { checkCredits, deductCredits } from '../../shared/credits';
|
||||
import { validateTextInput } from '../../config/analysisLimits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import type { InputType } from '../../shared/types/analysis-session';
|
||||
import { toClaimsResult } from '../../components/component-runner';
|
||||
import { getRedis } from './_init';
|
||||
import { extractTextContent } from './_media-extraction';
|
||||
import { createLLMClient } from './_llm-client';
|
||||
import { persistStandaloneResult } from './_standalone-session';
|
||||
|
||||
const router = Router();
|
||||
|
||||
async function dispatchClaimsAsync(req: Request, res: Response) {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_uid, user_email: body_email, plan_type = 1 } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
const validTypes = ['text', 'image', 'audio', 'video', 'url'];
|
||||
const inputType = media_type || (text ? 'text' : url ? 'url' : null);
|
||||
if (!inputType || !validTypes.includes(inputType)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, inputType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sessionId = randomUUID();
|
||||
log.info(`[${sessionId}] Claims analysis (async), type: ${inputType}`);
|
||||
|
||||
let content = text || '';
|
||||
if (inputType === 'url' && (url || media_url) && !text) {
|
||||
try {
|
||||
const extracted = await extractTextContent('url', { url: url || media_url, sessionId });
|
||||
content = extracted.text;
|
||||
} catch (e) {
|
||||
log.warn(`[${sessionId}] Failed to fetch URL: ${(e as Error).message}`);
|
||||
content = `URL: ${url || media_url}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (['text', 'url'].includes(inputType) && content && content.trim().length > 0) {
|
||||
const textValidation = validateTextInput(content);
|
||||
if (!textValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false, error: textValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: textValidation.error_code, stats: textValidation.stats },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const { dispatch } = await import('../../queue/dispatcher');
|
||||
type PlanType = 1 | 2 | 3 | 4 | 5 | 6;
|
||||
const planTypeNum = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
|
||||
const result = await dispatch(
|
||||
sessionId,
|
||||
{ content, url, mediaPath: media_url, userId: user_id, userEmail: user_email, inputType },
|
||||
planTypeNum,
|
||||
['claims'],
|
||||
);
|
||||
|
||||
if (!result.async) {
|
||||
const { ClaimsExecutor } = await import('../../components/claims/executor');
|
||||
const { wrapWithUsageTracker } = await import('../../components/component-runner');
|
||||
const r = getRedis();
|
||||
const usageTracker: any[] = [];
|
||||
const trackedClient = wrapWithUsageTracker(createLLMClient(), usageTracker);
|
||||
const executor = new ClaimsExecutor(r, trackedClient);
|
||||
const rawSyncResult = await executor.execute(content, sessionId);
|
||||
const syncResult = toClaimsResult(rawSyncResult);
|
||||
|
||||
persistStandaloneResult({
|
||||
sessionId, userId: user_id, userEmail: user_email,
|
||||
inputType: inputType as InputType,
|
||||
inputText: inputType === 'text' ? content : undefined,
|
||||
mediaUrl: media_url, result: syncResult, durationMs: Date.now() - Date.now(),
|
||||
llmUsage: usageTracker,
|
||||
});
|
||||
|
||||
return res.json({ success: true, async: false, data: { session_id: sessionId, media_type: inputType, result: syncResult } });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, inputType, sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType}`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType} err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
media_type: inputType,
|
||||
queued_components: result.queued,
|
||||
plan_type: planTypeNum,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_dispatch_async');
|
||||
}
|
||||
}
|
||||
|
||||
router.post('/analyze-async', dispatchClaimsAsync);
|
||||
router.post('/analyze', dispatchClaimsAsync);
|
||||
router.post('/analyze-media', dispatchClaimsAsync);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
/**
|
||||
* Claims config + introspection endpoints.
|
||||
*
|
||||
* GET /health
|
||||
* GET /config — manifest + claim types/statuses + scoring config
|
||||
* GET /types — claim_types from didi:framework:claims
|
||||
* GET /statuses — claim_statuses from didi:framework:claims
|
||||
* GET /models — available models
|
||||
* GET /stage-assignments — per-tier stage→model mapping
|
||||
* PUT /stage-assignments — zod-validated update
|
||||
* POST /test-model — connectivity probe (returns the upstream error
|
||||
* intentionally so the operator can debug)
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { FrameworkKeys } from '../../shared/redis/keys';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { TierStageAssignmentsSchema } from '../../shared/helpers/config-schemas';
|
||||
import { getRedis, REDIS_PREFIX } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/health', (_req: Request, res: Response) => {
|
||||
res.json({ success: true, service: 'claims-v1', version: '1.0.0', timestamp: new Date().toISOString() });
|
||||
});
|
||||
|
||||
router.get('/config', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const [manifest, frameworkClaims] = await Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:manifest`),
|
||||
r.get(FrameworkKeys.claims),
|
||||
]);
|
||||
|
||||
let claimTypes = null, claimStatuses = null, confidenceLevels = null;
|
||||
if (frameworkClaims) {
|
||||
const claims = JSON.parse(frameworkClaims);
|
||||
claimTypes = claims.types || null;
|
||||
claimStatuses = claims.status || null;
|
||||
confidenceLevels = claims.confidence || null;
|
||||
}
|
||||
|
||||
let scoringConfig = null;
|
||||
try {
|
||||
const scRaw = await r.get(`${REDIS_PREFIX}:scoring_config`);
|
||||
if (scRaw) scoringConfig = JSON.parse(scRaw);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
manifest: manifest ? JSON.parse(manifest) : null,
|
||||
claim_types: claimTypes,
|
||||
claim_statuses: claimStatuses,
|
||||
confidence_levels: confidenceLevels,
|
||||
scoring_config: scoringConfig,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_config');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/types', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(FrameworkKeys.claims);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'didi:framework:claims not found. Sync from didiFramework first.' });
|
||||
res.json({ success: true, data: JSON.parse(data).types || [] });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_types');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/statuses', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(FrameworkKeys.claims);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'didi:framework:claims not found. Sync from didiFramework first.' });
|
||||
res.json({ success: true, data: JSON.parse(data).status || [] });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_statuses');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/models', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'Models not configured in Redis' });
|
||||
res.json({ success: true, data: JSON.parse(data) });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_models');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/stage-assignments', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${REDIS_PREFIX}:stage_assignments`);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'Stage assignments not configured in Redis' });
|
||||
res.json({ success: true, data: JSON.parse(data) });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_get_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/stage-assignments', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const parsed = TierStageAssignmentsSchema.safeParse(req.body?.stage_assignments);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid stage_assignments payload',
|
||||
details: parsed.error.issues,
|
||||
});
|
||||
}
|
||||
const r = getRedis();
|
||||
await r.set(`${REDIS_PREFIX}:stage_assignments`, JSON.stringify(parsed.data));
|
||||
res.json({ success: true, message: 'Stage assignments updated' });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_put_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/test-model', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { model_key } = req.body;
|
||||
if (!model_key) return res.status(400).json({ success: false, error: 'model_key is required' });
|
||||
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
if (!modelsData) return res.status(404).json({ success: false, error: 'Models not configured' });
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === model_key);
|
||||
if (!model) return res.status(404).json({ success: false, error: `Model ${model_key} not found` });
|
||||
|
||||
const startTime = Date.now();
|
||||
const baseUrl = model.provider_config?.base_url || 'https://openrouter.ai/api/v1';
|
||||
const apiKey = process.env[`${model.provider?.toUpperCase()}_API_KEY`] || process.env.OPENROUTER_API_KEY;
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (model.provider_config?.auth_type === 'bearer') headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
if (model.provider === 'openrouter' || baseUrl.includes('openrouter.ai')) {
|
||||
headers['HTTP-Referer'] = 'https://didi.ai';
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ model: model.model_code, messages: [{ role: 'user', content: 'Reply with "OK"' }], max_tokens: 10 }),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
return res.json({ success: true, data: { model_key, status: 'error', error: `API error: ${response.status} - ${errorText.substring(0, 200)}` } });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: { model_key, status: 'connected', response_time_ms: Date.now() - startTime } });
|
||||
} catch (error) {
|
||||
res.json({ success: true, data: { model_key: req.body.model_key, status: 'error', error: (error as Error).message } });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/**
|
||||
* AGENT V3 — CLAIMS API barrel router.
|
||||
*
|
||||
* Original 668-line claims-routes.ts split into:
|
||||
* _init.ts — lazyRedis + persistService + REDIS_PREFIX
|
||||
* _standalone-session.ts — buildStandaloneSession + persistStandaloneResult (sync fallback persist)
|
||||
* _llm-client.ts — createLLMClient (used by sync fallback only)
|
||||
* _media-extraction.ts — image/audio/video/url → text helpers
|
||||
* config.ts — health, config, types, statuses, models, stage-assignments, test-model
|
||||
* analyze.ts — POST /analyze, /analyze-media, /analyze-async (shared dispatcher)
|
||||
* results.ts — GET /results/:sessionId
|
||||
*
|
||||
* Mounted at /api/v3/claims in src/index.ts.
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import configRouter from './config';
|
||||
import analyzeRouter from './analyze';
|
||||
import resultsRouter from './results';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(configRouter);
|
||||
router.use(analyzeRouter);
|
||||
router.use(resultsRouter);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* GET /results/:sessionId — fetch the per-stage claims results for a session
|
||||
* by SCAN-ing the agent's component-result keys (cursor-based, never KEYS).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { AgentKeys } from '../../shared/redis/keys';
|
||||
import { scanKeys } from '../../shared/redis/scan';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/results/:sessionId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const r = getRedis();
|
||||
|
||||
const keys = await scanKeys(r, AgentKeys.componentPattern(sessionId, 'claims'));
|
||||
const results: Record<string, any> = {};
|
||||
|
||||
for (const key of keys) {
|
||||
const stage = key.split(':').pop()!;
|
||||
const data = await r.get(key);
|
||||
results[stage] = data ? JSON.parse(data) : null;
|
||||
}
|
||||
|
||||
if (Object.keys(results).length === 0) {
|
||||
return res.status(404).json({ success: false, error: `No results found for session ${sessionId}` });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: { session_id: sessionId, results } });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'claims_results');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
335
backend/services/orchestration-layer/agent-v3/src/api/domain.ts
Normal file
335
backend/services/orchestration-layer/agent-v3/src/api/domain.ts
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
/**
|
||||
* AGENT V3 — DOMAIN routes (extracted from routes.ts).
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /domain/analyze-async — async via RabbitMQ
|
||||
* POST /domain/analyze — sync, calls Domain Check API directly
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { resolveUserId } from '../shared/auth/guards';
|
||||
import crypto from 'crypto';
|
||||
import { internalError } from '../shared/helpers/error-response';
|
||||
import { log } from '../shared/logger';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/domain/analyze-async - Async domain analysis via RabbitMQ
|
||||
// ============================================================================
|
||||
router.post('/domain/analyze-async', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { domain, url, plan_type = 1, user_id: body_uid, user_email: body_email } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
// Extract domain from URL if provided
|
||||
let targetDomain = domain;
|
||||
if (!targetDomain && url) {
|
||||
try {
|
||||
targetDomain = new URL(url).hostname.replace(/^www\./, '');
|
||||
} catch {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid URL provided',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetDomain) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'domain or url is required',
|
||||
});
|
||||
}
|
||||
|
||||
const sessionId = crypto.randomUUID();
|
||||
|
||||
// Import dispatcher
|
||||
const { dispatch } = await import('../queue/dispatcher');
|
||||
type PlanType = 1 | 2 | 3 | 4 | 5 | 6;
|
||||
|
||||
const planTypeNum = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
|
||||
const result = await dispatch(
|
||||
sessionId,
|
||||
{ content: '', url: url || `https://${targetDomain}`, userId: user_id, userEmail: user_email, inputType: 'url' },
|
||||
planTypeNum,
|
||||
['domain'] // Only domain component
|
||||
);
|
||||
|
||||
if (!result.async) {
|
||||
// Fallback to sync - domain analysis is fast, just do it sync
|
||||
// Call the sync domain analysis logic directly
|
||||
return res.redirect(307, '/api/v3/domain/analyze');
|
||||
}
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
queued_components: result.queued,
|
||||
plan_type: planTypeNum,
|
||||
domain: targetDomain,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Domain Async] Error:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/domain/analyze - Domain analysis for misinformation detection
|
||||
// ============================================================================
|
||||
router.post('/domain/analyze', async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const { domain, url } = req.body;
|
||||
|
||||
// Extract domain from URL if provided
|
||||
let targetDomain = domain;
|
||||
if (!targetDomain && url) {
|
||||
try {
|
||||
targetDomain = new URL(url).hostname.replace(/^www\./, '');
|
||||
} catch {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid URL provided',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetDomain) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'domain or url is required',
|
||||
});
|
||||
}
|
||||
|
||||
log.info(`[domain/analyze] Checking domain: ${targetDomain}`);
|
||||
|
||||
const DOMAIN_CHECK_API = process.env.DOMAIN_CHECK_API_URL || 'http://domain-check-api:11000/api/v1/check/check';
|
||||
|
||||
const response = await fetch(DOMAIN_CHECK_API, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
domain: targetDomain,
|
||||
check_options: {
|
||||
whois: true,
|
||||
dns: true,
|
||||
ssl: true,
|
||||
blacklist: true,
|
||||
ip_intelligence: true,
|
||||
http_analysis: true,
|
||||
},
|
||||
}),
|
||||
signal: AbortSignal.timeout(90000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Domain Check API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const apiData = await response.json() as {
|
||||
success: boolean;
|
||||
data?: {
|
||||
domain: string;
|
||||
whois: {
|
||||
creation_date: string | null;
|
||||
age_days: number | null;
|
||||
registrar: string | null;
|
||||
registrant_org: string | null;
|
||||
registrant_country: string | null;
|
||||
};
|
||||
dns: {
|
||||
a_records: string[];
|
||||
has_spf: boolean;
|
||||
has_dmarc: boolean;
|
||||
};
|
||||
ssl: {
|
||||
has_ssl: boolean;
|
||||
is_valid: boolean;
|
||||
is_self_signed: boolean;
|
||||
issuer: string | null;
|
||||
days_until_expiry: number | null;
|
||||
};
|
||||
blacklist: {
|
||||
is_blacklisted: boolean;
|
||||
reputation_score: number;
|
||||
risk_level: string;
|
||||
};
|
||||
ip_intelligence: {
|
||||
country: string | null;
|
||||
isp: string | null;
|
||||
is_datacenter: boolean;
|
||||
};
|
||||
risk_score: {
|
||||
total: number;
|
||||
level: string;
|
||||
is_suspicious: boolean;
|
||||
is_new_domain: boolean;
|
||||
};
|
||||
};
|
||||
error?: string;
|
||||
};
|
||||
|
||||
if (!apiData.success || !apiData.data) {
|
||||
throw new Error(apiData.error || 'Domain check failed');
|
||||
}
|
||||
|
||||
const d = apiData.data;
|
||||
|
||||
// Calculate trust score (inverse of risk)
|
||||
const trustScore = Math.max(0, 100 - (d.risk_score?.total ?? 50));
|
||||
|
||||
// Determine age category
|
||||
let ageCategory: string;
|
||||
const ageDays = d.whois?.age_days ?? null;
|
||||
if (ageDays === null) {
|
||||
ageCategory = 'UNKNOWN';
|
||||
} else if (ageDays < 30) {
|
||||
ageCategory = 'VERY_NEW';
|
||||
} else if (ageDays < 180) {
|
||||
ageCategory = 'NEW';
|
||||
} else if (ageDays < 365) {
|
||||
ageCategory = 'LESS_THAN_1_YEAR';
|
||||
} else if (ageDays < 730) {
|
||||
ageCategory = 'LESS_THAN_2_YEARS';
|
||||
} else if (ageDays < 1825) {
|
||||
ageCategory = 'ESTABLISHED';
|
||||
} else {
|
||||
ageCategory = 'WELL_ESTABLISHED';
|
||||
}
|
||||
|
||||
// Collect red flags for misinformation
|
||||
const redFlags: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Age-based flags
|
||||
if (ageDays !== null && ageDays < 30) {
|
||||
redFlags.push('DOMAIN_VERY_NEW');
|
||||
} else if (ageDays !== null && ageDays < 180) {
|
||||
warnings.push('DOMAIN_RELATIVELY_NEW');
|
||||
}
|
||||
|
||||
// Blacklist flags
|
||||
if (d.blacklist?.is_blacklisted) {
|
||||
redFlags.push('BLACKLISTED');
|
||||
}
|
||||
const reputationScore = d.blacklist?.reputation_score ?? 100;
|
||||
if (reputationScore < 30) {
|
||||
redFlags.push('LOW_REPUTATION');
|
||||
} else if (reputationScore < 60) {
|
||||
warnings.push('MEDIUM_REPUTATION');
|
||||
}
|
||||
|
||||
// SSL flags
|
||||
if (!d.ssl?.has_ssl) {
|
||||
redFlags.push('NO_SSL');
|
||||
} else if (!d.ssl?.is_valid) {
|
||||
redFlags.push('INVALID_SSL');
|
||||
} else if (d.ssl?.is_self_signed) {
|
||||
warnings.push('SELF_SIGNED_SSL');
|
||||
}
|
||||
|
||||
// DNS flags
|
||||
if ((d.dns?.a_records?.length ?? 0) === 0) {
|
||||
redFlags.push('NO_DNS_RECORDS');
|
||||
}
|
||||
if (!d.dns?.has_spf && !d.dns?.has_dmarc) {
|
||||
warnings.push('NO_EMAIL_SECURITY');
|
||||
}
|
||||
|
||||
// Risk flags
|
||||
if (d.risk_score?.is_suspicious) {
|
||||
redFlags.push('SUSPICIOUS_DOMAIN');
|
||||
}
|
||||
if (d.ip_intelligence?.is_datacenter) {
|
||||
warnings.push('HOSTED_IN_DATACENTER');
|
||||
}
|
||||
|
||||
// WHOIS privacy
|
||||
if (!d.whois?.registrant_org) {
|
||||
warnings.push('WHOIS_PRIVACY_ENABLED');
|
||||
}
|
||||
|
||||
// Determine overall verdict
|
||||
let verdict: 'TRUSTED' | 'NEUTRAL' | 'SUSPICIOUS' | 'UNTRUSTED';
|
||||
if (redFlags.length >= 2 || d.blacklist?.is_blacklisted) {
|
||||
verdict = 'UNTRUSTED';
|
||||
} else if (redFlags.length === 1 || warnings.length >= 3) {
|
||||
verdict = 'SUSPICIOUS';
|
||||
} else if (ageDays !== null && ageDays > 730 && reputationScore >= 80) {
|
||||
verdict = 'TRUSTED';
|
||||
} else {
|
||||
verdict = 'NEUTRAL';
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
domain: targetDomain,
|
||||
verdict,
|
||||
trust_score: trustScore,
|
||||
risk_level: d.risk_score?.level ?? 'UNKNOWN',
|
||||
|
||||
// Age info
|
||||
age: {
|
||||
days: ageDays,
|
||||
category: ageCategory,
|
||||
created_at: d.whois?.creation_date ?? null,
|
||||
},
|
||||
|
||||
// Blacklist info
|
||||
blacklist: {
|
||||
is_blacklisted: d.blacklist?.is_blacklisted ?? false,
|
||||
reputation_score: reputationScore,
|
||||
},
|
||||
|
||||
// SSL info
|
||||
ssl: {
|
||||
has_ssl: d.ssl?.has_ssl ?? false,
|
||||
is_valid: d.ssl?.is_valid ?? false,
|
||||
issuer: d.ssl?.issuer ?? null,
|
||||
},
|
||||
|
||||
// Ownership info
|
||||
ownership: {
|
||||
registrar: d.whois?.registrar ?? null,
|
||||
organization: d.whois?.registrant_org ?? null,
|
||||
country: d.whois?.registrant_country || d.ip_intelligence?.country || null,
|
||||
},
|
||||
|
||||
// Flags
|
||||
red_flags: redFlags,
|
||||
warnings,
|
||||
|
||||
// Metadata
|
||||
metadata: {
|
||||
duration_ms: duration,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[domain/analyze] Error:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
export default router;
|
||||
|
||||
172
backend/services/orchestration-layer/agent-v3/src/api/media.ts
Normal file
172
backend/services/orchestration-layer/agent-v3/src/api/media.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* AGENT V3 — MEDIA routes (extracted from routes.ts).
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /media/upload-url — presigned MinIO upload URL
|
||||
* POST /media/upload — direct buffer upload via multer
|
||||
* POST /media/download-url — presigned download URL
|
||||
* GET /media/file/:bucket/{*objectKey} — proxy stream
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { resolveUserId } from '../shared/auth/guards';
|
||||
import { internalError } from '../shared/helpers/error-response';
|
||||
import { log } from '../shared/logger';
|
||||
import { getMediaService, upload } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/media/upload-url - Generate presigned upload URL
|
||||
// ============================================================================
|
||||
router.post('/media/upload-url', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { filename, content_type, user_id: body_uid } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
|
||||
if (!filename) {
|
||||
return res.status(400).json({ success: false, error: 'filename is required' });
|
||||
}
|
||||
if (!user_id) {
|
||||
return res.status(400).json({ success: false, error: 'user_id (keycloak_id) is required' });
|
||||
}
|
||||
|
||||
const result = await getMediaService().getPresignedUploadUrl(user_id, filename, content_type);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
upload_url: result.upload_url,
|
||||
download_url: result.download_url,
|
||||
public_url: result.public_url,
|
||||
presigned_url: result.presigned_url,
|
||||
object_key: result.object_key,
|
||||
bucket: result.bucket,
|
||||
expires_in: result.expires_in,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[upload-url] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/media/upload - Direct file upload (multipart/form-data)
|
||||
// Frontend uploads file directly here, backend stores in MinIO
|
||||
// ============================================================================
|
||||
router.post('/media/upload', upload.single('file'), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const file = (req as any).file as Express.Multer.File | undefined;
|
||||
const userId = resolveUserId(req, req.body.user_id);
|
||||
|
||||
if (!file) {
|
||||
return res.status(400).json({ success: false, error: 'No file provided. Use multipart/form-data with field name "file"' });
|
||||
}
|
||||
if (!userId) {
|
||||
return res.status(400).json({ success: false, error: 'user_id (keycloak_id) is required' });
|
||||
}
|
||||
|
||||
const result = await getMediaService().uploadFile(userId, file.buffer, file.originalname, file.mimetype);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
download_url: result.download_url,
|
||||
public_url: result.public_url,
|
||||
object_key: result.object_key,
|
||||
bucket: result.bucket,
|
||||
filename: result.filename,
|
||||
size: result.size,
|
||||
content_type: result.content_type,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[upload] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/media/download-url - Generate presigned download URL
|
||||
// ============================================================================
|
||||
router.post('/media/download-url', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { object_key, bucket: requestBucket } = req.body;
|
||||
if (!object_key) {
|
||||
return res.status(400).json({ success: false, error: 'object_key is required' });
|
||||
}
|
||||
|
||||
const bucket = requestBucket || 'uploads';
|
||||
const result = await getMediaService().getPresignedDownloadUrl(object_key, bucket);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
download_url: result.download_url,
|
||||
public_url: result.public_url,
|
||||
presigned_url: result.presigned_url,
|
||||
object_key,
|
||||
expires_in: null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[download-url] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/media/file/:bucket/:objectKey - Proxy endpoint to serve MinIO files
|
||||
// This bypasses presigned URL signature issues by proxying through the backend
|
||||
// Public URL: https://didi365.eu/agent-v3/api/v3/media/file/uploads/{userId}/{filename}
|
||||
// ============================================================================
|
||||
router.get('/media/file/:bucket/{*objectKey}', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { bucket, objectKey } = req.params;
|
||||
const fullObjectKey = Array.isArray(objectKey) ? objectKey.join('/') : objectKey;
|
||||
|
||||
if (!bucket || !fullObjectKey) {
|
||||
return res.status(400).json({ success: false, error: 'bucket and objectKey are required' });
|
||||
}
|
||||
|
||||
log.info(`[media/file] Proxying ${bucket}/${fullObjectKey}`);
|
||||
|
||||
// Note: ownership check (ownerInternetUserId) can be added when auth middleware provides user info
|
||||
const result = await getMediaService().proxyFile(bucket, fullObjectKey, undefined, req.headers.range);
|
||||
|
||||
if (!result) {
|
||||
return res.status(403).json({ success: false, error: 'Access denied' });
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', result.contentType);
|
||||
res.setHeader('Accept-Ranges', 'bytes');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
|
||||
if (result.isPartial && result.rangeStart !== undefined && result.rangeEnd !== undefined) {
|
||||
res.status(206);
|
||||
res.setHeader('Content-Range', `bytes ${result.rangeStart}-${result.rangeEnd}/${result.totalSize}`);
|
||||
res.setHeader('Content-Length', result.contentLength);
|
||||
} else {
|
||||
res.setHeader('Content-Length', result.contentLength);
|
||||
}
|
||||
|
||||
(result.stream as any).pipe(res);
|
||||
} catch (error) {
|
||||
log.error('[media/file] Error:', error);
|
||||
const errMsg = (error as Error).message;
|
||||
|
||||
const errCode = (error as { code?: string }).code;
|
||||
if (
|
||||
errCode === 'NoSuchKey' ||
|
||||
errMsg.includes('not exist') ||
|
||||
errMsg.includes('NoSuchKey') ||
|
||||
errMsg.includes('Not Found')
|
||||
) {
|
||||
return res.status(404).json({ success: false, error: 'File not found' });
|
||||
}
|
||||
res.status(500).json({ success: false, error: errMsg });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Re-export of the new moderation barrel. Kept at this path so src/index.ts
|
||||
* (which imports `./api/moderation-routes`) continues to work unchanged after
|
||||
* the 422-LOC → 5-file split. See ./moderation/index.ts for the routing map.
|
||||
*/
|
||||
export { default } from './moderation';
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
/**
|
||||
* Brain gold-promotion helpers for the moderation /resolve endpoint.
|
||||
*
|
||||
* promoteAtomsToGold finds the analysis atoms produced for the given session,
|
||||
* applies any moderator corrections, and PATCHes them to gold tier in brain.
|
||||
*
|
||||
* All failures are logged but never thrown — gold promotion is best-effort and
|
||||
* must not block the moderator's resolve action.
|
||||
*/
|
||||
import { log } from '../../shared/logger';
|
||||
import { getPgPool } from '../../shared/persistence/pg-pool';
|
||||
import {
|
||||
computeContentHash,
|
||||
computePromptHash,
|
||||
lookupAnalysisAtom,
|
||||
patchAnalysisAtomGold,
|
||||
type AtomComponent,
|
||||
} from '../../shared/brain/client';
|
||||
|
||||
export async function promoteAtomsToGold(params: {
|
||||
sessionId: string;
|
||||
userId: string;
|
||||
corrections: Record<string, unknown> | null;
|
||||
isCorrected: boolean;
|
||||
}): Promise<void> {
|
||||
const pool = getPgPool();
|
||||
const r = await pool.query(
|
||||
`SELECT input_text, scenario_applied FROM bos_analysis.analysis_session WHERE session_id = $1`,
|
||||
[params.sessionId]
|
||||
);
|
||||
if (r.rows.length === 0 || !r.rows[0].input_text) return;
|
||||
const inputText = r.rows[0].input_text as string;
|
||||
const contentHash = computeContentHash(inputText);
|
||||
|
||||
// We don't know the original prompt_hash/framework_version used at analysis time.
|
||||
// Use current values — in practice the atom was likely written with these too.
|
||||
// If hash mismatch, we miss the lookup → no gold promotion (acceptable).
|
||||
// For tier, use 'premium' (only premium writes atoms).
|
||||
const tier = 'premium';
|
||||
|
||||
const components: AtomComponent[] = ['techniques', 'ai_tampered', 'claims'];
|
||||
for (const component of components) {
|
||||
if (params.isCorrected && params.corrections && !params.corrections[component]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const promptHash = await computePromptHashForComponent(component);
|
||||
if (!promptHash) continue;
|
||||
|
||||
const lookup = await lookupAnalysisAtom({
|
||||
content_hash: contentHash,
|
||||
component,
|
||||
tier,
|
||||
prompt_hash: promptHash,
|
||||
});
|
||||
if (!lookup?.atom) continue;
|
||||
|
||||
const componentCorrections = params.corrections?.[component];
|
||||
const updatedResult = params.isCorrected && componentCorrections
|
||||
? applyCorrections(lookup.atom.result_processed, componentCorrections as Record<string, unknown>)
|
||||
: null;
|
||||
|
||||
const patched = await patchAnalysisAtomGold(lookup.atom.atom_id, {
|
||||
human_validated: true,
|
||||
human_corrections: componentCorrections ? (componentCorrections as Record<string, unknown>) : null,
|
||||
validator_user_id: params.userId,
|
||||
result_processed: updatedResult,
|
||||
});
|
||||
if (patched) {
|
||||
log.info(`[Moderation] Brain atom ${patched.atom_id} (${component}) promoted to gold`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function computePromptHashForComponent(component: AtomComponent): Promise<string | null> {
|
||||
try {
|
||||
const { createRedisConnection } = await import('../../shared/redis/connection');
|
||||
const redis = createRedisConnection({ label: 'moderation-prompt-hash' });
|
||||
try {
|
||||
const versionMap: Record<string, string> = { techniques: 'v3', ai_tampered: 'v1', claims: 'v1' };
|
||||
const v = versionMap[component] ?? 'v1';
|
||||
const stage = component === 'claims' ? 'extraction' : 'screening';
|
||||
const raw = await redis.get(`didi:config:${component.replace('_', '-')}:${v}:prompts:${stage}`);
|
||||
if (!raw) return null;
|
||||
const parsed = JSON.parse(raw) as { system?: string; user_template?: string };
|
||||
return computePromptHash(parsed.system || '', parsed.user_template || '');
|
||||
} finally {
|
||||
redis.disconnect();
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply a JSONB diff to a result. Minimal implementation — handles top-level set/remove. */
|
||||
function applyCorrections(
|
||||
result: Record<string, any>,
|
||||
diff: Record<string, any>,
|
||||
): Record<string, any> {
|
||||
const out = { ...result };
|
||||
for (const [key, value] of Object.entries(diff)) {
|
||||
if (value && typeof value === 'object' && 'from' in value && 'to' in value) {
|
||||
out[key] = (value as { from: unknown; to: unknown }).to;
|
||||
} else if (value === null) {
|
||||
delete out[key];
|
||||
} else {
|
||||
out[key] = value;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
/**
|
||||
* Moderation auth middleware + role constants.
|
||||
*
|
||||
* Implementation moved to shared/auth/guards.ts so pipeline/history routes use
|
||||
* the same auth model; re-exported here to keep existing imports stable.
|
||||
*
|
||||
* Permission matrix per HIL plan Faza B:
|
||||
* - read endpoints (list/detail/stats): admin, moderator, senior_moderator
|
||||
* - write endpoints (claim/resolve) : moderator, senior_moderator
|
||||
* - /flag : any authenticated user (extension + UI)
|
||||
*/
|
||||
export { STAGING, ROLES_READ, ROLES_WRITE, requireRole, requireAuth } from '../../shared/auth/guards';
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* /flag (any authenticated user) + /stats (read-roles) endpoints.
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { getPgPool } from '../../shared/persistence/pg-pool';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import {
|
||||
enqueueForReview,
|
||||
getQueueStats,
|
||||
} from '../../components/moderation/queue-manager';
|
||||
import { requireAuth, requireRole, ROLES_READ } from './_middleware';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ─── POST /flag — user/extension reports content for review ───────────
|
||||
// NOT moderator-restricted: any authenticated user can flag.
|
||||
// Strict auth in production (requires JWT); in staging accepts user_id in body.
|
||||
|
||||
router.post('/flag', requireAuth(), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { session_id, reason, notes } = req.body ?? {};
|
||||
if (!session_id || typeof session_id !== 'string') {
|
||||
return res.status(400).json({ success: false, error: 'session_id (string) required' });
|
||||
}
|
||||
if (!['wrong_verdict', 'missing_techniques', 'wrong_claim', 'other'].includes(reason)) {
|
||||
return res.status(400).json({ success: false, error: 'reason must be one of: wrong_verdict, missing_techniques, wrong_claim, other' });
|
||||
}
|
||||
|
||||
const userId = req.jwtUserId ?? (req.body?.user_id as string | undefined);
|
||||
if (!userId) {
|
||||
return res.status(400).json({ success: false, error: 'No user identity' });
|
||||
}
|
||||
|
||||
const pool = getPgPool();
|
||||
const exists = await pool.query<{ session_id: string }>(
|
||||
'SELECT session_id FROM bos_analysis.analysis_session WHERE session_id = $1',
|
||||
[session_id]
|
||||
);
|
||||
if (exists.rows.length === 0) {
|
||||
return res.status(404).json({ success: false, error: `Session ${session_id} not found` });
|
||||
}
|
||||
|
||||
const queueId = await enqueueForReview({
|
||||
session_id,
|
||||
priority: 1, // user flag = highest priority
|
||||
enqueue_reason: 'flagged',
|
||||
enqueue_meta: { reported_by: userId, reason, notes: notes ?? null },
|
||||
});
|
||||
|
||||
res.json({ success: true, data: { queue_id: queueId }, message: 'Flagged for review' });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_flag');
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /stats — counts + averages for moderator dashboard ───────────
|
||||
|
||||
router.get('/stats', requireRole(...ROLES_READ), async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const stats = await getQueueStats();
|
||||
res.json({ success: true, data: stats });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_stats');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* AGENT V3 — MODERATION API barrel router.
|
||||
*
|
||||
* Original 422-line moderation-routes.ts split into:
|
||||
* _middleware.ts — requireRole/requireAuth + STAGING + ROLES_*
|
||||
* _gold-promotion.ts — promoteAtomsToGold (brain best-effort patch)
|
||||
* queue.ts — GET/POST/PUT /queue/* (list/detail/claim/resolve)
|
||||
* flag-stats.ts — POST /flag, GET /stats
|
||||
*
|
||||
* Mounted at /api/v3/moderation in src/index.ts.
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import queueRouter from './queue';
|
||||
import flagStatsRouter from './flag-stats';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(queueRouter);
|
||||
router.use(flagStatsRouter);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* Moderation queue endpoints — list / detail / claim / resolve.
|
||||
* Mounted at /api/v3/moderation. All endpoints require moderator role from JWT.
|
||||
*
|
||||
* Idempotency: enqueue is idempotent at queue-manager level. Resolve is
|
||||
* non-idempotent (intentional — second resolve attempt should error).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { getPgPool } from '../../shared/persistence/pg-pool';
|
||||
import { log } from '../../shared/logger';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import {
|
||||
listQueue,
|
||||
getQueueEntry,
|
||||
claimQueueEntry,
|
||||
resolveQueueEntry,
|
||||
} from '../../components/moderation/queue-manager';
|
||||
import { requireRole, ROLES_READ, ROLES_WRITE } from './_middleware';
|
||||
import { promoteAtomsToGold } from './_gold-promotion';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ─── GET /queue — list queue entries with filters ─────────────────────
|
||||
|
||||
router.get('/queue', requireRole(...ROLES_READ), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const status = (req.query.status as string | undefined)?.split(',').filter(Boolean);
|
||||
const priority = (req.query.priority as string | undefined)
|
||||
?.split(',')
|
||||
.map((p) => parseInt(p, 10))
|
||||
.filter((p) => !Number.isNaN(p));
|
||||
const assignedToParam = req.query.assigned_to as string | undefined;
|
||||
const limit = Math.min(parseInt((req.query.limit as string) || '20', 10), 100);
|
||||
const offset = parseInt((req.query.offset as string) || '0', 10);
|
||||
|
||||
const assigned_to = assignedToParam === 'me' ? req.jwtUserId : assignedToParam;
|
||||
|
||||
const result = await listQueue({
|
||||
status: status && status.length > 0 ? status : undefined,
|
||||
priority: priority && priority.length > 0 ? priority : undefined,
|
||||
assigned_to,
|
||||
limit,
|
||||
offset,
|
||||
});
|
||||
|
||||
res.json({ success: true, data: result.items, total: result.total, limit, offset });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_list_queue');
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /queue/:queueId — detail (queue + full session) ─────────────
|
||||
|
||||
router.get('/queue/:queueId', requireRole(...ROLES_READ), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const queueId = parseInt(req.params.queueId, 10);
|
||||
if (Number.isNaN(queueId)) {
|
||||
return res.status(400).json({ success: false, error: 'Invalid queueId' });
|
||||
}
|
||||
|
||||
const entry = await getQueueEntry(queueId);
|
||||
if (!entry) {
|
||||
return res.status(404).json({ success: false, error: `Queue entry ${queueId} not found` });
|
||||
}
|
||||
|
||||
const pool = getPgPool();
|
||||
const sessionRow = await pool.query(
|
||||
`SELECT s.*, t.* AS techniques, a.* AS ai_tampered, c.* AS claims, d.* AS domain, v.* AS verdict
|
||||
FROM bos_analysis.analysis_session s
|
||||
LEFT JOIN bos_analysis.analysis_techniques t ON s.session_id = t.session_id
|
||||
LEFT JOIN bos_analysis.analysis_ai_tampered a ON s.session_id = a.session_id
|
||||
LEFT JOIN bos_analysis.analysis_claims c ON s.session_id = c.session_id
|
||||
LEFT JOIN bos_analysis.analysis_domain d ON s.session_id = d.session_id
|
||||
LEFT JOIN bos_analysis.analysis_verdict v ON s.session_id = v.session_id
|
||||
WHERE s.session_id = $1`,
|
||||
[entry.session_id]
|
||||
);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
queue: entry,
|
||||
session: sessionRow.rows[0] ?? null,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_get_queue_entry');
|
||||
}
|
||||
});
|
||||
|
||||
// ─── POST /queue/:queueId/claim — atomic claim by current moderator ───
|
||||
|
||||
router.post('/queue/:queueId/claim', requireRole(...ROLES_WRITE), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const queueId = parseInt(req.params.queueId, 10);
|
||||
if (Number.isNaN(queueId)) {
|
||||
return res.status(400).json({ success: false, error: 'Invalid queueId' });
|
||||
}
|
||||
|
||||
const userId = req.jwtUserId ?? (req.body?.user_id as string | undefined);
|
||||
if (!userId) {
|
||||
return res.status(400).json({ success: false, error: 'No user identity (JWT or body.user_id required)' });
|
||||
}
|
||||
|
||||
const claimed = await claimQueueEntry(queueId, userId);
|
||||
if (!claimed) {
|
||||
return res.status(409).json({ success: false, error: 'Cannot claim — already in review or not pending' });
|
||||
}
|
||||
|
||||
const entry = await getQueueEntry(queueId);
|
||||
res.json({ success: true, data: entry, message: `Claimed by ${userId}` });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_claim');
|
||||
}
|
||||
});
|
||||
|
||||
// ─── PUT /queue/:queueId/resolve — moderator resolves with action ─────
|
||||
|
||||
router.put('/queue/:queueId/resolve', requireRole(...ROLES_WRITE), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const queueId = parseInt(req.params.queueId, 10);
|
||||
if (Number.isNaN(queueId)) {
|
||||
return res.status(400).json({ success: false, error: 'Invalid queueId' });
|
||||
}
|
||||
|
||||
const { action, corrections, notes } = req.body ?? {};
|
||||
if (!['approved', 'corrected', 'rejected'].includes(action)) {
|
||||
return res.status(400).json({ success: false, error: 'action must be approved | corrected | rejected' });
|
||||
}
|
||||
|
||||
const userId = req.jwtUserId ?? (req.body?.user_id as string | undefined);
|
||||
if (!userId) {
|
||||
return res.status(400).json({ success: false, error: 'No user identity' });
|
||||
}
|
||||
|
||||
if (action === 'corrected' && (!corrections || typeof corrections !== 'object')) {
|
||||
return res.status(400).json({ success: false, error: 'corrections (object) required when action=corrected' });
|
||||
}
|
||||
|
||||
const resolved = await resolveQueueEntry({
|
||||
queueId,
|
||||
userId,
|
||||
action,
|
||||
corrections: corrections ?? null,
|
||||
notes: notes ?? null,
|
||||
});
|
||||
|
||||
if (!resolved) {
|
||||
return res.status(404).json({ success: false, error: `Queue entry ${queueId} not found` });
|
||||
}
|
||||
|
||||
// BRAIN GOLD PROMOTION (fail-safe — never blocks resolve).
|
||||
if (action === 'approved' || action === 'corrected') {
|
||||
try {
|
||||
await promoteAtomsToGold({
|
||||
sessionId: resolved.session_id,
|
||||
userId,
|
||||
corrections: corrections ?? null,
|
||||
isCorrected: action === 'corrected',
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`[Moderation] Brain promotion failed for session ${resolved.session_id}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ success: true, data: resolved, message: `Resolved as '${action}'` });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'moderation_resolve');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Re-export of the new pipeline barrel. Kept at this path so src/index.ts
|
||||
* (which imports `./api/pipeline-routes`) continues to work unchanged after
|
||||
* the 1611-LOC → 5-file split. See ./pipeline/index.ts for the routing map.
|
||||
*/
|
||||
export { default } from './pipeline';
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/**
|
||||
* GARBAGE TEXT DETECTOR
|
||||
*
|
||||
* Heuristics that flag obvious non-content text from HTML scraping:
|
||||
* login walls (Facebook), cookie banners, navigation-only fragments.
|
||||
* Used to decide whether to fall back from M17/article scrape to yt-dlp
|
||||
* metadata extraction.
|
||||
*
|
||||
* Conservative on purpose — we'd rather analyze a noisy article than
|
||||
* silently swap valid (but short) content for metadata.
|
||||
*/
|
||||
|
||||
const LOGIN_WALL_PATTERNS = [
|
||||
/\blog\s*in\s*to\s*facebook\b/i,
|
||||
/\blog\s*in\s*or\s*sign\s*up\s*to\s*see\b/i,
|
||||
/\bsign\s*up\s*for\s*facebook\b/i,
|
||||
/\bconnect\s*with\s*friends\s*and\s*the\s*world\b/i,
|
||||
/\bsee\s*posts\s*[\w,\s]*photos\s*[\w,\s]*and\s*more\b/i,
|
||||
/\bjoin\s*facebook\s*to\s*connect\b/i,
|
||||
/\binstagram\s*is\s*a\s*simple\b/i,
|
||||
/\bsign\s*up\s*to\s*see\s*photos\b/i,
|
||||
/\bsee\s*more\s*on\s*instagram\b/i,
|
||||
/\bwatch\s*on\s*tiktok\b/i,
|
||||
/\bdownload\s*tiktok\s*to\b/i,
|
||||
];
|
||||
|
||||
const COOKIE_BANNER_PATTERNS = [
|
||||
/^we\s*use\s*cookies/i,
|
||||
/^accept\s*all\s*cookies/i,
|
||||
/^this\s*website\s*uses\s*cookies/i,
|
||||
/^by\s*clicking\s*"?accept/i,
|
||||
];
|
||||
|
||||
const NAV_TOKENS = [
|
||||
'home', 'login', 'log in', 'sign up', 'sign in', 'menu', 'search',
|
||||
'help center', 'privacy', 'terms', 'cookies', 'about us', 'contact',
|
||||
'download app', 'create account', 'forgot password',
|
||||
];
|
||||
|
||||
export function isLikelyGarbageText(text: string | null | undefined): boolean {
|
||||
if (!text) return true;
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return true;
|
||||
|
||||
if (trimmed.length < 80) {
|
||||
if (LOGIN_WALL_PATTERNS.some(rx => rx.test(trimmed))) return true;
|
||||
if (/\b(log\s*in|sign\s*in|sign\s*up)\b/i.test(trimmed) && trimmed.length < 60) return true;
|
||||
}
|
||||
|
||||
if (COOKIE_BANNER_PATTERNS.some(rx => rx.test(trimmed))) return true;
|
||||
|
||||
if (LOGIN_WALL_PATTERNS.some(rx => rx.test(trimmed)) && trimmed.length < 250) return true;
|
||||
|
||||
const lowered = trimmed.toLowerCase();
|
||||
let navHits = 0;
|
||||
for (const token of NAV_TOKENS) {
|
||||
if (lowered.includes(token)) navHits++;
|
||||
}
|
||||
if (navHits >= 5 && trimmed.length < 300) return true;
|
||||
|
||||
const linkPattern = /https?:\/\/\S+/g;
|
||||
const linkChars = (trimmed.match(linkPattern) || []).join('').length;
|
||||
if (linkChars > 0 && linkChars / trimmed.length > 0.7) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* URL handling helpers for /pipeline/analyze-url:
|
||||
* - detectUrlType: legacy 3-category adapter for url-probe
|
||||
* - fetchArticleText: M17 → fxtwitter → direct fetch chain
|
||||
* - isBoilerplate: navigation/boilerplate detector
|
||||
* - buildPlatformInfoField: shape used by empty-media response builder
|
||||
*/
|
||||
import { sanitizeUtf8, MAX_TEXT_LENGTH } from '../../../config/analysisLimits';
|
||||
import { sanitizeForLog } from '../../../shared/helpers/sanitize-log';
|
||||
import { validateExternalUrl } from '../../../shared/helpers/validate-url';
|
||||
import { identifyPlatform, type PlatformInfo, type ProbeResult } from '../../../shared/helpers/url-probe';
|
||||
import { requireEnv } from '../../../shared/helpers/env';
|
||||
import { log } from '../../../shared/logger';
|
||||
|
||||
/** Legacy adapter — maps url-probe platform info to old 3-category type for backward compat */
|
||||
export function detectUrlType(url: string): 'video_platform' | 'image' | 'article' {
|
||||
const platform = identifyPlatform(url);
|
||||
if (platform.contentHint === 'image') return 'image';
|
||||
if (platform.ytdlpSupported && (platform.contentHint === 'video' || platform.contentHint === 'mixed')) return 'video_platform';
|
||||
if (platform.ytdlpSupported && platform.platform !== 'other') return 'video_platform';
|
||||
return 'article';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch text content from a URL. Uses platform-aware extraction chain:
|
||||
* - Twitter/X: fxtwitter API (already extracted in probe, this is fallback)
|
||||
* - M17 Web API (primary for articles)
|
||||
* - Direct HTML fetch (last resort)
|
||||
*/
|
||||
export async function fetchArticleText(url: string, platform?: PlatformInfo): Promise<string> {
|
||||
validateExternalUrl(url);
|
||||
const plat = platform || identifyPlatform(url);
|
||||
|
||||
if (plat.platform === 'twitter') {
|
||||
try {
|
||||
const match = url.match(/(?:twitter\.com|x\.com)\/([^/]+)\/status\/(\d+)/i);
|
||||
if (match) {
|
||||
log.info(`[Pipeline URL] X.com detected, trying fxtwitter API`);
|
||||
const fxResp = await fetch(`https://api.fxtwitter.com/${match[1]}/status/${match[2]}`, {
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (fxResp.ok) {
|
||||
const fxData = await fxResp.json() as any;
|
||||
const tweet = fxData.tweet;
|
||||
if (tweet?.text) {
|
||||
const parts: string[] = [];
|
||||
if (tweet.author?.name) parts.push(`@${tweet.author.screen_name} (${tweet.author.name}):`);
|
||||
parts.push(tweet.text);
|
||||
if (tweet.created_at) parts.push(`\nPosted: ${tweet.created_at}`);
|
||||
const text = sanitizeUtf8(parts.join('\n'));
|
||||
log.info(`[Pipeline URL] fxtwitter extracted ${text.length} chars`);
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn(`[Pipeline URL] fxtwitter failed: ${(e as Error).message}, trying M17`);
|
||||
}
|
||||
}
|
||||
|
||||
const M17_API = requireEnv('M17_WEB_API_URL');
|
||||
|
||||
try {
|
||||
log.info(`[Pipeline URL] Fetching via M17: ${sanitizeForLog(url)}`);
|
||||
const response = await fetch(`${M17_API}/v1/fetch`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ urls: [url], extract_text: true }),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json() as {
|
||||
pages?: { url: string; title?: string; text: string }[];
|
||||
};
|
||||
|
||||
if (data.pages && data.pages.length > 0 && data.pages[0].text) {
|
||||
const page = data.pages[0];
|
||||
const title = page.title ? `Title: ${page.title}\n\n` : '';
|
||||
const extracted = sanitizeUtf8(title + page.text);
|
||||
log.info(`[Pipeline URL] M17 extracted ${extracted.length} chars`);
|
||||
return extracted;
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[Pipeline URL] M17 failed, falling back to direct fetch`);
|
||||
} catch (e) {
|
||||
log.warn(`[Pipeline URL] M17 error: ${e}, falling back to direct fetch`);
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const text = sanitizeUtf8(
|
||||
html
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.substring(0, MAX_TEXT_LENGTH),
|
||||
);
|
||||
|
||||
if (text.length < 100) {
|
||||
throw new Error('Could not extract meaningful content from page');
|
||||
}
|
||||
|
||||
if (isBoilerplate(text)) {
|
||||
throw new Error('Extracted content appears to be navigation/boilerplate, not article text');
|
||||
}
|
||||
|
||||
log.info(`[Pipeline URL] Direct fetch extracted ${text.length} chars`);
|
||||
return text;
|
||||
}
|
||||
|
||||
/** Detect if extracted text is mostly navigation/boilerplate rather than article content */
|
||||
export function isBoilerplate(text: string): boolean {
|
||||
const words = text.split(/\s+/);
|
||||
if (words.length < 30) return true;
|
||||
|
||||
const navPatterns = /^(skip to content|register|sign in|home|menu|navigation|cookie|privacy|terms)/i;
|
||||
if (navPatterns.test(text.trim())) return true;
|
||||
|
||||
const shortWords = words.filter(w => w.length <= 3).length;
|
||||
if (shortWords / words.length > 0.5) return true;
|
||||
|
||||
if (/javascript is (not available|disabled)/i.test(text)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Build platform_info object for empty media responses. */
|
||||
export function buildPlatformInfoField(platform: PlatformInfo, probe: ProbeResult) {
|
||||
return {
|
||||
platform: platform.platform,
|
||||
displayName: platform.displayName,
|
||||
contentHint: platform.contentHint,
|
||||
accessible: probe.accessible,
|
||||
probeMethod: probe.probeMethod,
|
||||
probeDurationMs: probe.durationMs,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* Vision-extraction helper used by /pipeline/analyze, /analyze-url and
|
||||
* /analyze-media when input is an image URL. Loads the prompt from Redis
|
||||
* (didi:config:vision:v1:prompts:extraction) and falls back to a hard-coded
|
||||
* default if the key is missing.
|
||||
*/
|
||||
import type Redis from 'ioredis';
|
||||
import { callVision } from '../../../shared/media/vision';
|
||||
import { ConfigKeys } from '../../../shared/redis/keys';
|
||||
import { log } from '../../../shared/logger';
|
||||
|
||||
/**
|
||||
* Extract text/description from an image via vision model.
|
||||
* Returns null on failure (caller decides how to degrade).
|
||||
*/
|
||||
export async function extractImageVision(
|
||||
redis: Redis,
|
||||
imageUrl: string,
|
||||
logPrefix = 'Pipeline API',
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<{ text: string; provider: string } | null> {
|
||||
try {
|
||||
const promptData = await redis.get(ConfigKeys.visionPromptExtraction);
|
||||
let systemPrompt = 'You are a text extraction specialist. Extract only the meaningful content from images. Ignore UI elements, buttons, menus, navigation bars, taskbars, browser chrome, and app interfaces.';
|
||||
let userPrompt = 'Extract the main text content from this image. Return ONLY the actual message, post, article text, or statement visible in the image. Do NOT describe the image layout, UI elements, buttons, or interface components. If the image contains a social media post, news article, or message, return just that text. If there is no meaningful text, respond with NO_TEXT_FOUND.';
|
||||
|
||||
if (promptData) {
|
||||
const parsed = JSON.parse(promptData);
|
||||
if (parsed.system) systemPrompt = parsed.system;
|
||||
if (parsed.user_template) userPrompt = parsed.user_template;
|
||||
} else {
|
||||
log.warn(`[${logPrefix}] Vision prompt not in Redis, using fallback`);
|
||||
}
|
||||
|
||||
const messages: any[] = [];
|
||||
if (systemPrompt) {
|
||||
messages.push({ role: 'system', content: systemPrompt });
|
||||
}
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: userPrompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
});
|
||||
|
||||
const visionResult = await callVision(redis, messages, { max_tokens: 1500 }, tier);
|
||||
log.info(`[${logPrefix}] Image vision (tier: ${tier}) extracted ${visionResult.content.length} chars via ${visionResult.provider}`);
|
||||
return { text: visionResult.content, provider: visionResult.provider };
|
||||
} catch (error) {
|
||||
log.warn(`[${logPrefix}] Image vision failed:`, (error as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Shared lazy-init singletons used by pipeline sub-routers (analyze, status,
|
||||
* history, extension, async). Extracted from the original pipeline-routes.ts
|
||||
* so each sub-router can reuse the same connections without duplicating setup.
|
||||
*
|
||||
* Redis label is 'pipeline-routes' (kept for connection-metadata continuity
|
||||
* with logs/dashboards from before the split).
|
||||
*/
|
||||
import multer from 'multer';
|
||||
import { lazyRedis } from '../../shared/redis/connection';
|
||||
import { MediaService } from '../../shared/media/media-service';
|
||||
import { PersistService, PgSessionAdapter, getPgPool } from '../../shared/persistence';
|
||||
import { SessionStore } from '../../shared/redis/session-store';
|
||||
import { optionalEnv } from '../../shared/helpers/env';
|
||||
|
||||
/** Multer upload — memory storage, 50MB max. Used by /pipeline/extension/upload. */
|
||||
export const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
limits: { fileSize: 50 * 1024 * 1024 },
|
||||
});
|
||||
|
||||
/** didi-framework hostname (internal). Safe default for local dev. */
|
||||
export const FRAMEWORK_API_URL = optionalEnv('FRAMEWORK_API_URL', 'http://didi-framework:3005');
|
||||
|
||||
export const getRedis = lazyRedis('pipeline-routes');
|
||||
|
||||
let _persistService: PersistService | null = null;
|
||||
export function getPersistService(): PersistService {
|
||||
if (!_persistService) {
|
||||
const redis = getRedis();
|
||||
_persistService = new PersistService(
|
||||
new SessionStore(redis),
|
||||
new PgSessionAdapter(getPgPool()),
|
||||
);
|
||||
}
|
||||
return _persistService;
|
||||
}
|
||||
|
||||
let _mediaService: MediaService | null = null;
|
||||
export function getMediaService(): MediaService {
|
||||
if (!_mediaService) _mediaService = new MediaService();
|
||||
return _mediaService;
|
||||
}
|
||||
|
|
@ -0,0 +1,580 @@
|
|||
/**
|
||||
* Pipeline sync analyze endpoints (extracted from pipeline-routes.ts):
|
||||
* POST /pipeline/analyze — Universal (text/url/image; rejects audio/video)
|
||||
* POST /pipeline/analyze-url — Smart URL analysis with platform probing
|
||||
* POST /pipeline/analyze-media — Convenience endpoint for image media
|
||||
*
|
||||
* Audio/video are rejected here with ASYNC_REQUIRED (use /pipeline/analyze-async).
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import crypto from 'crypto';
|
||||
import { PipelineExecutor } from '../../components/pipeline/executor';
|
||||
import { PipelineInput, MediaType } from '../../components/pipeline/types';
|
||||
import { validateTextInput, sanitizeUtf8, TextValidationResult } from '../../config/analysisLimits';
|
||||
import { processVideoUrl } from '../../shared/media/video-processor';
|
||||
import { extractUrlMetadata } from '../../shared/media/url-metadata';
|
||||
import { checkCredits, deductCredits, getSearchTier } from '../../shared/credits';
|
||||
import { dispatch } from '../../queue';
|
||||
import type { PlanType } from '../../queue';
|
||||
import { isLikelyGarbageText } from './_helpers/garbage-detector';
|
||||
import { buildEmptyMediaResponse } from '../../shared/helpers/empty-media-response';
|
||||
import { sanitizeForLog } from '../../shared/helpers/sanitize-log';
|
||||
import { validateExternalUrl } from '../../shared/helpers/validate-url';
|
||||
import {
|
||||
identifyPlatform,
|
||||
probeUrl,
|
||||
getExtractionStrategy,
|
||||
getUserFriendlyError,
|
||||
} from '../../shared/helpers/url-probe';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { resolveUserId, authRequired } from '../../shared/auth/guards';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis, getMediaService } from './_init';
|
||||
import { extractImageVision } from './_helpers/vision';
|
||||
import {
|
||||
detectUrlType,
|
||||
fetchArticleText,
|
||||
buildPlatformInfoField,
|
||||
} from './_helpers/url-helpers';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/pipeline/analyze - Universal analysis endpoint
|
||||
// ============================================================================
|
||||
|
||||
router.post('/analyze', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_user_id, user_email: body_user_email, options, user_flagged } = req.body;
|
||||
|
||||
const user_id = resolveUserId(req, body_user_id);
|
||||
const user_email = req.jwtEmail || body_user_email;
|
||||
const sessionId = crypto.randomUUID();
|
||||
|
||||
if (!user_id) {
|
||||
return authRequired(res);
|
||||
}
|
||||
|
||||
const validTypes: MediaType[] = ['text', 'url', 'image', 'audio', 'video'];
|
||||
if (!media_type || !validTypes.includes(media_type)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
let textValidation: TextValidationResult | null = null;
|
||||
if (text && (media_type === 'text' || media_type === 'url')) {
|
||||
textValidation = validateTextInput(text);
|
||||
if (!textValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: textValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: textValidation.error_code, stats: textValidation.stats },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const creditCheck = await checkCredits(user_id, media_type);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
let analyzedText = text;
|
||||
if (!text && media_url) {
|
||||
if (media_type === 'image') {
|
||||
const vision = await extractImageVision(getRedis(), media_url, 'Pipeline API', getSearchTier(creditCheck.planType));
|
||||
if (vision) analyzedText = vision.text;
|
||||
} else if (media_type === 'audio' || media_type === 'video') {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: `${media_type} content requires async processing due to transcription time`,
|
||||
code: 'ASYNC_REQUIRED',
|
||||
suggestion: 'Use POST /api/v3/pipeline/analyze-async for video and audio content',
|
||||
async_endpoint: '/api/v3/pipeline/analyze-async',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!text && media_url && (!analyzedText || analyzedText.trim().length === 0)) {
|
||||
return res.json(buildEmptyMediaResponse({
|
||||
sessionId,
|
||||
mediaType: media_type,
|
||||
mediaUrl: media_url,
|
||||
reason: 'no_text_content',
|
||||
message: 'No text content could be extracted from media',
|
||||
extractionDurationMs: 0,
|
||||
}));
|
||||
}
|
||||
|
||||
const searchTier = getSearchTier(creditCheck.planType);
|
||||
const input: PipelineInput = {
|
||||
text: analyzedText, media_url, url, media_type, session_id: sessionId, user_id, user_email,
|
||||
userFlagged: user_flagged === true,
|
||||
options: { ...options, skipClaims: textValidation?.skipClaims || options?.skipClaims },
|
||||
searchTier,
|
||||
};
|
||||
|
||||
log.info(`[Pipeline API] Starting analysis, type: ${media_type}, user: ${user_id}, searchTier: ${searchTier}`);
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const result = await executor.execute(input);
|
||||
|
||||
const deducted = await deductCredits(user_id, media_type, result.session_id);
|
||||
if (!deducted) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${user_id} type=${media_type}`);
|
||||
|
||||
const warnings: string[] = [...(textValidation?.warnings || [])];
|
||||
if (result.claims?.total_claims === 0 && !textValidation?.skipClaims) {
|
||||
warnings.push('No verifiable claims found in the text');
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result,
|
||||
...(warnings.length > 0 && { warnings }),
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/pipeline/analyze-url - Smart URL analysis with platform probing
|
||||
// ============================================================================
|
||||
|
||||
router.post('/analyze-url', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { url, user_id: body_user_id, user_email: body_user_email, options } = req.body;
|
||||
const user_id = resolveUserId(req, body_user_id);
|
||||
const user_email = req.jwtEmail || body_user_email;
|
||||
|
||||
if (!user_id) {
|
||||
return authRequired(res);
|
||||
}
|
||||
if (!url) {
|
||||
return res.status(400).json({ success: false, error: 'url is required' });
|
||||
}
|
||||
|
||||
log.info(`[Pipeline URL] Analyzing URL: ${sanitizeForLog(url)}`);
|
||||
|
||||
const platform = identifyPlatform(url);
|
||||
log.info(`[Pipeline URL] Platform: ${platform.displayName}, content: ${platform.contentHint}`);
|
||||
|
||||
const probe = await probeUrl(url, platform);
|
||||
log.info(`[Pipeline URL] Probe: accessible=${probe.accessible}, method=${probe.probeMethod}, text=${probe.extractedText?.length || 0} chars, ${probe.durationMs}ms`);
|
||||
|
||||
if (!probe.accessible) {
|
||||
const errorInfo = getUserFriendlyError(platform, probe.errorReason);
|
||||
log.info(`[Pipeline URL] Content inaccessible: ${probe.errorReason} (${platform.displayName})`);
|
||||
return res.json(buildEmptyMediaResponse({
|
||||
mediaType: platform.contentHint === 'video' ? 'video' : 'url',
|
||||
mediaUrl: url,
|
||||
reason: 'extraction_failed',
|
||||
message: errorInfo.message_en,
|
||||
extractionDurationMs: probe.durationMs,
|
||||
platformInfo: buildPlatformInfoField(platform, probe),
|
||||
userMessage: errorInfo.message_ro,
|
||||
userMessageEn: errorInfo.message_en,
|
||||
suggestion: errorInfo.suggestion_ro,
|
||||
suggestionEn: errorInfo.suggestion_en,
|
||||
}));
|
||||
}
|
||||
|
||||
const strategy = getExtractionStrategy(platform, probe);
|
||||
log.info(`[Pipeline URL] Strategy: primary=${strategy.primary}, fallbacks=[${strategy.fallbacks.join(',')}], mediaType=${strategy.expectedMediaType}`);
|
||||
|
||||
let input: PipelineInput;
|
||||
let extraWarnings: string[] = [];
|
||||
|
||||
if (strategy.primary === 'oembed_text' && probe.extractedText && probe.extractedText.length > 30) {
|
||||
const probeText = sanitizeUtf8(probe.extractedText);
|
||||
log.info(`[Pipeline URL] Using probe text (${probeText.length} chars from ${probe.probeMethod})`);
|
||||
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: probeText,
|
||||
media_type: 'url',
|
||||
options: { ...options, source_url: url },
|
||||
};
|
||||
|
||||
} else if (strategy.primary === 'yt-dlp' || (strategy.expectedMediaType === 'video' && platform.ytdlpSupported)) {
|
||||
// ── ASYNC PATH: dispatch video URLs to media_preprocess worker (non-blocking) ──
|
||||
// yt-dlp + ffmpeg + Whisper + vision can take 30-120s. We don't want to hold
|
||||
// the HTTP request open for that — Kong's 60s timeout would cut it anyway.
|
||||
// The worker runs the same processVideoUrl() flow, just on a separate process.
|
||||
const sessionId = crypto.randomUUID();
|
||||
const creditCheck = await checkCredits(user_id, 'video');
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
const planTypeValue: PlanType = (creditCheck.planType >= 1 && creditCheck.planType <= 6 ? creditCheck.planType : 1) as PlanType;
|
||||
|
||||
const dispatchResult = await dispatch(
|
||||
sessionId,
|
||||
{
|
||||
content: probe.extractedText ? sanitizeUtf8(probe.extractedText) : '',
|
||||
url,
|
||||
userId: user_id,
|
||||
userEmail: user_email,
|
||||
inputType: 'video',
|
||||
},
|
||||
planTypeValue,
|
||||
);
|
||||
|
||||
if (dispatchResult.async) {
|
||||
const deducted = await deductCredits(user_id, 'video', sessionId);
|
||||
if (!deducted) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=video`);
|
||||
|
||||
return res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
queued_components: dispatchResult.queued,
|
||||
plan_type: planTypeValue,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
url_metadata: {
|
||||
original_url: url,
|
||||
detected_type: detectUrlType(url),
|
||||
processed_as: 'video',
|
||||
platform: platform.platform,
|
||||
platform_display_name: platform.displayName,
|
||||
content_hint: platform.contentHint,
|
||||
probe_accessible: probe.accessible,
|
||||
probe_method: probe.probeMethod,
|
||||
probe_duration_ms: probe.durationMs,
|
||||
...(probe.title && { probe_title: probe.title }),
|
||||
...(probe.author && { probe_author: probe.author }),
|
||||
...(probe.thumbnailUrl && { probe_thumbnail_url: probe.thumbnailUrl }),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// RabbitMQ unavailable — fall back to inline sync video processing.
|
||||
// Same enrichment as the worker: video pipeline + metadata in parallel.
|
||||
log.warn(`[Pipeline URL] Queue unavailable (${dispatchResult.error}), falling back to sync video processing`);
|
||||
const [videoOutcome, metadataOutcome] = await Promise.allSettled([
|
||||
processVideoUrl(url, sessionId, {
|
||||
redis: getRedis(),
|
||||
logPrefix: 'Pipeline URL',
|
||||
}),
|
||||
extractUrlMetadata(url),
|
||||
]);
|
||||
|
||||
const videoResult = videoOutcome.status === 'fulfilled' ? videoOutcome.value : null;
|
||||
if (videoOutcome.status === 'rejected') {
|
||||
log.warn(`[Pipeline URL] yt-dlp failed for ${platform.displayName}: ${(videoOutcome.reason as Error).message}`);
|
||||
}
|
||||
const metadata = metadataOutcome.status === 'fulfilled' ? metadataOutcome.value : null;
|
||||
|
||||
const mergedParts: string[] = [];
|
||||
if (metadata?.combined_text) {
|
||||
const meta = metadata.uploader
|
||||
? `[POST METADATA — ${metadata.uploader}]\n${metadata.combined_text}`
|
||||
: `[POST METADATA]\n${metadata.combined_text}`;
|
||||
mergedParts.push(meta);
|
||||
}
|
||||
if (videoResult?.merged_text) mergedParts.push(videoResult.merged_text);
|
||||
const enrichedText = mergedParts.join('\n\n');
|
||||
|
||||
if (enrichedText && enrichedText.trim().length > 0) {
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: enrichedText,
|
||||
media_type: videoResult?.merged_text ? 'video' : 'url',
|
||||
options: { ...options, source_url: url },
|
||||
};
|
||||
if (!videoResult?.merged_text && metadata?.combined_text) {
|
||||
extraWarnings.push(`Video transcript unavailable; analysis based on ${platform.displayName} post metadata`);
|
||||
}
|
||||
} else if (probe.extractedText && probe.extractedText.length > 30) {
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: sanitizeUtf8(probe.extractedText),
|
||||
media_type: 'url',
|
||||
options: { ...options, source_url: url },
|
||||
};
|
||||
extraWarnings.push(`Video extraction failed, analyzed ${platform.displayName} description instead`);
|
||||
} else {
|
||||
try {
|
||||
const fallbackText = await fetchArticleText(url, platform);
|
||||
if (fallbackText && fallbackText.trim().length > 0 && !isLikelyGarbageText(fallbackText)) {
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: fallbackText,
|
||||
media_type: 'url',
|
||||
options: { ...options, source_url: url },
|
||||
};
|
||||
extraWarnings.push(`Video extraction failed, analyzed page text instead`);
|
||||
} else {
|
||||
const errorInfo = getUserFriendlyError(platform, 'platform_blocked');
|
||||
return res.json(buildEmptyMediaResponse({
|
||||
mediaType: 'video',
|
||||
mediaUrl: url,
|
||||
reason: 'extraction_failed',
|
||||
message: errorInfo.message_en,
|
||||
extractionDurationMs: probe.durationMs,
|
||||
platformInfo: buildPlatformInfoField(platform, probe),
|
||||
userMessage: errorInfo.message_ro,
|
||||
userMessageEn: errorInfo.message_en,
|
||||
suggestion: errorInfo.suggestion_ro,
|
||||
suggestionEn: errorInfo.suggestion_en,
|
||||
}));
|
||||
}
|
||||
} catch {
|
||||
const errorInfo = getUserFriendlyError(platform, 'platform_blocked');
|
||||
return res.json(buildEmptyMediaResponse({
|
||||
mediaType: 'video',
|
||||
mediaUrl: url,
|
||||
reason: 'extraction_failed',
|
||||
message: errorInfo.message_en,
|
||||
extractionDurationMs: probe.durationMs,
|
||||
platformInfo: buildPlatformInfoField(platform, probe),
|
||||
userMessage: errorInfo.message_ro,
|
||||
userMessageEn: errorInfo.message_en,
|
||||
suggestion: errorInfo.suggestion_ro,
|
||||
suggestionEn: errorInfo.suggestion_en,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
} else if (platform.contentHint === 'image' || strategy.expectedMediaType === 'image') {
|
||||
try {
|
||||
validateExternalUrl(url);
|
||||
const imageResponse = await fetch(url, { signal: AbortSignal.timeout(60000) });
|
||||
if (!imageResponse.ok) throw new Error(`Failed to download image: ${imageResponse.status}`);
|
||||
|
||||
const buffer = Buffer.from(await imageResponse.arrayBuffer());
|
||||
const mediaService = getMediaService();
|
||||
const uploaded = await mediaService.uploadFile(user_id, buffer, `pipeline-${Date.now()}.jpg`, 'image');
|
||||
|
||||
const visionExtract = await extractImageVision(getRedis(), uploaded.public_url, 'Pipeline URL', 'free');
|
||||
const imageText = visionExtract?.text || '';
|
||||
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: imageText || undefined,
|
||||
media_url: uploaded.public_url,
|
||||
media_type: 'image',
|
||||
options: { ...options, source_url: url },
|
||||
};
|
||||
} catch (imgError) {
|
||||
log.error(`[Pipeline URL] Image download failed:`, imgError);
|
||||
return internalError(res, imgError, 'pipeline_url_image_download');
|
||||
}
|
||||
|
||||
} else {
|
||||
try {
|
||||
let fetchedText = await fetchArticleText(url, platform);
|
||||
|
||||
// If the article scrape returned a login wall / cookie banner / nav-only
|
||||
// garbage (common on Facebook/Instagram/TikTok text posts), try yt-dlp
|
||||
// metadata as a more reliable fallback. Only kicks in when scrape failed.
|
||||
if (isLikelyGarbageText(fetchedText)) {
|
||||
log.info(`[Pipeline URL] Scraped text looks like garbage (${fetchedText?.length ?? 0} chars), trying yt-dlp metadata`);
|
||||
const metadata = await extractUrlMetadata(url);
|
||||
if (metadata?.combined_text) {
|
||||
fetchedText = metadata.combined_text;
|
||||
extraWarnings.push(
|
||||
`Article extraction blocked; analysis based on ${platform.displayName} post metadata`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const urlTextValidation = validateTextInput(fetchedText);
|
||||
if (!urlTextValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: urlTextValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: urlTextValidation.error_code, stats: urlTextValidation.stats, source: 'fetched_article' },
|
||||
});
|
||||
}
|
||||
|
||||
extraWarnings = [...extraWarnings, ...(urlTextValidation.warnings || [])];
|
||||
|
||||
input = {
|
||||
url, user_id, user_email,
|
||||
text: fetchedText,
|
||||
media_type: 'url',
|
||||
options: { ...options, source_url: url, skipClaims: urlTextValidation.skipClaims },
|
||||
};
|
||||
} catch (fetchError) {
|
||||
log.error(`[Pipeline URL] Article fetch failed:`, fetchError);
|
||||
const errorInfo = getUserFriendlyError(platform, 'platform_blocked');
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Could not fetch article content',
|
||||
error_code: 'ARTICLE_FETCH_FAILED',
|
||||
platform_info: buildPlatformInfoField(platform, probe),
|
||||
user_message: errorInfo.message_ro,
|
||||
user_message_en: errorInfo.message_en,
|
||||
suggestion: errorInfo.suggestion_ro,
|
||||
suggestion_en: errorInfo.suggestion_en,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const creditCheck = await checkCredits(user_id, input.media_type);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
input.searchTier = getSearchTier(creditCheck.planType);
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const result = await executor.execute(input);
|
||||
|
||||
const deductedUrl = await deductCredits(user_id, input.media_type, result.session_id);
|
||||
if (!deductedUrl) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${user_id} type=${input.media_type}`);
|
||||
|
||||
const warnings: string[] = [...extraWarnings];
|
||||
if (result.claims?.total_claims === 0) {
|
||||
warnings.push('No verifiable claims found in the text');
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
...result,
|
||||
url_metadata: {
|
||||
original_url: url,
|
||||
detected_type: detectUrlType(url),
|
||||
processed_as: input.media_type,
|
||||
platform: platform.platform,
|
||||
platform_display_name: platform.displayName,
|
||||
content_hint: platform.contentHint,
|
||||
probe_accessible: probe.accessible,
|
||||
probe_method: probe.probeMethod,
|
||||
probe_duration_ms: probe.durationMs,
|
||||
...(probe.title && { probe_title: probe.title }),
|
||||
...(probe.author && { probe_author: probe.author }),
|
||||
...(probe.thumbnailUrl && { probe_thumbnail_url: probe.thumbnailUrl }),
|
||||
},
|
||||
},
|
||||
...(warnings.length > 0 && { warnings }),
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline URL] URL error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/pipeline/analyze-media - Convenience endpoint for image media
|
||||
// ============================================================================
|
||||
|
||||
router.post('/analyze-media', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { media_url, media_type, user_id: body_user_id, user_email: body_user_email, options } = req.body;
|
||||
const user_id = resolveUserId(req, body_user_id);
|
||||
const user_email = req.jwtEmail || body_user_email;
|
||||
|
||||
if (!user_id) {
|
||||
return authRequired(res);
|
||||
}
|
||||
if (!media_url) {
|
||||
return res.status(400).json({ success: false, error: 'media_url is required' });
|
||||
}
|
||||
|
||||
const validTypes: MediaType[] = ['image', 'audio', 'video'];
|
||||
if (!media_type || !validTypes.includes(media_type)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required for media. Valid: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
log.info(`[Pipeline API] Analyzing ${media_type}: ${sanitizeForLog(media_url)}, user: ${user_id}`);
|
||||
|
||||
const creditCheck = await checkCredits(user_id, media_type);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
let extractedText: string | undefined;
|
||||
if (media_type === 'image' && media_url) {
|
||||
const vision = await extractImageVision(getRedis(), media_url, 'Pipeline Media', getSearchTier(creditCheck.planType));
|
||||
if (vision) extractedText = vision.text;
|
||||
} else if ((media_type === 'audio' || media_type === 'video') && media_url) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: `${media_type} content requires async processing due to transcription time`,
|
||||
code: 'ASYNC_REQUIRED',
|
||||
suggestion: 'Use POST /api/v3/pipeline/analyze-async for video and audio content',
|
||||
async_endpoint: '/api/v3/pipeline/analyze-async',
|
||||
});
|
||||
}
|
||||
|
||||
if (!extractedText || extractedText.trim().length === 0) {
|
||||
return res.json(buildEmptyMediaResponse({
|
||||
mediaType: media_type,
|
||||
mediaUrl: media_url,
|
||||
reason: 'no_text_content',
|
||||
message: 'No text content could be extracted from media',
|
||||
extractionDurationMs: 0,
|
||||
}));
|
||||
}
|
||||
|
||||
const input: PipelineInput = { text: extractedText, media_url, media_type, user_id, user_email, options, searchTier: getSearchTier(creditCheck.planType) };
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const result = await executor.execute(input);
|
||||
|
||||
const deductedMedia = await deductCredits(user_id, media_type, result.session_id);
|
||||
if (!deductedMedia) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${user_id} type=${media_type}`);
|
||||
|
||||
res.json({ success: true, data: result });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Media error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
/**
|
||||
* Async pipeline endpoints (extracted from pipeline-routes.ts):
|
||||
* POST /pipeline/analyze-async — dispatches via RabbitMQ
|
||||
* GET /pipeline/queue-health — RabbitMQ + queue health
|
||||
* GET /pipeline/:sessionId/queue-status — partial AnalysisSession + progress
|
||||
*
|
||||
* Async path falls back to sync execution if RabbitMQ is unavailable. Audio/video
|
||||
* are handled here (sync analyze rejects them with ASYNC_REQUIRED).
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import crypto from 'crypto';
|
||||
import { PipelineExecutor } from '../../components/pipeline/executor';
|
||||
import { PipelineInput, MediaType } from '../../components/pipeline/types';
|
||||
import { validateTextInput, TextValidationResult } from '../../config/analysisLimits';
|
||||
import { checkCredits, deductCredits, getSearchTier } from '../../shared/credits';
|
||||
import { getPgPool } from '../../shared/persistence';
|
||||
import {
|
||||
dispatch,
|
||||
getSessionState,
|
||||
healthCheck as queueHealthCheck,
|
||||
isRabbitMQAvailable,
|
||||
LEGACY_TIER_TO_PLAN,
|
||||
} from '../../queue';
|
||||
import type { LegacyTier, PlanType } from '../../queue';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis } from './_init';
|
||||
import { detectUrlType, fetchArticleText } from './_helpers/url-helpers';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/analyze-async', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_user_id, user_email: body_user_email, tier, plan_type, options } = req.body;
|
||||
const user_id = resolveUserId(req, body_user_id);
|
||||
const user_email = req.jwtEmail || body_user_email;
|
||||
|
||||
if (!user_id) {
|
||||
return res.status(400).json({ success: false, error: 'user_id is required' });
|
||||
}
|
||||
|
||||
const validTypes: MediaType[] = ['text', 'url', 'image', 'audio', 'video'];
|
||||
if (!media_type || !validTypes.includes(media_type)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
// Duration limit check (instant reject if frontend supplies media_duration_sec)
|
||||
const media_duration_sec = req.body.media_duration_sec;
|
||||
if (media_duration_sec != null) {
|
||||
const duration = Number(media_duration_sec);
|
||||
if (media_type === 'video' && duration > 180) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: `Video too long: ${Math.round(duration)}s. Maximum allowed: 3 minutes (180s).`,
|
||||
error_code: 'MEDIA_TOO_LONG',
|
||||
details: { duration_sec: duration, max_duration_sec: 180, media_type: 'video' },
|
||||
});
|
||||
}
|
||||
if (media_type === 'audio' && duration > 420) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: `Audio too long: ${Math.round(duration)}s. Maximum allowed: 7 minutes (420s).`,
|
||||
error_code: 'MEDIA_TOO_LONG',
|
||||
details: { duration_sec: duration, max_duration_sec: 420, media_type: 'audio' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let asyncTextValidation: TextValidationResult | null = null;
|
||||
if (text && (media_type === 'text' || media_type === 'url')) {
|
||||
asyncTextValidation = validateTextInput(text);
|
||||
if (!asyncTextValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: asyncTextValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: asyncTextValidation.error_code, stats: asyncTextValidation.stats },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const creditCheck = await checkCredits(user_id, media_type);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
const sessionId = crypto.randomUUID();
|
||||
|
||||
// Server-side planType from credit check is authoritative; request body overrides only if creditCheck returned plan 1 (free)
|
||||
let planTypeValue: PlanType = (creditCheck.planType >= 1 && creditCheck.planType <= 6 ? creditCheck.planType : 1) as PlanType;
|
||||
if (planTypeValue === 1 && plan_type && plan_type >= 1 && plan_type <= 6) {
|
||||
planTypeValue = plan_type as PlanType;
|
||||
} else if (planTypeValue === 1 && tier && LEGACY_TIER_TO_PLAN[tier as LegacyTier]) {
|
||||
planTypeValue = LEGACY_TIER_TO_PLAN[tier as LegacyTier];
|
||||
}
|
||||
|
||||
let content = text || '';
|
||||
let effectiveMediaType = media_type;
|
||||
|
||||
if (url && !text) {
|
||||
const urlType = detectUrlType(url);
|
||||
|
||||
if (urlType === 'video_platform') {
|
||||
log.info(`[Async] Video platform detected: ${url}, dispatching to workers (non-blocking)`);
|
||||
effectiveMediaType = 'video';
|
||||
} else {
|
||||
try {
|
||||
content = await fetchArticleText(url);
|
||||
} catch (e) {
|
||||
log.warn(`[Async] Failed to fetch URL content: ${(e as Error).message}`);
|
||||
content = `URL: ${url}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
// For media without text, OCR/transcription happens in workers — do NOT block here
|
||||
if (!content && media_url) {
|
||||
if (media_type === 'image') {
|
||||
log.info(`[Async] Image detected, OCR will be done by workers — dispatching immediately`);
|
||||
} else if (media_type === 'audio' || media_type === 'video') {
|
||||
log.info(`[Async] ${media_type} detected, workers will handle transcription — dispatching immediately`);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[Pipeline API Async] Starting async analysis, type: ${effectiveMediaType}, user: ${user_id}, plan: ${planTypeValue}`);
|
||||
|
||||
const dispatchResult = await dispatch(
|
||||
sessionId,
|
||||
{ content, url, mediaPath: media_url, userId: user_id, userEmail: user_email, inputType: effectiveMediaType },
|
||||
planTypeValue,
|
||||
);
|
||||
|
||||
if (!dispatchResult.async) {
|
||||
log.info(`[Pipeline API Async] Queue unavailable, falling back to sync`);
|
||||
|
||||
const input: PipelineInput = { text: content, media_url, url, media_type, session_id: sessionId, user_id, user_email, options, searchTier: getSearchTier(creditCheck.planType) };
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const result = await executor.execute(input);
|
||||
|
||||
const deductedSync = await deductCredits(user_id, media_type, result.session_id);
|
||||
if (!deductedSync) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${user_id} type=${media_type}`);
|
||||
|
||||
return res.json({
|
||||
success: true,
|
||||
async: false,
|
||||
data: result,
|
||||
message: 'Processed synchronously (queue unavailable)',
|
||||
});
|
||||
}
|
||||
|
||||
// Async dispatched — deduct credits immediately
|
||||
const deductedAsync = await deductCredits(user_id, media_type, sessionId);
|
||||
if (!deductedAsync) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${media_type}`);
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
queued_components: dispatchResult.queued,
|
||||
plan_type: planTypeValue,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API Async] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/queue-health', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const health = await queueHealthCheck();
|
||||
const available = await isRabbitMQAvailable();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: { ...health, available, mode: available ? 'async' : 'sync-fallback' },
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/:sessionId/queue-status', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const state = await getSessionState(sessionId);
|
||||
|
||||
if (!state) {
|
||||
// Session not in Redis queue — check PG to give a useful response
|
||||
try {
|
||||
const pool = getPgPool();
|
||||
const pgRow = await pool.query(
|
||||
'SELECT session_id, status, started_at, completed_at FROM bos_analysis.analysis_session WHERE session_id = $1',
|
||||
[sessionId],
|
||||
);
|
||||
if (pgRow.rows.length > 0) {
|
||||
const pg = pgRow.rows[0];
|
||||
if (pg.status === 'completed' || pg.status === 'failed') {
|
||||
return res.json({ success: true, data: { session_id: sessionId, status: pg.status, _queue: { progress: 100 } } });
|
||||
}
|
||||
// Session stuck (pending/running) and not in queue — mark as failed
|
||||
const startedAt = pg.started_at ? new Date(pg.started_at).getTime() : 0;
|
||||
const stuckMinutes = (Date.now() - startedAt) / 60_000;
|
||||
if (stuckMinutes > 10) {
|
||||
await pool.query(
|
||||
"UPDATE bos_analysis.analysis_session SET status = 'failed', completed_at = NOW() WHERE session_id = $1 AND status IN ('pending', 'running')",
|
||||
[sessionId],
|
||||
);
|
||||
log.warn(`[Queue Status] Marked zombie session ${sessionId} as failed (stuck ${Math.round(stuckMinutes)} min)`);
|
||||
}
|
||||
return res.json({
|
||||
success: true,
|
||||
data: { session_id: sessionId, status: 'failed', reason: 'session_expired_from_queue', _queue: { progress: 0 } },
|
||||
});
|
||||
}
|
||||
} catch (pgErr) {
|
||||
log.error('[Queue Status] PG fallback error:', (pgErr as Error).message);
|
||||
}
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
}
|
||||
|
||||
// Build partial AnalysisSession with completed components filled in
|
||||
const getResult = (comp: string) => {
|
||||
if (!state.completedComponents.includes(comp as any)) return null;
|
||||
const raw = state.results[comp as keyof typeof state.results]?.data as any;
|
||||
return raw?.result || raw || null;
|
||||
};
|
||||
|
||||
const allComponents: string[] = ['techniques', 'ai_tampered', 'claims', 'domain'];
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const session = {
|
||||
session_id: sessionId,
|
||||
user_id: state.userId || null,
|
||||
user_email: state.userEmail || null,
|
||||
input_type: state.inputType || 'text',
|
||||
input_text: state.inputText || null,
|
||||
input_url: state.inputUrl || null,
|
||||
input_media_url: state.mediaUrl || null,
|
||||
input_hash: null,
|
||||
status: state.status === 'completed' ? 'completed' : 'running',
|
||||
components_run: [...state.completedComponents],
|
||||
components_skipped: allComponents.filter(c => !state.completedComponents.includes(c as any) && state.status === 'completed'),
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: new Date(state.startTime).toISOString(),
|
||||
completed_at: state.status === 'completed' ? now : null,
|
||||
total_duration_ms: Date.now() - state.startTime,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'api',
|
||||
api_version: 'v3',
|
||||
created_at: new Date(state.startTime).toISOString(),
|
||||
techniques: getResult('techniques'),
|
||||
ai_tampered: getResult('ai_tampered'),
|
||||
claims: getResult('claims'),
|
||||
domain: getResult('domain'),
|
||||
verdict: null,
|
||||
_queue: {
|
||||
progress: Math.round((state.completedComponents.length / state.totalComponents) * 100),
|
||||
total_components: state.totalComponents,
|
||||
completed_components: state.completedComponents,
|
||||
elapsed_ms: Date.now() - state.startTime,
|
||||
plan_type: state.planType,
|
||||
},
|
||||
};
|
||||
|
||||
res.json({ success: true, data: session });
|
||||
} catch (error) {
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
/**
|
||||
* Session lifecycle control:
|
||||
* POST /pipeline/:sessionId/cancel — cancel a running session (Redis flag;
|
||||
* workers short-circuit, aggregator finalizes as 'canceled').
|
||||
* POST /pipeline/:sessionId/resume — resume from checkpoint: re-dispatch ONLY
|
||||
* the components that never completed (see dispatcher.resumeDispatch).
|
||||
*
|
||||
* Only the session owner (JWT identity, body/query user_id in staging) or an
|
||||
* admin can act. Cancel is idempotent.
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { QueueKeys } from '../../shared/redis/keys';
|
||||
import { resumeDispatch } from '../../queue/dispatcher';
|
||||
import { resolveUserId, authRequired, ROLES_ADMIN } from '../../shared/auth/guards';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
const CANCEL_FLAG_TTL = 3600; // 1h — outlives any in-flight component
|
||||
|
||||
router.post('/:sessionId/cancel', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const userId = resolveUserId(req, req.body?.user_id ?? req.query.user_id);
|
||||
if (!userId) return authRequired(res);
|
||||
|
||||
const redis = getRedis();
|
||||
const stateRaw = await redis.get(QueueKeys.sessionState(sessionId));
|
||||
if (!stateRaw) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Session not found or already finished (state expired)',
|
||||
});
|
||||
}
|
||||
|
||||
const state = JSON.parse(stateRaw);
|
||||
const isAdmin = (req.jwtRoles || []).some(r => ROLES_ADMIN.includes(r));
|
||||
if (state.userId && state.userId !== userId && !isAdmin) {
|
||||
return res.status(403).json({ success: false, error: 'Access denied' });
|
||||
}
|
||||
|
||||
if (state.status === 'completed' || state.status === 'failed') {
|
||||
return res.status(409).json({
|
||||
success: false,
|
||||
error: `Session already ${state.status} — nothing to cancel`,
|
||||
status: state.status,
|
||||
});
|
||||
}
|
||||
|
||||
await redis.set(QueueKeys.cancelFlag(sessionId), userId, 'EX', CANCEL_FLAG_TTL);
|
||||
log.info(`[Pipeline API] Session ${sessionId} canceled by ${userId}`);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'canceling',
|
||||
message: 'Cancel flag set — in-flight components will short-circuit and the session will finalize as canceled',
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Cancel error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/:sessionId/resume', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const userId = resolveUserId(req, req.body?.user_id ?? req.query.user_id);
|
||||
if (!userId) return authRequired(res);
|
||||
|
||||
const redis = getRedis();
|
||||
const stateRaw = await redis.get(QueueKeys.sessionState(sessionId));
|
||||
if (!stateRaw) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found or already finished (state expired)' });
|
||||
}
|
||||
const state = JSON.parse(stateRaw);
|
||||
const isAdmin = (req.jwtRoles || []).some(r => ROLES_ADMIN.includes(r));
|
||||
if (state.userId && state.userId !== userId && !isAdmin) {
|
||||
return res.status(403).json({ success: false, error: 'Access denied' });
|
||||
}
|
||||
|
||||
const result = await resumeDispatch(sessionId);
|
||||
if (!result.ok) {
|
||||
// 409 for terminal/expired states, so the caller can distinguish from auth errors.
|
||||
return res.status(409).json({ success: false, error: result.error, data: result });
|
||||
}
|
||||
|
||||
log.info(`[Pipeline API] Session ${sessionId} resumed by ${userId} — requeued: [${result.requeued.join(', ')}]`);
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'running',
|
||||
requeued_components: result.requeued,
|
||||
already_complete: result.alreadyComplete,
|
||||
message: result.requeued.length
|
||||
? `Resumed — re-dispatched ${result.requeued.length} component(s); completed work preserved`
|
||||
: 'Nothing to resume — all components already complete',
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Resume error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* POST /pipeline/dry-run — resolve the FULL execution plan without dispatching.
|
||||
*
|
||||
* Answers "ce ar rula pipeline-ul pentru acest input" using the exact same
|
||||
* decision code as the real execution (getComponentsToRun + Redis config):
|
||||
* - which components run / are skipped and WHY (media-type rules, skip list)
|
||||
* - the flow with explicit dependencies (intake → [media_preprocess] →
|
||||
* components → verdict_aggregator → persist)
|
||||
* - per component: queue name, timeout, stage→model chains (primary +
|
||||
* fallbacks per tier) and the prompt config keys that would be used
|
||||
* - the verdict profile (input_type_profile) with its weights
|
||||
*
|
||||
* Read-only: no session id allocated, no credits consumed, no messages
|
||||
* published. Modul 1 caiet: „execuție dry-run pentru validare".
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { getComponentsToRun, DEFAULT_COMPONENT_CONFIG, type ComponentConfig } from '../../components/pipeline/executor-helpers/component-config';
|
||||
import type { PipelineInput, MediaType } from '../../components/pipeline/types';
|
||||
import type { AnalysisComponent } from '../../shared/types/component-results';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import { getQueueName, MEDIA_QUEUE, PLAN_PRIORITY, RESULTS_QUEUE } from '../../shared/queue/constants';
|
||||
import type { PlanType } from '../../shared/queue/constants';
|
||||
import { resolveUserId, authRequired } from '../../shared/auth/guards';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
const STAGE_ASSIGNMENT_KEYS: Record<AnalysisComponent, string> = {
|
||||
techniques: ConfigKeys.techniquesStageAssignments,
|
||||
ai_tampered: ConfigKeys.aiTamperedStageAssignments,
|
||||
claims: ConfigKeys.claimsStageAssignments,
|
||||
domain: ConfigKeys.sourceAssessmentStageAssignments,
|
||||
};
|
||||
|
||||
// Redis prompt-key prefix per component (didi:config:<prefix>:prompts:*)
|
||||
const PROMPT_KEY_PATTERNS: Record<AnalysisComponent, string> = {
|
||||
techniques: 'didi:config:techniques:*:prompts:*',
|
||||
ai_tampered: 'didi:config:ai-tampered:*:prompts:*',
|
||||
claims: 'didi:config:claims:*:prompts:*',
|
||||
domain: 'didi:config:source-assessment:*:prompts:*',
|
||||
};
|
||||
|
||||
/** input profile (pipeline definition) inferred exactly like verdict calculation */
|
||||
function inferProfileCode(mediaType: string, hasUrl: boolean): string {
|
||||
if (mediaType === 'text') return hasUrl ? 'text_with_url' : 'text_no_url';
|
||||
return mediaType; // image | audio | video | url
|
||||
}
|
||||
|
||||
interface StageModelSummary {
|
||||
order: number;
|
||||
role: string;
|
||||
model_key: string;
|
||||
provider: string;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
router.post('/dry-run', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_user_id, plan_type, options } = req.body;
|
||||
|
||||
const user_id = resolveUserId(req, body_user_id);
|
||||
if (!user_id) return authRequired(res);
|
||||
|
||||
const validTypes: MediaType[] = ['text', 'url', 'image', 'audio', 'video'];
|
||||
if (!media_type || !validTypes.includes(media_type)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
const planType: PlanType = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
const redis = getRedis();
|
||||
|
||||
// 1. Component selection — SAME code path as PipelineExecutor.execute()
|
||||
const rawConfig = await redis.get(ConfigKeys.pipelineComponentConfig);
|
||||
const componentConfig: ComponentConfig = rawConfig ? JSON.parse(rawConfig) : DEFAULT_COMPONENT_CONFIG;
|
||||
const skipComponents: string[] = options?.skip_components || [];
|
||||
const input = { text, media_url, url, media_type, user_id, options } as PipelineInput;
|
||||
const componentsToRun = getComponentsToRun(input, skipComponents, componentConfig);
|
||||
|
||||
const allComponents: AnalysisComponent[] = ['techniques', 'ai_tampered', 'claims', 'domain'];
|
||||
const skipped = allComponents
|
||||
.filter(c => !componentsToRun.includes(c))
|
||||
.map(c => ({
|
||||
component: c,
|
||||
reason: skipComponents.includes(c)
|
||||
? 'skip_components (request option)'
|
||||
: !componentConfig.components[c]?.enabled
|
||||
? 'disabled in component_config'
|
||||
: `not applicable for media_type '${media_type}'`,
|
||||
}));
|
||||
|
||||
// 2. Per-component node plan: queue, timeout, stage→model chains, prompts
|
||||
const nodes = await Promise.all(componentsToRun.map(async (component) => {
|
||||
const cfg = componentConfig.components[component];
|
||||
|
||||
let stages: Record<string, Record<string, StageModelSummary[]>> = {};
|
||||
try {
|
||||
const raw = await redis.get(STAGE_ASSIGNMENT_KEYS[component]);
|
||||
if (raw) {
|
||||
const assignments = JSON.parse(raw);
|
||||
for (const [stageName, tiers] of Object.entries<any>(assignments)) {
|
||||
stages[stageName] = {};
|
||||
for (const [tier, tierCfg] of Object.entries<any>(tiers)) {
|
||||
stages[stageName][tier] = (tierCfg.models || []).map((m: any) => ({
|
||||
order: m.order, role: m.role, model_key: m.model_key,
|
||||
provider: m.provider, timeout_ms: m.timeout_ms,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn(`[DryRun] Stage assignments unavailable for ${component}: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
let promptKeys: string[] = [];
|
||||
try {
|
||||
promptKeys = await redis.keys(PROMPT_KEY_PATTERNS[component]);
|
||||
} catch { /* prompt listing is best-effort */ }
|
||||
|
||||
return {
|
||||
component,
|
||||
queue: getQueueName(component, planType),
|
||||
priority: PLAN_PRIORITY[planType],
|
||||
timeout_ms: cfg?.timeout_ms,
|
||||
depends_on: ['image', 'audio', 'video'].includes(media_type) ? ['media_preprocess'] : ['intake'],
|
||||
stages,
|
||||
prompt_config_keys: promptKeys.sort(),
|
||||
};
|
||||
}));
|
||||
|
||||
// 3. Media pre-processing node (video/audio/image only)
|
||||
const isMedia = ['image', 'audio', 'video'].includes(media_type);
|
||||
const mediaNode = isMedia
|
||||
? {
|
||||
component: 'media_preprocess',
|
||||
queue: MEDIA_QUEUE.queueName(planType),
|
||||
depends_on: ['intake'],
|
||||
produces: media_type === 'video'
|
||||
? ['transcript', 'frames', 'buster_verdict', 'forensic_features', 'metadata', 'ner', 'sentiment']
|
||||
: media_type === 'audio'
|
||||
? ['transcript', 'ner', 'sentiment']
|
||||
: ['ocr_text', 'ai_detection', 'forensic_features', 'metadata'],
|
||||
}
|
||||
: null;
|
||||
|
||||
// 4. Verdict profile (pipeline definition) — weights per component
|
||||
const profileCode = inferProfileCode(media_type, !!url);
|
||||
let verdictProfile: Record<string, unknown> | null = null;
|
||||
try {
|
||||
const frameworkUrl = process.env.FRAMEWORK_API_URL || 'http://didi-framework:3005';
|
||||
const resp = await fetch(`${frameworkUrl}/api/input-profiles/${profileCode}`, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
if (resp.ok) {
|
||||
const data = await resp.json() as any;
|
||||
const p = data.data;
|
||||
verdictProfile = {
|
||||
profile_code: p.profile_code,
|
||||
profile_name: p.profile_name,
|
||||
is_active: p.is_active,
|
||||
weights: {
|
||||
techniques: p.weight_techniques, claims: p.weight_claims,
|
||||
ai_tampered: p.weight_ai_tampered, source: p.weight_source,
|
||||
},
|
||||
min_components: p.min_components,
|
||||
override_cap: p.override_cap,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn(`[DryRun] Framework profile lookup failed: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
dry_run: true,
|
||||
data: {
|
||||
media_type,
|
||||
plan_type: planType,
|
||||
flow: [
|
||||
'intake',
|
||||
...(isMedia ? ['media_preprocess'] : []),
|
||||
`[${componentsToRun.join(' ∥ ')}]`,
|
||||
'verdict_aggregator',
|
||||
'persist',
|
||||
].join(' → '),
|
||||
results_queue: RESULTS_QUEUE,
|
||||
nodes: [...(mediaNode ? [mediaNode] : []), ...nodes],
|
||||
skipped,
|
||||
verdict_profile: verdictProfile,
|
||||
note: 'Plan resolved with live config (Redis + framework). Nothing was dispatched, no credits consumed.',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[DryRun] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,526 @@
|
|||
/**
|
||||
* Browser-extension API endpoints (extracted from pipeline-routes.ts):
|
||||
* POST /pipeline/extension/analyze — extension calls this with X-API-Key
|
||||
* POST /pipeline/extension/keys — admin: create key (proxies didiFramework)
|
||||
* GET /pipeline/extension/keys — admin: list keys
|
||||
* DELETE /pipeline/extension/keys/:id — admin: revoke key
|
||||
*
|
||||
* Extension uses an X-API-Key (validated against didiFramework via HTTP). The
|
||||
* admin endpoints require JWT (req.jwtUserId set by Kong upstream).
|
||||
*/
|
||||
import crypto from 'crypto';
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { PipelineExecutor } from '../../components/pipeline/executor';
|
||||
import { PipelineInput, MediaType } from '../../components/pipeline/types';
|
||||
import { checkCredits, deductCredits, getSearchTier } from '../../shared/credits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { dispatch, getSessionState } from '../../queue';
|
||||
import type { PlanType } from '../../queue';
|
||||
import { getPgPool } from '../../shared/persistence';
|
||||
import { FRAMEWORK_API_URL, getRedis, getMediaService, upload } from './_init';
|
||||
import { extractImageVision } from './_helpers/vision';
|
||||
import { fetchArticleText, detectUrlType } from './_helpers/url-helpers';
|
||||
|
||||
const router = Router();
|
||||
|
||||
const EXTENSION_KEYS_API = `${FRAMEWORK_API_URL}/api/extension-keys`;
|
||||
|
||||
async function validateApiKey(apiKey: string): Promise<{ valid: boolean; userId?: string; userEmail?: string; name?: string }> {
|
||||
if (!apiKey) return { valid: false };
|
||||
|
||||
try {
|
||||
const response = await fetch(`${EXTENSION_KEYS_API}/validate`, {
|
||||
method: 'GET',
|
||||
headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) return { valid: false };
|
||||
|
||||
const result = await response.json() as { success: boolean; data?: { valid: boolean; user_id?: string; user_email?: string; name?: string } };
|
||||
if (result.success && result.data?.valid) {
|
||||
// Fire-and-forget usage tracking — increments usage_count + last_used_at in PG
|
||||
fetch(`${EXTENSION_KEYS_API}/usage-by-key`, {
|
||||
method: 'POST',
|
||||
headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(3000),
|
||||
}).catch(err => log.warn('[Extension API] Usage tracking failed:', err.message));
|
||||
|
||||
return { valid: true, userId: result.data.user_id, userEmail: result.data.user_email, name: result.data.name };
|
||||
}
|
||||
return { valid: false };
|
||||
} catch (error) {
|
||||
log.error('[Extension API] Key validation error:', error);
|
||||
return { valid: false };
|
||||
}
|
||||
}
|
||||
|
||||
router.post('/extension/analyze', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const apiKey = req.headers['x-api-key'] as string || req.headers['authorization']?.replace('Bearer ', '');
|
||||
|
||||
if (!apiKey) {
|
||||
return res.status(401).json({ success: false, error: 'Missing API key. Use X-API-Key header or Authorization: Bearer <key>' });
|
||||
}
|
||||
|
||||
const keyInfo = await validateApiKey(apiKey);
|
||||
if (!keyInfo.valid) {
|
||||
return res.status(401).json({ success: false, error: 'Invalid API key' });
|
||||
}
|
||||
|
||||
log.info(`[Extension API] Request from: ${keyInfo.name} (${keyInfo.userId}, ${keyInfo.userEmail || 'no email'})`);
|
||||
|
||||
const { text, image_url, url, options } = req.body;
|
||||
|
||||
if (!text && !image_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'Provide text, image_url, or url' });
|
||||
}
|
||||
|
||||
const detectedMediaType: 'text' | 'image' | 'url' = image_url ? 'image' : (url ? 'url' : 'text');
|
||||
const analysisType = detectedMediaType;
|
||||
|
||||
log.info(`[Extension API] Analyzing ${analysisType} for ${keyInfo.userId}`);
|
||||
|
||||
const creditCheck = await checkCredits(keyInfo.userId!, detectedMediaType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false,
|
||||
error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
const extTier = getSearchTier(creditCheck.planType);
|
||||
|
||||
let input: PipelineInput;
|
||||
if (detectedMediaType === 'image') {
|
||||
const visionExt = await extractImageVision(getRedis(), image_url, 'Extension API', extTier);
|
||||
const imageText = visionExt?.text;
|
||||
input = {
|
||||
text: imageText, media_url: image_url, media_type: 'image', user_id: keyInfo.userId!, user_email: keyInfo.userEmail,
|
||||
options: { ...options, skip_components: options?.skip_components || [] },
|
||||
searchTier: extTier,
|
||||
};
|
||||
} else if (detectedMediaType === 'url') {
|
||||
try {
|
||||
const fetchedText = await fetchArticleText(url);
|
||||
input = {
|
||||
url, text: fetchedText, media_type: 'url', user_id: keyInfo.userId!, user_email: keyInfo.userEmail,
|
||||
options: { ...options, skip_components: options?.skip_components || [] },
|
||||
searchTier: extTier,
|
||||
};
|
||||
} catch (e) {
|
||||
log.warn('[Extension API] URL fetch failed:', (e as Error).message);
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Could not fetch URL content',
|
||||
error_code: 'URL_FETCH_FAILED',
|
||||
});
|
||||
}
|
||||
} else {
|
||||
input = {
|
||||
text, media_type: 'text', user_id: keyInfo.userId!, user_email: keyInfo.userEmail,
|
||||
options: { ...options, skip_components: options?.skip_components || [] },
|
||||
searchTier: extTier,
|
||||
};
|
||||
}
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const result = await executor.execute(input);
|
||||
|
||||
const deducted = await deductCredits(keyInfo.userId!, input.media_type, result.session_id);
|
||||
if (!deducted) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${keyInfo.userId} type=${input.media_type}`);
|
||||
|
||||
res.json({ success: true, data: { ...result, analysis_type: analysisType } });
|
||||
} catch (error) {
|
||||
log.error('[Extension API] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Extension media upload — X-API-Key auth (mirrors /media/upload but no JWT)
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
router.post('/extension/upload', upload.single('file'), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const apiKey = req.headers['x-api-key'] as string || req.headers['authorization']?.replace('Bearer ', '');
|
||||
if (!apiKey) {
|
||||
return res.status(401).json({ success: false, error: 'Missing API key. Use X-API-Key header.' });
|
||||
}
|
||||
const keyInfo = await validateApiKey(apiKey);
|
||||
if (!keyInfo.valid || !keyInfo.userId) {
|
||||
return res.status(401).json({ success: false, error: 'Invalid API key' });
|
||||
}
|
||||
|
||||
const file = (req as any).file as Express.Multer.File | undefined;
|
||||
if (!file) {
|
||||
return res.status(400).json({ success: false, error: 'No file provided. Use multipart/form-data with field "file"' });
|
||||
}
|
||||
|
||||
log.info(`[Extension API] Upload from ${keyInfo.userId}: ${file.originalname} (${file.size} bytes, ${file.mimetype})`);
|
||||
|
||||
const result = await getMediaService().uploadFile(keyInfo.userId, file.buffer, file.originalname, file.mimetype);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
download_url: result.download_url,
|
||||
public_url: result.public_url,
|
||||
object_key: result.object_key,
|
||||
bucket: result.bucket,
|
||||
filename: result.filename,
|
||||
size: result.size,
|
||||
content_type: result.content_type,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Extension API] Upload error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Extension async analyze — dispatches to RabbitMQ, returns 202 + session_id
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
router.post('/extension/analyze-async', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const apiKey = req.headers['x-api-key'] as string || req.headers['authorization']?.replace('Bearer ', '');
|
||||
if (!apiKey) {
|
||||
return res.status(401).json({ success: false, error: 'Missing API key. Use X-API-Key header.' });
|
||||
}
|
||||
const keyInfo = await validateApiKey(apiKey);
|
||||
if (!keyInfo.valid || !keyInfo.userId) {
|
||||
return res.status(401).json({ success: false, error: 'Invalid API key' });
|
||||
}
|
||||
|
||||
const { text, image_url, video_url, audio_url, url, options } = req.body as {
|
||||
text?: string; image_url?: string; video_url?: string; audio_url?: string; url?: string;
|
||||
options?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
// Detect media type from which URL field is present
|
||||
let mediaType: MediaType = 'text';
|
||||
let mediaUrl: string | undefined;
|
||||
if (image_url) { mediaType = 'image'; mediaUrl = image_url; }
|
||||
else if (video_url) { mediaType = 'video'; mediaUrl = video_url; }
|
||||
else if (audio_url) { mediaType = 'audio'; mediaUrl = audio_url; }
|
||||
else if (url) { mediaType = 'url'; }
|
||||
else if (text) { mediaType = 'text'; }
|
||||
else {
|
||||
return res.status(400).json({ success: false, error: 'Provide text, image_url, video_url, audio_url, or url' });
|
||||
}
|
||||
|
||||
const creditCheck = await checkCredits(keyInfo.userId, mediaType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(503).json({ success: false, error: 'Credit service temporarily unavailable.', error_code: 'CREDIT_SERVICE_UNAVAILABLE' });
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
|
||||
const planTypeValue: PlanType = (creditCheck.planType >= 1 && creditCheck.planType <= 6 ? creditCheck.planType : 1) as PlanType;
|
||||
const sessionId = crypto.randomUUID();
|
||||
|
||||
let content = text || '';
|
||||
let effectiveMediaType = mediaType;
|
||||
|
||||
if (url && !text) {
|
||||
const urlType = detectUrlType(url);
|
||||
if (urlType === 'video_platform') {
|
||||
effectiveMediaType = 'video';
|
||||
} else {
|
||||
try {
|
||||
content = await fetchArticleText(url);
|
||||
} catch (e) {
|
||||
log.warn(`[Extension Async] Failed to fetch URL: ${(e as Error).message}`);
|
||||
content = `URL: ${url}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[Extension Async] Dispatching ${effectiveMediaType} for user ${keyInfo.userId}, plan ${planTypeValue}`);
|
||||
|
||||
const dispatchResult = await dispatch(
|
||||
sessionId,
|
||||
{ content, url, mediaPath: mediaUrl, userId: keyInfo.userId, userEmail: keyInfo.userEmail, inputType: effectiveMediaType },
|
||||
planTypeValue,
|
||||
);
|
||||
|
||||
if (!dispatchResult.async) {
|
||||
// RabbitMQ down — fall back to sync (small text only, otherwise it'll hit Cloudflare timeout)
|
||||
log.warn('[Extension Async] Queue unavailable, sync fallback');
|
||||
const input: PipelineInput = {
|
||||
text: content, media_url: mediaUrl, url, media_type: mediaType, session_id: sessionId,
|
||||
user_id: keyInfo.userId, user_email: keyInfo.userEmail, options,
|
||||
searchTier: getSearchTier(creditCheck.planType),
|
||||
};
|
||||
const executor = new PipelineExecutor(getRedis());
|
||||
const result = await executor.execute(input);
|
||||
const deducted = await deductCredits(keyInfo.userId, mediaType, result.session_id);
|
||||
if (!deducted) log.error(`[Credits] BILLING_GAP session=${result.session_id} user=${keyInfo.userId}`);
|
||||
return res.json({ success: true, async: false, data: result, message: 'Processed synchronously (queue unavailable)' });
|
||||
}
|
||||
|
||||
const deducted = await deductCredits(keyInfo.userId, mediaType, sessionId);
|
||||
if (!deducted) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${keyInfo.userId}`);
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
queued_components: dispatchResult.queued,
|
||||
plan_type: planTypeValue,
|
||||
poll_url: `/api/v3/pipeline/extension/status/${sessionId}`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Extension Async] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Extension status polling — X-API-Key auth + ownership check
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
router.get('/extension/status/:sessionId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const apiKey = req.headers['x-api-key'] as string || req.headers['authorization']?.replace('Bearer ', '');
|
||||
if (!apiKey) {
|
||||
return res.status(401).json({ success: false, error: 'Missing API key' });
|
||||
}
|
||||
const keyInfo = await validateApiKey(apiKey);
|
||||
if (!keyInfo.valid || !keyInfo.userId) {
|
||||
return res.status(401).json({ success: false, error: 'Invalid API key' });
|
||||
}
|
||||
|
||||
const { sessionId } = req.params;
|
||||
const state = await getSessionState(sessionId);
|
||||
|
||||
// If session is in queue, build partial AnalysisSession from in-flight state
|
||||
if (state) {
|
||||
// Ownership check: the session must belong to the API key holder
|
||||
if (state.userId && state.userId !== keyInfo.userId) {
|
||||
return res.status(403).json({ success: false, error: 'Session does not belong to this API key' });
|
||||
}
|
||||
|
||||
// When state shows completed (or all components done) prefer the persisted
|
||||
// session — the queue state never carries verdict output and may be stale
|
||||
// when the aggregator marks the session failed without updating Redis.
|
||||
const allDone = state.completedComponents.length >= state.totalComponents;
|
||||
if (state.status === 'completed' || allDone) {
|
||||
const persist = (await import('./_init')).getPersistService();
|
||||
const full = (await persist.loadFromCache(sessionId)) || (await persist.loadFromDb(sessionId));
|
||||
if (full) return res.json({ success: true, data: full });
|
||||
|
||||
// Final guard: PG might be ahead of cache for a failed session. Check raw status.
|
||||
const pool = getPgPool();
|
||||
const pgRow = await pool.query(
|
||||
'SELECT status FROM bos_analysis.analysis_session WHERE session_id = $1',
|
||||
[sessionId],
|
||||
);
|
||||
if (pgRow.rows.length && (pgRow.rows[0].status === 'completed' || pgRow.rows[0].status === 'failed')) {
|
||||
return res.json({
|
||||
success: true,
|
||||
data: { session_id: sessionId, status: pgRow.rows[0].status, _queue: { progress: 100 } },
|
||||
});
|
||||
}
|
||||
// Persistence not yet flushed — fall through to partial state below
|
||||
}
|
||||
|
||||
const getResult = (comp: string) => {
|
||||
if (!state.completedComponents.includes(comp as any)) return null;
|
||||
const raw = state.results[comp as keyof typeof state.results]?.data as any;
|
||||
return raw?.result || raw || null;
|
||||
};
|
||||
const allComponents: string[] = ['techniques', 'ai_tampered', 'claims', 'domain'];
|
||||
const isDone = state.status === 'completed';
|
||||
const session = {
|
||||
session_id: sessionId,
|
||||
user_id: state.userId || null,
|
||||
input_type: state.inputType || 'text',
|
||||
input_text: state.inputText || null,
|
||||
input_url: state.inputUrl || null,
|
||||
input_media_url: state.mediaUrl || null,
|
||||
status: isDone ? 'completed' : 'running',
|
||||
components_run: [...state.completedComponents],
|
||||
components_skipped: allComponents.filter(c => !state.completedComponents.includes(c as any) && isDone),
|
||||
risk_score: null, risk_category: null, risk_level: null,
|
||||
confidence: null, confidence_level: null,
|
||||
started_at: new Date(state.startTime).toISOString(),
|
||||
completed_at: isDone ? new Date().toISOString() : null,
|
||||
total_duration_ms: Date.now() - state.startTime,
|
||||
api_version: 'v3',
|
||||
techniques: getResult('techniques'),
|
||||
ai_tampered: getResult('ai_tampered'),
|
||||
claims: getResult('claims'),
|
||||
domain: getResult('domain'),
|
||||
verdict: null,
|
||||
_queue: {
|
||||
progress: Math.round((state.completedComponents.length / state.totalComponents) * 100),
|
||||
total_components: state.totalComponents,
|
||||
completed_components: state.completedComponents,
|
||||
elapsed_ms: Date.now() - state.startTime,
|
||||
},
|
||||
};
|
||||
return res.json({ success: true, data: session });
|
||||
}
|
||||
|
||||
// Not in queue — check PG for completed session (ownership enforced via SQL)
|
||||
const pool = getPgPool();
|
||||
const pgRow = await pool.query(
|
||||
`SELECT session_id, user_id, status, started_at, completed_at FROM bos_analysis.analysis_session
|
||||
WHERE session_id = $1`,
|
||||
[sessionId],
|
||||
);
|
||||
if (pgRow.rows.length === 0) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
}
|
||||
const pg = pgRow.rows[0];
|
||||
if (pg.user_id && pg.user_id !== keyInfo.userId) {
|
||||
return res.status(403).json({ success: false, error: 'Session does not belong to this API key' });
|
||||
}
|
||||
|
||||
if (pg.status === 'completed') {
|
||||
// Fetch full session from PG so the extension gets the same payload as web app history
|
||||
const persist = (await import('./_init')).getPersistService();
|
||||
const full = (await persist.loadFromCache(sessionId)) || (await persist.loadFromDb(sessionId));
|
||||
if (full) return res.json({ success: true, data: full });
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: pg.status,
|
||||
started_at: pg.started_at,
|
||||
completed_at: pg.completed_at,
|
||||
_queue: { progress: pg.status === 'completed' ? 100 : 0 },
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Extension Status] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// Extension keys management (admin proxies to didiFramework)
|
||||
|
||||
router.post('/extension/keys', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const user_id = req.jwtUserId;
|
||||
if (!user_id) {
|
||||
return res.status(401).json({ success: false, error: 'Authentication required' });
|
||||
}
|
||||
|
||||
const { name } = req.body;
|
||||
if (!name) {
|
||||
return res.status(400).json({ success: false, error: 'name required' });
|
||||
}
|
||||
|
||||
// Pro tier gate: only plan_type >= 4 (Pro/Business/Enterprise) may generate keys.
|
||||
// Existing keys keep working post-downgrade — gate is at creation only.
|
||||
const creditCheck = await checkCredits(user_id, 'text');
|
||||
if (creditCheck === null) {
|
||||
return res.status(503).json({
|
||||
success: false, error: 'Subscription service temporarily unavailable. Please try again.',
|
||||
error_code: 'SUBSCRIPTION_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (creditCheck.planType < 4) {
|
||||
return res.status(403).json({
|
||||
success: false,
|
||||
error: 'Browser extension requires Pro tier or higher.',
|
||||
error_code: 'PRO_TIER_REQUIRED',
|
||||
data: { currentPlanType: creditCheck.planType, currentPlanName: creditCheck.planName, requiredMinPlanType: 4 },
|
||||
});
|
||||
}
|
||||
|
||||
const user_email = req.jwtEmail;
|
||||
const response = await fetch(EXTENSION_KEYS_API, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ user_id, user_email, name }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
const result = await response.json() as { success: boolean; data?: { api_key: string; [key: string]: any }; error?: string };
|
||||
if (!response.ok) {
|
||||
return res.status(response.status).json(result);
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
...result.data,
|
||||
usage: {
|
||||
endpoint: 'POST /api/v3/pipeline/extension/analyze',
|
||||
headers: { 'X-API-Key': result.data?.api_key || '', 'Content-Type': 'application/json' },
|
||||
body_examples: {
|
||||
text: { text: 'Content to analyze...' },
|
||||
image: { image_url: 'https://example.com/image.jpg' },
|
||||
url: { url: 'https://news-site.com/article' },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Extension API] Key generation error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/extension/keys', async (req: Request, res: Response) => {
|
||||
try {
|
||||
if (!req.jwtUserId) {
|
||||
return res.status(401).json({ success: false, error: 'Authentication required' });
|
||||
}
|
||||
|
||||
const response = await fetch(EXTENSION_KEYS_API, {
|
||||
method: 'GET',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
res.status(response.status).json(result);
|
||||
} catch (error) {
|
||||
log.error('[Extension API] List keys error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/extension/keys/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
if (!req.jwtUserId) {
|
||||
return res.status(401).json({ success: false, error: 'Authentication required' });
|
||||
}
|
||||
|
||||
const { id } = req.params;
|
||||
const response = await fetch(`${EXTENSION_KEYS_API}/${id}`, {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
res.status(response.status).json(result);
|
||||
} catch (error) {
|
||||
log.error('[Extension API] Delete key error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
/**
|
||||
* Pipeline history endpoints (extracted from pipeline-routes.ts):
|
||||
* GET /pipeline/history — user's own history (paginated)
|
||||
* GET /pipeline/history/admin — admin: all sessions with filters
|
||||
* GET /pipeline/history/admin/:id — admin: single session by id
|
||||
* DELETE /pipeline/history/admin/:id — admin: delete any session
|
||||
* GET /pipeline/history/:id — user: single (ownership-checked)
|
||||
* DELETE /pipeline/history/:id — user: delete (ownership-checked)
|
||||
*
|
||||
* Admin routes MUST be registered before /:id (Express matches in order — without
|
||||
* this, /admin would be captured as :id = 'admin').
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { resolveUserId, authRequired, requireRole, ROLES_ADMIN } from '../../shared/auth/guards';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getPersistService } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/history', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const userId = resolveUserId(req, req.query.user_id);
|
||||
const page = parseInt(req.query.page as string) || 1;
|
||||
const limit = Math.min(parseInt(req.query.limit as string) || 20, 100);
|
||||
|
||||
if (!userId) {
|
||||
return authRequired(res);
|
||||
}
|
||||
|
||||
const persist = getPersistService();
|
||||
const { items, total } = await persist.loadHistory(userId, page, limit);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
items,
|
||||
pagination: { page, limit, total, pages: Math.ceil(total / limit) },
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] History list error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// Admin routes — MUST be before /:id. Role-gated: admin JWT required in
|
||||
// production (staging soft-permits when no JWT is present — see guards.ts).
|
||||
|
||||
router.get('/history/admin', requireRole(...ROLES_ADMIN), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const page = parseInt(req.query.page as string) || 1;
|
||||
const limit = Math.min(parseInt(req.query.limit as string) || 20, 100);
|
||||
const filters = {
|
||||
search: (req.query.search as string) || undefined,
|
||||
risk_level: (req.query.risk_level as string) || undefined,
|
||||
status: (req.query.status as string) || undefined,
|
||||
from_date: (req.query.from_date as string) || undefined,
|
||||
to_date: (req.query.to_date as string) || undefined,
|
||||
};
|
||||
|
||||
const persist = getPersistService();
|
||||
const { items, total } = await persist.loadHistoryAdmin(page, limit, filters);
|
||||
const totalPages = Math.ceil(total / limit);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
items,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
total,
|
||||
total_pages: totalPages,
|
||||
has_next: page < totalPages,
|
||||
has_prev: page > 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Admin history list error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/history/admin/:id', requireRole(...ROLES_ADMIN), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
|
||||
const persist = getPersistService();
|
||||
const session = await persist.loadFromDb(id);
|
||||
|
||||
if (!session) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: session });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Admin history get error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/history/admin/:id', requireRole(...ROLES_ADMIN), async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
|
||||
const persist = getPersistService();
|
||||
const { pg } = await persist.deleteSession(id);
|
||||
|
||||
if (!pg) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found or delete failed' });
|
||||
}
|
||||
|
||||
res.json({ success: true, message: 'Session deleted' });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Admin history delete error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// User-scoped routes (ownership checked)
|
||||
|
||||
router.get('/history/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const userId = resolveUserId(req, req.query.user_id);
|
||||
|
||||
if (!userId) {
|
||||
return authRequired(res);
|
||||
}
|
||||
|
||||
const persist = getPersistService();
|
||||
const session = await persist.loadFromCache(id);
|
||||
|
||||
if (!session) {
|
||||
return res.status(404).json({ success: false, error: 'History entry not found' });
|
||||
}
|
||||
|
||||
if (session.user_id !== userId) {
|
||||
return res.status(403).json({ success: false, error: 'Access denied' });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: session });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] History get error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/history/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
const userId = resolveUserId(req, req.query.user_id);
|
||||
|
||||
if (!userId) {
|
||||
return authRequired(res);
|
||||
}
|
||||
|
||||
const persist = getPersistService();
|
||||
const session = await persist.loadFromCache(id);
|
||||
|
||||
if (!session) {
|
||||
return res.status(404).json({ success: false, error: 'History entry not found' });
|
||||
}
|
||||
|
||||
if (session.user_id !== userId) {
|
||||
return res.status(403).json({ success: false, error: 'Access denied' });
|
||||
}
|
||||
|
||||
const { pg } = await persist.deleteSession(id);
|
||||
if (!pg) {
|
||||
return res.status(500).json({ success: false, error: 'Failed to delete from database' });
|
||||
}
|
||||
|
||||
res.json({ success: true, message: 'History entry deleted' });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] History delete error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* AGENT V3 — PIPELINE API barrel router.
|
||||
*
|
||||
* The original 1611-line pipeline-routes.ts was split into 5 logical groups
|
||||
* (mounted under the same /pipeline prefix in src/index.ts).
|
||||
*
|
||||
* analyze.ts — POST /analyze, /analyze-url, /analyze-media (~530 LOC)
|
||||
* status.ts — GET /verdict-config + /:sessionId/{status,component/:name,result} (~150 LOC)
|
||||
* history.ts — GET/DELETE /history + admin variants (~165 LOC)
|
||||
* extension.ts — POST/GET/DELETE /extension/* (X-API-Key + admin) (~210 LOC)
|
||||
* async.ts — POST /analyze-async + /queue-health + /:sessionId/queue-status (~290 LOC)
|
||||
*
|
||||
* Shared infra is in _init.ts (lazy redis/persist/media singletons + FRAMEWORK_API_URL)
|
||||
* and _helpers/ (vision + url-helpers).
|
||||
*
|
||||
* Mount-order matters in two places:
|
||||
* 1) status.ts registers /verdict-config BEFORE /:sessionId/* (defensive).
|
||||
* 2) history.ts registers /history/admin BEFORE /history/:id (otherwise
|
||||
* `admin` would be captured as :id).
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import analyzeRouter from './analyze';
|
||||
import statusRouter from './status';
|
||||
import historyRouter from './history';
|
||||
import extensionRouter from './extension';
|
||||
import asyncRouter from './async';
|
||||
import dryRunRouter from './dry-run';
|
||||
import cancelRouter from './cancel';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(analyzeRouter);
|
||||
router.use(statusRouter);
|
||||
router.use(historyRouter);
|
||||
router.use(extensionRouter);
|
||||
router.use(asyncRouter);
|
||||
router.use(dryRunRouter);
|
||||
router.use(cancelRouter);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
/**
|
||||
* Pipeline status / result endpoints (extracted from pipeline-routes.ts):
|
||||
* GET /pipeline/verdict-config — read-only verdict config (admin dashboard)
|
||||
* GET /pipeline/:sessionId/status — poll component-level status
|
||||
* GET /pipeline/:sessionId/component/:name — single component result
|
||||
* GET /pipeline/:sessionId/result — full session (Redis → PG fallback)
|
||||
*
|
||||
* verdict-config is registered FIRST so it's never shadowed by /:sessionId/...
|
||||
* (current Express order would already disambiguate by path segment count, but
|
||||
* keeping it first is defensive against future single-segment routes).
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { PipelineExecutor } from '../../components/pipeline/executor';
|
||||
import { combineVideoProbability } from '../../shared/media/video-weighting';
|
||||
import { AgentKeys, ConfigKeys } from '../../shared/redis/keys';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis, getPersistService } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/pipeline/verdict-config - Read-only verdict config
|
||||
// ============================================================================
|
||||
|
||||
router.get('/verdict-config', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const redis = getRedis();
|
||||
const raw = await redis.get(ConfigKeys.pipelineComponentConfig.replace('component_config', 'verdict_config'));
|
||||
const config = raw ? JSON.parse(raw) : null;
|
||||
res.json({ success: true, data: config });
|
||||
} catch (err) {
|
||||
internalError(res, err, 'pipeline_verdict_config');
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/pipeline/:sessionId/status - Poll status
|
||||
// ============================================================================
|
||||
|
||||
router.get('/:sessionId/status', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const status = await executor.getStatus(sessionId);
|
||||
|
||||
if (!status) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: status });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Status error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/pipeline/:sessionId/component/:name - Get component result
|
||||
// ============================================================================
|
||||
|
||||
router.get('/:sessionId/component/:name', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId, name } = req.params;
|
||||
|
||||
const validComponents = ['domain', 'techniques', 'ai_tampered', 'claims', 'verdict'];
|
||||
if (!validComponents.includes(name)) {
|
||||
return res.status(400).json({ success: false, error: `Invalid component. Valid: ${validComponents.join(', ')}` });
|
||||
}
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
|
||||
const result = name === 'verdict'
|
||||
? await executor.getVerdict(sessionId)
|
||||
: await executor.getComponentResult(sessionId, name);
|
||||
|
||||
if (!result) {
|
||||
const status = await executor.getStatus(sessionId);
|
||||
if (!status) {
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
}
|
||||
return res.json({
|
||||
success: true,
|
||||
data: null,
|
||||
message: `Component ${name} not ready yet`,
|
||||
status: status.components[name as keyof typeof status.components]?.status || 'unknown',
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ success: true, data: result });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Component error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/pipeline/:sessionId/result - Get full result
|
||||
// ============================================================================
|
||||
|
||||
router.get('/:sessionId/result', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
|
||||
const redis = getRedis();
|
||||
const executor = new PipelineExecutor(redis);
|
||||
const session = await executor.getFullResult(sessionId);
|
||||
|
||||
if (session) {
|
||||
// For video: merge visual analysis (frame analysis) into ai_tampered result.
|
||||
// Uses shared combineVideoProbability — historical bug had this path inverted
|
||||
// 0.6/0.4 vs aggregator's 0.4/0.6, producing different scores for same video.
|
||||
if (session.ai_tampered) {
|
||||
const visualJson = await redis.get(AgentKeys.aiTamperedVisual(sessionId));
|
||||
if (visualJson) {
|
||||
const visual = JSON.parse(visualJson);
|
||||
session.ai_tampered.video_analysis = visual;
|
||||
const textProb = session.ai_tampered.ai_probability || 0;
|
||||
const visualIndicatesAI = visual.visual_analysis &&
|
||||
/\b(ai[- ]generated|deepfake|synthetic|artificial|sora|runway|midjourney)\b/i.test(visual.visual_analysis);
|
||||
const visualProb = visualIndicatesAI ? 70 : 20;
|
||||
session.ai_tampered.ai_probability_text = textProb;
|
||||
session.ai_tampered.ai_probability_visual = visualProb;
|
||||
session.ai_tampered.ai_probability = combineVideoProbability(textProb, visualProb);
|
||||
if (session.ai_tampered.ai_probability >= 80) session.ai_tampered.verdict = 'LIKELY_AI';
|
||||
else if (session.ai_tampered.ai_probability >= 50) session.ai_tampered.verdict = 'UNCERTAIN';
|
||||
else if (session.ai_tampered.ai_probability >= 20) session.ai_tampered.verdict = 'POSSIBLY_HUMAN';
|
||||
else session.ai_tampered.verdict = 'LIKELY_HUMAN';
|
||||
}
|
||||
}
|
||||
|
||||
return res.json({ success: true, data: session });
|
||||
}
|
||||
|
||||
log.info(`[Pipeline API] Session ${sessionId} not in Redis, trying PG fallback...`);
|
||||
try {
|
||||
const persist = getPersistService();
|
||||
const dbSession = await persist.loadFromDb(sessionId);
|
||||
if (dbSession) {
|
||||
log.info(`[Pipeline API] Session ${sessionId} found in PostgreSQL`);
|
||||
return res.json({ success: true, data: dbSession });
|
||||
}
|
||||
} catch (fallbackError) {
|
||||
log.error('[Pipeline API] PG fallback error:', (fallbackError as Error).message);
|
||||
}
|
||||
|
||||
return res.status(404).json({ success: false, error: 'Session not found' });
|
||||
} catch (error) {
|
||||
log.error('[Pipeline API] Result error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
167
backend/services/orchestration-layer/agent-v3/src/api/routes.ts
Normal file
167
backend/services/orchestration-layer/agent-v3/src/api/routes.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
/**
|
||||
* AGENT V3 — API barrel router.
|
||||
*
|
||||
* The original 1373-line file was split into three logical groups in this PR.
|
||||
* Consumers of this module (src/index.ts) keep importing `./routes` so the
|
||||
* mount path stays `/api/v3` — internally we now mount three sub-routers.
|
||||
*
|
||||
* /techniques/* → techniques.ts (~780 LOC)
|
||||
* /media/* → media.ts (~170 LOC)
|
||||
* /domain/* → domain.ts (~330 LOC)
|
||||
*
|
||||
* Health check stays here because it's not tied to any of the three groups.
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import techniquesRouter from './techniques';
|
||||
import mediaRouter from './media';
|
||||
import domainRouter from './domain';
|
||||
import { createRedisConnection } from '../shared/redis/connection';
|
||||
import { getPgPool } from '../shared/persistence/pg-pool';
|
||||
import { busterHealthCheck } from '../shared/media/buster';
|
||||
import { log } from '../shared/logger';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(techniquesRouter);
|
||||
router.use(mediaRouter);
|
||||
router.use(domainRouter);
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/health — Liveness probe (lightweight, always OK if process alive)
|
||||
// ============================================================================
|
||||
router.get('/health', (req: Request, res: Response) => {
|
||||
res.json({
|
||||
status: 'ok',
|
||||
service: 'agent-v3',
|
||||
version: '3.0.0',
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/health/all — Deep healthcheck (dependencies)
|
||||
// Cerință caiet criteriu E3 (health checks) + observabilitate runbook
|
||||
// ============================================================================
|
||||
router.get('/health/all', async (req: Request, res: Response) => {
|
||||
const checks: Record<string, { status: 'healthy' | 'unhealthy' | 'unknown'; latency_ms?: number; error?: string }> = {};
|
||||
const start = Date.now();
|
||||
|
||||
// Postgres
|
||||
try {
|
||||
const t = Date.now();
|
||||
const pool = getPgPool();
|
||||
await pool.query('SELECT 1');
|
||||
checks.postgres = { status: 'healthy', latency_ms: Date.now() - t };
|
||||
} catch (e) {
|
||||
checks.postgres = { status: 'unhealthy', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// Redis
|
||||
try {
|
||||
const t = Date.now();
|
||||
const redis = createRedisConnection();
|
||||
await redis.ping();
|
||||
redis.disconnect();
|
||||
checks.redis = { status: 'healthy', latency_ms: Date.now() - t };
|
||||
} catch (e) {
|
||||
checks.redis = { status: 'unhealthy', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// RabbitMQ mgmt API (separate from AMQP — uses dedicated MGMT creds)
|
||||
const rmqUrl = process.env.RABBITMQ_MGMT_URL || 'http://staging-dataLayer-rabbitmq:15672';
|
||||
const rmqUser = process.env.RABBITMQ_MGMT_USER || process.env.RABBITMQ_USER || 'admin';
|
||||
const rmqPass = process.env.RABBITMQ_MGMT_PASS || process.env.RABBITMQ_PASS || 'rabbitmq123';
|
||||
try {
|
||||
const t = Date.now();
|
||||
const r = await fetch(`${rmqUrl}/api/healthchecks/node`, {
|
||||
signal: AbortSignal.timeout(3000),
|
||||
headers: { Authorization: 'Basic ' + Buffer.from(`${rmqUser}:${rmqPass}`).toString('base64') },
|
||||
});
|
||||
checks.rabbitmq = r.ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy', error: `HTTP ${r.status}` };
|
||||
} catch (e) {
|
||||
checks.rabbitmq = { status: 'unknown', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// Brain (didi_brain) — fail-open dependency
|
||||
const brainUrl = process.env.DIDI_BRAIN_URL || 'http://10.11.10.12:8090';
|
||||
try {
|
||||
const t = Date.now();
|
||||
const r = await fetch(`${brainUrl}/health`, { signal: AbortSignal.timeout(3000) });
|
||||
checks.brain = r.ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy', error: `HTTP ${r.status}` };
|
||||
} catch (e) {
|
||||
checks.brain = { status: 'unknown', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// LLM router
|
||||
const llmUrl = process.env.LLM_ROUTER_URL || 'http://10.11.10.17:14011';
|
||||
try {
|
||||
const t = Date.now();
|
||||
const r = await fetch(`${llmUrl}/health`, { signal: AbortSignal.timeout(5000) });
|
||||
checks.llm = r.ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy', error: `HTTP ${r.status}` };
|
||||
} catch (e) {
|
||||
checks.llm = { status: 'unknown', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// Vision (BusterX video-analysis service) — fail-open
|
||||
try {
|
||||
const t = Date.now();
|
||||
const ok = await busterHealthCheck();
|
||||
checks.buster = ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy' };
|
||||
} catch (e) {
|
||||
checks.buster = { status: 'unknown', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// didiFramework
|
||||
const fwUrl = process.env.DIDI_FRAMEWORK_URL || 'http://didi-framework:3005';
|
||||
try {
|
||||
const t = Date.now();
|
||||
const r = await fetch(`${fwUrl}/health`, { signal: AbortSignal.timeout(3000) });
|
||||
checks.framework = r.ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy', error: `HTTP ${r.status}` };
|
||||
} catch (e) {
|
||||
checks.framework = { status: 'unhealthy', error: (e as Error).message };
|
||||
}
|
||||
|
||||
// ---- Servicii AI Lot 1 (integrare) — fail-open: 'unknown' nu invalideaza verdictul general ----
|
||||
// Corespunde tabelului de integrare din documentatia de arhitectura (§9) si testelor de integrare Lot1<->Lot2.
|
||||
const originOf = (u: string | undefined, fallback: string): string => {
|
||||
try { return new URL(u || fallback).origin; } catch { return (u || fallback).replace(/\/+$/, ''); }
|
||||
};
|
||||
const lot1Services: Array<[string, string]> = [
|
||||
['vision', originOf(process.env.VISION_LLM_URL, 'http://llm-api:14011')], // OCR / analiza imagine (Qwen vision)
|
||||
['whisper', originOf(process.env.M17_WHISPER_URL, 'http://audio-api:54300')], // transcriere audio
|
||||
['web', originOf(process.env.M17_WEB_API_URL, 'http://web-api:51100')], // cautare web pentru claims/surse
|
||||
['video', originOf(process.env.VIDEO_ANALYSIS_URL, 'http://video-api:54600')], // detectie deepfake (BusterX)
|
||||
['extractors', originOf(process.env.EXTRACTORS_URL, 'http://extractors:54400')],// EXIF/ELA/NER/YOLO/OCR
|
||||
['forensic', originOf(process.env.FORENSIC_API_URL, 'http://forensic:8080')], // trasaturi forensice media
|
||||
['domain_check', originOf(process.env.DOMAIN_CHECK_API_URL, 'http://domain-check-api:11000')], // WHOIS/DNS/SSL/blacklist (T4)
|
||||
];
|
||||
await Promise.all(lot1Services.map(async ([name, base]) => {
|
||||
try {
|
||||
const t = Date.now();
|
||||
const r = await fetch(`${base}/health`, { signal: AbortSignal.timeout(4000) });
|
||||
checks[name] = r.ok ? { status: 'healthy', latency_ms: Date.now() - t } : { status: 'unhealthy', error: `HTTP ${r.status}` };
|
||||
} catch (e) {
|
||||
checks[name] = { status: 'unknown', error: (e as Error).message };
|
||||
}
|
||||
}));
|
||||
|
||||
// Aggregate verdict
|
||||
const states = Object.values(checks).map(c => c.status);
|
||||
const unhealthyCount = states.filter(s => s === 'unhealthy').length;
|
||||
const unknownCount = states.filter(s => s === 'unknown').length;
|
||||
const overall =
|
||||
unhealthyCount === 0 && unknownCount === 0 ? 'healthy' :
|
||||
unhealthyCount === 0 ? 'degraded' :
|
||||
'unhealthy';
|
||||
|
||||
res.status(overall === 'unhealthy' ? 503 : 200).json({
|
||||
status: overall,
|
||||
service: 'agent-v3',
|
||||
version: '3.0.0',
|
||||
timestamp: new Date().toISOString(),
|
||||
total_check_ms: Date.now() - start,
|
||||
checks,
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Re-export of the new source-assessment barrel. Kept at this path so src/index.ts
|
||||
* (which imports `./api/source-assessment-routes`) continues to work unchanged after
|
||||
* the 480-LOC → 7-file split. See ./source-assessment/index.ts for the routing map.
|
||||
*/
|
||||
export { default } from './source-assessment';
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Shared lazy singletons + constants for source-assessment routes.
|
||||
*/
|
||||
import { lazyRedis } from '../../shared/redis/connection';
|
||||
import { createLLMClient } from '../../components/pipeline/executor';
|
||||
import type { LLMClient } from '../../components/component-runner';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
|
||||
let llmClient: LLMClient | null = null;
|
||||
|
||||
export const getRedis = lazyRedis('source-assessment-routes');
|
||||
|
||||
export function getLLMClient(): LLMClient {
|
||||
if (!llmClient) {
|
||||
llmClient = createLLMClient(getRedis());
|
||||
}
|
||||
return llmClient;
|
||||
}
|
||||
|
||||
// Route-level timeouts
|
||||
export const TEXT_TIMEOUT_MS = 60_000;
|
||||
export const MEDIA_TIMEOUT_MS = 300_000;
|
||||
|
||||
export const SA_REDIS_PREFIX = ConfigKeys.sourceAssessmentPrefix;
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Vision OCR helper used by /analyze-media when input is an image.
|
||||
* Loads the prompt from Redis (visionPromptExtraction key) with a sane default.
|
||||
*/
|
||||
import { callVision } from '../../shared/media/vision';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import { getRedis } from './_init';
|
||||
|
||||
export async function extractTextFromImage(
|
||||
imageUrl: string,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<string> {
|
||||
const r = getRedis();
|
||||
let userPrompt = 'Extract the main text content from this image. Return ONLY the actual message, post, article text visible in the image. If no text, respond with NO_TEXT_FOUND.';
|
||||
let systemPrompt = '';
|
||||
|
||||
try {
|
||||
const promptData = await r.get(ConfigKeys.visionPromptExtraction);
|
||||
if (promptData) {
|
||||
const parsed = JSON.parse(promptData);
|
||||
if (parsed.system) systemPrompt = parsed.system;
|
||||
if (parsed.user_template) userPrompt = parsed.user_template;
|
||||
}
|
||||
} catch { /* use defaults */ }
|
||||
|
||||
const messages: any[] = [];
|
||||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: userPrompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
});
|
||||
|
||||
const result = await callVision(r, messages, { max_tokens: 1500 }, tier);
|
||||
if (result.content.includes('NO_TEXT_FOUND')) return '';
|
||||
return result.content;
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* POST /analyze-async — RabbitMQ-backed async dispatch (text/media/url).
|
||||
*
|
||||
* Falls back to sync execution if dispatcher reports !async (typically when
|
||||
* RabbitMQ is unreachable — defensive path; queue health is the source of truth).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { SourceAssessmentExecutor } from '../../components/source-assessment/executor';
|
||||
import { checkCredits, deductCredits } from '../../shared/credits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis, getLLMClient } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/analyze-async', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_uid, user_email: body_email, plan_type = 1 } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
const validTypes = ['text', 'image', 'audio', 'video', 'url'];
|
||||
const inputType = media_type || (text ? 'text' : url ? 'url' : null);
|
||||
if (!inputType || !validTypes.includes(inputType)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, inputType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const crypto = await import('crypto');
|
||||
const sessionId = crypto.randomUUID();
|
||||
|
||||
const { dispatch } = await import('../../queue/dispatcher');
|
||||
type PlanType = 1 | 2 | 3 | 4 | 5 | 6;
|
||||
const planTypeNum = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
|
||||
const result = await dispatch(
|
||||
sessionId,
|
||||
{ content: text || '', url, mediaPath: media_url, userId: user_id, userEmail: user_email, inputType },
|
||||
planTypeNum,
|
||||
['domain'],
|
||||
);
|
||||
|
||||
if (!result.async) {
|
||||
const executor = new SourceAssessmentExecutor(getRedis(), getLLMClient());
|
||||
const syncResult = await executor.execute(text || '', url || null, sessionId);
|
||||
return res.json({ success: true, async: false, data: { session_id: sessionId, source_assessment: syncResult } });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, inputType, sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType}`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType} err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(202).json({
|
||||
success: true, async: true,
|
||||
data: {
|
||||
session_id: sessionId, status: 'processing', media_type: inputType,
|
||||
queued_components: ['domain'], plan_type: planTypeNum,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
internalError(res, err, 'source_assessment_analyze_async');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
/**
|
||||
* POST /analyze-media — sync source-assessment over image/audio/video URL.
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Extract text via vision OCR (image), Whisper (audio), or video processor (video).
|
||||
* 2. If extracted text is empty → return skipped:true response (no executor run).
|
||||
* 3. Otherwise run SourceAssessmentExecutor on the extracted text.
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { SourceAssessmentExecutor } from '../../components/source-assessment/executor';
|
||||
import { transcribe } from '../../shared/media/transcription';
|
||||
import { processVideoUrl } from '../../shared/media/video-processor';
|
||||
import { checkCredits, deductCredits } from '../../shared/credits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis, getLLMClient, MEDIA_TIMEOUT_MS } from './_init';
|
||||
import { extractTextFromImage } from './_media-extraction';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/analyze-media', async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
req.setTimeout(MEDIA_TIMEOUT_MS);
|
||||
res.setTimeout(MEDIA_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const { media_url, media_type, user_id: body_uid, user_email: body_email } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!media_url) {
|
||||
return res.status(400).json({ success: false, error: 'media_url is required' });
|
||||
}
|
||||
if (!media_type || !['image', 'audio', 'video'].includes(media_type)) {
|
||||
return res.status(400).json({ success: false, error: 'media_type must be: image, audio, video' });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, media_type);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sessionId = `sa-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
|
||||
log.info(`[SourceAssessment] ${sessionId}: Analyzing ${media_type} from ${media_url.substring(0, 80)}`);
|
||||
|
||||
// Step 1: Extract text from media
|
||||
let extractedText = '';
|
||||
let extractionMeta: Record<string, any> = {};
|
||||
|
||||
if (media_type === 'image') {
|
||||
extractedText = await extractTextFromImage(media_url);
|
||||
extractionMeta = { method: 'vision_ocr' };
|
||||
|
||||
} else if (media_type === 'audio') {
|
||||
const result = await transcribe(media_url, { logPrefix: `${sessionId} SourceAssessment` });
|
||||
if (result.success && result.text) {
|
||||
extractedText = result.text;
|
||||
extractionMeta = { method: 'whisper', provider: result.provider, duration_ms: result.duration_ms };
|
||||
}
|
||||
|
||||
} else if (media_type === 'video') {
|
||||
const result = await processVideoUrl(media_url, sessionId, {
|
||||
redis: getRedis(),
|
||||
logPrefix: 'SourceAssessment',
|
||||
visionContext: 'misinformation',
|
||||
});
|
||||
extractedText = result.merged_text || result.transcript || '';
|
||||
extractionMeta = {
|
||||
method: 'video_processing',
|
||||
has_transcript: !!result.transcript,
|
||||
has_visual_analysis: !!result.visual_analysis,
|
||||
};
|
||||
}
|
||||
|
||||
const extractionDuration = Date.now() - startTime;
|
||||
log.info(`[SourceAssessment] ${sessionId}: Extracted ${extractedText.length} chars in ${extractionDuration}ms`);
|
||||
|
||||
if (!extractedText || extractedText.trim().length === 0) {
|
||||
return res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'completed',
|
||||
input_type: media_type,
|
||||
skipped: true,
|
||||
skip_reason: 'no_text_content',
|
||||
message: `No text content extracted from ${media_type}`,
|
||||
source_assessment: null,
|
||||
media_metadata: { extraction_duration_ms: extractionDuration, ...extractionMeta },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Step 2: Run source assessment on extracted text
|
||||
const executor = new SourceAssessmentExecutor(getRedis(), getLLMClient());
|
||||
const result = await executor.execute(extractedText, null, sessionId);
|
||||
|
||||
const response = {
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
user_id: user_id || null,
|
||||
user_email: user_email || null,
|
||||
input_type: media_type,
|
||||
status: 'completed',
|
||||
source_assessment: result,
|
||||
media_metadata: { extraction_duration_ms: extractionDuration, ...extractionMeta },
|
||||
duration_ms: Date.now() - startTime,
|
||||
},
|
||||
};
|
||||
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, media_type, sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${media_type}`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${media_type} err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
internalError(res, err, 'source_assessment_analyze_media');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* POST /analyze — sync source-assessment over text (+ optional URL).
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { resolveUserId } from '../../shared/auth/guards';
|
||||
import { SourceAssessmentExecutor } from '../../components/source-assessment/executor';
|
||||
import { checkCredits, deductCredits } from '../../shared/credits';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { log } from '../../shared/logger';
|
||||
import { getRedis, getLLMClient, TEXT_TIMEOUT_MS } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.post('/analyze', async (req: Request, res: Response) => {
|
||||
const startTime = Date.now();
|
||||
req.setTimeout(TEXT_TIMEOUT_MS);
|
||||
res.setTimeout(TEXT_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const { text, url, user_id: body_uid, user_email: body_email } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!text && !url) {
|
||||
return res.status(400).json({ success: false, error: 'text or url is required' });
|
||||
}
|
||||
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, 'text');
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const executor = new SourceAssessmentExecutor(getRedis(), getLLMClient());
|
||||
const sessionId = `sa-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
|
||||
const result = await executor.execute(text || '', url || null, sessionId);
|
||||
|
||||
const response = {
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
user_id: user_id || null,
|
||||
user_email: user_email || null,
|
||||
input_type: url && !text ? 'url' : 'text',
|
||||
status: 'completed',
|
||||
source_assessment: result,
|
||||
duration_ms: Date.now() - startTime,
|
||||
},
|
||||
};
|
||||
|
||||
// Deduct credits BEFORE responding so a failure can't be lost behind a 200.
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, 'text', sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=text`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=text err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
internalError(res, err, 'source_assessment_analyze');
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* Source-assessment config + introspection endpoints.
|
||||
*
|
||||
* GET /health
|
||||
* GET /config — scoring config + framework sources from Redis
|
||||
* GET /models — available models for source-assessment
|
||||
* GET /stage-assignments — per-tier stage→model mapping
|
||||
* PUT /stage-assignments — update via zod-validated payload
|
||||
* POST /test-model — connectivity probe for a single model_key
|
||||
*/
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import { internalError } from '../../shared/helpers/error-response';
|
||||
import { TierStageAssignmentsSchema } from '../../shared/helpers/config-schemas';
|
||||
import { getRedis, SA_REDIS_PREFIX } from './_init';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.get('/health', (_req: Request, res: Response) => {
|
||||
res.json({ status: 'ok', component: 'source-assessment', version: 'v1' });
|
||||
});
|
||||
|
||||
router.get('/config', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const [scoringConfig, frameworkSources] = await Promise.all([
|
||||
r.get(ConfigKeys.sourceAssessmentScoringConfig),
|
||||
r.get('didi:framework:sources'),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
scoring_config: scoringConfig ? JSON.parse(scoringConfig) : null,
|
||||
framework_sources: frameworkSources ? JSON.parse(frameworkSources) : null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
internalError(res, error, 'source_assessment_config');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/models', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${SA_REDIS_PREFIX}:available_models`);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'Models not configured in Redis' });
|
||||
res.json({ success: true, data: JSON.parse(data) });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'source_assessment_models');
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/stage-assignments', async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${SA_REDIS_PREFIX}:stage_assignments`);
|
||||
if (!data) return res.status(404).json({ success: false, error: 'Stage assignments not configured in Redis' });
|
||||
res.json({ success: true, data: JSON.parse(data) });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'source_assessment_get_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/stage-assignments', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const parsed = TierStageAssignmentsSchema.safeParse(req.body?.stage_assignments);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid stage_assignments payload',
|
||||
details: parsed.error.issues,
|
||||
});
|
||||
}
|
||||
const r = getRedis();
|
||||
await r.set(`${SA_REDIS_PREFIX}:stage_assignments`, JSON.stringify(parsed.data));
|
||||
res.json({ success: true, message: 'Stage assignments updated' });
|
||||
} catch (error) {
|
||||
internalError(res, error, 'source_assessment_put_stage_assignments');
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/test-model', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { model_key } = req.body;
|
||||
if (!model_key) return res.status(400).json({ success: false, error: 'model_key is required' });
|
||||
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${SA_REDIS_PREFIX}:available_models`);
|
||||
if (!modelsData) return res.status(404).json({ success: false, error: 'Models not configured' });
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === model_key);
|
||||
if (!model) return res.status(404).json({ success: false, error: `Model ${model_key} not found` });
|
||||
|
||||
const startTime = Date.now();
|
||||
const baseUrl = model.provider_config?.base_url || 'https://openrouter.ai/api/v1';
|
||||
const apiKey = process.env[`${model.provider?.toUpperCase()}_API_KEY`] || process.env.OPENROUTER_API_KEY;
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (model.provider_config?.auth_type === 'bearer') headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
else if (model.provider_config?.auth_type === 'x-api-key') headers['x-api-key'] = apiKey || '';
|
||||
if (model.provider === 'openrouter' || baseUrl.includes('openrouter.ai')) {
|
||||
headers['HTTP-Referer'] = 'https://didi.ai';
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ model: model.model_code, messages: [{ role: 'user', content: 'Reply with "OK"' }], max_tokens: 10 }),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
return res.json({ success: true, data: { model_key, status: 'error', error: `API error: ${response.status} - ${errorText.substring(0, 200)}` } });
|
||||
}
|
||||
|
||||
res.json({ success: true, data: { model_key, status: 'connected', response_time_ms: Date.now() - startTime } });
|
||||
} catch (error) {
|
||||
res.json({ success: true, data: { model_key: req.body.model_key, status: 'error', error: (error as Error).message } });
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* AGENT V3 — SOURCE ASSESSMENT API barrel router.
|
||||
*
|
||||
* Original 480-line source-assessment-routes.ts split into:
|
||||
* _init.ts — lazy redis + llmClient + constants
|
||||
* _media-extraction.ts — extractTextFromImage helper (vision OCR with Redis prompt)
|
||||
* config.ts — health, config, models, stage-assignments, test-model
|
||||
* analyze.ts — POST /analyze (sync, text + optional URL)
|
||||
* analyze-async.ts — POST /analyze-async (RabbitMQ dispatch)
|
||||
* analyze-media.ts — POST /analyze-media (image/audio/video → text → executor)
|
||||
*
|
||||
* Mounted at /api/v3/source-assessment in src/index.ts.
|
||||
*/
|
||||
import { Router } from 'express';
|
||||
import configRouter from './config';
|
||||
import analyzeRouter from './analyze';
|
||||
import analyzeAsyncRouter from './analyze-async';
|
||||
import analyzeMediaRouter from './analyze-media';
|
||||
|
||||
const router = Router();
|
||||
|
||||
router.use(configRouter);
|
||||
router.use(analyzeRouter);
|
||||
router.use(analyzeAsyncRouter);
|
||||
router.use(analyzeMediaRouter);
|
||||
|
||||
export default router;
|
||||
|
|
@ -0,0 +1,782 @@
|
|||
/**
|
||||
* AGENT V3 — TECHNIQUES routes (extracted from routes.ts).
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /techniques/definitions
|
||||
* GET /techniques/config
|
||||
* GET /techniques/models
|
||||
* GET /techniques/stage-assignments
|
||||
* PUT /techniques/stage-assignments
|
||||
* POST /techniques/test-model
|
||||
* POST /techniques/test-openrouter
|
||||
* POST /techniques/analyze-async, /analyze, /analyze-media (all → dispatchTechniquesAsync)
|
||||
* GET /techniques/results/:sessionId
|
||||
*/
|
||||
import { Router, Request, Response } from 'express';
|
||||
import { resolveUserId } from '../shared/auth/guards';
|
||||
import crypto from 'crypto';
|
||||
import { validateTextInput, MAX_TEXT_LENGTH } from '../config/analysisLimits';
|
||||
import { callVision } from '../shared/media/vision';
|
||||
import { transcribe, type TranscriptionResult } from '../shared/media/transcription';
|
||||
import { processVideoUrl } from '../shared/media/video-processor';
|
||||
import type { InputType } from '../shared/types/analysis-session';
|
||||
import { ConfigKeys, AgentKeys } from '../shared/redis/keys';
|
||||
import { scanKeys } from '../shared/redis/scan';
|
||||
import { checkCredits, deductCredits } from '../shared/credits';
|
||||
import { toTechniquesResult } from '../components/component-runner';
|
||||
import { buildEmptyMediaResponse } from '../shared/helpers/empty-media-response';
|
||||
import { sanitizeForLog } from '../shared/helpers/sanitize-log';
|
||||
import { validateExternalUrl } from '../shared/helpers/validate-url';
|
||||
import { internalError } from '../shared/helpers/error-response';
|
||||
import { TierStageAssignmentsSchema } from '../shared/helpers/config-schemas';
|
||||
import { log } from '../shared/logger';
|
||||
import { getRedis, upload } from './_init';
|
||||
import { persistStandaloneResult } from './_helpers/standalone-session';
|
||||
|
||||
const router = Router();
|
||||
const REDIS_PREFIX = ConfigKeys.techniquesPrefix;
|
||||
const FRAMEWORK_PREFIX = 'didi:framework';
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/techniques/definitions - Technique definitions for frontend display
|
||||
// Returns: { technique_id, technique_name, dimension, subdimension, description, severity }
|
||||
// Cached in-memory for 5 minutes (data rarely changes).
|
||||
// ============================================================================
|
||||
let definitionsCache: { data: any; ts: number } | null = null;
|
||||
const DEFINITIONS_CACHE_MS = 5 * 60 * 1000;
|
||||
|
||||
router.get('/techniques/definitions', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const now = Date.now();
|
||||
if (definitionsCache && (now - definitionsCache.ts) < DEFINITIONS_CACHE_MS) {
|
||||
return res.json({ success: true, data: definitionsCache.data });
|
||||
}
|
||||
|
||||
const r = getRedis();
|
||||
const raw = await r.get('didi:framework:techniques');
|
||||
if (!raw) {
|
||||
return res.json({ success: true, data: [] });
|
||||
}
|
||||
|
||||
const framework = JSON.parse(raw);
|
||||
const definitions: any[] = [];
|
||||
|
||||
for (const dim of framework.dimensions || []) {
|
||||
for (const sub of dim.subdimensions || []) {
|
||||
for (const tech of sub.techniques || []) {
|
||||
definitions.push({
|
||||
technique_id: tech.technique_id,
|
||||
technique_name: tech.technique_name,
|
||||
dimension: dim.dimension_code,
|
||||
dimension_name: dim.dimension_name,
|
||||
subdimension: sub.subdimension_name,
|
||||
severity: tech.severity,
|
||||
description_en: typeof tech.description === 'object' ? tech.description?.en : tech.description,
|
||||
description_ro: typeof tech.description === 'object' ? tech.description?.ro : tech.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
definitionsCache = { data: definitions, ts: now };
|
||||
res.json({ success: true, data: definitions });
|
||||
} catch (error) {
|
||||
internalError(res, error);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/techniques/config - Get full configuration
|
||||
// ============================================================================
|
||||
router.get('/techniques/config', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
|
||||
const [manifest, availableModels, stageAssignments, prompts, schemas, scoringConfig] = await Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:manifest`),
|
||||
r.get(`${REDIS_PREFIX}:available_models`),
|
||||
r.get(`${REDIS_PREFIX}:stage_assignments`),
|
||||
Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:prompts:screening`),
|
||||
r.get(`${REDIS_PREFIX}:prompts:deep_analysis`),
|
||||
]),
|
||||
Promise.all([
|
||||
r.get(`${REDIS_PREFIX}:schemas:screening`),
|
||||
r.get(`${REDIS_PREFIX}:schemas:complete`),
|
||||
]),
|
||||
r.get(`${REDIS_PREFIX}:scoring_config`),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
manifest: manifest ? JSON.parse(manifest) : null,
|
||||
available_models: availableModels ? JSON.parse(availableModels) : null,
|
||||
stage_assignments: stageAssignments ? JSON.parse(stageAssignments) : null,
|
||||
prompts: {
|
||||
screening: prompts[0] ? JSON.parse(prompts[0]) : null,
|
||||
deep_analysis: prompts[1] ? JSON.parse(prompts[1]) : null,
|
||||
},
|
||||
schemas: {
|
||||
screening: schemas[0] ? JSON.parse(schemas[0]) : null,
|
||||
complete: schemas[1] ? JSON.parse(schemas[1]) : null,
|
||||
},
|
||||
scoring_config: scoringConfig ? JSON.parse(scoringConfig) : null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/techniques/models - Get available models from config
|
||||
// ============================================================================
|
||||
router.get('/techniques/models', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
|
||||
// First try component config, then fall back to didi:framework:providers
|
||||
let data = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
|
||||
if (data) {
|
||||
// Use component config format
|
||||
const { models } = JSON.parse(data);
|
||||
return res.json({
|
||||
success: true,
|
||||
data: { models },
|
||||
});
|
||||
}
|
||||
|
||||
// Fallback to didi:framework:providers
|
||||
data = await r.get(`${FRAMEWORK_PREFIX}:providers`);
|
||||
|
||||
if (!data) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Models not configured. Run sync-redis from didiFramework to load config.',
|
||||
});
|
||||
}
|
||||
|
||||
const { providers, models } = JSON.parse(data);
|
||||
|
||||
// Transform to expected format with model_key
|
||||
const transformedModels = models.map((m: any) => ({
|
||||
model_key: `${m.provider_code}:${m.model_code.split('/').pop()}`,
|
||||
provider: m.provider_code,
|
||||
provider_config: {
|
||||
base_url: providers.find((p: any) => p.provider_id === m.provider_id)?.base_url || '',
|
||||
auth_type: providers.find((p: any) => p.provider_id === m.provider_id)?.auth_type || 'bearer',
|
||||
},
|
||||
model_code: m.model_code,
|
||||
model_name: m.model_name,
|
||||
context_window: m.context_window,
|
||||
max_output_tokens: m.max_output_tokens,
|
||||
cost_input_1m: parseFloat(m.input_cost_per_1m) || 0,
|
||||
cost_output_1m: parseFloat(m.output_cost_per_1m) || 0,
|
||||
speed_tier: m.provider_code === 'groq' ? 'ultra_fast' : 'medium',
|
||||
quality_tier: parseFloat(m.input_cost_per_1m) >= 2 ? 'premium' : 'high',
|
||||
supports_vision: m.supports_vision,
|
||||
}));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
models: transformedModels,
|
||||
providers: providers,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/techniques/stage-assignments - Get stage assignments
|
||||
// ============================================================================
|
||||
router.get('/techniques/stage-assignments', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const data = await r.get(`${REDIS_PREFIX}:stage_assignments`);
|
||||
|
||||
if (!data) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Stage assignments not configured.',
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: JSON.parse(data),
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// PUT /api/v3/techniques/stage-assignments - Update stage assignments
|
||||
// ============================================================================
|
||||
router.put('/techniques/stage-assignments', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const r = getRedis();
|
||||
const parsed = TierStageAssignmentsSchema.safeParse(req.body?.stage_assignments);
|
||||
if (!parsed.success) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid stage_assignments payload',
|
||||
details: parsed.error.issues,
|
||||
});
|
||||
}
|
||||
|
||||
await r.set(`${REDIS_PREFIX}:stage_assignments`, JSON.stringify(parsed.data));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Stage assignments updated',
|
||||
data: parsed.data,
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/techniques/test-model - Test a specific model
|
||||
// ============================================================================
|
||||
router.post('/techniques/test-model', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { model_key, test_prompt, provider_routing } = req.body;
|
||||
|
||||
if (!model_key) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'model_key is required',
|
||||
});
|
||||
}
|
||||
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
|
||||
if (!modelsData) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Models not configured',
|
||||
});
|
||||
}
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === model_key);
|
||||
|
||||
if (!model) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: `Model ${model_key} not found`,
|
||||
});
|
||||
}
|
||||
|
||||
// Test the model with a simple prompt
|
||||
const prompt = test_prompt || 'Respond with JSON: {"status": "ok", "model": "your_model_name"}';
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const response = await callLLM(model, prompt, { provider_routing });
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
model_key,
|
||||
model_name: model.model_name,
|
||||
provider: model.provider,
|
||||
provider_routing: provider_routing || null,
|
||||
response_time_ms: duration,
|
||||
response: response.substring(0, 500), // Limit response size
|
||||
status: 'connected',
|
||||
},
|
||||
});
|
||||
} catch (llmError) {
|
||||
res.json({
|
||||
success: false,
|
||||
data: {
|
||||
model_key,
|
||||
model_name: model.model_name,
|
||||
provider: model.provider,
|
||||
provider_routing: provider_routing || null,
|
||||
status: 'error',
|
||||
error: (llmError as Error).message,
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// POST /api/v3/techniques/test-openrouter - Test OpenRouter with provider routing
|
||||
// ============================================================================
|
||||
router.post('/techniques/test-openrouter', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const {
|
||||
model_code = 'google/gemini-2.0-flash-001',
|
||||
provider_order = ['Google AI Studio'],
|
||||
allow_fallbacks = false,
|
||||
test_prompt
|
||||
} = req.body;
|
||||
|
||||
const apiKey = process.env.OPENROUTER_API_KEY;
|
||||
if (!apiKey) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'OPENROUTER_API_KEY not configured',
|
||||
});
|
||||
}
|
||||
|
||||
const prompt = test_prompt || 'Respond with JSON: {"status": "ok", "provider": "google-ai-studio", "model": "gemini-flash"}';
|
||||
const startTime = Date.now();
|
||||
|
||||
const body = {
|
||||
model: model_code,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: 500,
|
||||
temperature: 0.3,
|
||||
provider: {
|
||||
order: provider_order,
|
||||
allow_fallbacks,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'HTTP-Referer': 'https://didi.ai',
|
||||
'X-Title': 'DIDI Agent V3',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
return res.json({
|
||||
success: false,
|
||||
data: {
|
||||
model_code,
|
||||
provider_order,
|
||||
status: 'error',
|
||||
response_time_ms: duration,
|
||||
error: `${response.status} - ${errorText}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const data = await response.json() as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
model?: string;
|
||||
usage?: { prompt_tokens?: number; completion_tokens?: number };
|
||||
};
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
model_code,
|
||||
model_used: data.model,
|
||||
provider_order,
|
||||
allow_fallbacks,
|
||||
response_time_ms: duration,
|
||||
response: data.choices?.[0]?.message?.content?.substring(0, 500) || '',
|
||||
usage: data.usage,
|
||||
status: 'connected',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Helper: fetch URL content (M17 first, then direct fetch fallback)
|
||||
// ============================================================================
|
||||
async function fetchUrlContent(url: string): Promise<string> {
|
||||
validateExternalUrl(url);
|
||||
const M17_WEB_API = process.env.M17_WEB_API_URL;
|
||||
if (!M17_WEB_API) throw new Error('M17_WEB_API_URL env var is not set');
|
||||
|
||||
try {
|
||||
log.info(`[URL] Fetching via M17: ${sanitizeForLog(url)}`);
|
||||
const response = await fetch(`${M17_WEB_API}/v1/fetch`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ urls: [url], extract_text: true }),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json() as {
|
||||
pages?: { url: string; title?: string; text: string }[];
|
||||
total_fetched?: number;
|
||||
};
|
||||
|
||||
if (data.pages && data.pages.length > 0 && data.pages[0].text) {
|
||||
const page = data.pages[0];
|
||||
const title = page.title ? `Title: ${page.title}\n\n` : '';
|
||||
log.info(`[URL] M17 extracted ${page.text.length} chars`);
|
||||
return title + page.text;
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`[URL] M17 failed, falling back to direct fetch`);
|
||||
} catch (e) {
|
||||
log.warn(`[URL] M17 error: ${e}, falling back to direct fetch`);
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DIDI-Bot/1.0)' },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
return html
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.substring(0, MAX_TEXT_LENGTH);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shared async dispatch handler for all techniques endpoints
|
||||
// All routes (text, media, async) use the same async pattern:
|
||||
// validate → credit check → dispatch to queue → return 202
|
||||
// Media preprocessing (video/audio/image) handled by worker-media-preprocess
|
||||
// ============================================================================
|
||||
async function dispatchTechniquesAsync(req: Request, res: Response) {
|
||||
try {
|
||||
const { text, media_url, url, media_type, user_id: body_uid, user_email: body_email, plan_type = 1 } = req.body;
|
||||
const user_id = resolveUserId(req, body_uid);
|
||||
const user_email = req.jwtEmail || body_email;
|
||||
|
||||
if (!text && !media_url && !url) {
|
||||
return res.status(400).json({ success: false, error: 'One of text, media_url, or url is required' });
|
||||
}
|
||||
|
||||
const validTypes = ['text', 'image', 'audio', 'video', 'url'];
|
||||
const inputType = media_type || (text ? 'text' : url ? 'url' : null);
|
||||
if (!inputType || !validTypes.includes(inputType)) {
|
||||
return res.status(400).json({ success: false, error: `media_type is required. Valid types: ${validTypes.join(', ')}` });
|
||||
}
|
||||
|
||||
// Credit check
|
||||
if (user_id) {
|
||||
const creditCheck = await checkCredits(user_id, inputType);
|
||||
if (creditCheck === null) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Credit service temporarily unavailable. Please try again in a few moments.',
|
||||
error_code: 'CREDIT_SERVICE_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (!creditCheck.hasEnoughCredits) {
|
||||
return res.status(402).json({
|
||||
success: false, error: 'Insufficient credits',
|
||||
data: { creditsRemained: creditCheck.creditsRemained, creditCost: creditCheck.creditCost, planName: creditCheck.planName },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const sessionId = crypto.randomUUID();
|
||||
log.info(`[${sessionId}] Techniques analysis (async), type: ${inputType}`);
|
||||
|
||||
// For text/url: validate and pass content; for media: workers handle extraction
|
||||
let content = text || '';
|
||||
if (inputType === 'url' && (url || media_url) && !text) {
|
||||
try {
|
||||
content = await fetchUrlContent(url || media_url);
|
||||
} catch (e) {
|
||||
log.warn(`[${sessionId}] Failed to fetch URL: ${(e as Error).message}`);
|
||||
content = `URL: ${url || media_url}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Validate text for text/url inputs (media extraction done by workers)
|
||||
if (['text', 'url'].includes(inputType) && content) {
|
||||
const textValidation = validateTextInput(content);
|
||||
if (!textValidation.valid) {
|
||||
return res.status(400).json({
|
||||
success: false, error: textValidation.error,
|
||||
error_code: 'INVALID_TEXT_INPUT',
|
||||
details: { validation_error: textValidation.error_code, stats: textValidation.stats },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const { dispatch } = await import('../queue/dispatcher');
|
||||
type PlanType = 1 | 2 | 3 | 4 | 5 | 6;
|
||||
const planTypeNum = (plan_type >= 1 && plan_type <= 6 ? plan_type : 1) as PlanType;
|
||||
|
||||
const result = await dispatch(
|
||||
sessionId,
|
||||
{ content, url, mediaPath: media_url, userId: user_id, userEmail: user_email, inputType },
|
||||
planTypeNum,
|
||||
['techniques'],
|
||||
);
|
||||
|
||||
if (!result.async) {
|
||||
// Fallback to sync if RabbitMQ unavailable
|
||||
const { ComponentRunner } = await import('../components/component-runner');
|
||||
const r = getRedis();
|
||||
const llmClient = createLLMClient();
|
||||
const runner = new ComponentRunner(r, llmClient);
|
||||
const syncResult = await runner.runTechniques({ text: content || undefined, media_url, media_type: inputType, sessionId });
|
||||
const startTime = Date.now();
|
||||
|
||||
persistStandaloneResult({
|
||||
sessionId, userId: user_id, userEmail: user_email,
|
||||
inputType: inputType as InputType,
|
||||
inputText: inputType === 'text' ? content : undefined,
|
||||
mediaUrl: media_url, result: syncResult, durationMs: Date.now() - startTime,
|
||||
llmUsage: runner.getLastUsageTracker(),
|
||||
});
|
||||
|
||||
return res.json({ success: true, async: false, data: { session_id: sessionId, media_type: inputType, result: syncResult } });
|
||||
}
|
||||
|
||||
// Deduct credits BEFORE responding so a failure can't be lost behind a 202.
|
||||
// We don't refund here even if dispatch already happened — the task is in the
|
||||
// queue and a BILLING_GAP log is the operational signal.
|
||||
if (user_id) {
|
||||
try {
|
||||
const ok = await deductCredits(user_id, inputType, sessionId);
|
||||
if (!ok) log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType}`);
|
||||
} catch (e) {
|
||||
log.error(`[Credits] BILLING_GAP session=${sessionId} user=${user_id} type=${inputType} err=`, (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(202).json({
|
||||
success: true,
|
||||
async: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
status: 'processing',
|
||||
media_type: inputType,
|
||||
queued_components: result.queued,
|
||||
plan_type: planTypeNum,
|
||||
poll_url: `/api/v3/pipeline/${sessionId}/queue-status`,
|
||||
result_url: `/api/v3/pipeline/${sessionId}/result`,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
log.error('[Techniques] Error:', error);
|
||||
internalError(res, error);
|
||||
}
|
||||
}
|
||||
|
||||
// POST /api/v3/techniques/analyze-async
|
||||
router.post('/techniques/analyze-async', dispatchTechniquesAsync);
|
||||
|
||||
// POST /api/v3/techniques/analyze — now async (same handler)
|
||||
router.post('/techniques/analyze', dispatchTechniquesAsync);
|
||||
|
||||
// POST /api/v3/techniques/analyze-media — now async (same handler)
|
||||
router.post('/techniques/analyze-media', dispatchTechniquesAsync);
|
||||
|
||||
|
||||
// ============================================================================
|
||||
// GET /api/v3/techniques/results/:sessionId - Get analysis results
|
||||
// ============================================================================
|
||||
router.get('/techniques/results/:sessionId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
const r = getRedis();
|
||||
|
||||
const keys = await scanKeys(r, AgentKeys.componentPattern(sessionId, 'techniques'));
|
||||
const results: Record<string, any> = {};
|
||||
|
||||
for (const key of keys) {
|
||||
const stage = key.split(':').pop()!;
|
||||
const data = await r.get(key);
|
||||
results[stage] = data ? JSON.parse(data) : null;
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
session_id: sessionId,
|
||||
results,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// LLM Client Implementation
|
||||
// ============================================================================
|
||||
|
||||
interface LLMCallOptions {
|
||||
provider_routing?: {
|
||||
order?: string[];
|
||||
allow_fallbacks?: boolean;
|
||||
};
|
||||
max_tokens?: number;
|
||||
temperature?: number;
|
||||
}
|
||||
|
||||
async function callLLM(model: any, prompt: string, options?: LLMCallOptions): Promise<string> {
|
||||
const { provider, provider_config, model_code } = model;
|
||||
|
||||
// Get API key from environment
|
||||
const apiKeyEnvName = `${provider.toUpperCase()}_API_KEY`;
|
||||
const apiKey = process.env[apiKeyEnvName] || process.env.OPENROUTER_API_KEY;
|
||||
|
||||
if (!apiKey && provider_config.auth_type !== 'none') {
|
||||
throw new Error(`API key not found for provider ${provider}. Set ${apiKeyEnvName}`);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
|
||||
if (provider_config.auth_type === 'bearer') {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
} else if (provider_config.auth_type === 'x-api-key') {
|
||||
headers['x-api-key'] = apiKey!;
|
||||
} else if (provider_config.auth_type === 'api_key') {
|
||||
headers['x-goog-api-key'] = apiKey!;
|
||||
}
|
||||
|
||||
// OpenRouter specific headers
|
||||
if (provider === 'openrouter') {
|
||||
headers['HTTP-Referer'] = 'https://didi.ai';
|
||||
headers['X-Title'] = 'DIDI Agent V3';
|
||||
}
|
||||
|
||||
// Build request body
|
||||
const body: Record<string, any> = {
|
||||
model: model_code,
|
||||
messages: [
|
||||
{ role: 'user', content: prompt }
|
||||
],
|
||||
max_tokens: options?.max_tokens || 500,
|
||||
temperature: options?.temperature || 0.3,
|
||||
};
|
||||
|
||||
// OpenRouter Provider Routing - specify which provider to use
|
||||
// See: https://openrouter.ai/docs/provider-routing
|
||||
if (provider === 'openrouter' && options?.provider_routing) {
|
||||
body.provider = {
|
||||
order: options.provider_routing.order || [],
|
||||
allow_fallbacks: options.provider_routing.allow_fallbacks ?? true,
|
||||
};
|
||||
}
|
||||
|
||||
const response = await fetch(`${provider_config.base_url}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`LLM API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number };
|
||||
};
|
||||
|
||||
// Track usage if caller provided a tracker
|
||||
if ((options as any)?._usage_tracker && Array.isArray((options as any)._usage_tracker) && data.usage) {
|
||||
(options as any)._usage_tracker.push({
|
||||
model: model_code,
|
||||
prompt_tokens: data.usage.prompt_tokens || 0,
|
||||
completion_tokens: data.usage.completion_tokens || 0,
|
||||
total_tokens: data.usage.total_tokens || (data.usage.prompt_tokens || 0) + (data.usage.completion_tokens || 0),
|
||||
});
|
||||
}
|
||||
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
|
||||
function createLLMClient() {
|
||||
return {
|
||||
async call(prompt: string, systemPrompt: string, options: any): Promise<string> {
|
||||
// Get model info from available_models
|
||||
const r = getRedis();
|
||||
const modelsData = await r.get(`${REDIS_PREFIX}:available_models`);
|
||||
|
||||
if (!modelsData) {
|
||||
throw new Error('Models not configured');
|
||||
}
|
||||
|
||||
const { models } = JSON.parse(modelsData);
|
||||
const model = models.find((m: any) => m.model_key === options.model_key);
|
||||
|
||||
if (!model) {
|
||||
throw new Error(`Model ${options.model_key} not found`);
|
||||
}
|
||||
|
||||
const fullPrompt = systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt;
|
||||
|
||||
// Pass provider_routing from model config or options
|
||||
const providerRouting = options.provider_routing || model.provider_routing;
|
||||
const llmOptions: LLMCallOptions & { _usage_tracker?: any[] } = {
|
||||
temperature: options.temperature,
|
||||
max_tokens: options.max_tokens,
|
||||
};
|
||||
|
||||
if (providerRouting && providerRouting.length > 0) {
|
||||
llmOptions.provider_routing = {
|
||||
order: providerRouting,
|
||||
allow_fallbacks: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Pass through usage tracker if present
|
||||
if (options._usage_tracker) {
|
||||
llmOptions._usage_tracker = options._usage_tracker;
|
||||
}
|
||||
|
||||
return callLLM(model, fullPrompt, llmOptions);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
export default router;
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue