livrare lot 2
This commit is contained in:
commit
8ecc78e729
763 changed files with 164593 additions and 0 deletions
|
|
@ -0,0 +1,522 @@
|
|||
/**
|
||||
* Tests for VerdictAggregator (Task 7.3)
|
||||
*
|
||||
* Tests the refactored aggregator that uses:
|
||||
* - VerdictCalculator (pure function)
|
||||
* - VerdictExplanation (LLM, with fallback)
|
||||
* - PersistService (Redis + PG)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Mock Redis before importing aggregator
|
||||
vi.mock('ioredis', () => {
|
||||
const store = new Map<string, string>();
|
||||
const MockRedis = vi.fn(() => ({
|
||||
get: vi.fn((key: string) => Promise.resolve(store.get(key) || null)),
|
||||
set: vi.fn((key: string, value: string, ...args: any[]) => {
|
||||
store.set(key, value);
|
||||
return Promise.resolve('OK');
|
||||
}),
|
||||
setex: vi.fn((key: string, _ttl: number, value: string) => {
|
||||
store.set(key, value);
|
||||
return Promise.resolve('OK');
|
||||
}),
|
||||
del: vi.fn((key: string) => {
|
||||
store.delete(key);
|
||||
return Promise.resolve(1);
|
||||
}),
|
||||
on: vi.fn(),
|
||||
once: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
status: 'ready',
|
||||
_store: store,
|
||||
}));
|
||||
return { default: MockRedis };
|
||||
});
|
||||
|
||||
// Mock connection
|
||||
vi.mock('../connection', () => ({
|
||||
getChannel: vi.fn(() => Promise.resolve({
|
||||
prefetch: vi.fn(),
|
||||
assertQueue: vi.fn(),
|
||||
bindQueue: vi.fn(),
|
||||
consume: vi.fn(),
|
||||
ack: vi.fn(),
|
||||
nack: vi.fn(),
|
||||
close: vi.fn(),
|
||||
on: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
// Mock pg
|
||||
vi.mock('pg', () => ({
|
||||
Pool: vi.fn(() => ({
|
||||
connect: vi.fn(() => Promise.resolve({
|
||||
query: vi.fn(() => Promise.resolve({ rows: [] })),
|
||||
release: vi.fn(),
|
||||
})),
|
||||
end: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
import { VerdictAggregator } from '../aggregator';
|
||||
import { QueueKeys, AgentKeys } from '../../shared/redis/keys';
|
||||
import type { SessionState, AnalysisResultMessage } from '../types';
|
||||
import type { AnalysisComponent } from '../../shared/queue/constants';
|
||||
import {
|
||||
sampleTechniques,
|
||||
sampleAiTampered,
|
||||
sampleClaims,
|
||||
sampleDomain,
|
||||
} from '../../shared/test-utils/fixtures';
|
||||
|
||||
// ============================================================================
|
||||
// HELPERS
|
||||
// ============================================================================
|
||||
|
||||
function makeResultMessage(
|
||||
sessionId: string,
|
||||
component: AnalysisComponent,
|
||||
data: any,
|
||||
): AnalysisResultMessage {
|
||||
return {
|
||||
sessionId,
|
||||
component,
|
||||
success: true,
|
||||
score: 50,
|
||||
data,
|
||||
processingTime: 1000,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function makeSessionState(
|
||||
sessionId: string,
|
||||
components: AnalysisComponent[] = ['techniques', 'ai_tampered', 'claims', 'domain'],
|
||||
): SessionState {
|
||||
return {
|
||||
sessionId,
|
||||
planType: 1 as any,
|
||||
totalComponents: components.length,
|
||||
completedComponents: [] as AnalysisComponent[],
|
||||
results: {} as any,
|
||||
startTime: Date.now() - 5000,
|
||||
status: 'processing',
|
||||
userId: 'usr_test',
|
||||
userEmail: 'test@example.com',
|
||||
inputType: 'text',
|
||||
inputText: 'Test content for analysis',
|
||||
inputUrl: undefined,
|
||||
mediaUrl: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCompletedState(sessionId: string): SessionState {
|
||||
const state = makeSessionState(sessionId);
|
||||
state.completedComponents = ['techniques', 'ai_tampered', 'claims', 'domain'];
|
||||
state.results = {
|
||||
techniques: makeResultMessage(sessionId, 'techniques', sampleTechniques),
|
||||
ai_tampered: makeResultMessage(sessionId, 'ai_tampered', sampleAiTampered),
|
||||
claims: makeResultMessage(sessionId, 'claims', sampleClaims),
|
||||
domain: makeResultMessage(sessionId, 'domain', sampleDomain),
|
||||
};
|
||||
return state;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TESTS
|
||||
// ============================================================================
|
||||
|
||||
describe('VerdictAggregator', () => {
|
||||
let aggregator: VerdictAggregator;
|
||||
let mockLlmClient: any;
|
||||
let mockPersistService: any;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
|
||||
mockLlmClient = {
|
||||
call: vi.fn().mockResolvedValue('RO: Explicatie test\nEN: Test explanation'),
|
||||
};
|
||||
|
||||
mockPersistService = {
|
||||
persist: vi.fn().mockResolvedValue({ redis: true, pg: true }),
|
||||
loadFromCache: vi.fn().mockResolvedValue(null),
|
||||
loadFromDb: vi.fn().mockResolvedValue(null),
|
||||
loadHistory: vi.fn().mockResolvedValue({ items: [], total: 0 }),
|
||||
cleanupRedis: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
aggregator = new VerdictAggregator(10, {
|
||||
llmClient: mockLlmClient,
|
||||
persistService: mockPersistService,
|
||||
});
|
||||
});
|
||||
|
||||
describe('start', () => {
|
||||
it('should start and bind to result queues', async () => {
|
||||
await aggregator.start();
|
||||
// Just verifying it doesn't throw
|
||||
await aggregator.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateVerdict (via handleMessage)', () => {
|
||||
it('should calculate verdict when all components complete', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440000';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
// Setup Redis with session state
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
// Simulate the last component result arriving
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'domain', sampleDomain)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
// Verify PersistService was called
|
||||
expect(mockPersistService.persist).toHaveBeenCalledTimes(1);
|
||||
|
||||
const persistedSession = mockPersistService.persist.mock.calls[0][0];
|
||||
expect(persistedSession.session_id).toBe(sessionId);
|
||||
expect(persistedSession.status).toBe('completed');
|
||||
expect(persistedSession.verdict).not.toBeNull();
|
||||
expect(persistedSession.verdict.risk_score).toBeGreaterThanOrEqual(0);
|
||||
expect(persistedSession.verdict.risk_score).toBeLessThanOrEqual(100);
|
||||
expect(persistedSession.risk_score).toBe(persistedSession.verdict.risk_score);
|
||||
expect(persistedSession.risk_category).toBe(persistedSession.verdict.risk_category);
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
|
||||
it('should include VerdictExplanation when llmClient is provided', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440001';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'domain', sampleDomain)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
expect(mockLlmClient.call).toHaveBeenCalled();
|
||||
|
||||
const persistedSession = mockPersistService.persist.mock.calls[0][0];
|
||||
expect(persistedSession.verdict.explanation_ro).toBe('Explicatie test');
|
||||
expect(persistedSession.verdict.explanation_en).toBe('Test explanation');
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
|
||||
it('should persist without explanation when VerdictExplanation fails', async () => {
|
||||
mockLlmClient.call.mockRejectedValue(new Error('LLM unavailable'));
|
||||
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440002';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'domain', sampleDomain)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
// Should still persist
|
||||
expect(mockPersistService.persist).toHaveBeenCalledTimes(1);
|
||||
|
||||
const persistedSession = mockPersistService.persist.mock.calls[0][0];
|
||||
expect(persistedSession.verdict).not.toBeNull();
|
||||
// Explanation should be null (LLM failed)
|
||||
expect(persistedSession.verdict.explanation_ro).toBeNull();
|
||||
expect(persistedSession.verdict.explanation_en).toBeNull();
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
|
||||
it('should work without llmClient (no explanation)', async () => {
|
||||
// Create aggregator without LLM client
|
||||
const aggregatorNoLlm = new VerdictAggregator(10, {
|
||||
persistService: mockPersistService,
|
||||
});
|
||||
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440003';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
await aggregatorNoLlm.start();
|
||||
const redis = (aggregatorNoLlm as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'domain', sampleDomain)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregatorNoLlm as any).handleMessage(msg);
|
||||
|
||||
expect(mockPersistService.persist).toHaveBeenCalledTimes(1);
|
||||
const persistedSession = mockPersistService.persist.mock.calls[0][0];
|
||||
expect(persistedSession.verdict.explanation_ro).toBeNull();
|
||||
expect(persistedSession.verdict.explanation_en).toBeNull();
|
||||
|
||||
await aggregatorNoLlm.stop();
|
||||
});
|
||||
|
||||
it('should not calculate verdict when not all components done', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440004';
|
||||
const state = makeSessionState(sessionId);
|
||||
// Only 1 component done out of 4
|
||||
state.completedComponents = ['techniques'];
|
||||
state.results = {
|
||||
techniques: makeResultMessage(sessionId, 'techniques', sampleTechniques),
|
||||
} as any;
|
||||
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'ai_tampered', sampleAiTampered)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
// Should NOT have called persist (only 2/4 components)
|
||||
expect(mockPersistService.persist).not.toHaveBeenCalled();
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
|
||||
it('should handle lock conflict by requeueing', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440005';
|
||||
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
const channel = (aggregator as any).channel;
|
||||
|
||||
// Pre-set the lock (simulating another aggregator)
|
||||
const lockKey = QueueKeys.aggregatorLock(sessionId);
|
||||
await redis.set(lockKey, '1', 'EX', 30, 'NX');
|
||||
|
||||
// Now override set to return null (lock already exists)
|
||||
redis.set = vi.fn().mockResolvedValue(null);
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'techniques', sampleTechniques)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
// Should requeue (nack with requeue=true)
|
||||
expect(channel.nack).toHaveBeenCalledWith(msg, false, true);
|
||||
expect(mockPersistService.persist).not.toHaveBeenCalled();
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
|
||||
it('should handle missing session state gracefully', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440006';
|
||||
|
||||
await aggregator.start();
|
||||
const channel = (aggregator as any).channel;
|
||||
|
||||
// No session state in Redis
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'techniques', sampleTechniques)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
// Should ack (skip, no state to process)
|
||||
expect(channel.ack).toHaveBeenCalled();
|
||||
expect(mockPersistService.persist).not.toHaveBeenCalled();
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSession', () => {
|
||||
it('should build AnalysisSession from SessionState', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440010';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
const session = await (aggregator as any).buildSession(sessionId, state);
|
||||
|
||||
expect(session.session_id).toBe(sessionId);
|
||||
expect(session.user_id).toBe('usr_test');
|
||||
expect(session.user_email).toBe('test@example.com');
|
||||
expect(session.input_type).toBe('text');
|
||||
expect(session.status).toBe('completed');
|
||||
expect(session.techniques).toEqual(sampleTechniques);
|
||||
expect(session.ai_tampered).toEqual(sampleAiTampered);
|
||||
expect(session.claims).toEqual(sampleClaims);
|
||||
expect(session.domain).toEqual(sampleDomain);
|
||||
expect(session.verdict).toBeNull(); // Set later by calculateVerdict
|
||||
expect(session.source_app).toBe('web');
|
||||
expect(session.api_version).toBe('v3');
|
||||
});
|
||||
|
||||
it('should handle partial results (some components failed)', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440011';
|
||||
const state = makeSessionState(sessionId, ['techniques', 'claims']);
|
||||
state.completedComponents = ['techniques', 'claims'];
|
||||
state.results = {
|
||||
techniques: makeResultMessage(sessionId, 'techniques', sampleTechniques),
|
||||
claims: { ...makeResultMessage(sessionId, 'claims', null), success: false, error: 'timeout' },
|
||||
} as any;
|
||||
|
||||
const session = await (aggregator as any).buildSession(sessionId, state);
|
||||
|
||||
expect(session.techniques).toEqual(sampleTechniques);
|
||||
expect(session.claims).toBeNull(); // Failed component → null
|
||||
expect(session.ai_tampered).toBeNull();
|
||||
expect(session.domain).toBeNull();
|
||||
});
|
||||
|
||||
it('should handle invalid input type gracefully', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440012';
|
||||
const state = makeCompletedState(sessionId);
|
||||
state.inputType = 'invalid_type'; // Not in valid list
|
||||
|
||||
const session = await (aggregator as any).buildSession(sessionId, state);
|
||||
expect(session.input_type).toBe('text'); // Defaults to text
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeVisualAnalysis', () => {
|
||||
it('should merge visual analysis into ai_tampered result', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440020';
|
||||
const redis = (aggregator as any).getRedis();
|
||||
|
||||
// Set visual analysis in Redis
|
||||
const visual = {
|
||||
visual_analysis: 'AI-generated content detected with deepfake artifacts',
|
||||
frames_analyzed: 10,
|
||||
};
|
||||
await redis.setex(AgentKeys.aiTamperedVisual(sessionId), 3600, JSON.stringify(visual));
|
||||
|
||||
const result = await (aggregator as any).mergeVisualAnalysis(sessionId, { ...sampleAiTampered });
|
||||
|
||||
expect(result.image_analysis).not.toBeNull();
|
||||
expect(result.image_analysis.indicators).toContain('AI-generated content detected in frames');
|
||||
expect(result.image_analysis.indicators).toContain('Deepfake indicators detected');
|
||||
expect(result.image_analysis.frames_analyzed).toBe(10);
|
||||
// Visual indicates AI → combined probability should be higher
|
||||
expect(result.ai_probability).toBeGreaterThan(sampleAiTampered.ai_probability);
|
||||
});
|
||||
|
||||
it('should return unchanged result when no visual analysis exists', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440021';
|
||||
const result = await (aggregator as any).mergeVisualAnalysis(sessionId, { ...sampleAiTampered });
|
||||
expect(result).toEqual(sampleAiTampered);
|
||||
});
|
||||
|
||||
it('should handle visual analysis parse error gracefully', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440022';
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(AgentKeys.aiTamperedVisual(sessionId), 3600, 'invalid json{{{');
|
||||
|
||||
const result = await (aggregator as any).mergeVisualAnalysis(sessionId, { ...sampleAiTampered });
|
||||
// Should return original unchanged
|
||||
expect(result.ai_probability).toBe(sampleAiTampered.ai_probability);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PersistService output matches sync path', () => {
|
||||
it('should produce AnalysisSession with all required fields', async () => {
|
||||
const sessionId = '550e8400-e29b-41d4-a716-446655440030';
|
||||
const state = makeCompletedState(sessionId);
|
||||
|
||||
await aggregator.start();
|
||||
const redis = (aggregator as any).getRedis();
|
||||
await redis.setex(QueueKeys.sessionState(sessionId), 3600, JSON.stringify(state));
|
||||
|
||||
const msg = {
|
||||
content: Buffer.from(JSON.stringify(
|
||||
makeResultMessage(sessionId, 'domain', sampleDomain)
|
||||
)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
|
||||
await (aggregator as any).handleMessage(msg);
|
||||
|
||||
const session = mockPersistService.persist.mock.calls[0][0];
|
||||
|
||||
// Verify all mandatory AnalysisSession fields
|
||||
expect(session.session_id).toBeTruthy();
|
||||
expect(session.input_type).toBeTruthy();
|
||||
expect(session.status).toBe('completed');
|
||||
expect(session.components_run).toBeInstanceOf(Array);
|
||||
expect(session.components_skipped).toBeInstanceOf(Array);
|
||||
expect(session.started_at).toBeTruthy();
|
||||
expect(session.completed_at).toBeTruthy();
|
||||
expect(session.source_app).toBe('web');
|
||||
expect(session.api_version).toBe('v3');
|
||||
|
||||
// Verdict fields
|
||||
expect(session.verdict.risk_score).toBeGreaterThanOrEqual(0);
|
||||
expect(session.verdict.risk_score).toBeLessThanOrEqual(100);
|
||||
expect(session.verdict.risk_category).toBeTruthy();
|
||||
expect(session.verdict.risk_level).toBeTruthy();
|
||||
expect(session.verdict.confidence).toBeGreaterThanOrEqual(0);
|
||||
expect(session.verdict.confidence).toBeLessThanOrEqual(100);
|
||||
expect(session.verdict.components_used).toBeInstanceOf(Array);
|
||||
|
||||
// Denormalized fields on session match verdict
|
||||
expect(session.risk_score).toBe(session.verdict.risk_score);
|
||||
expect(session.risk_category).toBe(session.verdict.risk_category);
|
||||
expect(session.risk_level).toBe(session.verdict.risk_level);
|
||||
expect(session.confidence).toBe(session.verdict.confidence);
|
||||
|
||||
await aggregator.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('stop', () => {
|
||||
it('should clean up channel and redis', async () => {
|
||||
await aggregator.start();
|
||||
await aggregator.stop();
|
||||
// Should not throw on double stop
|
||||
await aggregator.stop();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* Tests for Queue Dispatcher (Task 7.4)
|
||||
*
|
||||
* Tests that the dispatcher:
|
||||
* - Uses QueueKeys from shared (no hardcoded keys)
|
||||
* - Saves initial session via PersistService (replaces saveHistoryEntry)
|
||||
* - Publishes messages to correct queues
|
||||
* - Falls back to sync when RabbitMQ unavailable
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// Mock Redis
|
||||
const redisStore = new Map<string, string>();
|
||||
vi.mock('ioredis', () => {
|
||||
const MockRedis = vi.fn(() => ({
|
||||
get: vi.fn((key: string) => Promise.resolve(redisStore.get(key) || null)),
|
||||
set: vi.fn((key: string, value: string) => {
|
||||
redisStore.set(key, value);
|
||||
return Promise.resolve('OK');
|
||||
}),
|
||||
setex: vi.fn((key: string, _ttl: number, value: string) => {
|
||||
redisStore.set(key, value);
|
||||
return Promise.resolve('OK');
|
||||
}),
|
||||
del: vi.fn((key: string) => {
|
||||
redisStore.delete(key);
|
||||
return Promise.resolve(1);
|
||||
}),
|
||||
zadd: vi.fn(() => Promise.resolve(1)),
|
||||
expire: vi.fn(() => Promise.resolve(1)),
|
||||
on: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
}));
|
||||
return { default: MockRedis };
|
||||
});
|
||||
|
||||
// Mock connection. dispatcher now publishes via publishConfirmed() which
|
||||
// awaits a broker ack — the mock resolves immediately and still records the
|
||||
// underlying publish() call so existing assertions on args keep working.
|
||||
const mockPublish = vi.fn().mockReturnValue(true);
|
||||
vi.mock('../connection', () => ({
|
||||
getConfirmChannel: vi.fn(() => Promise.resolve({
|
||||
publish: mockPublish,
|
||||
})),
|
||||
publishConfirmed: vi.fn((ch: any, ex: string, rk: string, content: Buffer, opts: unknown) => {
|
||||
// Real publishConfirmed rejects on broker nack. The tests drive failure via
|
||||
// publish() returning false, so bridge that to a rejection here.
|
||||
const ok = ch.publish(ex, rk, content, opts);
|
||||
return ok === false ? Promise.reject(new Error('nack')) : Promise.resolve();
|
||||
}),
|
||||
isRabbitMQAvailable: vi.fn(() => Promise.resolve(true)),
|
||||
}));
|
||||
|
||||
// Mock pg
|
||||
vi.mock('pg', () => ({
|
||||
Pool: vi.fn(() => ({
|
||||
connect: vi.fn(() => Promise.resolve({
|
||||
query: vi.fn(() => Promise.resolve({ rows: [] })),
|
||||
release: vi.fn(),
|
||||
})),
|
||||
end: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
import { dispatch, dispatchWithTier, getSessionState, updateSessionState } from '../dispatcher';
|
||||
import type { DispatchInput } from '../dispatcher';
|
||||
import { QueueKeys } from '../../shared/redis/keys';
|
||||
import { isRabbitMQAvailable } from '../connection';
|
||||
|
||||
// ============================================================================
|
||||
// HELPERS
|
||||
// ============================================================================
|
||||
|
||||
const sampleInput: DispatchInput = {
|
||||
content: 'Test content for analysis',
|
||||
url: 'https://example.com/article',
|
||||
userId: 'usr_test',
|
||||
userEmail: 'test@example.com',
|
||||
inputType: 'text',
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// TESTS
|
||||
// ============================================================================
|
||||
|
||||
describe('Dispatcher', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
redisStore.clear();
|
||||
mockPublish.mockReturnValue(true);
|
||||
});
|
||||
|
||||
describe('dispatch', () => {
|
||||
it('should publish tasks to all component queues', async () => {
|
||||
const result = await dispatch('session-1', sampleInput, 1);
|
||||
|
||||
expect(result.async).toBe(true);
|
||||
expect(result.sessionId).toBe('session-1');
|
||||
expect(result.queued).toHaveLength(4);
|
||||
expect(result.queued).toContain('techniques');
|
||||
expect(result.queued).toContain('ai_tampered');
|
||||
expect(result.queued).toContain('claims');
|
||||
expect(result.queued).toContain('domain');
|
||||
expect(mockPublish).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it('should publish to specific components only', async () => {
|
||||
const result = await dispatch('session-2', sampleInput, 1, ['techniques', 'claims']);
|
||||
|
||||
expect(result.async).toBe(true);
|
||||
expect(result.queued).toHaveLength(2);
|
||||
expect(result.queued).toContain('techniques');
|
||||
expect(result.queued).toContain('claims');
|
||||
expect(mockPublish).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should save session state to Redis using QueueKeys', async () => {
|
||||
await dispatch('session-3', sampleInput, 1);
|
||||
|
||||
const key = QueueKeys.sessionState('session-3');
|
||||
const stateJson = redisStore.get(key);
|
||||
expect(stateJson).toBeTruthy();
|
||||
|
||||
const state = JSON.parse(stateJson!);
|
||||
expect(state.sessionId).toBe('session-3');
|
||||
expect(state.status).toBe('processing');
|
||||
expect(state.userId).toBe('usr_test');
|
||||
expect(state.userEmail).toBe('test@example.com');
|
||||
expect(state.totalComponents).toBe(4);
|
||||
});
|
||||
|
||||
it('should use correct priority from plan type', async () => {
|
||||
await dispatch('session-4', sampleInput, 6); // enterprise plan
|
||||
|
||||
// Check that publish was called with priority 10 (enterprise)
|
||||
const publishCall = mockPublish.mock.calls[0];
|
||||
const options = publishCall[3];
|
||||
expect(options.priority).toBe(10);
|
||||
});
|
||||
|
||||
it('should fall back to sync when RabbitMQ unavailable', async () => {
|
||||
vi.mocked(isRabbitMQAvailable).mockResolvedValueOnce(false);
|
||||
|
||||
const result = await dispatch('session-5', sampleInput, 1);
|
||||
|
||||
expect(result.async).toBe(false);
|
||||
expect(result.queued).toHaveLength(0);
|
||||
expect(result.error).toBe('RabbitMQ unavailable');
|
||||
});
|
||||
|
||||
it('should handle partial publish failures', async () => {
|
||||
// First 2 succeed, last 2 fail
|
||||
mockPublish
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false)
|
||||
.mockReturnValueOnce(false);
|
||||
|
||||
const result = await dispatch('session-6', sampleInput, 1);
|
||||
|
||||
expect(result.async).toBe(false); // Not all queued
|
||||
expect(result.queued).toHaveLength(2);
|
||||
expect(result.error).toBe('Some components failed to queue');
|
||||
});
|
||||
|
||||
it('should include correct message headers', async () => {
|
||||
await dispatch('session-7', sampleInput, 3, ['techniques']);
|
||||
|
||||
const publishCall = mockPublish.mock.calls[0];
|
||||
const options = publishCall[3];
|
||||
expect(options.headers.sessionId).toBe('session-7');
|
||||
expect(options.headers.component).toBe('techniques');
|
||||
expect(options.headers.planType).toBe(3);
|
||||
expect(options.persistent).toBe(true);
|
||||
expect(options.contentType).toBe('application/json');
|
||||
});
|
||||
});
|
||||
|
||||
describe('dispatchWithTier', () => {
|
||||
it('should map legacy tier to plan type', async () => {
|
||||
const result = await dispatchWithTier('session-10', sampleInput, 'enterprise');
|
||||
expect(result.async).toBe(true);
|
||||
// enterprise → plan 6 → priority 10
|
||||
const options = mockPublish.mock.calls[0][3];
|
||||
expect(options.priority).toBe(10);
|
||||
});
|
||||
|
||||
it('should map free tier to plan type 1', async () => {
|
||||
const result = await dispatchWithTier('session-11', sampleInput, 'free');
|
||||
expect(result.async).toBe(true);
|
||||
const options = mockPublish.mock.calls[0][3];
|
||||
expect(options.priority).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSessionState', () => {
|
||||
it('should return session state from Redis', async () => {
|
||||
// First dispatch to create state
|
||||
await dispatch('session-20', sampleInput, 1);
|
||||
|
||||
const state = await getSessionState('session-20');
|
||||
expect(state).not.toBeNull();
|
||||
expect(state!.sessionId).toBe('session-20');
|
||||
expect(state!.userId).toBe('usr_test');
|
||||
});
|
||||
|
||||
it('should return null for non-existent session', async () => {
|
||||
const state = await getSessionState('non-existent');
|
||||
expect(state).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateSessionState', () => {
|
||||
it('should update state in Redis', async () => {
|
||||
await dispatch('session-30', sampleInput, 1);
|
||||
|
||||
const state = await getSessionState('session-30');
|
||||
state!.status = 'completed';
|
||||
await updateSessionState(state!);
|
||||
|
||||
const updated = await getSessionState('session-30');
|
||||
expect(updated!.status).toBe('completed');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* Pure helpers that turn the worker-message SessionState into the canonical
|
||||
* AnalysisSession shape consumed by VerdictCalculator + PersistService.
|
||||
*
|
||||
* extractComponentResult unwraps the `{ data: { result: ... } }` shape used
|
||||
* by ComponentWorker (Task 7.2) AND the legacy `{ data: ... }` shape from
|
||||
* BaseWorker — both are still in flight. Returns null on missing/failed.
|
||||
*
|
||||
* buildLLMUsage aggregates per-component token counts into a flat
|
||||
* `{ total, by_component }` summary; returns null if no LLM calls happened.
|
||||
*
|
||||
* buildSession needs `mergeVisualAnalysis` injected because it needs Redis
|
||||
* I/O — keeping that dependency at the call site keeps these helpers pure
|
||||
* and individually testable.
|
||||
*/
|
||||
import { ANALYSIS_COMPONENTS, type AnalysisResultMessage, type SessionState } from '../types';
|
||||
import type { AnalysisSession, InputType, SourceApp } from '../../shared/types/analysis-session';
|
||||
import type {
|
||||
TechniquesResult,
|
||||
AiTamperedResult,
|
||||
ClaimsResult,
|
||||
SourceAssessmentResult,
|
||||
} from '../../shared/types/component-results';
|
||||
|
||||
/**
|
||||
* Extract typed component result from worker message.
|
||||
* Workers wrap results in { data: { result: ... } } or { data: ... }.
|
||||
*/
|
||||
export function extractComponentResult<T>(message: AnalysisResultMessage | undefined): T | null {
|
||||
if (!message || !message.success || !message.data) return null;
|
||||
// ComponentWorker (Task 7.2) sends result directly in data;
|
||||
// legacy BaseWorker wraps in { result: ... }. Probe with bracket-typed
|
||||
// access — both shapes are accepted.
|
||||
const data = message.data as { result?: T } & T;
|
||||
return data?.result ?? (data as T | null) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build LLM usage summary from worker result messages.
|
||||
* Returns null when no component reported any LLM calls.
|
||||
*/
|
||||
export function buildLLMUsage(state: SessionState): any {
|
||||
const byComponent: Record<string, { calls: number; prompt_tokens: number; completion_tokens: number; total_tokens: number; models_used: string[] }> = {};
|
||||
let totalCalls = 0, totalPrompt = 0, totalCompletion = 0, totalTokens = 0;
|
||||
|
||||
for (const comp of ANALYSIS_COMPONENTS) {
|
||||
const msg = state.results[comp];
|
||||
if (!msg?.llm_usage?.length) continue;
|
||||
const compKey = comp === 'domain' ? 'source_assessment' : comp;
|
||||
const models = [...new Set(msg.llm_usage.map((e: any) => e.model))];
|
||||
const summary = { calls: msg.llm_usage.length, prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, models_used: models };
|
||||
for (const e of msg.llm_usage) {
|
||||
summary.prompt_tokens += e.prompt_tokens;
|
||||
summary.completion_tokens += e.completion_tokens;
|
||||
summary.total_tokens += e.total_tokens;
|
||||
}
|
||||
byComponent[compKey] = summary;
|
||||
totalCalls += summary.calls;
|
||||
totalPrompt += summary.prompt_tokens;
|
||||
totalCompletion += summary.completion_tokens;
|
||||
totalTokens += summary.total_tokens;
|
||||
}
|
||||
|
||||
if (totalCalls === 0) return null;
|
||||
return {
|
||||
total: { calls: totalCalls, prompt_tokens: totalPrompt, completion_tokens: totalCompletion, total_tokens: totalTokens },
|
||||
by_component: byComponent,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a full AnalysisSession from SessionState + component results.
|
||||
*
|
||||
* `mergeVisualAnalysis` is passed in (rather than imported) so this helper
|
||||
* stays free of Redis dependency. Caller (aggregator class) provides the
|
||||
* visual-merge function bound to its own Redis client.
|
||||
*/
|
||||
export async function buildSession(
|
||||
sessionId: string,
|
||||
state: SessionState,
|
||||
mergeVisualAnalysis: (sessionId: string, ai: AiTamperedResult) => Promise<AiTamperedResult>,
|
||||
): Promise<AnalysisSession> {
|
||||
const now = new Date().toISOString();
|
||||
const processingTime = Date.now() - state.startTime;
|
||||
|
||||
const techniques = extractComponentResult<TechniquesResult>(state.results.techniques);
|
||||
let aiTampered = extractComponentResult<AiTamperedResult>(state.results.ai_tampered);
|
||||
const claims = extractComponentResult<ClaimsResult>(state.results.claims);
|
||||
// Domain worker now returns SourceAssessmentResult (not DomainResult)
|
||||
const sourceAssessment = extractComponentResult<SourceAssessmentResult>(state.results.domain);
|
||||
|
||||
// For video: merge visual analysis into ai_tampered result
|
||||
if (aiTampered) {
|
||||
aiTampered = await mergeVisualAnalysis(sessionId, aiTampered);
|
||||
}
|
||||
|
||||
// completedComponents = received result (success or failure) — they RAN
|
||||
// skipped = never dispatched / never received — they did NOT run
|
||||
const componentsRun = state.completedComponents;
|
||||
const componentsSkipped = ANALYSIS_COMPONENTS.filter(
|
||||
c => !state.completedComponents.includes(c)
|
||||
);
|
||||
|
||||
const validInputTypes = ['text', 'url', 'image', 'audio', 'video'];
|
||||
const inputType = (validInputTypes.includes(state.inputType || '') ? state.inputType : 'text') as InputType;
|
||||
|
||||
return {
|
||||
session_id: sessionId,
|
||||
user_id: state.userId || null,
|
||||
user_email: state.userEmail || null,
|
||||
input_type: inputType,
|
||||
input_text: state.inputText || null,
|
||||
input_url: state.inputUrl || null,
|
||||
input_media_url: state.mediaUrl || null,
|
||||
input_hash: null,
|
||||
status: 'completed',
|
||||
components_run: componentsRun,
|
||||
components_skipped: componentsSkipped,
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: new Date(state.startTime).toISOString(),
|
||||
completed_at: now,
|
||||
total_duration_ms: processingTime,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'web' as SourceApp,
|
||||
api_version: 'v3',
|
||||
created_at: new Date(state.startTime).toISOString(),
|
||||
|
||||
techniques,
|
||||
ai_tampered: aiTampered,
|
||||
claims,
|
||||
domain: null, // legacy, null on new analyses
|
||||
source_assessment: sourceAssessment,
|
||||
verdict: null, // Set after VerdictCalculator
|
||||
llm_usage: buildLLMUsage(state),
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* PersistService factory for the aggregator.
|
||||
*
|
||||
* Two modes depending on env:
|
||||
* - PG_HOST set: build a real PG pool (max=5) and PgSessionAdapter.
|
||||
* - PG_HOST unset: dummy adapter that throws on use, so the service stays
|
||||
* Redis-only for backward compat with environments that haven't yet
|
||||
* migrated to the cluster.
|
||||
*
|
||||
* `requireEnv`/`optionalEnv` are used dynamically (not top-level imports) so
|
||||
* the path that doesn't need PG can run without throwing at module load.
|
||||
*/
|
||||
import type { Pool } from 'pg';
|
||||
import type Redis from 'ioredis';
|
||||
import { PersistService } from '../../shared/persistence/persist-service';
|
||||
import { SessionStore } from '../../shared/redis/session-store';
|
||||
import { PgSessionAdapter } from '../../shared/persistence/pg-adapter';
|
||||
|
||||
export function createPersistService(redis: Redis): PersistService {
|
||||
const sessionStore = new SessionStore(redis);
|
||||
const pgHost = process.env.PG_HOST;
|
||||
|
||||
if (pgHost) {
|
||||
// Eager-require — only happens when PG_HOST is set, so we don't pay
|
||||
// the cost in PG-less smoke tests.
|
||||
const { Pool: PgPool } = require('pg');
|
||||
const { requireEnv, optionalEnv } = require('../../shared/helpers/env') as typeof import('../../shared/helpers/env');
|
||||
const pool = new PgPool({
|
||||
host: pgHost,
|
||||
port: parseInt(optionalEnv('PG_PORT', '5000'), 10),
|
||||
database: requireEnv('PG_DATABASE'),
|
||||
user: requireEnv('PG_USER'),
|
||||
password: requireEnv('PG_PASSWORD'),
|
||||
max: 5,
|
||||
});
|
||||
const pgAdapter = new PgSessionAdapter(pool);
|
||||
return new PersistService(sessionStore, pgAdapter);
|
||||
}
|
||||
|
||||
// No PG configured — dummy adapter that always fails (Redis-only mode).
|
||||
// `as unknown as Pool` is intentional duck-typing: dummy is never called
|
||||
// in practice, just satisfies the structural interface.
|
||||
const dummyPool = { connect: () => { throw new Error('PG not configured'); } } as unknown as Pool;
|
||||
const pgAdapter = new PgSessionAdapter(dummyPool);
|
||||
return new PersistService(sessionStore, pgAdapter);
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Merge per-frame visual analysis (from media-preprocess worker) into the
|
||||
* ai_tampered component result. Reads from Redis key
|
||||
* `agent:result:{sid}:ai-tampered:visual` written upstream by the worker.
|
||||
*
|
||||
* Combines text + visual probabilities using the canonical 0.4/0.6 weights
|
||||
* (visual weighted higher because frames are direct evidence). Falls back to
|
||||
* the unmodified ai_tampered result if no visual data exists or parse fails.
|
||||
*
|
||||
* Indicator extraction is a small regex pass over the visual-analysis text —
|
||||
* not authoritative, just for surfacing humans-readable cues alongside the
|
||||
* combined probability.
|
||||
*/
|
||||
import type Redis from 'ioredis';
|
||||
import type { AiTamperedResult } from '../../shared/types/component-results';
|
||||
import { AgentKeys } from '../../shared/redis/keys';
|
||||
import { combineVideoProbability, probabilityToVerdict } from '../../shared/media/video-weighting';
|
||||
import { log } from '../../shared/logger';
|
||||
|
||||
export async function mergeVisualAnalysis(
|
||||
redis: Redis,
|
||||
sessionId: string,
|
||||
aiTampered: AiTamperedResult,
|
||||
): Promise<AiTamperedResult> {
|
||||
const visualJson = await redis.get(AgentKeys.aiTamperedVisual(sessionId));
|
||||
if (!visualJson) return aiTampered;
|
||||
|
||||
try {
|
||||
const visual = JSON.parse(visualJson);
|
||||
const indicators: string[] = [];
|
||||
|
||||
const visualText = visual.visual_analysis || '';
|
||||
if (/ai[- ]generated/i.test(visualText)) indicators.push('AI-generated content detected in frames');
|
||||
if (/deepfake/i.test(visualText)) indicators.push('Deepfake indicators detected');
|
||||
if (/watermark/i.test(visualText)) indicators.push('AI watermark detected');
|
||||
if (/artifact/i.test(visualText)) indicators.push('Visual artifacts typical of AI generation');
|
||||
if (/smear|blur|smooth/i.test(visualText)) indicators.push('Unnatural texture/smoothing detected');
|
||||
|
||||
// Combine text + visual using shared helper (canonical 40/60 weights).
|
||||
const textProb = aiTampered.ai_probability || 0;
|
||||
const visualIndicatesAI = /\b(ai[- ]generated|deepfake|synthetic|artificial|sora|runway|midjourney)\b/i.test(visualText);
|
||||
const visualProb = visualIndicatesAI ? 70 : 20;
|
||||
const combinedProb = combineVideoProbability(textProb, visualProb);
|
||||
const verdict: string = probabilityToVerdict(combinedProb);
|
||||
|
||||
// Recalculate risk_score (0-100 scale)
|
||||
const disclosureImpact = aiTampered.disclosure_detected ? 0.7 : 1.0;
|
||||
const riskScore = Math.round((combinedProb / 100) * disclosureImpact * 100);
|
||||
|
||||
log.info(`[VerdictAggregator] Merged visual analysis for ${sessionId}: text=${textProb}% visual=${visualProb}% → combined=${combinedProb}%`);
|
||||
|
||||
return {
|
||||
...aiTampered,
|
||||
ai_probability: combinedProb,
|
||||
verdict,
|
||||
risk_score: riskScore,
|
||||
image_analysis: {
|
||||
indicators,
|
||||
evidence: visualText,
|
||||
model_used: 'qwen-vision',
|
||||
frames_analyzed: visual.frames_analyzed || 0,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
log.warn(`[VerdictAggregator] Failed to merge visual analysis:`, (e as Error).message);
|
||||
return aiTampered;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,532 @@
|
|||
/**
|
||||
* Verdict Aggregator (refactored 2026-05-08, helpers extracted)
|
||||
*
|
||||
* Fan-in worker that collects results from all component workers
|
||||
* and calculates the final verdict when all components complete.
|
||||
*
|
||||
* Uses: VerdictCalculator (pure), VerdictExplanation (LLM), PersistService (Redis+PG).
|
||||
* ZERO manual Redis writes for results. ZERO HTTP sync to didiFramework.
|
||||
*
|
||||
* Pure helpers extracted to ./aggregator-helpers/:
|
||||
* build-session.ts — buildSession + buildLLMUsage + extractComponentResult
|
||||
* visual-analysis.ts — mergeVisualAnalysis (text+frames AI probability merge)
|
||||
* persist-init.ts — createPersistService factory (lazy PG pool)
|
||||
*/
|
||||
|
||||
import { getChannel } from './connection';
|
||||
import {
|
||||
AnalysisResultMessage,
|
||||
ANALYSIS_COMPONENTS,
|
||||
EXCHANGE_NAME,
|
||||
RESULTS_QUEUE,
|
||||
SessionState,
|
||||
} from './types';
|
||||
import { QueueKeys, REDIS_TTL } from '../shared/redis/keys';
|
||||
import { acquireLock, releaseLock } from '../shared/redis/lock';
|
||||
import { WORKER_CONFIG } from '../shared/queue/constants';
|
||||
import { VerdictCalculator } from '../components/pipeline/verdict-calculator';
|
||||
import { calculateVirality } from '../components/pipeline/virality-calculator';
|
||||
import { VerdictExplanation } from '../components/pipeline/verdict-explanation';
|
||||
import type { LLMClient } from '../components/component-runner';
|
||||
import { PersistService } from '../shared/persistence/persist-service';
|
||||
import { shouldEnqueueForReview } from '../components/moderation/triage';
|
||||
import { enqueueForReview } from '../components/moderation/queue-manager';
|
||||
import type { AiTamperedResult } from '../shared/types/component-results';
|
||||
import type Redis from 'ioredis';
|
||||
import { createRedisConnection } from '../shared/redis/connection';
|
||||
import { analysisCompleted, analysisFailed, pipelineDuration } from '../shared/observability/metrics';
|
||||
import { log } from '../shared/logger';
|
||||
|
||||
import { buildSession } from './aggregator-helpers/build-session';
|
||||
import { mergeVisualAnalysis } from './aggregator-helpers/visual-analysis';
|
||||
import { createPersistService } from './aggregator-helpers/persist-init';
|
||||
|
||||
// Message type from amqplib
|
||||
interface ConsumeMessage {
|
||||
content: Buffer;
|
||||
fields: any;
|
||||
properties: any;
|
||||
}
|
||||
|
||||
export class VerdictAggregator {
|
||||
private channel: any = null;
|
||||
private redis: Redis | null = null;
|
||||
private isRunning = false;
|
||||
private prefetch: number;
|
||||
private persistService: PersistService | null = null;
|
||||
private llmClient: LLMClient | null = null;
|
||||
|
||||
private resubscribeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(prefetch = 10, options?: { llmClient?: LLMClient; persistService?: PersistService }) {
|
||||
this.prefetch = prefetch;
|
||||
this.llmClient = options?.llmClient ?? null;
|
||||
this.persistService = options?.persistService ?? null;
|
||||
}
|
||||
|
||||
/** Re-subscribe after connection loss — see ComponentWorker for rationale. */
|
||||
private scheduleResubscribe(): void {
|
||||
if (!this.isRunning || this.resubscribeTimer) return;
|
||||
const RETRY_MS = 5000;
|
||||
const attempt = async (): Promise<void> => {
|
||||
this.resubscribeTimer = null;
|
||||
if (!this.isRunning) return;
|
||||
try {
|
||||
await this.start();
|
||||
log.info('[VerdictAggregator] Re-subscribed after connection loss');
|
||||
} catch (err) {
|
||||
log.warn(`[VerdictAggregator] Re-subscribe failed (${(err as Error).message}), retrying in ${RETRY_MS}ms`);
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
};
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
|
||||
private getRedis(): Redis {
|
||||
if (!this.redis) {
|
||||
this.redis = createRedisConnection({
|
||||
label: 'aggregator',
|
||||
overrides: { lazyConnect: false },
|
||||
});
|
||||
this.redis.on('error', (err) => {
|
||||
log.error('[VerdictAggregator] Redis error:', err.message);
|
||||
});
|
||||
this.redis.on('ready', () => {
|
||||
log.info('[VerdictAggregator] Redis ready');
|
||||
});
|
||||
}
|
||||
return this.redis;
|
||||
}
|
||||
|
||||
private getPersistService(): PersistService {
|
||||
if (!this.persistService) {
|
||||
this.persistService = createPersistService(this.getRedis());
|
||||
}
|
||||
return this.persistService;
|
||||
}
|
||||
|
||||
/** Bound visual-merge — used by buildSession helper that needs Redis I/O. */
|
||||
private mergeVisual = (sessionId: string, aiTampered: AiTamperedResult): Promise<AiTamperedResult> => {
|
||||
return mergeVisualAnalysis(this.getRedis(), sessionId, aiTampered);
|
||||
};
|
||||
|
||||
async start(): Promise<void> {
|
||||
log.info('[VerdictAggregator] Starting...');
|
||||
|
||||
this.channel = await getChannel();
|
||||
if (!this.channel) {
|
||||
throw new Error('Failed to get RabbitMQ channel');
|
||||
}
|
||||
|
||||
this.channel.on('close', () => {
|
||||
if (this.isRunning) {
|
||||
log.warn('[VerdictAggregator] Channel closed — scheduling re-subscribe');
|
||||
this.channel = null;
|
||||
this.scheduleResubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
await this.channel.prefetch(this.prefetch);
|
||||
|
||||
await this.channel.assertQueue(RESULTS_QUEUE, { durable: true });
|
||||
|
||||
// Bind to all component result routing keys
|
||||
for (const component of ANALYSIS_COMPONENTS) {
|
||||
const routingKey = `${RESULTS_QUEUE}.${component}`;
|
||||
await this.channel.bindQueue(RESULTS_QUEUE, EXCHANGE_NAME, routingKey);
|
||||
log.info(`[VerdictAggregator] Bound to ${routingKey}`);
|
||||
}
|
||||
|
||||
await this.channel.consume(RESULTS_QUEUE, (msg: ConsumeMessage | null) => this.handleMessage(msg), {
|
||||
noAck: false,
|
||||
});
|
||||
|
||||
this.isRunning = true;
|
||||
log.info(`[VerdictAggregator] Started, prefetch: ${this.prefetch}`);
|
||||
}
|
||||
|
||||
private async handleMessage(msg: ConsumeMessage | null): Promise<void> {
|
||||
if (!msg || !this.channel) return;
|
||||
|
||||
try {
|
||||
const result = JSON.parse(msg.content.toString()) as AnalysisResultMessage;
|
||||
const { sessionId, component } = result;
|
||||
|
||||
log.info(`[VerdictAggregator] Received ${component} result for ${sessionId}`);
|
||||
|
||||
// Acquire aggregator lock to prevent race conditions.
|
||||
// Uses fenced lock (token-based release) so we never delete a lock that
|
||||
// expired and was re-acquired by another worker mid-calculation.
|
||||
const lockKey = QueueKeys.aggregatorLock(sessionId);
|
||||
const redisClient = this.getRedis();
|
||||
const lock = await acquireLock(redisClient, lockKey, REDIS_TTL.LOCK_AGGREGATOR);
|
||||
|
||||
if (!lock) {
|
||||
log.info(`[VerdictAggregator] Lock exists for ${sessionId}, waiting...`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
this.channel.nack(msg, false, true);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get session state
|
||||
const stateKey = QueueKeys.sessionState(sessionId);
|
||||
const stateData = await redisClient.get(stateKey);
|
||||
|
||||
if (!stateData) {
|
||||
log.error(`[VerdictAggregator] No session state for ${sessionId} — marking failed in PG`);
|
||||
// Session state expired from Redis — mark as failed in PG so frontend stops polling
|
||||
try {
|
||||
const { getPgPool } = await import('../shared/persistence/pg-pool');
|
||||
const pool = getPgPool();
|
||||
await pool.query(
|
||||
"UPDATE bos_analysis.analysis_session SET status = 'failed', completed_at = NOW() WHERE session_id = $1 AND status IN ('pending', 'running')",
|
||||
[sessionId],
|
||||
);
|
||||
log.warn(`[VerdictAggregator] Marked orphaned session ${sessionId} as failed in PG`);
|
||||
} catch (pgErr) {
|
||||
log.error(`[VerdictAggregator] Failed to mark orphaned session in PG:`, (pgErr as Error).message);
|
||||
}
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
const state = JSON.parse(stateData) as SessionState;
|
||||
|
||||
// Cancel check — user canceled: finalize as 'canceled' (idempotent) and
|
||||
// drop further results instead of computing a verdict.
|
||||
if (state.status !== 'canceled' && await redisClient.get(QueueKeys.cancelFlag(sessionId))) {
|
||||
await this.markSessionCanceled(sessionId, state);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
if (state.status === 'canceled') {
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Store result
|
||||
state.results[component] = result;
|
||||
|
||||
if (!state.completedComponents.includes(component)) {
|
||||
state.completedComponents.push(component);
|
||||
}
|
||||
|
||||
await redisClient.setex(stateKey, WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
log.info(
|
||||
`[VerdictAggregator] ${sessionId}: ${state.completedComponents.length}/${state.totalComponents} components`
|
||||
);
|
||||
|
||||
// Check if all components are done (and session not already finalized)
|
||||
if (state.completedComponents.length === state.totalComponents && state.status !== 'completed') {
|
||||
const hasAtLeastOneSuccessfulComponent = state.completedComponents.some(
|
||||
c => state.results[c]?.success === true
|
||||
);
|
||||
|
||||
if (hasAtLeastOneSuccessfulComponent) {
|
||||
await this.calculateVerdict(sessionId, state);
|
||||
} else {
|
||||
await this.markSessionFailed(sessionId, state, 'all_components_failed');
|
||||
}
|
||||
}
|
||||
|
||||
this.channel.ack(msg);
|
||||
} finally {
|
||||
// Compare-and-delete: only release if our token still owns the key.
|
||||
// Logs a warning if lock was already gone (TTL expired during calc).
|
||||
const released = await releaseLock(redisClient, lock);
|
||||
if (!released) {
|
||||
log.warn(
|
||||
`[VerdictAggregator] Lock for ${sessionId} expired before release — calculation may have exceeded ${REDIS_TTL.LOCK_AGGREGATOR}s TTL`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.error('[VerdictAggregator] Error:', (err as Error).message);
|
||||
this.channel.nack(msg, false, false); // Send to DLQ
|
||||
}
|
||||
}
|
||||
|
||||
private async calculateVerdict(sessionId: string, state: SessionState): Promise<void> {
|
||||
log.info(`[VerdictAggregator] Calculating verdict for ${sessionId}`);
|
||||
|
||||
const redisClient = this.getRedis();
|
||||
|
||||
try {
|
||||
// 1. Build AnalysisSession from SessionState (extracts components, merges visual)
|
||||
const session = await buildSession(sessionId, state, this.mergeVisual);
|
||||
|
||||
// 2. Load framework data and calculate verdict (pure function)
|
||||
const framework = await VerdictCalculator.loadFramework(redisClient);
|
||||
const calculator = new VerdictCalculator(framework);
|
||||
const verdict = calculator.calculate(
|
||||
session.techniques,
|
||||
session.ai_tampered,
|
||||
session.claims,
|
||||
session.domain,
|
||||
{ input_type: session.input_type, source_assessment: session.source_assessment },
|
||||
);
|
||||
|
||||
// 2b. Virality risk (pure function, no I/O)
|
||||
const virality = calculateVirality(session.techniques, session.claims);
|
||||
verdict.virality_score = virality.virality_score;
|
||||
verdict.virality_level = virality.virality_level;
|
||||
verdict.virality_factors = virality.factors;
|
||||
|
||||
// 3. LLM verdict review (RO + EN explanations + potential score adjustment)
|
||||
session.verdict = verdict;
|
||||
if (this.llmClient) {
|
||||
try {
|
||||
const tier: 'free' | 'premium' = (state.planType >= 4 && state.planType <= 6) ? 'premium' : 'free';
|
||||
const explainer = new VerdictExplanation(this.llmClient, this.redis || undefined);
|
||||
const explanation = await explainer.generate(session, framework, tier);
|
||||
|
||||
// Explanations always applied
|
||||
verdict.explanation_ro = explanation.explanation_ro;
|
||||
verdict.explanation_en = explanation.explanation_en;
|
||||
|
||||
// v2 (Bug #5): persist the structured verdict_summary in
|
||||
// context_summary (JSONB, zero PG migration). Frontend Bug #6 will
|
||||
// read this for the new TL;DR + key_findings layout. Until then,
|
||||
// the legacy explanation_ro/_en strings (derived from the summary)
|
||||
// continue to render in the old UI.
|
||||
if (explanation.verdict_summary) {
|
||||
verdict.context_summary['verdict_summary'] = explanation.verdict_summary;
|
||||
}
|
||||
|
||||
// If LLM adjusted the verdict, merge over mathematical baseline
|
||||
if (explanation.adjusted && explanation.adjusted_risk_score != null) {
|
||||
const oldScore = verdict.risk_score;
|
||||
verdict.risk_score = explanation.adjusted_risk_score;
|
||||
|
||||
// Re-map category/level/severity from framework using new score
|
||||
const remap = calculator.remapScore(explanation.adjusted_risk_score);
|
||||
verdict.risk_category = remap.risk_category;
|
||||
verdict.risk_category_color = remap.risk_category_color;
|
||||
verdict.risk_level = remap.risk_level;
|
||||
verdict.risk_level_color = remap.risk_level_color;
|
||||
verdict.severity = remap.severity;
|
||||
verdict.recommended_action = remap.recommended_action;
|
||||
|
||||
if (explanation.adjusted_confidence != null) {
|
||||
verdict.confidence = explanation.adjusted_confidence;
|
||||
if (explanation.adjusted_confidence >= 75) verdict.confidence_level = 'HIGH';
|
||||
else if (explanation.adjusted_confidence >= 50) verdict.confidence_level = 'MEDIUM';
|
||||
else verdict.confidence_level = 'LOW';
|
||||
}
|
||||
|
||||
// Store LLM review metadata in context_summary (JSONB, no PG migration).
|
||||
// context_summary is Record<string, unknown> so bracket access types cleanly.
|
||||
verdict.context_summary['llm_review'] = {
|
||||
adjusted: true,
|
||||
original_risk_score: oldScore,
|
||||
adjusted_risk_score: explanation.adjusted_risk_score,
|
||||
reasoning: explanation.reasoning,
|
||||
model_used: explanation.model_used,
|
||||
};
|
||||
|
||||
log.info(`[VerdictAggregator] LLM adjusted verdict for ${sessionId}: ${oldScore} → ${explanation.adjusted_risk_score} (${verdict.risk_category})`);
|
||||
} else {
|
||||
log.info(`[VerdictAggregator] LLM reviewed verdict for ${sessionId} (no adjustment, model: ${explanation.model_used})`);
|
||||
}
|
||||
|
||||
// v2 per-component score autonomy: apply LLM's adjusted_scores to
|
||||
// verdict.score_* fields with audit trail. Each adjustment was
|
||||
// already capped to ±20 from raw by the parser. We track raw vs
|
||||
// adjusted in context_summary['score_adjustments'] so admins can
|
||||
// see WHY a number was overridden.
|
||||
const summary = explanation.verdict_summary;
|
||||
if (summary) {
|
||||
const adjustedScores = summary.adjusted_scores;
|
||||
const auditTrail: Record<string, { raw: number | null; adjusted: number | null; reason: string }> = {};
|
||||
const applyAdj = (
|
||||
key: 'manipulation' | 'claims' | 'ai' | 'source',
|
||||
scoreField: 'score_manipulation' | 'score_claims' | 'score_ai' | 'score_source',
|
||||
) => {
|
||||
const adj = adjustedScores[key];
|
||||
if (!adj) return;
|
||||
const rawValue = verdict[scoreField];
|
||||
if (adj.value !== null && adj.value !== undefined && adj.value !== rawValue) {
|
||||
auditTrail[key] = { raw: rawValue, adjusted: adj.value, reason: adj.reason };
|
||||
verdict[scoreField] = adj.value;
|
||||
log.info(`[VerdictAggregator] LLM adjusted score_${key}: ${rawValue} → ${adj.value} (${adj.reason})`);
|
||||
}
|
||||
};
|
||||
applyAdj('manipulation', 'score_manipulation');
|
||||
applyAdj('claims', 'score_claims');
|
||||
applyAdj('ai', 'score_ai');
|
||||
applyAdj('source', 'score_source');
|
||||
if (Object.keys(auditTrail).length > 0) {
|
||||
verdict.context_summary['score_adjustments'] = auditTrail;
|
||||
}
|
||||
|
||||
// Sanity guard: surface possible LLM blind-spots for HIL review.
|
||||
// If claims have ≥3 verified-false but LLM lowered claims score
|
||||
// dramatically, log a warning so a moderator can spot-check it.
|
||||
const claimsAdj = adjustedScores.claims;
|
||||
const claimsFalse = (session.claims?.verified_false ?? 0);
|
||||
if (claimsAdj?.value !== null && claimsAdj?.value !== undefined
|
||||
&& claimsAdj.raw !== null && claimsAdj.raw !== undefined
|
||||
&& claimsFalse >= 3
|
||||
&& claimsAdj.value < 50
|
||||
&& claimsAdj.value < claimsAdj.raw - 10) {
|
||||
log.warn(`[VerdictAggregator] Suspicious LLM softening: claims_false=${claimsFalse} but score adjusted ${claimsAdj.raw}→${claimsAdj.value}. Reason: "${claimsAdj.reason}"`);
|
||||
verdict.context_summary['llm_review_flag'] = 'claims_softening_suspicious';
|
||||
}
|
||||
}
|
||||
} catch (explainErr) {
|
||||
log.warn(`[VerdictAggregator] VerdictReview failed for ${sessionId}:`, (explainErr as Error).message);
|
||||
verdict.llm_reviewed = false;
|
||||
// confidence is set by VerdictCalculator.calculate() (line 396); no fallback needed.
|
||||
verdict.confidence = Math.max(0, verdict.confidence - 15);
|
||||
if (verdict.confidence >= 75) verdict.confidence_level = 'HIGH';
|
||||
else if (verdict.confidence >= 50) verdict.confidence_level = 'MEDIUM';
|
||||
else verdict.confidence_level = 'LOW';
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Finalize session with verdict and denormalized fields
|
||||
session.verdict = verdict;
|
||||
session.status = 'completed';
|
||||
session.risk_score = verdict.risk_score;
|
||||
session.risk_category = verdict.risk_category;
|
||||
session.risk_level = verdict.risk_level;
|
||||
session.confidence = verdict.confidence;
|
||||
session.confidence_level = verdict.confidence_level;
|
||||
|
||||
// 5. Persist to Redis + PG via PersistService
|
||||
const persistService = this.getPersistService();
|
||||
const { redis: redisOk, pg: pgOk } = await persistService.persist(session);
|
||||
log.info(`[VerdictAggregator] Persisted ${sessionId}: redis=${redisOk}, pg=${pgOk}`);
|
||||
|
||||
// 5.1 Fire-and-forget — emit a summary event to AI platform dashboard.
|
||||
try {
|
||||
const { emitSessionEvent } = await import('../shared/dashboard/event-sink');
|
||||
emitSessionEvent(session);
|
||||
} catch {
|
||||
/* dashboard outages must not affect aggregation */
|
||||
}
|
||||
|
||||
// 5.5 HIL triage — decide if session needs human review (fail-safe, never blocks)
|
||||
try {
|
||||
const triage = await shouldEnqueueForReview({ session, userFlagged: state.userFlagged ?? false });
|
||||
if (triage.needsReview) {
|
||||
const queueId = await enqueueForReview({
|
||||
session_id: session.session_id,
|
||||
priority: triage.priority,
|
||||
enqueue_reason: triage.reason,
|
||||
enqueue_meta: triage.meta,
|
||||
});
|
||||
log.info(`[VerdictAggregator] ${sessionId} enqueued for review (queue_id=${queueId}, reason=${triage.reason}, priority=${triage.priority})`);
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[VerdictAggregator] Triage failed for ${sessionId}:`, (err as Error).message);
|
||||
}
|
||||
|
||||
// 6. Update queue session state
|
||||
state.status = 'completed';
|
||||
await redisClient.setex(QueueKeys.sessionState(sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
const tier = state.planType >= 4 ? 'premium' : 'free';
|
||||
analysisCompleted.inc({
|
||||
component: 'pipeline',
|
||||
tier,
|
||||
media_type: state.inputType || 'text',
|
||||
verdict: verdict.risk_category || 'unknown',
|
||||
});
|
||||
if (session.total_duration_ms) {
|
||||
pipelineDuration.observe({ component: 'pipeline', tier }, session.total_duration_ms / 1000);
|
||||
}
|
||||
|
||||
log.info(
|
||||
`[VerdictAggregator] Verdict for ${sessionId}: ${verdict.risk_category} (${verdict.risk_score}%) in ${session.total_duration_ms}ms`
|
||||
);
|
||||
} catch (err) {
|
||||
log.error(`[VerdictAggregator] Failed to calculate verdict for ${sessionId}:`, (err as Error).message);
|
||||
analysisFailed.inc({
|
||||
component: 'pipeline',
|
||||
tier: state.planType >= 4 ? 'premium' : 'free',
|
||||
media_type: state.inputType || 'text',
|
||||
reason: 'verdict_calculation_failed',
|
||||
});
|
||||
|
||||
state.status = 'failed';
|
||||
await redisClient.setex(QueueKeys.sessionState(sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
// Try to persist a failed session via PersistService
|
||||
try {
|
||||
const failedSession = await buildSession(sessionId, state, this.mergeVisual);
|
||||
failedSession.status = 'failed';
|
||||
const persistService = this.getPersistService();
|
||||
await persistService.persist(failedSession);
|
||||
} catch (persistErr) {
|
||||
log.error(`[VerdictAggregator] Failed to persist error state for ${sessionId}:`, (persistErr as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalize a user-canceled session: state + PG status = 'canceled'. */
|
||||
private async markSessionCanceled(sessionId: string, state: SessionState): Promise<void> {
|
||||
log.info(`[VerdictAggregator] Session ${sessionId} canceled by user — finalizing`);
|
||||
|
||||
const redisClient = this.getRedis();
|
||||
state.status = 'canceled';
|
||||
await redisClient.setex(QueueKeys.sessionState(sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
try {
|
||||
const canceledSession = await buildSession(sessionId, state, this.mergeVisual);
|
||||
canceledSession.status = 'canceled';
|
||||
canceledSession.completed_at = new Date().toISOString();
|
||||
await this.getPersistService().persist(canceledSession);
|
||||
} catch (persistErr) {
|
||||
log.error(`[VerdictAggregator] Failed to persist canceled session ${sessionId}:`, (persistErr as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
private async markSessionFailed(sessionId: string, state: SessionState, reason: string): Promise<void> {
|
||||
log.warn(`[VerdictAggregator] Marking session ${sessionId} as failed: ${reason}`);
|
||||
|
||||
analysisFailed.inc({
|
||||
component: 'pipeline',
|
||||
tier: state.planType >= 4 ? 'premium' : 'free',
|
||||
media_type: state.inputType || 'text',
|
||||
reason,
|
||||
});
|
||||
|
||||
const redisClient = this.getRedis();
|
||||
state.status = 'failed';
|
||||
await redisClient.setex(QueueKeys.sessionState(sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
try {
|
||||
const failedSession = await buildSession(sessionId, state, this.mergeVisual);
|
||||
failedSession.status = 'failed';
|
||||
failedSession.completed_at = new Date().toISOString();
|
||||
|
||||
const persistService = this.getPersistService();
|
||||
const { redis: redisOk, pg: pgOk } = await persistService.persist(failedSession);
|
||||
log.warn(`[VerdictAggregator] Persisted failed session ${sessionId}: redis=${redisOk}, pg=${pgOk}`);
|
||||
} catch (persistErr) {
|
||||
log.error(`[VerdictAggregator] Failed to persist failed session ${sessionId}:`, (persistErr as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
log.info('[VerdictAggregator] Stopping...');
|
||||
this.isRunning = false;
|
||||
|
||||
if (this.channel) {
|
||||
await this.channel.close();
|
||||
this.channel = null;
|
||||
}
|
||||
|
||||
if (this.redis) {
|
||||
this.redis.disconnect();
|
||||
this.redis = null;
|
||||
}
|
||||
|
||||
log.info('[VerdictAggregator] Stopped');
|
||||
}
|
||||
}
|
||||
|
||||
export default VerdictAggregator;
|
||||
|
|
@ -0,0 +1,234 @@
|
|||
/**
|
||||
* RabbitMQ Connection Manager
|
||||
*
|
||||
* Lazy initialization with graceful degradation.
|
||||
* When RabbitMQ is unavailable, async operations fall back to sync.
|
||||
*/
|
||||
|
||||
import amqp from 'amqplib';
|
||||
import { getRabbitMQUrl, getRabbitMQConfig, EXCHANGE_NAME } from '../shared/queue/constants';
|
||||
import { log } from '../shared/logger';
|
||||
|
||||
// Use any to avoid amqplib type issues
|
||||
let connection: any = null;
|
||||
let channel: any = null;
|
||||
let confirmChannel: any = null;
|
||||
let isConnecting = false;
|
||||
let lastConnectionAttempt = 0;
|
||||
// 2s between retries (was 5s). Tuned for HA clusters where HAProxy may need
|
||||
// a few seconds to reroute after failover; shorter backoff = faster recovery.
|
||||
const CONNECTION_RETRY_DELAY = 2000;
|
||||
|
||||
/**
|
||||
* Get or create RabbitMQ connection
|
||||
*/
|
||||
export async function getConnection(): Promise<any> {
|
||||
if (connection) {
|
||||
return connection;
|
||||
}
|
||||
|
||||
// Prevent concurrent connection attempts
|
||||
if (isConnecting) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
return connection;
|
||||
}
|
||||
|
||||
// Rate limit connection attempts
|
||||
const now = Date.now();
|
||||
if (now - lastConnectionAttempt < CONNECTION_RETRY_DELAY) {
|
||||
return null;
|
||||
}
|
||||
|
||||
isConnecting = true;
|
||||
lastConnectionAttempt = now;
|
||||
|
||||
try {
|
||||
const url = getRabbitMQUrl();
|
||||
const cfg = getRabbitMQConfig();
|
||||
const safeTarget = `${cfg.host}:${cfg.port}${cfg.vhost} (user: ${cfg.user})`;
|
||||
log.info(`[RabbitMQ] Connecting to ${safeTarget}...`);
|
||||
|
||||
connection = await amqp.connect(url);
|
||||
|
||||
connection.on('error', (err: Error) => {
|
||||
log.error(`[RabbitMQ] Connection error to ${cfg.host}${cfg.vhost}:`, err.message);
|
||||
connection = null;
|
||||
channel = null;
|
||||
confirmChannel = null;
|
||||
});
|
||||
|
||||
connection.on('close', () => {
|
||||
log.warn(`[RabbitMQ] Connection closed to ${cfg.host}${cfg.vhost}`);
|
||||
connection = null;
|
||||
channel = null;
|
||||
confirmChannel = null;
|
||||
});
|
||||
|
||||
connection.on('blocked', (reason: string) => {
|
||||
log.warn(`[RabbitMQ] Connection blocked by broker: ${reason}`);
|
||||
});
|
||||
connection.on('unblocked', () => {
|
||||
log.info('[RabbitMQ] Connection unblocked');
|
||||
});
|
||||
|
||||
log.info(`[RabbitMQ] ✓ Connected to ${safeTarget}`);
|
||||
return connection;
|
||||
} catch (err) {
|
||||
log.warn('[RabbitMQ] Connection failed:', (err as Error).message);
|
||||
return null;
|
||||
} finally {
|
||||
isConnecting = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create channel for consuming messages
|
||||
*/
|
||||
export async function getChannel(): Promise<any> {
|
||||
if (channel) {
|
||||
return channel;
|
||||
}
|
||||
|
||||
const conn = await getConnection();
|
||||
if (!conn) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
channel = await conn.createChannel();
|
||||
|
||||
// Ensure exchange exists
|
||||
await channel.assertExchange(EXCHANGE_NAME, 'topic', { durable: true });
|
||||
|
||||
channel.on('error', (err: Error) => {
|
||||
log.error('[RabbitMQ] Channel error:', err.message);
|
||||
channel = null;
|
||||
});
|
||||
|
||||
channel.on('close', () => {
|
||||
log.warn('[RabbitMQ] Channel closed');
|
||||
channel = null;
|
||||
});
|
||||
|
||||
return channel;
|
||||
} catch (err) {
|
||||
log.error('[RabbitMQ] Failed to create channel:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create confirm channel for publishing with confirms
|
||||
*/
|
||||
export async function getConfirmChannel(): Promise<any> {
|
||||
if (confirmChannel) {
|
||||
return confirmChannel;
|
||||
}
|
||||
|
||||
const conn = await getConnection();
|
||||
if (!conn) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
confirmChannel = await conn.createConfirmChannel();
|
||||
|
||||
// Ensure exchange exists
|
||||
await confirmChannel.assertExchange(EXCHANGE_NAME, 'topic', { durable: true });
|
||||
|
||||
confirmChannel.on('error', (err: Error) => {
|
||||
log.error('[RabbitMQ] Confirm channel error:', err.message);
|
||||
confirmChannel = null;
|
||||
});
|
||||
|
||||
confirmChannel.on('close', () => {
|
||||
log.warn('[RabbitMQ] Confirm channel closed');
|
||||
confirmChannel = null;
|
||||
});
|
||||
|
||||
return confirmChannel;
|
||||
} catch (err) {
|
||||
log.error('[RabbitMQ] Failed to create confirm channel:', (err as Error).message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish on a confirm channel and WAIT for the broker ack.
|
||||
*
|
||||
* Without this, publishes on a confirm channel are fire-and-forget: the broker
|
||||
* can nack (queue overflow, internal error) or the connection can drop before
|
||||
* delivery, and the message is silently lost while the caller reports success.
|
||||
* Throws on nack/error so callers can fall back (sync path, error result).
|
||||
*/
|
||||
export function publishConfirmed(
|
||||
ch: any,
|
||||
exchange: string,
|
||||
routingKey: string,
|
||||
content: Buffer,
|
||||
options: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
ch.publish(exchange, routingKey, content, options, (err: unknown) => {
|
||||
if (err) reject(err instanceof Error ? err : new Error(String(err)));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if RabbitMQ is available
|
||||
*/
|
||||
export async function isRabbitMQAvailable(): Promise<boolean> {
|
||||
const conn = await getConnection();
|
||||
return conn !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close all connections gracefully
|
||||
*/
|
||||
export async function closeConnection(): Promise<void> {
|
||||
try {
|
||||
if (channel) {
|
||||
await channel.close();
|
||||
channel = null;
|
||||
}
|
||||
if (confirmChannel) {
|
||||
await confirmChannel.close();
|
||||
confirmChannel = null;
|
||||
}
|
||||
if (connection) {
|
||||
await connection.close();
|
||||
connection = null;
|
||||
}
|
||||
log.info('[RabbitMQ] Connections closed');
|
||||
} catch (err) {
|
||||
log.error('[RabbitMQ] Error closing connections:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Health check
|
||||
*/
|
||||
export async function healthCheck(): Promise<{
|
||||
connected: boolean;
|
||||
host: string;
|
||||
error?: string;
|
||||
}> {
|
||||
const url = getRabbitMQUrl();
|
||||
const host = url.replace(/amqp:\/\/[^@]+@/, '').replace(/\/.*$/, '');
|
||||
|
||||
try {
|
||||
const conn = await getConnection();
|
||||
return {
|
||||
connected: conn !== null,
|
||||
host,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
connected: false,
|
||||
host,
|
||||
error: (err as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,473 @@
|
|||
/**
|
||||
* Queue Dispatcher (refactored Task 7.4)
|
||||
*
|
||||
* Publishes analysis tasks to RabbitMQ queues.
|
||||
* Implements graceful degradation - falls back to sync when RabbitMQ unavailable.
|
||||
*
|
||||
* Uses:
|
||||
* - QueueKeys from shared (no hardcoded Redis keys)
|
||||
* - PersistService for initial session save (replaces saveHistoryEntry Redis sorted set)
|
||||
* - Unified AnalysisSession types
|
||||
*/
|
||||
|
||||
import type { Pool } from 'pg';
|
||||
import { getConfirmChannel, isRabbitMQAvailable, publishConfirmed } from './connection';
|
||||
import {
|
||||
type AnalysisComponent,
|
||||
type AnalysisTaskMessage,
|
||||
ANALYSIS_COMPONENTS,
|
||||
EXCHANGE_NAME,
|
||||
getQueueName,
|
||||
type PlanType,
|
||||
PLAN_PRIORITY,
|
||||
LEGACY_TIER_TO_PLAN,
|
||||
type LegacyTier,
|
||||
type SessionState,
|
||||
} from './types';
|
||||
import { QueueKeys } from '../shared/redis/keys';
|
||||
import { WORKER_CONFIG, MEDIA_QUEUE } from '../shared/queue/constants';
|
||||
import { PersistService } from '../shared/persistence/persist-service';
|
||||
import { SessionStore } from '../shared/redis/session-store';
|
||||
import { PgSessionAdapter } from '../shared/persistence/pg-adapter';
|
||||
import type { AnalysisSession, InputType, SourceApp } from '../shared/types/analysis-session';
|
||||
import type Redis from 'ioredis';
|
||||
import { createRedisConnection } from '../shared/redis/connection';
|
||||
import { log } from '../shared/logger';
|
||||
|
||||
// ============================================================================
|
||||
// REDIS SINGLETON
|
||||
// ============================================================================
|
||||
|
||||
let redis: Redis | null = null;
|
||||
|
||||
function getRedis(): Redis {
|
||||
if (!redis) {
|
||||
redis = createRedisConnection({ label: 'dispatcher' });
|
||||
}
|
||||
return redis;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PERSIST SERVICE (lazy init)
|
||||
// ============================================================================
|
||||
|
||||
let persistService: PersistService | null = null;
|
||||
|
||||
function getPersistService(): PersistService {
|
||||
if (!persistService) {
|
||||
const redisClient = getRedis();
|
||||
const sessionStore = new SessionStore(redisClient);
|
||||
|
||||
const pgHost = process.env.PG_HOST;
|
||||
if (pgHost) {
|
||||
const { Pool } = require('pg');
|
||||
// PG_HOST gates this entire branch; once it's set we require the rest.
|
||||
const { requireEnv, optionalEnv } = require('../shared/helpers/env') as typeof import('../shared/helpers/env');
|
||||
const pool = new Pool({
|
||||
host: pgHost,
|
||||
port: parseInt(optionalEnv('PG_PORT', '5000'), 10),
|
||||
database: requireEnv('PG_DATABASE'),
|
||||
user: requireEnv('PG_USER'),
|
||||
password: requireEnv('PG_PASSWORD'),
|
||||
max: 3,
|
||||
});
|
||||
const pgAdapter = new PgSessionAdapter(pool);
|
||||
persistService = new PersistService(sessionStore, pgAdapter);
|
||||
} else {
|
||||
// No PG configured - Redis-only mode (intentional duck-typing).
|
||||
const dummyPool = { connect: () => { throw new Error('PG not configured'); } } as unknown as Pool;
|
||||
const pgAdapter = new PgSessionAdapter(dummyPool);
|
||||
persistService = new PersistService(sessionStore, pgAdapter);
|
||||
}
|
||||
}
|
||||
return persistService;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
export interface DispatchResult {
|
||||
async: boolean;
|
||||
sessionId: string;
|
||||
queued: AnalysisComponent[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface DispatchInput {
|
||||
content: string;
|
||||
url?: string;
|
||||
mediaPath?: string;
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType?: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SAVE INITIAL SESSION
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Save initial pending session via PersistService.
|
||||
* Replaces the old saveHistoryEntry() Redis sorted set approach.
|
||||
* Saves to Redis (for status polling) + PG if configured (for history).
|
||||
*/
|
||||
async function saveInitialSession(
|
||||
sessionId: string,
|
||||
input: DispatchInput,
|
||||
components: AnalysisComponent[],
|
||||
): Promise<void> {
|
||||
if (!input.userId) return;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const validInputTypes = ['text', 'url', 'image', 'audio', 'video'];
|
||||
const inputType = (validInputTypes.includes(input.inputType || '') ? input.inputType : 'text') as InputType;
|
||||
|
||||
const session: AnalysisSession = {
|
||||
session_id: sessionId,
|
||||
user_id: input.userId,
|
||||
user_email: input.userEmail || null,
|
||||
input_type: inputType,
|
||||
input_text: input.content?.substring(0, 2000) || null,
|
||||
input_url: input.url || null,
|
||||
input_media_url: input.mediaPath || null,
|
||||
input_hash: null,
|
||||
status: 'pending',
|
||||
components_run: [],
|
||||
components_skipped: [],
|
||||
risk_score: null,
|
||||
risk_category: null,
|
||||
risk_level: null,
|
||||
confidence: null,
|
||||
confidence_level: null,
|
||||
started_at: now,
|
||||
completed_at: null,
|
||||
total_duration_ms: null,
|
||||
scenario_applied: null,
|
||||
topic_applied: null,
|
||||
source_app: 'web' as SourceApp,
|
||||
api_version: 'v3',
|
||||
created_at: now,
|
||||
|
||||
techniques: null,
|
||||
ai_tampered: null,
|
||||
claims: null,
|
||||
domain: null,
|
||||
source_assessment: null,
|
||||
verdict: null,
|
||||
};
|
||||
|
||||
try {
|
||||
const service = getPersistService();
|
||||
const { redis: redisOk, pg: pgOk } = await service.persist(session);
|
||||
log.info(`[Dispatcher] Saved initial session ${sessionId}: redis=${redisOk}, pg=${pgOk}`);
|
||||
} catch (err) {
|
||||
log.error(`[Dispatcher] Failed to save initial session ${sessionId}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DISPATCH
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Dispatch analysis tasks to queues.
|
||||
*
|
||||
* @param sessionId - Unique session identifier
|
||||
* @param input - Analysis input data
|
||||
* @param planType - Plan type (1-6) from subscription
|
||||
* @param components - Components to analyze (default: all)
|
||||
* @returns DispatchResult with async flag
|
||||
*/
|
||||
export async function dispatch(
|
||||
sessionId: string,
|
||||
input: DispatchInput,
|
||||
planType: PlanType,
|
||||
components: AnalysisComponent[] = ANALYSIS_COMPONENTS
|
||||
): Promise<DispatchResult> {
|
||||
// Check RabbitMQ availability
|
||||
const available = await isRabbitMQAvailable();
|
||||
|
||||
if (!available) {
|
||||
log.warn('[Dispatcher] RabbitMQ unavailable, falling back to sync');
|
||||
return {
|
||||
async: false,
|
||||
sessionId,
|
||||
queued: [],
|
||||
error: 'RabbitMQ unavailable',
|
||||
};
|
||||
}
|
||||
|
||||
const channel = await getConfirmChannel();
|
||||
if (!channel) {
|
||||
return {
|
||||
async: false,
|
||||
sessionId,
|
||||
queued: [],
|
||||
error: 'Failed to get channel',
|
||||
};
|
||||
}
|
||||
|
||||
// Initialize session state in Redis (for aggregator fan-in)
|
||||
const sessionState: SessionState = {
|
||||
sessionId,
|
||||
planType,
|
||||
totalComponents: components.length,
|
||||
completedComponents: [],
|
||||
results: {} as Record<AnalysisComponent, never>,
|
||||
startTime: Date.now(),
|
||||
status: 'pending',
|
||||
userId: input.userId,
|
||||
userEmail: input.userEmail,
|
||||
inputType: input.inputType,
|
||||
inputText: input.content?.substring(0, 500),
|
||||
inputUrl: input.url,
|
||||
mediaUrl: input.mediaPath,
|
||||
};
|
||||
|
||||
const redisClient = getRedis();
|
||||
await redisClient.setex(
|
||||
QueueKeys.sessionState(sessionId),
|
||||
WORKER_CONFIG.SESSION_STATE_TTL,
|
||||
JSON.stringify(sessionState)
|
||||
);
|
||||
|
||||
// Persist the FULL input (session state truncates inputText to 500 chars) so
|
||||
// resume-from-checkpoint can re-dispatch missing components with real content.
|
||||
await redisClient.setex(
|
||||
QueueKeys.resumeInput(sessionId),
|
||||
WORKER_CONFIG.SESSION_STATE_TTL,
|
||||
JSON.stringify({ input, planType, requestedComponents: components }),
|
||||
);
|
||||
|
||||
const priority = PLAN_PRIORITY[planType];
|
||||
const queued: AnalysisComponent[] = [];
|
||||
|
||||
// Source assessment now runs on all input types (text, URL, media)
|
||||
const effectiveComponents = components;
|
||||
sessionState.totalComponents = effectiveComponents.length;
|
||||
|
||||
// Route media (video/audio/image) through pre-processing worker first.
|
||||
// Text and URL go directly to component queues (no media processing needed).
|
||||
const isMedia = ['video', 'audio', 'image'].includes(input.inputType || '');
|
||||
|
||||
if (isMedia) {
|
||||
// ── MEDIA PATH: single task → media_preprocess → caches in Redis → dispatches components ──
|
||||
const message: AnalysisTaskMessage = {
|
||||
sessionId,
|
||||
component: 'techniques' as AnalysisComponent, // placeholder — media worker ignores this
|
||||
planType,
|
||||
priority,
|
||||
input,
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
requestedComponents: effectiveComponents,
|
||||
};
|
||||
|
||||
const mediaQueueName = MEDIA_QUEUE.queueName(planType);
|
||||
|
||||
try {
|
||||
// Await broker confirm — an unconfirmed publish can vanish silently
|
||||
// (queue overflow, connection drop) while we report the session queued.
|
||||
await publishConfirmed(
|
||||
channel,
|
||||
EXCHANGE_NAME,
|
||||
mediaQueueName,
|
||||
Buffer.from(JSON.stringify(message)),
|
||||
{
|
||||
persistent: true,
|
||||
priority,
|
||||
contentType: 'application/json',
|
||||
timestamp: Date.now(),
|
||||
headers: { sessionId, planType, inputType: input.inputType },
|
||||
},
|
||||
);
|
||||
|
||||
// Report all components as queued — media worker will dispatch them
|
||||
queued.push(...effectiveComponents);
|
||||
log.info(`[Dispatcher] Published media_preprocess task for ${sessionId} (${input.inputType}), components: [${effectiveComponents.join(', ')}]`);
|
||||
} catch (err) {
|
||||
log.error(`[Dispatcher] Error publishing media_preprocess:`, (err as Error).message);
|
||||
}
|
||||
} else {
|
||||
// ── TEXT/URL PATH: direct dispatch to component queues (unchanged) ──
|
||||
for (const component of effectiveComponents) {
|
||||
const message: AnalysisTaskMessage = {
|
||||
sessionId,
|
||||
component,
|
||||
planType,
|
||||
priority,
|
||||
input,
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
};
|
||||
|
||||
const queueName = getQueueName(component, planType);
|
||||
|
||||
try {
|
||||
await publishConfirmed(
|
||||
channel,
|
||||
EXCHANGE_NAME,
|
||||
queueName,
|
||||
Buffer.from(JSON.stringify(message)),
|
||||
{
|
||||
persistent: true,
|
||||
priority,
|
||||
contentType: 'application/json',
|
||||
timestamp: Date.now(),
|
||||
headers: {
|
||||
sessionId,
|
||||
component,
|
||||
planType,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
queued.push(component);
|
||||
log.info(`[Dispatcher] Published ${component} task for session ${sessionId}`);
|
||||
} catch (err) {
|
||||
log.error(`[Dispatcher] Error publishing ${component}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update session status and save initial session
|
||||
if (queued.length > 0) {
|
||||
sessionState.status = 'processing';
|
||||
await redisClient.setex(
|
||||
QueueKeys.sessionState(sessionId),
|
||||
WORKER_CONFIG.SESSION_STATE_TTL,
|
||||
JSON.stringify(sessionState)
|
||||
);
|
||||
|
||||
// Save initial pending session (fire and forget)
|
||||
saveInitialSession(sessionId, input, queued).catch(err => {
|
||||
log.error(`[Dispatcher] Failed to save initial session: ${err.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
async: queued.length === effectiveComponents.length,
|
||||
sessionId,
|
||||
queued,
|
||||
error: queued.length < effectiveComponents.length ? 'Some components failed to queue' : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch with legacy tier (for backward compatibility)
|
||||
*/
|
||||
export async function dispatchWithTier(
|
||||
sessionId: string,
|
||||
input: DispatchInput,
|
||||
tier: LegacyTier,
|
||||
components?: AnalysisComponent[]
|
||||
): Promise<DispatchResult> {
|
||||
const planType = LEGACY_TIER_TO_PLAN[tier];
|
||||
return dispatch(sessionId, input, planType, components);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session state from Redis
|
||||
*/
|
||||
export async function getSessionState(sessionId: string): Promise<SessionState | null> {
|
||||
const redisClient = getRedis();
|
||||
const data = await redisClient.get(QueueKeys.sessionState(sessionId));
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(data) as SessionState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update session state in Redis
|
||||
*/
|
||||
export async function updateSessionState(state: SessionState): Promise<void> {
|
||||
const redisClient = getRedis();
|
||||
await redisClient.setex(QueueKeys.sessionState(state.sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
}
|
||||
|
||||
export interface ResumeResult {
|
||||
ok: boolean;
|
||||
sessionId: string;
|
||||
requeued: AnalysisComponent[];
|
||||
alreadyComplete: AnalysisComponent[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume from checkpoint — re-dispatch ONLY the components that never completed.
|
||||
*
|
||||
* Reads the surviving session state + the full input stored at dispatch time,
|
||||
* computes `requested - completed`, and republishes just those tasks straight
|
||||
* to their component queues (media preprocessing output is already cached in
|
||||
* Redis from the first run, so components read it without re-extraction).
|
||||
* Completed components and their results are preserved; the aggregator finalizes
|
||||
* once the fan-in count is reached again.
|
||||
*
|
||||
* Returns ok=false (with a reason) when the session is gone, already finished,
|
||||
* or its input payload has expired (can't safely re-run).
|
||||
*/
|
||||
export async function resumeDispatch(sessionId: string): Promise<ResumeResult> {
|
||||
const redisClient = getRedis();
|
||||
|
||||
const stateRaw = await redisClient.get(QueueKeys.sessionState(sessionId));
|
||||
if (!stateRaw) {
|
||||
return { ok: false, sessionId, requeued: [], alreadyComplete: [], error: 'Session state expired or not found — cannot resume' };
|
||||
}
|
||||
const state = JSON.parse(stateRaw) as SessionState;
|
||||
|
||||
if (state.status === 'completed') {
|
||||
return { ok: false, sessionId, requeued: [], alreadyComplete: state.completedComponents, error: 'Session already completed' };
|
||||
}
|
||||
|
||||
const resumeRaw = await redisClient.get(QueueKeys.resumeInput(sessionId));
|
||||
if (!resumeRaw) {
|
||||
return { ok: false, sessionId, requeued: [], alreadyComplete: state.completedComponents, error: 'Resume payload expired — full input no longer available' };
|
||||
}
|
||||
const { input, planType, requestedComponents } = JSON.parse(resumeRaw) as {
|
||||
input: DispatchInput; planType: PlanType; requestedComponents: AnalysisComponent[];
|
||||
};
|
||||
|
||||
const requested = requestedComponents?.length ? requestedComponents : [...ANALYSIS_COMPONENTS];
|
||||
const done = new Set(state.completedComponents);
|
||||
const missing = requested.filter(c => !done.has(c));
|
||||
|
||||
if (missing.length === 0) {
|
||||
return { ok: true, sessionId, requeued: [], alreadyComplete: [...done] };
|
||||
}
|
||||
|
||||
const available = await isRabbitMQAvailable();
|
||||
const channel = available ? await getConfirmChannel() : null;
|
||||
if (!channel) {
|
||||
return { ok: false, sessionId, requeued: [], alreadyComplete: [...done], error: 'RabbitMQ unavailable' };
|
||||
}
|
||||
|
||||
// Clear any stale cancel flag and mark in-flight again before re-dispatch.
|
||||
await redisClient.del(QueueKeys.cancelFlag(sessionId));
|
||||
state.status = 'processing';
|
||||
await redisClient.setex(QueueKeys.sessionState(sessionId), WORKER_CONFIG.SESSION_STATE_TTL, JSON.stringify(state));
|
||||
|
||||
const priority = PLAN_PRIORITY[planType];
|
||||
const requeued: AnalysisComponent[] = [];
|
||||
|
||||
for (const component of missing) {
|
||||
const message: AnalysisTaskMessage = {
|
||||
sessionId, component, planType, priority, input,
|
||||
timestamp: Date.now(), retryCount: 0,
|
||||
};
|
||||
try {
|
||||
await publishConfirmed(
|
||||
channel, EXCHANGE_NAME, getQueueName(component, planType),
|
||||
Buffer.from(JSON.stringify(message)),
|
||||
{ persistent: true, priority, contentType: 'application/json', timestamp: Date.now(), headers: { sessionId, component, planType, resumed: 'true' } },
|
||||
);
|
||||
requeued.push(component);
|
||||
log.info(`[Dispatcher] Resumed ${component} for session ${sessionId}`);
|
||||
} catch (err) {
|
||||
log.error(`[Dispatcher] Resume publish failed for ${component}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: requeued.length > 0, sessionId, requeued, alreadyComplete: [...done], error: requeued.length === 0 ? 'Failed to re-dispatch any component' : undefined };
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Queue Module Exports
|
||||
*/
|
||||
|
||||
// Types and constants
|
||||
export * from './types';
|
||||
|
||||
// Connection management
|
||||
export {
|
||||
getConnection,
|
||||
getChannel,
|
||||
getConfirmChannel,
|
||||
isRabbitMQAvailable,
|
||||
closeConnection,
|
||||
healthCheck,
|
||||
} from './connection';
|
||||
|
||||
// Dispatcher
|
||||
export { dispatch, dispatchWithTier, getSessionState, updateSessionState } from './dispatcher';
|
||||
export type { DispatchResult, DispatchInput } from './dispatcher';
|
||||
|
||||
// Aggregator
|
||||
export { VerdictAggregator } from './aggregator';
|
||||
|
||||
// Workers (ComponentWorker based)
|
||||
export { ComponentWorker } from './workers/component-worker';
|
||||
export { TechniquesWorker } from './workers/techniques-worker';
|
||||
export { AITamperedWorker } from './workers/ai-tampered-worker';
|
||||
export { ClaimsWorker } from './workers/claims-worker';
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/**
|
||||
* Queue Message Types
|
||||
*
|
||||
* RabbitMQ message schemas for async analysis processing.
|
||||
* Constants and config have been moved to shared/queue/constants.ts (Task 7.1).
|
||||
*/
|
||||
|
||||
// Re-export everything from shared/queue/constants for backward compatibility
|
||||
export {
|
||||
type PlanType,
|
||||
type LegacyTier,
|
||||
type AnalysisComponent,
|
||||
type RabbitMQConfig,
|
||||
PLAN_PRIORITY,
|
||||
PLAN_NAMES,
|
||||
LEGACY_TIER_TO_PLAN,
|
||||
ANALYSIS_COMPONENTS,
|
||||
QUEUE,
|
||||
EXCHANGE_NAME,
|
||||
RESULTS_QUEUE,
|
||||
DLQ_NAME,
|
||||
WORKER_CONFIG,
|
||||
getQueueName,
|
||||
getAllQueuesForComponent,
|
||||
getAllQueues,
|
||||
getRabbitMQConfig,
|
||||
getRabbitMQUrl,
|
||||
} from '../shared/queue/constants';
|
||||
|
||||
// Re-export QueueKeys from shared/redis/keys for backward compat
|
||||
export { QueueKeys as QUEUE_REDIS_KEYS } from '../shared/redis/keys';
|
||||
|
||||
// ============================================================================
|
||||
// MESSAGE TYPES (queue-specific, not shared)
|
||||
// ============================================================================
|
||||
|
||||
import type { AnalysisComponent, PlanType } from '../shared/queue/constants';
|
||||
|
||||
/** Message published to component queues by dispatcher */
|
||||
export interface AnalysisTaskMessage {
|
||||
sessionId: string;
|
||||
component: AnalysisComponent;
|
||||
planType: PlanType;
|
||||
priority: number;
|
||||
input: {
|
||||
content: string;
|
||||
url?: string;
|
||||
mediaPath?: string;
|
||||
inputType?: string;
|
||||
};
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
timestamp: number;
|
||||
retryCount: number;
|
||||
/** Components requested for this session (used by media preprocess to dispatch only what was asked) */
|
||||
requestedComponents?: AnalysisComponent[];
|
||||
}
|
||||
|
||||
/** Result message published by workers to aggregator */
|
||||
export interface AnalysisResultMessage {
|
||||
sessionId: string;
|
||||
component: AnalysisComponent;
|
||||
success: boolean;
|
||||
score?: number;
|
||||
data?: unknown;
|
||||
error?: string;
|
||||
llm_usage?: Array<{ model: string; prompt_tokens: number; completion_tokens: number; total_tokens: number }>;
|
||||
processingTime: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Session state for fan-in aggregation.
|
||||
* @deprecated Use AnalysisSession from shared/types for new code.
|
||||
* Kept for backward compat until Task 7.3 (aggregator refactor).
|
||||
*/
|
||||
export interface SessionState {
|
||||
sessionId: string;
|
||||
planType: PlanType;
|
||||
totalComponents: number;
|
||||
completedComponents: AnalysisComponent[];
|
||||
results: Record<AnalysisComponent, AnalysisResultMessage>;
|
||||
startTime: number;
|
||||
status: 'pending' | 'processing' | 'completed' | 'failed' | 'canceled';
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
inputType?: string;
|
||||
inputText?: string;
|
||||
inputUrl?: string;
|
||||
mediaUrl?: string;
|
||||
userFlagged?: boolean; // HIL: user reported this content for review
|
||||
}
|
||||
|
|
@ -0,0 +1,366 @@
|
|||
/**
|
||||
* Tests for ComponentWorker
|
||||
*
|
||||
* Verifies: direct ComponentRunner invocation (no HTTP), lock mechanism,
|
||||
* result publishing, retry/DLQ, score extraction.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
// ============================================================================
|
||||
// MOCKS - must be declared before vi.mock due to hoisting
|
||||
// ============================================================================
|
||||
|
||||
// Mock Redis
|
||||
vi.mock('ioredis', () => {
|
||||
const mockSet = vi.fn().mockResolvedValue('OK');
|
||||
const mockGet = vi.fn().mockResolvedValue(null); // cancel flag absent by default
|
||||
const Redis = vi.fn().mockImplementation(() => ({
|
||||
set: mockSet,
|
||||
get: mockGet,
|
||||
disconnect: vi.fn(),
|
||||
_mockSet: mockSet,
|
||||
_mockGet: mockGet,
|
||||
}));
|
||||
return { default: Redis, __mockSet: mockSet, __mockGet: mockGet };
|
||||
});
|
||||
|
||||
// Mock RabbitMQ connection
|
||||
vi.mock('../../connection', () => {
|
||||
const channel = {
|
||||
prefetch: vi.fn(),
|
||||
assertQueue: vi.fn(),
|
||||
bindQueue: vi.fn(),
|
||||
consume: vi.fn(),
|
||||
ack: vi.fn(),
|
||||
nack: vi.fn(),
|
||||
close: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
// publishConfirmed invokes the callback arg (err-first) — mock calls it with no error.
|
||||
const confirmChannel = {
|
||||
assertQueue: vi.fn(),
|
||||
publish: vi.fn((_ex: string, _rk: string, _content: Buffer, _opts: unknown, cb?: (err?: unknown) => void) => {
|
||||
if (typeof cb === 'function') cb(undefined);
|
||||
return true;
|
||||
}),
|
||||
};
|
||||
return {
|
||||
getChannel: vi.fn().mockResolvedValue(channel),
|
||||
getConfirmChannel: vi.fn().mockResolvedValue(confirmChannel),
|
||||
publishConfirmed: vi.fn((ch: any, ex: string, rk: string, content: Buffer, opts: unknown) => {
|
||||
ch.publish(ex, rk, content, opts);
|
||||
return Promise.resolve();
|
||||
}),
|
||||
__channel: channel,
|
||||
__confirmChannel: confirmChannel,
|
||||
};
|
||||
});
|
||||
|
||||
import { ComponentWorker } from '../component-worker';
|
||||
import type { AnalysisTaskMessage } from '../../types';
|
||||
|
||||
// Get mock references
|
||||
const connectionMock = await import('../../connection') as any;
|
||||
const mockChannel = connectionMock.__channel;
|
||||
const mockConfirmChannel = connectionMock.__confirmChannel;
|
||||
|
||||
const ioredisMock = await import('ioredis') as any;
|
||||
const mockRedisSet = ioredisMock.__mockSet;
|
||||
|
||||
// ============================================================================
|
||||
// MOCK COMPONENT RUNNER
|
||||
// ============================================================================
|
||||
|
||||
const mockRunTechniques = vi.fn();
|
||||
const mockRunAiTampered = vi.fn();
|
||||
const mockRunClaims = vi.fn();
|
||||
const mockRunSourceAssessment = vi.fn();
|
||||
const mockGetLastUsageTracker = vi.fn().mockReturnValue([]);
|
||||
const mockGetLLMClient = vi.fn().mockReturnValue({ call: vi.fn() });
|
||||
|
||||
const mockComponentRunner = {
|
||||
runTechniques: mockRunTechniques,
|
||||
runAiTampered: mockRunAiTampered,
|
||||
runClaims: mockRunClaims,
|
||||
// 'domain' component is now routed via runSourceAssessment (renamed).
|
||||
runSourceAssessment: mockRunSourceAssessment,
|
||||
getLastUsageTracker: mockGetLastUsageTracker,
|
||||
getLLMClient: mockGetLLMClient,
|
||||
} as any;
|
||||
|
||||
// ============================================================================
|
||||
// HELPERS
|
||||
// ============================================================================
|
||||
|
||||
function makeTaskMessage(overrides: Partial<AnalysisTaskMessage> = {}): AnalysisTaskMessage {
|
||||
return {
|
||||
sessionId: 'test-session-123',
|
||||
component: 'techniques',
|
||||
planType: 1,
|
||||
priority: 1,
|
||||
input: {
|
||||
content: 'Test content for analysis',
|
||||
url: 'https://example.com/article',
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeConsumeMessage(task: AnalysisTaskMessage) {
|
||||
return {
|
||||
content: Buffer.from(JSON.stringify(task)),
|
||||
fields: {},
|
||||
properties: {},
|
||||
};
|
||||
}
|
||||
|
||||
async function triggerMessage(worker: ComponentWorker, task: AnalysisTaskMessage) {
|
||||
await worker.start();
|
||||
|
||||
// Get the last consume callback registered
|
||||
const lastCall = mockChannel.consume.mock.calls[mockChannel.consume.mock.calls.length - 1];
|
||||
const callback = lastCall[1];
|
||||
|
||||
await callback(makeConsumeMessage(task));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// TESTS
|
||||
// ============================================================================
|
||||
|
||||
describe('ComponentWorker', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Default: lock acquired successfully
|
||||
mockRedisSet.mockResolvedValue('OK');
|
||||
});
|
||||
|
||||
describe('start()', () => {
|
||||
it('should subscribe to all 6 plan type queues', async () => {
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner, 5);
|
||||
await worker.start();
|
||||
|
||||
// 6 plan types
|
||||
expect(mockChannel.assertQueue).toHaveBeenCalledTimes(6);
|
||||
expect(mockChannel.bindQueue).toHaveBeenCalledTimes(6);
|
||||
expect(mockChannel.consume).toHaveBeenCalledTimes(6);
|
||||
|
||||
expect(mockChannel.assertQueue).toHaveBeenCalledWith(
|
||||
'analysis.techniques.1',
|
||||
expect.objectContaining({
|
||||
durable: true,
|
||||
arguments: expect.objectContaining({
|
||||
'x-max-priority': 10,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
|
||||
it('should set prefetch', async () => {
|
||||
const worker = new ComponentWorker('claims', mockComponentRunner, 3);
|
||||
await worker.start();
|
||||
expect(mockChannel.prefetch).toHaveBeenCalledWith(3);
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMessage - techniques', () => {
|
||||
it('should call ComponentRunner.runTechniques and publish result', async () => {
|
||||
const techniqueResult = {
|
||||
manipulation_score: 75,
|
||||
total_severity: 150,
|
||||
dimensions_affected: ['D1', 'D3'],
|
||||
techniques_count: 3,
|
||||
};
|
||||
mockRunTechniques.mockResolvedValue(techniqueResult);
|
||||
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
const task = makeTaskMessage({ component: 'techniques' });
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
expect(mockRunTechniques).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
text: 'Test content for analysis',
|
||||
sessionId: 'test-session-123',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockConfirmChannel.publish).toHaveBeenCalledWith(
|
||||
'analysis',
|
||||
'analysis.results.techniques',
|
||||
expect.any(Buffer),
|
||||
expect.objectContaining({ persistent: true }),
|
||||
);
|
||||
|
||||
const publishedBuffer = mockConfirmChannel.publish.mock.calls[0][2];
|
||||
const published = JSON.parse(publishedBuffer.toString());
|
||||
expect(published.success).toBe(true);
|
||||
expect(published.score).toBe(75);
|
||||
expect(published.component).toBe('techniques');
|
||||
|
||||
expect(mockChannel.ack).toHaveBeenCalled();
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMessage - ai_tampered', () => {
|
||||
it('should extract ai_probability as score', async () => {
|
||||
mockRunAiTampered.mockResolvedValue({
|
||||
ai_probability: 85,
|
||||
verdict: 'LIKELY_AI',
|
||||
risk_score: 60,
|
||||
});
|
||||
|
||||
const worker = new ComponentWorker('ai_tampered', mockComponentRunner);
|
||||
const task = makeTaskMessage({ component: 'ai_tampered' });
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
expect(mockRunAiTampered).toHaveBeenCalled();
|
||||
|
||||
const publishedBuffer = mockConfirmChannel.publish.mock.calls[0][2];
|
||||
const published = JSON.parse(publishedBuffer.toString());
|
||||
expect(published.score).toBe(85);
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMessage - claims', () => {
|
||||
it('should invert credibility_score for risk score', async () => {
|
||||
mockRunClaims.mockResolvedValue({
|
||||
credibility_score: 80,
|
||||
total_claims: 5,
|
||||
});
|
||||
|
||||
const worker = new ComponentWorker('claims', mockComponentRunner);
|
||||
const task = makeTaskMessage({ component: 'claims' });
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
const publishedBuffer = mockConfirmChannel.publish.mock.calls[0][2];
|
||||
const published = JSON.parse(publishedBuffer.toString());
|
||||
expect(published.score).toBe(20); // 100 - 80
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleMessage - domain', () => {
|
||||
it('should invert trust_score for risk score', async () => {
|
||||
// 'domain' component is now routed to runSourceAssessment (renamed).
|
||||
// Worker still extracts score from trust_score on the result.
|
||||
mockRunSourceAssessment.mockResolvedValue({
|
||||
domain: 'reuters.com',
|
||||
trust_score: 70,
|
||||
verdict: 'TRUSTED',
|
||||
});
|
||||
|
||||
const worker = new ComponentWorker('domain', mockComponentRunner);
|
||||
const task = makeTaskMessage({ component: 'domain' });
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
const publishedBuffer = mockConfirmChannel.publish.mock.calls[0][2];
|
||||
const published = JSON.parse(publishedBuffer.toString());
|
||||
expect(published.score).toBe(30); // 100 - 70
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('lock mechanism', () => {
|
||||
it('should skip processing when lock already exists', async () => {
|
||||
mockRedisSet.mockResolvedValue(null);
|
||||
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
const task = makeTaskMessage();
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
expect(mockRunTechniques).not.toHaveBeenCalled();
|
||||
expect(mockChannel.ack).toHaveBeenCalled();
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
|
||||
it('should acquire lock with correct key and TTL', async () => {
|
||||
mockRedisSet.mockResolvedValue('OK');
|
||||
mockRunTechniques.mockResolvedValue({ manipulation_score: 50 });
|
||||
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
const task = makeTaskMessage();
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
expect(mockRedisSet).toHaveBeenCalledWith(
|
||||
'didi:queue:lock:test-session-123:techniques',
|
||||
'1',
|
||||
'EX',
|
||||
300,
|
||||
'NX',
|
||||
);
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('should requeue on failure when retryCount < max', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
mockRunTechniques.mockRejectedValue(new Error('LLM timeout'));
|
||||
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
const task = makeTaskMessage({ retryCount: 1 });
|
||||
|
||||
// The worker now ack+republish to the input queue (with retryCount++)
|
||||
// instead of nack-with-requeue, so RabbitMQ persists progress correctly.
|
||||
// Backoff is linear (retry N waits N*1000ms, capped at 5000ms) — advance
|
||||
// timers manually in fake-timers mode.
|
||||
const triggered = triggerMessage(worker, task);
|
||||
await vi.advanceTimersByTimeAsync(2500);
|
||||
await triggered;
|
||||
|
||||
expect(mockChannel.ack).toHaveBeenCalled();
|
||||
|
||||
// Republished to the priority queue with incremented retryCount
|
||||
const republishCall = mockConfirmChannel.publish.mock.calls[0];
|
||||
expect(republishCall[1]).toBe('analysis.techniques.1');
|
||||
const republished = JSON.parse(republishCall[2].toString());
|
||||
expect(republished.retryCount).toBe(2);
|
||||
|
||||
await worker.stop();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('should send to DLQ after max retries', async () => {
|
||||
mockRunTechniques.mockRejectedValue(new Error('Persistent failure'));
|
||||
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
const task = makeTaskMessage({ retryCount: 3 });
|
||||
await triggerMessage(worker, task);
|
||||
|
||||
expect(mockChannel.nack).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
false,
|
||||
false, // no requeue → DLQ
|
||||
);
|
||||
|
||||
await worker.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe('stop()', () => {
|
||||
it('should close channel and disconnect Redis', async () => {
|
||||
const worker = new ComponentWorker('techniques', mockComponentRunner);
|
||||
await worker.start();
|
||||
await worker.stop();
|
||||
|
||||
expect(mockChannel.close).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,369 @@
|
|||
/**
|
||||
* Task 2.4: Verify DomainWorker output conforms to DomainResult schema.
|
||||
*
|
||||
* Tests the mapping from DomainAnalysisResult (executor nested format) →
|
||||
* DomainResult (unified flat format from component-results.ts).
|
||||
* trust_score is already 0-100, main change is structure flattening.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'vitest';
|
||||
import { DomainResultSchema } from '../../../shared/types/component-results';
|
||||
import type { DomainResult } from '../../../shared/types/component-results';
|
||||
|
||||
// ============================================================================
|
||||
// Helper: Convert DomainAnalysisResult (executor output) → DomainResult (unified)
|
||||
// This mapping will be used in ComponentRunner (Task 3.1)
|
||||
// ============================================================================
|
||||
|
||||
interface DomainAnalysisResult {
|
||||
domain: string;
|
||||
verdict: 'TRUSTED' | 'NEUTRAL' | 'SUSPICIOUS' | 'UNTRUSTED' | 'NOT_APPLICABLE';
|
||||
trust_score: number; // 0-100
|
||||
risk_level: string;
|
||||
age: {
|
||||
days: number | null;
|
||||
category: string;
|
||||
created_at: string | null;
|
||||
};
|
||||
blacklist: {
|
||||
is_blacklisted: boolean;
|
||||
reputation_score: number | null;
|
||||
};
|
||||
ssl: {
|
||||
has_ssl: boolean;
|
||||
is_valid: boolean;
|
||||
issuer: string | null;
|
||||
};
|
||||
ownership: {
|
||||
registrar: string | null;
|
||||
organization: string | null;
|
||||
country: string | null;
|
||||
};
|
||||
red_flags: string[];
|
||||
warnings: string[];
|
||||
metadata: {
|
||||
duration_ms: number;
|
||||
};
|
||||
}
|
||||
|
||||
function toDomainResult(exec: DomainAnalysisResult): DomainResult {
|
||||
return {
|
||||
domain: exec.domain,
|
||||
verdict: exec.verdict,
|
||||
trust_score: exec.trust_score,
|
||||
risk_level: exec.risk_level,
|
||||
age_days: exec.age.days,
|
||||
age_category: exec.age.category,
|
||||
domain_created_at: exec.age.created_at,
|
||||
is_blacklisted: exec.blacklist.is_blacklisted,
|
||||
reputation_score: exec.blacklist.reputation_score,
|
||||
has_ssl: exec.ssl.has_ssl,
|
||||
ssl_valid: exec.ssl.is_valid,
|
||||
ssl_issuer: exec.ssl.issuer,
|
||||
registrar: exec.ownership.registrar,
|
||||
organization: exec.ownership.organization,
|
||||
country: exec.ownership.country,
|
||||
red_flags: exec.red_flags,
|
||||
warnings: exec.warnings,
|
||||
duration_ms: exec.metadata.duration_ms,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test fixtures simulating executor output
|
||||
// ============================================================================
|
||||
|
||||
const trustedResult: DomainAnalysisResult = {
|
||||
domain: 'reuters.com',
|
||||
verdict: 'TRUSTED',
|
||||
trust_score: 70,
|
||||
risk_level: 'LOW',
|
||||
age: { days: null, category: 'UNKNOWN', created_at: null },
|
||||
blacklist: { is_blacklisted: false, reputation_score: 80 },
|
||||
ssl: { has_ssl: true, is_valid: true, issuer: null },
|
||||
ownership: { registrar: null, organization: null, country: null },
|
||||
red_flags: [],
|
||||
warnings: [],
|
||||
metadata: { duration_ms: 45 },
|
||||
};
|
||||
|
||||
const suspiciousResult: DomainAnalysisResult = {
|
||||
domain: 'news123.xyz',
|
||||
verdict: 'SUSPICIOUS',
|
||||
trust_score: 30,
|
||||
risk_level: 'HIGH',
|
||||
age: { days: null, category: 'UNKNOWN', created_at: null },
|
||||
blacklist: { is_blacklisted: false, reputation_score: 80 },
|
||||
ssl: { has_ssl: true, is_valid: true, issuer: null },
|
||||
ownership: { registrar: null, organization: null, country: null },
|
||||
red_flags: ['SUSPICIOUS_TLD', 'NUMERIC_DOMAIN'],
|
||||
warnings: ['Source type could not be determined'],
|
||||
metadata: { duration_ms: 38 },
|
||||
};
|
||||
|
||||
const socialResult: DomainAnalysisResult = {
|
||||
domain: 'twitter.com',
|
||||
verdict: 'NEUTRAL',
|
||||
trust_score: 40,
|
||||
risk_level: 'MODERATE',
|
||||
age: { days: null, category: 'UNKNOWN', created_at: null },
|
||||
blacklist: { is_blacklisted: false, reputation_score: 80 },
|
||||
ssl: { has_ssl: true, is_valid: true, issuer: null },
|
||||
ownership: { registrar: null, organization: null, country: null },
|
||||
red_flags: [],
|
||||
warnings: ['Social media content may lack editorial oversight'],
|
||||
metadata: { duration_ms: 32 },
|
||||
};
|
||||
|
||||
const notApplicableResult: DomainAnalysisResult = {
|
||||
domain: 'unknown',
|
||||
verdict: 'NOT_APPLICABLE',
|
||||
trust_score: 0,
|
||||
risk_level: 'NONE',
|
||||
age: { days: null, category: 'UNKNOWN', created_at: null },
|
||||
blacklist: { is_blacklisted: false, reputation_score: null },
|
||||
ssl: { has_ssl: false, is_valid: false, issuer: null },
|
||||
ownership: { registrar: null, organization: null, country: null },
|
||||
red_flags: [],
|
||||
warnings: ['Domain analysis not applicable for text-only input'],
|
||||
metadata: { duration_ms: 2 },
|
||||
};
|
||||
|
||||
const blacklistedResult: DomainAnalysisResult = {
|
||||
domain: 'bit.ly',
|
||||
verdict: 'UNTRUSTED',
|
||||
trust_score: 20,
|
||||
risk_level: 'CRITICAL',
|
||||
age: { days: null, category: 'UNKNOWN', created_at: null },
|
||||
blacklist: { is_blacklisted: true, reputation_score: 20 },
|
||||
ssl: { has_ssl: true, is_valid: true, issuer: null },
|
||||
ownership: { registrar: null, organization: null, country: null },
|
||||
red_flags: ['URL_SHORTENER'],
|
||||
warnings: [],
|
||||
metadata: { duration_ms: 15 },
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// TESTS
|
||||
// ============================================================================
|
||||
|
||||
describe('DomainWorker output (Task 2.4)', () => {
|
||||
describe('toDomainResult mapping', () => {
|
||||
test('trusted result conforms to DomainResult schema', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
const parsed = DomainResultSchema.safeParse(result);
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
|
||||
test('suspicious result conforms to DomainResult schema', () => {
|
||||
const result = toDomainResult(suspiciousResult);
|
||||
const parsed = DomainResultSchema.safeParse(result);
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
|
||||
test('social result conforms to DomainResult schema', () => {
|
||||
const result = toDomainResult(socialResult);
|
||||
const parsed = DomainResultSchema.safeParse(result);
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
|
||||
test('not applicable result conforms to DomainResult schema', () => {
|
||||
const result = toDomainResult(notApplicableResult);
|
||||
const parsed = DomainResultSchema.safeParse(result);
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
|
||||
test('blacklisted result conforms to DomainResult schema', () => {
|
||||
const result = toDomainResult(blacklistedResult);
|
||||
const parsed = DomainResultSchema.safeParse(result);
|
||||
expect(parsed.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('trust_score is 0-100 (already was)', () => {
|
||||
test('trusted: trust_score = 70', () => {
|
||||
expect(trustedResult.trust_score).toBe(70);
|
||||
expect(trustedResult.trust_score).toBeGreaterThanOrEqual(0);
|
||||
expect(trustedResult.trust_score).toBeLessThanOrEqual(100);
|
||||
});
|
||||
|
||||
test('suspicious: trust_score = 30', () => {
|
||||
expect(suspiciousResult.trust_score).toBe(30);
|
||||
});
|
||||
|
||||
test('not applicable: trust_score = 0 (NOT -1)', () => {
|
||||
expect(notApplicableResult.trust_score).toBe(0);
|
||||
expect(notApplicableResult.trust_score).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('blacklisted: trust_score = 20', () => {
|
||||
expect(blacklistedResult.trust_score).toBe(20);
|
||||
});
|
||||
|
||||
test('trust_score is integer', () => {
|
||||
expect(Number.isInteger(trustedResult.trust_score)).toBe(true);
|
||||
expect(Number.isInteger(suspiciousResult.trust_score)).toBe(true);
|
||||
expect(Number.isInteger(notApplicableResult.trust_score)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('structure flattening (nested → flat)', () => {
|
||||
test('age fields flatten correctly', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.age_days).toBeNull();
|
||||
expect(result.age_category).toBe('UNKNOWN');
|
||||
expect(result.domain_created_at).toBeNull();
|
||||
});
|
||||
|
||||
test('blacklist fields flatten correctly', () => {
|
||||
const result = toDomainResult(blacklistedResult);
|
||||
expect(result.is_blacklisted).toBe(true);
|
||||
expect(result.reputation_score).toBe(20);
|
||||
});
|
||||
|
||||
test('ssl fields flatten correctly', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.has_ssl).toBe(true);
|
||||
expect(result.ssl_valid).toBe(true);
|
||||
expect(result.ssl_issuer).toBeNull();
|
||||
});
|
||||
|
||||
test('ownership fields flatten correctly', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.registrar).toBeNull();
|
||||
expect(result.organization).toBeNull();
|
||||
expect(result.country).toBeNull();
|
||||
});
|
||||
|
||||
test('metadata.duration_ms → duration_ms', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.duration_ms).toBe(45);
|
||||
});
|
||||
});
|
||||
|
||||
describe('red_flags and warnings', () => {
|
||||
test('no flags: empty arrays', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.red_flags).toEqual([]);
|
||||
expect(result.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
test('suspicious domain: multiple red_flags', () => {
|
||||
const result = toDomainResult(suspiciousResult);
|
||||
expect(result.red_flags).toContain('SUSPICIOUS_TLD');
|
||||
expect(result.red_flags).toContain('NUMERIC_DOMAIN');
|
||||
expect(result.red_flags).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('social domain: has warning', () => {
|
||||
const result = toDomainResult(socialResult);
|
||||
expect(result.warnings).toHaveLength(1);
|
||||
expect(result.warnings[0]).toContain('Social media');
|
||||
});
|
||||
|
||||
test('not applicable: has warning', () => {
|
||||
const result = toDomainResult(notApplicableResult);
|
||||
expect(result.warnings).toContain('Domain analysis not applicable for text-only input');
|
||||
});
|
||||
});
|
||||
|
||||
describe('verdict and risk_level consistency', () => {
|
||||
test('TRUSTED → LOW risk', () => {
|
||||
expect(trustedResult.verdict).toBe('TRUSTED');
|
||||
expect(trustedResult.risk_level).toBe('LOW');
|
||||
});
|
||||
|
||||
test('SUSPICIOUS → HIGH risk', () => {
|
||||
expect(suspiciousResult.verdict).toBe('SUSPICIOUS');
|
||||
expect(suspiciousResult.risk_level).toBe('HIGH');
|
||||
});
|
||||
|
||||
test('UNTRUSTED → CRITICAL risk', () => {
|
||||
expect(blacklistedResult.verdict).toBe('UNTRUSTED');
|
||||
expect(blacklistedResult.risk_level).toBe('CRITICAL');
|
||||
});
|
||||
|
||||
test('NOT_APPLICABLE → NONE risk', () => {
|
||||
expect(notApplicableResult.verdict).toBe('NOT_APPLICABLE');
|
||||
expect(notApplicableResult.risk_level).toBe('NONE');
|
||||
});
|
||||
});
|
||||
|
||||
describe('domain string', () => {
|
||||
test('normal domain preserved', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.domain).toBe('reuters.com');
|
||||
});
|
||||
|
||||
test('not applicable: domain = "unknown"', () => {
|
||||
const result = toDomainResult(notApplicableResult);
|
||||
expect(result.domain).toBe('unknown');
|
||||
});
|
||||
|
||||
test('domain fits VARCHAR(255)', () => {
|
||||
const result = toDomainResult(trustedResult);
|
||||
expect(result.domain.length).toBeLessThanOrEqual(255);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// TRUST SCORE CALCULATION (replicated from domain-worker)
|
||||
// ============================================================================
|
||||
|
||||
describe('trust_score calculation', () => {
|
||||
const PLATFORM_SCORES: Record<string, number> = {
|
||||
PLAT_TWITTER: 40, PLAT_FACEBOOK: 45, PLAT_YOUTUBE: 50,
|
||||
PLAT_REDDIT: 35, PLAT_TIKTOK: 30, PLAT_INSTAGRAM: 40,
|
||||
};
|
||||
|
||||
const SOURCE_TYPE_SCORES: Record<string, number> = {
|
||||
news: 70, official: 85, social: 40, blog: 50, unknown: 50,
|
||||
};
|
||||
|
||||
function calculateTrustScore(
|
||||
sourceType: string,
|
||||
platformCode: string | null,
|
||||
redFlagCount: number,
|
||||
): number {
|
||||
let trustScore = SOURCE_TYPE_SCORES[sourceType] || 50;
|
||||
if (platformCode && PLATFORM_SCORES[platformCode] !== undefined) {
|
||||
trustScore = (trustScore + PLATFORM_SCORES[platformCode]) / 2;
|
||||
}
|
||||
for (let i = 0; i < redFlagCount; i++) {
|
||||
trustScore = Math.max(0, trustScore - 10);
|
||||
}
|
||||
return Math.round(trustScore);
|
||||
}
|
||||
|
||||
test('news source: trust = 70', () => {
|
||||
expect(calculateTrustScore('news', null, 0)).toBe(70);
|
||||
});
|
||||
|
||||
test('official source: trust = 85', () => {
|
||||
expect(calculateTrustScore('official', null, 0)).toBe(85);
|
||||
});
|
||||
|
||||
test('social + twitter platform: trust = (40+40)/2 = 40', () => {
|
||||
expect(calculateTrustScore('social', 'PLAT_TWITTER', 0)).toBe(40);
|
||||
});
|
||||
|
||||
test('unknown source with 2 red flags: 50 - 20 = 30', () => {
|
||||
expect(calculateTrustScore('unknown', null, 2)).toBe(30);
|
||||
});
|
||||
|
||||
test('unknown source with 6 red flags: floors at 0', () => {
|
||||
expect(calculateTrustScore('unknown', null, 6)).toBe(0);
|
||||
});
|
||||
|
||||
test('result is always integer', () => {
|
||||
const result = calculateTrustScore('social', 'PLAT_YOUTUBE', 1);
|
||||
expect(Number.isInteger(result)).toBe(true);
|
||||
});
|
||||
|
||||
test('result is always 0-100', () => {
|
||||
const result = calculateTrustScore('unknown', null, 100);
|
||||
expect(result).toBeGreaterThanOrEqual(0);
|
||||
expect(result).toBeLessThanOrEqual(100);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/**
|
||||
* AI Tampered Worker - config-only subclass of ComponentWorker.
|
||||
*/
|
||||
import { ComponentWorker } from './component-worker';
|
||||
import type { ComponentRunner } from '../../components/component-runner';
|
||||
|
||||
export class AITamperedWorker extends ComponentWorker {
|
||||
constructor(runner: ComponentRunner) {
|
||||
super('ai_tampered', runner, 5);
|
||||
}
|
||||
}
|
||||
|
||||
export default AITamperedWorker;
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/**
|
||||
* Claims Worker - config-only subclass of ComponentWorker.
|
||||
*/
|
||||
import { ComponentWorker } from './component-worker';
|
||||
import type { ComponentRunner } from '../../components/component-runner';
|
||||
|
||||
export class ClaimsWorker extends ComponentWorker {
|
||||
constructor(runner: ComponentRunner) {
|
||||
super('claims', runner, 3);
|
||||
}
|
||||
}
|
||||
|
||||
export default ClaimsWorker;
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/**
|
||||
* Extract the primary score (0-100) from a typed component result.
|
||||
*
|
||||
* For credibility/trust components (claims/domain) we INVERT — the worker
|
||||
* publishes a "risk score" upstream, so high credibility / high trust both
|
||||
* map to low risk. Scores are already 0-100 (post Task 2.x standardization).
|
||||
*/
|
||||
import type { AnalysisComponent } from '../../../shared/queue/constants';
|
||||
|
||||
export function extractScore(component: AnalysisComponent, result: any): number {
|
||||
switch (component) {
|
||||
case 'techniques':
|
||||
return result.manipulation_score ?? 0;
|
||||
case 'ai_tampered':
|
||||
return result.ai_probability ?? 0;
|
||||
case 'claims':
|
||||
// Invert: high credibility = low risk
|
||||
return result.credibility_score != null ? 100 - result.credibility_score : 50;
|
||||
case 'domain':
|
||||
// Invert: high trust = low risk
|
||||
return result.trust_score != null ? 100 - result.trust_score : 50;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Parse a cached Vision-AI-detection response into a fully-formed
|
||||
* AiTamperedResult, so the worker can short-circuit a fresh Vision call when
|
||||
* the media-preprocess worker has already done it.
|
||||
*
|
||||
* Same JSON shape and same result fields as
|
||||
* ComponentRunner.runAiTamperedImage — keeps consumers from caring whether
|
||||
* the result came from cache or live Vision.
|
||||
*
|
||||
* Returns null on parse failure; caller falls back to a live Vision call.
|
||||
*/
|
||||
import { log } from '../../../shared/logger';
|
||||
|
||||
export function parseImageAiDetectionCache(
|
||||
visionResponse: string,
|
||||
_sessionId: string,
|
||||
startTime: number,
|
||||
): any | null {
|
||||
try {
|
||||
let aiProb = 50;
|
||||
let indicators: string[] = ['Analysis from cache'];
|
||||
let evidence = '';
|
||||
|
||||
const jsonMatch = visionResponse.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
const parsed = JSON.parse(jsonMatch[0]);
|
||||
aiProb = parsed.ai_generated_probability || 50;
|
||||
indicators = parsed.indicators || [];
|
||||
evidence = parsed.evidence || '';
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
const verdict = aiProb >= 70 ? 'LIKELY_AI' : aiProb >= 50 ? 'POSSIBLY_AI' : aiProb >= 30 ? 'MIXED' : 'LIKELY_HUMAN';
|
||||
|
||||
return {
|
||||
ai_probability: aiProb,
|
||||
verdict,
|
||||
risk_score: aiProb,
|
||||
categories_affected: [],
|
||||
indicators_count: indicators.length,
|
||||
disclosure_detected: false,
|
||||
disclosure_explicit: false,
|
||||
disclosure_text: null,
|
||||
indicators_detected: indicators.map((ind: string, i: number) => ({
|
||||
id: `IMG.${i + 1}`, category: 'IMG', name: ind, confidence: aiProb, evidence,
|
||||
})),
|
||||
coupling_context: {
|
||||
for_verdict: { ai_risk_score: aiProb, undisclosed_ai: aiProb > 50, confidence_level: aiProb > 70 ? 'HIGH' : 'MEDIUM', needs_manual_review: aiProb > 50 },
|
||||
for_source_assessment: { synthetic_content_detected: aiProb > 50, disclosure_rating: 0 },
|
||||
},
|
||||
llm_screening: 'cache',
|
||||
llm_deep: 'none',
|
||||
screening_duration_ms: durationMs,
|
||||
deep_analysis_duration_ms: 0,
|
||||
total_duration_ms: durationMs,
|
||||
fallbacks_screening: 0,
|
||||
fallbacks_deep: 0,
|
||||
content_type: 'image',
|
||||
image_analysis: { ai_generated_probability: aiProb, indicators, evidence },
|
||||
};
|
||||
} catch (err) {
|
||||
log.warn(`[ai_tamperedWorker] Failed to parse cached image AI detection: ${(err as Error).message}`);
|
||||
return null; // fallback: ComponentRunner will do its own Vision call
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Read pre-processed media results from Redis cache.
|
||||
*
|
||||
* MediaPreprocessWorker writes (in order):
|
||||
* - MediaCacheKeys.transcript(sid) — Whisper output
|
||||
* - MediaCacheKeys.mergedText(sid) — transcript + visual text
|
||||
* - MediaCacheKeys.vision(sid, 'misinformation') — vision text for techniques/claims
|
||||
* - MediaCacheKeys.vision(sid, 'ai_detection') — vision JSON for ai_tampered
|
||||
* - MediaCacheKeys.ready(sid) — sentinel, last
|
||||
*
|
||||
* `ready` flag is checked first; if absent, return hit:false without further reads.
|
||||
* On Redis errors, fail-open with hit:false so the worker falls back to local processing.
|
||||
*/
|
||||
import type Redis from 'ioredis';
|
||||
import { MediaCacheKeys } from '../../../shared/redis/keys';
|
||||
import { log } from '../../../shared/logger';
|
||||
|
||||
export interface MediaCacheResult {
|
||||
hit: boolean;
|
||||
transcript: string | null;
|
||||
mergedText: string | null;
|
||||
visionMisinformation: string | null;
|
||||
visionAiDetection: string | null;
|
||||
}
|
||||
|
||||
const EMPTY_RESULT: MediaCacheResult = {
|
||||
hit: false,
|
||||
transcript: null,
|
||||
mergedText: null,
|
||||
visionMisinformation: null,
|
||||
visionAiDetection: null,
|
||||
};
|
||||
|
||||
export async function readMediaCache(
|
||||
sessionId: string,
|
||||
redis: Redis,
|
||||
componentLabel: string,
|
||||
): Promise<MediaCacheResult> {
|
||||
try {
|
||||
const ready = await redis.get(MediaCacheKeys.ready(sessionId));
|
||||
if (!ready) return EMPTY_RESULT;
|
||||
|
||||
const [transcript, mergedText, visionMisinfo, visionAi] = await Promise.all([
|
||||
redis.get(MediaCacheKeys.transcript(sessionId)),
|
||||
redis.get(MediaCacheKeys.mergedText(sessionId)),
|
||||
redis.get(MediaCacheKeys.vision(sessionId, 'misinformation')),
|
||||
redis.get(MediaCacheKeys.vision(sessionId, 'ai_detection')),
|
||||
]);
|
||||
|
||||
const hasData = !!(transcript || mergedText || visionMisinfo || visionAi);
|
||||
|
||||
if (hasData) {
|
||||
log.info(`[${componentLabel}Worker] Media cache: transcript=${transcript?.length || 0}, merged=${mergedText?.length || 0}, vision_misinfo=${visionMisinfo?.length || 0}, vision_ai=${visionAi?.length || 0}`);
|
||||
}
|
||||
|
||||
return {
|
||||
hit: hasData,
|
||||
transcript,
|
||||
mergedText,
|
||||
visionMisinformation: visionMisinfo,
|
||||
visionAiDetection: visionAi,
|
||||
};
|
||||
} catch (err) {
|
||||
log.warn(`[${componentLabel}Worker] Redis cache read failed: ${(err as Error).message}`);
|
||||
return EMPTY_RESULT;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
/**
|
||||
* 2-track AI-tampered analysis for VIDEO inputs.
|
||||
*
|
||||
* Track 1 (text): AITamperedExecutor on transcript.
|
||||
* Skipped when transcript is missing or shorter than
|
||||
* MIN_TEXT_FOR_AI_TRACK (configurable, default 200).
|
||||
* Track 2 (visual): AI_CONFIDENCE field parsed from vision-analysis text;
|
||||
* falls back to keyword heuristic
|
||||
* (ai-generated/deepfake/synthetic/artifact).
|
||||
*
|
||||
* Final probability blends the two with weights from
|
||||
* `didi:config:pipeline:v1:component_config.video_track_weights`, falling
|
||||
* back to canonical DEFAULT_VIDEO_TRACK_WEIGHTS (0.4 / 0.6).
|
||||
*
|
||||
* If text track was skipped, visual gets 100% weight.
|
||||
*
|
||||
* Result shape mirrors toAiTamperedResult() so the worker publishes the
|
||||
* same shape regardless of which track ran.
|
||||
*/
|
||||
import type Redis from 'ioredis';
|
||||
import { AITamperedExecutor } from '../../../components/ai-tampered/executor';
|
||||
import { toAiTamperedResult, type AnalysisInput, type LLMClient } from '../../../components/component-runner';
|
||||
import { DEFAULT_VIDEO_TRACK_WEIGHTS, type VideoTrackWeights } from '../../../shared/media/video-weighting';
|
||||
import { MediaCacheKeys } from '../../../shared/redis/keys';
|
||||
import type { BusterResult } from '../../../shared/media/buster';
|
||||
import { log } from '../../../shared/logger';
|
||||
|
||||
export async function runAiTamperedVideo(
|
||||
redis: Redis,
|
||||
llmClient: LLMClient,
|
||||
input: AnalysisInput,
|
||||
visualAnalysis: string,
|
||||
workerStartTime: number,
|
||||
): Promise<any> {
|
||||
const videoIndicators: any[] = [];
|
||||
let aiProbability = 0;
|
||||
let baseResult: any = null;
|
||||
let busterAvailable = true; // devine false dacă BusterX era așteptat dar a eșuat (santinelă UNAVAILABLE)
|
||||
|
||||
// Track weights from Redis pipeline config (fallback to canonical defaults
|
||||
// shared with aggregator + pipeline-routes via shared/media/video-weighting).
|
||||
const weights: VideoTrackWeights = { ...DEFAULT_VIDEO_TRACK_WEIGHTS };
|
||||
let MIN_TEXT_FOR_AI_TRACK = 200;
|
||||
try {
|
||||
const pipelineCfg = await redis.get('didi:config:pipeline:v1:component_config');
|
||||
if (pipelineCfg) {
|
||||
const parsed = JSON.parse(pipelineCfg);
|
||||
const vtw = parsed.video_track_weights;
|
||||
if (vtw) {
|
||||
weights.text = vtw.text ?? DEFAULT_VIDEO_TRACK_WEIGHTS.text;
|
||||
weights.visual = vtw.visual ?? DEFAULT_VIDEO_TRACK_WEIGHTS.visual;
|
||||
}
|
||||
if (parsed.video_min_text_chars) MIN_TEXT_FOR_AI_TRACK = parsed.video_min_text_chars;
|
||||
}
|
||||
} catch { /* use defaults */ }
|
||||
const TEXT_WEIGHT = weights.text;
|
||||
const VISUAL_WEIGHT = weights.visual;
|
||||
const hasUsableText = input.text && input.text.trim().length >= MIN_TEXT_FOR_AI_TRACK;
|
||||
|
||||
if (hasUsableText) {
|
||||
const executor = new AITamperedExecutor(redis, llmClient);
|
||||
const rawTextAnalysis = await executor.execute(input.text!, input.sessionId);
|
||||
baseResult = toAiTamperedResult(rawTextAnalysis);
|
||||
aiProbability = Math.round(baseResult.ai_probability * TEXT_WEIGHT);
|
||||
videoIndicators.push(...(baseResult.indicators_detected || []));
|
||||
log.info(`[ai_tamperedWorker] Track 1 (text): ${baseResult.ai_probability}% × ${TEXT_WEIGHT} = ${aiProbability}%`);
|
||||
} else {
|
||||
const reason = !input.text ? 'no transcript' : `transcript too short (${input.text.trim().length} < ${MIN_TEXT_FOR_AI_TRACK} chars)`;
|
||||
log.info(`[ai_tamperedWorker] Track 1 (text): skipped — ${reason}`);
|
||||
}
|
||||
|
||||
// Track 2: Visual AI detection — parse AI_CONFIDENCE or keyword fallback
|
||||
if (visualAnalysis) {
|
||||
let visualScore = 0;
|
||||
|
||||
const confMatch = visualAnalysis.match(/AI_CONFIDENCE:\s*(\d+)/i);
|
||||
if (confMatch) {
|
||||
visualScore = Math.min(100, Math.max(0, parseInt(confMatch[1], 10)));
|
||||
log.info(`[ai_tamperedWorker] Track 2 (visual): AI_CONFIDENCE parsed = ${visualScore}%`);
|
||||
} else {
|
||||
// Keyword fallback (same as sync path)
|
||||
const visualLower = visualAnalysis.toLowerCase();
|
||||
if (visualLower.includes('ai-generated') || visualLower.includes('deepfake')) {
|
||||
visualScore = 75;
|
||||
} else if (visualLower.includes('artifact') || visualLower.includes('synthetic')) {
|
||||
visualScore = 40;
|
||||
} else {
|
||||
visualScore = 10;
|
||||
}
|
||||
log.info(`[ai_tamperedWorker] Track 2 (visual): keyword fallback = ${visualScore}%`);
|
||||
}
|
||||
|
||||
if (!hasUsableText) {
|
||||
aiProbability = visualScore;
|
||||
log.info(`[ai_tamperedWorker] No usable text — visual is 100% weight: ${aiProbability}%`);
|
||||
} else {
|
||||
aiProbability += Math.round(visualScore * VISUAL_WEIGHT);
|
||||
log.info(`[ai_tamperedWorker] Blended: text(${TEXT_WEIGHT * 100}%) + visual(${VISUAL_WEIGHT * 100}%) = ${aiProbability}%`);
|
||||
}
|
||||
|
||||
if (visualScore > 30) {
|
||||
videoIndicators.push({
|
||||
id: 'VID.1', category: 'VID',
|
||||
name: visualScore >= 60 ? 'Visual AI indicators detected' : 'Possible visual artifacts',
|
||||
confidence: visualScore,
|
||||
evidence: visualAnalysis.substring(0, 500),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Track 3 (BusterX deepfake): specialized video classifier written to Redis
|
||||
// by media-preprocess-worker. BusterX is a dedicated deepfake model, so a
|
||||
// confident verdict deterministically overrides the blended text+visual
|
||||
// score. INCONCLUSIVE is a no-op. Fail-open on read/parse error.
|
||||
try {
|
||||
const busterRaw = await redis.get(MediaCacheKeys.busterVerdict(input.sessionId));
|
||||
if (busterRaw) {
|
||||
const buster = JSON.parse(busterRaw) as BusterResult;
|
||||
const before = aiProbability;
|
||||
// 'UNAVAILABLE' e o santinelă injectată de media-preprocess (nu un verdict BusterX
|
||||
// real), deci cast la string pentru comparație fără a extinde tipul de domeniu.
|
||||
if ((buster.verdict as string) === 'UNAVAILABLE') {
|
||||
// BusterX era așteptat dar a eșuat — NU tăcem. Marcăm degradarea vizibil, ca
|
||||
// verdictul să nu pară o verificare deepfake completă când de fapt nu a rulat.
|
||||
busterAvailable = false;
|
||||
videoIndicators.push({
|
||||
id: 'VID.2', category: 'VID',
|
||||
name: 'Verificare deepfake (BusterX) INDISPONIBILĂ — recomandată verificare manuală',
|
||||
confidence: 0,
|
||||
evidence: (buster.explanation || 'BusterX nu a returnat rezultat').substring(0, 500),
|
||||
});
|
||||
log.warn('[ai_tamperedWorker] Track 3 (BusterX): UNAVAILABLE — analiză marcată pentru verificare manuală');
|
||||
} else {
|
||||
if (buster.verdict === 'FAKE') {
|
||||
aiProbability = Math.max(aiProbability, 85);
|
||||
} else if (buster.verdict === 'REAL') {
|
||||
aiProbability = Math.min(aiProbability, 20);
|
||||
}
|
||||
if (buster.verdict !== 'INCONCLUSIVE') {
|
||||
videoIndicators.push({
|
||||
id: 'VID.2', category: 'VID',
|
||||
name: `BusterX deepfake detection: ${buster.verdict}`,
|
||||
confidence: buster.verdict === 'FAKE' ? 90 : 80,
|
||||
evidence: (buster.explanation || '').substring(0, 500),
|
||||
});
|
||||
}
|
||||
log.info(`[ai_tamperedWorker] Track 3 (BusterX): ${buster.verdict} — ai_probability ${before}% → ${aiProbability}%`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[ai_tamperedWorker] BusterX verdict read/parse failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
aiProbability = Math.min(100, Math.max(0, aiProbability));
|
||||
const durationMs = Date.now() - workerStartTime;
|
||||
|
||||
// Build result in same shape as toAiTamperedResult() — identical output structure
|
||||
const verdict = aiProbability >= 70 ? 'LIKELY_AI'
|
||||
: aiProbability >= 50 ? 'POSSIBLY_AI'
|
||||
: aiProbability >= 30 ? 'MIXED'
|
||||
: 'LIKELY_HUMAN';
|
||||
|
||||
// coupling_context (persistat în DB). Dacă BusterX a fost indisponibil, forțăm
|
||||
// needs_manual_review și marcăm deepfake_check='unavailable' — semnal onest că
|
||||
// verdictul video NU include verificarea deepfake dedicată.
|
||||
const couplingContext = baseResult?.coupling_context || {
|
||||
for_verdict: { ai_risk_score: aiProbability, undisclosed_ai: aiProbability > 50, confidence_level: 'MEDIUM', needs_manual_review: true },
|
||||
for_source_assessment: { synthetic_content_detected: aiProbability > 50, disclosure_rating: 0 },
|
||||
};
|
||||
if (!busterAvailable) {
|
||||
couplingContext.for_verdict = {
|
||||
...(couplingContext.for_verdict || {}),
|
||||
needs_manual_review: true,
|
||||
deepfake_check: 'unavailable',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ai_probability: aiProbability,
|
||||
verdict,
|
||||
risk_score: aiProbability,
|
||||
categories_affected: baseResult?.categories_affected || [],
|
||||
indicators_count: videoIndicators.length,
|
||||
disclosure_detected: baseResult?.disclosure_detected || false,
|
||||
disclosure_explicit: baseResult?.disclosure_explicit || false,
|
||||
disclosure_text: baseResult?.disclosure_text || null,
|
||||
indicators_detected: videoIndicators,
|
||||
coupling_context: couplingContext,
|
||||
llm_screening: baseResult?.llm_screening || 'none',
|
||||
llm_deep: baseResult?.llm_deep || 'none',
|
||||
screening_duration_ms: durationMs,
|
||||
deep_analysis_duration_ms: 0,
|
||||
total_duration_ms: durationMs,
|
||||
fallbacks_screening: baseResult?.fallbacks_screening || 0,
|
||||
fallbacks_deep: 0,
|
||||
content_type: 'video',
|
||||
image_analysis: null,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,461 @@
|
|||
/**
|
||||
* ComponentWorker - Generic queue worker that uses ComponentRunner directly.
|
||||
*
|
||||
* Replaces BaseWorker (HTTP) + TechniquesWorker/AITamperedWorker/ClaimsWorker/DomainWorker.
|
||||
* Zero HTTP calls - invokes executors directly via ComponentRunner.
|
||||
*
|
||||
* Helpers extracted to ./component-worker-helpers/:
|
||||
* extract-score.ts — primary score extraction per component
|
||||
* media-cache.ts — readMediaCache (Redis lookup of preprocess results)
|
||||
* image-ai-cache.ts — parseImageAiDetectionCache (cached Vision JSON → result)
|
||||
* video-2-track.ts — runAiTamperedVideo (text + visual blended scoring)
|
||||
*/
|
||||
|
||||
import { getChannel, getConfirmChannel, publishConfirmed } from '../connection';
|
||||
import {
|
||||
EXCHANGE_NAME,
|
||||
getQueueName,
|
||||
WORKER_CONFIG,
|
||||
type PlanType,
|
||||
} from '../../shared/queue/constants';
|
||||
import { QueueKeys } from '../../shared/redis/keys';
|
||||
import type { AnalysisTaskMessage, AnalysisResultMessage } from '../types';
|
||||
import { RESULTS_QUEUE } from '../../shared/queue/constants';
|
||||
import type { AnalysisComponent } from '../../shared/queue/constants';
|
||||
import { ComponentRunner, type AnalysisInput } from '../../components/component-runner';
|
||||
import { getSearchTier } from '../../shared/credits';
|
||||
import { pipelineDuration, analysisFailed } from '../../shared/observability/metrics';
|
||||
import { processVideoUrl } from '../../shared/media/video-processor';
|
||||
import { transcribe } from '../../shared/media/transcription';
|
||||
import { sanitizeUtf8 } from '../../config/analysisLimits';
|
||||
import type Redis from 'ioredis';
|
||||
import { createRedisConnection } from '../../shared/redis/connection';
|
||||
import { isLogicalInputError } from '../../shared/helpers/errors';
|
||||
import { log } from '../../shared/logger';
|
||||
|
||||
import { extractScore } from './component-worker-helpers/extract-score';
|
||||
import { readMediaCache } from './component-worker-helpers/media-cache';
|
||||
import { parseImageAiDetectionCache } from './component-worker-helpers/image-ai-cache';
|
||||
import { runAiTamperedVideo } from './component-worker-helpers/video-2-track';
|
||||
|
||||
interface ConsumeMessage {
|
||||
content: Buffer;
|
||||
fields: any;
|
||||
properties: any;
|
||||
}
|
||||
|
||||
export class ComponentWorker {
|
||||
protected channel: any = null;
|
||||
protected redis: Redis | null = null;
|
||||
protected isRunning = false;
|
||||
private resubscribeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private component: AnalysisComponent,
|
||||
private componentRunner: ComponentRunner,
|
||||
private prefetch: number = 5,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Re-subscribe after connection/channel loss. Without this, a dropped
|
||||
* RabbitMQ connection leaves the worker running but consuming nothing —
|
||||
* silent until someone notices sessions piling up.
|
||||
*/
|
||||
private scheduleResubscribe(): void {
|
||||
if (!this.isRunning || this.resubscribeTimer) return;
|
||||
const RETRY_MS = 5000;
|
||||
const attempt = async (): Promise<void> => {
|
||||
this.resubscribeTimer = null;
|
||||
if (!this.isRunning) return;
|
||||
try {
|
||||
await this.start();
|
||||
log.info(`[${this.component}Worker] Re-subscribed after connection loss`);
|
||||
} catch (err) {
|
||||
log.warn(`[${this.component}Worker] Re-subscribe failed (${(err as Error).message}), retrying in ${RETRY_MS}ms`);
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
};
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
|
||||
protected getRedis(): Redis {
|
||||
if (!this.redis) {
|
||||
this.redis = createRedisConnection({ label: `component-worker:${this.component}` });
|
||||
}
|
||||
return this.redis;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
log.info(`[${this.component}Worker] Starting...`);
|
||||
|
||||
this.channel = await getChannel();
|
||||
if (!this.channel) {
|
||||
throw new Error('Failed to get RabbitMQ channel');
|
||||
}
|
||||
|
||||
this.channel.on('close', () => {
|
||||
if (this.isRunning) {
|
||||
log.warn(`[${this.component}Worker] Channel closed — scheduling re-subscribe`);
|
||||
this.channel = null;
|
||||
this.scheduleResubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
await this.channel.prefetch(this.prefetch);
|
||||
|
||||
const planTypes: PlanType[] = [1, 2, 3, 4, 5, 6];
|
||||
|
||||
for (const planType of planTypes) {
|
||||
const queueName = getQueueName(this.component, planType);
|
||||
|
||||
await this.channel.assertQueue(queueName, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-max-priority': WORKER_CONFIG.MAX_PRIORITY,
|
||||
'x-dead-letter-exchange': '',
|
||||
'x-dead-letter-routing-key': 'analysis_dlq',
|
||||
'x-message-ttl': WORKER_CONFIG.MESSAGE_TTL,
|
||||
},
|
||||
});
|
||||
|
||||
await this.channel.bindQueue(queueName, EXCHANGE_NAME, queueName);
|
||||
|
||||
await this.channel.consume(
|
||||
queueName,
|
||||
(msg: ConsumeMessage | null) => this.handleMessage(msg),
|
||||
{ noAck: false },
|
||||
);
|
||||
|
||||
log.info(`[${this.component}Worker] Consuming from ${queueName}`);
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
log.info(`[${this.component}Worker] Started, prefetch: ${this.prefetch}`);
|
||||
}
|
||||
|
||||
private async handleMessage(msg: ConsumeMessage | null): Promise<void> {
|
||||
if (!msg || !this.channel) return;
|
||||
|
||||
const startTime = Date.now();
|
||||
let taskMessage: AnalysisTaskMessage | null = null;
|
||||
|
||||
try {
|
||||
taskMessage = JSON.parse(msg.content.toString()) as AnalysisTaskMessage;
|
||||
const { sessionId, component } = taskMessage;
|
||||
|
||||
log.info(`[${this.component}Worker] Processing ${sessionId}`);
|
||||
|
||||
// Cancel check — user canceled the session; publish a canceled result so
|
||||
// the aggregator's fan-in still completes and finalizes as 'canceled'.
|
||||
if (await this.getRedis().get(QueueKeys.cancelFlag(sessionId))) {
|
||||
log.info(`[${this.component}Worker] ${sessionId} canceled — skipping execution`);
|
||||
await this.publishResult({
|
||||
sessionId, component: this.component,
|
||||
success: false, error: 'canceled_by_user',
|
||||
processingTime: 0, timestamp: Date.now(),
|
||||
});
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Acquire lock to prevent duplicate processing
|
||||
const lockKey = QueueKeys.componentLock(sessionId, component);
|
||||
const acquired = await this.getRedis().set(
|
||||
lockKey, '1', 'EX', WORKER_CONFIG.LOCK_TTL_COMPONENT, 'NX',
|
||||
);
|
||||
|
||||
if (!acquired) {
|
||||
log.warn(`[${this.component}Worker] Lock exists for ${sessionId}, skipping`);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
const input: AnalysisInput = {
|
||||
text: taskMessage.input.content,
|
||||
url: taskMessage.input.url,
|
||||
media_url: taskMessage.input.mediaPath,
|
||||
media_type: taskMessage.input.inputType,
|
||||
sessionId,
|
||||
searchTier: getSearchTier(taskMessage.planType),
|
||||
};
|
||||
|
||||
const inputType = taskMessage.input.inputType || 'text';
|
||||
|
||||
// ================================================================
|
||||
// MEDIA PRE-PROCESSING: try Redis cache first, fallback to local
|
||||
// MediaPreprocessWorker should have already cached results.
|
||||
// If cache miss (e.g. preprocess worker failed), process locally.
|
||||
// ================================================================
|
||||
let videoVisualAnalysis: string | null = null; // stored for ai_tampered 2-track
|
||||
let cachedImageAiResult: any = null; // stored for ai_tampered image from cache
|
||||
if (!input.text || input.text.trim().length === 0) {
|
||||
const redisClient = this.getRedis();
|
||||
|
||||
if (inputType === 'video' && (input.url || input.media_url)) {
|
||||
const cached = await readMediaCache(sessionId, redisClient, this.component);
|
||||
|
||||
if (cached.hit) {
|
||||
videoVisualAnalysis = cached.visionAiDetection;
|
||||
// ai_tampered needs transcript only (for 2-track: text AI vs visual AI)
|
||||
// techniques + claims get merged_text (transcript + visual text from frames)
|
||||
const useTranscriptOnly = this.component === 'ai_tampered';
|
||||
let textToUse = useTranscriptOnly ? cached.transcript : cached.mergedText;
|
||||
// Claims on media: truncate to 4000 chars to reduce extraction count
|
||||
if (this.component === 'claims' && textToUse && textToUse.length > 4000) {
|
||||
log.info(`[${this.component}Worker] Truncating for claims: ${textToUse.length} → 4000 chars`);
|
||||
textToUse = textToUse.substring(0, 4000);
|
||||
}
|
||||
if (textToUse && textToUse.trim().length > 0) {
|
||||
input.text = sanitizeUtf8(textToUse);
|
||||
log.info(`[${this.component}Worker] Cache HIT — ${useTranscriptOnly ? 'transcript' : 'merged'}: ${input.text.length} chars`);
|
||||
} else {
|
||||
log.warn(`[${this.component}Worker] Cache HIT but no text content`);
|
||||
}
|
||||
} else {
|
||||
// Cache MISS — fallback: process video locally (safety net)
|
||||
const videoSource = input.url || input.media_url!;
|
||||
log.info(`[${this.component}Worker] Cache MISS — fallback: downloading + processing: ${videoSource}`);
|
||||
try {
|
||||
const videoResult = await processVideoUrl(videoSource, sessionId, {
|
||||
redis: redisClient,
|
||||
logPrefix: `${this.component}Worker`,
|
||||
visionContext: this.component === 'ai_tampered' ? 'ai_detection' : 'misinformation',
|
||||
});
|
||||
|
||||
if (videoResult.visual_analysis) {
|
||||
videoVisualAnalysis = videoResult.visual_analysis;
|
||||
}
|
||||
|
||||
const useTranscriptOnly = this.component === 'ai_tampered';
|
||||
const textToUse = useTranscriptOnly ? videoResult.transcript : videoResult.merged_text;
|
||||
|
||||
if (textToUse && textToUse.trim().length > 0) {
|
||||
input.text = sanitizeUtf8(textToUse);
|
||||
log.info(`[${this.component}Worker] Fallback video text (${useTranscriptOnly ? 'transcript-only' : 'merged'}): ${input.text.length} chars`);
|
||||
} else {
|
||||
log.warn(`[${this.component}Worker] Video had no extractable speech/text`);
|
||||
}
|
||||
} catch (videoErr) {
|
||||
log.error(`[${this.component}Worker] Video download failed: ${(videoErr as Error).message}`);
|
||||
}
|
||||
}
|
||||
} else if (inputType === 'audio' && (input.media_url || input.url)) {
|
||||
const cached = await readMediaCache(sessionId, redisClient, this.component);
|
||||
|
||||
if (cached.hit && cached.transcript) {
|
||||
input.text = sanitizeUtf8(cached.transcript);
|
||||
log.info(`[${this.component}Worker] Cache HIT — audio transcript: ${input.text.length} chars`);
|
||||
} else {
|
||||
// Cache MISS — fallback: transcribe locally
|
||||
const audioSource = input.media_url || input.url!;
|
||||
log.info(`[${this.component}Worker] Cache MISS — fallback: transcribing: ${audioSource}`);
|
||||
try {
|
||||
const transcription = await transcribe(audioSource, { logPrefix: `${this.component}Worker`, tier: getSearchTier(taskMessage.planType) });
|
||||
if (transcription.success && transcription.text) {
|
||||
input.text = sanitizeUtf8(transcription.text);
|
||||
log.info(`[${this.component}Worker] Fallback audio transcribed: ${input.text.length} chars`);
|
||||
}
|
||||
} catch (audioErr) {
|
||||
log.error(`[${this.component}Worker] Audio transcription failed: ${(audioErr as Error).message}`);
|
||||
}
|
||||
}
|
||||
} else if (inputType === 'image' && (input.media_url || input.url)) {
|
||||
const cached = await readMediaCache(sessionId, redisClient, this.component);
|
||||
|
||||
if (cached.hit) {
|
||||
// OCR text for techniques + claims
|
||||
if (cached.transcript) {
|
||||
input.text = sanitizeUtf8(cached.transcript);
|
||||
log.info(`[${this.component}Worker] Cache HIT — image OCR text: ${input.text.length} chars`);
|
||||
}
|
||||
|
||||
// AI detection result for ai_tampered — build result from cache, skip Vision call
|
||||
if (this.component === 'ai_tampered' && cached.visionAiDetection) {
|
||||
cachedImageAiResult = parseImageAiDetectionCache(cached.visionAiDetection, sessionId, startTime);
|
||||
if (cachedImageAiResult) {
|
||||
log.info(`[${this.component}Worker] Cache HIT — image AI detection: prob=${cachedImageAiResult.ai_probability}%`);
|
||||
}
|
||||
}
|
||||
}
|
||||
// If cache miss, ComponentRunner.runTechniques/runClaims will do OCR itself
|
||||
// and runAiTampered will do image AI detection itself — no fallback needed here
|
||||
}
|
||||
}
|
||||
|
||||
// Run component directly via ComponentRunner (NO HTTP)
|
||||
let result: any;
|
||||
switch (this.component) {
|
||||
case 'techniques':
|
||||
result = await this.componentRunner.runTechniques(input);
|
||||
break;
|
||||
case 'ai_tampered':
|
||||
if (cachedImageAiResult) {
|
||||
// IMAGE from cache — result already built from cached Vision AI detection
|
||||
result = cachedImageAiResult;
|
||||
} else if (inputType === 'video' && videoVisualAnalysis) {
|
||||
// VIDEO 2-TRACK: separate text analysis + visual analysis, then blend
|
||||
result = await runAiTamperedVideo(
|
||||
this.getRedis(),
|
||||
this.componentRunner.getLLMClient(),
|
||||
input,
|
||||
videoVisualAnalysis,
|
||||
startTime,
|
||||
);
|
||||
} else {
|
||||
result = await this.componentRunner.runAiTampered(input);
|
||||
}
|
||||
break;
|
||||
case 'claims':
|
||||
result = await this.componentRunner.runClaims(input);
|
||||
break;
|
||||
case 'domain':
|
||||
result = await this.componentRunner.runSourceAssessment(input);
|
||||
break;
|
||||
}
|
||||
|
||||
const score = extractScore(this.component, result);
|
||||
|
||||
// Capture LLM usage from the last run
|
||||
const llmUsage = this.componentRunner.getLastUsageTracker();
|
||||
|
||||
const resultMessage: AnalysisResultMessage = {
|
||||
sessionId,
|
||||
component: this.component,
|
||||
success: true,
|
||||
score,
|
||||
data: result,
|
||||
llm_usage: llmUsage.length > 0 ? llmUsage : undefined,
|
||||
processingTime: Date.now() - startTime,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
await this.publishResult(resultMessage);
|
||||
this.channel.ack(msg);
|
||||
|
||||
pipelineDuration.observe(
|
||||
{ component: this.component, tier: getSearchTier(taskMessage.planType) },
|
||||
(Date.now() - startTime) / 1000,
|
||||
);
|
||||
|
||||
log.info(
|
||||
`[${this.component}Worker] Completed ${sessionId} in ${Date.now() - startTime}ms (score: ${score})`,
|
||||
);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
log.error(`[${this.component}Worker] Error:`, error.message);
|
||||
|
||||
// Logical errors (missing input) should not retry — publish error immediately.
|
||||
// Detect by error class so we don't depend on message wording.
|
||||
const isLogicalError = isLogicalInputError(err);
|
||||
if (isLogicalError && taskMessage) {
|
||||
log.info(`[${this.component}Worker] Logical error for ${taskMessage.sessionId}, skipping retries`);
|
||||
const errorResult: AnalysisResultMessage = {
|
||||
sessionId: taskMessage.sessionId, component: this.component,
|
||||
success: false, error: error.message,
|
||||
processingTime: Date.now() - startTime, timestamp: Date.now(),
|
||||
};
|
||||
await this.publishResult(errorResult);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check retry count
|
||||
if (taskMessage && taskMessage.retryCount < WORKER_CONFIG.RETRY_MAX) {
|
||||
// Ack original message and republish with incremented retryCount
|
||||
// (nack+requeue would put back the original message with retryCount=0 forever).
|
||||
// Apply linear backoff so transient failures (LLM rate limits, network blips)
|
||||
// don't cascade — retry 1 waits 1s, retry 2 waits 2s, ... capped at 5s.
|
||||
this.channel.ack(msg);
|
||||
const nextRetry = taskMessage.retryCount + 1;
|
||||
const backoffMs = Math.min(5000, nextRetry * 1000);
|
||||
await new Promise(r => setTimeout(r, backoffMs));
|
||||
try {
|
||||
const retryMessage: AnalysisTaskMessage = { ...taskMessage, retryCount: nextRetry };
|
||||
const queueName = getQueueName(this.component, taskMessage.planType as PlanType);
|
||||
const confirmChannel = await getConfirmChannel();
|
||||
if (!confirmChannel) throw new Error('No confirm channel for retry');
|
||||
await publishConfirmed(confirmChannel, EXCHANGE_NAME, queueName, Buffer.from(JSON.stringify(retryMessage)), {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
priority: taskMessage.priority || 1,
|
||||
});
|
||||
log.info(`[${this.component}Worker] Retrying ${taskMessage.sessionId} after ${backoffMs}ms (attempt ${retryMessage.retryCount}/${WORKER_CONFIG.RETRY_MAX})`);
|
||||
} catch (retryErr) {
|
||||
// Original message is already acked — if the retry publish is lost
|
||||
// too, the session hangs until zombie cleanup. Fall back to an error
|
||||
// result so the aggregator can finalize.
|
||||
log.error(`[${this.component}Worker] Retry publish failed:`, (retryErr as Error).message);
|
||||
const errorResult: AnalysisResultMessage = {
|
||||
sessionId: taskMessage.sessionId, component: this.component,
|
||||
success: false, error: error.message,
|
||||
processingTime: Date.now() - startTime, timestamp: Date.now(),
|
||||
};
|
||||
await this.publishResult(errorResult);
|
||||
}
|
||||
} else {
|
||||
// No more retries — publish error result to aggregator so it can finalize
|
||||
analysisFailed.inc({
|
||||
component: this.component,
|
||||
tier: taskMessage ? getSearchTier(taskMessage.planType) : 'unknown',
|
||||
media_type: taskMessage?.input?.inputType || 'text',
|
||||
reason: 'retries_exhausted',
|
||||
});
|
||||
if (taskMessage) {
|
||||
const errorResult: AnalysisResultMessage = {
|
||||
sessionId: taskMessage.sessionId,
|
||||
component: this.component,
|
||||
success: false,
|
||||
error: error.message,
|
||||
processingTime: Date.now() - startTime, timestamp: Date.now(),
|
||||
};
|
||||
await this.publishResult(errorResult);
|
||||
}
|
||||
this.channel.nack(msg, false, false); // Send to DLQ
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async publishResult(result: AnalysisResultMessage): Promise<void> {
|
||||
const confirmChannel = await getConfirmChannel();
|
||||
if (!confirmChannel) {
|
||||
log.error(`[${this.component}Worker] Failed to get confirm channel for results`);
|
||||
return;
|
||||
}
|
||||
|
||||
await confirmChannel.assertQueue(RESULTS_QUEUE, { durable: true });
|
||||
|
||||
const routingKey = `${RESULTS_QUEUE}.${result.component}`;
|
||||
try {
|
||||
await publishConfirmed(confirmChannel, EXCHANGE_NAME, routingKey, Buffer.from(JSON.stringify(result)), {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
headers: {
|
||||
sessionId: result.sessionId,
|
||||
component: result.component,
|
||||
},
|
||||
});
|
||||
log.info(`[${this.component}Worker] Published result for ${result.sessionId}`);
|
||||
} catch (err) {
|
||||
// Lost result = aggregator never reaches N/N and the session hangs until
|
||||
// zombie cleanup marks it failed. Log loudly with session id for triage.
|
||||
log.error(`[${this.component}Worker] RESULT_PUBLISH_LOST session=${result.sessionId} component=${result.component}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
log.info(`[${this.component}Worker] Stopping...`);
|
||||
this.isRunning = false;
|
||||
|
||||
if (this.channel) {
|
||||
await this.channel.close();
|
||||
this.channel = null;
|
||||
}
|
||||
|
||||
if (this.redis) {
|
||||
this.redis.disconnect();
|
||||
this.redis = null;
|
||||
}
|
||||
|
||||
log.info(`[${this.component}Worker] Stopped`);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
/**
|
||||
* Dead-letter queue monitor — drains `analysis_dlq`, makes failures visible.
|
||||
*
|
||||
* Before this existed NOBODY asserted the DLQ queue: component queues declare
|
||||
* x-dead-letter-routing-key=analysis_dlq, but a routing key with no matching
|
||||
* queue means the broker DROPS dead-lettered messages silently. This monitor:
|
||||
* 1. asserts the queue (so dead-letters are actually retained),
|
||||
* 2. consumes it: error log + didi_dlq_messages_total metric + last 100
|
||||
* entries in Redis (didi:queue:dlq:recent) for triage from the dashboard.
|
||||
*
|
||||
* Runs inside the verdict-aggregator container (always-on, single instance
|
||||
* semantics not required — consuming is idempotent).
|
||||
*/
|
||||
import type Redis from 'ioredis';
|
||||
import { getChannel } from '../connection';
|
||||
import { DLQ_NAME } from '../../shared/queue/constants';
|
||||
import { dlqMessages } from '../../shared/observability/metrics';
|
||||
import { log } from '../../shared/logger';
|
||||
|
||||
const DLQ_RECENT_KEY = 'didi:queue:dlq:recent';
|
||||
const DLQ_RECENT_MAX = 100;
|
||||
|
||||
interface ConsumeMessage {
|
||||
content: Buffer;
|
||||
fields: any;
|
||||
properties: any;
|
||||
}
|
||||
|
||||
export class DlqMonitor {
|
||||
private channel: any = null;
|
||||
private isRunning = false;
|
||||
private resubscribeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(private redis: Redis) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
this.channel = await getChannel();
|
||||
if (!this.channel) {
|
||||
throw new Error('Failed to get RabbitMQ channel for DLQ monitor');
|
||||
}
|
||||
|
||||
this.channel.on('close', () => {
|
||||
if (this.isRunning) {
|
||||
log.warn('[DlqMonitor] Channel closed — scheduling re-subscribe');
|
||||
this.channel = null;
|
||||
this.scheduleResubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
// Dead-letters arrive via the DEFAULT exchange with routing key
|
||||
// analysis_dlq — the queue just has to exist with that name.
|
||||
await this.channel.assertQueue(DLQ_NAME, { durable: true });
|
||||
|
||||
await this.channel.consume(
|
||||
DLQ_NAME,
|
||||
(msg: ConsumeMessage | null) => this.handleMessage(msg),
|
||||
{ noAck: false },
|
||||
);
|
||||
|
||||
this.isRunning = true;
|
||||
log.info(`[DlqMonitor] Draining ${DLQ_NAME}`);
|
||||
}
|
||||
|
||||
private async handleMessage(msg: ConsumeMessage | null): Promise<void> {
|
||||
if (!msg || !this.channel) return;
|
||||
|
||||
try {
|
||||
let sessionId = 'unknown';
|
||||
let component = 'unknown';
|
||||
let retryCount: number | undefined;
|
||||
try {
|
||||
const body = JSON.parse(msg.content.toString());
|
||||
sessionId = body.sessionId ?? sessionId;
|
||||
component = body.component ?? component;
|
||||
retryCount = body.retryCount;
|
||||
} catch {
|
||||
// non-JSON payload — record raw
|
||||
}
|
||||
|
||||
const deathInfo = msg.properties?.headers?.['x-death']?.[0];
|
||||
const entry = {
|
||||
ts: new Date().toISOString(),
|
||||
sessionId,
|
||||
component,
|
||||
retryCount,
|
||||
sourceQueue: deathInfo?.queue,
|
||||
reason: deathInfo?.reason,
|
||||
};
|
||||
|
||||
log.error(`[DlqMonitor] DEAD_LETTER session=${sessionId} component=${component} source=${entry.sourceQueue ?? '?'} reason=${entry.reason ?? '?'}`);
|
||||
dlqMessages.inc({ component });
|
||||
|
||||
await this.redis
|
||||
.multi()
|
||||
.lpush(DLQ_RECENT_KEY, JSON.stringify(entry))
|
||||
.ltrim(DLQ_RECENT_KEY, 0, DLQ_RECENT_MAX - 1)
|
||||
.exec();
|
||||
} catch (err) {
|
||||
log.error('[DlqMonitor] Failed to record dead-letter:', (err as Error).message);
|
||||
} finally {
|
||||
// Always ack — the DLQ is a terminal sink; requeueing would loop forever.
|
||||
try { this.channel?.ack(msg); } catch { /* channel died mid-ack */ }
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleResubscribe(): void {
|
||||
if (!this.isRunning || this.resubscribeTimer) return;
|
||||
const RETRY_MS = 5000;
|
||||
const attempt = async (): Promise<void> => {
|
||||
this.resubscribeTimer = null;
|
||||
if (!this.isRunning) return;
|
||||
try {
|
||||
await this.start();
|
||||
log.info('[DlqMonitor] Re-subscribed after connection loss');
|
||||
} catch (err) {
|
||||
log.warn(`[DlqMonitor] Re-subscribe failed (${(err as Error).message}), retrying in ${RETRY_MS}ms`);
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
};
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.isRunning = false;
|
||||
if (this.channel) {
|
||||
// Shared channel — the aggregator owns closing it; just drop our ref.
|
||||
this.channel = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,707 @@
|
|||
/**
|
||||
* MediaPreprocessWorker - Pre-processes video/audio/image ONCE per session.
|
||||
*
|
||||
* Runs BEFORE component workers. Downloads media, extracts frames (ffmpeg),
|
||||
* transcribes audio (Whisper), runs Vision API (2 prompts), and caches
|
||||
* all results in Redis. Then dispatches component tasks.
|
||||
*
|
||||
* This eliminates 3x duplicate downloads, 3x ffmpeg, 3x transcription,
|
||||
* and 2x redundant Vision calls that happened when each component worker
|
||||
* processed media independently.
|
||||
*
|
||||
* Redis cache keys (TTL 1h):
|
||||
* agent:media:{sessionId}:transcript - audio transcript
|
||||
* agent:media:{sessionId}:vision:misinformation - frame analysis (manipulation)
|
||||
* agent:media:{sessionId}:vision:ai_detection - frame analysis (AI detection)
|
||||
* agent:media:{sessionId}:merged_text - transcript + vision misinformation
|
||||
* agent:media:{sessionId}:ready - "1" signal for component workers
|
||||
*/
|
||||
|
||||
import { getChannel, getConfirmChannel, publishConfirmed } from '../connection';
|
||||
import {
|
||||
EXCHANGE_NAME,
|
||||
getQueueName,
|
||||
MEDIA_QUEUE,
|
||||
WORKER_CONFIG,
|
||||
type PlanType,
|
||||
type AnalysisComponent,
|
||||
ANALYSIS_COMPONENTS,
|
||||
} from '../../shared/queue/constants';
|
||||
import { MediaCacheKeys, QueueKeys, REDIS_TTL } from '../../shared/redis/keys';
|
||||
import type { AnalysisTaskMessage } from '../types';
|
||||
import { processVideoUrl, type VideoAnalysisResult } from '../../shared/media/video-processor';
|
||||
import { extractUrlMetadata } from '../../shared/media/url-metadata';
|
||||
import { transcribe, type TranscriptionResult } from '../../shared/media/transcription';
|
||||
import { callVision } from '../../shared/media/vision';
|
||||
import { analyzeBuster } from '../../shared/media/buster';
|
||||
import { analyzeForensic, formatForensicForVisionPrompt } from '../../shared/media/forensic';
|
||||
import { analyzeMetadata, formatMetadataForPrompt, analyzeNer, analyzeSentiment, formatFeaturesForPrompt } from '../../shared/media/extractors';
|
||||
import { getSearchTier } from '../../shared/credits';
|
||||
import { sanitizeUtf8 } from '../../config/analysisLimits';
|
||||
import { ConfigKeys } from '../../shared/redis/keys';
|
||||
import type Redis from 'ioredis';
|
||||
import { createRedisConnection } from '../../shared/redis/connection';
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import { log } from '../../shared/logger';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
// Duration limits (seconds)
|
||||
const MAX_VIDEO_DURATION = 180; // 3 minutes
|
||||
const MAX_AUDIO_DURATION = 420; // 7 minutes
|
||||
|
||||
// Re-use the same task message type from component workers
|
||||
// The dispatcher will send the same shape, just to a different queue
|
||||
|
||||
interface ConsumeMessage {
|
||||
content: Buffer;
|
||||
fields: any;
|
||||
properties: any;
|
||||
}
|
||||
|
||||
/** What the media preprocess worker produces */
|
||||
export interface MediaPreprocessResult {
|
||||
transcript: string;
|
||||
vision_misinformation: string;
|
||||
vision_ai_detection: string;
|
||||
merged_text: string;
|
||||
frames_extracted: number;
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export class MediaPreprocessWorker {
|
||||
private channel: any = null;
|
||||
private redis: Redis | null = null;
|
||||
private isRunning = false;
|
||||
|
||||
private resubscribeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private prefetch: number = 2,
|
||||
private redisInstance?: Redis,
|
||||
) {}
|
||||
|
||||
/** Re-subscribe after connection loss — see ComponentWorker for rationale. */
|
||||
private scheduleResubscribe(): void {
|
||||
if (!this.isRunning || this.resubscribeTimer) return;
|
||||
const RETRY_MS = 5000;
|
||||
const attempt = async (): Promise<void> => {
|
||||
this.resubscribeTimer = null;
|
||||
if (!this.isRunning) return;
|
||||
try {
|
||||
await this.start();
|
||||
log.info('[MediaPreprocess] Re-subscribed after connection loss');
|
||||
} catch (err) {
|
||||
log.warn(`[MediaPreprocess] Re-subscribe failed (${(err as Error).message}), retrying in ${RETRY_MS}ms`);
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
};
|
||||
this.resubscribeTimer = setTimeout(attempt, RETRY_MS);
|
||||
}
|
||||
|
||||
private getRedis(): Redis {
|
||||
if (this.redisInstance) return this.redisInstance;
|
||||
if (!this.redis) {
|
||||
this.redis = createRedisConnection({ label: 'media-preprocess-worker' });
|
||||
}
|
||||
return this.redis;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
log.info('[MediaPreprocess] Starting...');
|
||||
|
||||
this.channel = await getChannel();
|
||||
if (!this.channel) {
|
||||
throw new Error('Failed to get RabbitMQ channel');
|
||||
}
|
||||
|
||||
this.channel.on('close', () => {
|
||||
if (this.isRunning) {
|
||||
log.warn('[MediaPreprocess] Channel closed — scheduling re-subscribe');
|
||||
this.channel = null;
|
||||
this.scheduleResubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
await this.channel.prefetch(this.prefetch);
|
||||
|
||||
const planTypes: PlanType[] = [1, 2, 3, 4, 5, 6];
|
||||
|
||||
for (const planType of planTypes) {
|
||||
const queueName = MEDIA_QUEUE.queueName(planType);
|
||||
|
||||
await this.channel.assertQueue(queueName, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-max-priority': WORKER_CONFIG.MAX_PRIORITY,
|
||||
'x-dead-letter-exchange': '',
|
||||
'x-dead-letter-routing-key': 'analysis_dlq',
|
||||
'x-message-ttl': WORKER_CONFIG.MESSAGE_TTL,
|
||||
},
|
||||
});
|
||||
|
||||
await this.channel.bindQueue(queueName, EXCHANGE_NAME, queueName);
|
||||
|
||||
await this.channel.consume(
|
||||
queueName,
|
||||
(msg: ConsumeMessage | null) => this.handleMessage(msg),
|
||||
{ noAck: false },
|
||||
);
|
||||
|
||||
log.info(`[MediaPreprocess] Consuming from ${queueName}`);
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
log.info(`[MediaPreprocess] Started, prefetch: ${this.prefetch}`);
|
||||
}
|
||||
|
||||
private async handleMessage(msg: ConsumeMessage | null): Promise<void> {
|
||||
if (!msg || !this.channel) return;
|
||||
|
||||
const startTime = Date.now();
|
||||
let taskMessage: AnalysisTaskMessage | null = null;
|
||||
|
||||
try {
|
||||
taskMessage = JSON.parse(msg.content.toString()) as AnalysisTaskMessage;
|
||||
const { sessionId, input } = taskMessage;
|
||||
const inputType = input.inputType || 'text';
|
||||
|
||||
log.info(`[MediaPreprocess] Processing ${sessionId} (type: ${inputType})`);
|
||||
|
||||
// Cancel check — skip the expensive extraction (yt-dlp/Whisper/vision)
|
||||
// and dispatch components directly: they short-circuit on the same flag
|
||||
// and publish canceled results, so the aggregator finalizes the session.
|
||||
if (await this.getRedis().get(QueueKeys.cancelFlag(sessionId))) {
|
||||
log.info(`[MediaPreprocess] ${sessionId} canceled — skipping extraction`);
|
||||
await this.dispatchComponentTasks(taskMessage);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Acquire lock to prevent duplicate processing
|
||||
const lockKey = MediaCacheKeys.lock(sessionId);
|
||||
const redisClient = this.getRedis();
|
||||
const acquired = await redisClient.set(lockKey, '1', 'EX', WORKER_CONFIG.LOCK_TTL_COMPONENT, 'NX');
|
||||
|
||||
if (!acquired) {
|
||||
log.warn(`[MediaPreprocess] Lock exists for ${sessionId}, skipping`);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
const mediaSource = input.url || input.mediaPath;
|
||||
|
||||
if (!mediaSource) {
|
||||
log.error(`[MediaPreprocess] No media source for ${sessionId}`);
|
||||
// Dispatch components anyway — they'll handle missing text
|
||||
await this.dispatchComponentTasks(taskMessage);
|
||||
this.channel.ack(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// PROCESS MEDIA (one time, for all components)
|
||||
// ================================================================
|
||||
|
||||
const tier = getSearchTier(taskMessage.planType);
|
||||
|
||||
if (inputType === 'video') {
|
||||
await this.processVideo(sessionId, mediaSource, redisClient, tier);
|
||||
} else if (inputType === 'audio') {
|
||||
await this.processAudio(sessionId, mediaSource, redisClient, tier);
|
||||
} else if (inputType === 'image') {
|
||||
await this.processImage(sessionId, mediaSource, redisClient, tier);
|
||||
}
|
||||
|
||||
// Signal that pre-processing is complete
|
||||
await redisClient.setex(
|
||||
MediaCacheKeys.ready(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
'1',
|
||||
);
|
||||
|
||||
log.info(`[MediaPreprocess] Cached results for ${sessionId} in ${Date.now() - startTime}ms`);
|
||||
|
||||
// ================================================================
|
||||
// DISPATCH COMPONENT TASKS
|
||||
// ================================================================
|
||||
|
||||
await this.dispatchComponentTasks(taskMessage);
|
||||
|
||||
this.channel.ack(msg);
|
||||
|
||||
log.info(`[MediaPreprocess] Completed ${sessionId} in ${Date.now() - startTime}ms`);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
log.error(`[MediaPreprocess] Error:`, error.message);
|
||||
|
||||
// On failure, still dispatch components — they have fallback processing
|
||||
if (taskMessage) {
|
||||
try {
|
||||
await this.dispatchComponentTasks(taskMessage);
|
||||
} catch (dispatchErr) {
|
||||
log.error(`[MediaPreprocess] Failed to dispatch after error:`, (dispatchErr as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
this.channel.ack(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// ======================================================================
|
||||
// VIDEO: download → ffmpeg frames → ffmpeg audio → transcribe → 2× vision
|
||||
// ======================================================================
|
||||
|
||||
private async processVideo(
|
||||
sessionId: string,
|
||||
videoSource: string,
|
||||
redis: Redis,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<void> {
|
||||
log.info(`[MediaPreprocess] Video: downloading + processing (tier: ${tier}): ${videoSource}`);
|
||||
|
||||
// Run video pipeline + metadata extraction in parallel.
|
||||
// Metadata gives us caption/description even when full download fails or
|
||||
// when audio/visual lacks the surrounding context (hashtags, sponsored
|
||||
// disclosures, etc.) — both signals matter for the analysis.
|
||||
const isPlatformUrl = /^https?:/i.test(videoSource)
|
||||
&& /youtube|youtu\.be|tiktok|twitter|x\.com|instagram|facebook|fb\.watch|vimeo|dailymotion|twitch/i.test(videoSource);
|
||||
|
||||
const [videoOutcome, metadataOutcome] = await Promise.allSettled([
|
||||
processVideoUrl(videoSource, sessionId, {
|
||||
redis,
|
||||
logPrefix: 'MediaPreprocess',
|
||||
visionContexts: ['misinformation', 'ai_detection'],
|
||||
tier,
|
||||
}),
|
||||
isPlatformUrl ? extractUrlMetadata(videoSource) : Promise.resolve(null),
|
||||
]);
|
||||
|
||||
let result: VideoAnalysisResult;
|
||||
if (videoOutcome.status === 'fulfilled') {
|
||||
result = videoOutcome.value;
|
||||
} else {
|
||||
log.warn(`[MediaPreprocess] Video pipeline failed: ${(videoOutcome.reason as Error)?.message}`);
|
||||
result = {
|
||||
transcript: '',
|
||||
visual_analysis: '',
|
||||
merged_text: '',
|
||||
frames_extracted: 0,
|
||||
duration_ms: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const metadata = metadataOutcome.status === 'fulfilled' ? metadataOutcome.value : null;
|
||||
|
||||
// Cache transcript
|
||||
if (result.transcript) {
|
||||
await redis.setex(
|
||||
MediaCacheKeys.transcript(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
result.transcript,
|
||||
);
|
||||
log.info(`[MediaPreprocess] Cached transcript: ${result.transcript.length} chars`);
|
||||
}
|
||||
|
||||
// Cache vision outputs per context
|
||||
if (result.visual_analyses) {
|
||||
for (const [ctx, analysis] of Object.entries(result.visual_analyses)) {
|
||||
if (analysis) {
|
||||
await redis.setex(
|
||||
MediaCacheKeys.vision(sessionId, ctx),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
analysis,
|
||||
);
|
||||
log.info(`[MediaPreprocess] Cached vision(${ctx}): ${analysis.length} chars`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// BusterX + Forensic Features (specialized) — fail-open, run în paralel
|
||||
// pe acelaşi videoBuffer pentru economie I/O.
|
||||
//
|
||||
// BusterX: deepfake video classifier (Qwen2.5-VL-7B fine-tuned). Opt-in
|
||||
// via env BUSTER_ENABLED=true.
|
||||
// Forensic: m25-m29 measurements (puls, lip-sync, NPR, blending heatmap,
|
||||
// lighting 3D). Output text + PNG heatmap-uri pe care LLM ai-tampered
|
||||
// le primește prin Vision Cascade.
|
||||
//
|
||||
// Ambele rulează DOAR pentru direct video URLs (NOT yt-dlp platform).
|
||||
// ai-tampered executor degradează grațios dacă output-ul lipseşte.
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
const busterEnabled = process.env.BUSTER_ENABLED === 'true';
|
||||
const forensicEnabled = process.env.FORENSIC_ENABLED !== 'false'; // default ON
|
||||
const extractorsEnabled = process.env.EXTRACTORS_ENABLED !== 'false'; // default ON
|
||||
const needsVideoBuffer = (busterEnabled || forensicEnabled || extractorsEnabled)
|
||||
&& result.frames_extracted > 0
|
||||
&& !isPlatformUrl
|
||||
&& /^https?:/i.test(videoSource);
|
||||
|
||||
if (needsVideoBuffer) {
|
||||
try {
|
||||
const vResp = await fetch(videoSource, { signal: AbortSignal.timeout(60000) });
|
||||
if (vResp.ok) {
|
||||
const videoBuffer = Buffer.from(await vResp.arrayBuffer());
|
||||
const filename = videoSource.split('/').pop()?.split('?')[0] || 'video.mp4';
|
||||
|
||||
// Run în paralel — Buster (opt-in) + Forensic (default on) + Metadata (default on)
|
||||
const [buster, forensic, fileMeta] = await Promise.all([
|
||||
busterEnabled ? analyzeBuster(videoBuffer, filename).catch(() => null) : Promise.resolve(null),
|
||||
forensicEnabled ? analyzeForensic(videoBuffer, filename, {
|
||||
modules: tier === 'premium'
|
||||
? ['m25', 'm26', 'm27', 'm28', 'm29'] // toate cele 5 pe premium
|
||||
: ['m27', 'm28'], // doar core pe free (economie CPU)
|
||||
encodeImages: true,
|
||||
}).catch(() => null) : Promise.resolve(null),
|
||||
extractorsEnabled ? analyzeMetadata(videoBuffer, filename).catch(() => null) : Promise.resolve(null),
|
||||
]);
|
||||
|
||||
if (buster) {
|
||||
await redis.setex(
|
||||
MediaCacheKeys.busterVerdict(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
JSON.stringify(buster),
|
||||
);
|
||||
log.info(`[MediaPreprocess] BusterX: ${buster.verdict} (${buster.latency_ms}ms)`);
|
||||
} else if (busterEnabled) {
|
||||
// BusterX era activat dar NU a returnat rezultat (timeout / eroare / 5xx).
|
||||
// Scriem o santinelă explicită INDISPONIBIL, ca track 3 din ai-tampered să
|
||||
// semnaleze degradarea (indicator + verificare manuală) în loc să o ascundă
|
||||
// producând un verdict care pare complet. Vezi video-2-track.ts.
|
||||
await redis.setex(
|
||||
MediaCacheKeys.busterVerdict(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
JSON.stringify({
|
||||
verdict: 'UNAVAILABLE',
|
||||
explanation: 'Serviciul BusterX (detecție deepfake) nu a returnat un rezultat — verificarea deepfake a fost indisponibilă pentru acest video.',
|
||||
}),
|
||||
);
|
||||
log.warn('[MediaPreprocess] BusterX activat dar fără rezultat — marcat UNAVAILABLE (degradare vizibilă)');
|
||||
}
|
||||
|
||||
if (forensic) {
|
||||
// Cache JSON full (pentru audit/debug)
|
||||
await redis.setex(
|
||||
MediaCacheKeys.forensicEvidence(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
JSON.stringify({
|
||||
summary: forensic.summary,
|
||||
modules: forensic.modules,
|
||||
fusion: forensic.fusion,
|
||||
instruction_for_llm: forensic.instruction_for_llm,
|
||||
execution_time_ms: forensic.execution_time_ms,
|
||||
modules_run: forensic.modules_run,
|
||||
auto_skipped: forensic.auto_skipped,
|
||||
}),
|
||||
);
|
||||
// Cache imagini base64 separat (pentru deep-analysis Vision pass 2)
|
||||
if (forensic.images?.length) {
|
||||
await redis.setex(
|
||||
MediaCacheKeys.forensicImages(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
JSON.stringify(forensic.images),
|
||||
);
|
||||
}
|
||||
// Append evidence_text la vision:ai_detection cache, astfel încât
|
||||
// LLM ai-tampered downstream să vadă măsurătorile în context automat.
|
||||
const aiDetectionKey = MediaCacheKeys.vision(sessionId, 'ai_detection');
|
||||
const existingAi = await redis.get(aiDetectionKey);
|
||||
const enrichedAi = (existingAi || '') + formatForensicForVisionPrompt(forensic);
|
||||
await redis.setex(aiDetectionKey, REDIS_TTL.INTERMEDIATE, enrichedAi);
|
||||
log.info(`[MediaPreprocess] Forensic: ${forensic.summary.overall_label} score=${forensic.summary.overall_score} modules=${forensic.modules_run.length} images=${forensic.images.length} (${forensic.execution_time_ms}ms)`);
|
||||
}
|
||||
|
||||
if (fileMeta) {
|
||||
// Cache JSON full (audit/debug + downstream persistence)
|
||||
await redis.setex(
|
||||
MediaCacheKeys.mediaMetadata(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
JSON.stringify(fileMeta),
|
||||
);
|
||||
// Append metadata/integrity segment la vision:ai_detection, la fel ca
|
||||
// forensic — LLM ai-tampered downstream vede semnalele EXIF/ELA/C2PA
|
||||
// în context automat (fail-open dacă nu există semnal).
|
||||
const segment = formatMetadataForPrompt(fileMeta);
|
||||
if (segment) {
|
||||
const aiDetectionKey = MediaCacheKeys.vision(sessionId, 'ai_detection');
|
||||
const existingAi = await redis.get(aiDetectionKey);
|
||||
await redis.setex(aiDetectionKey, REDIS_TTL.INTERMEDIATE, (existingAi || '') + segment);
|
||||
}
|
||||
log.info(`[MediaPreprocess] Metadata: ${fileMeta.media_type} sha=${fileMeta.sha256.slice(0, 12)} anomalies=${fileMeta.anomalies.length} evidence=${fileMeta.evidence.length} (${fileMeta.execution_time_ms}ms)`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`[MediaPreprocess] Buster/Forensic call failed (fail-open): ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Build merged text: caption/metadata FIRST (gives platform context like
|
||||
// hashtags, sponsored disclosures, post date), then transcript + visual.
|
||||
const parts: string[] = [];
|
||||
if (metadata?.combined_text) {
|
||||
const meta = metadata.uploader
|
||||
? `[POST METADATA — ${metadata.uploader}]\n${metadata.combined_text}`
|
||||
: `[POST METADATA]\n${metadata.combined_text}`;
|
||||
parts.push(meta);
|
||||
}
|
||||
if (result.merged_text) parts.push(result.merged_text);
|
||||
const mergedText = parts.join('\n\n');
|
||||
|
||||
if (mergedText) {
|
||||
// Text extractors (NER + sentiment) — entities to fact-check + emotional
|
||||
// tone (manipulation signal). Appended la merged_text ca techniques/claims
|
||||
// să le vadă în context. Fail-open.
|
||||
let finalText = mergedText;
|
||||
if (extractorsEnabled) {
|
||||
const [ner, sentiment] = await Promise.all([
|
||||
analyzeNer(mergedText).catch(() => null),
|
||||
analyzeSentiment(mergedText).catch(() => null),
|
||||
]);
|
||||
const seg = formatFeaturesForPrompt([ner, sentiment], 'TEXT EXTRACTORS — entities (NER) + sentiment');
|
||||
if (seg) finalText += seg;
|
||||
}
|
||||
await redis.setex(
|
||||
MediaCacheKeys.mergedText(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
finalText,
|
||||
);
|
||||
const sources: string[] = [];
|
||||
if (metadata?.combined_text) sources.push(`metadata:${metadata.combined_text.length}`);
|
||||
if (result.transcript) sources.push(`transcript:${result.transcript.length}`);
|
||||
if (result.visual_analysis) sources.push(`visual:${result.visual_analysis.length}`);
|
||||
log.info(`[MediaPreprocess] Cached merged_text: ${mergedText.length} chars (${sources.join(', ')})`);
|
||||
} else {
|
||||
log.warn(`[MediaPreprocess] No text content from any source for ${sessionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ======================================================================
|
||||
// AUDIO: download → transcribe (no vision needed)
|
||||
// ======================================================================
|
||||
|
||||
private async processAudio(
|
||||
sessionId: string,
|
||||
audioSource: string,
|
||||
redis: Redis,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<void> {
|
||||
log.info(`[MediaPreprocess] Audio: transcribing (tier: ${tier}): ${audioSource}`);
|
||||
|
||||
const result: TranscriptionResult = await transcribe(audioSource, {
|
||||
logPrefix: 'MediaPreprocess',
|
||||
tier,
|
||||
});
|
||||
|
||||
if (result.success && result.text) {
|
||||
const sanitized = sanitizeUtf8(result.text);
|
||||
await redis.setex(
|
||||
MediaCacheKeys.transcript(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
sanitized,
|
||||
);
|
||||
// For audio, merged_text = transcript (no visual) — enriched cu NER+sentiment
|
||||
let finalText = sanitized;
|
||||
const extractorsEnabled = process.env.EXTRACTORS_ENABLED !== 'false';
|
||||
if (extractorsEnabled) {
|
||||
const [ner, sentiment] = await Promise.all([
|
||||
analyzeNer(sanitized).catch(() => null),
|
||||
analyzeSentiment(sanitized).catch(() => null),
|
||||
]);
|
||||
const seg = formatFeaturesForPrompt([ner, sentiment], 'TEXT EXTRACTORS — entities (NER) + sentiment');
|
||||
if (seg) finalText += seg;
|
||||
}
|
||||
await redis.setex(
|
||||
MediaCacheKeys.mergedText(sessionId),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
finalText,
|
||||
);
|
||||
log.info(`[MediaPreprocess] Cached audio transcript: ${sanitized.length} chars`);
|
||||
} else {
|
||||
log.warn(`[MediaPreprocess] Audio transcription failed: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ======================================================================
|
||||
// IMAGE: 2× vision (OCR extraction + AI detection)
|
||||
// ======================================================================
|
||||
|
||||
private async processImage(
|
||||
sessionId: string,
|
||||
imageUrl: string,
|
||||
redis: Redis,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<void> {
|
||||
log.info(`[MediaPreprocess] Image: running vision (tier: ${tier}): ${imageUrl}`);
|
||||
|
||||
// Vision 1: OCR / text extraction (for techniques + claims)
|
||||
await this.runImageExtraction(sessionId, imageUrl, redis, tier);
|
||||
|
||||
// Vision 2: AI detection (for ai_tampered)
|
||||
await this.runImageAiDetection(sessionId, imageUrl, redis, tier);
|
||||
}
|
||||
|
||||
private async runImageExtraction(
|
||||
sessionId: string,
|
||||
imageUrl: string,
|
||||
redis: Redis,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<void> {
|
||||
let userPrompt = 'Extract the main text content from this image. Return ONLY the actual message, post, article text, or statement visible in the image. Do NOT describe UI elements. If there is no meaningful text, respond with NO_TEXT_FOUND.';
|
||||
let systemPrompt = '';
|
||||
|
||||
try {
|
||||
const promptData = await redis.get(ConfigKeys.visionPromptExtraction);
|
||||
if (promptData) {
|
||||
const parsed = JSON.parse(promptData);
|
||||
if (parsed.system) systemPrompt = parsed.system;
|
||||
if (parsed.user_template) userPrompt = parsed.user_template;
|
||||
}
|
||||
} catch { /* use fallback */ }
|
||||
|
||||
const messages: any[] = [];
|
||||
if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: userPrompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await callVision(redis, messages, { max_tokens: 1500 }, tier);
|
||||
log.info(`[MediaPreprocess] Image OCR via ${result.provider}`);
|
||||
|
||||
if (result.content && !result.content.includes('NO_TEXT_FOUND')) {
|
||||
const sanitized = sanitizeUtf8(result.content);
|
||||
// Store as "transcript" (extracted text) and merged_text
|
||||
await redis.setex(MediaCacheKeys.transcript(sessionId), REDIS_TTL.INTERMEDIATE, sanitized);
|
||||
await redis.setex(MediaCacheKeys.mergedText(sessionId), REDIS_TTL.INTERMEDIATE, sanitized);
|
||||
await redis.setex(MediaCacheKeys.vision(sessionId, 'misinformation'), REDIS_TTL.INTERMEDIATE, sanitized);
|
||||
}
|
||||
} catch (err) {
|
||||
log.error(`[MediaPreprocess] Image OCR failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async runImageAiDetection(
|
||||
sessionId: string,
|
||||
imageUrl: string,
|
||||
redis: Redis,
|
||||
tier: 'free' | 'premium' = 'free',
|
||||
): Promise<void> {
|
||||
const prompt = `Analyze this image to determine if it was AI-generated (by DALL-E, Midjourney, Stable Diffusion, etc.) or is a real photograph/human-created image.
|
||||
|
||||
Look for these AI generation indicators:
|
||||
1. **Anatomical errors**: Extra fingers, merged hands, distorted faces, asymmetric eyes
|
||||
2. **Texture anomalies**: Overly smooth skin, plastic-like appearance, inconsistent textures
|
||||
3. **Background artifacts**: Blurred or nonsensical backgrounds, floating objects
|
||||
4. **Lighting inconsistencies**: Shadows going different directions, incorrect reflections
|
||||
5. **Text/writing errors**: Garbled text, nonsensical letters
|
||||
6. **Repetitive patterns**: Unnatural repetition in textures or elements
|
||||
7. **Watermarks/signatures**: AI tool watermarks (Midjourney, DALL-E signatures)
|
||||
8. **Style indicators**: Characteristic AI art styles, over-processed look
|
||||
9. **Edge artifacts**: Unnatural edges, halos around objects
|
||||
10. **Composition issues**: Unnatural object placement, perspective errors
|
||||
|
||||
Return JSON only:
|
||||
{
|
||||
"ai_generated_probability": 75,
|
||||
"indicators": ["extra fingers visible", "plastic skin texture", "background artifacts"],
|
||||
"evidence": "Detailed explanation of what you observed"
|
||||
}`;
|
||||
|
||||
try {
|
||||
const result = await callVision(redis, [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: prompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
}], { max_tokens: 1500, temperature: 0.2 }, tier);
|
||||
|
||||
log.info(`[MediaPreprocess] Image AI detection via ${result.provider}`);
|
||||
|
||||
await redis.setex(
|
||||
MediaCacheKeys.vision(sessionId, 'ai_detection'),
|
||||
REDIS_TTL.INTERMEDIATE,
|
||||
result.content,
|
||||
);
|
||||
} catch (err) {
|
||||
log.error(`[MediaPreprocess] Image AI detection failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ======================================================================
|
||||
// DISPATCH COMPONENT TASKS (same logic as original dispatcher)
|
||||
// ======================================================================
|
||||
|
||||
private async dispatchComponentTasks(originalTask: AnalysisTaskMessage): Promise<void> {
|
||||
const confirmChannel = await getConfirmChannel();
|
||||
if (!confirmChannel) {
|
||||
log.error('[MediaPreprocess] No confirm channel for component dispatch');
|
||||
return;
|
||||
}
|
||||
|
||||
const { sessionId, planType, priority, input } = originalTask;
|
||||
|
||||
// Dispatch only the components that were requested for this session
|
||||
// (e.g. techniques-only image analysis sends ['techniques'], full pipeline sends all 4)
|
||||
const components: AnalysisComponent[] = originalTask.requestedComponents?.length
|
||||
? originalTask.requestedComponents
|
||||
: [...ANALYSIS_COMPONENTS];
|
||||
|
||||
for (const component of components) {
|
||||
const message: AnalysisTaskMessage = {
|
||||
sessionId,
|
||||
component,
|
||||
planType,
|
||||
priority: priority || 1,
|
||||
input,
|
||||
timestamp: Date.now(),
|
||||
retryCount: 0,
|
||||
};
|
||||
|
||||
const queueName = getQueueName(component, planType as PlanType);
|
||||
|
||||
try {
|
||||
await publishConfirmed(
|
||||
confirmChannel,
|
||||
EXCHANGE_NAME,
|
||||
queueName,
|
||||
Buffer.from(JSON.stringify(message)),
|
||||
{
|
||||
persistent: true,
|
||||
priority: priority || 1,
|
||||
contentType: 'application/json',
|
||||
headers: { sessionId, component, planType },
|
||||
},
|
||||
);
|
||||
log.info(`[MediaPreprocess] Dispatched ${component} for ${sessionId}`);
|
||||
} catch (err) {
|
||||
// Missing component = aggregator never reaches N/N → session hangs
|
||||
// until zombie cleanup. Log loudly with ids for triage.
|
||||
log.error(`[MediaPreprocess] DISPATCH_LOST session=${sessionId} component=${component}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
log.info('[MediaPreprocess] Stopping...');
|
||||
this.isRunning = false;
|
||||
|
||||
if (this.channel) {
|
||||
await this.channel.close();
|
||||
this.channel = null;
|
||||
}
|
||||
|
||||
if (this.redis) {
|
||||
this.redis.disconnect();
|
||||
this.redis = null;
|
||||
}
|
||||
|
||||
log.info('[MediaPreprocess] Stopped');
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/**
|
||||
* Techniques Worker - config-only subclass of ComponentWorker.
|
||||
*/
|
||||
import { ComponentWorker } from './component-worker';
|
||||
import type { ComponentRunner } from '../../components/component-runner';
|
||||
|
||||
export class TechniquesWorker extends ComponentWorker {
|
||||
constructor(runner: ComponentRunner) {
|
||||
super('techniques', runner, 5);
|
||||
}
|
||||
}
|
||||
|
||||
export default TechniquesWorker;
|
||||
Loading…
Add table
Add a link
Reference in a new issue