239 lines
8.7 KiB
TypeScript
239 lines
8.7 KiB
TypeScript
/**
|
|
* Redis cluster readiness verification.
|
|
*
|
|
* Runs a battery of checks against the Redis instance configured via env vars
|
|
* (REDIS_HOST/PORT/USERNAME/PASSWORD/DB). Intended to be run:
|
|
* - Pre-cutover: confirm the new cluster has all required keys before switching env
|
|
* - Post-cutover smoke test: confirm nothing got lost
|
|
* - CI / healthcheck: block container start until cluster is ready
|
|
*
|
|
* Exit codes:
|
|
* 0 — all checks pass
|
|
* 1 — at least one FATAL check fails (connectivity, missing framework keys)
|
|
* 2 — only WARNING-level issues (missing optional keys, empty scan results)
|
|
*
|
|
* Usage:
|
|
* REDIS_HOST=10.11.50.100 REDIS_PORT=16379 REDIS_USERNAME=didi \
|
|
* REDIS_PASSWORD=... npx tsx scripts/verify-redis-cluster.ts
|
|
*/
|
|
|
|
import { createRedisConnection } from '../src/shared/redis/connection';
|
|
import { FrameworkKeys, ConfigKeys } from '../src/shared/redis/keys';
|
|
|
|
type Severity = 'OK' | 'WARN' | 'FAIL';
|
|
interface CheckResult {
|
|
name: string;
|
|
severity: Severity;
|
|
detail: string;
|
|
}
|
|
|
|
const results: CheckResult[] = [];
|
|
let fatalCount = 0;
|
|
let warnCount = 0;
|
|
|
|
function record(name: string, severity: Severity, detail: string) {
|
|
results.push({ name, severity, detail });
|
|
if (severity === 'FAIL') fatalCount++;
|
|
else if (severity === 'WARN') warnCount++;
|
|
}
|
|
|
|
async function main() {
|
|
const host = process.env.REDIS_HOST || 'didi-cache';
|
|
const port = process.env.REDIS_PORT || '6379';
|
|
const user = process.env.REDIS_USERNAME || '(legacy-auth)';
|
|
|
|
console.log(`\n${'='.repeat(70)}`);
|
|
console.log(`Redis cluster readiness check — ${host}:${port} (user: ${user})`);
|
|
console.log('='.repeat(70));
|
|
|
|
const redis = createRedisConnection({
|
|
label: 'verify-cluster',
|
|
overrides: { maxRetriesPerRequest: 3 },
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 1. Connectivity + auth
|
|
// -------------------------------------------------------------------------
|
|
try {
|
|
const pong = await redis.ping();
|
|
if (pong !== 'PONG') throw new Error(`unexpected PING reply: ${pong}`);
|
|
record('connectivity', 'OK', 'PING → PONG');
|
|
} catch (err: any) {
|
|
record('connectivity', 'FAIL', `cannot reach cluster: ${err.message}`);
|
|
await finalize(redis);
|
|
return;
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 2. Server info (master/replica, memory)
|
|
// -------------------------------------------------------------------------
|
|
try {
|
|
const info = await redis.info('replication');
|
|
const role = /role:(\w+)/.exec(info)?.[1] || 'unknown';
|
|
const slaves = /connected_slaves:(\d+)/.exec(info)?.[1] || '0';
|
|
record('replication', role === 'master' ? 'OK' : 'WARN',
|
|
`role=${role}, connected_slaves=${slaves}`);
|
|
} catch (err: any) {
|
|
record('replication', 'WARN', `INFO replication failed: ${err.message}`);
|
|
}
|
|
|
|
try {
|
|
const mem = await redis.info('memory');
|
|
const used = /used_memory_human:(\S+)/.exec(mem)?.[1] || '?';
|
|
const max = /maxmemory_human:(\S+)/.exec(mem)?.[1] || '?';
|
|
const policy = /maxmemory_policy:(\S+)/.exec(mem)?.[1] || '?';
|
|
record('memory', 'OK', `used=${used}, max=${max}, policy=${policy}`);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3. Framework keys (REQUIRED — agent-v3 fails without these)
|
|
// -------------------------------------------------------------------------
|
|
const requiredFrameworkKeys = [
|
|
FrameworkKeys.manifest,
|
|
FrameworkKeys.techniques,
|
|
FrameworkKeys.sources,
|
|
FrameworkKeys.claims,
|
|
FrameworkKeys.verdicts,
|
|
FrameworkKeys.weights,
|
|
FrameworkKeys.dimensionsCompact,
|
|
];
|
|
|
|
for (const key of requiredFrameworkKeys) {
|
|
const exists = await redis.exists(key);
|
|
if (exists) {
|
|
const val = await redis.get(key);
|
|
const size = val ? val.length : 0;
|
|
record(`framework:${key.split(':').pop()}`, 'OK', `present (${size} bytes)`);
|
|
} else {
|
|
record(`framework:${key.split(':').pop()}`, 'FAIL', `MISSING: ${key}`);
|
|
}
|
|
}
|
|
|
|
// Optional
|
|
const providers = await redis.exists(FrameworkKeys.providers);
|
|
record('framework:providers', providers ? 'OK' : 'WARN',
|
|
providers ? 'present (optional)' : 'absent (optional, OK)');
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 4. Component config keys (REQUIRED for tier routing)
|
|
// -------------------------------------------------------------------------
|
|
const stageAssignmentKeys = [
|
|
{ label: 'techniques', key: ConfigKeys.techniquesStageAssignments },
|
|
{ label: 'ai-tampered', key: ConfigKeys.aiTamperedStageAssignments },
|
|
{ label: 'claims', key: ConfigKeys.claimsStageAssignments },
|
|
{ label: 'source-assessment', key: ConfigKeys.sourceAssessmentStageAssignments },
|
|
{ label: 'vision', key: 'didi:config:vision:v1:stage_assignments' },
|
|
{ label: 'verdict', key: 'didi:config:verdict:v1:stage_assignments' },
|
|
];
|
|
|
|
for (const { label, key } of stageAssignmentKeys) {
|
|
const val = await redis.get(key);
|
|
if (!val) {
|
|
record(`stage_assignments:${label}`, 'FAIL', `MISSING: ${key}`);
|
|
continue;
|
|
}
|
|
try {
|
|
const parsed = JSON.parse(val);
|
|
const stages = Object.keys(parsed);
|
|
if (stages.length === 0) {
|
|
record(`stage_assignments:${label}`, 'FAIL', `empty object in ${key}`);
|
|
continue;
|
|
}
|
|
// Verify tier-nested structure on first stage
|
|
const firstStage = parsed[stages[0]];
|
|
const hasFree = firstStage?.free?.models && Array.isArray(firstStage.free.models);
|
|
const hasPremium = firstStage?.premium?.models && Array.isArray(firstStage.premium.models);
|
|
if (hasFree && hasPremium) {
|
|
record(`stage_assignments:${label}`, 'OK',
|
|
`${stages.length} stage(s), tier-nested (free+premium) ✓`);
|
|
} else if (hasFree) {
|
|
record(`stage_assignments:${label}`, 'WARN',
|
|
`${stages.length} stage(s), free only (premium missing — fallback works)`);
|
|
} else {
|
|
record(`stage_assignments:${label}`, 'FAIL',
|
|
`${stages.length} stage(s), not tier-nested — old format? Re-run sync-redis.`);
|
|
}
|
|
} catch (err: any) {
|
|
record(`stage_assignments:${label}`, 'FAIL', `invalid JSON: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
// Available models (union, used for UI)
|
|
const availableModelKeys = [
|
|
ConfigKeys.techniquesAvailableModels,
|
|
ConfigKeys.aiTamperedAvailableModels,
|
|
ConfigKeys.claimsAvailableModels,
|
|
ConfigKeys.sourceAssessmentAvailableModels,
|
|
];
|
|
let missingModels = 0;
|
|
for (const key of availableModelKeys) {
|
|
if (!(await redis.exists(key))) missingModels++;
|
|
}
|
|
record('available_models',
|
|
missingModels === 0 ? 'OK' : missingModels < availableModelKeys.length ? 'WARN' : 'FAIL',
|
|
`${availableModelKeys.length - missingModels}/${availableModelKeys.length} keys present`);
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 5. Pipeline config + input profiles (REQUIRED by verdict calculator)
|
|
// -------------------------------------------------------------------------
|
|
const pipelineKeys = [
|
|
ConfigKeys.pipelineComponentConfig,
|
|
ConfigKeys.pipelineSessionConfig,
|
|
ConfigKeys.pipelineInputProfiles,
|
|
];
|
|
for (const key of pipelineKeys) {
|
|
const exists = await redis.exists(key);
|
|
const label = key.split(':').slice(-1)[0];
|
|
record(`pipeline:${label}`, exists ? 'OK' : 'FAIL',
|
|
exists ? 'present' : `MISSING: ${key}`);
|
|
}
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 6. Aggregate stats
|
|
// -------------------------------------------------------------------------
|
|
try {
|
|
const dbsize = await redis.dbsize();
|
|
record('dbsize', 'OK', `${dbsize} keys total`);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
await finalize(redis);
|
|
}
|
|
|
|
async function finalize(redis: any) {
|
|
redis.disconnect();
|
|
|
|
// Pretty print
|
|
console.log('');
|
|
for (const r of results) {
|
|
const badge =
|
|
r.severity === 'OK' ? ' ✓ ' :
|
|
r.severity === 'WARN' ? ' ⚠ ' : ' ✘ ';
|
|
console.log(`${badge} [${r.severity.padEnd(4)}] ${r.name.padEnd(35)} ${r.detail}`);
|
|
}
|
|
|
|
console.log('');
|
|
console.log('='.repeat(70));
|
|
const passed = results.filter(r => r.severity === 'OK').length;
|
|
console.log(`Results: ${passed} OK · ${warnCount} WARN · ${fatalCount} FAIL`);
|
|
console.log('='.repeat(70));
|
|
|
|
if (fatalCount > 0) {
|
|
console.log(`\n✘ NOT READY — ${fatalCount} fatal issue(s). Run sync-redis against this cluster before cutover.\n`);
|
|
process.exit(1);
|
|
}
|
|
if (warnCount > 0) {
|
|
console.log(`\n⚠ READY with warnings — ${warnCount} non-blocking issue(s).\n`);
|
|
process.exit(2);
|
|
}
|
|
console.log(`\n✓ CLUSTER READY — safe to cutover.\n`);
|
|
process.exit(0);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error('Unhandled error:', err);
|
|
process.exit(1);
|
|
});
|