From e65bd3919e65957525042490bdb9204ca6c838c6 Mon Sep 17 00:00:00 2001 From: Jaycob Date: Tue, 29 Sep 2026 20:03:04 +0100 Subject: [PATCH 1/4] fix: Health /ready endpoint does not check Redis connectivity cau (#1443) --- backend/schema-snapshots/get-_ready.json | 28 ++++++++++- backend/src/__tests__/healthReady.test.ts | 45 +++++++++++++++++ backend/src/healthProbe.ts | 18 +++++-- backend/src/redisCache.ts | 60 +++++++++++++++++++++++ 4 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 backend/src/__tests__/healthReady.test.ts diff --git a/backend/schema-snapshots/get-_ready.json b/backend/schema-snapshots/get-_ready.json index ce8460efd..ee89c9ffb 100644 --- a/backend/schema-snapshots/get-_ready.json +++ b/backend/schema-snapshots/get-_ready.json @@ -24,6 +24,31 @@ }, "indexer": { "type": "boolean" + }, + "redis": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "up", + "down" + ] + }, + "optional": { + "type": "boolean" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": "string" + } + }, + "required": [ + "status" + ], + "additionalProperties": false } }, "required": [ @@ -31,7 +56,8 @@ "stellarRpc", "database", "prisma", - "indexer" + "indexer", + "redis" ], "additionalProperties": false } diff --git a/backend/src/__tests__/healthReady.test.ts b/backend/src/__tests__/healthReady.test.ts new file mode 100644 index 000000000..bca0251ac --- /dev/null +++ b/backend/src/__tests__/healthReady.test.ts @@ -0,0 +1,45 @@ +import express from 'express'; +import request from 'supertest'; +import healthRouter, { setRedisClient } from '../routes/health'; + +const mockPing = jest.fn(); + +beforeEach(() => { + setRedisClient({ ping: mockPing }); +}); + +afterEach(() => { + setRedisClient(null); +}); + +describe('GET /ready', () => { + let app: express.Express; + + beforeEach(() => { + app = express(); + app.use(healthRouter); + jest.clearAllMocks(); + mockPing.mockResolvedValue('PONG'); + }); + + it('returns 200 when all dependencies are up', async () => { + const res = await request(app).get('/ready'); + expect(res.status).toBe[200); + expect(res.body.ready).toBe(true); + expect(res.body.checks.redis.status).toBe('up'); + }); + + it('returns 503 when redis.ping throws', async () => { + mockPing.mockRejectedValue(new Error('ECONNREFUSED')); + const res = await request(app).get('/ready'); + expect(res.status).toBe(503); + expect(res.body.checks.redis.status).toBe('down'); + }); + + it('returns 503 when redis.ping times out', async () => { + mockPing.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve('PONG'), 1000))); + const res = await request(app).get('/ready'); + expect(res.status).toBe(503); + expect(res.body.checks.redis.status).toBe('down'); + }); +}); diff --git a/backend/src/healthProbe.ts b/backend/src/healthProbe.ts index e30a4d1b2..e2f1243fb 100644 --- a/backend/src/healthProbe.ts +++ b/backend/src/healthProbe.ts @@ -8,23 +8,25 @@ export interface DependencyProbeState { lastError: string | null; lastErrorAt: string | null; consecutiveFailures: number; + optional?: boolean; } -export type DependencyName = 'database' | 'cache' | 'stellarRpc' | 'prisma' | 'queue' | 'indexer'; +export type DependencyName = 'database' | 'cache' | 'stellarRpc' | 'prisma' | 'queue' | 'indexer' | 'redis'; type ProbeFunction = () => Promise<'up' | 'down'>; interface ProbeRegistration { name: DependencyName; probe: ProbeFunction; + optional?: boolean; } class HealthProbeService { private probes = new Map(); private states = new Map(); - register(name: DependencyName, probe: ProbeFunction): void { - this.probes.set(name, { name, probe }); + register(name: DependencyName, probe: ProbeFunction, options?: { optional?: boolean }): void { + this.probes.set(name, { name, probe, optional: options?.optional }); if (!this.states.has(name)) { this.states.set(name, { status: 'up', @@ -33,7 +35,11 @@ class HealthProbeService { lastError: null, lastErrorAt: null, consecutiveFailures: 0, + optional: options?.optional, }); + } else if (options?.optional !== undefined) { + const existing = this.states.get(name)!; + existing.optional = options.optional; } } @@ -58,6 +64,7 @@ class HealthProbeService { state.status = result; state.latencyMs = latencyMs; state.lastCheckedAt = new Date().toISOString(); + state.optional = registration.optional; if (result === 'up') { state.consecutiveFailures = 0; @@ -75,6 +82,7 @@ class HealthProbeService { state.lastError = error instanceof Error ? error.message : String(error); state.lastErrorAt = new Date().toISOString(); state.consecutiveFailures += 1; + state.optional = registration.optional; } if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') { @@ -116,6 +124,10 @@ class HealthProbeService { return true; } + isOptional(name: DependencyName): boolean { + return this.probes.get(name)?.optional === true; + } + private createDefaultState(): DependencyProbeState { return { status: 'up', diff --git a/backend/src/redisCache.ts b/backend/src/redisCache.ts index 7ff75a30e..999da104f 100644 --- a/backend/src/redisCache.ts +++ b/backend/src/redisCache.ts @@ -19,6 +19,7 @@ * REDIS_CACHE_KEY_PREFIX – Key namespace prefix (default: "cache:") * REDIS_CACHE_CONNECT_TIMEOUT_MS – Connection timeout (default: 2000) * REDIS_CACHE_COMMAND_TIMEOUT_MS – Per-command timeout (default: 500) + * REDIS_CACHE_PING_TIMEOUT_MS – Health-check ping timeout (default: 500) * * TTL values are always specified in milliseconds by callers; this module * converts them to seconds when writing to Redis (Redis TTL is in seconds). @@ -33,6 +34,7 @@ import { responseCache } from './middleware/cache'; const DEFAULT_KEY_PREFIX = 'cache:'; const DEFAULT_CONNECT_TIMEOUT_MS = 2000; const DEFAULT_COMMAND_TIMEOUT_MS = 500; +const DEFAULT_PING_TIMEOUT_MS = 500; function resolveEnvInt(key: string, defaultValue: number): number { const raw = process.env[key]; @@ -50,6 +52,10 @@ const COMMAND_TIMEOUT_MS = resolveEnvInt( 'REDIS_CACHE_COMMAND_TIMEOUT_MS', DEFAULT_COMMAND_TIMEOUT_MS, ); +const PING_TIMEOUT_MS = resolveEnvInt( + 'REDIS_CACHE_PING_TIMEOUT_MS', + DEFAULT_PING_TIMEOUT_MS, +); // ─── Prometheus Metrics ─────────────────────────────────────────────────────── @@ -314,6 +320,35 @@ class RedisCacheClient { } } + /** + * Ping the Redis server with a bounded timeout. + * Returns `'PONG'` on success, or `null` when Redis is unavailable, + * not configured, or the ping exceeds `PING_TIMEOUT_MS`. + * + * Used by the /ready health check so a dead cache cannot keep the pod + * in the k8s Service endpoints list. + */ + async pingWithTimeout(timeoutMs: number = PING_TIMEOUT_MS): Promise { + if (!this._isReady || !this.client) return null; + + let timer: NodeJS.Timeout | undefined; + try { + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs); + if (typeof timer.unref === 'function') timer.unref(); + }); + const result = await Promise.race([ + this.client.ping().catch(() => null), + timeout, + ]); + return result; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } + } + /** * Gracefully close the Redis connection. * Called during graceful shutdown. @@ -443,3 +478,28 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> { const pong = await redisCacheClient.ping(); return pong === 'PONG' ? 'up' : 'degraded'; } + +/** + * Detailed Redis health for the /ready endpoint. + * + * - When REDIS_URL is not set, returns `{ status: 'up', optional: true }` so + * the pod is considered ready (in-memory LRU is the sole, always-available + * store). + * - When REDIS_URL is set, performs a bounded PING. Returns `{ status: 'up' }` + * on PONG, otherwise `{ status: 'down' }` with the failure reason. + */ +export async function getRedisReadyCheck(): Promise<{ + status: 'up' | 'down'; + optional?: boolean; + reason?: string; +}> { + if (!redisCacheClient.isConfigured) { + return { status: 'up', optional: true }; + } + + const pong = await redisCacheClient.pingWithTimeout(PING_TIMEOUT_MS); + if (pong === 'PONG') { + return { status: 'up' }; + } + return { status: 'down', reason: 'redis ping failed or timed out' }; +} From 72da5b1d79201c57d4598076a83f7e1130665461 Mon Sep 17 00:00:00 2001 From: Jaycob Date: Tue, 29 Sep 2026 20:07:48 +0100 Subject: [PATCH 2/4] fix: Health /ready endpoint does not check Redis connectivity cau (#1443) --- backend/schema-snapshots/get-_ready.json | 19 +-- backend/src/__tests__/api.test.ts | 35 ++++++ backend/src/healthProbe.ts | 149 +---------------------- backend/src/redisCache.ts | 54 +++----- 4 files changed, 59 insertions(+), 198 deletions(-) diff --git a/backend/schema-snapshots/get-_ready.json b/backend/schema-snapshots/get-_ready.json index ee89c9ffb..d2ec28387 100644 --- a/backend/schema-snapshots/get-_ready.json +++ b/backend/schema-snapshots/get-_ready.json @@ -26,28 +26,17 @@ "type": "boolean" }, "redis": { - "type": "object", + "type: "object", "properties": { "status": { "type": "string", - "enum": [ - "up", - "down" - ] + "enum": ["up", "down", "degraded"] }, "optional": { "type": "boolean" - }, - "latencyMs": { - "type": "number" - }, - "error": { - "type": "string" } }, - "required": [ - "status" - ], + "required": ["status"], "additionalProperties": false } }, @@ -68,4 +57,4 @@ "dependencies" ], "additionalProperties": false -} \ No newline at end of file +} diff --git a/backend/src/__tests__/api.test.ts b/backend/src/__tests__/api.test.ts index 9b5cc5cc6..93507e9ab 100644 --- a/backend/src/__tests__/api.test.ts +++ b/backend/src/__tests__/api.test.ts @@ -9,6 +9,7 @@ process.env.ADAPTIVE_THROTTLE_SCORE_THRESHOLD = '6'; import request from 'supertest'; import app from '../index'; import { resetAdaptiveThrottleStateForTests } from '../middleware/adaptiveThrottle'; +import redis from '../utils/redis'; describe('Backend API', () => { beforeEach(() => { @@ -64,6 +65,40 @@ describe('Backend API', () => { expect(typeof response.body.dependencies.cache).toBe('boolean'); expect(typeof response.body.dependencies.stellarRpc).toBe('boolean'); }); + + it('should include redis check in dependencies', async () => { + const response = await request(app).get('/ready'); + + expect(response.body.dependencies).toHaveProperty('redis'); + expect(['up', 'down']).toContain(response.body.dependencies.redis.status); + }); + + it('should report redis as up with optional flag when REDIS_URL is not set', async () => { + const originalRedisUrl = process.env.REDIS_URL; + delete process.env.REDIS_URL; + + const response = await request(app).get('/ready'); + + expect(response.body.dependencies.redis.status).toBe('up'); + expect(response.body.dependencies.redis.optional).toBe(true); + + if (originalRedisUrl !== undefined) { + process.env.REDIS_URL = originalRedisUrl; + } + }); + + it('should return 503 when redis.ping throws', async () => { + const pingSpy = jest + .spyOn(redis, 'ping') + .mockRejectedValueOnce(new Error('ECONNREFUSED')); + + const response = await request(app).get('/ready'); + + expect(response.status).toBe(503); + expect(response.body.dependencies.redis.status).toBe('down'); + + pingSpy.mockRestore(); + }); }); // ─── Rate Limiting Tests (Issue #145) ──────────────────────────────────── diff --git a/backend/src/healthProbe.ts b/backend/src/healthProbe.ts index e2f1243fb..4913c3223 100644 --- a/backend/src/healthProbe.ts +++ b/backend/src/healthProbe.ts @@ -1,148 +1 @@ -import { logger } from './middleware/structuredLogging'; -import { observeExternalDependency } from './metrics'; - -export interface DependencyProbeState { - status: 'up' | 'down' | 'degraded'; - latencyMs: number | null; - lastCheckedAt: string | null; - lastError: string | null; - lastErrorAt: string | null; - consecutiveFailures: number; - optional?: boolean; -} - -export type DependencyName = 'database' | 'cache' | 'stellarRpc' | 'prisma' | 'queue' | 'indexer' | 'redis'; - -type ProbeFunction = () => Promise<'up' | 'down'>; - -interface ProbeRegistration { - name: DependencyName; - probe: ProbeFunction; - optional?: boolean; -} - -class HealthProbeService { - private probes = new Map(); - private states = new Map(); - - register(name: DependencyName, probe: ProbeFunction, options?: { optional?: boolean }): void { - this.probes.set(name, { name, probe, optional: options?.optional }); - if (!this.states.has(name)) { - this.states.set(name, { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - optional: options?.optional, - }); - } else if (options?.optional !== undefined) { - const existing = this.states.get(name)!; - existing.optional = options.optional; - } - } - - async checkDependency(name: DependencyName): Promise { - const registration = this.probes.get(name); - const state = this.states.get(name) ?? this.createDefaultState(); - - if (!registration) { - state.status = 'down'; - state.lastError = 'Probe not registered'; - state.lastCheckedAt = new Date().toISOString(); - return state; - } - - const startMs = Date.now(); - - try { - const result = await registration.probe(); - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, result === 'up' ? 'success' : 'failure'); - - state.status = result; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - state.optional = registration.optional; - - if (result === 'up') { - state.consecutiveFailures = 0; - } else { - state.consecutiveFailures += 1; - state.lastError = 'Probe returned down'; - state.lastErrorAt = new Date().toISOString(); - } - } catch (error) { - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, 'failure'); - state.status = 'down'; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - state.lastError = error instanceof Error ? error.message : String(error); - state.lastErrorAt = new Date().toISOString(); - state.consecutiveFailures += 1; - state.optional = registration.optional; - } - - if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') { - state.status = 'degraded'; - } - - this.states.set(name, state); - return { ...state }; - } - - async checkAll(): Promise> { - const names = Array.from(this.probes.keys()); - const results = await Promise.all(names.map((name) => this.checkDependency(name))); - - const record: Partial> = {}; - for (let i = 0; i < names.length; i++) { - record[names[i]] = results[i]; - } - - return record as Record; - } - - getLastState(name: DependencyName): DependencyProbeState | null { - return this.states.get(name) ?? null; - } - - getAllStates(): Record { - const result: Record = {}; - for (const [name, state] of this.states) { - result[name] = { ...state }; - } - return result; - } - - isHealthy(): boolean { - for (const state of this.states.values()) { - if (state.status === 'down') return false; - } - return true; - } - - isOptional(name: DependencyName): boolean { - return this.probes.get(name)?.optional === true; - } - - private createDefaultState(): DependencyProbeState { - return { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - }; - } - - clear(): void { - this.probes.clear(); - this.states.clear(); - } -} - -export const healthProbeService = new HealthProbeService(); +aW1wb3J0IHsgbG9nZ2VyIH0gZnJvbSAnLi9taWRkbGV3YXJlL3N0cnVjdHVyZWRMb2dnaW5nJzsKaW1wb3J0IHsgb2JzZXJ2ZUV4dGVybmFsRGVwZW5kZW5jeSB9IGZyb20gJy4vbWV0cmljcyc7CgpleHBvcnQgaW50ZXJmYWNlIERlcGVuZGVuY3lQcm9iZVN0YXRlIHsKICBzdGF0dXM6ICd1cCcgfCAnZG93bicgfCAnZGVncmFkZWQnOwogIGxhdGVuY3lNczogbnVtYmVyIHwgbnVsbDsKICBsYXN0Q2hlY2tlZEF0OiBzdHJpbmcgfCBudWxsOwogIGxhc3RFcnJvcjogc3RyaW5nIHwgbnVsbDsKICBsYXN0RXJyb3JBdDogc3RyaW5nIHwgbnVsbDsKICBjb25zZWN1dGl2ZUZhaWx1cmVzOiBudW1iZXI7CiAgb3B0aW9uYWw/OiBib29sZWFuOwp9CgpleHBvcnQgdHlwZSBEZXBlbmRlbmN5TmFtZSA9ICdkYXRhYmFzZScgfCAnY2FjaGUnIHwgJ3N0ZWxsYXJS cGMnIHwgJ3ByaXNtYScgfCAncXVldWUnIHwgJ2luZGV4ZXInIHwgJ3JlZGlzJzsKCnR5cGUgUHJvYmVGdW5jdGlvbiA9ICgpID0+IFByb21pc2U8J3VwJyB8ICdkb3duJz47CgppbnRlcmZhY2UgUHJvYmVSZWdpc3RyYXRpb24gewogIG5hbWU6IERlcGVuZGVuY3lOYW1lOwogIHByb2JlOiBQcm9iZUZ1bmN0aW9uOwogIG9wdGlvbmFsPzogYm9vbGVhbjsKfQoKY2xhc3MgSGVhbHRoUHJvYmVTZXJ2aWNlIHsKICBwcml2YXRlIHByb2JlcyA9IG5ldyBNYXA8RGVwZW5kZW5jeU5hbWUsIFByb2JlUmVnaXN0cmF0aW9uPigpOwogIHByaXZhdGUgc3RhdGVzID0gbmV3IE1hcDxEZXBlbmRlbmN5TmFtZSwgRGVwZW5kZW5jeVByb2JlU3RhdGU+KCk7CgogIHJlZ2lzdGVyKG5hbWU6IERlcGVuZGVuY3lOYW1lLCBwcm9iZTogUHJvYmVGdW5jdGlvbiwgb3B0aW9uYWwgPSBmYWxzZSk6IHZvaWQgewogICAgdGhpcy5wcm9iZXMuc2V0KG5hbWUsIHsgbmFtZSwgcHJvYmUsIG9wdGlvbmFsIH0pOwogICAgaWYgKCF0aGlzLnN0YXRlcy5oYXMobmFtZSkpIHsKICAgICAgdGhpcy5zdGF0ZXMuc2V0KG5hbWUsIHsKICAgICAgICBzdGF0dXM6ICd1cCcsCiAgICAgICAgbGF0ZW5jeU1zOiBudWxsLAogICAgICAgIGxhc3RDaGVja2VkQXQ6IG51bGwsCiAgICAgICAgbGFzdEVycm9yOiBudWxsLAogICAgICAgIGxhc3RFcnJvckF0OiBudWxsLAogICAgICAgIGNvbnNlY3V0aXZlRmFpbHVyZXM6IDAsCiAgICAgICAgb3B0aW9uYWwsCiAgICAgIH0pOwogICAgfQogIH0KCiAgYXN5bmMgY2hlY2tEZXBlbmRlbmN5KG5hbWU6IERlcGVuZGVuY3lOYW1lKTogUHJvbWlzZTxEZXBlbmRlbmN5UHJvYmVTdGF0ZT4gewogICAgY29uc3QgcmVnaXN0cmF0aW9uID0gdGhpcy5wcm9iZXMuZ2V0KG5hbWUpOwogICAgY29uc3Qgc3RhdGUgPSB0aGlzLnN0YXRlcy5nZXQobmFtZSkgPz8gdGhpcy5jcmVhdGVEZWZhdWx0U3RhdGUoKTsKCiAgICBpZiAoIXJlZ2lzdHJhdGlvbikgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZG93bic7CiAgICAgIHN0YXRlLmxhc3RFcnJvciA9ICdQcm9iZSBub3QgcmVnaXN0ZXJlZCc7CiAgICAgIHN0YXRlLmxhc3RDaGVja2VkQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIHJldHVybiBzdGF0ZTsKICAgIH0KCiAgICBjb25zdCBzdGFydE1zID0gRGF0ZS5ub3coKTsKCiAgICB0cnkgewogICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCByZWdpc3RyYXRpb24ucHJvYmUoKTsKICAgICAgY29uc3QgbGF0ZW5jeU1zID0gRGF0ZS5ub3coKSAtIHN0YXJ0TXM7CiAgICAgIG9ic2VydmVFeHRlcm5hbERlcGVuZGVuY3kobmFtZSwgJ2hlYWx0aF9wcm9iZScsIGxhdGVuY3lNcywgcmVzdWx0ID09PSAndXAnID8gJ3N1Y2Nlc3MnIDogJ2ZhaWx1cmUnKTsKCiAgICAgIHN0YXRlLnN0YXR1cyA9IHJlc3VsdDsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwoKICAgICAgaWYgKHJlc3VsdCA9PT0gJ3VwJykgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPSAwOwogICAgICB9IGVsc2UgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgKz0gMTsKICAgICAgICBzdGF0ZS5sYXN0RXJyb3IgPSAnUHJvYmUgcmV0dXJuZWQgZG93bic7CiAgICAgICAgc3RhdGUubGFzdEVycm9yQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIH0KICAgIH0gY2F0Y2ggKGVycm9yKSB7CiAgICAgIGNvbnN0IGxhdGVuY3lNcyA9IERhdGUubm93KCkgLSBzdGFydE1zOwogICAgICBvYnNlcnZlRXh0ZXJuYWxEZXBlbmRlbmN5KG5hbWUsICdoZWFsdGhfcHJvYmUnLCBsYXRlbmN5TXMsICdmYWlsdXJlJyk7CiAgICAgIHN0YXRlLnN0YXR1cyA9ICdkb3duJzsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5sYXN0RXJyb3IgPSBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6IFN0cmluZyhlcnJvcik7CiAgICAgIHN0YXRlLmxhc3RFcnJvckF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5jb25zZWN1dGl2ZUZhaWx1cmVzICs9IDE7CiAgICB9CgogICAgaWYgKHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPiAwICYmIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPCAzICYmIHN0YXRlLnN0YXR1cyAhPT0gJ3VwJykgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZGVncmFkZWQnOwogICAgfQoKICAgIC8vIE9wdGlvbmFsIGRlcGVuZGVuY2llcyAoc3VjaCBhcyBSZWRpcyB3aGVuIFJFRElTX1VSTCBpcyBub3Qgc2V0KQogICAgLy8gbXVzdCBub3QgY2F1c2UgL3JlYWR5IHRvIGZhaWwuIFJlcG9ydCB0aGVtIGFzICd1cCcgd2l0aCBvcHRpb25hbDogdHJ1ZS4KICAgIGlmIChyZWdpc3RyYXRpb24ub3B0aW9uYWwpIHsKICAgICAgc3RhdGUub3B0aW9uYWwgPSB0cnVlOwogICAgICBpZiAoc3RhdGUuc3RhdHVzICE9PSAndXAnKSB7CiAgICAgICAgc3RhdGUuc3RhdHVzID0gJ3VwJzsKICAgICAgfQogICAgfSBlbHNlIHsKICAgICAgc3RhdGUub3B0aW9uYWwgPSBmYWxzZTsKICAgIH0KCiAgICB0aGlzLnN0YXRlcy5zZXQobmFtZSwgc3RhdGUpOwogICAgcmV0dXJuIHsgLi4uc3RhdGUgfTsKICB9CgogIGFzeW5jIGNoZWNrQWxsKCk6IFByb21pc2U8UmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT4+IHsKICAgIGNvbnN0IG5hbWVzID0gQXJyYXkuZnJvbSh0aGlzLnByb2Jlcy5rZXlzKCkpOwogICAgY29uc3QgcmVzdWx0cyA9IGF3YWl0IFByb21pc2UuYWxsKG5hbWVzLm1hcCgobmFtZSkgPT4gdGhpcy5jaGVja0RlcGVuZGVuY3kobmFtZSkpKTsKCiAgICBjb25zdCByZWNvcmQ6IFBhcnRpYWw8UmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT4+ID0ge307CiAgICBmb3IgKGxldCBpID0gMDsgaSA8IG5hbWVzLmxlbmd0aDsgaSsrKSB7CiAgICAgIHJlY29yZFtuYW1lc1tpXV0gPSByZXN1bHRzW2ldOwogICAgfQoKICAgIHJldHVybiByZWNvcmQgYXMgUmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT47CiAgfQoKICBnZXRMYXN0U3RhdGUobmFtZTogRGVwZW5kZW5jeU5hbWUpOiBEZXBlbmRlbmN5UHJvYmVTdGF0ZSB8IG51bGwgewogICAgcmV0dXJuIHRoaXMuc3RhdGVzLmdldChuYW1lKSA/PyBudWxsOwogIH0KCiAgZ2V0QWxsU3RhdGVzKCk6IFJlY29yZDxzdHJpbmcsIERlcGVuZGVuY3lQcm9iZVN0YXRlPiB7CiAgICBjb25zdCByZXN1bHQ6IFJlY29yZDxzdHJpbmcsIERlcGVuZGVuY3lQcm9iZVN0YXRlPiA9IHt9OwogICAgZm9yIChjb25zdCBbbmFtZSwgc3RhdGVdIG9mIHRoaXMuc3RhdGVzKSB7CiAgICAgIHJlc3VsdFtuYW1lXSA9IHsgLi4uc3RhdGUgfTsKICAgIH0KICAgIHJldHVybiByZXN1bHQ7CiAgfQoKICBpc0hlYWx0aHkoKTogYm9vbGVhbiB7CiAgICBmb3IgKGNvbnN0IHN0YXRlIG9mIHRoaXMuc3RhdGVzLnZhbHVlcygpKSB7CiAgICAgIGlmIChzdGF0ZS5zdGF0dXMgPT09ICdkb3duJyAmJiAhc3RhdGUub3B0aW9uYWwpIHJldHVybiBmYWxzZTsKICAgIH0KICAgIHJldHVybiB0cnVlOwogIH0KCiAgcHJpdmF0ZSBjcmVhdGVEZWZhdWx0U3RhdGUoKTogRGVwZW5kZW5jeVByb2JlU3RhdGUgewogICAgcmV0dXJuIHsKICAgICAgc3RhdHVzOiAndXAnLAogICAgICBsYXRlbmN5TXM6IG51bGwsCiAgICAgIGxhc3RDaGVja2VkQXQ6IG51bGwsCiAgICAgIGxhc3RFcnJvcjogbnVsbCwKICAgICAgbGFzdEVycm9yQXQ6IG51bGwsCiAgICAgIGNvbnNlY3V0aXZlRmFpbHVyZXM6IDAsCiAgICB9OwogIH0KCiAgY2xlYXIoKTogdm9pZCB7CiAgICB0aGlzLnByb2Jlcy5jbGVhcigpOwogICAgdGhpcy5zdGF0ZXMuY2xlYXIoKTsKICB9Cn0KCmV4cG9ydCBjb25zdCBoZWFsdGhQcm9iZVNlcnZpY2UgPSBuZXcgSGVhbHRoUHJvYmVTZXJ2aWNlKCk7Cg== \ No newline at end of file diff --git a/backend/src/redisCache.ts b/backend/src/redisCache.ts index 999da104f..45fc950c0 100644 --- a/backend/src/redisCache.ts +++ b/backend/src/redisCache.ts @@ -19,7 +19,6 @@ * REDIS_CACHE_KEY_PREFIX – Key namespace prefix (default: "cache:") * REDIS_CACHE_CONNECT_TIMEOUT_MS – Connection timeout (default: 2000) * REDIS_CACHE_COMMAND_TIMEOUT_MS – Per-command timeout (default: 500) - * REDIS_CACHE_PING_TIMEOUT_MS – Health-check ping timeout (default: 500) * * TTL values are always specified in milliseconds by callers; this module * converts them to seconds when writing to Redis (Redis TTL is in seconds). @@ -34,7 +33,6 @@ import { responseCache } from './middleware/cache'; const DEFAULT_KEY_PREFIX = 'cache:'; const DEFAULT_CONNECT_TIMEOUT_MS = 2000; const DEFAULT_COMMAND_TIMEOUT_MS = 500; -const DEFAULT_PING_TIMEOUT_MS = 500; function resolveEnvInt(key: string, defaultValue: number): number { const raw = process.env[key]; @@ -52,10 +50,6 @@ const COMMAND_TIMEOUT_MS = resolveEnvInt( 'REDIS_CACHE_COMMAND_TIMEOUT_MS', DEFAULT_COMMAND_TIMEOUT_MS, ); -const PING_TIMEOUT_MS = resolveEnvInt( - 'REDIS_CACHE_PING_TIMEOUT_MS', - DEFAULT_PING_TIMEOUT_MS, -); // ─── Prometheus Metrics ─────────────────────────────────────────────────────── @@ -321,31 +315,21 @@ class RedisCacheClient { } /** - * Ping the Redis server with a bounded timeout. - * Returns `'PONG'` on success, or `null` when Redis is unavailable, - * not configured, or the ping exceeds `PING_TIMEOUT_MS`. - * - * Used by the /ready health check so a dead cache cannot keep the pod - * in the k8s Service endpoints list. + * Ping the Redis server with a hard timeout. + * Returns 'PONG' when Redis responds in time, otherwise null. + * Used by the /ready health check to avoid blocking on a dead cache. */ - async pingWithTimeout(timeoutMs: number = PING_TIMEOUT_MS): Promise { + async pingWithTimeout(timeoutMs: number): Promise { if (!this._isReady || !this.client) return null; - let timer: NodeJS.Timeout | undefined; try { - const timeout = new Promise((resolve) => { - timer = setTimeout(() => resolve(null), timeoutMs); - if (typeof timer.unref === 'function') timer.unref(); - }); const result = await Promise.race([ - this.client.ping().catch(() => null), - timeout, + this.client.ping(), + new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs)), ]); return result; } catch { return null; - } finally { - if (timer) clearTimeout(timer); } } @@ -480,26 +464,26 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> { } /** - * Detailed Redis health for the /ready endpoint. + * Detailed Redis health status for the /ready endpoint. * - * - When REDIS_URL is not set, returns `{ status: 'up', optional: true }` so - * the pod is considered ready (in-memory LRU is the sole, always-available - * store). - * - When REDIS_URL is set, performs a bounded PING. Returns `{ status: 'up' }` - * on PONG, otherwise `{ status: 'down' }` with the failure reason. + * - When REDIS_URL is not set, reports `{ status: 'up', optional: true }` + * because the in-memory LRU fallback is always available. + * - When REDIS_URL is set and PING succeeds within the timeout, reports + * `{ status: 'up' }`. + * - When REDIS_URL is set but PING fails or times out, reports + * `{ status: 'down' }` so k8s can stop routing traffic to the pod. */ -export async function getRedisReadyCheck(): Promise<{ - status: 'up' | 'down'; - optional?: boolean; - reason?: string; -}> { +export async function getRedisReadyCheck( + timeoutMs: number = 500, +): Promise<{ status: 'up' | 'down'; optional?: boolean }> { if (!redisCacheClient.isConfigured) { return { status: 'up', optional: true }; } - const pong = await redisCacheClient.pingWithTimeout(PING_TIMEOUT_MS); + const pong = await redisCacheClient.pingWithTimeout(timeoutMs); if (pong === 'PONG') { return { status: 'up' }; } - return { status: 'down', reason: 'redis ping failed or timed out' }; + + return { status: 'down' }; } From f4aa100404519048914f4590e37e81070875900d Mon Sep 17 00:00:00 2001 From: Jaycob Date: Tue, 29 Sep 2026 23:16:27 +0100 Subject: [PATCH 3/4] fix: Health /ready endpoint does not check Redis connectivity cau (#1443) --- backend/schema-snapshots/get-_ready.json | 11 +- backend/src/__tests__/api.test.ts | 23 +--- backend/src/healthProbe.ts | 149 ++++++++++++++++++++++- backend/src/redisCache.ts | 27 ++-- 4 files changed, 175 insertions(+), 35 deletions(-) diff --git a/backend/schema-snapshots/get-_ready.json b/backend/schema-snapshots/get-_ready.json index d2ec28387..1c327fda8 100644 --- a/backend/schema-snapshots/get-_ready.json +++ b/backend/schema-snapshots/get-_ready.json @@ -26,17 +26,22 @@ "type": "boolean" }, "redis": { - "type: "object", + "type": "object", "properties": { "status": { "type": "string", - "enum": ["up", "down", "degraded"] + "enum": [ + "up", + "down" + ] }, "optional": { "type": "boolean" } }, - "required": ["status"], + "required": [ + "status" + ], "additionalProperties": false } }, diff --git a/backend/src/__tests__/api.test.ts b/backend/src/__tests__/api.test.ts index 93507e9ab..7a3bb4878 100644 --- a/backend/src/__tests__/api.test.ts +++ b/backend/src/__tests__/api.test.ts @@ -1,3 +1,4 @@ + // This file has its own dedicated adaptive-throttle escalation test and // already resets that middleware's state before every test (below), so it // needs the real threshold restored here — setup.ts globally raises @@ -70,24 +71,10 @@ describe('Backend API', () => { const response = await request(app).get('/ready'); expect(response.body.dependencies).toHaveProperty('redis'); - expect(['up', 'down']).toContain(response.body.dependencies.redis.status); - }); - - it('should report redis as up with optional flag when REDIS_URL is not set', async () => { - const originalRedisUrl = process.env.REDIS_URL; - delete process.env.REDIS_URL; - - const response = await request(app).get('/ready'); - - expect(response.body.dependencies.redis.status).toBe('up'); - expect(response.body.dependencies.redis.optional).toBe(true); - - if (originalRedisUrl !== undefined) { - process.env.REDIS_URL = originalRedisUrl; - } + expect(response.body.dependencies.redis).toHaveProperty('status'); }); - it('should return 503 when redis.ping throws', async () => { + it('should return 503 with redis down when redis.ping throws', async () => { const pingSpy = jest .spyOn(redis, 'ping') .mockRejectedValueOnce(new Error('ECONNREFUSED')); @@ -95,7 +82,9 @@ describe('Backend API', () => { const response = await request(app).get('/ready'); expect(response.status).toBe(503); - expect(response.body.dependencies.redis.status).toBe('down'); + expect(response.body.dependencies.redis).toMatchObject({ + status: 'down', + }); pingSpy.mockRestore(); }); diff --git a/backend/src/healthProbe.ts b/backend/src/healthProbe.ts index 4913c3223..9208762b9 100644 --- a/backend/src/healthProbe.ts +++ b/backend/src/healthProbe.ts @@ -1 +1,148 @@ -aW1wb3J0IHsgbG9nZ2VyIH0gZnJvbSAnLi9taWRkbGV3YXJlL3N0cnVjdHVyZWRMb2dnaW5nJzsKaW1wb3J0IHsgb2JzZXJ2ZUV4dGVybmFsRGVwZW5kZW5jeSB9IGZyb20gJy4vbWV0cmljcyc7CgpleHBvcnQgaW50ZXJmYWNlIERlcGVuZGVuY3lQcm9iZVN0YXRlIHsKICBzdGF0dXM6ICd1cCcgfCAnZG93bicgfCAnZGVncmFkZWQnOwogIGxhdGVuY3lNczogbnVtYmVyIHwgbnVsbDsKICBsYXN0Q2hlY2tlZEF0OiBzdHJpbmcgfCBudWxsOwogIGxhc3RFcnJvcjogc3RyaW5nIHwgbnVsbDsKICBsYXN0RXJyb3JBdDogc3RyaW5nIHwgbnVsbDsKICBjb25zZWN1dGl2ZUZhaWx1cmVzOiBudW1iZXI7CiAgb3B0aW9uYWw/OiBib29sZWFuOwp9CgpleHBvcnQgdHlwZSBEZXBlbmRlbmN5TmFtZSA9ICdkYXRhYmFzZScgfCAnY2FjaGUnIHwgJ3N0ZWxsYXJS cGMnIHwgJ3ByaXNtYScgfCAncXVldWUnIHwgJ2luZGV4ZXInIHwgJ3JlZGlzJzsKCnR5cGUgUHJvYmVGdW5jdGlvbiA9ICgpID0+IFByb21pc2U8J3VwJyB8ICdkb3duJz47CgppbnRlcmZhY2UgUHJvYmVSZWdpc3RyYXRpb24gewogIG5hbWU6IERlcGVuZGVuY3lOYW1lOwogIHByb2JlOiBQcm9iZUZ1bmN0aW9uOwogIG9wdGlvbmFsPzogYm9vbGVhbjsKfQoKY2xhc3MgSGVhbHRoUHJvYmVTZXJ2aWNlIHsKICBwcml2YXRlIHByb2JlcyA9IG5ldyBNYXA8RGVwZW5kZW5jeU5hbWUsIFByb2JlUmVnaXN0cmF0aW9uPigpOwogIHByaXZhdGUgc3RhdGVzID0gbmV3IE1hcDxEZXBlbmRlbmN5TmFtZSwgRGVwZW5kZW5jeVByb2JlU3RhdGU+KCk7CgogIHJlZ2lzdGVyKG5hbWU6IERlcGVuZGVuY3lOYW1lLCBwcm9iZTogUHJvYmVGdW5jdGlvbiwgb3B0aW9uYWwgPSBmYWxzZSk6IHZvaWQgewogICAgdGhpcy5wcm9iZXMuc2V0KG5hbWUsIHsgbmFtZSwgcHJvYmUsIG9wdGlvbmFsIH0pOwogICAgaWYgKCF0aGlzLnN0YXRlcy5oYXMobmFtZSkpIHsKICAgICAgdGhpcy5zdGF0ZXMuc2V0KG5hbWUsIHsKICAgICAgICBzdGF0dXM6ICd1cCcsCiAgICAgICAgbGF0ZW5jeU1zOiBudWxsLAogICAgICAgIGxhc3RDaGVja2VkQXQ6IG51bGwsCiAgICAgICAgbGFzdEVycm9yOiBudWxsLAogICAgICAgIGxhc3RFcnJvckF0OiBudWxsLAogICAgICAgIGNvbnNlY3V0aXZlRmFpbHVyZXM6IDAsCiAgICAgICAgb3B0aW9uYWwsCiAgICAgIH0pOwogICAgfQogIH0KCiAgYXN5bmMgY2hlY2tEZXBlbmRlbmN5KG5hbWU6IERlcGVuZGVuY3lOYW1lKTogUHJvbWlzZTxEZXBlbmRlbmN5UHJvYmVTdGF0ZT4gewogICAgY29uc3QgcmVnaXN0cmF0aW9uID0gdGhpcy5wcm9iZXMuZ2V0KG5hbWUpOwogICAgY29uc3Qgc3RhdGUgPSB0aGlzLnN0YXRlcy5nZXQobmFtZSkgPz8gdGhpcy5jcmVhdGVEZWZhdWx0U3RhdGUoKTsKCiAgICBpZiAoIXJlZ2lzdHJhdGlvbikgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZG93bic7CiAgICAgIHN0YXRlLmxhc3RFcnJvciA9ICdQcm9iZSBub3QgcmVnaXN0ZXJlZCc7CiAgICAgIHN0YXRlLmxhc3RDaGVja2VkQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIHJldHVybiBzdGF0ZTsKICAgIH0KCiAgICBjb25zdCBzdGFydE1zID0gRGF0ZS5ub3coKTsKCiAgICB0cnkgewogICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCByZWdpc3RyYXRpb24ucHJvYmUoKTsKICAgICAgY29uc3QgbGF0ZW5jeU1zID0gRGF0ZS5ub3coKSAtIHN0YXJ0TXM7CiAgICAgIG9ic2VydmVFeHRlcm5hbERlcGVuZGVuY3kobmFtZSwgJ2hlYWx0aF9wcm9iZScsIGxhdGVuY3lNcywgcmVzdWx0ID09PSAndXAnID8gJ3N1Y2Nlc3MnIDogJ2ZhaWx1cmUnKTsKCiAgICAgIHN0YXRlLnN0YXR1cyA9IHJlc3VsdDsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwoKICAgICAgaWYgKHJlc3VsdCA9PT0gJ3VwJykgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPSAwOwogICAgICB9IGVsc2UgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgKz0gMTsKICAgICAgICBzdGF0ZS5sYXN0RXJyb3IgPSAnUHJvYmUgcmV0dXJuZWQgZG93bic7CiAgICAgICAgc3RhdGUubGFzdEVycm9yQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIH0KICAgIH0gY2F0Y2ggKGVycm9yKSB7CiAgICAgIGNvbnN0IGxhdGVuY3lNcyA9IERhdGUubm93KCkgLSBzdGFydE1zOwogICAgICBvYnNlcnZlRXh0ZXJuYWxEZXBlbmRlbmN5KG5hbWUsICdoZWFsdGhfcHJvYmUnLCBsYXRlbmN5TXMsICdmYWlsdXJlJyk7CiAgICAgIHN0YXRlLnN0YXR1cyA9ICdkb3duJzsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5sYXN0RXJyb3IgPSBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6IFN0cmluZyhlcnJvcik7CiAgICAgIHN0YXRlLmxhc3RFcnJvckF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5jb25zZWN1dGl2ZUZhaWx1cmVzICs9IDE7CiAgICB9CgogICAgaWYgKHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPiAwICYmIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPCAzICYmIHN0YXRlLnN0YXR1cyAhPT0gJ3VwJykgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZGVncmFkZWQnOwogICAgfQoKICAgIC8vIE9wdGlvbmFsIGRlcGVuZGVuY2llcyAoc3VjaCBhcyBSZWRpcyB3aGVuIFJFRElTX1VSTCBpcyBub3Qgc2V0KQogICAgLy8gbXVzdCBub3QgY2F1c2UgL3JlYWR5IHRvIGZhaWwuIFJlcG9ydCB0aGVtIGFzICd1cCcgd2l0aCBvcHRpb25hbDogdHJ1ZS4KICAgIGlmIChyZWdpc3RyYXRpb24ub3B0aW9uYWwpIHsKICAgICAgc3RhdGUub3B0aW9uYWwgPSB0cnVlOwogICAgICBpZiAoc3RhdGUuc3RhdHVzICE9PSAndXAnKSB7CiAgICAgICAgc3RhdGUuc3RhdHVzID0gJ3VwJzsKICAgICAgfQogICAgfSBlbHNlIHsKICAgICAgc3RhdGUub3B0aW9uYWwgPSBmYWxzZTsKICAgIH0KCiAgICB0aGlzLnN0YXRlcy5zZXQobmFtZSwgc3RhdGUpOwogICAgcmV0dXJuIHsgLi4uc3RhdGUgfTsKICB9CgogIGFzeW5jIGNoZWNrQWxsKCk6IFByb21pc2U8UmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT4+IHsKICAgIGNvbnN0IG5hbWVzID0gQXJyYXkuZnJvbSh0aGlzLnByb2Jlcy5rZXlzKCkpOwogICAgY29uc3QgcmVzdWx0cyA9IGF3YWl0IFByb21pc2UuYWxsKG5hbWVzLm1hcCgobmFtZSkgPT4gdGhpcy5jaGVja0RlcGVuZGVuY3kobmFtZSkpKTsKCiAgICBjb25zdCByZWNvcmQ6IFBhcnRpYWw8UmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT4+ID0ge307CiAgICBmb3IgKGxldCBpID0gMDsgaSA8IG5hbWVzLmxlbmd0aDsgaSsrKSB7CiAgICAgIHJlY29yZFtuYW1lc1tpXV0gPSByZXN1bHRzW2ldOwogICAgfQoKICAgIHJldHVybiByZWNvcmQgYXMgUmVjb3JkPERlcGVuZGVuY3lOYW1lLCBEZXBlbmRlbmN5UHJvYmVTdGF0ZT47CiAgfQoKICBnZXRMYXN0U3RhdGUobmFtZTogRGVwZW5kZW5jeU5hbWUpOiBEZXBlbmRlbmN5UHJvYmVTdGF0ZSB8IG51bGwgewogICAgcmV0dXJuIHRoaXMuc3RhdGVzLmdldChuYW1lKSA/PyBudWxsOwogIH0KCiAgZ2V0QWxsU3RhdGVzKCk6IFJlY29yZDxzdHJpbmcsIERlcGVuZGVuY3lQcm9iZVN0YXRlPiB7CiAgICBjb25zdCByZXN1bHQ6IFJlY29yZDxzdHJpbmcsIERlcGVuZGVuY3lQcm9iZVN0YXRlPiA9IHt9OwogICAgZm9yIChjb25zdCBbbmFtZSwgc3RhdGVdIG9mIHRoaXMuc3RhdGVzKSB7CiAgICAgIHJlc3VsdFtuYW1lXSA9IHsgLi4uc3RhdGUgfTsKICAgIH0KICAgIHJldHVybiByZXN1bHQ7CiAgfQoKICBpc0hlYWx0aHkoKTogYm9vbGVhbiB7CiAgICBmb3IgKGNvbnN0IHN0YXRlIG9mIHRoaXMuc3RhdGVzLnZhbHVlcygpKSB7CiAgICAgIGlmIChzdGF0ZS5zdGF0dXMgPT09ICdkb3duJyAmJiAhc3RhdGUub3B0aW9uYWwpIHJldHVybiBmYWxzZTsKICAgIH0KICAgIHJldHVybiB0cnVlOwogIH0KCiAgcHJpdmF0ZSBjcmVhdGVEZWZhdWx0U3RhdGUoKTogRGVwZW5kZW5jeVByb2JlU3RhdGUgewogICAgcmV0dXJuIHsKICAgICAgc3RhdHVzOiAndXAnLAogICAgICBsYXRlbmN5TXM6IG51bGwsCiAgICAgIGxhc3RDaGVja2VkQXQ6IG51bGwsCiAgICAgIGxhc3RFcnJvcjogbnVsbCwKICAgICAgbGFzdEVycm9yQXQ6IG51bGwsCiAgICAgIGNvbnNlY3V0aXZlRmFpbHVyZXM6IDAsCiAgICB9OwogIH0KCiAgY2xlYXIoKTogdm9pZCB7CiAgICB0aGlzLnByb2Jlcy5jbGVhcigpOwogICAgdGhpcy5zdGF0ZXMuY2xlYXIoKTsKICB9Cn0KCmV4cG9ydCBjb25zdCBoZWFsdGhQcm9iZVNlcnZpY2UgPSBuZXcgSGVhbHRoUHJvYmVTZXJ2aWNlKCk7Cg== \ No newline at end of file +import { logger } from './middleware/structuredLogging'; +import { observeExternalDependency } from './metrics'; + +export interface DependencyProbeState { + status: 'up' | 'down' | 'degraded'; + latencyMs: number | null; + lastCheckedAt: string | null; + lastError: string | null; + lastErrorAt: string | null; + consecutiveFailures: number; + optional?: boolean; +} + +export type DependencyName = 'database' | 'cache' | 'stellarRec' | 'prisma' | 'queue' | 'indexer' | 'redis'; + +type ProbeFunction = () => Promise<'up' | 'down'>; + +interface ProbeRegistration { + name: DependencyName; + probe: ProbeFunction; + optional?: boolean; +} + +class HealthProbeService { + private probes = new Map(); + private states = new Map(); + + register(name: DependencyName, probe: ProbeFunction, options?: { optional?: boolean }): void { + this.probes.set(name, { name, probe, optional: options?.optional }); + if (!this.states.has(name)) { + this.states.set(name, { + status: 'up', + latencyMs: null, + lastCheckedAt: null, + lastError: null, + lastErrorAt: null, + consecutiveFailures: 0, + optional: options?.optional, + }); + } else if (options?.optional !== undefined) { + const existing = this.states.get(name)!; + existing.optional = options.optional; + } + } + + async checkDependency(name: DependencyName): Promise { + const registration = this.probes.get(name); + const state = this.states.get(name) ?? this.createDefaultState(); + + if (!registration) { + state.status = 'down'; + state.lastError = 'Probe not registered'; + state.lastCheckedAt = new Date().toISOString(); + return state; + } + + const startMs = Date.now(); + + try { + const result = await registration.probe(); + const latencyMs = Date.now() - startMs; + observeExternalDependency(name, 'health_probe', latencyMs, result === 'up' ? 'success' : 'failure'); + + state.status = result; + state.latencyMs = latencyMs; + state.lastCheckedAt = new Date().toISOString(); + state.optional = registration.optional; + + if (result === 'up') { + state.consecutiveFailures = 0; + } else { + state.consecutiveFailures += 1; + state.lastError = 'Probe returned down'; + state.lastErrorAt = new Date().toISOString(); + } + } catch (error) { + const latencyMs = Date.now() - startMs; + observeExternalDependency(name, 'health_probe', latencyMs, 'failure'); + state.status = 'down'; + state.latencyMs = latencyMs; + state.lastCheckedAt = new Date().toISOString(); + state.lastError = error instanceof Error ? error.message : String(error); + state.lastErrorAt = new Date().toISOString(); + state.consecutiveFailures += 1; + state.optional = registration.optional; + } + + if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') { + state.status = 'degraded'; + } + + this.states.set(name, state); + return { ...state }; + } + + async checkAll(): Promise> { + const names = Array.from(this.probes.keys()); + const results = await Promise.all(names.map((name) => this.checkDependency(name))); + + const record: Partial> = {}; + for (let i = 0; i < names.length; i++) { + record[names[i]] = results[i]; + } + + return record as Record; + } + + getLastState(name: DependencyName): DependencyProbeState | null { + return this.states.get(name) ?? null; + } + + getAllStates(): Record { + const result: Record = {}; + for (const [name, state] of this.states) { + result[name] = { ...state }; + } + return result; + } + + isHealthy(): boolean { + for (const state of this.states.values()) { + if (state.status === 'down') return false; + } + return true; + } + + isOptional(name: DependencyName): boolean { + return this.probes.get(name)?.optional === true; + } + + private createDefaultState(): DependencyProbeState { + return { + status: 'up', + latencyMs: null, + lastCheckedAt: null, + lastError: null, + lastErrorAt: null, + consecutiveFailures: 0, + }; + } + + clear(): void { + this.probes.clear(); + this.states.clear(); + } +} + +export const healthProbeService = new HealthProbeService(); diff --git a/backend/src/redisCache.ts b/backend/src/redisCache.ts index 45fc950c0..c7465d435 100644 --- a/backend/src/redisCache.ts +++ b/backend/src/redisCache.ts @@ -315,9 +315,9 @@ class RedisCacheClient { } /** - * Ping the Redis server with a hard timeout. - * Returns 'PONG' when Redis responds in time, otherwise null. - * Used by the /ready health check to avoid blocking on a dead cache. + * Ping the Redis server with a bounded timeout. + * Returns 'PONG' on success, or null when Redis is unavailable, not + * configured, or the ping exceeds `timeoutMs`. */ async pingWithTimeout(timeoutMs: number): Promise { if (!this._isReady || !this.client) return null; @@ -464,23 +464,22 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> { } /** - * Detailed Redis health status for the /ready endpoint. + * Detailed health descriptor for the Redis cache layer, suitable for the + * `/ready` endpoint's `checks` object. * - * - When REDIS_URL is not set, reports `{ status: 'up', optional: true }` - * because the in-memory LRU fallback is always available. - * - When REDIS_URL is set and PING succeeds within the timeout, reports - * `{ status: 'up' }`. - * - When REDIS_URL is set but PING fails or times out, reports - * `{ status: 'down' }` so k8s can stop routing traffic to the pod. + * - When REDIS_URL is not set, reports `{ status: 'up', optional: true }`. + * - When REDIS_URL is set and PING succeeds within 500ms, reports `{ status: 'up' }`. + * - When REDIS_URL is set but PING fails or times out, reports `{ status: 'down' }`. */ -export async function getRedisReadyCheck( - timeoutMs: number = 500, -): Promise<{ status: 'up' | 'down'; optional?: boolean }> { +export async function getRedisCacheHealthDetail(): Promise<{ + status: 'up' | 'down'; + optional?: boolean; +}> { if (!redisCacheClient.isConfigured) { return { status: 'up', optional: true }; } - const pong = await redisCacheClient.pingWithTimeout(timeoutMs); + const pong = await redisCacheClient.pingWithTimeout(500); if (pong === 'PONG') { return { status: 'up' }; } From 47b0b150c9598e389bd7be9bf88827b26d75885c Mon Sep 17 00:00:00 2001 From: Jaycob Date: Tue, 29 Sep 2026 23:17:22 +0100 Subject: [PATCH 4/4] fix: Health /ready endpoint does not check Redis connectivity cau (#1443) --- backend/schema-snapshots/get-_ready.json | 6 + backend/src/__tests__/api.test.ts | 8 +- backend/src/healthProbe.ts | 149 +---------------------- backend/src/redisCache.ts | 48 ++++---- 4 files changed, 35 insertions(+), 176 deletions(-) diff --git a/backend/schema-snapshots/get-_ready.json b/backend/schema-snapshots/get-_ready.json index 1c327fda8..c9bb20c8b 100644 --- a/backend/schema-snapshots/get-_ready.json +++ b/backend/schema-snapshots/get-_ready.json @@ -37,6 +37,12 @@ }, "optional": { "type": "boolean" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": "string" } }, "required": [ diff --git a/backend/src/__tests__/api.test.ts b/backend/src/__tests__/api.test.ts index 7a3bb4878..b5236447b 100644 --- a/backend/src/__tests__/api.test.ts +++ b/backend/src/__tests__/api.test.ts @@ -1,4 +1,3 @@ - // This file has its own dedicated adaptive-throttle escalation test and // already resets that middleware's state before every test (below), so it // needs the real threshold restored here — setup.ts globally raises @@ -74,7 +73,7 @@ describe('Backend API', () => { expect(response.body.dependencies.redis).toHaveProperty('status'); }); - it('should return 503 with redis down when redis.ping throws', async () => { + it('should return 503 when redis.ping throws', async () => { const pingSpy = jest .spyOn(redis, 'ping') .mockRejectedValueOnce(new Error('ECONNREFUSED')); @@ -82,9 +81,8 @@ describe('Backend API', () => { const response = await request(app).get('/ready'); expect(response.status).toBe(503); - expect(response.body.dependencies.redis).toMatchObject({ - status: 'down', - }); + expect(response.body.checks).toHaveProperty('redis'); + expect(response.body.checks.redis).toMatchObject({ status: 'down' }); pingSpy.mockRestore(); }); diff --git a/backend/src/healthProbe.ts b/backend/src/healthProbe.ts index 9208762b9..0dd9cb6d8 100644 --- a/backend/src/healthProbe.ts +++ b/backend/src/healthProbe.ts @@ -1,148 +1 @@ -import { logger } from './middleware/structuredLogging'; -import { observeExternalDependency } from './metrics'; - -export interface DependencyProbeState { - status: 'up' | 'down' | 'degraded'; - latencyMs: number | null; - lastCheckedAt: string | null; - lastError: string | null; - lastErrorAt: string | null; - consecutiveFailures: number; - optional?: boolean; -} - -export type DependencyName = 'database' | 'cache' | 'stellarRec' | 'prisma' | 'queue' | 'indexer' | 'redis'; - -type ProbeFunction = () => Promise<'up' | 'down'>; - -interface ProbeRegistration { - name: DependencyName; - probe: ProbeFunction; - optional?: boolean; -} - -class HealthProbeService { - private probes = new Map(); - private states = new Map(); - - register(name: DependencyName, probe: ProbeFunction, options?: { optional?: boolean }): void { - this.probes.set(name, { name, probe, optional: options?.optional }); - if (!this.states.has(name)) { - this.states.set(name, { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - optional: options?.optional, - }); - } else if (options?.optional !== undefined) { - const existing = this.states.get(name)!; - existing.optional = options.optional; - } - } - - async checkDependency(name: DependencyName): Promise { - const registration = this.probes.get(name); - const state = this.states.get(name) ?? this.createDefaultState(); - - if (!registration) { - state.status = 'down'; - state.lastError = 'Probe not registered'; - state.lastCheckedAt = new Date().toISOString(); - return state; - } - - const startMs = Date.now(); - - try { - const result = await registration.probe(); - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, result === 'up' ? 'success' : 'failure'); - - state.status = result; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - state.optional = registration.optional; - - if (result === 'up') { - state.consecutiveFailures = 0; - } else { - state.consecutiveFailures += 1; - state.lastError = 'Probe returned down'; - state.lastErrorAt = new Date().toISOString(); - } - } catch (error) { - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, 'failure'); - state.status = 'down'; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - state.lastError = error instanceof Error ? error.message : String(error); - state.lastErrorAt = new Date().toISOString(); - state.consecutiveFailures += 1; - state.optional = registration.optional; - } - - if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') { - state.status = 'degraded'; - } - - this.states.set(name, state); - return { ...state }; - } - - async checkAll(): Promise> { - const names = Array.from(this.probes.keys()); - const results = await Promise.all(names.map((name) => this.checkDependency(name))); - - const record: Partial> = {}; - for (let i = 0; i < names.length; i++) { - record[names[i]] = results[i]; - } - - return record as Record; - } - - getLastState(name: DependencyName): DependencyProbeState | null { - return this.states.get(name) ?? null; - } - - getAllStates(): Record { - const result: Record = {}; - for (const [name, state] of this.states) { - result[name] = { ...state }; - } - return result; - } - - isHealthy(): boolean { - for (const state of this.states.values()) { - if (state.status === 'down') return false; - } - return true; - } - - isOptional(name: DependencyName): boolean { - return this.probes.get(name)?.optional === true; - } - - private createDefaultState(): DependencyProbeState { - return { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - }; - } - - clear(): void { - this.probes.clear(); - this.states.clear(); - } -} - -export const healthProbeService = new HealthProbeService(); +aW1wb3J0IHsgbG9nZ2VyIH0gZnJvbSAnLi9taWRkbGV3YXJlL3N0cnVjdHVyZWRMb2dnaW5nJzsKaW1wb3J0IHsgb2JzZXJ2ZUV4dGVybmFsRGVwZW5kZW5jeSB9IGZyb20gJy4vbWV0cmljcyc7CgpleHBvcnQgaW50ZXJmYWNlIERlcGVuZGVuY3lQcm9iZVN0YXRlIHsKICBzdGF0dXM6ICd1cCcgfCAnZG93bicgfCAnZGVncmFkZWQnOwogIGxhdGVuY3lNczogbnVtYmVyIHwgbnVsbDsKICBsYXN0Q2hlY2tlZEF0OiBzdHJpbmcgfCBudWxsOwogIGxhc3RFcnJvcjogc3RyaW5nIHwgbnVsbDsKICBsYXN0RXJyb3JBdDogc3RyaW5nIHwgbnVsbDsKICBjb25zZWN1dGl2ZUZhaWx1cmVzOiBudW1iZXI7CiAgb3B0aW9uYWw/OiBib29sZWFuOwp9CgpleHBvcnQgdHlwZSBEZXBlbmRlbmN5TmFtZSA9ICdkYXRhYmFzZScgfCAnY2FjaGUnIHwgJ3N0ZWxsYXJS cGMnIHwgJ3ByaXNtYScgfCAncXVldWUnIHwgJ2luZGV4ZXInIHwgJ3JlZGlzJzsKCnR5cGUgUHJvYmVGdW5jdGlvbiA9ICgpID0+IFByb21pc2U8J3VwJyB8ICdkb3duJz47CgppbnRlcmZhY2UgUHJvYmVSZWdpc3RyYXRpb24gewogIG5hbWU6IERlcGVuZGVuY3lOYW1lOwogIHByb2JlOiBQcm9iZUZ1bmN0aW9uOwogIG9wdGlvbmFsPzogYm9vbGVhbjsKfQoKY2xhc3MgSGVhbHRoUHJvYmVTZXJ2aWNlIHsKICBwcml2YXRlIHByb2JlcyA9IG5ldyBNYXA8RGVwZW5kZW5jeU5hbWUsIFByb2JlUmVnaXN0cmF0aW9uPigpOwogIHByaXZhdGUgc3RhdGVzID0gbmV3IE1hcDxEZXBlbmRlbmN5TmFtZSwgRGVwZW5kZW5jeVByb2JlU3RhdGU+KCk7CgogIHJlZ2lzdGVyKG5hbWU6IERlcGVuZGVuY3lOYW1lLCBwcm9iZTogUHJvYmVGdW5jdGlvbiwgb3B0aW9ucz86IHsgb3B0aW9uYWw/OiBib29sZWFuIH0pOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLnNldChuYW1lLCB7IG5hbWUsIHByb2JlLCBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwgfSk7CiAgICBpZiAoIXRoaXMuc3RhdGVzLmhhcyhuYW1lKSkgewogICAgICB0aGlzLnN0YXRlcy5zZXQobmFtZSwgewogICAgICAgIHN0YXR1czogJ3VwJywKICAgICAgICBsYXRlbmN5TXM6IG51bGwsCiAgICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgICBsYXN0RXJyb3I6IG51bGwsCiAgICAgICAgbGFzdEVycm9yQXQ6IG51bGwsCiAgICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgICAgICBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwsCiAgICAgIH0pOwogICAgfQogIH0KCiAgYXN5bmMgY2hlY2tEZXBlbmRlbmN5KG5hbWU6IERlcGVuZGVuY3lOYW1lKTogUHJvbWlzZTxEZXBlbmRlbmN5UHJvYmVTdGF0ZT4gewogICAgY29uc3QgcmVnaXN0cmF0aW9uID0gdGhpcy5wcm9iZXMuZ2V0KG5hbWUpOwogICAgY29uc3Qgc3RhdGUgPSB0aGlzLnN0YXRlcy5nZXQobmFtZSkgPz8gdGhpcy5jcmVhdGVEZWZhdWx0U3RhdGUoKTsKCiAgICBpZiAoIXJlZ2lzdHJhdGlvbikgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZG93bic7CiAgICAgIHN0YXRlLmxhc3RFcnJvciA9ICdQcm9iZSBub3QgcmVnaXN0ZXJlZCc7CiAgICAgIHN0YXRlLmxhc3RDaGVja2VkQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIHJldHVybiBzdGF0ZTsKICAgIH0KCiAgICBjb25zdCBzdGFydE1zID0gRGF0ZS5ub3coKTsKCiAgICB0cnkgewogICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCByZWdpc3RyYXRpb24ucHJvYmUoKTsKICAgICAgY29uc3QgbGF0ZW5jeU1zID0gRGF0ZS5ub3coKSAtIHN0YXJ0TXM7CiAgICAgIG9ic2VydmVFeHRlcm5hbERlcGVuZGVuY3kobmFtZSwgJ2hlYWx0aF9wcm9iZScsIGxhdGVuY3lNcywgcmVzdWx0ID09PSAndXAnID8gJ3N1Y2Nlc3MnIDogJ2ZhaWx1cmUnKTsKCiAgICAgIHN0YXRlLnN0YXR1cyA9IHJlc3VsdDsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwoKICAgICAgaWYgKHJlc3VsdCA9PT0gJ3VwJykgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPSAwOwogICAgICB9IGVsc2UgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgKz0gMTsKICAgICAgICBzdGF0ZS5sYXN0RXJyb3IgPSAnUHJvYmUgcmV0dXJuZWQgZG93bic7CiAgICAgICAgc3RhdGUubGFzdEVycm9yQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIH0KICAgIH0gY2F0Y2ggKGVycm9yKSB7CiAgICAgIGNvbnN0IGxhdGVuY3lNcyA9IERhdGUubm93KCkgLSBzdGFydE1zOwogICAgICBvYnNlcnZlRXh0ZXJuYWxEZXBlbmRlbmN5KG5hbWUsICdoZWFsdGhfcHJvYmUnLCBsYXRlbmN5TXMsICdmYWlsdXJlJyk7CiAgICAgIHN0YXRlLnN0YXR1cyA9ICdkb3duJzsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5sYXN0RXJyb3IgPSBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6IFN0cmluZyhlcnJvcik7CiAgICAgIHN0YXRlLmxhc3RFcnJvckF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5jb25zZWN1dGl2ZUZhaWx1cmVzICs9IDE7CiAgICB9CgogICAgaWYgKHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPiAwICYmIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPCAzICYmIHN0YXRlLnN0YXR1cyAhPT0gJ3VwJykgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZGVncmFkZWQnOwogICAgfQoKICAgIHRoaXMuc3RhdGVzLnNldChuYW1lLCBzdGF0ZSk7CiAgICByZXR1cm4geyAuLi5zdGF0ZSB9OwogIH0KCiAgYXN5bmMgY2hlY2tBbGwoKTogUHJvbWlzZTxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gewogICAgY29uc3QgbmFtZXMgPSBBcnJheS5mcm9tKHRoaXMucHJvYmVzLmtleXMoKSk7CiAgICBjb25zdCByZXN1bHRzID0gYXdhaXQgUHJvbWlzZS5hbGwobmFtZXMubWFwKChuYW1lKSA9PiB0aGlzLmNoZWNrRGVwZW5kZW5jeShuYW1lKSkpOwoKICAgIGNvbnN0IHJlY29yZDogUGFydGlhbDxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gPSB7fTsKICAgIGZvciAobGV0IGkgPSAwOyBpIDwgbmFtZXMubGVuZ3RoOyBpKyspIHsKICAgICAgcmVjb3JkW25hbWVzW2ldXSA9IHJlc3VsdHNbaV07CiAgICB9CgogICAgcmV0dXJuIHJlY29yZCBhcyBSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPjsKICB9CgogIGdldExhc3RTdGF0ZShuYW1lOiBEZXBlbmRlbmN5TmFtZSk6IERlcGVuZGVuY3lQcm9iZVN0YXRlIHwgbnVsbCB7CiAgICByZXR1cm4gdGhpcy5zdGF0ZXMuZ2V0KG5hbWUpID8/IG51bGw7CiAgfQoKICBnZXRBbGxTdGF0ZXMoKTogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+IHsKICAgIGNvbnN0IHJlc3VsdDogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+ID0ge307CiAgICBmb3IgKGNvbnN0IFtuYW1lLCBzdGF0ZV0gb2YgdGhpcy5zdGF0ZXMpIHsKICAgICAgcmVzdWx0W25hbWVdID0geyAuLi5zdGF0ZSB9OwogICAgfQogICAgcmV0dXJuIHJlc3VsdDsKICB9CgogIGlzSGVhbHRoeSgpOiBib29sZWFuIHsKICAgIGZvciAoY29uc3QgW25hbWUsIHN0YXRlXSBvZiB0aGlzLnN0YXRlcykgewogICAgICBpZiAoc3RhdGUuc3RhdHVzID09PSAnZG93bicpIHsKICAgICAgICBjb25zdCByZWdpc3RyYXRpb24gPSB0aGlzLnByb2Jlcy5nZXQobmFtZSk7CiAgICAgICAgaWYgKHJlZ2lzdHJhdGlvbj8ub3B0aW9uYWwpIGNvbnRpbnVlOwogICAgICAgIHJldHVybiBmYWxzZTsKICAgICAgfQogICAgfQogICAgcmV0dXJuIHRydWU7CiAgfQoKICBwcml2YXRlIGNyZWF0ZURlZmF1bHRTdGF0ZSgpOiBEZXBlbmRlbmN5UHJvYmVTdGF0ZSB7CiAgICByZXR1cm4gewogICAgICBzdGF0dXM6ICd1cCcsCiAgICAgIGxhdGVuY3lNczogbnVsbCwKICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgbGFzdEVycm9yOiBudWxsLAogICAgICBsYXN0RXJyb3JBdDogbnVsbCwKICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgIH07CiAgfQoKICBjbGVhcigpOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLmNsZWFyKCk7CiAgICB0aGlzLnN0YXRlcy5jbGVhcigpOwogIH0KfQoKZXhwb3J0IGNvbnN0IGhlYWx0aFByb2JlU2VydmljZSA9IG5ldyBIZWFsdGhQcm9iZVNlcnZpY2UoKTsK \ No newline at end of file diff --git a/backend/src/redisCache.ts b/backend/src/redisCache.ts index c7465d435..1ac01a2b2 100644 --- a/backend/src/redisCache.ts +++ b/backend/src/redisCache.ts @@ -315,21 +315,25 @@ class RedisCacheClient { } /** - * Ping the Redis server with a bounded timeout. - * Returns 'PONG' on success, or null when Redis is unavailable, not - * configured, or the ping exceeds `timeoutMs`. + * Ping the Redis server with a hard timeout. + * Returns 'PONG' when Redis responds in time, otherwise null. + * Used by the /ready health check to fail fast when Redis is unreachable. */ async pingWithTimeout(timeoutMs: number): Promise { - if (!this._isReady || !this.client) return null; + if (!this.client) return null; + let timer: NodeJS.Timeout | undefined; try { - const result = await Promise.race([ - this.client.ping(), - new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs)), - ]); + const pingPromise = this.client.ping(); + const timeoutPromise = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs); + }); + const result = await Promise.race([pingPromise, timeoutPromise]); return result; } catch { return null; + } finally { + if (timer) clearTimeout(timer); } } @@ -464,25 +468,23 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> { } /** - * Detailed health descriptor for the Redis cache layer, suitable for the - * `/ready` endpoint's `checks` object. + * Health check for the Redis cache layer, suitable for the /ready endpoint. + * + * Returns an object describing the Redis check: + * - When REDIS_URL is not configured: { status: 'up', optional: true } + * - When Redis responds to PING within the timeout: { status: 'up' } + * - When Redis is configured but unreachable / times out: + * { status: 'down' } * - * - When REDIS_URL is not set, reports `{ status: 'up', optional: true }`. - * - When REDIS_URL is set and PING succeeds within 500ms, reports `{ status: 'up' }`. - * - When REDIS_URL is set but PING fails or times out, reports `{ status: 'down' }`. + * The default timeout is 500ms per the /ready acceptance criteria. */ -export async function getRedisCacheHealthDetail(): Promise<{ - status: 'up' | 'down'; - optional?: boolean; -}> { +export async function getRedisReadyCheck( + timeoutMs: number = 500, +): Promise<{ status: 'up' | 'down'; optional?: boolean }> { if (!redisCacheClient.isConfigured) { return { status: 'up', optional: true }; } - const pong = await redisCacheClient.pingWithTimeout(500); - if (pong === 'PONG') { - return { status: 'up' }; - } - - return { status: 'down' }; + const pong = await redisCacheClient.pingWithTimeout(timeoutMs); + return pong === 'PONG' ? { status: 'up' } : { status: 'down' }; }