diff --git a/backend/schema-snapshots/get-_ready.json b/backend/schema-snapshots/get-_ready.json index ce8460efd..c9bb20c8b 100644 --- a/backend/schema-snapshots/get-_ready.json +++ b/backend/schema-snapshots/get-_ready.json @@ -24,6 +24,31 @@ }, "indexer": { "type": "boolean" + }, + "redis": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "up", + "down" + ] + }, + "optional": { + "type": "boolean" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": "string" + } + }, + "required": [ + "status" + ], + "additionalProperties": false } }, "required": [ @@ -31,7 +56,8 @@ "stellarRpc", "database", "prisma", - "indexer" + "indexer", + "redis" ], "additionalProperties": false } @@ -42,4 +68,4 @@ "dependencies" ], "additionalProperties": false -} \ No newline at end of file +} diff --git a/backend/src/__tests__/api.test.ts b/backend/src/__tests__/api.test.ts index 9b5cc5cc6..b5236447b 100644 --- a/backend/src/__tests__/api.test.ts +++ b/backend/src/__tests__/api.test.ts @@ -9,6 +9,7 @@ process.env.ADAPTIVE_THROTTLE_SCORE_THRESHOLD = '6'; import request from 'supertest'; import app from '../index'; import { resetAdaptiveThrottleStateForTests } from '../middleware/adaptiveThrottle'; +import redis from '../utils/redis'; describe('Backend API', () => { beforeEach(() => { @@ -64,6 +65,27 @@ describe('Backend API', () => { expect(typeof response.body.dependencies.cache).toBe('boolean'); expect(typeof response.body.dependencies.stellarRpc).toBe('boolean'); }); + + it('should include redis check in dependencies', async () => { + const response = await request(app).get('/ready'); + + expect(response.body.dependencies).toHaveProperty('redis'); + expect(response.body.dependencies.redis).toHaveProperty('status'); + }); + + it('should return 503 when redis.ping throws', async () => { + const pingSpy = jest + .spyOn(redis, 'ping') + .mockRejectedValueOnce(new Error('ECONNREFUSED')); + + const response = await request(app).get('/ready'); + + expect(response.status).toBe(503); + expect(response.body.checks).toHaveProperty('redis'); + expect(response.body.checks.redis).toMatchObject({ status: 'down' }); + + pingSpy.mockRestore(); + }); }); // ─── Rate Limiting Tests (Issue #145) ──────────────────────────────────── diff --git a/backend/src/__tests__/healthReady.test.ts b/backend/src/__tests__/healthReady.test.ts new file mode 100644 index 000000000..bca0251ac --- /dev/null +++ b/backend/src/__tests__/healthReady.test.ts @@ -0,0 +1,45 @@ +import express from 'express'; +import request from 'supertest'; +import healthRouter, { setRedisClient } from '../routes/health'; + +const mockPing = jest.fn(); + +beforeEach(() => { + setRedisClient({ ping: mockPing }); +}); + +afterEach(() => { + setRedisClient(null); +}); + +describe('GET /ready', () => { + let app: express.Express; + + beforeEach(() => { + app = express(); + app.use(healthRouter); + jest.clearAllMocks(); + mockPing.mockResolvedValue('PONG'); + }); + + it('returns 200 when all dependencies are up', async () => { + const res = await request(app).get('/ready'); + expect(res.status).toBe[200); + expect(res.body.ready).toBe(true); + expect(res.body.checks.redis.status).toBe('up'); + }); + + it('returns 503 when redis.ping throws', async () => { + mockPing.mockRejectedValue(new Error('ECONNREFUSED')); + const res = await request(app).get('/ready'); + expect(res.status).toBe(503); + expect(res.body.checks.redis.status).toBe('down'); + }); + + it('returns 503 when redis.ping times out', async () => { + mockPing.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve('PONG'), 1000))); + const res = await request(app).get('/ready'); + expect(res.status).toBe(503); + expect(res.body.checks.redis.status).toBe('down'); + }); +}); diff --git a/backend/src/healthProbe.ts b/backend/src/healthProbe.ts index e30a4d1b2..0dd9cb6d8 100644 --- a/backend/src/healthProbe.ts +++ b/backend/src/healthProbe.ts @@ -1,136 +1 @@ -import { logger } from './middleware/structuredLogging'; -import { observeExternalDependency } from './metrics'; - -export interface DependencyProbeState { - status: 'up' | 'down' | 'degraded'; - latencyMs: number | null; - lastCheckedAt: string | null; - lastError: string | null; - lastErrorAt: string | null; - consecutiveFailures: number; -} - -export type DependencyName = 'database' | 'cache' | 'stellarRpc' | 'prisma' | 'queue' | 'indexer'; - -type ProbeFunction = () => Promise<'up' | 'down'>; - -interface ProbeRegistration { - name: DependencyName; - probe: ProbeFunction; -} - -class HealthProbeService { - private probes = new Map(); - private states = new Map(); - - register(name: DependencyName, probe: ProbeFunction): void { - this.probes.set(name, { name, probe }); - if (!this.states.has(name)) { - this.states.set(name, { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - }); - } - } - - async checkDependency(name: DependencyName): Promise { - const registration = this.probes.get(name); - const state = this.states.get(name) ?? this.createDefaultState(); - - if (!registration) { - state.status = 'down'; - state.lastError = 'Probe not registered'; - state.lastCheckedAt = new Date().toISOString(); - return state; - } - - const startMs = Date.now(); - - try { - const result = await registration.probe(); - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, result === 'up' ? 'success' : 'failure'); - - state.status = result; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - - if (result === 'up') { - state.consecutiveFailures = 0; - } else { - state.consecutiveFailures += 1; - state.lastError = 'Probe returned down'; - state.lastErrorAt = new Date().toISOString(); - } - } catch (error) { - const latencyMs = Date.now() - startMs; - observeExternalDependency(name, 'health_probe', latencyMs, 'failure'); - state.status = 'down'; - state.latencyMs = latencyMs; - state.lastCheckedAt = new Date().toISOString(); - state.lastError = error instanceof Error ? error.message : String(error); - state.lastErrorAt = new Date().toISOString(); - state.consecutiveFailures += 1; - } - - if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') { - state.status = 'degraded'; - } - - this.states.set(name, state); - return { ...state }; - } - - async checkAll(): Promise> { - const names = Array.from(this.probes.keys()); - const results = await Promise.all(names.map((name) => this.checkDependency(name))); - - const record: Partial> = {}; - for (let i = 0; i < names.length; i++) { - record[names[i]] = results[i]; - } - - return record as Record; - } - - getLastState(name: DependencyName): DependencyProbeState | null { - return this.states.get(name) ?? null; - } - - getAllStates(): Record { - const result: Record = {}; - for (const [name, state] of this.states) { - result[name] = { ...state }; - } - return result; - } - - isHealthy(): boolean { - for (const state of this.states.values()) { - if (state.status === 'down') return false; - } - return true; - } - - private createDefaultState(): DependencyProbeState { - return { - status: 'up', - latencyMs: null, - lastCheckedAt: null, - lastError: null, - lastErrorAt: null, - consecutiveFailures: 0, - }; - } - - clear(): void { - this.probes.clear(); - this.states.clear(); - } -} - -export const healthProbeService = new HealthProbeService(); +aW1wb3J0IHsgbG9nZ2VyIH0gZnJvbSAnLi9taWRkbGV3YXJlL3N0cnVjdHVyZWRMb2dnaW5nJzsKaW1wb3J0IHsgb2JzZXJ2ZUV4dGVybmFsRGVwZW5kZW5jeSB9IGZyb20gJy4vbWV0cmljcyc7CgpleHBvcnQgaW50ZXJmYWNlIERlcGVuZGVuY3lQcm9iZVN0YXRlIHsKICBzdGF0dXM6ICd1cCcgfCAnZG93bicgfCAnZGVncmFkZWQnOwogIGxhdGVuY3lNczogbnVtYmVyIHwgbnVsbDsKICBsYXN0Q2hlY2tlZEF0OiBzdHJpbmcgfCBudWxsOwogIGxhc3RFcnJvcjogc3RyaW5nIHwgbnVsbDsKICBsYXN0RXJyb3JBdDogc3RyaW5nIHwgbnVsbDsKICBjb25zZWN1dGl2ZUZhaWx1cmVzOiBudW1iZXI7CiAgb3B0aW9uYWw/OiBib29sZWFuOwp9CgpleHBvcnQgdHlwZSBEZXBlbmRlbmN5TmFtZSA9ICdkYXRhYmFzZScgfCAnY2FjaGUnIHwgJ3N0ZWxsYXJS cGMnIHwgJ3ByaXNtYScgfCAncXVldWUnIHwgJ2luZGV4ZXInIHwgJ3JlZGlzJzsKCnR5cGUgUHJvYmVGdW5jdGlvbiA9ICgpID0+IFByb21pc2U8J3VwJyB8ICdkb3duJz47CgppbnRlcmZhY2UgUHJvYmVSZWdpc3RyYXRpb24gewogIG5hbWU6IERlcGVuZGVuY3lOYW1lOwogIHByb2JlOiBQcm9iZUZ1bmN0aW9uOwogIG9wdGlvbmFsPzogYm9vbGVhbjsKfQoKY2xhc3MgSGVhbHRoUHJvYmVTZXJ2aWNlIHsKICBwcml2YXRlIHByb2JlcyA9IG5ldyBNYXA8RGVwZW5kZW5jeU5hbWUsIFByb2JlUmVnaXN0cmF0aW9uPigpOwogIHByaXZhdGUgc3RhdGVzID0gbmV3IE1hcDxEZXBlbmRlbmN5TmFtZSwgRGVwZW5kZW5jeVByb2JlU3RhdGU+KCk7CgogIHJlZ2lzdGVyKG5hbWU6IERlcGVuZGVuY3lOYW1lLCBwcm9iZTogUHJvYmVGdW5jdGlvbiwgb3B0aW9ucz86IHsgb3B0aW9uYWw/OiBib29sZWFuIH0pOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLnNldChuYW1lLCB7IG5hbWUsIHByb2JlLCBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwgfSk7CiAgICBpZiAoIXRoaXMuc3RhdGVzLmhhcyhuYW1lKSkgewogICAgICB0aGlzLnN0YXRlcy5zZXQobmFtZSwgewogICAgICAgIHN0YXR1czogJ3VwJywKICAgICAgICBsYXRlbmN5TXM6IG51bGwsCiAgICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgICBsYXN0RXJyb3I6IG51bGwsCiAgICAgICAgbGFzdEVycm9yQXQ6IG51bGwsCiAgICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgICAgICBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwsCiAgICAgIH0pOwogICAgfQogIH0KCiAgYXN5bmMgY2hlY2tEZXBlbmRlbmN5KG5hbWU6IERlcGVuZGVuY3lOYW1lKTogUHJvbWlzZTxEZXBlbmRlbmN5UHJvYmVTdGF0ZT4gewogICAgY29uc3QgcmVnaXN0cmF0aW9uID0gdGhpcy5wcm9iZXMuZ2V0KG5hbWUpOwogICAgY29uc3Qgc3RhdGUgPSB0aGlzLnN0YXRlcy5nZXQobmFtZSkgPz8gdGhpcy5jcmVhdGVEZWZhdWx0U3RhdGUoKTsKCiAgICBpZiAoIXJlZ2lzdHJhdGlvbikgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZG93bic7CiAgICAgIHN0YXRlLmxhc3RFcnJvciA9ICdQcm9iZSBub3QgcmVnaXN0ZXJlZCc7CiAgICAgIHN0YXRlLmxhc3RDaGVja2VkQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIHJldHVybiBzdGF0ZTsKICAgIH0KCiAgICBjb25zdCBzdGFydE1zID0gRGF0ZS5ub3coKTsKCiAgICB0cnkgewogICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCByZWdpc3RyYXRpb24ucHJvYmUoKTsKICAgICAgY29uc3QgbGF0ZW5jeU1zID0gRGF0ZS5ub3coKSAtIHN0YXJ0TXM7CiAgICAgIG9ic2VydmVFeHRlcm5hbERlcGVuZGVuY3kobmFtZSwgJ2hlYWx0aF9wcm9iZScsIGxhdGVuY3lNcywgcmVzdWx0ID09PSAndXAnID8gJ3N1Y2Nlc3MnIDogJ2ZhaWx1cmUnKTsKCiAgICAgIHN0YXRlLnN0YXR1cyA9IHJlc3VsdDsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwoKICAgICAgaWYgKHJlc3VsdCA9PT0gJ3VwJykgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPSAwOwogICAgICB9IGVsc2UgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgKz0gMTsKICAgICAgICBzdGF0ZS5sYXN0RXJyb3IgPSAnUHJvYmUgcmV0dXJuZWQgZG93bic7CiAgICAgICAgc3RhdGUubGFzdEVycm9yQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIH0KICAgIH0gY2F0Y2ggKGVycm9yKSB7CiAgICAgIGNvbnN0IGxhdGVuY3lNcyA9IERhdGUubm93KCkgLSBzdGFydE1zOwogICAgICBvYnNlcnZlRXh0ZXJuYWxEZXBlbmRlbmN5KG5hbWUsICdoZWFsdGhfcHJvYmUnLCBsYXRlbmN5TXMsICdmYWlsdXJlJyk7CiAgICAgIHN0YXRlLnN0YXR1cyA9ICdkb3duJzsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5sYXN0RXJyb3IgPSBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6IFN0cmluZyhlcnJvcik7CiAgICAgIHN0YXRlLmxhc3RFcnJvckF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5jb25zZWN1dGl2ZUZhaWx1cmVzICs9IDE7CiAgICB9CgogICAgaWYgKHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPiAwICYmIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPCAzICYmIHN0YXRlLnN0YXR1cyAhPT0gJ3VwJykgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZGVncmFkZWQnOwogICAgfQoKICAgIHRoaXMuc3RhdGVzLnNldChuYW1lLCBzdGF0ZSk7CiAgICByZXR1cm4geyAuLi5zdGF0ZSB9OwogIH0KCiAgYXN5bmMgY2hlY2tBbGwoKTogUHJvbWlzZTxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gewogICAgY29uc3QgbmFtZXMgPSBBcnJheS5mcm9tKHRoaXMucHJvYmVzLmtleXMoKSk7CiAgICBjb25zdCByZXN1bHRzID0gYXdhaXQgUHJvbWlzZS5hbGwobmFtZXMubWFwKChuYW1lKSA9PiB0aGlzLmNoZWNrRGVwZW5kZW5jeShuYW1lKSkpOwoKICAgIGNvbnN0IHJlY29yZDogUGFydGlhbDxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gPSB7fTsKICAgIGZvciAobGV0IGkgPSAwOyBpIDwgbmFtZXMubGVuZ3RoOyBpKyspIHsKICAgICAgcmVjb3JkW25hbWVzW2ldXSA9IHJlc3VsdHNbaV07CiAgICB9CgogICAgcmV0dXJuIHJlY29yZCBhcyBSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPjsKICB9CgogIGdldExhc3RTdGF0ZShuYW1lOiBEZXBlbmRlbmN5TmFtZSk6IERlcGVuZGVuY3lQcm9iZVN0YXRlIHwgbnVsbCB7CiAgICByZXR1cm4gdGhpcy5zdGF0ZXMuZ2V0KG5hbWUpID8/IG51bGw7CiAgfQoKICBnZXRBbGxTdGF0ZXMoKTogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+IHsKICAgIGNvbnN0IHJlc3VsdDogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+ID0ge307CiAgICBmb3IgKGNvbnN0IFtuYW1lLCBzdGF0ZV0gb2YgdGhpcy5zdGF0ZXMpIHsKICAgICAgcmVzdWx0W25hbWVdID0geyAuLi5zdGF0ZSB9OwogICAgfQogICAgcmV0dXJuIHJlc3VsdDsKICB9CgogIGlzSGVhbHRoeSgpOiBib29sZWFuIHsKICAgIGZvciAoY29uc3QgW25hbWUsIHN0YXRlXSBvZiB0aGlzLnN0YXRlcykgewogICAgICBpZiAoc3RhdGUuc3RhdHVzID09PSAnZG93bicpIHsKICAgICAgICBjb25zdCByZWdpc3RyYXRpb24gPSB0aGlzLnByb2Jlcy5nZXQobmFtZSk7CiAgICAgICAgaWYgKHJlZ2lzdHJhdGlvbj8ub3B0aW9uYWwpIGNvbnRpbnVlOwogICAgICAgIHJldHVybiBmYWxzZTsKICAgICAgfQogICAgfQogICAgcmV0dXJuIHRydWU7CiAgfQoKICBwcml2YXRlIGNyZWF0ZURlZmF1bHRTdGF0ZSgpOiBEZXBlbmRlbmN5UHJvYmVTdGF0ZSB7CiAgICByZXR1cm4gewogICAgICBzdGF0dXM6ICd1cCcsCiAgICAgIGxhdGVuY3lNczogbnVsbCwKICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgbGFzdEVycm9yOiBudWxsLAogICAgICBsYXN0RXJyb3JBdDogbnVsbCwKICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgIH07CiAgfQoKICBjbGVhcigpOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLmNsZWFyKCk7CiAgICB0aGlzLnN0YXRlcy5jbGVhcigpOwogIH0KfQoKZXhwb3J0IGNvbnN0IGhlYWx0aFByb2JlU2VydmljZSA9IG5ldyBIZWFsdGhQcm9iZVNlcnZpY2UoKTsK \ No newline at end of file diff --git a/backend/src/redisCache.ts b/backend/src/redisCache.ts index 7ff75a30e..1ac01a2b2 100644 --- a/backend/src/redisCache.ts +++ b/backend/src/redisCache.ts @@ -314,6 +314,29 @@ class RedisCacheClient { } } + /** + * Ping the Redis server with a hard timeout. + * Returns 'PONG' when Redis responds in time, otherwise null. + * Used by the /ready health check to fail fast when Redis is unreachable. + */ + async pingWithTimeout(timeoutMs: number): Promise { + if (!this.client) return null; + + let timer: NodeJS.Timeout | undefined; + try { + const pingPromise = this.client.ping(); + const timeoutPromise = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), timeoutMs); + }); + const result = await Promise.race([pingPromise, timeoutPromise]); + return result; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } + } + /** * Gracefully close the Redis connection. * Called during graceful shutdown. @@ -443,3 +466,25 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> { const pong = await redisCacheClient.ping(); return pong === 'PONG' ? 'up' : 'degraded'; } + +/** + * Health check for the Redis cache layer, suitable for the /ready endpoint. + * + * Returns an object describing the Redis check: + * - When REDIS_URL is not configured: { status: 'up', optional: true } + * - When Redis responds to PING within the timeout: { status: 'up' } + * - When Redis is configured but unreachable / times out: + * { status: 'down' } + * + * The default timeout is 500ms per the /ready acceptance criteria. + */ +export async function getRedisReadyCheck( + timeoutMs: number = 500, +): Promise<{ status: 'up' | 'down'; optional?: boolean }> { + if (!redisCacheClient.isConfigured) { + return { status: 'up', optional: true }; + } + + const pong = await redisCacheClient.pingWithTimeout(timeoutMs); + return pong === 'PONG' ? { status: 'up' } : { status: 'down' }; +}