Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions backend/schema-snapshots/get-_ready.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,40 @@
},
"indexer": {
"type": "boolean"
},
"redis": {
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": [
"up",
"down"
]
},
"optional": {
"type": "boolean"
},
"latencyMs": {
"type": "number"
},
"error": {
"type": "string"
}
},
"required": [
"status"
],
"additionalProperties": false
}
},
"required": [
"cache",
"stellarRpc",
"database",
"prisma",
"indexer"
"indexer",
"redis"
],
"additionalProperties": false
}
Expand All @@ -42,4 +68,4 @@
"dependencies"
],
"additionalProperties": false
}
}
22 changes: 22 additions & 0 deletions backend/src/__tests__/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ process.env.ADAPTIVE_THROTTLE_SCORE_THRESHOLD = '6';
import request from 'supertest';
import app from '../index';
import { resetAdaptiveThrottleStateForTests } from '../middleware/adaptiveThrottle';
import redis from '../utils/redis';

describe('Backend API', () => {
beforeEach(() => {
Expand Down Expand Up @@ -64,6 +65,27 @@ describe('Backend API', () => {
expect(typeof response.body.dependencies.cache).toBe('boolean');
expect(typeof response.body.dependencies.stellarRpc).toBe('boolean');
});

it('should include redis check in dependencies', async () => {
const response = await request(app).get('/ready');

expect(response.body.dependencies).toHaveProperty('redis');
expect(response.body.dependencies.redis).toHaveProperty('status');
});

it('should return 503 when redis.ping throws', async () => {
const pingSpy = jest
.spyOn(redis, 'ping')
.mockRejectedValueOnce(new Error('ECONNREFUSED'));

const response = await request(app).get('/ready');

expect(response.status).toBe(503);
expect(response.body.checks).toHaveProperty('redis');
expect(response.body.checks.redis).toMatchObject({ status: 'down' });

pingSpy.mockRestore();
});
});

// ─── Rate Limiting Tests (Issue #145) ────────────────────────────────────
Expand Down
45 changes: 45 additions & 0 deletions backend/src/__tests__/healthReady.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import express from 'express';
import request from 'supertest';
import healthRouter, { setRedisClient } from '../routes/health';

const mockPing = jest.fn();

beforeEach(() => {
setRedisClient({ ping: mockPing });
});

afterEach(() => {
setRedisClient(null);
});

describe('GET /ready', () => {
let app: express.Express;

beforeEach(() => {
app = express();
app.use(healthRouter);
jest.clearAllMocks();
mockPing.mockResolvedValue('PONG');
});

it('returns 200 when all dependencies are up', async () => {
const res = await request(app).get('/ready');
expect(res.status).toBe[200);
expect(res.body.ready).toBe(true);
expect(res.body.checks.redis.status).toBe('up');
});

it('returns 503 when redis.ping throws', async () => {
mockPing.mockRejectedValue(new Error('ECONNREFUSED'));
const res = await request(app).get('/ready');
expect(res.status).toBe(503);
expect(res.body.checks.redis.status).toBe('down');
});

it('returns 503 when redis.ping times out', async () => {
mockPing.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve('PONG'), 1000)));
const res = await request(app).get('/ready');
expect(res.status).toBe(503);
expect(res.body.checks.redis.status).toBe('down');
});
});
137 changes: 1 addition & 136 deletions backend/src/healthProbe.ts
Original file line number Diff line number Diff line change
@@ -1,136 +1 @@
import { logger } from './middleware/structuredLogging';
import { observeExternalDependency } from './metrics';

export interface DependencyProbeState {
status: 'up' | 'down' | 'degraded';
latencyMs: number | null;
lastCheckedAt: string | null;
lastError: string | null;
lastErrorAt: string | null;
consecutiveFailures: number;
}

export type DependencyName = 'database' | 'cache' | 'stellarRpc' | 'prisma' | 'queue' | 'indexer';

type ProbeFunction = () => Promise<'up' | 'down'>;

interface ProbeRegistration {
name: DependencyName;
probe: ProbeFunction;
}

class HealthProbeService {
private probes = new Map<DependencyName, ProbeRegistration>();
private states = new Map<DependencyName, DependencyProbeState>();

register(name: DependencyName, probe: ProbeFunction): void {
this.probes.set(name, { name, probe });
if (!this.states.has(name)) {
this.states.set(name, {
status: 'up',
latencyMs: null,
lastCheckedAt: null,
lastError: null,
lastErrorAt: null,
consecutiveFailures: 0,
});
}
}

async checkDependency(name: DependencyName): Promise<DependencyProbeState> {
const registration = this.probes.get(name);
const state = this.states.get(name) ?? this.createDefaultState();

if (!registration) {
state.status = 'down';
state.lastError = 'Probe not registered';
state.lastCheckedAt = new Date().toISOString();
return state;
}

const startMs = Date.now();

try {
const result = await registration.probe();
const latencyMs = Date.now() - startMs;
observeExternalDependency(name, 'health_probe', latencyMs, result === 'up' ? 'success' : 'failure');

state.status = result;
state.latencyMs = latencyMs;
state.lastCheckedAt = new Date().toISOString();

if (result === 'up') {
state.consecutiveFailures = 0;
} else {
state.consecutiveFailures += 1;
state.lastError = 'Probe returned down';
state.lastErrorAt = new Date().toISOString();
}
} catch (error) {
const latencyMs = Date.now() - startMs;
observeExternalDependency(name, 'health_probe', latencyMs, 'failure');
state.status = 'down';
state.latencyMs = latencyMs;
state.lastCheckedAt = new Date().toISOString();
state.lastError = error instanceof Error ? error.message : String(error);
state.lastErrorAt = new Date().toISOString();
state.consecutiveFailures += 1;
}

if (state.consecutiveFailures > 0 && state.consecutiveFailures < 3 && state.status !== 'up') {
state.status = 'degraded';
}

this.states.set(name, state);
return { ...state };
}

async checkAll(): Promise<Record<DependencyName, DependencyProbeState>> {
const names = Array.from(this.probes.keys());
const results = await Promise.all(names.map((name) => this.checkDependency(name)));

const record: Partial<Record<DependencyName, DependencyProbeState>> = {};
for (let i = 0; i < names.length; i++) {
record[names[i]] = results[i];
}

return record as Record<DependencyName, DependencyProbeState>;
}

getLastState(name: DependencyName): DependencyProbeState | null {
return this.states.get(name) ?? null;
}

getAllStates(): Record<string, DependencyProbeState> {
const result: Record<string, DependencyProbeState> = {};
for (const [name, state] of this.states) {
result[name] = { ...state };
}
return result;
}

isHealthy(): boolean {
for (const state of this.states.values()) {
if (state.status === 'down') return false;
}
return true;
}

private createDefaultState(): DependencyProbeState {
return {
status: 'up',
latencyMs: null,
lastCheckedAt: null,
lastError: null,
lastErrorAt: null,
consecutiveFailures: 0,
};
}

clear(): void {
this.probes.clear();
this.states.clear();
}
}

export const healthProbeService = new HealthProbeService();
aW1wb3J0IHsgbG9nZ2VyIH0gZnJvbSAnLi9taWRkbGV3YXJlL3N0cnVjdHVyZWRMb2dnaW5nJzsKaW1wb3J0IHsgb2JzZXJ2ZUV4dGVybmFsRGVwZW5kZW5jeSB9IGZyb20gJy4vbWV0cmljcyc7CgpleHBvcnQgaW50ZXJmYWNlIERlcGVuZGVuY3lQcm9iZVN0YXRlIHsKICBzdGF0dXM6ICd1cCcgfCAnZG93bicgfCAnZGVncmFkZWQnOwogIGxhdGVuY3lNczogbnVtYmVyIHwgbnVsbDsKICBsYXN0Q2hlY2tlZEF0OiBzdHJpbmcgfCBudWxsOwogIGxhc3RFcnJvcjogc3RyaW5nIHwgbnVsbDsKICBsYXN0RXJyb3JBdDogc3RyaW5nIHwgbnVsbDsKICBjb25zZWN1dGl2ZUZhaWx1cmVzOiBudW1iZXI7CiAgb3B0aW9uYWw/OiBib29sZWFuOwp9CgpleHBvcnQgdHlwZSBEZXBlbmRlbmN5TmFtZSA9ICdkYXRhYmFzZScgfCAnY2FjaGUnIHwgJ3N0ZWxsYXJS cGMnIHwgJ3ByaXNtYScgfCAncXVldWUnIHwgJ2luZGV4ZXInIHwgJ3JlZGlzJzsKCnR5cGUgUHJvYmVGdW5jdGlvbiA9ICgpID0+IFByb21pc2U8J3VwJyB8ICdkb3duJz47CgppbnRlcmZhY2UgUHJvYmVSZWdpc3RyYXRpb24gewogIG5hbWU6IERlcGVuZGVuY3lOYW1lOwogIHByb2JlOiBQcm9iZUZ1bmN0aW9uOwogIG9wdGlvbmFsPzogYm9vbGVhbjsKfQoKY2xhc3MgSGVhbHRoUHJvYmVTZXJ2aWNlIHsKICBwcml2YXRlIHByb2JlcyA9IG5ldyBNYXA8RGVwZW5kZW5jeU5hbWUsIFByb2JlUmVnaXN0cmF0aW9uPigpOwogIHByaXZhdGUgc3RhdGVzID0gbmV3IE1hcDxEZXBlbmRlbmN5TmFtZSwgRGVwZW5kZW5jeVByb2JlU3RhdGU+KCk7CgogIHJlZ2lzdGVyKG5hbWU6IERlcGVuZGVuY3lOYW1lLCBwcm9iZTogUHJvYmVGdW5jdGlvbiwgb3B0aW9ucz86IHsgb3B0aW9uYWw/OiBib29sZWFuIH0pOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLnNldChuYW1lLCB7IG5hbWUsIHByb2JlLCBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwgfSk7CiAgICBpZiAoIXRoaXMuc3RhdGVzLmhhcyhuYW1lKSkgewogICAgICB0aGlzLnN0YXRlcy5zZXQobmFtZSwgewogICAgICAgIHN0YXR1czogJ3VwJywKICAgICAgICBsYXRlbmN5TXM6IG51bGwsCiAgICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgICBsYXN0RXJyb3I6IG51bGwsCiAgICAgICAgbGFzdEVycm9yQXQ6IG51bGwsCiAgICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgICAgICBvcHRpb25hbDogb3B0aW9ucz8ub3B0aW9uYWwsCiAgICAgIH0pOwogICAgfQogIH0KCiAgYXN5bmMgY2hlY2tEZXBlbmRlbmN5KG5hbWU6IERlcGVuZGVuY3lOYW1lKTogUHJvbWlzZTxEZXBlbmRlbmN5UHJvYmVTdGF0ZT4gewogICAgY29uc3QgcmVnaXN0cmF0aW9uID0gdGhpcy5wcm9iZXMuZ2V0KG5hbWUpOwogICAgY29uc3Qgc3RhdGUgPSB0aGlzLnN0YXRlcy5nZXQobmFtZSkgPz8gdGhpcy5jcmVhdGVEZWZhdWx0U3RhdGUoKTsKCiAgICBpZiAoIXJlZ2lzdHJhdGlvbikgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZG93bic7CiAgICAgIHN0YXRlLmxhc3RFcnJvciA9ICdQcm9iZSBub3QgcmVnaXN0ZXJlZCc7CiAgICAgIHN0YXRlLmxhc3RDaGVja2VkQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIHJldHVybiBzdGF0ZTsKICAgIH0KCiAgICBjb25zdCBzdGFydE1zID0gRGF0ZS5ub3coKTsKCiAgICB0cnkgewogICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCByZWdpc3RyYXRpb24ucHJvYmUoKTsKICAgICAgY29uc3QgbGF0ZW5jeU1zID0gRGF0ZS5ub3coKSAtIHN0YXJ0TXM7CiAgICAgIG9ic2VydmVFeHRlcm5hbERlcGVuZGVuY3kobmFtZSwgJ2hlYWx0aF9wcm9iZScsIGxhdGVuY3lNcywgcmVzdWx0ID09PSAndXAnID8gJ3N1Y2Nlc3MnIDogJ2ZhaWx1cmUnKTsKCiAgICAgIHN0YXRlLnN0YXR1cyA9IHJlc3VsdDsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwoKICAgICAgaWYgKHJlc3VsdCA9PT0gJ3VwJykgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPSAwOwogICAgICB9IGVsc2UgewogICAgICAgIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgKz0gMTsKICAgICAgICBzdGF0ZS5sYXN0RXJyb3IgPSAnUHJvYmUgcmV0dXJuZWQgZG93bic7CiAgICAgICAgc3RhdGUubGFzdEVycm9yQXQgPSBuZXcgRGF0ZSgpLnRvSVNPU3RyaW5nKCk7CiAgICAgIH0KICAgIH0gY2F0Y2ggKGVycm9yKSB7CiAgICAgIGNvbnN0IGxhdGVuY3lNcyA9IERhdGUubm93KCkgLSBzdGFydE1zOwogICAgICBvYnNlcnZlRXh0ZXJuYWxEZXBlbmRlbmN5KG5hbWUsICdoZWFsdGhfcHJvYmUnLCBsYXRlbmN5TXMsICdmYWlsdXJlJyk7CiAgICAgIHN0YXRlLnN0YXR1cyA9ICdkb3duJzsKICAgICAgc3RhdGUubGF0ZW5jeU1zID0gbGF0ZW5jeU1zOwogICAgICBzdGF0ZS5sYXN0Q2hlY2tlZEF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5sYXN0RXJyb3IgPSBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6IFN0cmluZyhlcnJvcik7CiAgICAgIHN0YXRlLmxhc3RFcnJvckF0ID0gbmV3IERhdGUoKS50b0lTT1N0cmluZygpOwogICAgICBzdGF0ZS5jb25zZWN1dGl2ZUZhaWx1cmVzICs9IDE7CiAgICB9CgogICAgaWYgKHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPiAwICYmIHN0YXRlLmNvbnNlY3V0aXZlRmFpbHVyZXMgPCAzICYmIHN0YXRlLnN0YXR1cyAhPT0gJ3VwJykgewogICAgICBzdGF0ZS5zdGF0dXMgPSAnZGVncmFkZWQnOwogICAgfQoKICAgIHRoaXMuc3RhdGVzLnNldChuYW1lLCBzdGF0ZSk7CiAgICByZXR1cm4geyAuLi5zdGF0ZSB9OwogIH0KCiAgYXN5bmMgY2hlY2tBbGwoKTogUHJvbWlzZTxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gewogICAgY29uc3QgbmFtZXMgPSBBcnJheS5mcm9tKHRoaXMucHJvYmVzLmtleXMoKSk7CiAgICBjb25zdCByZXN1bHRzID0gYXdhaXQgUHJvbWlzZS5hbGwobmFtZXMubWFwKChuYW1lKSA9PiB0aGlzLmNoZWNrRGVwZW5kZW5jeShuYW1lKSkpOwoKICAgIGNvbnN0IHJlY29yZDogUGFydGlhbDxSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPj4gPSB7fTsKICAgIGZvciAobGV0IGkgPSAwOyBpIDwgbmFtZXMubGVuZ3RoOyBpKyspIHsKICAgICAgcmVjb3JkW25hbWVzW2ldXSA9IHJlc3VsdHNbaV07CiAgICB9CgogICAgcmV0dXJuIHJlY29yZCBhcyBSZWNvcmQ8RGVwZW5kZW5jeU5hbWUsIERlcGVuZGVuY3lQcm9iZVN0YXRlPjsKICB9CgogIGdldExhc3RTdGF0ZShuYW1lOiBEZXBlbmRlbmN5TmFtZSk6IERlcGVuZGVuY3lQcm9iZVN0YXRlIHwgbnVsbCB7CiAgICByZXR1cm4gdGhpcy5zdGF0ZXMuZ2V0KG5hbWUpID8/IG51bGw7CiAgfQoKICBnZXRBbGxTdGF0ZXMoKTogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+IHsKICAgIGNvbnN0IHJlc3VsdDogUmVjb3JkPHN0cmluZywgRGVwZW5kZW5jeVByb2JlU3RhdGU+ID0ge307CiAgICBmb3IgKGNvbnN0IFtuYW1lLCBzdGF0ZV0gb2YgdGhpcy5zdGF0ZXMpIHsKICAgICAgcmVzdWx0W25hbWVdID0geyAuLi5zdGF0ZSB9OwogICAgfQogICAgcmV0dXJuIHJlc3VsdDsKICB9CgogIGlzSGVhbHRoeSgpOiBib29sZWFuIHsKICAgIGZvciAoY29uc3QgW25hbWUsIHN0YXRlXSBvZiB0aGlzLnN0YXRlcykgewogICAgICBpZiAoc3RhdGUuc3RhdHVzID09PSAnZG93bicpIHsKICAgICAgICBjb25zdCByZWdpc3RyYXRpb24gPSB0aGlzLnByb2Jlcy5nZXQobmFtZSk7CiAgICAgICAgaWYgKHJlZ2lzdHJhdGlvbj8ub3B0aW9uYWwpIGNvbnRpbnVlOwogICAgICAgIHJldHVybiBmYWxzZTsKICAgICAgfQogICAgfQogICAgcmV0dXJuIHRydWU7CiAgfQoKICBwcml2YXRlIGNyZWF0ZURlZmF1bHRTdGF0ZSgpOiBEZXBlbmRlbmN5UHJvYmVTdGF0ZSB7CiAgICByZXR1cm4gewogICAgICBzdGF0dXM6ICd1cCcsCiAgICAgIGxhdGVuY3lNczogbnVsbCwKICAgICAgbGFzdENoZWNrZWRBdDogbnVsbCwKICAgICAgbGFzdEVycm9yOiBudWxsLAogICAgICBsYXN0RXJyb3JBdDogbnVsbCwKICAgICAgY29uc2VjdXRpdmVGYWlsdXJlczogMCwKICAgIH07CiAgfQoKICBjbGVhcigpOiB2b2lkIHsKICAgIHRoaXMucHJvYmVzLmNsZWFyKCk7CiAgICB0aGlzLnN0YXRlcy5jbGVhcigpOwogIH0KfQoKZXhwb3J0IGNvbnN0IGhlYWx0aFByb2JlU2VydmljZSA9IG5ldyBIZWFsdGhQcm9iZVNlcnZpY2UoKTsK
45 changes: 45 additions & 0 deletions backend/src/redisCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,29 @@ class RedisCacheClient {
}
}

/**
* Ping the Redis server with a hard timeout.
* Returns 'PONG' when Redis responds in time, otherwise null.
* Used by the /ready health check to fail fast when Redis is unreachable.
*/
async pingWithTimeout(timeoutMs: number): Promise<string | null> {
if (!this.client) return null;

let timer: NodeJS.Timeout | undefined;
try {
const pingPromise = this.client.ping();
const timeoutPromise = new Promise<null>((resolve) => {
timer = setTimeout(() => resolve(null), timeoutMs);
});
const result = await Promise.race([pingPromise, timeoutPromise]);
return result;
} catch {
return null;
} finally {
if (timer) clearTimeout(timer);
}
}

/**
* Gracefully close the Redis connection.
* Called during graceful shutdown.
Expand Down Expand Up @@ -443,3 +466,25 @@ export async function getRedisCacheHealth(): Promise<'up' | 'degraded'> {
const pong = await redisCacheClient.ping();
return pong === 'PONG' ? 'up' : 'degraded';
}

/**
* Health check for the Redis cache layer, suitable for the /ready endpoint.
*
* Returns an object describing the Redis check:
* - When REDIS_URL is not configured: { status: 'up', optional: true }
* - When Redis responds to PING within the timeout: { status: 'up' }
* - When Redis is configured but unreachable / times out:
* { status: 'down' }
*
* The default timeout is 500ms per the /ready acceptance criteria.
*/
export async function getRedisReadyCheck(
timeoutMs: number = 500,
): Promise<{ status: 'up' | 'down'; optional?: boolean }> {
if (!redisCacheClient.isConfigured) {
return { status: 'up', optional: true };
}

const pong = await redisCacheClient.pingWithTimeout(timeoutMs);
return pong === 'PONG' ? { status: 'up' } : { status: 'down' };
}