feat(gql): add configurable soft deadline to GraphQL fan-out

docker-compose.yaml

@@ -346,6 +346,8 @@ services:
       - GATEWAYS_GQL_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE=${GATEWAYS_GQL_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE:-}
       - GATEWAYS_GQL_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS=${GATEWAYS_GQL_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS:-}
       - GATEWAYS_GQL_CIRCUIT_BREAKER_RESET_TIMEOUT_MS=${GATEWAYS_GQL_CIRCUIT_BREAKER_RESET_TIMEOUT_MS:-}
+      - GATEWAYS_GQL_SOFT_DEADLINE_ENABLED=${GATEWAYS_GQL_SOFT_DEADLINE_ENABLED:-}
+      - GATEWAYS_GQL_SOFT_DEADLINE_MS=${GATEWAYS_GQL_SOFT_DEADLINE_MS:-}
       - HYPERBEAM_ENDPOINT=${HYPERBEAM_ENDPOINT:-}
       - HYPERBEAM_REQUEST_TIMEOUT_MS=${HYPERBEAM_REQUEST_TIMEOUT_MS:-}
       - HYPERBEAM_ROOT_TX_RATE_LIMIT_BURST_SIZE=${HYPERBEAM_ROOT_TX_RATE_LIMIT_BURST_SIZE:-}

docs/envs.md

@@ -491,6 +491,8 @@ When enabled, the `transaction(id)` GraphQL query can resolve unindexed data ite
 | GATEWAYS_GQL_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE | Number        | 80                                            | Error rate (0–100) within the rolling window that trips the breaker open                                                                                              |
 | GATEWAYS_GQL_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS | Number           | 60000                                         | Rolling window in milliseconds over which error rate is measured                                                                                                      |
 | GATEWAYS_GQL_CIRCUIT_BREAKER_RESET_TIMEOUT_MS    | Number               | 30000                                         | How long an open breaker stays open before transitioning to half-open for a trial request                                                                             |
+| GATEWAYS_GQL_SOFT_DEADLINE_ENABLED               | Boolean              | false                                         | When `true`, fan-out list queries (transactions/blocks) stop waiting for slow upstreams once at least one has answered and `GATEWAYS_GQL_SOFT_DEADLINE_MS` elapses, returning a partial merge with an `UPSTREAM_SOFT_DEADLINE` warning per unfinished source. Abandoned requests keep running so the circuit breaker still sees their true outcome. When `false` (default), the fan-out waits for every upstream (bounded by `GATEWAYS_GQL_REQUEST_TIMEOUT_MS`) |
+| GATEWAYS_GQL_SOFT_DEADLINE_MS                    | Number               | 2000                                          | Soft-deadline window in milliseconds, armed once the first upstream returns a usable result. Only used when `GATEWAYS_GQL_SOFT_DEADLINE_ENABLED` is `true`                                                             |
 
 ## HTTPSIG Response Signing
 

src/config.ts

@@ -421,6 +421,24 @@ export const GATEWAYS_GQL_CIRCUIT_BREAKER_RESET_TIMEOUT_MS =
     30 * 1_000,
   );
 
+// Soft deadline for the GraphQL fan-out list queries (transactions/blocks).
+// When enabled, once at least one upstream has returned a usable result the
+// merge waits at most this long for the remaining upstreams, then returns a
+// partial merge with an `UPSTREAM_SOFT_DEADLINE` warning for each source that
+// had not yet responded. The abandoned requests keep running in the
+// background so the per-upstream circuit breaker still observes their true
+// outcome (a real success or timeout). The deadline is only armed after the
+// first usable result, so an all-slow moment never turns a would-succeed
+// query into a failure. When disabled (the default), the fan-out waits for
+// every upstream to settle (bounded by GATEWAYS_GQL_REQUEST_TIMEOUT_MS),
+// preserving the prior behavior.
+export const GATEWAYS_GQL_SOFT_DEADLINE_ENABLED =
+  env.varOrDefault('GATEWAYS_GQL_SOFT_DEADLINE_ENABLED', 'false') === 'true';
+export const GATEWAYS_GQL_SOFT_DEADLINE_MS = env.positiveIntOrDefault(
+  'GATEWAYS_GQL_SOFT_DEADLINE_MS',
+  2_000,
+);
+
 // GraphQL root TX lookup rate limiting
 export const GRAPHQL_ROOT_TX_RATE_LIMIT_BURST_SIZE = +env.varOrDefault(
   'GRAPHQL_ROOT_TX_RATE_LIMIT_BURST_SIZE',

src/database/gateways-gql-queryable.test.ts

@@ -920,3 +920,171 @@ describe('RemoteGqlQueryable circuit breaker', () => {
     );
   });
 });
+
+describe('GatewaysGqlQueryable soft deadline', () => {
+  class DelayedTxQueryable extends FakeQueryable {
+    constructor(
+      data: { transactions?: GqlTransaction[]; throws?: Error },
+      private readonly delayMs: number,
+    ) {
+      super(data);
+    }
+    async getGqlTransactions(args: {
+      pageSize: number;
+      cursor?: string;
+      sortOrder?: 'HEIGHT_DESC' | 'HEIGHT_ASC';
+    }): Promise<GqlTransactionsResult> {
+      await new Promise((r) => setTimeout(r, this.delayMs));
+      return super.getGqlTransactions(args);
+    }
+  }
+
+  const fastSource = () =>
+    new FakeQueryable({ transactions: [txAt({ id: 'fast', height: 100 })] });
+
+  it('returns the fast source without waiting for a laggard, with a warning', async () => {
+    const merger = GatewaysGqlQueryable.forTesting({
+      log,
+      sources: [
+        fastSource(),
+        new DelayedTxQueryable(
+          { transactions: [txAt({ id: 'slow', height: 99 })] },
+          // Abandoned laggard: kept short (not multi-second) so its still-ref'd
+          // timer can't leave the process hanging at exit, but well above the
+          // 30ms deadline so the assertion proves we returned via the deadline.
+          500,
+        ),
+      ],
+      labels: ['<local>', 'http://slow.example'],
+      softDeadlineEnabled: true,
+      softDeadlineMs: 30,
+    });
+
+    const started = Date.now();
+    const result = await merger.getGqlTransactions({
+      pageSize: 10,
+      sortOrder: 'HEIGHT_DESC',
+      tags: [],
+    });
+    const elapsed = Date.now() - started;
+
+    assert.deepEqual(
+      result.edges.map((e) => e.node.id),
+      ['fast'],
+    );
+    assert.ok(
+      elapsed < 300,
+      `expected to return via the 30ms soft deadline, not wait for the 500ms laggard, took ${elapsed}ms`,
+    );
+    const warnings = result.warnings ?? [];
+    assert.equal(warnings.length, 1);
+    assert.equal(warnings[0].code, 'UPSTREAM_SOFT_DEADLINE');
+    assert.equal(warnings[0].source, 'http://slow.example');
+  });
+
+  it('waits for the laggard and omits the warning when disabled (default)', async () => {
+    const merger = GatewaysGqlQueryable.forTesting({
+      log,
+      sources: [
+        fastSource(),
+        new DelayedTxQueryable(
+          { transactions: [txAt({ id: 'slow', height: 99 })] },
+          40,
+        ),
+      ],
+      // softDeadlineEnabled defaults to false -> prior behavior
+    });
+    const result = await merger.getGqlTransactions({
+      pageSize: 10,
+      sortOrder: 'HEIGHT_DESC',
+      tags: [],
+    });
+    assert.deepEqual(
+      result.edges.map((e) => e.node.id),
+      ['fast', 'slow'],
+    );
+    assert.equal(result.warnings, undefined);
+  });
+
+  it('arms the deadline only after the first result, so an all-slow moment is not a failure', async () => {
+    // Both sources take ~200ms; the deadline is 100ms. Armed at call time the
+    // 100ms deadline would fire before either source (200ms) responds, cut
+    // both, and throw "all sources failed". Armed after the first fulfillment
+    // it fires at ~300ms, well after the second source (~200ms) settles, so
+    // both results are captured. The deadline must stay below the source delay
+    // — that gap is what distinguishes the two arming strategies — while the
+    // ~100ms slack between the second source and the timer keeps the test
+    // robust against CI scheduling jitter.
+    const merger = GatewaysGqlQueryable.forTesting({
+      log,
+      sources: [
+        new DelayedTxQueryable(
+          { transactions: [txAt({ id: 'a', height: 100 })] },
+          200,
+        ),
+        new DelayedTxQueryable(
+          { transactions: [txAt({ id: 'b', height: 99 })] },
+          200,
+        ),
+      ],
+      softDeadlineEnabled: true,
+      softDeadlineMs: 100,
+    });
+    const result = await merger.getGqlTransactions({
+      pageSize: 10,
+      sortOrder: 'HEIGHT_DESC',
+      tags: [],
+    });
+    assert.deepEqual(
+      result.edges.map((e) => e.node.id),
+      ['a', 'b'],
+    );
+    assert.equal(result.warnings, undefined);
+  });
+
+  it('applies the soft deadline to block queries too', async () => {
+    class DelayedBlockQueryable extends FakeQueryable {
+      constructor(
+        data: { blocks?: GqlBlock[] },
+        private readonly delayMs: number,
+      ) {
+        super(data);
+      }
+      async getGqlBlocks(args: {
+        pageSize: number;
+        sortOrder?: 'HEIGHT_DESC' | 'HEIGHT_ASC';
+      }): Promise<GqlBlocksResult> {
+        await new Promise((r) => setTimeout(r, this.delayMs));
+        return super.getGqlBlocks(args);
+      }
+    }
+    const merger = GatewaysGqlQueryable.forTesting({
+      log,
+      sources: [
+        new FakeQueryable({
+          blocks: [{ id: 'b100', height: 100, timestamp: 0, previous: '' }],
+        }),
+        new DelayedBlockQueryable(
+          { blocks: [{ id: 'b99', height: 99, timestamp: 0, previous: '' }] },
+          // Abandoned laggard kept short so its ref'd timer can't hang exit.
+          500,
+        ),
+      ],
+      labels: ['<local>', 'http://slow.example'],
+      softDeadlineEnabled: true,
+      softDeadlineMs: 30,
+    });
+    const result = await merger.getGqlBlocks({
+      pageSize: 10,
+      sortOrder: 'HEIGHT_DESC',
+    });
+    assert.deepEqual(
+      result.edges.map((e) => e.node.id),
+      ['b100'],
+    );
+    const warnings = result.warnings ?? [];
+    assert.equal(warnings.length, 1);
+    assert.equal(warnings[0].code, 'UPSTREAM_SOFT_DEADLINE');
+    assert.equal(warnings[0].source, 'http://slow.example');
+  });
+});