test(e2e): Avoid rate limits when polling Sentry in send-to-sentry tests

dev-packages/test-utils/src/cli.ts

@@ -8,7 +8,7 @@ import type { SpawnSyncReturns } from 'node:child_process';
 export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] };
 
 /**
- * A node of the span tree returned by `sentry trace view`. Spans, errors and occurrences all
+ * A node of the span tree returned by the trace endpoint. Spans, errors and occurrences all
  * share this shape and are discriminated by `event_type`.
  */
 export interface TraceItem {
@@ -59,37 +59,49 @@ function runSentryCli(args: string[]): SpawnSyncReturns<string> {
   return result;
 }
 
+/**
+ * The API rate limit is shared by every GET request of the E2E token's user, so concurrent jobs can
+ * exhaust it. The callers poll anyway, so a rate limited request is treated like data that has not
+ * landed yet instead of failing the test.
+ */
+function isRateLimited(result: SpawnSyncReturns<string>): boolean {
+  return /\b429\b|too frequently/i.test(`${result.stdout}${result.stderr}`);
+}
+
 /**
  * Fetch a trace of the E2E test project through the `sentry` CLI, which the calling test app has to
  * list as a dev dependency. Returns an empty list while the trace has not landed yet.
+ *
+ * This calls the trace endpoint directly instead of `sentry trace view --json`, which also looks up
+ * the project and fetches the details of every span, one request each. That turns every poll into
+ * dozens of requests and gets the E2E token rate limited.
  */
 export function fetchTrace(traceId: string): TraceItem[] {
-  const target = traceTarget(traceId);
-  const result = runSentryCli(['trace', 'view', target, '--json', '--fresh']);
+  const params = new URLSearchParams({ project: '-1', statsPeriod: '1h', limit: '10000' });
+  const path = `/organizations/${process.env['E2E_TEST_SENTRY_ORG_SLUG']}/trace/${traceId}/?${params}`;
+  const result = runSentryCli(['api', path]);
 
   if (result.status === 0) {
-    return (JSON.parse(result.stdout) as { spans?: TraceItem[] }).spans ?? [];
+    return JSON.parse(result.stdout) as TraceItem[];
+  }
+
+  if (isRateLimited(result)) {
+    return [];
   }
 
-  // Exit codes 10-19 are auth errors, and a rejected token also surfaces as an API error (exit 30)
-  // with a 401 in the message. Neither resolves by waiting, so fail loudly instead of polling until
-  // the timeout and reporting it as a missing event. The trace endpoint is org scoped, so the token
-  // needs `org:read` on top of the project scopes.
+  // Exit codes 10-19 are auth errors, and a rejected token also surfaces as an API error with a 401
+  // or 403 in the message. Neither resolves by waiting, so fail loudly instead of polling until the
+  // timeout and reporting it as a missing event. The trace endpoint is org scoped, so the token needs
+  // `org:read` on top of the project scopes.
   const isAuthError = result.status !== null && result.status >= 10 && result.status < 20;
-  if (isAuthError || /\b40[13]\b/.test(result.stderr)) {
+  if (isAuthError || /\b40[13]\b/.test(`${result.stdout}${result.stderr}`)) {
     throw new Error(
-      `sentry trace view ${target} failed with exit code ${result.status}: ${result.stderr}` +
+      `sentry api ${path} failed with exit code ${result.status}: ${result.stdout}${result.stderr}` +
         'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.',
     );
   }
 
-  const traceMissing = result.status === 23 && result.stderr.includes(`Trace '${traceId}' not found`);
-
-  if (traceMissing) {
-    return [];
-  }
-
-  throw new Error(`sentry trace view ${target} exited with ${result.status}: ${result.stderr}`);
+  throw new Error(`sentry api ${path} exited with ${result.status}: ${result.stdout}${result.stderr}`);
 }
 
 /**
@@ -110,7 +122,7 @@ export function fetchSpanAttributes(traceId: string, spanId: string): Record<str
     return Object.fromEntries(attributes.map(({ name, value }) => [name, value]));
   }
 
-  if (result.stdout.includes('"Not found."')) {
+  if (result.stdout.includes('"Not found."') || isRateLimited(result)) {
     return undefined;
   }
 
@@ -136,7 +148,7 @@ export function fetchEvent(eventId: string): ApiEvent | undefined {
     return JSON.parse(result.stdout) as ApiEvent;
   }
 
-  if (result.stdout.includes('"Event not found"')) {
+  if (result.stdout.includes('"Event not found"') || isRateLimited(result)) {
     return undefined;
   }
 
@@ -165,6 +177,10 @@ export function findTraceIdOfSpan(query: string): string | undefined {
     return (JSON.parse(result.stdout) as { data: { trace: string }[] }).data[0]?.trace;
   }
 
+  if (isRateLimited(result)) {
+    return undefined;
+  }
+
   throw new Error(`sentry api ${path} exited with ${result.status}: ${result.stdout}${result.stderr}`);
 }