feat: paginate /models in UI + "Fetch All Pages" option

src/app/api/models/route.ts

@@ -1,20 +1,13 @@
 import { NextResponse } from 'next/server';
 import { loadConfig } from '@/lib/config';
+import { fetchAllPages } from '@/lib/pagination';
 
 // Force dynamic to prevent caching - config can change
 export const dynamic = 'force-dynamic';
 
-// deAPI paginates /models: default 25 per page, `limit` is capped at 50 server-side.
-// Fetch every page so the models cache is always complete.
-const PAGE_LIMIT = 50;
-const MAX_PAGES = 20;
-
-interface ModelsPage {
-  data?: unknown[];
-  meta?: { current_page?: number; last_page?: number; total?: number };
-}
-
-// GET /api/models - Proxy to deAPI /models endpoint
+// GET /api/models - Proxy to deAPI /models endpoint.
+// Walks every page (see lib/pagination) so the models cache is always complete —
+// deAPI caps `limit` at 50, so a single request can never hold them all.
 export async function GET() {
   try {
     const config = loadConfig();
@@ -26,42 +19,25 @@ export async function GET() {
       );
     }
 
-    const baseUrl = `${config.apiUrl.replace(/\/$/, '')}/models`;
-    const models: unknown[] = [];
-    let lastMeta: ModelsPage['meta'];
-    let page = 1;
-    let lastPage = 1;
+    const result = await fetchAllPages(
+      `${config.apiUrl.replace(/\/$/, '')}/models`,
+      config.apiToken
+    );
 
-    while (page <= lastPage && page <= MAX_PAGES) {
-      const url = `${baseUrl}?limit=${PAGE_LIMIT}&page=${page}`;
-      const response = await fetch(url, {
-        headers: {
-          'Authorization': `Bearer ${config.apiToken}`,
-          'Accept': 'application/json',
+    if (!result.ok) {
+      const { error, message } = result.body;
+      return NextResponse.json(
+        {
+          error:
+            (typeof error === 'string' && error) ||
+            (typeof message === 'string' && message) ||
+            `HTTP ${result.status}`,
         },
-      });
-
-      const data = await response.json();
-
-      if (!response.ok) {
-        return NextResponse.json(
-          { error: data.error || data.message || `HTTP ${response.status}` },
-          { status: response.status }
-        );
-      }
-
-      const pageData = data as ModelsPage;
-      models.push(...(pageData.data ?? []));
-      lastMeta = pageData.meta;
-      lastPage = pageData.meta?.last_page ?? 1;
-      page += 1;
-    }
-
-    if (page > MAX_PAGES && page <= lastPage) {
-      console.warn(`[deapi-tester] /models pagination stopped at ${MAX_PAGES} pages (last_page=${lastPage})`);
+        { status: result.status }
+      );
     }
 
-    return NextResponse.json({ data: models, meta: lastMeta }, {
+    return NextResponse.json({ data: result.data, meta: result.meta }, {
       headers: {
         'Cache-Control': 'no-store, no-cache, must-revalidate',
       },

src/app/api/proxy/route.ts

@@ -3,6 +3,7 @@ import { loadConfig } from '@/lib/config';
 import { addJob, generateJobId, updateJob } from '@/lib/storage';
 import { saveUploadedFile } from '@/lib/upload-storage';
 import { getEndpointById } from '@/lib/endpoint-registry';
+import { fetchAllPages, PAGE_LIMIT } from '@/lib/pagination';
 import { Job, JsonValue, UploadedFile } from '@/lib/types';
 
 // POST /api/proxy - Proxy request to deAPI
@@ -48,6 +49,13 @@ export async function POST(request: Request) {
     const providedJobId = typeof params._jobId === 'string' ? params._jobId : undefined;
     delete params._jobId;
 
+    // Tester-only control: walk every page of a paginated GET and merge the
+    // results into one response. deAPI caps `limit` at 50, so this is the only
+    // way to see more than 50 items at once. Deleted from params so it is never
+    // forwarded to the API as a query param.
+    const fetchAllRequested = params._fetchAll === true || params._fetchAll === 'true';
+    delete params._fetchAll;
+
     // Validate endpoint
     const endpoint = getEndpointById(endpointId);
     if (!endpoint) {
@@ -137,6 +145,10 @@ export async function POST(request: Request) {
       bodyForLog = params;
     }
 
+    // Only paginate GETs — the flag is meaningless for a POST body, and a price
+    // pre-calculation is a single POST regardless.
+    const shouldFetchAllPages = fetchAllRequested && endpoint.method === 'GET' && !isPriceCalc;
+
     // Add query params for GET requests
     let finalUrl = url;
     if (endpoint.method === 'GET' && Object.keys(params).length > 0) {
@@ -149,6 +161,23 @@ export async function POST(request: Request) {
       finalUrl = url + '?' + queryParams.toString();
     }
 
+    // In "fetch all pages" mode the walker owns page/limit. Log the first page
+    // it will actually request; `_tester.pages_fetched` on the response records
+    // how many followed.
+    const pagedQuery: Record<string, string> = {};
+    if (shouldFetchAllPages) {
+      Object.entries(params).forEach(([key, value]) => {
+        if (key === 'page' || key === 'limit') return;
+        if (value !== undefined && value !== null && value !== '') {
+          pagedQuery[key] = String(value);
+        }
+      });
+      const firstPage = new URLSearchParams(pagedQuery);
+      firstPage.set('limit', String(PAGE_LIMIT));
+      firstPage.set('page', '1');
+      finalUrl = `${url}?${firstPage.toString()}`;
+    }
+
     // Fetch estimated price if endpoint supports price calculation (skip if this IS a price calc request)
     let estimatedPrice: number | undefined;
     if (!isPriceCalc && endpoint.hasPriceCalc && endpoint.priceCalcPath) {
@@ -267,56 +296,93 @@ export async function POST(request: Request) {
 
     // Make request to deAPI
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 30000);
+    // The page walker makes up to MAX_PAGES sequential requests, so it gets a
+    // larger budget than a single call.
+    const timeoutId = setTimeout(() => controller.abort(), shouldFetchAllPages ? 60000 : 30000);
 
-    const response = await fetch(finalUrl, {
-      ...fetchOptions,
-      signal: controller.signal,
-    });
+    let rawResponse;
+    const rawResponseHeaders: Record<string, string> = {};
+    let responseOk: boolean;
+    let responseStatus: number;
 
-    clearTimeout(timeoutId);
+    if (shouldFetchAllPages) {
+      const paged = await fetchAllPages(url, config.apiToken, {
+        query: pagedQuery,
+        signal: controller.signal,
+      });
+      clearTimeout(timeoutId);
 
-    // deAPI (or a gateway/proxy in front of it) can return a non-JSON body on
-    // errors — an HTML 500/502 page, a plain string. response.json() would throw
-    // a cryptic "Unexpected token '<'". Read the body as text and parse
-    // defensively so we surface the real HTTP status + a body snippet instead.
-    const responseText = await response.text();
-    let rawResponse;
-    try {
-      rawResponse = JSON.parse(responseText);
-    } catch {
-      rawResponse = {
-        error: `Non-JSON response from API (HTTP ${response.status})`,
-        status: response.status,
-        body: responseText.slice(0, 4000),
-      };
-    }
+      responseOk = paged.ok;
+      if (paged.ok) {
+        responseStatus = 200;
+        rawResponse = {
+          data: paged.data,
+          meta: paged.meta,
+          // Marker so the inspector never reads this as a verbatim API
+          // response — it is N page responses merged by the tester. `meta` is
+          // carried over from the LAST page, hence current_page === last_page.
+          _tester: {
+            merged_pages: true,
+            pages_fetched: paged.pagesFetched,
+            items: paged.data.length,
+            truncated: paged.truncated,
+          },
+        };
+      } else {
+        responseStatus = paged.status;
+        rawResponse = paged.body;
+      }
+    } else {
+      const response = await fetch(finalUrl, {
+        ...fetchOptions,
+        signal: controller.signal,
+      });
 
-    // Capture response headers so the UI can optionally display them
-    const rawResponseHeaders: Record<string, string> = {};
-    response.headers.forEach((value, key) => {
-      rawResponseHeaders[key] = value;
-    });
+      clearTimeout(timeoutId);
+
+      responseOk = response.ok;
+      responseStatus = response.status;
+
+      // deAPI (or a gateway/proxy in front of it) can return a non-JSON body on
+      // errors — an HTML 500/502 page, a plain string. response.json() would throw
+      // a cryptic "Unexpected token '<'". Read the body as text and parse
+      // defensively so we surface the real HTTP status + a body snippet instead.
+      const responseText = await response.text();
+      try {
+        rawResponse = JSON.parse(responseText);
+      } catch {
+        rawResponse = {
+          error: `Non-JSON response from API (HTTP ${response.status})`,
+          status: response.status,
+          body: responseText.slice(0, 4000),
+        };
+      }
+
+      // Capture response headers so the UI can optionally display them
+      response.headers.forEach((value, key) => {
+        rawResponseHeaders[key] = value;
+      });
+    }
 
     // Update job with response
-    if (!response.ok) {
+    if (!responseOk) {
       if (!isPriceCalc) {
         updateJob(jobId, {
           rawResponse,
           rawResponseHeaders,
           status: 'failed',
-          error: rawResponse.error || rawResponse.message || `HTTP ${response.status}`,
+          error: rawResponse.error || rawResponse.message || `HTTP ${responseStatus}`,
           completedAt: new Date().toISOString(),
         });
       }
 
       return NextResponse.json({
         success: false,
         jobId,
-        error: rawResponse.error || rawResponse.message || `HTTP ${response.status}`,
+        error: rawResponse.error || rawResponse.message || `HTTP ${responseStatus}`,
         rawRequest: job.rawRequest,
         rawResponse,
-      }, { status: response.status });
+      }, { status: responseStatus });
     }
 
     // For async endpoints, extract request_id

src/lib/endpoint-registry.ts

@@ -1047,7 +1047,47 @@ export const ENDPOINTS: EndpointDefinition[] = [
     contentType: 'json',
     isAsync: false,
     hasPriceCalc: false,
-    params: [],
+    // deAPI paginates this endpoint (Laravel-style `meta.current_page` /
+    // `meta.last_page`). Defaults are page 1 / 25 per page, so without these
+    // controls the tester could only ever see the first 25 models. Note the
+    // param is `limit` — `per_page` is accepted but silently ignored — and the
+    // server caps it at 50 (limit=100 still returns per_page=50), so more than
+    // 50 models always requires walking pages.
+    params: [
+      {
+        name: '_fetchAll',
+        label: 'Fetch All Pages',
+        type: 'boolean',
+        required: false,
+        default: false,
+        description:
+          'Tester-side: walk every page and merge into one response. Not sent to deAPI — the response is marked with _tester.merged_pages.',
+      },
+      {
+        name: 'page',
+        label: 'Page',
+        type: 'number',
+        required: false,
+        min: 1,
+        step: 1,
+        placeholder: '1',
+        description: 'Page number. Check meta.last_page in the response for the page count.',
+        visibleWhen: { field: '_fetchAll', values: ['false'], matchEmpty: true },
+      },
+      {
+        name: 'limit',
+        label: 'Limit',
+        type: 'number',
+        required: false,
+        default: 50,
+        min: 1,
+        max: 50,
+        step: 1,
+        placeholder: '25',
+        description: 'Models per page. Server caps this at 50 (deAPI default is 25).',
+        visibleWhen: { field: '_fetchAll', values: ['false'], matchEmpty: true },
+      },
+    ],
   },
 
   {