diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts index 69e62b0..e30534c 100644 --- a/src/app/api/models/route.ts +++ b/src/app/api/models/route.ts @@ -1,20 +1,13 @@ import { NextResponse } from 'next/server'; import { loadConfig } from '@/lib/config'; +import { fetchAllPages } from '@/lib/pagination'; // Force dynamic to prevent caching - config can change export const dynamic = 'force-dynamic'; -// deAPI paginates /models: default 25 per page, `limit` is capped at 50 server-side. -// Fetch every page so the models cache is always complete. -const PAGE_LIMIT = 50; -const MAX_PAGES = 20; - -interface ModelsPage { - data?: unknown[]; - meta?: { current_page?: number; last_page?: number; total?: number }; -} - -// GET /api/models - Proxy to deAPI /models endpoint +// GET /api/models - Proxy to deAPI /models endpoint. +// Walks every page (see lib/pagination) so the models cache is always complete — +// deAPI caps `limit` at 50, so a single request can never hold them all. export async function GET() { try { const config = loadConfig(); @@ -26,42 +19,25 @@ export async function GET() { ); } - const baseUrl = `${config.apiUrl.replace(/\/$/, '')}/models`; - const models: unknown[] = []; - let lastMeta: ModelsPage['meta']; - let page = 1; - let lastPage = 1; + const result = await fetchAllPages( + `${config.apiUrl.replace(/\/$/, '')}/models`, + config.apiToken + ); - while (page <= lastPage && page <= MAX_PAGES) { - const url = `${baseUrl}?limit=${PAGE_LIMIT}&page=${page}`; - const response = await fetch(url, { - headers: { - 'Authorization': `Bearer ${config.apiToken}`, - 'Accept': 'application/json', + if (!result.ok) { + const { error, message } = result.body; + return NextResponse.json( + { + error: + (typeof error === 'string' && error) || + (typeof message === 'string' && message) || + `HTTP ${result.status}`, }, - }); - - const data = await response.json(); - - if (!response.ok) { - return NextResponse.json( - { error: data.error || data.message || `HTTP ${response.status}` }, - { status: response.status } - ); - } - - const pageData = data as ModelsPage; - models.push(...(pageData.data ?? [])); - lastMeta = pageData.meta; - lastPage = pageData.meta?.last_page ?? 1; - page += 1; - } - - if (page > MAX_PAGES && page <= lastPage) { - console.warn(`[deapi-tester] /models pagination stopped at ${MAX_PAGES} pages (last_page=${lastPage})`); + { status: result.status } + ); } - return NextResponse.json({ data: models, meta: lastMeta }, { + return NextResponse.json({ data: result.data, meta: result.meta }, { headers: { 'Cache-Control': 'no-store, no-cache, must-revalidate', }, diff --git a/src/app/api/proxy/route.ts b/src/app/api/proxy/route.ts index 4f051bd..654445a 100644 --- a/src/app/api/proxy/route.ts +++ b/src/app/api/proxy/route.ts @@ -3,6 +3,7 @@ import { loadConfig } from '@/lib/config'; import { addJob, generateJobId, updateJob } from '@/lib/storage'; import { saveUploadedFile } from '@/lib/upload-storage'; import { getEndpointById } from '@/lib/endpoint-registry'; +import { fetchAllPages, PAGE_LIMIT } from '@/lib/pagination'; import { Job, JsonValue, UploadedFile } from '@/lib/types'; // POST /api/proxy - Proxy request to deAPI @@ -48,6 +49,13 @@ export async function POST(request: Request) { const providedJobId = typeof params._jobId === 'string' ? params._jobId : undefined; delete params._jobId; + // Tester-only control: walk every page of a paginated GET and merge the + // results into one response. deAPI caps `limit` at 50, so this is the only + // way to see more than 50 items at once. Deleted from params so it is never + // forwarded to the API as a query param. + const fetchAllRequested = params._fetchAll === true || params._fetchAll === 'true'; + delete params._fetchAll; + // Validate endpoint const endpoint = getEndpointById(endpointId); if (!endpoint) { @@ -137,6 +145,10 @@ export async function POST(request: Request) { bodyForLog = params; } + // Only paginate GETs — the flag is meaningless for a POST body, and a price + // pre-calculation is a single POST regardless. + const shouldFetchAllPages = fetchAllRequested && endpoint.method === 'GET' && !isPriceCalc; + // Add query params for GET requests let finalUrl = url; if (endpoint.method === 'GET' && Object.keys(params).length > 0) { @@ -149,6 +161,23 @@ export async function POST(request: Request) { finalUrl = url + '?' + queryParams.toString(); } + // In "fetch all pages" mode the walker owns page/limit. Log the first page + // it will actually request; `_tester.pages_fetched` on the response records + // how many followed. + const pagedQuery: Record = {}; + if (shouldFetchAllPages) { + Object.entries(params).forEach(([key, value]) => { + if (key === 'page' || key === 'limit') return; + if (value !== undefined && value !== null && value !== '') { + pagedQuery[key] = String(value); + } + }); + const firstPage = new URLSearchParams(pagedQuery); + firstPage.set('limit', String(PAGE_LIMIT)); + firstPage.set('page', '1'); + finalUrl = `${url}?${firstPage.toString()}`; + } + // Fetch estimated price if endpoint supports price calculation (skip if this IS a price calc request) let estimatedPrice: number | undefined; if (!isPriceCalc && endpoint.hasPriceCalc && endpoint.priceCalcPath) { @@ -267,45 +296,82 @@ export async function POST(request: Request) { // Make request to deAPI const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 30000); + // The page walker makes up to MAX_PAGES sequential requests, so it gets a + // larger budget than a single call. + const timeoutId = setTimeout(() => controller.abort(), shouldFetchAllPages ? 60000 : 30000); - const response = await fetch(finalUrl, { - ...fetchOptions, - signal: controller.signal, - }); + let rawResponse; + const rawResponseHeaders: Record = {}; + let responseOk: boolean; + let responseStatus: number; - clearTimeout(timeoutId); + if (shouldFetchAllPages) { + const paged = await fetchAllPages(url, config.apiToken, { + query: pagedQuery, + signal: controller.signal, + }); + clearTimeout(timeoutId); - // deAPI (or a gateway/proxy in front of it) can return a non-JSON body on - // errors — an HTML 500/502 page, a plain string. response.json() would throw - // a cryptic "Unexpected token '<'". Read the body as text and parse - // defensively so we surface the real HTTP status + a body snippet instead. - const responseText = await response.text(); - let rawResponse; - try { - rawResponse = JSON.parse(responseText); - } catch { - rawResponse = { - error: `Non-JSON response from API (HTTP ${response.status})`, - status: response.status, - body: responseText.slice(0, 4000), - }; - } + responseOk = paged.ok; + if (paged.ok) { + responseStatus = 200; + rawResponse = { + data: paged.data, + meta: paged.meta, + // Marker so the inspector never reads this as a verbatim API + // response — it is N page responses merged by the tester. `meta` is + // carried over from the LAST page, hence current_page === last_page. + _tester: { + merged_pages: true, + pages_fetched: paged.pagesFetched, + items: paged.data.length, + truncated: paged.truncated, + }, + }; + } else { + responseStatus = paged.status; + rawResponse = paged.body; + } + } else { + const response = await fetch(finalUrl, { + ...fetchOptions, + signal: controller.signal, + }); - // Capture response headers so the UI can optionally display them - const rawResponseHeaders: Record = {}; - response.headers.forEach((value, key) => { - rawResponseHeaders[key] = value; - }); + clearTimeout(timeoutId); + + responseOk = response.ok; + responseStatus = response.status; + + // deAPI (or a gateway/proxy in front of it) can return a non-JSON body on + // errors — an HTML 500/502 page, a plain string. response.json() would throw + // a cryptic "Unexpected token '<'". Read the body as text and parse + // defensively so we surface the real HTTP status + a body snippet instead. + const responseText = await response.text(); + try { + rawResponse = JSON.parse(responseText); + } catch { + rawResponse = { + error: `Non-JSON response from API (HTTP ${response.status})`, + status: response.status, + body: responseText.slice(0, 4000), + }; + } + + // Capture response headers so the UI can optionally display them + response.headers.forEach((value, key) => { + rawResponseHeaders[key] = value; + }); + } // Update job with response - if (!response.ok) { + if (!responseOk) { if (!isPriceCalc) { updateJob(jobId, { rawResponse, rawResponseHeaders, status: 'failed', - error: rawResponse.error || rawResponse.message || `HTTP ${response.status}`, + error: rawResponse.error || rawResponse.message || `HTTP ${responseStatus}`, completedAt: new Date().toISOString(), }); } @@ -313,10 +379,10 @@ export async function POST(request: Request) { return NextResponse.json({ success: false, jobId, - error: rawResponse.error || rawResponse.message || `HTTP ${response.status}`, + error: rawResponse.error || rawResponse.message || `HTTP ${responseStatus}`, rawRequest: job.rawRequest, rawResponse, - }, { status: response.status }); + }, { status: responseStatus }); } // For async endpoints, extract request_id diff --git a/src/lib/endpoint-registry.ts b/src/lib/endpoint-registry.ts index baf1c3c..c1f0a5a 100644 --- a/src/lib/endpoint-registry.ts +++ b/src/lib/endpoint-registry.ts @@ -1047,7 +1047,47 @@ export const ENDPOINTS: EndpointDefinition[] = [ contentType: 'json', isAsync: false, hasPriceCalc: false, - params: [], + // deAPI paginates this endpoint (Laravel-style `meta.current_page` / + // `meta.last_page`). Defaults are page 1 / 25 per page, so without these + // controls the tester could only ever see the first 25 models. Note the + // param is `limit` — `per_page` is accepted but silently ignored — and the + // server caps it at 50 (limit=100 still returns per_page=50), so more than + // 50 models always requires walking pages. + params: [ + { + name: '_fetchAll', + label: 'Fetch All Pages', + type: 'boolean', + required: false, + default: false, + description: + 'Tester-side: walk every page and merge into one response. Not sent to deAPI — the response is marked with _tester.merged_pages.', + }, + { + name: 'page', + label: 'Page', + type: 'number', + required: false, + min: 1, + step: 1, + placeholder: '1', + description: 'Page number. Check meta.last_page in the response for the page count.', + visibleWhen: { field: '_fetchAll', values: ['false'], matchEmpty: true }, + }, + { + name: 'limit', + label: 'Limit', + type: 'number', + required: false, + default: 50, + min: 1, + max: 50, + step: 1, + placeholder: '25', + description: 'Models per page. Server caps this at 50 (deAPI default is 25).', + visibleWhen: { field: '_fetchAll', values: ['false'], matchEmpty: true }, + }, + ], }, { diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts new file mode 100644 index 0000000..d6a9425 --- /dev/null +++ b/src/lib/pagination.ts @@ -0,0 +1,109 @@ +// Shared pagination walker for deAPI list endpoints. +// +// deAPI paginates Laravel-style: responses carry `meta.current_page` / +// `meta.last_page`, and the query accepts `page` + `limit`. Two quirks, both +// verified against the live API: +// - `per_page` is accepted but SILENTLY IGNORED — only `limit` has an effect. +// - `limit` is capped at 50 server-side (limit=100 still returns per_page=50), +// so collecting more than 50 items ALWAYS requires walking pages. +export const PAGE_LIMIT = 50; +export const MAX_PAGES = 20; + +export interface PaginationMeta { + current_page?: number; + last_page?: number; + per_page?: number; + total?: number; + [key: string]: unknown; +} + +interface PaginatedPage { + data?: unknown[]; + meta?: PaginationMeta; +} + +export type FetchAllPagesResult = + | { + ok: true; + data: unknown[]; + meta?: PaginationMeta; + pagesFetched: number; + truncated: boolean; + } + | { ok: false; status: number; body: Record }; + +interface FetchAllPagesOptions { + // Extra query params carried onto every page request. `page` and `limit` are + // always set by this function and cannot be overridden. + query?: Record; + signal?: AbortSignal; +} + +// Walk every page of a paginated deAPI GET endpoint and concatenate the items. +// Stops early (with `truncated: true`) at MAX_PAGES so a bad `last_page` can +// never spin forever. Any non-OK / non-JSON page aborts the walk and is +// returned verbatim so the caller can surface the real API error. +export async function fetchAllPages( + baseUrl: string, + token: string, + options?: FetchAllPagesOptions +): Promise { + const data: unknown[] = []; + let meta: PaginationMeta | undefined; + let page = 1; + let lastPage = 1; + + while (page <= lastPage && page <= MAX_PAGES) { + const query = new URLSearchParams(options?.query); + query.set('limit', String(PAGE_LIMIT)); + query.set('page', String(page)); + + const response = await fetch(`${baseUrl}?${query.toString()}`, { + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/json', + }, + signal: options?.signal, + }); + + // Parse defensively — a gateway can answer with an HTML error page, and + // response.json() would throw a cryptic "Unexpected token '<'". + const text = await response.text(); + let body: unknown; + let parsed = true; + try { + body = JSON.parse(text); + } catch { + parsed = false; + body = { + error: `Non-JSON response from API (HTTP ${response.status})`, + status: response.status, + body: text.slice(0, 4000), + }; + } + + if (!response.ok || !parsed) { + return { + ok: false, + // A 200 that isn't JSON is an upstream fault, not a client error. + status: response.ok ? 502 : response.status, + body: body as Record, + }; + } + + const pageBody = body as PaginatedPage; + data.push(...(pageBody.data ?? [])); + meta = pageBody.meta; + lastPage = pageBody.meta?.last_page ?? 1; + page += 1; + } + + const truncated = page > MAX_PAGES && page <= lastPage; + if (truncated) { + console.warn( + `[deapi-tester] pagination stopped at ${MAX_PAGES} pages (last_page=${lastPage}) for ${baseUrl}` + ); + } + + return { ok: true, data, meta, pagesFetched: page - 1, truncated }; +}