@@ -5,8 +5,10 @@ import { githubConnectorMeta } from '@/connectors/github/meta'
55import type { ConnectorConfig , ExternalDocument , ExternalDocumentList } from '@/connectors/types'
66import {
77 CONNECTOR_MAX_FILE_BYTES ,
8+ ConnectorFileTooLargeError ,
89 markSkipped ,
910 parseTagDate ,
11+ readBodyWithLimit ,
1012 sizeLimitSkipReason ,
1113 stubOrSkipBySize ,
1214 takeIndexableWithinCap ,
@@ -156,7 +158,7 @@ async function fetchBlobContent(
156158 const response = await fetchWithRetry ( url , {
157159 method : 'GET' ,
158160 headers : {
159- Accept : 'application/vnd.github+json' ,
161+ Accept : 'application/vnd.github.raw +json' ,
160162 Authorization : `Bearer ${ accessToken } ` ,
161163 'X-GitHub-Api-Version' : '2022-11-28' ,
162164 } ,
@@ -166,25 +168,12 @@ async function fetchBlobContent(
166168 throw new Error ( `Failed to fetch git blob ${ sha } : ${ response . status } ` )
167169 }
168170
169- const data = await response . json ( )
170- const content = ( data . content as string ) || ''
171- const encoding = data . encoding as string | undefined
172-
173- if ( encoding === 'base64' ) {
174- const buf = Buffer . from ( content , 'base64' )
175- if ( isBinaryBuffer ( buf ) ) return null
176- return buf . toString ( 'utf8' )
171+ const buffer = await readBodyWithLimit ( response , MAX_FILE_SIZE )
172+ if ( ! buffer ) {
173+ throw new ConnectorFileTooLargeError ( MAX_FILE_SIZE )
177174 }
178- /**
179- * `GET /repos/{owner}/{repo}/git/blobs/{sha}` documents a single response
180- * encoding: "The `content` in the response will always be Base64 encoded."
181- * The "Currently, `utf-8` and `base64` are supported" sentence belongs to the
182- * `encoding` REQUEST parameter of `POST .../git/blobs` (Create a blob) and does
183- * not describe this response, so no `utf-8` branch is warranted here. Any other
184- * encoding would silently persist empty content, so it throws and surfaces as a
185- * failed document instead.
186- */
187- throw new Error ( `Unexpected git blob encoding for ${ sha } : ${ encoding ?? 'undefined' } ` )
175+ if ( isBinaryBuffer ( buffer ) ) return null
176+ return buffer . toString ( 'utf8' )
188177}
189178
190179/**
@@ -325,33 +314,14 @@ export const githubConnector: ConnectorConfig = {
325314 const response = await fetchWithRetry ( url , {
326315 method : 'GET' ,
327316 headers : {
328- Accept : 'application/vnd.github+json' ,
317+ Accept : 'application/vnd.github.object +json' ,
329318 Authorization : `Bearer ${ accessToken } ` ,
330319 'X-GitHub-Api-Version' : '2022-11-28' ,
331320 } ,
332321 } )
333322
334323 if ( ! response . ok ) {
335324 if ( response . status === 404 ) return null
336- /**
337- * A rate-limit 403 never reaches here: `fetchWithRetry` treats a 403 carrying
338- * `retry-after` or `x-ratelimit-remaining: 0` as retryable and throws once the
339- * retries are spent, so it lands in the catch below as a failure.
340- *
341- * A 403 that survives is usually an authorization denial, but NOT always: this
342- * request sends `application/vnd.github+json`, and the Contents API documents
343- * that files between 1-100 MB support "only the `raw` or `object` custom media
344- * types". A >1 MB text file therefore also lands here and is dropped, which on
345- * an `add` is silent (a fulfilled `null` records no failure). Reconciliation is
346- * unaffected — the file is already in `seenExternalIds` from the listing.
347- */
348- if ( response . status === 403 ) {
349- logger . info ( 'Skipping GitHub file rejected by Contents API' , {
350- path,
351- status : response . status ,
352- } )
353- return null
354- }
355325 throw new Error ( `Failed to fetch file ${ path } : ${ response . status } ` )
356326 }
357327
@@ -392,15 +362,18 @@ export const githubConnector: ConnectorConfig = {
392362 * "only the `raw` or `object` custom media types are supported", and it is
393363 * specifically "when using the `object` media type" that "the `content` field
394364 * will be an empty string and the `encoding` field will be `none`".
395- *
396- * This request sends `application/vnd.github+json`, so that precondition does
397- * not hold and this branch is currently unreachable — such files 403 above
398- * instead. Reaching it would require requesting
399- * `application/vnd.github.object+json`. The fallback itself is correct: the Git
400- * Blobs API returns the same blob as JSON and is documented to support blobs up
401- * to 100 MB.
365+ * The Git Blobs fallback streams that same blob through GitHub's documented raw
366+ * media type and supports blobs up to 100 MB.
402367 */
403- const blobContent = await fetchBlobContent ( accessToken , owner , repo , data . sha as string )
368+ let blobContent : string | null
369+ try {
370+ blobContent = await fetchBlobContent ( accessToken , owner , repo , data . sha as string )
371+ } catch ( error ) {
372+ if ( error instanceof ConnectorFileTooLargeError ) {
373+ return markSkipped ( stub , sizeLimitSkipReason ( MAX_FILE_SIZE ) )
374+ }
375+ throw error
376+ }
404377 if ( blobContent === null ) {
405378 logger . info ( 'Skipping binary GitHub file' , { path, size } )
406379 return markSkipped ( stub , BINARY_SKIP_REASON )
0 commit comments