diff --git a/src/processing/svg.ts b/src/processing/svg.ts index 19b0d8f6..a23eb46b 100644 --- a/src/processing/svg.ts +++ b/src/processing/svg.ts @@ -107,7 +107,7 @@ export class SvgComposer { const perLine = Math.floor((this.config.width - (preset.container?.sidePadding || 0) * 2) / preset.boxWidth) for (let i = 0; i < Math.ceil(sponsors.length / perLine); i++) { - await this.addSponsorLine(sponsors.slice(i * perLine, (i + 1) * perLine), preset) + await this.addSponsorLine(sponsors.slice(i * perLine, (i + 1) * perLine), preset) // NOSONAR(typescript:S9382): Each row uses the height advanced by the previous row. } return this diff --git a/src/providers/github.ts b/src/providers/github.ts index 96c8d054..d7d5eb34 100644 --- a/src/providers/github.ts +++ b/src/providers/github.ts @@ -77,7 +77,7 @@ export async function fetchGitHubSponsors( const tiers = config.tiers?.filter(tier => tier.monthlyDollars && tier.monthlyDollars > 0).sort((a, b) => b.monthlyDollars! - a.monthlyDollars!) do { const query = makeQuery(login, type, !config.includePastSponsors, cursor) - const data = await requestGitHubGraphQL(token, query) + const data = await requestGitHubGraphQL(token, query) // NOSONAR(typescript:S9382): Each page requires the previous response's endCursor. sponsors.push( ...(data.data[type].sponsorshipsAsMaintainer.nodes || []), @@ -241,7 +241,7 @@ async function fetchGitHubSponsoringNodes( do { const query = makeSponsoringQuery(login, type, activeOnly, cursor) - const data = await requestGitHubGraphQL(token, query) + const data = await requestGitHubGraphQL(token, query) // NOSONAR(typescript:S9382): Each page requires the previous response's endCursor. const page = data.data?.[type]?.sponsorshipsAsSponsor if (!page) throw new Error('Invalid GitHub response: `sponsorshipsAsSponsor` is missing') diff --git a/src/providers/githubContributions.ts b/src/providers/githubContributions.ts index 00957480..04c64a5d 100644 --- a/src/providers/githubContributions.ts +++ b/src/providers/githubContributions.ts @@ -156,7 +156,7 @@ async function discoverReposFromMergedPRs( do { type SearchResponse = { data: { search: { pageInfo: { hasNextPage: boolean; endCursor: string | null }; edges: Array<{ node: { repository?: RepoNode } }> } } } - const response: SearchResponse = await graphqlFetch({ + const response: SearchResponse = await graphqlFetch({ // NOSONAR(typescript:S9382): Each search page requires the previous response's endCursor. query: ` query($searchQuery: String!, $after: String) { search(query: $searchQuery, type: ISSUE, first: 100, after: $after) { @@ -214,20 +214,21 @@ async function fetchMergedPRCounts( ): Promise> { console.log(`[contribkit][githubContributions] fetching merged PR counts per repository...`) const repoPRs = new Map() - const batchSize = 10 - - for (let i = 0; i < allRepos.length; i += batchSize) { - const batch = allRepos.slice(i, i + batchSize) - const counts = await Promise.all(batch.map(repo => fetchPRCountForRepo(graphqlFetch, repo, login))) - - for (let index = 0; index < batch.length; index++) { - const count = counts[index] - if (count > 0) - repoPRs.set(batch[index].nameWithOwner, count) - } - - if (i + batchSize < allRepos.length) - console.log(`[contribkit][githubContributions] processed PR batches for ${Math.min(i + batchSize, allRepos.length)}/${allRepos.length} repos...`) + const pLimit = await import('p-limit').then(r => r.default) + const limit = pLimit(10) + let processed = 0 + const counts = await Promise.all(allRepos.map(repo => limit(async () => { + const count = await fetchPRCountForRepo(graphqlFetch, repo, login) + processed++ + if (processed % 10 === 0 && processed < allRepos.length) + console.log(`[contribkit][githubContributions] processed PR counts for ${processed}/${allRepos.length} repos...`) + return count + }))) + + for (let index = 0; index < allRepos.length; index++) { + const count = counts[index] + if (count > 0) + repoPRs.set(allRepos[index].nameWithOwner, count) } console.log(`[contribkit][githubContributions] found merged PR counts for ${repoPRs.size} repositories`) diff --git a/src/providers/opencollective.ts b/src/providers/opencollective.ts index 9a0b542e..d52146fb 100644 --- a/src/providers/opencollective.ts +++ b/src/providers/opencollective.ts @@ -116,7 +116,7 @@ async function fetchOpenCollectivePages( const nodes: any[] = [] let offset: number | undefined = 0 while (offset !== undefined) { - const data = await fetchOpenCollectivePage(key, makeQuery(offset)) + const data = await fetchOpenCollectivePage(key, makeQuery(offset)) // NOSONAR(typescript:S9382): The next offset depends on the returned node count and total. const connection = getConnection(data) const pageNodes = connection.nodes ?? [] nodes.push(...pageNodes) diff --git a/src/providers/patreon.ts b/src/providers/patreon.ts index 515f4311..6fffe111 100644 --- a/src/providers/patreon.ts +++ b/src/providers/patreon.ts @@ -36,7 +36,7 @@ export async function fetchPatreonSponsors(token: string | undefined): Promise s.provider = provider.name) - sponsors = (await config.onSponsorsFetched?.(sponsors, provider.name)) ?? sponsors + sponsors = (await config.onSponsorsFetched?.(sponsors, provider.name)) ?? sponsors // NOSONAR(typescript:S9382): Provider hooks run in configured order before subsequent fetches. t.success(`${sponsors.length} sponsorships fetched from ${provider.name}`) allSponsors.push(...sponsors) } diff --git a/tests/async-work.test.js b/tests/async-work.test.js index 5300f969..74bfbab4 100644 --- a/tests/async-work.test.js +++ b/tests/async-work.test.js @@ -22,6 +22,10 @@ describe('independent asynchronous work', () => { name: 'retains successful years when one contribution query fails', scenario: 'github-failure', }, + { + name: 'keeps PR-count requests capped and fills free slots without waiting for a batch', + scenario: 'github-counts', + }, { name: 'overlaps GitLab lookups with a cap, preserving contributor order and partial results', scenario: 'gitlab', diff --git a/tests/fixtures/async-work.mjs b/tests/fixtures/async-work.mjs index 29e13e27..5ec554a2 100644 --- a/tests/fixtures/async-work.mjs +++ b/tests/fixtures/async-work.mjs @@ -108,6 +108,78 @@ async function github(failYear) { assert.equal(warnings.length, failYear ? 1 : 0) } +async function githubCounts() { + const repos = Array.from({ length: 25 }, (_, i) => ({ + ...repository(new Date().getFullYear()), + nameWithOwner: `owner${i}/repo`, + owner: { ...repository(0).owner, login: `owner${i}` }, + })) + const gate = requestGate(10) + const warnings = [] + const completed = [] + let active = 0 + let maximum = 0 + let releaseSlowRequest + const nextRequestStarted = new Promise(resolve => releaseSlowRequest = resolve) + console.warn = (...args) => warnings.push(args.join(' ')) + + globalThis.fetch = async (_url, options) => { + const { query, variables } = JSON.parse(options.body) + if (query.includes('createdAt')) + return json({ data: { user: { createdAt: `${new Date().getFullYear()}-01-01T00:00:00Z` } } }) + if (query.includes('contributionsCollection')) + return json({ data: { user: { contributionsCollection: { + commitContributionsByRepository: repos.map(repository => ({ repository })), + } } } }) + if (query.includes('pageInfo')) + return json({ data: { search: { pageInfo: { hasNextPage: false, endCursor: null }, edges: [] } } }) + + const index = Number(/repo:owner(\d+)\/repo/.exec(variables.q)[1]) + active++ + maximum = Math.max(maximum, active) + try { + await gate.wait() + // A batch barrier would keep request 10 queued behind request 0. + if (index === 0) { + let timer + try { + await Promise.race([ + nextRequestStarted, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('PR-count requests stalled behind a batch barrier')), 1000) + }), + ]) + } + finally { + clearTimeout(timer) + } + } + if (index === 10) + releaseSlowRequest() + completed.push(index) + if (index === 3) + return new Response('count unavailable', { status: 400 }) + return json({ data: { search: { issueCount: index === 4 ? 0 : 1 } } }) + } + finally { + active-- + } + } + + const { fetchGitHubContributions } = await import('../../src/providers/githubContributions.ts') + const result = fetchGitHubContributions('test-token', 'contributor') + await gate.open() + const sponsorships = await result + assert.equal(active, 0) + assert.equal(maximum, 10) + assert.equal(completed.length, repos.length) + assert.ok(completed.indexOf(10) < completed.indexOf(0)) + assert.deepEqual(sponsorships.map(ship => ship.sponsor.login), + repos.filter((_, i) => i !== 3 && i !== 4).map(repo => repo.owner.login)) + assert.ok(sponsorships.every(ship => ship.monthlyDollars === 1)) + assert.equal(warnings.length, 1) +} + async function gitlab() { const contributors = Array.from({ length: 101 }, (_, i) => ({ name: `user${i}`, email: `${i}@example.test`, commits: i < 24 ? 2 : 0, @@ -193,6 +265,7 @@ async function circles(failImage) { const scenarios = { 'github': () => github(false), 'github-failure': () => github(true), + 'github-counts': githubCounts, 'gitlab': gitlab, 'circles': () => circles(false), 'circles-failure': () => circles(true),