From fd6cc583ad813c8df86f68e94e64f34268742479 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 4 Oct 2026 13:42:44 -0400 Subject: [PATCH] fix: bound concurrency for independent async work --- src/providers/githubContributions.ts | 15 +- src/providers/gitlabContributors.ts | 75 +++++----- src/renders/circles.ts | 9 +- tests/async-work.test.js | 40 ++++++ tests/fixtures/async-work.mjs | 202 +++++++++++++++++++++++++++ 5 files changed, 294 insertions(+), 47 deletions(-) create mode 100644 tests/async-work.test.js create mode 100644 tests/fixtures/async-work.mjs diff --git a/src/providers/githubContributions.ts b/src/providers/githubContributions.ts index 3c835b0c..00957480 100644 --- a/src/providers/githubContributions.ts +++ b/src/providers/githubContributions.ts @@ -120,16 +120,21 @@ async function discoverReposFromContributions( console.log(`[contribkit][githubContributions] querying contributions across ${years.length} years...`) - for (const { from, to } of years) { + const pLimit = await import('p-limit').then(r => r.default) + const limit = pLimit(10) + const reposByYear = await Promise.all(years.map(({ from, to }) => limit(async () => { try { - const repos = await fetchContributionsForYear(graphqlFetch, login, from, to) - for (const repo of repos) { - repoMap.set(repo.nameWithOwner, repo) - } + return await fetchContributionsForYear(graphqlFetch, login, from, to) } catch (e: any) { console.warn(`[contribkit][githubContributions] failed contributions query for ${from.slice(0, 4)}:`, e.message) + return [] } + }))) + + for (const repos of reposByYear) { + for (const repo of repos) + repoMap.set(repo.nameWithOwner, repo) } } catch (e: any) { diff --git a/src/providers/gitlabContributors.ts b/src/providers/gitlabContributors.ts index 8c44cdd6..054f5886 100644 --- a/src/providers/gitlabContributors.ts +++ b/src/providers/gitlabContributors.ts @@ -71,46 +71,45 @@ export async function fetchGitlabContributors( page++ } - const sponsorships: Sponsorship[] = [] - - for (const contributor of allContributors) { - if (contributor.commits < minContributions) - continue - - try { - const userDetails = await $fetch('https://gitlab.com/api/v4/users', { - query: { - search: contributor.email, - }, - headers: { - 'PRIVATE-TOKEN': token, - 'Content-Type': 'application/json', - }, - }) - - if (userDetails && userDetails.length > 0) { - const user = userDetails[0] - sponsorships.push({ - sponsor: { - type: 'User', - login: user.username, - name: user.username, // user.name is also available - avatarUrl: user.avatar_url, - linkUrl: user.web_url, + const pLimit = await import('p-limit').then(r => r.default) + const limit = pLimit(10) + const sponsorships = await Promise.all(allContributors + .filter(contributor => contributor.commits >= minContributions) + .map(contributor => limit(async (): Promise => { + try { + const userDetails = await $fetch('https://gitlab.com/api/v4/users', { + query: { + search: contributor.email, + }, + headers: { + 'PRIVATE-TOKEN': token, + 'Content-Type': 'application/json', }, - isOneTime: false, - monthlyDollars: contributor.commits, - privacyLevel: 'PUBLIC', - tierName: 'Contributor', - createdAt: new Date().toISOString(), - provider: 'gitlabContributors', }) + + if (userDetails && userDetails.length > 0) { + const user = userDetails[0] + return { + sponsor: { + type: 'User', + login: user.username, + name: user.username, // user.name is also available + avatarUrl: user.avatar_url, + linkUrl: user.web_url, + }, + isOneTime: false, + monthlyDollars: contributor.commits, + privacyLevel: 'PUBLIC', + tierName: 'Contributor', + createdAt: new Date().toISOString(), + provider: 'gitlabContributors', + } + } } - } - catch (error) { - console.warn(`Failed to fetch user details for ${contributor.email}:`, error) - } - } + catch (error) { + console.warn(`Failed to fetch user details for ${contributor.email}:`, error) + } + }))) - return sponsorships + return sponsorships.filter(sponsorship => sponsorship !== undefined) } diff --git a/src/renders/circles.ts b/src/renders/circles.ts index 64f8351f..7013aef8 100644 --- a/src/renders/circles.ts +++ b/src/renders/circles.ts @@ -39,8 +39,9 @@ export const circlesRenderer: ContribkitRenderer = { p.padding(config.width / 400) const circles = p(root).descendants().slice(1) - for (const circle of circles) { - composer.addRaw(await generateBadge( + const pLimit = await import('p-limit').then(r => r.default) + const limit = pLimit(15) + const badges = await Promise.all(circles.map(circle => limit(() => generateBadge( circle.x - circle.r, circle.y - circle.r, circle.data.sponsor, @@ -54,8 +55,8 @@ export const circlesRenderer: ContribkitRenderer = { }, 0.5, config.imageFormat, - )) - } + )))) + composer.addRaw(badges.join('')) composer.height = config.width diff --git a/tests/async-work.test.js b/tests/async-work.test.js new file mode 100644 index 00000000..5300f969 --- /dev/null +++ b/tests/async-work.test.js @@ -0,0 +1,40 @@ +import { execFileSync } from 'node:child_process' +import { resolve } from 'node:path' +import { describe, expect, it } from '@jest/globals' + +// Run the TypeScript modules with the same loader as the development CLI. +// A separate process lets each case intercept fetch before ofetch is imported. +function runScenario(scenario) { + execFileSync(process.execPath, [ + '--import', 'tsx', + resolve('tests/fixtures/async-work.mjs'), + scenario, + ], { encoding: 'utf8', timeout: 10000 }) +} + +describe('independent asynchronous work', () => { + it.each([ + { + name: 'overlaps yearly queries with a cap and keeps year order when deduplicating', + scenario: 'github', + }, + { + name: 'retains successful years when one contribution query fails', + scenario: 'github-failure', + }, + { + name: 'overlaps GitLab lookups with a cap, preserving contributor order and partial results', + scenario: 'gitlab', + }, + { + name: 'renders circle badges in packed order with unchanged SVG output', + scenario: 'circles', + }, + { + name: 'propagates circle image processing failures', + scenario: 'circles-failure', + }, + ])('$name', ({ scenario }) => { + expect(() => runScenario(scenario)).not.toThrow() + }) +}) diff --git a/tests/fixtures/async-work.mjs b/tests/fixtures/async-work.mjs new file mode 100644 index 00000000..29e13e27 --- /dev/null +++ b/tests/fixtures/async-work.mjs @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' + +function json(data) { + return new Response(JSON.stringify(data), { + headers: { 'Content-Type': 'application/json' }, + }) +} + +// Hold the first group of requests so concurrency and ordering checks do not +// depend on elapsed-time comparisons or external services. +function requestGate(size) { + let active = 0 + let maximum = 0 + let started = 0 + let release + let ready + const pending = new Promise(resolve => release = resolve) + const filled = new Promise(resolve => ready = resolve) + + return { + async wait() { + active++ + started++ + maximum = Math.max(maximum, active) + if (started === size) + ready() + try { + await pending + } + finally { + active-- + } + }, + async open() { + let timer + try { + await Promise.race([ + filled, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Only ${started} requests started before the gate opened`)), 1000) + }), + ]) + } + finally { + clearTimeout(timer) + } + // Let queued microtasks run so an unbounded implementation exceeds size. + await delay(0) + assert.equal(started, size) + release() + }, + verify() { + assert.equal(active, 0) + assert.equal(maximum, size) + }, + } +} + +function repository(year) { + return { + name: 'repo', + nameWithOwner: 'owner/repo', + url: `https://github.com/owner/repo?year=${year}`, + owner: { login: 'owner', url: 'https://github.com/owner', avatarUrl: 'avatar', __typename: 'User' }, + } +} + +async function github(failYear) { + const finalYear = new Date().getFullYear() + const firstYear = finalYear - 13 + const gate = requestGate(10) + const warnings = [] + const completed = [] + console.warn = (...args) => warnings.push(args.join(' ')) + + globalThis.fetch = async (_url, options) => { + const { query, variables } = JSON.parse(options.body) + if (query.includes('createdAt')) + return json({ data: { user: { createdAt: `${firstYear}-01-01T00:00:00Z` } } }) + if (query.includes('contributionsCollection')) { + const year = Number(variables.from.slice(0, 4)) + await gate.wait() + // The first year finishes last in the initial group. + if (year === firstYear) + await delay(30) + completed.push(year) + if (failYear && year === finalYear) + throw new Error('year unavailable') + return json({ data: { user: { contributionsCollection: { + commitContributionsByRepository: [{ repository: repository(year) }], + } } } }) + } + if (query.includes('pageInfo')) + return json({ data: { search: { pageInfo: { hasNextPage: false, endCursor: null }, edges: [] } } }) + return json({ data: { search: { issueCount: 3 } } }) + } + + const { fetchGitHubContributions } = await import('../../src/providers/githubContributions.ts') + const result = fetchGitHubContributions('test-token', 'contributor') + await gate.open() + const sponsorships = await result + gate.verify() + assert.notEqual(completed[0], firstYear) + assert.equal(sponsorships.length, 1) + assert.equal(sponsorships[0].sponsor.linkUrl, repository(failYear ? finalYear - 1 : finalYear).url) + assert.equal(sponsorships[0].monthlyDollars, 3) + assert.equal(warnings.length, failYear ? 1 : 0) +} + +async function gitlab() { + const contributors = Array.from({ length: 101 }, (_, i) => ({ + name: `user${i}`, email: `${i}@example.test`, commits: i < 24 ? 2 : 0, + })) + const gate = requestGate(10) + const pages = [] + const searched = [] + const warnings = [] + console.warn = (...args) => warnings.push(args.join(' ')) + + globalThis.fetch = async (input) => { + const url = new URL(input) + if (url.pathname.endsWith('/contributors')) { + const page = Number(url.searchParams.get('page')) + pages.push(page) + return json(contributors.slice((page - 1) * 100, page * 100)) + } + const index = Number(url.searchParams.get('search').split('@')[0]) + searched.push(index) + await gate.wait() + if (index === 0) + await delay(30) + if (index === 3) + return new Response('user unavailable', { status: 400 }) + return json(index === 4 ? [] : [{ + id: index, username: `user${index}`, name: `User ${index}`, + avatar_url: `avatar${index}`, web_url: `https://gitlab.com/user${index}`, + }]) + } + + const { fetchGitlabContributors } = await import('../../src/providers/gitlabContributors.ts') + const result = fetchGitlabContributors('test-token', 123, 2) + await gate.open() + const sponsorships = await result + gate.verify() + assert.deepEqual(pages, [1, 2]) + assert.equal(searched.length, 24) + assert.deepEqual(sponsorships.map(ship => ship.sponsor.login), + Array.from({ length: 24 }, (_, i) => i).filter(i => i !== 3 && i !== 4).map(i => `user${i}`)) + assert.ok(sponsorships.every(ship => ship.monthlyDollars === 2)) + assert.equal(warnings.length, 1) +} + +async function circles(failImage) { + const { default: sharp } = await import('sharp') + const { hierarchy, pack } = await import('d3-hierarchy') + const { circlesRenderer } = await import('../../src/renders/circles.ts') + const { generateBadge, SvgComposer } = await import('../../src/processing/svg.ts') + const avatar = await sharp({ create: { width: 4, height: 4, channels: 4, background: 'red' } }).png().toBuffer() + const sponsors = Array.from({ length: 20 }, (_, i) => ({ + sponsor: { + type: 'User', login: `user${i}`, name: `User ${i}`, avatarUrl: 'avatar', + avatarBuffer: failImage && i === 0 ? Buffer.from('invalid image') : avatar, + }, + monthlyDollars: 20 - i, + })) + const config = { + width: 200, imageFormat: 'png', svgInlineCSS: '', includePastSponsors: true, + circles: { weightInterop: ship => ship.monthlyDollars }, + } + if (failImage) { + await assert.rejects(circlesRenderer.renderSVG(config, sponsors), /unsupported image format/i) + return + } + + const svg = await circlesRenderer.renderSVG(config, sponsors) + const root = hierarchy({ ...sponsors[0], children: sponsors, id: 'root' }) + .sum(config.circles.weightInterop) + .sort((a, b) => (b.value ?? 0) - (a.value ?? 0)) + const packed = pack().size([config.width, config.width]).padding(config.width / 400)(root).descendants().slice(1) + const composer = new SvgComposer(config) + // Reference the original badge coordinates and packed order. + const badges = await Promise.all(packed.map(circle => + generateBadge(circle.x - circle.r, circle.y - circle.r, circle.data.sponsor, { + name: false, boxHeight: circle.r * 2, boxWidth: circle.r * 2, avatar: { size: circle.r * 2 }, + }, 0.5, config.imageFormat))) + badges.forEach(badge => composer.addRaw(badge)) + composer.height = config.width + assert.equal(svg, composer.generateSvg()) + assert.equal(await circlesRenderer.renderSVG(config, []), new SvgComposer(config).addSpan(config.width).generateSvg()) +} + +const scenarios = { + 'github': () => github(false), + 'github-failure': () => github(true), + 'gitlab': gitlab, + 'circles': () => circles(false), + 'circles-failure': () => circles(true), +} + +assert.ok(Object.hasOwn(scenarios, process.argv[2]), 'Unknown test scenario') +await scenarios[process.argv[2]]()