From f6c54dc621a225ace5276d7a3a9e0c4b4b8dece3 Mon Sep 17 00:00:00 2001 From: Sachin Panayil Date: Mon, 4 May 2026 13:17:22 -0400 Subject: [PATCH] fixing linting errors --- agency-indexes/DoE/magpie.json | 22 +++--- assets/_common/js/filters.js | 27 +++++--- assets/_common/js/issue-filters.js | 15 +++-- config/updateCodeGov.js | 48 ++++++++----- config/updateIssuePool.js | 104 ++++++++++++++++------------- 5 files changed, 125 insertions(+), 91 deletions(-) diff --git a/agency-indexes/DoE/magpie.json b/agency-indexes/DoE/magpie.json index 60467ea8..9e2d8976 100644 --- a/agency-indexes/DoE/magpie.json +++ b/agency-indexes/DoE/magpie.json @@ -1,31 +1,31 @@ { "contact": { - "email": "christopher.kowalczyk@inl.gov" + "email": "holdensanchez2@llnl.gov" }, "date": { - "created": "2018-04-24", - "metadataLastUpdated": "2023-03-15" + "created": "2018-07-30", + "metadataLastUpdated": "2020-02-05" }, - "description": "Mesoscale Atomistic Glue Program for Integrated Execution", - "laborHours": 14333.6, + "description": "Magpie contains a number of scripts for running Big Data software in HPC environments, including Hadoop and Spark. There is support for Lustre, Slurm, Moab, Torque. and LSF.\n", + "laborHours": 52607.2, "languages": [], - "name": "magpie", - "organization": "Idaho National Laboratory (INL)", + "name": "Magpie", + "organization": "Lawrence Livermore National Laboratory (LLNL)", "permissions": { "exemptionText": null, "licenses": [ { - "URL": "https://api.github.com/licenses/lgpl-2.1", - "name": "LGPL-2.1" + "URL": "https://api.github.com/licenses/gpl-2.0", + "name": "GPL-2.0" } ], "usageType": "openSource" }, - "repositoryURL": "https://github.com/idaholab/magpie", + "repositoryURL": "https://github.com/LLNL/magpie", "status": "Production", "tags": [ "DOE CODE", - "Idaho National Laboratory (INL)" + "Lawrence Livermore National Laboratory (LLNL)" ], "vcs": "git" } \ No newline at end of file diff --git a/assets/_common/js/filters.js b/assets/_common/js/filters.js index 2974e14e..44a81af2 100644 --- a/assets/_common/js/filters.js +++ b/assets/_common/js/filters.js @@ -59,16 +59,18 @@ const applySelectFilters = (data, filters, targetType) => Object.entries(filters switch (key) { case 'language': - actualKey = 'languages' + actualKey = 'languages'; break; case 'category': - actualKey = 'categories' + actualKey = 'categories'; break; case 'platform': - actualKey = 'platforms' + actualKey = 'platforms'; break; case 'agency': - actualKey = 'agencyName' + actualKey = 'agencyName'; + break; + default: break; } @@ -107,10 +109,15 @@ const sortData = (data, sortBy) => { return 0; } + if (aVal === bVal) { + return 0; + } + if (direction === 'desc') { - return aVal < bVal ? 1 : aVal > bVal ? -1 : 0; + return aVal < bVal ? 1 : -1; } - return aVal > bVal ? 1 : aVal < bVal ? -1 : 0; + + return aVal > bVal ? 1 : -1; }); }; @@ -165,7 +172,7 @@ const createProjectCardHTML = (project) => { if (Array.isArray(value)) return value; if (typeof value === 'string') return [value]; return []; - } + }; return `
@@ -213,7 +220,7 @@ const createCard = (item, targetType) => { const cardElement = document.createElement('div'); cardElement.innerHTML = targetType === 'agencies' ? createAgencyCardHTML(item) - : createProjectCardHTML(item) + : createProjectCardHTML(item); return cardElement.firstElementChild; }; @@ -229,7 +236,7 @@ const updateDisplay = (state) => { state.filteredData.forEach(item => { const card = createCard(item, state.targetType); - container.appendChild(card) + container.appendChild(card); }); }; @@ -353,4 +360,4 @@ document.addEventListener('DOMContentLoaded', () => { } } } -}); \ No newline at end of file +}); diff --git a/assets/_common/js/issue-filters.js b/assets/_common/js/issue-filters.js index 808fd6bd..30ddf4aa 100644 --- a/assets/_common/js/issue-filters.js +++ b/assets/_common/js/issue-filters.js @@ -81,10 +81,15 @@ const sortData = (data, sortBy) => { return 0; } + if (aVal === bVal) { + return 0; + } + if (direction === 'desc') { - return aVal < bVal ? 1 : aVal > bVal ? -1 : 0; + return aVal < bVal ? 1 : -1; } - return aVal > bVal ? 1 : aVal < bVal ? -1 : 0; + + return aVal > bVal ? 1 : -1; }); }; @@ -159,7 +164,7 @@ const createIssueCardHTML = (issue) => { `; }; -const createCard = (item, targetType) => { +const createCard = (item) => { const cardElement = document.createElement('div'); cardElement.innerHTML = createIssueCardHTML(item); return cardElement.firstElementChild; @@ -176,7 +181,7 @@ const updateDisplay = (state) => { container.innerHTML = ''; state.filteredData.forEach(item => { - const card = createCard(item, state.targetType); + const card = createCard(item); container.appendChild(card); }); }; @@ -342,4 +347,4 @@ document.addEventListener('DOMContentLoaded', () => { } } } -}); \ No newline at end of file +}); diff --git a/config/updateCodeGov.js b/config/updateCodeGov.js index 03608c9d..719a37e8 100644 --- a/config/updateCodeGov.js +++ b/config/updateCodeGov.js @@ -19,25 +19,38 @@ async function updateCodeGov() { // read all files in the directory const filenames = await fs.readdir(CONFIG.agencyDirectory) - - // we know that the directory will only contain json files so dont need to check for non jsons - for (const file of filenames) { - const filePath = path.join(CONFIG.agencyDirectory, file) - - try { - const content = await fs.readFile(filePath, 'utf-8') - const jsonData = JSON.parse(content) - - // store the agency name only for readability in codegov.json - const matches = file.match(CONFIG.regex) - const agencyName = matches[1] + const processedFiles = await Promise.all( + filenames.map(async (file) => { + const filePath = path.join(CONFIG.agencyDirectory, file) + + try { + const content = await fs.readFile(filePath, 'utf-8') + const jsonData = JSON.parse(content) + + // store the agency name only for readability in codegov.json + const matches = file.match(CONFIG.regex) + const agencyName = matches ? matches[1] : null + + if (!agencyName) { + return null + } + + console.log(`✅ Successfully processed: ${file}`) + return [agencyName, jsonData] + } catch (error) { + console.error(`❌ Error processing file: ${file}`, error) + return null + } + }) + ) + + processedFiles.forEach((entry) => { + if (entry) { + const [agencyName, jsonData] = entry updatedJSON[agencyName] = jsonData - console.log(`✅ Successfully processed: ${file}`) - } catch (error) { - console.error(`❌ Error processing file: ${file}`, error) } - } + }) // actually update the codegov.json file const jsonString = JSON.stringify(updatedJSON, null, 2) @@ -46,7 +59,8 @@ async function updateCodeGov() { return updatedJSON } catch (error) { console.error('❌ Failed to update codegov.json:', error) + return null } } -updateCodeGov() \ No newline at end of file +updateCodeGov() diff --git a/config/updateIssuePool.js b/config/updateIssuePool.js index 2c347e2b..4ada8a3a 100644 --- a/config/updateIssuePool.js +++ b/config/updateIssuePool.js @@ -4,7 +4,7 @@ const path = require('path') const CONFIG = { repoFilePath: path.resolve(__dirname, '../codegov.json'), issueFilePath: path.resolve(__dirname, '../issue-pool.json'), - regex: /https?:\/\/github\.com\/([^\/]+)\/([^\/]+)/, + regex: /https?:\/\/github\.com\/([^/]+)\/([^/]+)/, githubToken: process.env.GITHUB_TOKEN, requiredLabel: 'code-gov', concurrentRepos: 6, // processing 6 repos at once but need to find the sweetspot because at this rate, it takes 18 minutes for the entire script to run through codegov.json. the "bathtub curve" is what we have here and what we need to experiment with and solve 👀 @@ -26,7 +26,9 @@ async function fetchWithRateLimit(url, options = {}) { if (CONFIG.rateLimitRemaining <= 10 && Date.now() < CONFIG.rateLimitReset) { const waitTime = CONFIG.rateLimitReset - Date.now() + 1000 // add 1 second buffer console.log(`Rate limit low (${CONFIG.rateLimitRemaining} remaining). Waiting ${Math.round(waitTime/1000)}s...`) - await new Promise(resolve => setTimeout(resolve, waitTime)) + await new Promise((resolve) => { + setTimeout(resolve, waitTime) + }) } const response = await fetch(url, options) @@ -34,8 +36,8 @@ async function fetchWithRateLimit(url, options = {}) { const remainingHeader = response.headers.get('X-RateLimit-Remaining') const resetHeader = response.headers.get('X-RateLimit-Reset') - if (remainingHeader) CONFIG.rateLimitRemaining = parseInt(remainingHeader) - if (resetHeader) CONFIG.rateLimitReset = parseInt(resetHeader) * 1000 + if (remainingHeader) CONFIG.rateLimitRemaining = parseInt(remainingHeader, 10) + if (resetHeader) CONFIG.rateLimitReset = parseInt(resetHeader, 10) * 1000 return response } @@ -47,29 +49,30 @@ async function getRepoInfo() { // dont know how i feel about this double loop se const content = await fs.readFile(CONFIG.repoFilePath, 'utf-8') const jsonData = JSON.parse(content) - for (const agencyKey in jsonData) { - const agency = jsonData[agencyKey] + Object.values(jsonData).forEach((agency) => { + if (!agency.releases) { + return + } - if (agency.releases) { - for (const organization of agency.releases) { + agency.releases.forEach((organization) => { + if (!organization.repositoryURL) { + return + } - if (organization.repositoryURL) { - const match = organization.repositoryURL.match(CONFIG.regex) + const match = organization.repositoryURL.match(CONFIG.regex) - if (match) { - const [url, owner, repo] = match - - repoInfo.push({ - ownerName: owner, - repoName: repo - }) - } else { - console.warn(`No match found for URL: ${organization.repositoryURL}`) - } - } + if (match) { + const [, owner, repo] = match + + repoInfo.push({ + ownerName: owner, + repoName: repo + }) + } else { + console.warn(`No match found for URL: ${organization.repositoryURL}`) } - } - } + }) + }) } catch (error) { console.error('Error in getting repo information:', error) } @@ -148,37 +151,34 @@ async function processSingleRepository(repo, headers) { const repoData = await repoResponse.json() const repoLanguage = repoData.language || '' - let page = 1 - let hasMore = true - - while (hasMore) { + const fetchIssuePage = async (page) => { const issuesUrl = `https://api.github.com/repos/${repo.ownerName}/${repo.repoName}/issues?page=${page}&per_page=100&state=open&labels=${CONFIG.requiredLabel}` const issuesResponse = await fetchWithRateLimit(issuesUrl, { headers }) if (!issuesResponse.ok) { console.error(`Failed to fetch issues for ${repo.ownerName}/${repo.repoName}: ${issuesResponse.status}`) - break + return } const issues = await issuesResponse.json() // endpoint always returns both issues and pull requests so we ignore the PRs - for (const [index, issue] of issues.entries()) { + issues.forEach((issue, index) => { if (issue.pull_request) { - continue + return } - + const transformedIssue = transformIssue(issue, repo, repoLanguage) repoIssues[transformedIssue.id] = transformedIssue // is having the ID is the best key name? console.log(`✅ Processed ${index + 1}/${issues.length}: ${repo.ownerName}/${repo.repoName}`) - } + }) - if (issues.length < 100) { - hasMore = false + if (issues.length === 100) { + await fetchIssuePage(page + 1) } - - page++ } + + await fetchIssuePage(1) } catch (error) { console.error(`❌ Error processing ${repo.ownerName}/${repo.repoName}:`, error) } @@ -193,25 +193,33 @@ async function updateIssuePool() { const headers = getHeaders() // process repositories in chunks of 3 for parallel processing - for (let i = 0; i < repoInfo.length; i += CONFIG.concurrentRepos) { - const chunk = repoInfo.slice(i, i + CONFIG.concurrentRepos) - console.log(`Processing chunk ${Math.floor(i/CONFIG.concurrentRepos) + 1}/${Math.ceil(repoInfo.length/CONFIG.concurrentRepos)} (${chunk.length} repos)`) - - const chunkPromises = chunk.map(repo => processSingleRepository(repo, headers)) + const chunkCount = Math.ceil(repoInfo.length / CONFIG.concurrentRepos) + const chunks = Array.from({ length: chunkCount }, (_, index) => { + const start = index * CONFIG.concurrentRepos + return repoInfo.slice(start, start + CONFIG.concurrentRepos) + }) + + await chunks.reduce(async (previousChunk, chunk, index) => { + await previousChunk + console.log(`Processing chunk ${index + 1}/${chunks.length} (${chunk.length} repos)`) + + const chunkPromises = chunk.map((repo) => processSingleRepository(repo, headers)) const chunkResults = await Promise.allSettled(chunkPromises) - - chunkResults.forEach((result, index) => { + + chunkResults.forEach((result, resultIndex) => { if (result.status === 'fulfilled') { Object.assign(issuePool, result.value) } else { - console.error(`Failed ${chunk[index].ownerName}/${chunk[index].repoName}:`, result.reason) + console.error(`Failed ${chunk[resultIndex].ownerName}/${chunk[resultIndex].repoName}:`, result.reason) } }) - if (i + CONFIG.concurrentRepos < repoInfo.length) { - await new Promise(resolve => setTimeout(resolve, 1000)) + if (index + 1 < chunks.length) { + await new Promise((resolve) => { + setTimeout(resolve, 1000) + }) } - } + }, Promise.resolve()) try { await fs.writeFile(CONFIG.issueFilePath, JSON.stringify(issuePool, null, 2)) @@ -224,4 +232,4 @@ async function updateIssuePool() { } // getRepoInfo() -updateIssuePool() \ No newline at end of file +updateIssuePool()