perf(vulnfeeds): make vulnfeeds tests go faster

vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go

@@ -8,6 +8,7 @@ import (
 	"flag"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"path/filepath"
 	"slices"
@@ -21,6 +22,7 @@ import (
 	"github.com/google/osv.dev/vulnfeeds/conversion/cve5"
 	"github.com/google/osv.dev/vulnfeeds/conversion/writer"
 	"github.com/google/osv.dev/vulnfeeds/gcs-tools"
+	"github.com/google/osv.dev/vulnfeeds/git"
 	"github.com/google/osv.dev/vulnfeeds/models"
 	"github.com/google/osv.dev/vulnfeeds/utility/logger"
 )
@@ -120,10 +122,11 @@ func main() {
 		logger.Info("GCS Upload Pool initialized", slog.String("bucket", *outputBucket))
 	}
 
+	repoTagsCache := git.NewRepoTagsCache()
 	// Start the worker pool.
 	for range *workers {
 		wg.Add(1)
-		go worker(&wg, jobs, gcsHelper, *localOutputDir, actualMetricsDir, cnaList, *rejectFailed, *outputMetrics, *gcsMetricsPrefix)
+		go worker(&wg, jobs, gcsHelper, *localOutputDir, actualMetricsDir, cnaList, *rejectFailed, *outputMetrics, *gcsMetricsPrefix, repoTagsCache)
 	}
 
 	// Discover files and send them to the workers.
@@ -180,7 +183,7 @@ func main() {
 }
 
 // worker is a function that processes CVE files from the jobs channel.
-func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDir string, metricsDir string, cnas []string, rejectFailed bool, outputMetrics bool, gcsMetricsPrefix string) {
+func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDir string, metricsDir string, cnas []string, rejectFailed bool, outputMetrics bool, gcsMetricsPrefix string, cache git.RepoTagsCache) {
 	defer wg.Done()
 	for path := range jobs {
 		data, err := os.ReadFile(path)
@@ -211,7 +214,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
 		}
 
 		if gcsHelper != nil {
-			vuln, metrics := cve5.CVEToOSV(cve, sourceLink)
+			vuln, metrics := cve5.CVEToOSV(cve, sourceLink, cache, http.DefaultClient)
 			if metrics.Outcome == models.Successful {
 				successfulConversionsCount.Add(1)
 			}
@@ -263,7 +266,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
 			}
 
 			// Perform the conversion and export the results.
-			metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsSink, sourceLink)
+			metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsSink, sourceLink, cache, http.DefaultClient)
 			if err != nil {
 				logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 			} else {

vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go

@@ -9,8 +9,11 @@ import (
 
 	"github.com/google/osv.dev/vulnfeeds/conversion/cve5"
 	"github.com/google/osv.dev/vulnfeeds/conversion/writer"
+	"github.com/google/osv.dev/vulnfeeds/git"
 	"github.com/google/osv.dev/vulnfeeds/models"
 	"github.com/google/osv.dev/vulnfeeds/utility/logger"
+
+	"net/http"
 )
 
 var (
@@ -64,7 +67,8 @@ func main() {
 	}
 
 	// Perform the conversion and export the results.
-	if metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, ""); err != nil {
+	cache := git.NewRepoTagsCache()
+	if metrics, err := cve5.ConvertAndExportCVEToOSV(cve, osvFile, metricsFile, "", cache, http.DefaultClient); err != nil {
 		logger.Warn("Failed to generate an OSV record", slog.String("cve", string(cveID)), slog.Any("err", err))
 	} else {
 		logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))

vulnfeeds/conversion/common.go

@@ -171,7 +171,7 @@ func ConductAnalysisAndUpload(prefix string, year string, metricsDir string, csv
 
 // GitVersionsToCommits examines repos and tries to convert versions to commits by treating them as Git tags.
 // Returns the resolved ranges, unresolved ranges, and successful repos involved.
-func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
+func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache, httpClient *http.Client) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
 	var newVersionRanges []models.RangeWithMetadata
 	unresolvedRanges := versionRanges
 	var successfulRepos []string
@@ -204,7 +204,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 			continue
 		}
 
-		repo, err := git.FindCanonicalLink(repo, http.DefaultClient, cache)
+		repo, err := git.FindCanonicalLink(repo, httpClient, cache)
 		if err != nil {
 			metrics.AddNote("Failed to find canonical link - %s %v", repo, err)
 			if git.IsRateLimit(err) {
@@ -729,12 +729,12 @@ func AddFieldToDatabaseSpecific(ds *structpb.Struct, field string, value any) er
 }
 
 // ProcessRanges attempts to resolve the given ranges to commits and updates the metrics accordingly.
-func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
+func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *models.ConversionMetrics, cache git.RepoTagsCache, httpClient *http.Client) ([]models.RangeWithMetadata, []models.RangeWithMetadata, []string) {
 	if len(ranges) == 0 {
 		return nil, nil, nil
 	}
 
-	r, un, sR := GitVersionsToCommits(ranges, repos, metrics, cache)
+	r, un, sR := GitVersionsToCommits(ranges, repos, metrics, cache, httpClient)
 	if len(r) > 0 {
 		metrics.ResolvedRangesCount += len(r)
 		metrics.SetOutcome(models.Successful)

vulnfeeds/conversion/common_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -747,7 +748,7 @@ func TestGitVersionsToCommits_Canonicalization(t *testing.T) {
 			}
 
 			metrics := &models.ConversionMetrics{}
-			gotResolved, gotUnresolved, gotSuccessful := GitVersionsToCommits(tt.versionRanges, tt.repos, metrics, cache)
+			gotResolved, gotUnresolved, gotSuccessful := GitVersionsToCommits(tt.versionRanges, tt.repos, metrics, cache, http.DefaultClient)
 
 			if len(gotResolved) != tt.wantResolved {
 				t.Errorf("GitVersionsToCommits() gotResolved count = %v, want %v", len(gotResolved), tt.wantResolved)