From 68c9ec7ac340a0643b37b121297f3c0bb0dc1fd6 Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Mon, 3 Aug 2026 14:14:16 +0530 Subject: [PATCH 1/4] Existing Index data structure added --- .../pkg/har/migrate/types/existing_index.go | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 modules/har/pkg/har/migrate/types/existing_index.go diff --git a/modules/har/pkg/har/migrate/types/existing_index.go b/modules/har/pkg/har/migrate/types/existing_index.go new file mode 100644 index 0000000..19fa495 --- /dev/null +++ b/modules/har/pkg/har/migrate/types/existing_index.go @@ -0,0 +1,124 @@ +package types + +import ( + "strings" + "sync" +) + +// ExistingIndex is a read-only-after-build snapshot of what already exists at the +// destination registry: pkg -> version -> set of LOWERCASED destination (HAR) +// file paths, exactly as returned by GetArtifactFiles. +// +// Lookups query by source-relative path (types.File.Uri); HasFile owns the +// reverse conversion from a stored HAR path back to source form (see +// harToSourcePath), so every package-type path rewrite lives in one place and +// the index build can store HAR paths verbatim. +// +// Concurrency: AddFile takes mu during the concurrent build. After +// BuildExistingIndex returns, the struct is treated as immutable and all reads +// are lock-free. +type ExistingIndex struct { + files map[string]map[string]map[string]struct{} + mu sync.Mutex +} + +func NewExistingIndex() *ExistingIndex { + return &ExistingIndex{ + files: map[string]map[string]map[string]struct{}{}, + } +} + +// AddFile records a destination (HAR) file path under (pkg, version); the path +// is lowercased for case-insensitive matching. +func (i *ExistingIndex) AddFile(pkg, version, harPath string) { + i.mu.Lock() + defer i.mu.Unlock() + if i.files[pkg] == nil { + i.files[pkg] = map[string]map[string]struct{}{} + } + if i.files[pkg][version] == nil { + i.files[pkg][version] = map[string]struct{}{} + } + i.files[pkg][version][strings.ToLower(harPath)] = struct{}{} +} + +// HasFile reports whether the source-relative filePath already exists at the +// destination. The index stores destination (HAR) paths, so HasFile converts +// stored paths back to source form (harToSourcePath) before comparing. +func (i *ExistingIndex) HasFile(pkg, version, filePath string, artifactType ArtifactType) bool { + lower := strings.ToLower(filePath) + + // NPM and MAVEN flatten all packages/versions under one pseudo-bucket in + // the source tree; scan every bucket converting stored HAR paths back. + if artifactType == NPM || artifactType == MAVEN { + for p, fv := range i.files { + for v, fs := range fv { + for harPath := range fs { + if harToSourcePath(artifactType, harPath, p, v) == lower { + return true + } + } + } + } + return false + } + + fs := i.files[pkg][version] + if fs == nil { + return false + } + + // O(1) direct lookup covers GENERIC/RAW/PYTHON/DART/PUPPET/CONAN etc. + if _, ok := fs[lower]; ok { + return true + } + + // Types with a HAR prefix rewrite (NUGET) need per-entry conversion. + if needsPathRewrite(artifactType) { + for harPath := range fs { + if harToSourcePath(artifactType, harPath, pkg, version) == lower { + return true + } + } + } + return false +} + +// harToSourcePath converts a stored HAR file path back to source-relative form +// so HasFile can compare against a source-tree query. +func harToSourcePath(artifactType ArtifactType, harPath, pkg, version string) string { + switch artifactType { + case NUGET: + return stripLeadingSegments(harPath, 2) + case NPM: + p := strings.ToLower(pkg) + prefix := "/" + p + "/" + strings.ToLower(version) + "/" + if rest, ok := strings.CutPrefix(harPath, prefix); ok { + return "/" + p + "/-/" + rest + } + return harPath + default: + return harPath + } +} + +func needsPathRewrite(artifactType ArtifactType) bool { + return artifactType == NUGET +} + +func stripLeadingSegments(p string, n int) string { + parts := strings.Split(strings.TrimPrefix(p, "/"), "/") + if len(parts) <= n { + return p + } + return "/" + strings.Join(parts[n:], "/") +} + +// FilesFor returns the lowercased HAR file-path set for (pkg, version), or nil. +// The returned map must be treated as read-only. +func (i *ExistingIndex) FilesFor(pkg, version string) map[string]struct{} { + if fv, ok := i.files[pkg]; ok { + return fv[version] + } + return nil +} From 39cbc228f68dd8c40733e307bf58a5a4c2becc7b Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Mon, 3 Aug 2026 14:16:08 +0530 Subject: [PATCH 2/4] Added BuildeExistingIndex to the adapters interfaces --- modules/har/pkg/har/migrate/adapter/adapter.go | 1 + modules/har/pkg/har/migrate/adapter/har/adapter.go | 4 ++++ modules/har/pkg/har/migrate/adapter/jfrog/adapter.go | 4 ++++ modules/har/pkg/har/migrate/adapter/nexus/adapter.go | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/modules/har/pkg/har/migrate/adapter/adapter.go b/modules/har/pkg/har/migrate/adapter/adapter.go index 07b9471..f126234 100644 --- a/modules/har/pkg/har/migrate/adapter/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/adapter.go @@ -66,6 +66,7 @@ type Adapter interface { files []*types.PackageFiles, metadata map[string]interface{}, ) error + BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) } var registry = map[types.RegistryType]Factory{} diff --git a/modules/har/pkg/har/migrate/adapter/har/adapter.go b/modules/har/pkg/har/migrate/adapter/har/adapter.go index b2fbad7..c8ac134 100644 --- a/modules/har/pkg/har/migrate/adapter/har/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/har/adapter.go @@ -172,3 +172,7 @@ func (a *harAdapter) CreateVersion(registry string, artifactName string, version return fmt.Errorf("not implemented") } } + +func (a *harAdapter) BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) { + return a.client.buildExistingIndex(ctx, registryRef, concurrency) +} diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go index d02ea8d..9bd1328 100644 --- a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go @@ -1243,6 +1243,10 @@ func (a *adapter) CreateVersion( return nil } +func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) { + return nil, nil +} + // getPythonVersionsFromTree extracts Python package versions by scanning the // file tree. This is used as a fallback when the .pypi index HTML files are // not available (e.g. packages deployed directly, not via the PyPI API). diff --git a/modules/har/pkg/har/migrate/adapter/nexus/adapter.go b/modules/har/pkg/har/migrate/adapter/nexus/adapter.go index 07c7ea8..c6fb9c6 100644 --- a/modules/har/pkg/har/migrate/adapter/nexus/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/nexus/adapter.go @@ -440,3 +440,7 @@ func (a *adapter) CreateVersion( ) error { return nil } + +func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) { + return nil, nil +} From cbffd00f87dfbe8848b4aad1e4c5c41e5c29be1a Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Mon, 3 Aug 2026 14:18:02 +0530 Subject: [PATCH 3/4] Actual buildExistingIndex method using v1 pagination --- .../har/pkg/har/migrate/adapter/har/client.go | 94 +++++++++++++++++++ .../pkg/har/migrate/adapter/harbor/adapter.go | 4 + 2 files changed, 98 insertions(+) diff --git a/modules/har/pkg/har/migrate/adapter/har/client.go b/modules/har/pkg/har/migrate/adapter/har/client.go index 9dbeccd..80f805c 100644 --- a/modules/har/pkg/har/migrate/adapter/har/client.go +++ b/modules/har/pkg/har/migrate/adapter/har/client.go @@ -11,6 +11,7 @@ import ( http2 "net/http" "path/filepath" "strings" + "sync" "time" "github.com/harness/cli/modules/har/pkg/har/migrate/adapter/har/arapi" @@ -19,6 +20,7 @@ import ( "github.com/google/uuid" retryablehttp "github.com/hashicorp/go-retryablehttp" + "github.com/rs/zerolog/log" ) type xAPIKeyTransport struct { @@ -1088,3 +1090,95 @@ func (c *client) uploadConanFile( } return nil } + +func (c *client) buildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) { + // Step 1: collect all artifact (package) names for this registry. + page := int64(0) + size := int64(100) + var artifactNames []string + for { + resp, err := c.apiClient.GetAllArtifactsByRegistryWithResponse(ctx, registryRef, + &arapi.GetAllArtifactsByRegistryParams{Page: &page, Size: &size}) + if err != nil { + return nil, fmt.Errorf("failed to list artifacts for index: %w", err) + } + if resp.StatusCode() != http2.StatusOK || resp.JSON200 == nil { + return nil, fmt.Errorf("failed to list artifacts for index: %s", resp.Status()) + } + for _, a := range resp.JSON200.Data.Artifacts { + artifactNames = append(artifactNames, a.Name) + } + data := resp.JSON200.Data + if len(data.Artifacts) < int(size) || + (data.PageCount != nil && data.PageIndex != nil && *data.PageIndex+1 >= *data.PageCount) { + break + } + page++ + } + + // Step 2: for each artifact, collect all (pkg, version) pairs. + type pkgVersion struct{ pkg, version string } + var pvPairs []pkgVersion + for _, name := range artifactNames { + p := int64(0) + for { + resp, err := c.apiClient.GetAllArtifactVersionsWithResponse(ctx, registryRef, name, + &arapi.GetAllArtifactVersionsParams{Page: &p, Size: &size}) + if err != nil { + log.Warn().Err(err).Str("artifact", name).Msg("buildExistingIndex: failed to list versions, skipping artifact") + break + } + if resp.StatusCode() != http2.StatusOK || resp.JSON200 == nil { + log.Warn().Str("artifact", name).Str("status", resp.Status()).Msg("buildExistingIndex: unexpected status listing versions, skipping artifact") + break + } + if resp.JSON200.Data.ArtifactVersions != nil { + for _, v := range *resp.JSON200.Data.ArtifactVersions { + if v.FileCount != nil && *v.FileCount == 0 { + continue + } + pvPairs = append(pvPairs, pkgVersion{name, v.Name}) + } + } + data := resp.JSON200.Data + if data.ArtifactVersions == nil || len(*data.ArtifactVersions) < int(size) || + (data.PageCount != nil && data.PageIndex != nil && *data.PageIndex+1 >= *data.PageCount) { + break + } + p++ + } + } + + // Step 3: fetch files per (pkg, version) concurrently, bounded by concurrency. + idx := types.NewExistingIndex() + if len(pvPairs) == 0 { + return idx, nil + } + + limit := concurrency + if limit <= 0 { + limit = 4 + } + sem := make(chan struct{}, limit) + var wg sync.WaitGroup + for _, pv := range pvPairs { + pv := pv + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + names, err := c.artifactGetFilesForVersion(ctx, registryRef, pv.pkg, pv.version) + if err != nil { + log.Warn().Err(err).Str("pkg", pv.pkg).Str("version", pv.version).Msg("buildExistingIndex: failed to list files for version, skipping") + return + } + for _, name := range names { + idx.AddFile(pv.pkg, pv.version, name) + } + }() + } + wg.Wait() + + return idx, nil +} diff --git a/modules/har/pkg/har/migrate/adapter/harbor/adapter.go b/modules/har/pkg/har/migrate/adapter/harbor/adapter.go index 03a9c0b..5b87cc1 100644 --- a/modules/har/pkg/har/migrate/adapter/harbor/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/harbor/adapter.go @@ -162,3 +162,7 @@ func (a *adapter) GetAllFilesForVersion(_ context.Context, _, _, _ string) ([]st func (a *adapter) CreateVersion(_ string, _ string, _ string, _ types.ArtifactType, _ []*types.PackageFiles, _ map[string]interface{}) error { return fmt.Errorf("CreateVersion not implemented for HARBOR") } + +func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) { + return nil, nil +} From 2781469925272dc423a58264ddd0cf816b23254a Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Mon, 3 Aug 2026 14:18:54 +0530 Subject: [PATCH 4/4] added existingIndex from registry --- .../har/pkg/har/migrate/migratable/package.go | 5 +- .../pkg/har/migrate/migratable/registry.go | 24 +++++++++- .../har/pkg/har/migrate/migratable/version.go | 46 ++++++++++++------- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/modules/har/pkg/har/migrate/migratable/package.go b/modules/har/pkg/har/migrate/migratable/package.go index dc58be7..5e037f0 100644 --- a/modules/har/pkg/har/migrate/migratable/package.go +++ b/modules/har/pkg/har/migrate/migratable/package.go @@ -53,6 +53,7 @@ type Package struct { config *types.Config registry types.RegistryInfo dryRunStats *types.DryRunStats + existingIndex *types.ExistingIndex } func NewPackageJob( @@ -69,6 +70,7 @@ func NewPackageJob( config *types.Config, registry types.RegistryInfo, dryRunStats *types.DryRunStats, + existingIndex *types.ExistingIndex, ) engine.Job { jobID := uuid.New().String() @@ -95,6 +97,7 @@ func NewPackageJob( config: config, registry: registry, dryRunStats: dryRunStats, + existingIndex: existingIndex, } } @@ -315,7 +318,7 @@ func (r *Package) Migrate(ctx context.Context) error { return fmt.Errorf("get version failed: %w", err) } job := NewVersionJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.destRegistry, r.artifactType, r.pkg, - version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats) + version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats, r.existingIndex) jobs = append(jobs, job) } diff --git a/modules/har/pkg/har/migrate/migratable/registry.go b/modules/har/pkg/har/migrate/migratable/registry.go index e71f137..dd32969 100644 --- a/modules/har/pkg/har/migrate/migratable/registry.go +++ b/modules/har/pkg/har/migrate/migratable/registry.go @@ -191,6 +191,28 @@ func (r *Registry) Migrate(ctx context.Context) error { } } + // Build a registry-wide snapshot of existing files so re-runs can skip + // already-migrated files without per-version API calls in version.go. + // Only applicable for types whose Migrate path consults the index. + indexApplicable := func(t types.ArtifactType) bool { + switch t { + case types.GENERIC, types.RAW, types.MAVEN, types.PYTHON, + types.NUGET, types.NPM, types.DART, types.PUPPET: + return true + } + return false + } + var existingIndex *types.ExistingIndex + if !r.config.DryRun && !r.config.Overwrite && indexApplicable(r.artifactType) { + idx, idxErr := r.destAdapter.BuildExistingIndex(ctx, r.registry.Path, r.config.Concurrency) + if idxErr != nil { + logger.Warn().Err(idxErr).Msg("Failed to build existing index, falling back to per-version checks") + } else { + existingIndex = idx + logger.Info().Msg("Built existing index for destination registry") + } + } + var jobs []engine.Job for _, pkg := range pkgs { treeNode, err2 := tree.GetNodeForPath(root, pkg.Path) @@ -199,7 +221,7 @@ func (r *Registry) Migrate(ctx context.Context) error { return fmt.Errorf("get node for path %s failed: %w", pkg.Path, err2) } job := NewPackageJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.sourcePackageHostname, r.destRegistry, r.artifactType, pkg, treeNode, - r.stats, r.mapping, r.config, r.registry, r.dryRunStats) + r.stats, r.mapping, r.config, r.registry, r.dryRunStats, existingIndex) jobs = append(jobs, job) } diff --git a/modules/har/pkg/har/migrate/migratable/version.go b/modules/har/pkg/har/migrate/migratable/version.go index fd51939..1b03bff 100644 --- a/modules/har/pkg/har/migrate/migratable/version.go +++ b/modules/har/pkg/har/migrate/migratable/version.go @@ -34,6 +34,7 @@ type Version struct { registry types.RegistryInfo existingFileMap map[string]bool dryRunStats *types.DryRunStats + existingIndex *types.ExistingIndex } func NewVersionJob( @@ -50,6 +51,7 @@ func NewVersionJob( config *types.Config, registry types.RegistryInfo, dryRunStats *types.DryRunStats, + existingIndex *types.ExistingIndex, ) engine.Job { jobID := uuid.New().String() @@ -78,6 +80,7 @@ func NewVersionJob( registry: registry, existingFileMap: make(map[string]bool), dryRunStats: dryRunStats, + existingIndex: existingIndex, } } @@ -104,20 +107,20 @@ func (r *Version) Pre(ctx context.Context) error { return nil } - // reading all existing files for this version from destination - + // If an upfront index was built by registry.go, skip the per-version API + // call entirely — the index already has everything. Fall back to the + // per-version lookup only when no index is available. if !r.config.Overwrite && (r.artifactType != types.MAVEN && r.artifactType != types.NPM && r.pkg.Name != "" && r.version.Name != "") { - - existingFiles, err := r.getAllExistingFilesForThisVersion(ctx) - - if err != nil { - logger.Warn().Err(err).Msg("Failed to get existing files, will proceed with migration") - } else { - // Populate existingFileMap with file name - for _, fileName := range existingFiles { - r.existingFileMap[fileName] = true + if r.existingIndex == nil { + existingFiles, err := r.getAllExistingFilesForThisVersion(ctx) + if err != nil { + logger.Warn().Err(err).Msg("Failed to get existing files, will proceed with migration") + } else { + for _, fileName := range existingFiles { + r.existingFileMap[strings.ToLower(fileName)] = true + } + logger.Info().Msgf("Found %d existing files for version %s", len(r.existingFileMap), r.version.Name) } - logger.Info().Msgf("Found %d existing files for version %s", len(r.existingFileMap), r.version.Name) } } logger.Info(). @@ -174,16 +177,25 @@ func (r *Version) Migrate(ctx context.Context) error { continue } } - // Check if file already exists in destination - - lowerCaseNormalizeFileName := strings.ToLower(file.Name) - if r.existingFileMap[lowerCaseNormalizeFileName] { + // Check if file already exists in destination (index takes priority). + // GENERIC/RAW: v1 HAR stores full path in Name; use Uri to match. + // All other types: Name is a basename; use it directly. + fileKey := file.Name + if r.artifactType == types.GENERIC || r.artifactType == types.RAW { + fileKey = strings.TrimPrefix(file.Uri, "/") + } + alreadyExists := false + if r.existingIndex != nil { + alreadyExists = r.existingIndex.HasFile(r.pkg.Name, r.version.Name, fileKey, r.artifactType) + } else { + alreadyExists = r.existingFileMap[strings.ToLower(fileKey)] + } + if alreadyExists { util.GetSkipPrinter().Println(fmt.Sprintf("Registry [%s], Package [%s/%s], File [%s] already exists", r.destRegistry, r.pkg.Name, r.version.Name, file.Name)) logger.Info().Msgf("Skipping file %s as it already exists in destination", file.Uri) - // Add to statistics stat := types.FileStat{ Name: file.Name, Registry: r.srcRegistry,