feat(migrate): add ExistingIndex for registry-wide skip on re-runs

modules/har/pkg/har/migrate/adapter/adapter.go

@@ -66,6 +66,7 @@ type Adapter interface {
 		files []*types.PackageFiles,
 		metadata map[string]interface{},
 	) error
+	BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error)
 }
 
 var registry = map[types.RegistryType]Factory{}

modules/har/pkg/har/migrate/adapter/har/adapter.go

@@ -172,3 +172,7 @@ func (a *harAdapter) CreateVersion(registry string, artifactName string, version
 		return fmt.Errorf("not implemented")
 	}
 }
+
+func (a *harAdapter) BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) {
+	return a.client.buildExistingIndex(ctx, registryRef, concurrency)
+}

modules/har/pkg/har/migrate/adapter/har/client.go

@@ -11,6 +11,7 @@ import (
 	http2 "net/http"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/harness/cli/modules/har/pkg/har/migrate/adapter/har/arapi"
@@ -19,6 +20,7 @@ import (
 
 	"github.com/google/uuid"
 	retryablehttp "github.com/hashicorp/go-retryablehttp"
+	"github.com/rs/zerolog/log"
 )
 
 type xAPIKeyTransport struct {
@@ -1088,3 +1090,95 @@ func (c *client) uploadConanFile(
 	}
 	return nil
 }
+
+func (c *client) buildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) {
+	// Step 1: collect all artifact (package) names for this registry.
+	page := int64(0)
+	size := int64(100)
+	var artifactNames []string
+	for {
+		resp, err := c.apiClient.GetAllArtifactsByRegistryWithResponse(ctx, registryRef,
+			&arapi.GetAllArtifactsByRegistryParams{Page: &page, Size: &size})
+		if err != nil {
+			return nil, fmt.Errorf("failed to list artifacts for index: %w", err)
+		}
+		if resp.StatusCode() != http2.StatusOK || resp.JSON200 == nil {
+			return nil, fmt.Errorf("failed to list artifacts for index: %s", resp.Status())
+		}
+		for _, a := range resp.JSON200.Data.Artifacts {
+			artifactNames = append(artifactNames, a.Name)
+		}
+		data := resp.JSON200.Data
+		if len(data.Artifacts) < int(size) ||
+			(data.PageCount != nil && data.PageIndex != nil && *data.PageIndex+1 >= *data.PageCount) {
+			break
+		}
+		page++
+	}
+
+	// Step 2: for each artifact, collect all (pkg, version) pairs.
+	type pkgVersion struct{ pkg, version string }
+	var pvPairs []pkgVersion
+	for _, name := range artifactNames {
+		p := int64(0)
+		for {
+			resp, err := c.apiClient.GetAllArtifactVersionsWithResponse(ctx, registryRef, name,
+				&arapi.GetAllArtifactVersionsParams{Page: &p, Size: &size})
+			if err != nil {
+				log.Warn().Err(err).Str("artifact", name).Msg("buildExistingIndex: failed to list versions, skipping artifact")
+				break
+			}
+			if resp.StatusCode() != http2.StatusOK || resp.JSON200 == nil {
+				log.Warn().Str("artifact", name).Str("status", resp.Status()).Msg("buildExistingIndex: unexpected status listing versions, skipping artifact")
+				break
+			}
+			if resp.JSON200.Data.ArtifactVersions != nil {
+				for _, v := range *resp.JSON200.Data.ArtifactVersions {
+					if v.FileCount != nil && *v.FileCount == 0 {
+						continue
+					}
+					pvPairs = append(pvPairs, pkgVersion{name, v.Name})
+				}
+			}
+			data := resp.JSON200.Data
+			if data.ArtifactVersions == nil || len(*data.ArtifactVersions) < int(size) ||
+				(data.PageCount != nil && data.PageIndex != nil && *data.PageIndex+1 >= *data.PageCount) {
+				break
+			}
+			p++
+		}
+	}
+
+	// Step 3: fetch files per (pkg, version) concurrently, bounded by concurrency.
+	idx := types.NewExistingIndex()
+	if len(pvPairs) == 0 {
+		return idx, nil
+	}
+
+	limit := concurrency
+	if limit <= 0 {
+		limit = 4
+	}
+	sem := make(chan struct{}, limit)
+	var wg sync.WaitGroup
+	for _, pv := range pvPairs {
+		pv := pv
+		wg.Add(1)
+		sem <- struct{}{}
+		go func() {
+			defer wg.Done()
+			defer func() { <-sem }()
+			names, err := c.artifactGetFilesForVersion(ctx, registryRef, pv.pkg, pv.version)
+			if err != nil {
+				log.Warn().Err(err).Str("pkg", pv.pkg).Str("version", pv.version).Msg("buildExistingIndex: failed to list files for version, skipping")
+				return
+			}
+			for _, name := range names {
+				idx.AddFile(pv.pkg, pv.version, name)
+			}
+		}()
+	}
+	wg.Wait()
+
+	return idx, nil
+}

modules/har/pkg/har/migrate/adapter/harbor/adapter.go

@@ -162,3 +162,7 @@ func (a *adapter) GetAllFilesForVersion(_ context.Context, _, _, _ string) ([]st
 func (a *adapter) CreateVersion(_ string, _ string, _ string, _ types.ArtifactType, _ []*types.PackageFiles, _ map[string]interface{}) error {
 	return fmt.Errorf("CreateVersion not implemented for HARBOR")
 }
+
+func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
+	return nil, nil
+}

modules/har/pkg/har/migrate/adapter/jfrog/adapter.go

@@ -1243,6 +1243,10 @@ func (a *adapter) CreateVersion(
 	return nil
 }
 
+func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
+	return nil, nil
+}
+
 // getPythonVersionsFromTree extracts Python package versions by scanning the
 // file tree. This is used as a fallback when the .pypi index HTML files are
 // not available (e.g. packages deployed directly, not via the PyPI API).

modules/har/pkg/har/migrate/adapter/nexus/adapter.go

@@ -440,3 +440,7 @@ func (a *adapter) CreateVersion(
 ) error {
 	return nil
 }
+
+func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
+	return nil, nil
+}

modules/har/pkg/har/migrate/migratable/package.go

@@ -53,6 +53,7 @@ type Package struct {
 	config                *types.Config
 	registry              types.RegistryInfo
 	dryRunStats           *types.DryRunStats
+	existingIndex         *types.ExistingIndex
 }
 
 func NewPackageJob(
@@ -69,6 +70,7 @@ func NewPackageJob(
 	config *types.Config,
 	registry types.RegistryInfo,
 	dryRunStats *types.DryRunStats,
+	existingIndex *types.ExistingIndex,
 ) engine.Job {
 	jobID := uuid.New().String()
 
@@ -95,6 +97,7 @@ func NewPackageJob(
 		config:                config,
 		registry:              registry,
 		dryRunStats:           dryRunStats,
+		existingIndex:         existingIndex,
 	}
 }
 
@@ -315,7 +318,7 @@ func (r *Package) Migrate(ctx context.Context) error {
 				return fmt.Errorf("get version failed: %w", err)
 			}
 			job := NewVersionJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.destRegistry, r.artifactType, r.pkg,
-				version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats)
+				version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats, r.existingIndex)
 			jobs = append(jobs, job)
 		}
 

modules/har/pkg/har/migrate/migratable/registry.go

@@ -191,6 +191,28 @@ func (r *Registry) Migrate(ctx context.Context) error {
 		}
 	}
 
+	// Build a registry-wide snapshot of existing files so re-runs can skip
+	// already-migrated files without per-version API calls in version.go.
+	// Only applicable for types whose Migrate path consults the index.
+	indexApplicable := func(t types.ArtifactType) bool {
+		switch t {
+		case types.GENERIC, types.RAW, types.MAVEN, types.PYTHON,
+			types.NUGET, types.NPM, types.DART, types.PUPPET:
+			return true
+		}
+		return false
+	}
+	var existingIndex *types.ExistingIndex
+	if !r.config.DryRun && !r.config.Overwrite && indexApplicable(r.artifactType) {
+		idx, idxErr := r.destAdapter.BuildExistingIndex(ctx, r.registry.Path, r.config.Concurrency)
+		if idxErr != nil {
+			logger.Warn().Err(idxErr).Msg("Failed to build existing index, falling back to per-version checks")
+		} else {
+			existingIndex = idx
+			logger.Info().Msg("Built existing index for destination registry")
+		}
+	}
+
 	var jobs []engine.Job
 	for _, pkg := range pkgs {
 		treeNode, err2 := tree.GetNodeForPath(root, pkg.Path)
@@ -199,7 +221,7 @@ func (r *Registry) Migrate(ctx context.Context) error {
 			return fmt.Errorf("get node for path %s failed: %w", pkg.Path, err2)
 		}
 		job := NewPackageJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.sourcePackageHostname, r.destRegistry, r.artifactType, pkg, treeNode,
-			r.stats, r.mapping, r.config, r.registry, r.dryRunStats)
+			r.stats, r.mapping, r.config, r.registry, r.dryRunStats, existingIndex)
 		jobs = append(jobs, job)
 	}
 

modules/har/pkg/har/migrate/migratable/version.go

@@ -34,6 +34,7 @@ type Version struct {
 	registry        types.RegistryInfo
 	existingFileMap map[string]bool
 	dryRunStats     *types.DryRunStats
+	existingIndex   *types.ExistingIndex
 }
 
 func NewVersionJob(
@@ -50,6 +51,7 @@ func NewVersionJob(
 	config *types.Config,
 	registry types.RegistryInfo,
 	dryRunStats *types.DryRunStats,
+	existingIndex *types.ExistingIndex,
 ) engine.Job {
 	jobID := uuid.New().String()
 
@@ -78,6 +80,7 @@ func NewVersionJob(
 		registry:        registry,
 		existingFileMap: make(map[string]bool),
 		dryRunStats:     dryRunStats,
+		existingIndex:   existingIndex,
 	}
 }
 
@@ -104,20 +107,20 @@ func (r *Version) Pre(ctx context.Context) error {
 		return nil
 	}
 
-	// reading all existing files for this version from destination
-
+	// If an upfront index was built by registry.go, skip the per-version API
+	// call entirely — the index already has everything. Fall back to the
+	// per-version lookup only when no index is available.
 	if !r.config.Overwrite && (r.artifactType != types.MAVEN && r.artifactType != types.NPM && r.pkg.Name != "" && r.version.Name != "") {
-
-		existingFiles, err := r.getAllExistingFilesForThisVersion(ctx)
-
-		if err != nil {
-			logger.Warn().Err(err).Msg("Failed to get existing files, will proceed with migration")
-		} else {
-			// Populate existingFileMap with file name
-			for _, fileName := range existingFiles {
-				r.existingFileMap[fileName] = true
+		if r.existingIndex == nil {
+			existingFiles, err := r.getAllExistingFilesForThisVersion(ctx)
+			if err != nil {
+				logger.Warn().Err(err).Msg("Failed to get existing files, will proceed with migration")
+			} else {
+				for _, fileName := range existingFiles {
+					r.existingFileMap[strings.ToLower(fileName)] = true
+				}
+				logger.Info().Msgf("Found %d existing files for version %s", len(r.existingFileMap), r.version.Name)
 			}
-			logger.Info().Msgf("Found %d existing files for version %s", len(r.existingFileMap), r.version.Name)
 		}
 	}
 	logger.Info().
@@ -174,16 +177,25 @@ func (r *Version) Migrate(ctx context.Context) error {
 					continue
 				}
 			}
-			// Check if file already exists in destination
-
-			lowerCaseNormalizeFileName := strings.ToLower(file.Name)
-			if r.existingFileMap[lowerCaseNormalizeFileName] {
+			// Check if file already exists in destination (index takes priority).
+			// GENERIC/RAW: v1 HAR stores full path in Name; use Uri to match.
+			// All other types: Name is a basename; use it directly.
+			fileKey := file.Name
+			if r.artifactType == types.GENERIC || r.artifactType == types.RAW {
+				fileKey = strings.TrimPrefix(file.Uri, "/")
+			}
+			alreadyExists := false
+			if r.existingIndex != nil {
+				alreadyExists = r.existingIndex.HasFile(r.pkg.Name, r.version.Name, fileKey, r.artifactType)
+			} else {
+				alreadyExists = r.existingFileMap[strings.ToLower(fileKey)]
+			}
+			if alreadyExists {
 				util.GetSkipPrinter().Println(fmt.Sprintf("Registry [%s], Package [%s/%s], File [%s] already exists",
 					r.destRegistry,
 					r.pkg.Name, r.version.Name, file.Name))
 				logger.Info().Msgf("Skipping file %s as it already exists in destination", file.Uri)
 
-				// Add to statistics
 				stat := types.FileStat{
 					Name:     file.Name,
 					Registry: r.srcRegistry,