feat(har): port date filter and dry-run summary improvements from legacy hc

modules/har/pkg/har/migrate/adapter/adapter.go

@@ -66,6 +66,7 @@ type Adapter interface {
 		files []*types.PackageFiles,
 		metadata map[string]interface{},
 	) error
+	SearchFiles(registry string) ([]types.SearchedFile, error)
 	BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error)
 }
 

modules/har/pkg/har/migrate/adapter/har/adapter.go

@@ -173,6 +173,9 @@ func (a *harAdapter) CreateVersion(registry string, artifactName string, version
 	}
 }
 
+func (a *harAdapter) SearchFiles(_ string) ([]types.SearchedFile, error) {
+	return nil, fmt.Errorf("date filter (SearchFiles) is not supported for this source adapter")
+}
 func (a *harAdapter) BuildExistingIndex(ctx context.Context, registryRef string, concurrency int) (*types.ExistingIndex, error) {
 	return a.client.buildExistingIndex(ctx, registryRef, concurrency)
 }

modules/har/pkg/har/migrate/adapter/harbor/adapter.go

@@ -163,6 +163,9 @@ func (a *adapter) CreateVersion(_ string, _ string, _ string, _ types.ArtifactTy
 	return fmt.Errorf("CreateVersion not implemented for HARBOR")
 }
 
+func (a *adapter) SearchFiles(_ string) ([]types.SearchedFile, error) {
+	return nil, fmt.Errorf("date filter (SearchFiles) is not supported for this source adapter")
+}
 func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
 	return nil, nil
 }

modules/har/pkg/har/migrate/adapter/jfrog/adapter.go

@@ -1243,6 +1243,14 @@ func (a *adapter) CreateVersion(
 	return nil
 }
 
+func (a *adapter) SearchFiles(registry string) ([]types.SearchedFile, error) {
+	files, err := a.client.SearchFiles(registry)
+	if err != nil {
+		log.Error().Msgf("Failed to search files from registry: %v", err)
+		return nil, fmt.Errorf("failed to search files from registry: %w", err)
+	}
+	return files, nil
+}
 func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
 	return nil, nil
 }

modules/har/pkg/har/migrate/adapter/jfrog/client.go

@@ -49,6 +49,7 @@ type Client interface {
 	GetFile(registry string, path string) (io.ReadCloser, http.Header, error)
 	GetFiles(registry string) ([]types.File, error)
 	GetCatalog(registry string) ([]string, error)
+	SearchFiles(registry string) ([]types.SearchedFile, error)
 }
 
 // newClient constructs a jfrog client
@@ -291,3 +292,39 @@ func (c *client) catalog(url string) ([]string, string, error) {
 	}
 	return repositories.Repositories, parseNextLink(resp.Header.Get("Link")), nil
 }
+
+func (c *client) SearchFiles(registry string) ([]types.SearchedFile, error) {
+	aqlURL := fmt.Sprintf("%s/artifactory/api/search/aql", c.url)
+	query := fmt.Sprintf(`items.find({"repo": "%s", "type": "file"}).include("repo", "path", "name", "created", "modified", "stat.downloaded")`, registry)
+
+	req, err := http.NewRequest(http.MethodPost, aqlURL, strings.NewReader(query))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AQL request: %w", err)
+	}
+	req.Header.Set("Content-Type", "text/plain")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute AQL search for registry %q: %w", registry, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("AQL search failed for registry %q, status: %d", registry, resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read AQL response: %w", err)
+	}
+
+	type aqlResponse struct {
+		Results []types.SearchedFile `json:"results"`
+	}
+	var result aqlResponse
+	if err := json.Unmarshal(body, &result); err != nil {
+		return nil, fmt.Errorf("failed to parse AQL response: %w", err)
+	}
+
+	return result.Results, nil
+}

modules/har/pkg/har/migrate/adapter/mock_jfrog/mock_client.go

@@ -26,6 +26,7 @@ type mockClient struct {
 	catalogs      map[string][]string
 	fileContent   map[string][]byte // keyed by "registry/path"
 	binaryContent map[string][]byte
+	searchedFiles map[string][]types.SearchedFile
 }
 
 // NewMockClient creates a mock implementation of jfrog.Client backed by
@@ -37,6 +38,7 @@ func NewMockClient() jfrog.Client {
 		catalogs:      make(map[string][]string),
 		fileContent:   make(map[string][]byte),
 		binaryContent: make(map[string][]byte),
+		searchedFiles: make(map[string][]types.SearchedFile),
 	}
 	c.loadRegistries()
 	c.loadFiles()
@@ -46,6 +48,11 @@ func NewMockClient() jfrog.Client {
 	return c
 }
 
+// SetSearchedFiles registers per-registry AQL result data for date-filter tests.
+func (c *mockClient) SetSearchedFiles(registry string, files []types.SearchedFile) {
+	c.searchedFiles[registry] = files
+}
+
 func (c *mockClient) loadRegistries() {
 	data, err := testdataFS.ReadFile("testdata/registries.json")
 	if err != nil {
@@ -302,6 +309,13 @@ func (c *mockClient) GetFiles(registry string) ([]types.File, error) {
 	}, nil
 }
 
+func (c *mockClient) SearchFiles(registry string) ([]types.SearchedFile, error) {
+	if files, exists := c.searchedFiles[registry]; exists {
+		return files, nil
+	}
+	return nil, fmt.Errorf("no search data found for registry '%s'", registry)
+}
+
 func (c *mockClient) GetCatalog(registry string) ([]string, error) {
 	if catalog, exists := c.catalogs[registry]; exists {
 		return catalog, nil

modules/har/pkg/har/migrate/adapter/nexus/adapter.go

@@ -441,6 +441,9 @@ func (a *adapter) CreateVersion(
 	return nil
 }
 
+func (a *adapter) SearchFiles(_ string) ([]types.SearchedFile, error) {
+	return nil, fmt.Errorf("date filter (SearchFiles) is not supported for this source adapter")
+}
 func (a *adapter) BuildExistingIndex(_ context.Context, _ string, _ int) (*types.ExistingIndex, error) {
 	return nil, nil
 }

modules/har/pkg/har/migrate/migratable/package.go

@@ -53,6 +53,7 @@ type Package struct {
 	config                *types.Config
 	registry              types.RegistryInfo
 	dryRunStats           *types.DryRunStats
+	unfilteredRoot        *types.TreeNode
 	existingIndex         *types.ExistingIndex
 }
 
@@ -70,6 +71,7 @@ func NewPackageJob(
 	config *types.Config,
 	registry types.RegistryInfo,
 	dryRunStats *types.DryRunStats,
+	unfilteredRoot *types.TreeNode,
 	existingIndex *types.ExistingIndex,
 ) engine.Job {
 	jobID := uuid.New().String()
@@ -97,6 +99,7 @@ func NewPackageJob(
 		config:                config,
 		registry:              registry,
 		dryRunStats:           dryRunStats,
+		unfilteredRoot:        unfilteredRoot,
 		existingIndex:         existingIndex,
 	}
 }
@@ -314,11 +317,13 @@ func (r *Package) Migrate(ctx context.Context) error {
 		for _, version := range versions {
 			versionNode, err := tree.GetNodeForPath(r.node, version.Path)
 			if err != nil {
-				logger.Error().Msg("Failed to get node for version")
-				return fmt.Errorf("get version failed: %w", err)
+				// Version path not in the filtered tree — it was entirely pruned by date
+				// or pattern filters. Skip gracefully rather than aborting the package.
+				logger.Debug().Str("version", version.Name).Msg("version not in filtered tree, skipping (out of date-filter window)")
+				continue
 			}
 			job := NewVersionJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.destRegistry, r.artifactType, r.pkg,
-				version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats, r.existingIndex)
+				version, versionNode, r.stats, r.mapping, r.config, r.registry, r.dryRunStats, r.unfilteredRoot, r.existingIndex)
 			jobs = append(jobs, job)
 		}
 
@@ -667,28 +672,12 @@ func check(err error, context string) {
 	}
 }
 
-// addPackageToDryRunDirectory adds package to the directory structure
+// addPackageToDryRunDirectory adds package to the directory structure (thread-safe).
 func (r *Package) addPackageToDryRunDirectory() {
 	if r.dryRunStats == nil {
 		return
 	}
-
-	// Ensure registry entry exists (should already be created by Registry)
-	if r.dryRunStats.Directories[r.srcRegistry] == nil {
-		r.dryRunStats.Directories[r.srcRegistry] = &types.DryRunDirectoryEntry{
-			Registry: r.srcRegistry,
-			Packages: make(map[string]*types.DryRunPackageEntry),
-		}
-	}
-	dirEntry := r.dryRunStats.Directories[r.srcRegistry]
-
-	// Add package entry if not exists
-	if dirEntry.Packages[r.pkg.Name] == nil {
-		dirEntry.Packages[r.pkg.Name] = &types.DryRunPackageEntry{
-			Name:     r.pkg.Name,
-			Versions: make(map[string]*types.DryRunVersionEntry),
-		}
-	}
+	r.dryRunStats.EnsurePackage(r.srcRegistry, r.pkg.Name)
 }
 
 // Post Any post processing work

modules/har/pkg/har/migrate/migratable/registry.go

@@ -12,6 +12,7 @@ import (
 	"github.com/harness/cli/modules/har/pkg/har/migrate/util"
 
 	"github.com/google/uuid"
+	"github.com/pterm/pterm"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
@@ -136,32 +137,83 @@ func (r *Registry) Migrate(ctx context.Context) error {
 		return fmt.Errorf("get files from registry %s failed: %w", r.srcRegistry, err2)
 	}
 
-	// In dry-run mode, collect all files and initialize directory entry for this registry
+	pterm.Info.Println(fmt.Sprintf("Pulled %d file(s) from registry %s", len(files), r.srcRegistry))
+
+	// Keep a copy of the original file list before any filtering. This is used
+	// to build an unfilteredRoot for PYTHON so version enumeration is not broken
+	// when date filtering prunes .pypi index files.
+	originalFiles := files
+
+	// In dry-run mode, collect all files using thread-safe methods.
 	if r.config.DryRun && r.dryRunStats != nil {
-		// Add all files to the file list
+		var entries []types.DryRunFileEntry
 		for _, file := range files {
-			entry := types.DryRunFileEntry{
+			entries = append(entries, types.DryRunFileEntry{
 				Registry:     r.srcRegistry,
 				Name:         file.Name,
 				Uri:          file.Uri,
 				Size:         file.Size,
 				LastModified: file.LastModified,
-			}
-			r.dryRunStats.Files = append(r.dryRunStats.Files, entry)
+			})
 		}
+		r.dryRunStats.AddFiles(entries...)
+		r.dryRunStats.EnsureRegistry(r.srcRegistry)
 		logger.Info().Msgf("Dry-run: collected %d files from registry %s", len(files), r.srcRegistry)
+	}
 
-		// Initialize directory entry for this registry
-		if r.dryRunStats.Directories[r.srcRegistry] == nil {
-			r.dryRunStats.Directories[r.srcRegistry] = &types.DryRunDirectoryEntry{
-				Registry: r.srcRegistry,
-				Packages: make(map[string]*types.DryRunPackageEntry),
+	// Date-based filtering: query the source for file timestamps and narrow the
+	// file list to files that satisfy the configured createdAfter/downloadedAfter
+	// thresholds. Index/metadata files (PyPI .pypi/) are always preserved so
+	// package enumeration is not broken.
+	currArtifactType := r.artifactType
+	var dateFilteredFiles []types.File
+	dateFilterActive := false
+
+	if util.IsTimeBasedFilterPresent(r.mapping) {
+		dateFilterActive = true
+		df := r.mapping.DateFilter
+		if err := util.ValidateDateFilter(df); err != nil {
+			logger.Error().Err(err).Msg("Date filter validation failed")
+			return err
+		}
+
+		searchedFiles, searchErr := r.srcAdapter.SearchFiles(r.srcRegistry)
+		if searchErr != nil {
+			logger.Error().Msgf("Failed to search files from registry %s", r.srcRegistry)
+			return fmt.Errorf("search files from registry %s failed: %w", r.srcRegistry, searchErr)
+		}
+		logger.Info().Msgf("Applying time based filter (match: %s)", df.Match)
+		filteredURIs := util.CreateMapOfFilteredFile(searchedFiles, r.mapping)
+		logger.Info().Msgf("Time-based filter includes %d file(s) out of %d", len(filteredURIs), len(searchedFiles))
+
+		// Preserve index/metadata files regardless of date — enumeration reads them.
+		indexCount := 0
+		for _, f := range files {
+			if util.IsPackageIndexFile(r.artifactType, f.Uri) {
+				if _, ok := filteredURIs[f.Uri]; !ok {
+					filteredURIs[f.Uri] = struct{}{}
+					indexCount++
+				}
 			}
 		}
+		if indexCount > 0 {
+			logger.Info().Msgf("Preserving %d index/metadata file(s) exempt from date filter", indexCount)
+		}
+
+		dateFilteredFiles = util.FilterFilesByDate(files, filteredURIs)
+		logger.Info().Msgf("Count of filtered files by date filter: %d -> %d", len(files), len(dateFilteredFiles))
+		skippedByFilter := len(files) - len(dateFilteredFiles)
+		pterm.Info.Println(fmt.Sprintf("Registry %s: %d file(s) pulled, %d under skip condition (date/pattern filters)",
+			r.srcRegistry, len(files), skippedByFilter))
+
+		// Narrow the tree for all types EXCEPT metadata-driven types (RPM, DEBIAN)
+		// which need the full file tree to read repomd.xml / Packages.gz metadata.
+		if !util.IsMetadataDrivenArtifact(currArtifactType) {
+			files = dateFilteredFiles
+		}
 	}
 
 	// Filter files based on include/exclude patterns
-	currArtifactType := r.artifactType
 	if util.IsFileLevelFilterableArtifact(currArtifactType) {
 		if len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0 {
 			originalCount := len(files)
@@ -174,12 +226,33 @@ func (r *Registry) Migrate(ctx context.Context) error {
 
 	root := tree.TransformToTree(files)
 
+	// For PYTHON (IsAtomicVersionArtifact), build an unfilteredRoot from the
+	// original file list so version.go can recover distributions that were pruned
+	// by date or pattern filters. Other types pass nil.
+	var unfilteredRoot *types.TreeNode
+	if dateFilterActive && util.IsAtomicVersionArtifact(currArtifactType) {
+		recoveryFiles := originalFiles
+		if util.IsFileLevelFilterableArtifact(currArtifactType) &&
+			(len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0) {
+			recoveryFiles = util.FilterFilesByPatterns(originalFiles, r.mapping.IncludePatterns, r.mapping.ExcludePatterns)
+		}
+		unfilteredRoot = tree.TransformToTree(recoveryFiles)
+	}
+
 	pkgs, err := r.srcAdapter.GetPackages(r.srcRegistry, r.artifactType, root)
 	if err != nil {
 		logger.Error().Msg("Failed to get packages")
 		return fmt.Errorf("get packages failed: %w", err)
 	}
 
+	// For metadata-driven types, re-apply date filter at the package level.
+	if dateFilterActive && util.IsMetadataDrivenArtifact(currArtifactType) {
+		originalPkgCount := len(pkgs)
+		pkgs = util.FilterPackagesByFileName(pkgs, dateFilteredFiles)
+		logger.Info().Msgf("Date filter (post-GetPackages): %d -> %d packages for %s",
+			originalPkgCount, len(pkgs), currArtifactType)
+	}
+
 	// applying package level filter
 	if util.IsPackageLevelFilterableArtifact(currArtifactType) {
 		if len(r.mapping.IncludePatterns) > 0 || len(r.mapping.ExcludePatterns) > 0 {
@@ -221,7 +294,7 @@ func (r *Registry) Migrate(ctx context.Context) error {
 			return fmt.Errorf("get node for path %s failed: %w", pkg.Path, err2)
 		}
 		job := NewPackageJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.sourcePackageHostname, r.destRegistry, r.artifactType, pkg, treeNode,
-			r.stats, r.mapping, r.config, r.registry, r.dryRunStats, existingIndex)
+			r.stats, r.mapping, r.config, r.registry, r.dryRunStats, unfilteredRoot,existingIndex)
 		jobs = append(jobs, job)
 	}