From 55b293e0256234174b3a99f1cdc4e80326a49420 Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Tue, 4 Aug 2026 18:39:43 +0530 Subject: [PATCH 1/2] fix(har): fix Maven URL, raw/generic %2F encoding, NuGet subdir, nested RPM repos (hc 417d71b, 329d705, f549396, ed236dd) --- .../har/pkg/har/migrate/adapter/har/client.go | 109 ++++++++++-------- .../pkg/har/migrate/adapter/jfrog/adapter.go | 90 +++++++++++---- 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/modules/har/pkg/har/migrate/adapter/har/client.go b/modules/har/pkg/har/migrate/adapter/har/client.go index 9dbeccd..63a972c 100644 --- a/modules/har/pkg/har/migrate/adapter/har/client.go +++ b/modules/har/pkg/har/migrate/adapter/har/client.go @@ -99,22 +99,34 @@ type client struct { accountID string } +// uploadGenericFile, headRawFile, and uploadRawFile build the request URL manually +// and issue it directly (instead of going through the generated pkgClient) because +// the generated client percent-encodes "/" in path params (%2F), which the server +// answers with a 307 redirect for nested file paths. func (c *client) uploadGenericFile(registry, artifactName, version string, f *types.File, file io.ReadCloser) error { // For generic, include package/version as path segments: {package}/{version}/{filepath} fileUri := strings.TrimPrefix(f.Uri, "/") fullPath := fmt.Sprintf("%s/%s/%s", artifactName, version, fileUri) defer file.Close() - _, err2 := c.pkgClient.UploadGenericFileToPathWithBodyWithResponse( - context.Background(), - c.accountID, - registry, - fullPath, - "application/octet-stream", - file) + base := strings.TrimRight(c.url, "/") + url := fmt.Sprintf("%s/pkg/%s/%s/files/%s", base, c.accountID, registry, fullPath) + req, err := http2.NewRequest(http2.MethodPut, url, file) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/octet-stream") - if err2 != nil { - return fmt.Errorf("failed to upload file '%s/%s': %w", artifactName, version, err2) + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("failed to upload file '%s/%s': %w", artifactName, version, err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to upload file '%s/%s', status code: %d, response: %s", + artifactName, version, resp.StatusCode, string(body)) } return nil @@ -124,48 +136,58 @@ func (c *client) headRawFile(registryRef string, fileUri string) (bool, error) { fileUri = strings.TrimPrefix(fileUri, "/") parts := strings.Split(registryRef, "/") registry := parts[len(parts)-1] - resp, err := c.pkgClient.HeadGenericFileAtPathWithResponse( - context.Background(), - c.accountID, - registry, - fileUri, - ) + + base := strings.TrimRight(c.url, "/") + url := fmt.Sprintf("%s/pkg/%s/%s/files/%s", base, c.accountID, registry, fileUri) + req, err := http2.NewRequest(http2.MethodHead, url, nil) + if err != nil { + return false, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.client.Do(req) if err != nil { return false, fmt.Errorf("failed to HEAD raw file '%s': %w", fileUri, err) } + defer resp.Body.Close() - if resp.StatusCode() == http2.StatusOK { + switch resp.StatusCode { + case http2.StatusOK: return true, nil - } - if resp.StatusCode() == http2.StatusNotFound { + case http2.StatusNotFound: return false, nil + default: + return false, fmt.Errorf("unexpected status code %d for HEAD on raw file '%s'", resp.StatusCode, fileUri) } - return false, fmt.Errorf("unexpected status code %d for HEAD on raw file '%s'", resp.StatusCode(), fileUri) } func (c *client) uploadRawFile(registry string, f *types.File, file io.ReadCloser) error { fileUri := strings.TrimPrefix(f.Uri, "/") defer file.Close() - resp, err := c.pkgClient.UploadGenericFileToPathWithBodyWithResponse( - context.Background(), - c.accountID, - registry, - fileUri, - "application/octet-stream", - file, - ) + base := strings.TrimRight(c.url, "/") + url := fmt.Sprintf("%s/pkg/%s/%s/files/%s", base, c.accountID, registry, fileUri) + req, err := http2.NewRequest(http2.MethodPut, url, file) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := c.client.Do(req) if err != nil { return fmt.Errorf("failed to upload raw file '%s': %w", fileUri, err) } - if resp.StatusCode() == http2.StatusConflict { + defer resp.Body.Close() + + switch { + case resp.StatusCode == http2.StatusConflict: return types.ErrArtifactAlreadyExists + case resp.StatusCode >= 200 && resp.StatusCode <= 299: + return nil + default: + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to upload raw file '%s', status code: %d, response: %s", + fileUri, resp.StatusCode, string(body)) } - if resp.StatusCode() < 200 || resp.StatusCode() > 299 { - return fmt.Errorf("failed to upload raw file '%s', status %d", fileUri, resp.StatusCode()) - } - - return nil } func (c *client) uploadMavenFile( @@ -176,7 +198,7 @@ func (c *client) uploadMavenFile( file io.ReadCloser, ) error { fileUri := strings.TrimPrefix(f.Uri, "/") - url := fmt.Sprintf("%s/maven/%s/%s/%s", c.url, c.accountID, registry, fileUri) + url := fmt.Sprintf("%s/pkg/%s/%s/maven/%s", c.url, c.accountID, registry, fileUri) // Create request req, err := http2.NewRequest(http2.MethodPut, url, file) if err != nil { @@ -266,24 +288,21 @@ func (c *client) uploadNugetFile( return nil } -// nugetSubDir extracts the subdirectory prefix from a NuGet file URI. -// JFrog stores NuGet packages as: /{subdir}/{packageId}/{version}/{filename} -// The last 3 segments are always packageId/version/filename. -// Returns the subdirectory with a trailing slash, or empty string if none. +// nugetSubDir extracts the directory prefix from a NuGet file URI. +// Returns everything except the filename (last segment), with a trailing slash, +// or empty string if the file is at the root. // // Examples: // -// "foo/company.grpc.pkg/1.0.0/company.grpc.pkg.1.0.0.nupkg" → "foo/" -// "a/b/pkg/1.0.0/pkg.1.0.0.nupkg" → "a/b/" -// "company.grpc.pkg/1.0.0/company.grpc.pkg.1.0.0.nupkg" → "" +// "a/b/c/d/proto-bindings.0.8.662.nupkg" → "a/b/c/d/" +// "foo/company.grpc.pkg.1.0.0.nupkg" → "foo/" +// "company.grpc.pkg.1.0.0.nupkg" → "" func nugetSubDir(fileUri string) string { parts := strings.Split(strings.TrimPrefix(fileUri, "/"), "/") - // Need at least 4 segments for there to be a subdirectory - // (subdir + packageId + version + filename) - if len(parts) <= 3 { + if len(parts) <= 1 { return "" } - return strings.Join(parts[:len(parts)-3], "/") + "/" + return strings.Join(parts[:len(parts)-1], "/") + "/" } func (c *client) uploadNPMFile( diff --git a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go index d02ea8d..c8f5007 100644 --- a/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go +++ b/modules/har/pkg/har/migrate/adapter/jfrog/adapter.go @@ -230,33 +230,83 @@ func (a *adapter) GetPackages(registry string, artifactType types.ArtifactType, return packages, nil } else if artifactType == types.RPM { - node, err := tree.GetNodeForPath(root, "/repodata/repomd.xml") - if err != nil { - return nil, fmt.Errorf("get node for path: %w", err) - } - file, _, err := a.DownloadFile(registry, node.File.Uri) + // Find all repodata/repomd.xml files in the tree. A single JFrog repo + // can contain multiple RPM repos at different paths (nested repos). + files, err := tree.GetAllFiles(root) if err != nil { - return nil, fmt.Errorf("download repomd.xml: %w", err) + return nil, fmt.Errorf("get all files: %w", err) } - defer file.Close() - primaryLocation, err := extractPrimaryLocation(file) - if err != nil { - return nil, fmt.Errorf("extract primary location: %w", err) - } + var allPackages []types.Package + seenRepomdPaths := make(map[string]bool) + successfulRepos := 0 - primaryFile, _, err := a.DownloadFile(registry, primaryLocation) - if err != nil { - return nil, fmt.Errorf("download primary file: %w", err) + for _, file := range files { + if file.Folder { + continue + } + // Look for all repodata/repomd.xml files at any depth + if !strings.HasSuffix(file.Uri, "/repodata/repomd.xml") { + continue + } + if seenRepomdPaths[file.Uri] { + continue + } + seenRepomdPaths[file.Uri] = true + + log.Info().Msgf("Found RPM repository metadata at: %s", file.Uri) + + repomdFile, _, err := a.DownloadFile(registry, file.Uri) + if err != nil { + log.Warn().Msgf("Failed to download repomd.xml at %s: %v", file.Uri, err) + continue + } + + primaryLocation, err := extractPrimaryLocation(repomdFile) + repomdFile.Close() + if err != nil { + log.Warn().Msgf("Failed to extract primary location from %s: %v", file.Uri, err) + continue + } + + // Resolve relative primary location path against repomd.xml location. + // The primary location in repomd.xml is relative to the RPM repo root + // (parent of repodata/), not to the repodata/ directory itself. + primaryPath := primaryLocation + if !strings.HasPrefix(primaryLocation, "/") { + repomdDir := path.Dir(file.Uri) // e.g., "/repodata" or "/centos7/repodata" + rpmRepoRoot := path.Dir(repomdDir) // e.g., "/" or "/centos7" + primaryPath = path.Join(rpmRepoRoot, primaryLocation) + } + + primaryFile, _, err := a.DownloadFile(registry, primaryPath) + if err != nil { + log.Warn().Msgf("Failed to download primary file at %s: %v", primaryPath, err) + continue + } + + // Extract package URLs from primary.xml.gz + rpmPackages, err := extractRPMPackages(primaryFile, registry) + primaryFile.Close() + if err != nil { + log.Warn().Msgf("Failed to extract RPM packages from %s: %v", primaryPath, err) + continue + } + + log.Info().Msgf("Extracted %d packages from RPM repo at %s", len(rpmPackages), file.Uri) + allPackages = append(allPackages, rpmPackages...) + successfulRepos++ } - defer primaryFile.Close() - // Extract package URLs from primary.xml.gz - packages, err := extractRPMPackages(primaryFile, registry) - if err != nil { - return nil, fmt.Errorf("extract RPM package URLs: %w", err) + if len(seenRepomdPaths) == 0 { + return nil, fmt.Errorf("no repodata/repomd.xml found in repository") } - return packages, nil + if successfulRepos == 0 { + return nil, fmt.Errorf("all %d RPM repositories failed to process", len(seenRepomdPaths)) + } + + log.Info().Msgf("Found total of %d RPM packages across %d repositories", len(allPackages), len(seenRepomdPaths)) + return allPackages, nil } else if artifactType == types.GO { leaves, _ := tree.GetAllFiles(root) packageMap := make(map[string]bool) From c97a97d47d4d029d307f4969036067564c596ff1 Mon Sep 17 00:00:00 2001 From: naman-nirwan Date: Tue, 4 Aug 2026 18:39:50 +0530 Subject: [PATCH 2/2] fix(har): add retry to push artifact HTTP client (hc 2e1fa5d) --- modules/har/pkg/har/harutil.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/har/pkg/har/harutil.go b/modules/har/pkg/har/harutil.go index 1c372d5..b3242c1 100644 --- a/modules/har/pkg/har/harutil.go +++ b/modules/har/pkg/har/harutil.go @@ -18,6 +18,8 @@ import ( "time" "github.com/harness/cli/pkg/auth" + + retryablehttp "github.com/hashicorp/go-retryablehttp" ) // atomicWrite writes content to path via a temp file + rename, ensuring an @@ -61,9 +63,19 @@ func parseRegistryAndName(id string) (registry, name string, err error) { return parts[0], parts[1], nil } -// newHTTPClient returns an HTTP client with a 10-minute timeout suitable for large artifact uploads/downloads. +// newHTTPClient returns a retry-enabled HTTP client with a 10-minute timeout, +// suitable for large artifact uploads/downloads that may hit transient network errors. func newHTTPClient() *http.Client { - return &http.Client{Timeout: 10 * time.Minute} + rc := retryablehttp.NewClient() + rc.RetryMax = 5 + rc.RetryWaitMin = 200 * time.Millisecond + rc.RetryWaitMax = 1 * time.Minute + rc.Backoff = retryablehttp.RateLimitLinearJitterBackoff + rc.Logger = nil + + client := rc.StandardClient() // returns *http.Client using a retrying RoundTripper + client.Timeout = 10 * time.Minute + return client } // setAuthHeader sets the appropriate auth header on req (Bearer for SSO, x-api-key for PAT).