diff --git a/README.md b/README.md index 2dfbb13..fb654ba 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Resolution order: package override, then ecosystem override, then global default | Alpine | Alpine Linux | | ✓ | | Arch | Arch Linux | | ✗ | | Chef | Chef | | ✗ | -| Generic | Any | | ✗ | +| Generic | Any | | ✓ | | Helm | Kubernetes | | ✗ | | Vagrant | Vagrant | | ✗ | @@ -494,6 +494,32 @@ http://localhost:8080/apk/private apk appends the architecture and index filename to each repository line itself. +### GitHub Releases / mise (aqua backend) + +Configure named generic upstreams: + +```yaml +upstream: + generic: + github: "https://github.com" + github-api: "https://api.github.com" +``` + +Then rewrite GitHub URLs in mise's settings (`~/.config/mise/config.toml`, mise ≥ 2025.9.3): + +```toml +[settings.url_replacements] +"regex:^https://github\\.com/([^/]+)/([^/]+)/releases/download/(.+)" = "http://localhost:8080/generic/github/$1/$2/releases/download/$3" +"regex:^https://api\\.github\\.com/(.*)" = "http://localhost:8080/generic/github-api/$1" +``` + +Release assets are cached permanently after the first download and keep +installing while GitHub is down. Tag lookups through `api.github.com` are +cached for `metadata_ttl` and served stale during an outage or rate limit. +Commit a `mise.lock` and install with `mise install --locked` so pinned +installs need no API call at all. Add a bearer token for `https://api.github.com` +under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. + ## Configuration The proxy can be configured via: @@ -792,6 +818,7 @@ Recently cached: | `GET /helm/{repository}/*` | HTTP Helm chart repository protocol | | `GET /v2/*` | OCI/Docker registry protocol | | `GET /apk/{repository}/*` | Alpine APK repository protocol | +| `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | | `GET /debian/*` | Debian/APT repository protocol | | `GET /rpm/*` | RPM/Yum repository protocol | diff --git a/config.example.yaml b/config.example.yaml index 277b58c..b259a74 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -193,6 +193,14 @@ upstream: # alpine: "https://dl-cdn.alpinelinux.org/alpine" # private: "https://apk.example.com" + # Named generic HTTP upstreams (used by /generic/{name}/). The remaining + # request path and query are appended to the upstream URL. GitHub release + # assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached + # immutably; other paths use the metadata cache with stale-on-error. + # generic: + # github: "https://github.com" + # github-api: "https://api.github.com" + # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path # segment boundaries must match; the longest matching scope wins. diff --git a/docs/configuration.md b/docs/configuration.md index 1b70410..55ca7c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -209,6 +209,37 @@ repository's `index.yaml` so chart archives are downloaded through the proxy. Chart archives are retained only when their SHA-256 digest matches the digest listed in the index. Relative and absolute chart URLs are both supported. +Generic HTTP upstreams proxy plain downloads from fixed base URLs: + +```yaml +upstream: + # Named HTTP upstreams, served at /generic/{name}/. The rest of the + # request path and the query string are appended to the upstream URL. + generic: + github: "https://github.com" + github-api: "https://api.github.com" + auth: + # Optional: raise the GitHub API rate limit. Scoped to this host only, + # so the token is never sent to the object store GitHub redirects to. + "https://api.github.com": + type: bearer + token: "${GITHUB_TOKEN}" +``` + +Only configured upstreams are reachable, so this is not an open HTTP proxy. +Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are +version-pinned GitHub release assets: they are stored in the artifact cache +and served from it without revalidation, including while the upstream is +down. Every other path is served through the metadata cache (`cache_metadata` +must be enabled for offline fallback): fresh within `metadata_ttl`, then +revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with +a `Warning: 110` header when the upstream fails, refuses or rate-limits the +request. Metadata responses are buffered up to `metadata_max_size`, so keep +large mutable downloads (`releases/latest/download/...`) off this route. + +This is the cache behind [mise](https://mise.jdx.dev)'s aqua backend; see the +mise section in the README for the client-side `url_replacements`. + `upstream.oci_default` sets the registry used by unprefixed `/v2` requests, while `upstream.oci` selects named registries through the `upstream/{name}/` repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart` diff --git a/internal/config/config.go b/internal/config/config.go index 61aa490..e49e558 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -436,6 +436,14 @@ type UpstreamConfig struct { // oci://proxy.example.com/upstream/ghcr/owner/chart. OCI map[string]string `json:"oci" yaml:"oci"` + // Generic maps names to plain HTTP upstream base URLs, served at + // /generic/{name}/. The remaining request path and query string are + // appended to the upstream URL. GitHub release asset paths + // ({owner}/{repo}/releases/download/{tag}/{asset}) are cached in the + // artifact cache; everything else goes through the metadata cache. + // Example: {"github": "https://github.com", "github-api": "https://api.github.com"}. + Generic map[string]string `json:"generic" yaml:"generic"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -485,6 +493,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } + if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { + return err + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e64391..2527814 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1062,12 +1062,30 @@ func TestValidateNamedUpstreams(t *testing.T) { wantErr bool }{ { - name: "valid Helm, OCI, and APK upstreams", + name: "valid Helm, OCI, APK, and generic upstreams", modify: func(cfg *Config) { cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} cfg.Upstream.APK = map[string]string{"alpine": "https://dl-cdn.alpinelinux.org/alpine"} + cfg.Upstream.Generic = map[string]string{ + "github": "https://github.com", + "github-api": "https://api.github.com", + } + }, + }, + { + name: "generic upstream name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"github/releases": "https://github.com"} }, + wantErr: true, + }, + { + name: "generic upstream URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"github": "github.com"} + }, + wantErr: true, }, { name: "Helm upstream name contains path separator", diff --git a/internal/handler/generic.go b/internal/handler/generic.go new file mode 100644 index 0000000..5d7e6f4 --- /dev/null +++ b/internal/handler/generic.go @@ -0,0 +1,166 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "regexp" + "strings" +) + +const ( + genericEcosystem = "generic" + // genericAcceptAny is sent upstream when the client did not send an + // Accept header, so generic upstreams are not asked for JSON by default. + genericAcceptAny = "*/*" + // githubReleaseAssetMatchCount is the full match plus owner, repository, + // tag and asset filename. + githubReleaseAssetMatchCount = 5 +) + +// githubReleaseAssetPattern matches the path of a GitHub release asset +// download, {owner}/{repo}/releases/download/{tag}/{asset}. A tag pins the +// asset to one release, so these downloads are cached in the artifact cache +// and served without revalidation once fetched. +var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) + +// GenericHandler proxies plain HTTP downloads from configured upstream base +// URLs. Each configured upstream is mounted at /generic/{name}/ and the +// remaining request path (and query string) is appended to the upstream URL. +// +// Only configured upstreams are reachable, so the proxy is not an open HTTP +// proxy. The handler is the caching layer behind tools that download from +// fixed URL shapes, such as mise's aqua backend fetching GitHub release +// assets, and is pointed at by URL-rewriting settings on the client. +// +// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}) are +// version-pinned and cached in the shared artifact cache, so they keep being +// served when the upstream is unreachable. Every other path is served through +// the metadata cache: fresh within the metadata TTL, revalidated with the +// upstream's validators after that, and served stale when the upstream fails +// or refuses the request. That covers API responses such as +// api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. +type GenericHandler struct { + proxy *Proxy + repositories map[string]string +} + +// NewGenericHandler creates a generic HTTP download proxy handler. +func NewGenericHandler(proxy *Proxy, repositories map[string]string) *GenericHandler { + h := &GenericHandler{ + proxy: proxy, + repositories: make(map[string]string, len(repositories)), + } + for name, upstreamURL := range repositories { + h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") + } + return h +} + +// Routes returns the HTTP handler for generic download requests. +// Mount this at /generic on your router. +func (h *GenericHandler) Routes() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + path := strings.TrimPrefix(r.URL.Path, "/") + + if containsPathTraversal(path) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + repository, rest, ok := strings.Cut(path, "/") + upstreamURL, found := h.repositories[repository] + if !ok || rest == "" || !found { + http.NotFound(w, r) + return + } + + if asset, ok := parseGitHubReleaseAsset(rest); ok { + h.handleReleaseAsset(w, r, repository, upstreamURL, rest, asset) + return + } + + h.handleMetadata(w, r, repository, upstreamURL, rest) + }) +} + +// githubReleaseAsset is the identity of a version-pinned release download. +type githubReleaseAsset struct { + owner string + repo string + tag string + filename string +} + +// parseGitHubReleaseAsset extracts the release identity from a path shaped +// like {owner}/{repo}/releases/download/{tag}/{asset}. +func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { + matches := githubReleaseAssetPattern.FindStringSubmatch(path) + if len(matches) != githubReleaseAssetMatchCount { + return githubReleaseAsset{}, false + } + return githubReleaseAsset{ + owner: matches[1], + repo: matches[2], + tag: matches[3], + filename: matches[4], + }, true +} + +// handleReleaseAsset fetches and caches a version-pinned release asset in the +// artifact cache. The configured upstream name is part of the cache identity +// so two upstreams serving the same path never share bytes. +func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, asset githubReleaseAsset) { + name := asset.owner + "/" + asset.repo + downloadURL := upstreamURL + "/" + path + cacheFilename := repository + "/" + asset.filename + + h.proxy.Logger.Info("generic release asset download", + "repository", repository, "name", name, "version", asset.tag, "filename", asset.filename) + + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), genericEcosystem, name, asset.tag, cacheFilename, downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch release asset") + return + } + + if result.ContentType == "" { + result.ContentType = "application/octet-stream" + } + serveArtifact(w, r.Method, result) +} + +// handleMetadata serves any other path through the metadata cache. The query +// string is forwarded and is part of the cache identity, and the client's +// Accept header is replayed so content-negotiated upstreams (the GitHub API) +// cache the representation the client asked for. +func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + target := upstreamURL + "/" + path + if r.URL.RawQuery != "" { + target += "?" + r.URL.RawQuery + } + + accept := r.Header.Get("Accept") + if accept == "" { + accept = genericAcceptAny + } + + h.proxy.ProxyCached(w, r, target, genericEcosystem, + h.metadataCacheKey(repository, upstreamURL, path, r.URL.RawQuery), accept) +} + +// metadataCacheKey derives the metadata cache key from the upstream name, its +// URL, the request path and query. Hashing the identity keeps distinct +// upstreams from sharing entries and drops cached entries when an upstream is +// repointed, mirroring APKHandler.metadataCacheKey. +func (h *GenericHandler) metadataCacheKey(repository, upstreamURL, path, query string) string { + identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go new file mode 100644 index 0000000..23b2cb6 --- /dev/null +++ b/internal/handler/generic_test.go @@ -0,0 +1,339 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/registries/fetch" +) + +const testReleaseAssetPath = "/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" + +func TestParseGitHubReleaseAsset(t *testing.T) { + tests := []struct { + path string + want githubReleaseAsset + ok bool + }{ + { + "jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", + githubReleaseAsset{owner: "jqlang", repo: "jq", tag: "jq-1.7.1", filename: "jq-linux-amd64"}, + true, + }, + { + "cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.tar.gz", + githubReleaseAsset{owner: "cli", repo: "cli", tag: "v2.63.2", filename: "gh_2.63.2_linux_amd64.tar.gz"}, + true, + }, + // Mutable: resolves to whatever is latest today. + {"jqlang/jq/releases/latest/download/jq-linux-amd64", githubReleaseAsset{}, false}, + // API lookups and tag listings are not assets. + {"repos/jqlang/jq/releases/tags/jq-1.7.1", githubReleaseAsset{}, false}, + {"jqlang/jq/releases/tag/jq-1.7.1", githubReleaseAsset{}, false}, + // Source archives are a different shape. + {"jqlang/jq/archive/refs/tags/jq-1.7.1.tar.gz", githubReleaseAsset{}, false}, + // Extra or missing segments. + {"jqlang/jq/releases/download/jq-1.7.1", githubReleaseAsset{}, false}, + {"jqlang/jq/releases/download/jq-1.7.1/dir/asset", githubReleaseAsset{}, false}, + {"", githubReleaseAsset{}, false}, + } + + for _, tt := range tests { + got, ok := parseGitHubReleaseAsset(tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("parseGitHubReleaseAsset(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) + } + } +} + +func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { + h := NewGenericHandler(testProxy(), map[string]string{"github": "https://github.com"}) + + tests := []struct { + name string + method string + target string + want int + }{ + {"unknown upstream", http.MethodGet, "/gitlab/owner/repo/releases/download/v1/asset", http.StatusNotFound}, + {"missing path", http.MethodGet, "/github", http.StatusNotFound}, + {"missing path with slash", http.MethodGet, "/github/", http.StatusNotFound}, + {"traversal", http.MethodGet, "/github/../etc/passwd", http.StatusBadRequest}, + {"encoded traversal", http.MethodGet, "/github/%2e%2e/etc/passwd", http.StatusBadRequest}, + {"post", http.MethodPost, "/github/owner/repo/releases/download/v1/asset", http.StatusMethodNotAllowed}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(tt.method, tt.target, nil)) + if w.Code != tt.want { + t.Errorf("status = %d, want %d", w.Code, tt.want) + } + }) + } +} + +func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing.T) { + asset := []byte("jq binary bytes") + var available atomic.Bool + available.Store(true) + var upstreamRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != testReleaseAssetPath { + http.NotFound(w, r) + return + } + upstreamRequests.Add(1) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(asset) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("body = %q, want %q", got, asset) + } + if got := upstreamRequests.Load(); got != 1 { + t.Fatalf("upstream requests = %d, want 1", got) + } + + // Second request must be served from cache, even with the upstream down. + available.Store(false) + w = serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("cached: status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("cached: body = %q, want %q", got, asset) + } + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream requests after cache hit = %d, want 1", got) + } + + // HEAD is answered from the same cache entry without a body. + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/github"+testReleaseAssetPath, nil)) + if w.Code != http.StatusOK { + t.Fatalf("HEAD: status = %d, want 200", w.Code) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD: body length = %d, want 0", w.Body.Len()) + } +} + +func TestGenericHandler_ReleaseAssetNotFoundIsNotCached(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", w.Code, w.Body.String()) + } +} + +func TestGenericHandler_MetadataForwardsAcceptAndQueryAndServesStaleOnThrottle(t *testing.T) { + const apiPath = "/repos/jqlang/jq/releases/tags/jq-1.7.1" + body := `{"tag_name":"jq-1.7.1"}` + var throttled atomic.Bool + var gotAccept, gotQuery string + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != apiPath { + http.NotFound(w, r) + return + } + gotAccept = r.Header.Get("Accept") + gotQuery = r.URL.RawQuery + if throttled.Load() { + w.Header().Set("Retry-After", "60") + http.Error(w, `{"message":"API rate limit exceeded"}`, http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/vnd.github+json") + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte(body)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + // A tiny TTL so the second request is past freshness and has to consult + // the upstream, and the served copy is marked stale. + proxy.MetadataTTL = time.Millisecond + + h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL}) + + req := httptest.NewRequest(http.MethodGet, "/github-api"+apiPath+"?per_page=1", nil) + req.Header.Set("Accept", "application/vnd.github+json") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != body { + t.Errorf("body = %q, want %q", got, body) + } + if gotAccept != "application/vnd.github+json" { + t.Errorf("upstream Accept = %q, want the client's header replayed", gotAccept) + } + if gotQuery != "per_page=1" { + t.Errorf("upstream query = %q, want %q", gotQuery, "per_page=1") + } + if ct := w.Header().Get("Content-Type"); ct != "application/vnd.github+json" { + t.Errorf("Content-Type = %q, want upstream's", ct) + } + + // The upstream now throttles us: the cached body must be served stale + // rather than the 429 being passed through. + throttled.Store(true) + time.Sleep(5 * time.Millisecond) + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("throttled: status = %d, want 200 stale: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != body { + t.Errorf("throttled: body = %q, want cached %q", got, body) + } + if warning := w.Header().Get("Warning"); !strings.Contains(warning, "110") { + t.Errorf("throttled: Warning = %q, want a 110 stale warning", warning) + } +} + +func TestGenericHandler_MetadataWithoutAcceptAsksForAnything(t *testing.T) { + var gotAccept string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAccept = r.Header.Get("Accept") + _, _ = w.Write([]byte("checksums")) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github/jqlang/jq/releases/latest/download/sha256sum.txt") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if gotAccept != genericAcceptAny { + t.Errorf("upstream Accept = %q, want %q", gotAccept, genericAcceptAny) + } +} + +func TestGenericHandler_DistinctUpstreamsDoNotShareCache(t *testing.T) { + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("from first")) + })) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("from second")) + })) + defer second.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(first.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"one": first.URL, "two": second.URL}) + + w := serveGenericRequest(h, "/one"+testReleaseAssetPath) + if got := w.Body.String(); got != "from first" { + t.Fatalf("one: body = %q, want %q", got, "from first") + } + w = serveGenericRequest(h, "/two"+testReleaseAssetPath) + if got := w.Body.String(); got != "from second" { + t.Fatalf("two: body = %q, want %q (must not reuse the first upstream's cache entry)", got, "from second") + } +} + +func TestGenericHandler_UpstreamAuthIsScopedToTheConfiguredHost(t *testing.T) { + asset := []byte("private asset") + var storageAuth atomic.Value + storageAuth.Store("unset") + + // The object store the release host redirects to must never see the + // token configured for the release host. + objectStore := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + storageAuth.Store(r.Header.Get("Authorization")) + _, _ = w.Write(asset) + })) + defer objectStore.Close() + + var releaseAuth string + releaseHost := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + releaseAuth = r.Header.Get("Authorization") + if releaseAuth != "Bearer github-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + http.Redirect(w, r, objectStore.URL+"/signed"+r.URL.Path, http.StatusFound) + })) + defer releaseHost.Close() + + proxy, _, _, _ := setupTestProxy(t) + authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, + upstreamhttp.AuthFunc(func(url string) (string, string) { + if strings.HasPrefix(url, releaseHost.URL) { + return "Authorization", "Bearer github-token" + } + return "", "" + }))} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": releaseHost.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("body = %q, want %q", got, asset) + } + if releaseAuth != "Bearer github-token" { + t.Errorf("release host Authorization = %q, want the configured token", releaseAuth) + } + if got := storageAuth.Load(); got != "" { + t.Errorf("object store Authorization = %q, want none after the cross-host redirect", got) + } +} + +func serveGenericRequest(h *GenericHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} diff --git a/internal/server/server.go b/internal/server/server.go index 6fd56bd..f672a34 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -314,6 +314,7 @@ func (s *Server) serve(listener net.Listener) error { apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) @@ -336,6 +337,7 @@ func (s *Server) serve(listener net.Listener) error { r.Mount("/apk", http.StripPrefix("/apk", apkHandler.Routes())) r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) + r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) // Health, stats, and metrics endpoints r.Get("/health", s.handleHealth)