From 0067b1781e52796cec63eb13e34370989d56b756 Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Wed, 2 Sep 2026 16:30:56 +0100 Subject: [PATCH] Add generic HTTP download proxy for GitHub release assets Adds a /generic/{name}/ route backed by a new upstream.generic named-upstream map, so tools that download from fixed URL shapes (mise's aqua backend fetching GitHub release assets, and its tag lookups on api.github.com) can be pointed at the proxy with client-side URL rewriting. Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like {owner}/{repo}/releases/download/{tag}/{asset} are version-pinned and go through the artifact cache: fetched once, hashed, served without revalidation, and still served when the upstream is down. Every other path goes through the metadata cache with the client's Accept header and query string replayed, so API responses are fresh within metadata_ttl, revalidated after that, and served stale when the upstream fails or rate-limits the request. Tests cover path classification, unknown upstreams and traversal, cache hits with the upstream down, HEAD, 404 pass-through, Accept and query forwarding, stale-on-429, cache isolation between upstreams, and that an upstream token scoped to the release host is not sent to the object store it redirects to. Closes #183. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RDjDeq27CKzEP2o3GWBY7F --- README.md | 29 ++- config.example.yaml | 8 + docs/configuration.md | 31 +++ internal/config/config.go | 11 + internal/config/config_test.go | 20 +- internal/handler/generic.go | 166 +++++++++++++++ internal/handler/generic_test.go | 339 +++++++++++++++++++++++++++++++ internal/server/server.go | 2 + 8 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 internal/handler/generic.go create mode 100644 internal/handler/generic_test.go diff --git a/README.md b/README.md index 2dfbb13..fb654ba 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Resolution order: package override, then ecosystem override, then global default | Alpine | Alpine Linux | | ✓ | | Arch | Arch Linux | | ✗ | | Chef | Chef | | ✗ | -| Generic | Any | | ✗ | +| Generic | Any | | ✓ | | Helm | Kubernetes | | ✗ | | Vagrant | Vagrant | | ✗ | @@ -494,6 +494,32 @@ http://localhost:8080/apk/private apk appends the architecture and index filename to each repository line itself. +### GitHub Releases / mise (aqua backend) + +Configure named generic upstreams: + +```yaml +upstream: + generic: + github: "https://github.com" + github-api: "https://api.github.com" +``` + +Then rewrite GitHub URLs in mise's settings (`~/.config/mise/config.toml`, mise ≥ 2025.9.3): + +```toml +[settings.url_replacements] +"regex:^https://github\\.com/([^/]+)/([^/]+)/releases/download/(.+)" = "http://localhost:8080/generic/github/$1/$2/releases/download/$3" +"regex:^https://api\\.github\\.com/(.*)" = "http://localhost:8080/generic/github-api/$1" +``` + +Release assets are cached permanently after the first download and keep +installing while GitHub is down. Tag lookups through `api.github.com` are +cached for `metadata_ttl` and served stale during an outage or rate limit. +Commit a `mise.lock` and install with `mise install --locked` so pinned +installs need no API call at all. Add a bearer token for `https://api.github.com` +under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. + ## Configuration The proxy can be configured via: @@ -792,6 +818,7 @@ Recently cached: | `GET /helm/{repository}/*` | HTTP Helm chart repository protocol | | `GET /v2/*` | OCI/Docker registry protocol | | `GET /apk/{repository}/*` | Alpine APK repository protocol | +| `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | | `GET /debian/*` | Debian/APT repository protocol | | `GET /rpm/*` | RPM/Yum repository protocol | diff --git a/config.example.yaml b/config.example.yaml index 277b58c..b259a74 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -193,6 +193,14 @@ upstream: # alpine: "https://dl-cdn.alpinelinux.org/alpine" # private: "https://apk.example.com" + # Named generic HTTP upstreams (used by /generic/{name}/). The remaining + # request path and query are appended to the upstream URL. GitHub release + # assets ({owner}/{repo}/releases/download/{tag}/{asset}) are cached + # immutably; other paths use the metadata cache with stale-on-error. + # generic: + # github: "https://github.com" + # github-api: "https://api.github.com" + # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path # segment boundaries must match; the longest matching scope wins. diff --git a/docs/configuration.md b/docs/configuration.md index 1b70410..55ca7c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -209,6 +209,37 @@ repository's `index.yaml` so chart archives are downloaded through the proxy. Chart archives are retained only when their SHA-256 digest matches the digest listed in the index. Relative and absolute chart URLs are both supported. +Generic HTTP upstreams proxy plain downloads from fixed base URLs: + +```yaml +upstream: + # Named HTTP upstreams, served at /generic/{name}/. The rest of the + # request path and the query string are appended to the upstream URL. + generic: + github: "https://github.com" + github-api: "https://api.github.com" + auth: + # Optional: raise the GitHub API rate limit. Scoped to this host only, + # so the token is never sent to the object store GitHub redirects to. + "https://api.github.com": + type: bearer + token: "${GITHUB_TOKEN}" +``` + +Only configured upstreams are reachable, so this is not an open HTTP proxy. +Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are +version-pinned GitHub release assets: they are stored in the artifact cache +and served from it without revalidation, including while the upstream is +down. Every other path is served through the metadata cache (`cache_metadata` +must be enabled for offline fallback): fresh within `metadata_ttl`, then +revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with +a `Warning: 110` header when the upstream fails, refuses or rate-limits the +request. Metadata responses are buffered up to `metadata_max_size`, so keep +large mutable downloads (`releases/latest/download/...`) off this route. + +This is the cache behind [mise](https://mise.jdx.dev)'s aqua backend; see the +mise section in the README for the client-side `url_replacements`. + `upstream.oci_default` sets the registry used by unprefixed `/v2` requests, while `upstream.oci` selects named registries through the `upstream/{name}/` repository prefix. For example, `oci://proxy.example.com/upstream/ghcr/owner/chart` diff --git a/internal/config/config.go b/internal/config/config.go index 61aa490..e49e558 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -436,6 +436,14 @@ type UpstreamConfig struct { // oci://proxy.example.com/upstream/ghcr/owner/chart. OCI map[string]string `json:"oci" yaml:"oci"` + // Generic maps names to plain HTTP upstream base URLs, served at + // /generic/{name}/. The remaining request path and query string are + // appended to the upstream URL. GitHub release asset paths + // ({owner}/{repo}/releases/download/{tag}/{asset}) are cached in the + // artifact cache; everything else goes through the metadata cache. + // Example: {"github": "https://github.com", "github-api": "https://api.github.com"}. + Generic map[string]string `json:"generic" yaml:"generic"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -485,6 +493,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } + if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { + return err + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e64391..2527814 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1062,12 +1062,30 @@ func TestValidateNamedUpstreams(t *testing.T) { wantErr bool }{ { - name: "valid Helm, OCI, and APK upstreams", + name: "valid Helm, OCI, APK, and generic upstreams", modify: func(cfg *Config) { cfg.Upstream.Helm = map[string]string{"bitnami": "https://charts.bitnami.com/bitnami"} cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} cfg.Upstream.APK = map[string]string{"alpine": "https://dl-cdn.alpinelinux.org/alpine"} + cfg.Upstream.Generic = map[string]string{ + "github": "https://github.com", + "github-api": "https://api.github.com", + } + }, + }, + { + name: "generic upstream name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"github/releases": "https://github.com"} }, + wantErr: true, + }, + { + name: "generic upstream URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"github": "github.com"} + }, + wantErr: true, }, { name: "Helm upstream name contains path separator", diff --git a/internal/handler/generic.go b/internal/handler/generic.go new file mode 100644 index 0000000..5d7e6f4 --- /dev/null +++ b/internal/handler/generic.go @@ -0,0 +1,166 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "regexp" + "strings" +) + +const ( + genericEcosystem = "generic" + // genericAcceptAny is sent upstream when the client did not send an + // Accept header, so generic upstreams are not asked for JSON by default. + genericAcceptAny = "*/*" + // githubReleaseAssetMatchCount is the full match plus owner, repository, + // tag and asset filename. + githubReleaseAssetMatchCount = 5 +) + +// githubReleaseAssetPattern matches the path of a GitHub release asset +// download, {owner}/{repo}/releases/download/{tag}/{asset}. A tag pins the +// asset to one release, so these downloads are cached in the artifact cache +// and served without revalidation once fetched. +var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) + +// GenericHandler proxies plain HTTP downloads from configured upstream base +// URLs. Each configured upstream is mounted at /generic/{name}/ and the +// remaining request path (and query string) is appended to the upstream URL. +// +// Only configured upstreams are reachable, so the proxy is not an open HTTP +// proxy. The handler is the caching layer behind tools that download from +// fixed URL shapes, such as mise's aqua backend fetching GitHub release +// assets, and is pointed at by URL-rewriting settings on the client. +// +// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}) are +// version-pinned and cached in the shared artifact cache, so they keep being +// served when the upstream is unreachable. Every other path is served through +// the metadata cache: fresh within the metadata TTL, revalidated with the +// upstream's validators after that, and served stale when the upstream fails +// or refuses the request. That covers API responses such as +// api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. +type GenericHandler struct { + proxy *Proxy + repositories map[string]string +} + +// NewGenericHandler creates a generic HTTP download proxy handler. +func NewGenericHandler(proxy *Proxy, repositories map[string]string) *GenericHandler { + h := &GenericHandler{ + proxy: proxy, + repositories: make(map[string]string, len(repositories)), + } + for name, upstreamURL := range repositories { + h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") + } + return h +} + +// Routes returns the HTTP handler for generic download requests. +// Mount this at /generic on your router. +func (h *GenericHandler) Routes() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + path := strings.TrimPrefix(r.URL.Path, "/") + + if containsPathTraversal(path) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + repository, rest, ok := strings.Cut(path, "/") + upstreamURL, found := h.repositories[repository] + if !ok || rest == "" || !found { + http.NotFound(w, r) + return + } + + if asset, ok := parseGitHubReleaseAsset(rest); ok { + h.handleReleaseAsset(w, r, repository, upstreamURL, rest, asset) + return + } + + h.handleMetadata(w, r, repository, upstreamURL, rest) + }) +} + +// githubReleaseAsset is the identity of a version-pinned release download. +type githubReleaseAsset struct { + owner string + repo string + tag string + filename string +} + +// parseGitHubReleaseAsset extracts the release identity from a path shaped +// like {owner}/{repo}/releases/download/{tag}/{asset}. +func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { + matches := githubReleaseAssetPattern.FindStringSubmatch(path) + if len(matches) != githubReleaseAssetMatchCount { + return githubReleaseAsset{}, false + } + return githubReleaseAsset{ + owner: matches[1], + repo: matches[2], + tag: matches[3], + filename: matches[4], + }, true +} + +// handleReleaseAsset fetches and caches a version-pinned release asset in the +// artifact cache. The configured upstream name is part of the cache identity +// so two upstreams serving the same path never share bytes. +func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, asset githubReleaseAsset) { + name := asset.owner + "/" + asset.repo + downloadURL := upstreamURL + "/" + path + cacheFilename := repository + "/" + asset.filename + + h.proxy.Logger.Info("generic release asset download", + "repository", repository, "name", name, "version", asset.tag, "filename", asset.filename) + + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), genericEcosystem, name, asset.tag, cacheFilename, downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch release asset") + return + } + + if result.ContentType == "" { + result.ContentType = "application/octet-stream" + } + serveArtifact(w, r.Method, result) +} + +// handleMetadata serves any other path through the metadata cache. The query +// string is forwarded and is part of the cache identity, and the client's +// Accept header is replayed so content-negotiated upstreams (the GitHub API) +// cache the representation the client asked for. +func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { + target := upstreamURL + "/" + path + if r.URL.RawQuery != "" { + target += "?" + r.URL.RawQuery + } + + accept := r.Header.Get("Accept") + if accept == "" { + accept = genericAcceptAny + } + + h.proxy.ProxyCached(w, r, target, genericEcosystem, + h.metadataCacheKey(repository, upstreamURL, path, r.URL.RawQuery), accept) +} + +// metadataCacheKey derives the metadata cache key from the upstream name, its +// URL, the request path and query. Hashing the identity keeps distinct +// upstreams from sharing entries and drops cached entries when an upstream is +// repointed, mirroring APKHandler.metadataCacheKey. +func (h *GenericHandler) metadataCacheKey(repository, upstreamURL, path, query string) string { + identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go new file mode 100644 index 0000000..23b2cb6 --- /dev/null +++ b/internal/handler/generic_test.go @@ -0,0 +1,339 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/registries/fetch" +) + +const testReleaseAssetPath = "/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" + +func TestParseGitHubReleaseAsset(t *testing.T) { + tests := []struct { + path string + want githubReleaseAsset + ok bool + }{ + { + "jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", + githubReleaseAsset{owner: "jqlang", repo: "jq", tag: "jq-1.7.1", filename: "jq-linux-amd64"}, + true, + }, + { + "cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.tar.gz", + githubReleaseAsset{owner: "cli", repo: "cli", tag: "v2.63.2", filename: "gh_2.63.2_linux_amd64.tar.gz"}, + true, + }, + // Mutable: resolves to whatever is latest today. + {"jqlang/jq/releases/latest/download/jq-linux-amd64", githubReleaseAsset{}, false}, + // API lookups and tag listings are not assets. + {"repos/jqlang/jq/releases/tags/jq-1.7.1", githubReleaseAsset{}, false}, + {"jqlang/jq/releases/tag/jq-1.7.1", githubReleaseAsset{}, false}, + // Source archives are a different shape. + {"jqlang/jq/archive/refs/tags/jq-1.7.1.tar.gz", githubReleaseAsset{}, false}, + // Extra or missing segments. + {"jqlang/jq/releases/download/jq-1.7.1", githubReleaseAsset{}, false}, + {"jqlang/jq/releases/download/jq-1.7.1/dir/asset", githubReleaseAsset{}, false}, + {"", githubReleaseAsset{}, false}, + } + + for _, tt := range tests { + got, ok := parseGitHubReleaseAsset(tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("parseGitHubReleaseAsset(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) + } + } +} + +func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { + h := NewGenericHandler(testProxy(), map[string]string{"github": "https://github.com"}) + + tests := []struct { + name string + method string + target string + want int + }{ + {"unknown upstream", http.MethodGet, "/gitlab/owner/repo/releases/download/v1/asset", http.StatusNotFound}, + {"missing path", http.MethodGet, "/github", http.StatusNotFound}, + {"missing path with slash", http.MethodGet, "/github/", http.StatusNotFound}, + {"traversal", http.MethodGet, "/github/../etc/passwd", http.StatusBadRequest}, + {"encoded traversal", http.MethodGet, "/github/%2e%2e/etc/passwd", http.StatusBadRequest}, + {"post", http.MethodPost, "/github/owner/repo/releases/download/v1/asset", http.StatusMethodNotAllowed}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(tt.method, tt.target, nil)) + if w.Code != tt.want { + t.Errorf("status = %d, want %d", w.Code, tt.want) + } + }) + } +} + +func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing.T) { + asset := []byte("jq binary bytes") + var available atomic.Bool + available.Store(true) + var upstreamRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != testReleaseAssetPath { + http.NotFound(w, r) + return + } + upstreamRequests.Add(1) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(asset) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("body = %q, want %q", got, asset) + } + if got := upstreamRequests.Load(); got != 1 { + t.Fatalf("upstream requests = %d, want 1", got) + } + + // Second request must be served from cache, even with the upstream down. + available.Store(false) + w = serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("cached: status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("cached: body = %q, want %q", got, asset) + } + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream requests after cache hit = %d, want 1", got) + } + + // HEAD is answered from the same cache entry without a body. + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/github"+testReleaseAssetPath, nil)) + if w.Code != http.StatusOK { + t.Fatalf("HEAD: status = %d, want 200", w.Code) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD: body length = %d, want 0", w.Body.Len()) + } +} + +func TestGenericHandler_ReleaseAssetNotFoundIsNotCached(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", w.Code, w.Body.String()) + } +} + +func TestGenericHandler_MetadataForwardsAcceptAndQueryAndServesStaleOnThrottle(t *testing.T) { + const apiPath = "/repos/jqlang/jq/releases/tags/jq-1.7.1" + body := `{"tag_name":"jq-1.7.1"}` + var throttled atomic.Bool + var gotAccept, gotQuery string + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != apiPath { + http.NotFound(w, r) + return + } + gotAccept = r.Header.Get("Accept") + gotQuery = r.URL.RawQuery + if throttled.Load() { + w.Header().Set("Retry-After", "60") + http.Error(w, `{"message":"API rate limit exceeded"}`, http.StatusTooManyRequests) + return + } + w.Header().Set("Content-Type", "application/vnd.github+json") + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte(body)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + // A tiny TTL so the second request is past freshness and has to consult + // the upstream, and the served copy is marked stale. + proxy.MetadataTTL = time.Millisecond + + h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL}) + + req := httptest.NewRequest(http.MethodGet, "/github-api"+apiPath+"?per_page=1", nil) + req.Header.Set("Accept", "application/vnd.github+json") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != body { + t.Errorf("body = %q, want %q", got, body) + } + if gotAccept != "application/vnd.github+json" { + t.Errorf("upstream Accept = %q, want the client's header replayed", gotAccept) + } + if gotQuery != "per_page=1" { + t.Errorf("upstream query = %q, want %q", gotQuery, "per_page=1") + } + if ct := w.Header().Get("Content-Type"); ct != "application/vnd.github+json" { + t.Errorf("Content-Type = %q, want upstream's", ct) + } + + // The upstream now throttles us: the cached body must be served stale + // rather than the 429 being passed through. + throttled.Store(true) + time.Sleep(5 * time.Millisecond) + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("throttled: status = %d, want 200 stale: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != body { + t.Errorf("throttled: body = %q, want cached %q", got, body) + } + if warning := w.Header().Get("Warning"); !strings.Contains(warning, "110") { + t.Errorf("throttled: Warning = %q, want a 110 stale warning", warning) + } +} + +func TestGenericHandler_MetadataWithoutAcceptAsksForAnything(t *testing.T) { + var gotAccept string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAccept = r.Header.Get("Accept") + _, _ = w.Write([]byte("checksums")) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github/jqlang/jq/releases/latest/download/sha256sum.txt") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if gotAccept != genericAcceptAny { + t.Errorf("upstream Accept = %q, want %q", gotAccept, genericAcceptAny) + } +} + +func TestGenericHandler_DistinctUpstreamsDoNotShareCache(t *testing.T) { + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("from first")) + })) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("from second")) + })) + defer second.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(first.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"one": first.URL, "two": second.URL}) + + w := serveGenericRequest(h, "/one"+testReleaseAssetPath) + if got := w.Body.String(); got != "from first" { + t.Fatalf("one: body = %q, want %q", got, "from first") + } + w = serveGenericRequest(h, "/two"+testReleaseAssetPath) + if got := w.Body.String(); got != "from second" { + t.Fatalf("two: body = %q, want %q (must not reuse the first upstream's cache entry)", got, "from second") + } +} + +func TestGenericHandler_UpstreamAuthIsScopedToTheConfiguredHost(t *testing.T) { + asset := []byte("private asset") + var storageAuth atomic.Value + storageAuth.Store("unset") + + // The object store the release host redirects to must never see the + // token configured for the release host. + objectStore := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + storageAuth.Store(r.Header.Get("Authorization")) + _, _ = w.Write(asset) + })) + defer objectStore.Close() + + var releaseAuth string + releaseHost := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + releaseAuth = r.Header.Get("Authorization") + if releaseAuth != "Bearer github-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + http.Redirect(w, r, objectStore.URL+"/signed"+r.URL.Path, http.StatusFound) + })) + defer releaseHost.Close() + + proxy, _, _, _ := setupTestProxy(t) + authClient := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, + upstreamhttp.AuthFunc(func(url string) (string, string) { + if strings.HasPrefix(url, releaseHost.URL) { + return "Authorization", "Bearer github-token" + } + return "", "" + }))} + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(authClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github": releaseHost.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Body.String(); got != string(asset) { + t.Errorf("body = %q, want %q", got, asset) + } + if releaseAuth != "Bearer github-token" { + t.Errorf("release host Authorization = %q, want the configured token", releaseAuth) + } + if got := storageAuth.Load(); got != "" { + t.Errorf("object store Authorization = %q, want none after the cross-host redirect", got) + } +} + +func serveGenericRequest(h *GenericHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} diff --git a/internal/server/server.go b/internal/server/server.go index 6fd56bd..f672a34 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -314,6 +314,7 @@ func (s *Server) serve(listener net.Listener) error { apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) @@ -336,6 +337,7 @@ func (s *Server) serve(listener net.Listener) error { r.Mount("/apk", http.StripPrefix("/apk", apkHandler.Routes())) r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) + r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) // Health, stats, and metrics endpoints r.Get("/health", s.handleHealth)