diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79a..fd82fcd29 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "slices" "time" @@ -11,6 +12,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" ) var shortCutHelp = "Cut a tree with selected slices" @@ -121,9 +123,29 @@ func (cmd *cmdCut) Execute(args []string) error { } } + stores := make(map[string]store.Store) + for storeName, storeInfo := range release.Stores { + openStore, err := store.Open(&store.Options{ + Arch: cmd.Arch, + CacheDir: cache.DefaultDir("chisel"), + Kind: storeInfo.Kind, + Version: storeInfo.Version, + }) + if err != nil { + var unknownStoreKindError *store.UnknownStoreKindError + if errors.As(err, &unknownStoreKindError) { + logf("Store %q ignored: %v", storeName, err) + continue + } + return err + } + stores[storeName] = openStore + } + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: cmd.RootDir, }) return err diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index 569c07ab9..0450e901c 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -16,6 +16,7 @@ import ( "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/tarball" //"github.com/canonical/chisel/internal/logger" ) @@ -328,6 +329,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + store.SetLogger(log.Default()) tarball.SetLogger(log.Default()) SetLogger(log.Default()) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 7c90a41dd..c88a30bda 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -26,10 +26,12 @@ type Archive interface { } type PackageInfo struct { - Name string - Version string - Arch string - SHA256 string + Name string + Version string + Arch string + SHA256 string + Revision int + SHA384 string } type Options struct { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index fcc16d673..0982e0804 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -20,14 +20,16 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/tarball" ) -const manifestMode fs.FileMode = 0644 +const manifestMode fs.FileMode = 0o644 type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive + Stores map[string]store.Store TargetDir string } @@ -99,7 +101,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgSources, err := resolvePkgSources(options.Archives, options.Selection) + pkgSources, err := resolvePkgSources(options.Archives, options.Stores, options.Selection) if err != nil { return err } @@ -248,6 +250,12 @@ func Run(options *RunOptions) error { if reader == nil { continue } + // src := pkgSources[slice.Package] + // Store packages are distributed as plain tarballs, whose extraction + // is not yet implemented. Fail until the format support is in place. + // if src.kind != sourceArchive { + // return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName) + // } err := tarball.Extract(reader, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], @@ -360,8 +368,7 @@ func Run(options *RunOptions) error { return generateManifests(targetDir, options.Selection, report, pkgInfos) } -func generateManifests(targetDir string, selection *setup.Selection, - report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { +func generateManifests(targetDir string, selection *setup.Selection, report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { manifestSlices := manifestutil.FindPaths(selection.Slices) if len(manifestSlices) == 0 { // Nothing to do. @@ -502,10 +509,9 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent // resolvePkgSources determines the source for each package in the selection. // For archive packages it selects the highest priority archive containing the // package unless a particular archive is pinned within the slice definition -// file. For store packages it records a fetch function that returns an error -// until store support is implemented. It returns a map of pkgSource indexed by -// package names. -func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Selection) (map[string]pkgSource, error) { +// file. For store packages it records the opened store handle. It returns a map +// of pkgSourceInfo indexed by package names. +func resolvePkgSources(archives map[string]archive.Archive, stores map[string]store.Store, selection *setup.Selection) (map[string]pkgSource, error) { sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) for _, archive := range selection.Release.Archives { if archive.Priority < 0 { @@ -526,10 +532,19 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel } pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { + storeHandle := stores[pkg.Store] + if storeHandle == nil { + return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store) + } + // The store channel track is "-", + // e.g. "3.1-26.10". The version pins the release series. + // Risk is left unspecified for now; the store applies its default. + // In the future the risk will optionnaly come from the CLI. + track := pkg.DefaultTrack + "-" + storeHandle.Options().Version pkgSources[pkg.Name] = pkgSource{ - // TODO: set the arch when implementing fetching from the store. + arch: storeHandle.Options().Arch, fetch: func() (io.ReadSeekCloser, *archive.PackageInfo, error) { - return nil, nil, fmt.Errorf("cannot fetch package %q from store %q: not implemented", pkg.Name, pkg.Store) + return storeHandle.Fetch(pkg.RealName, track, "") }, } continue diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 954602de4..df88fa31a 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -18,6 +18,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/testutil" "github.com/canonical/chisel/public/manifest" ) @@ -1982,6 +1983,14 @@ var slicerTests = []slicerTest{{ summary: "Store package fetching not yet implemented", slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, arch: "amd64", + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }, { + Name: "store-pkg", + Store: "bin", + Data: testutil.PackageData["test-package"], + }}, release: map[string]string{ "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/mydir/test-package.yaml": ` @@ -2001,7 +2010,7 @@ var slicerTests = []slicerTest{{ /dir/store-file: `, }, - error: `cannot fetch package "bin-store-pkg" from store "bin": not implemented`, + error: `cannot extract package "store-pkg" from store: store packages are not yet supported`, }} func (s *S) TestRun(c *C) { @@ -2109,6 +2118,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { for name, setupArchive := range release.Archives { pkgs := make(map[string]*testutil.TestPackage) for _, pkg := range test.pkgs { + if pkg.Store != "" { + continue + } if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { pkgs[pkg.Name] = pkg } @@ -2127,9 +2139,26 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { archives[name] = archive } + stores := map[string]store.Store{} + for name, relStore := range release.Stores { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if pkg.Store == name { + pkgs[pkg.Name] = pkg + } + } + stores[name] = &testutil.TestStore{ + Packages: pkgs, + Opts: store.Options{ + Version: relStore.Version, + }, + } + } + options := slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: c.MkDir(), } if test.hackopt != nil { diff --git a/internal/store/export_test.go b/internal/store/export_test.go new file mode 100644 index 000000000..cdaf5d1d6 --- /dev/null +++ b/internal/store/export_test.go @@ -0,0 +1,19 @@ +package store + +import "net/http" + +var ( + ValidateDownloadURL = validateDownloadURL + BinStagingEnvVar = binStagingEnvVar +) + +func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) { + _httpDo := httpDo + _bulkDo := bulkDo + httpDo = do + bulkDo = do + return func() { + httpDo = _httpDo + bulkDo = _bulkDo + } +} diff --git a/internal/store/log.go b/internal/store/log.go new file mode 100644 index 000000000..d8c11a443 --- /dev/null +++ b/internal/store/log.go @@ -0,0 +1,53 @@ +package store + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 000000000..9611651ae --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,316 @@ +package store + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/deb" +) + +// Store provides access to packages from the Store API. +type Store interface { + Options() *Options + Fetch(name, track, risk string) (io.ReadSeekCloser, *archive.PackageInfo, error) +} + +type Options struct { + Arch string + CacheDir string + Kind string + Version string +} + +type storeKind string + +const storeKindBin storeKind = "bin" + +const defaultRisk = "stable" + +type binStore struct { + options Options + cache *cache.Cache + apiURL string + downloadHost string +} + +const ( + binAPIBase = "https://api.snapcraft.io/v2" + binAPIBaseStaging = "https://api.staging.snapcraft.io/v2" + binDownloadHost = "api.snapcraft.io" + binDownloadHostStaging = "api.staging.snapcraft.io" + binStagingEnvVar = "CHISEL_BIN_STAGING" +) + +var httpClient = &http.Client{ + Timeout: 30 * time.Second, +} + +var httpDo = httpClient.Do + +var bulkClient = &http.Client{ + Timeout: 5 * time.Minute, +} + +var bulkDo = bulkClient.Do + +// nameExp matches a valid package name. It is used to validate the name +// before sending it to the store API. Unlike other payload values (track, +// risk, arch) which are validated upstream, the package name comes directly +// from the release YAML without format validation. +var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) + +type UnknownStoreKindError struct { + kind string +} + +func (e *UnknownStoreKindError) Error() string { + return fmt.Sprintf("unsupported store kind %q", e.kind) +} + +func Open(options *Options) (Store, error) { + var err error + if options.Arch == "" { + options.Arch, err = deb.InferArch() + } else { + err = deb.ValidateArch(options.Arch) + } + if err != nil { + return nil, err + } + + switch storeKind(options.Kind) { + case storeKindBin: + apiURL := binAPIBase + downloadHost := binDownloadHost + if os.Getenv(binStagingEnvVar) != "" { + apiURL = binAPIBaseStaging + downloadHost = binDownloadHostStaging + } + return &binStore{ + options: *options, + cache: &cache.Cache{Dir: options.CacheDir}, + apiURL: apiURL, + downloadHost: downloadHost, + }, nil + default: + return nil, &UnknownStoreKindError{kind: options.Kind} + } +} + +// resolveRequest is the body sent to the revisions/resolve endpoint. +type resolveRequest struct { + Packages []resolvePackage `json:"packages"` +} + +type resolvePackage struct { + InstanceKey string `json:"instance-key"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Channel string `json:"channel"` + Platform binPlatform `json:"platform"` +} + +type resolveResponse struct { + PackageResults []resolveResult `json:"package-results"` +} + +type resolveResult struct { + InstanceKey string `json:"instance-key"` + Status string `json:"status"` + Error *resolveError `json:"error"` + Result *resolveEntry `json:"result"` +} + +type resolveError struct { + Message string `json:"message"` +} + +type resolveEntry struct { + Revision binRevision `json:"revision"` +} + +type binPlatform struct { + Architecture string `json:"architecture"` +} + +type binRevision struct { + Version string `json:"version"` + Revision int `json:"revision"` + Download binDownload `json:"download"` +} + +type binDownload struct { + URL string `json:"url"` + SHA384 string `json:"sha3-384"` +} + +// resolveRevision resolves a single package revision via the store API. It +// returns the matching revision or an error if the package is not found or has +// no release for the requested channel and architecture. +func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, error) { + if !nameExp.MatchString(name) { + return nil, fmt.Errorf("invalid package name %q", name) + } + u, err := url.Parse(s.apiURL) + if err != nil { + return nil, fmt.Errorf("internal error: cannot parse bin store URL: %v", err) + } + u = u.JoinPath("revisions", "resolve") + + reqBody, err := json.Marshal(resolveRequest{ + Packages: []resolvePackage{{ + InstanceKey: name, + Namespace: string(storeKindBin), + Name: name, + Channel: track + "/" + risk, + Platform: binPlatform{Architecture: s.options.Arch}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("internal error: cannot encode resolve request: %v", err) + } + + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := httpDo(req) + if err != nil { + return nil, fmt.Errorf("cannot talk to store: %v", err) + } + defer resp.Body.Close() + + // The resolve endpoint returns 200 for both success and per-package errors + // (e.g. package not found). Request-level errors (e.g. malformed channel) + // return 4xx. A simple status check is sufficient here because per-package + // errors are handled below via the result status field. + if resp.StatusCode != 200 { + return nil, fmt.Errorf("cannot fetch from store: %v", resp.Status) + } + + var res resolveResponse + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + return nil, fmt.Errorf("cannot decode store response: %v", err) + } + + if len(res.PackageResults) == 0 { + return nil, fmt.Errorf("package %q not found", name) + } + if len(res.PackageResults) > 1 { + return nil, fmt.Errorf("internal error: store returned %d results for package %q", len(res.PackageResults), name) + } + result := &res.PackageResults[0] + if result.Status != "ok" || result.Result == nil { + if result.Error != nil { + return nil, fmt.Errorf("package %q not found: %s", name, result.Error.Message) + } + return nil, fmt.Errorf("package %q not found", name) + } + rev := &result.Result.Revision + if rev.Download.SHA384 == "" { + return nil, fmt.Errorf("package %q has no download digest", name) + } + return rev, nil +} + +func (s *binStore) Options() *Options { + return &s.options +} + +func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *archive.PackageInfo, error) { + if risk == "" { + risk = defaultRisk + } + logf("Fetching package %s %s/%s...", name, track, risk) + + rev, err := s.resolveRevision(name, track, risk) + if err != nil { + return nil, nil, err + } + + digest := rev.Download.SHA384 + info := &archive.PackageInfo{ + Name: name, + Version: rev.Version, + Revision: rev.Revision, + SHA384: rev.Download.SHA384, + } + + const digestKind = cache.SHA384 + // Check cache first. + reader, err := s.cache.Open(digestKind, digest) + if err == nil { + logf("Using cached package %s", name) + return reader, info, nil + } else if err != cache.ErrMiss { + return nil, nil, err + } + + // Download the package. + err = validateDownloadURL(rev.Download.URL, s.downloadHost) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequest("GET", rev.Download.URL, nil) + if err != nil { + return nil, nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + + httpResp, err := bulkDo(req) + if err != nil { + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode != 200 { + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, httpResp.Status) + } + + writer := s.cache.Create(digestKind, digest) + defer writer.Close() + + _, err = io.Copy(writer, httpResp.Body) + if err == nil { + err = writer.Close() + } + if err != nil { + return nil, nil, fmt.Errorf("cannot fetch package %q: %v", name, err) + } + + reader, err = s.cache.Open(digestKind, writer.Digest()) + if err != nil { + return nil, nil, err + } + return reader, info, nil +} + +// validateDownloadURL checks that the download URL is HTTPS and from the +// allowed host. +func validateDownloadURL(downloadURL, allowedHost string) error { + u, err := url.Parse(downloadURL) + if err != nil { + return fmt.Errorf("cannot parse download URL: %v", err) + } + if u.Scheme != "https" { + return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) + } + host := strings.ToLower(u.Hostname()) + if host == allowedHost { + return nil + } + return fmt.Errorf("download URL has untrusted host %q", host) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 000000000..76d016110 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,527 @@ +package store_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/sha3" + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/store" +) + +type storeSuite struct { + tempDir string + cacheDir string + fakeDoFunc func(req *http.Request) (*http.Response, error) + restore func() + envRestore func() +} + +var _ = Suite(&storeSuite{}) + +func (s *storeSuite) SetUpTest(c *C) { + s.tempDir = c.MkDir() + s.cacheDir = filepath.Join(s.tempDir, "cache") + c.Assert(os.MkdirAll(s.cacheDir, 0o755), IsNil) + + s.envRestore = fakeEnv(store.BinStagingEnvVar, "") + s.restore = store.FakeDo(s.doRequest) + s.fakeDoFunc = nil +} + +func (s *storeSuite) TearDownTest(c *C) { + s.restore() + s.envRestore() +} + +func (s *storeSuite) doRequest(req *http.Request) (*http.Response, error) { + if s.fakeDoFunc != nil { + return s.fakeDoFunc(req) + } + return nil, fmt.Errorf("unexpected HTTP request: %s", req.URL.String()) +} + +func fakeEnv(name, value string) (restore func()) { + origValue, origSet := os.LookupEnv(name) + os.Setenv(name, value) + return func() { + if origSet { + os.Setenv(name, origValue) + } else { + os.Unsetenv(name) + } + } +} + +func sha384Hash(data []byte) string { + h := sha3.New384() + h.Write(data) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +type resolveBodyOptions struct { + name string + track string + risk string + arch string + version string + revision int + sha384 string + downloadURL string +} + +func makeResolveBody(opts resolveBodyOptions) []byte { + if opts.downloadURL == "" { + opts.downloadURL = "https://api.snapcraft.io/api/v1/bins/download/" + opts.name + ".bin" + } + return []byte(fmt.Sprintf(`{ + "craft-results": [], + "package-results": [ + { + "instance-key": %q, + "status": "ok", + "result": { + "channel": { + "name": %q, + "risk": %q, + "track": %q, + "platform": {"architecture": %q} + }, + "revision": { + "version": %q, + "revision": %d, + "download": { + "url": %q, + "sha3-384": %q, + "size": 1024 + } + } + } + } + ] + }`, opts.name, opts.track+"/"+opts.risk, opts.risk, opts.track, opts.arch, opts.version, opts.revision, opts.downloadURL, opts.sha384)) +} + +func makeResolveErrorBody(name, code, message string) []byte { + return []byte(fmt.Sprintf(`{ + "craft-results": [], + "package-results": [ + { + "instance-key": %q, + "status": "error", + "error": {"code": %q, "message": %q}, + "result": null + } + ] + }`, name, code, message)) +} + +func (s *storeSuite) TestValidateDownloadURL(c *C) { + tests := []struct { + summary string + url string + allowedHost string + error string + }{ + {"Valid production host", "https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, + { + "HTTP rejected", + "http://api.snapcraft.io/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, + }, + { + "Untrusted host", + "https://evil.example.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "evil.example.com"`, + }, + { + "Untrusted host case-insensitive", + "https://Evil.Example.Com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "evil.example.com"`, + }, + { + "Host suffix attack", + "https://api.snapcraft.io.evil.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "api.snapcraft.io.evil.com"`, + }, + {"Invalid URL", "://invalid-url", "api.snapcraft.io", `cannot parse download URL: .*`}, + } + for _, test := range tests { + c.Logf("Summary: %s", test.summary) + err := store.ValidateDownloadURL(test.url, test.allowedHost) + if test.error == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, ErrorMatches, test.error) + } + } +} + +func (s *storeSuite) TestOpenOptionErrors(c *C) { + tests := []struct { + summary string + options store.Options + error string + }{{ + summary: "Valid amd64", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "bin"}, + }, { + summary: "Invalid architecture", + options: store.Options{Arch: "invalid", CacheDir: s.cacheDir, Kind: "bin"}, + error: "invalid package architecture: invalid", + }, { + summary: "Unsupported store kind", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "snap"}, + error: `unsupported store kind "snap"`, + }} + for _, test := range tests { + c.Logf("Summary: %s", test.summary) + _, err := store.Open(&test.options) + if test.error == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, ErrorMatches, test.error) + } + } +} + +type fetchTest struct { + summary string + risk string + status int + statusText string + body string + error string +} + +var fetchTests = []fetchTest{{ + summary: "Defaults to stable risk when unspecified", + risk: "", + status: 200, + // Uses a real digest so the cache verification passes. + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + })), +}, { + summary: "Package not found", + risk: "stable", + status: 200, + body: string(makeResolveErrorBody("curl", "package-not-found", "Package not found")), + error: `package "curl" not found: Package not found`, +}, { + summary: "Server error", + risk: "stable", + status: 500, + statusText: "500 Internal Server Error", + body: "boom", + error: "cannot fetch from store: 500 Internal Server Error", +}, { + summary: "Malformed response body", + risk: "stable", + status: 200, + body: "not json", + error: "cannot decode store response: .*", +}, { + summary: "Missing download digest", + risk: "stable", + status: 200, + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + })), + error: `package "curl" has no download digest`, +}} + +func (s *storeSuite) TestFetch(c *C) { + tarData := []byte("fake tar.xz content") + for _, test := range fetchTests { + c.Logf("Summary: %s", test.summary) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: test.status, + Status: test.statusText, + Body: io.NopCloser(strings.NewReader(test.body)), + }, nil + } + // Download URL. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", test.risk) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + } else { + c.Assert(err, IsNil) + } + } +} + +func (s *storeSuite) TestResolveRequest(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/v2/revisions/resolve" { + // Download request. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + c.Assert(req.Method, Equals, "POST") + c.Assert(req.Header.Get("Content-Type"), Equals, "application/json") + c.Assert(req.Header.Get("Accept"), Equals, "application/json") + + var body map[string]any + err := json.NewDecoder(req.Body).Decode(&body) + c.Assert(err, IsNil) + pkgs := body["packages"].([]any) + c.Assert(pkgs, HasLen, 1) + pkg := pkgs[0].(map[string]any) + c.Assert(pkg["instance-key"], Equals, "curl") + c.Assert(pkg["namespace"], Equals, "bin") + c.Assert(pkg["name"], Equals, "curl") + c.Assert(pkg["channel"], Equals, "latest/stable") + c.Assert(pkg["platform"].(map[string]any)["architecture"], Equals, "amd64") + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) +} + +func (s *storeSuite) TestFetchCacheMiss(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + callCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + callCount++ + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + // Download URL. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.Version, Equals, "8.5.0") + c.Assert(info.SHA384, Equals, digest) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) + c.Assert(callCount, Equals, 2) + + // Verify it's in the cache. + cc := cache.Cache{Dir: s.cacheDir} + cached, err := cc.Read(cache.SHA384, digest) + c.Assert(err, IsNil) + c.Assert(cached, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchCacheHit(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + // Pre-populate the cache. + cc := cache.Cache{Dir: s.cacheDir} + err := cc.Write(cache.SHA384, digest, tarData) + c.Assert(err, IsNil) + + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + resolveCallCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + resolveCallCount++ + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return nil, fmt.Errorf("download should not be called for cache hit") + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.SHA384, Equals, digest) + c.Assert(resolveCallCount, Equals, 1) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { + // Override the download URL to an invalid one. + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + downloadURL: "http://evil.example.com/bins/curl.tar.xz", + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) +} + +func (s *storeSuite) TestStagingEnvVar(c *C) { + s.envRestore() + s.envRestore = fakeEnv(store.BinStagingEnvVar, "1") + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + downloadURL: "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin", + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + // Verify staging URL is used. + c.Assert(req.URL.Host, Equals, "api.staging.snapcraft.io") + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) +} + +func (s *storeSuite) TestFetchDownloadError(c *C) { + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return &http.Response{ + StatusCode: 500, + Status: "500 Internal Server Error", + Body: io.NopCloser(strings.NewReader("boom")), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `cannot download package "curl": 500 Internal Server Error`) +} diff --git a/internal/store/suite_test.go b/internal/store/suite_test.go new file mode 100644 index 000000000..2eac508f2 --- /dev/null +++ b/internal/store/suite_test.go @@ -0,0 +1,24 @@ +package store_test + +import ( + "testing" + + "github.com/canonical/chisel/internal/store" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + store.SetDebug(true) + store.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + store.SetDebug(false) + store.SetLogger(nil) +} diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..ccac36890 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -13,15 +13,6 @@ type TestArchive struct { Packages map[string]*TestPackage } -type TestPackage struct { - Name string - Version string - Hash string - Arch string - Data []byte - Archives []string -} - func (a *TestArchive) Options() *archive.Options { return &a.Opts } diff --git a/internal/testutil/package.go b/internal/testutil/package.go new file mode 100644 index 000000000..561eb105c --- /dev/null +++ b/internal/testutil/package.go @@ -0,0 +1,17 @@ +package testutil + +// TestPackage is a source-agnostic package fixture that can be served by either +// a TestArchive or a TestStore. +type TestPackage struct { + Name string + Version string + Hash string + Arch string + Data []byte + // Archives lists the archives the package belongs to. It is only relevant + // for archive-backed packages. + Archives []string + // Store names the store the package is served from. When set, the package + // is store-backed rather than archive-backed. + Store string +} diff --git a/internal/testutil/store.go b/internal/testutil/store.go new file mode 100644 index 000000000..44f4fedce --- /dev/null +++ b/internal/testutil/store.go @@ -0,0 +1,32 @@ +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/store" +) + +type TestStore struct { + Opts store.Options + Packages map[string]*TestPackage +} + +func (s *TestStore) Options() *store.Options { + return &s.Opts +} + +func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *archive.PackageInfo, error) { + pkg, ok := s.Packages[name] + if !ok { + return nil, nil, fmt.Errorf("cannot find package %q in store", name) + } + info := &archive.PackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA384: pkg.Hash, + } + return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil +}