diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79a..fd82fcd29 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "slices" "time" @@ -11,6 +12,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" ) var shortCutHelp = "Cut a tree with selected slices" @@ -121,9 +123,29 @@ func (cmd *cmdCut) Execute(args []string) error { } } + stores := make(map[string]store.Store) + for storeName, storeInfo := range release.Stores { + openStore, err := store.Open(&store.Options{ + Arch: cmd.Arch, + CacheDir: cache.DefaultDir("chisel"), + Kind: storeInfo.Kind, + Version: storeInfo.Version, + }) + if err != nil { + var unknownStoreKindError *store.UnknownStoreKindError + if errors.As(err, &unknownStoreKindError) { + logf("Store %q ignored: %v", storeName, err) + continue + } + return err + } + stores[storeName] = openStore + } + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: cmd.RootDir, }) return err diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index fa5485c6c..67ede89a8 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -123,9 +123,12 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* } else { releasePkg := release.Packages[pkgName] pkg = &setup.Package{ - Name: releasePkg.Name, - Archive: releasePkg.Archive, - Slices: make(map[string]*setup.Slice), + Name: releasePkg.Name, + RealName: releasePkg.RealName, + Archive: releasePkg.Archive, + Store: releasePkg.Store, + DefaultTrack: releasePkg.DefaultTrack, + Slices: make(map[string]*setup.Slice), } for _, sliceName := range pkgSlices[pkgName] { pkg.Slices[sliceName] = releasePkg.Slices[sliceName] diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index 441d1a543..0450e901c 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -16,6 +16,8 @@ import ( "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" + "github.com/canonical/chisel/internal/tarball" //"github.com/canonical/chisel/internal/logger" ) @@ -327,6 +329,8 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + store.SetLogger(log.Default()) + tarball.SetLogger(log.Default()) SetLogger(log.Default()) parser := Parser() diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 8ca840e59..30d65a481 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -17,8 +17,9 @@ import ( "strings" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/archive/testarchive" "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/archive/testarchive" + "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -856,10 +857,11 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) - err = deb.Extract(pkg, &deb.ExtractOptions{ + err = tarball.Extract(pkg, &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, - Extract: map[string][]deb.ExtractInfo{ + OpenData: deb.DataReader, + Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 67bee3f3d..bf543c2d8 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -1,384 +1,15 @@ package deb import ( - "archive/tar" - "bytes" "compress/gzip" "fmt" "io" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - "syscall" "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" - - "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/strdist" ) -type ExtractOptions struct { - Package string - TargetDir string - Extract map[string][]ExtractInfo - // Create can optionally be set to control the creation of extracted entries. - // extractInfos is set to the matching entries in Extract, and is nil in cases where - // the created entry is implicit and unlisted (for example, parent directories). - Create func(extractInfos []ExtractInfo, options *fsutil.CreateOptions) error -} - -type ExtractInfo struct { - Path string - Mode uint - Optional bool - Context any -} - -func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { - for extractPath, extractInfos := range options.Extract { - isGlob := strings.ContainsAny(extractPath, "*?") - if isGlob { - for _, extractInfo := range extractInfos { - if extractInfo.Path != extractPath || extractInfo.Mode != 0 { - return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) - } - } - } - } - - if options.Create == nil { - validOpts := *options - validOpts.Create = func(_ []ExtractInfo, o *fsutil.CreateOptions) error { - _, err := fsutil.Create(o) - return err - } - return &validOpts, nil - } - - return options, nil -} - -func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) - } - }() - - logf("Extracting files from package %q...", options.Package) - - validOpts, err := getValidOptions(options) - if err != nil { - return err - } - - _, err = os.Stat(validOpts.TargetDir) - if os.IsNotExist(err) { - return fmt.Errorf("target directory does not exist") - } else if err != nil { - return err - } - - return extractData(pkgReader, validOpts) -} - -func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := DataReader(pkgReader) - if err != nil { - return err - } - defer dataReader.Close() - - oldUmask := syscall.Umask(0) - defer func() { - syscall.Umask(oldUmask) - }() - - pendingPaths := make(map[string]bool) - for extractPath, extractInfos := range options.Extract { - for _, extractInfo := range extractInfos { - if !extractInfo.Optional { - pendingPaths[extractPath] = true - break - } - } - } - - // Store the hard links that we cannot extract when we first iterate over - // the tarball. - // - // This happens because the tarball only stores the contents once in the - // first entry and the rest of them point to the first one. Therefore, we - // cannot tell whether we need to extract the content until after we get to - // a hard link. In this case, we need a second pass. - pendingHardLinks := make(map[string][]pendingHardLink) - - // When creating a file we will iterate through its parent directories and - // create them with the permissions defined in the tarball. - // - // The assumption is that the tar entries of the parent directories appear - // before the entry for the file itself. This is the case for .deb files but - // not for all tarballs. - tarDirMode := make(map[string]fs.FileMode) - tarReader := tar.NewReader(dataReader) - for { - tarHeader, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - sourcePath, ok := sanitizeTarPath(tarHeader.Name) - if !ok { - continue - } - - sourceIsDir := sourcePath[len(sourcePath)-1] == '/' - if sourceIsDir { - tarDirMode[sourcePath] = tarHeader.FileInfo().Mode() - } - - // Find all globs and copies that require this source, and map them by - // their target paths on disk. - targetPaths := map[string][]ExtractInfo{} - for extractPath, extractInfos := range options.Extract { - if extractPath == "" { - continue - } - if strings.ContainsAny(extractPath, "*?") { - if strdist.GlobPath(extractPath, sourcePath) { - targetPaths[sourcePath] = append(targetPaths[sourcePath], extractInfos...) - delete(pendingPaths, extractPath) - } - } else if extractPath == sourcePath { - for _, extractInfo := range extractInfos { - targetPaths[extractInfo.Path] = append(targetPaths[extractInfo.Path], extractInfo) - } - delete(pendingPaths, extractPath) - } - } - if len(targetPaths) == 0 { - // Nothing to do. - continue - } - - var contentCache []byte - var contentIsCached = len(targetPaths) > 1 && !sourceIsDir - if contentIsCached { - // Read and cache the content so it may be reused. - // As an alternative, to avoid having an entire file in - // memory at once this logic might open the first file - // written and copy it every time. For now, the choice - // is speed over memory efficiency. - data, err := io.ReadAll(tarReader) - if err != nil { - return err - } - contentCache = data - } - - var pathReader io.Reader = tarReader - for targetPath, extractInfos := range targetPaths { - if contentIsCached { - pathReader = bytes.NewReader(contentCache) - } - mode := extractInfos[0].Mode - for _, extractInfo := range extractInfos { - if extractInfo.Mode != mode { - if mode < extractInfo.Mode { - mode, extractInfo.Mode = extractInfo.Mode, mode - } - return fmt.Errorf("path %s requested twice with diverging mode: 0%03o != 0%03o", targetPath, mode, extractInfo.Mode) - } - } - if mode != 0 { - tarHeader.Mode = int64(mode) - } - // Create the parent directories using the permissions from the tarball. - parents := parentDirs(targetPath) - for _, path := range parents { - if path == "/" { - continue - } - mode, ok := tarDirMode[path] - if !ok { - continue - } - delete(tarDirMode, path) - - createOptions := &fsutil.CreateOptions{ - Root: options.TargetDir, - Path: path, - Mode: mode, - MakeParents: true, - } - err := options.Create(nil, createOptions) - if err != nil { - return err - } - } - link := tarHeader.Linkname - if tarHeader.Typeflag == tar.TypeLink { - // A hard link requires the real path of the target file. - link = filepath.Join(options.TargetDir, link) - } - - // Create the entry itself. - createOptions := &fsutil.CreateOptions{ - Root: options.TargetDir, - Path: targetPath, - Mode: tarHeader.FileInfo().Mode(), - Data: pathReader, - Link: link, - MakeParents: true, - OverrideMode: true, - } - err := options.Create(extractInfos, createOptions) - if err != nil && os.IsNotExist(err) && tarHeader.Typeflag == tar.TypeLink { - // The hard link could not be created because the content - // was not extracted previously. Add this hard link entry - // to the pending list to extract later. - relLinkPath, ok := sanitizeTarPath(tarHeader.Linkname) - if !ok { - return fmt.Errorf("invalid link target %s", tarHeader.Linkname) - } - info := pendingHardLink{ - path: targetPath, - extractInfos: extractInfos, - } - pendingHardLinks[relLinkPath] = append(pendingHardLinks[relLinkPath], info) - } else if err != nil { - return err - } - } - } - - if len(pendingHardLinks) > 0 { - // Go over the tarball again to textract the pending hard links. - extractHardLinkOptions := &extractHardLinkOptions{ - ExtractOptions: options, - pendingLinks: pendingHardLinks, - } - _, err := pkgReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = extractHardLinks(pkgReader, extractHardLinkOptions) - if err != nil { - return err - } - } - - if len(pendingPaths) > 0 { - pendingList := make([]string, 0, len(pendingPaths)) - for pendingPath := range pendingPaths { - pendingList = append(pendingList, pendingPath) - } - if len(pendingList) == 1 { - return fmt.Errorf("no content at %s", pendingList[0]) - } else { - sort.Strings(pendingList) - return fmt.Errorf("no content at:\n- %s", strings.Join(pendingList, "\n- ")) - } - } - - return nil -} - -type pendingHardLink struct { - path string - extractInfos []ExtractInfo -} - -type extractHardLinkOptions struct { - *ExtractOptions - pendingLinks map[string][]pendingHardLink -} - -// extractHardLinks iterates through the tarball a second time to extract the -// hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := DataReader(pkgReader) - if err != nil { - return err - } - defer dataReader.Close() - - tarReader := tar.NewReader(dataReader) - for { - tarHeader, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - sourcePath, ok := sanitizeTarPath(tarHeader.Name) - if !ok { - continue - } - - links := opts.pendingLinks[sourcePath] - if len(links) == 0 { - continue - } - - // For a target path, the first hard link will be created as a file with - // the content of the target path. If there are more pending hard links, - // the remaining ones will be created as hard links with the newly - // created file as their target. - absLink := filepath.Join(opts.TargetDir, links[0].path) - // Extract the content to the first hard link path. - createOptions := &fsutil.CreateOptions{ - Root: opts.TargetDir, - Path: links[0].path, - Mode: tarHeader.FileInfo().Mode(), - Data: tarReader, - } - err = opts.Create(links[0].extractInfos, createOptions) - if err != nil { - return err - } - - // Create the remaining hard links. - for _, link := range links[1:] { - createOptions := &fsutil.CreateOptions{ - Root: opts.TargetDir, - Path: link.path, - Mode: tarHeader.FileInfo().Mode(), - // Link to the first file extracted for the hard links. - Link: absLink, - } - err := opts.Create(link.extractInfos, createOptions) - if err != nil { - return err - } - } - delete(opts.pendingLinks, sourcePath) - } - - // If there are pending links, that means the link targets do not come from - // this package. - if len(opts.pendingLinks) > 0 { - var targets []string - for target := range opts.pendingLinks { - targets = append(targets, target) - } - sort.Strings(targets) - link := opts.pendingLinks[targets[0]][0] - return fmt.Errorf("cannot create hard link %s: no content at %s", link.path, targets[0]) - } - - return nil -} - // DataReader takes a Reader for the ar file belonging to a Debian package and // returns a Reader to the inner tarball. func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { @@ -416,25 +47,3 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { return dataReader, nil } - -func parentDirs(path string) []string { - path = filepath.Clean(path) - parents := make([]string, strings.Count(path, "/")) - count := 0 - for i, c := range path { - if c == '/' { - parents[count] = path[:i+1] - count++ - } - } - return parents -} - -// sanitizeTarPath removes the leading "./" from the source path in the tarball, -// and verifies that the path is not empty. -func sanitizeTarPath(path string) (string, bool) { - if len(path) < 3 || path[0] != '.' || path[1] != '/' { - return "", false - } - return path[1:], true -} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index aaa8d1e55..67c478f62 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,6 +16,14 @@ import ( "github.com/canonical/chisel/internal/strdist" ) +// Store is the location from which packages are obtained via a store API. +type Store struct { + Name string + Kind string + Version string + DefaultPrefix string +} + // Release is a collection of package slices targeting a particular // distribution version. type Release struct { @@ -23,6 +31,7 @@ type Release struct { Path string Packages map[string]*Package Archives map[string]*Archive + Stores map[string]*Store Maintenance *Maintenance } @@ -51,10 +60,16 @@ type Archive struct { // Package holds a collection of slices that represent parts of themselves. type Package struct { - Name string - Path string - Archive string - Slices map[string]*Slice + // Name is the unique package identifier (e.g. "bin-curl" for store packages, + // "curl" for archive packages). + Name string + // RealName is the bare name visible in the Debian archive (e.g. "curl"). + RealName string + Path string + Archive string + Store string + DefaultTrack string + Slices map[string]*Slice } // Slice holds the details about a package slice. @@ -441,20 +456,20 @@ func readSlices(release *Release, baseDir, dirName string) error { pkgName := match[1] pkgPath := filepath.Join(dirName, entry.Name()) - if pkg, ok := release.Packages[pkgName]; ok { - return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, pkg.Path, stripBase(baseDir, pkgPath)) - } data, err := os.ReadFile(pkgPath) if err != nil { // Errors from package os generally include the path. return fmt.Errorf("cannot read slice definition file: %v", err) } - pkg, err := parsePackage(release.Format, pkgName, stripBase(baseDir, pkgPath), data) + pkg, err := parsePackage(release, pkgName, stripBase(baseDir, pkgPath), data) if err != nil { return err } + if existing, ok := release.Packages[pkg.Name]; ok { + return fmt.Errorf("package %q slices defined more than once: %s and %s", pkg.Name, existing.Path, stripBase(baseDir, pkgPath)) + } release.Packages[pkg.Name] = pkg } return nil @@ -502,6 +517,21 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error new, newPath, newInfo.Generate) } } + // An invalid store kind should only throw an error if a slice references it. + // Hence, the check is here. + pkg := release.Packages[new.Package] + if pkg.Store == "" { + continue + } + store, ok := selection.Release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("internal error: slice %s refers to missing store %q", new, pkg.Store) + } + switch store.Kind { + case "bin": + default: + return nil, fmt.Errorf("slice %s refers to store %q with unknown kind %q", new, pkg.Store, store.Kind) + } } return selection, nil diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d669943f6..0ba3fd814 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -92,9 +92,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -139,8 +140,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -204,8 +206,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -270,9 +273,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -327,9 +331,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -582,8 +587,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -800,8 +806,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -843,8 +850,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -887,8 +895,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -960,9 +969,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1166,8 +1176,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1241,9 +1252,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1354,8 +1366,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "jq": { - Name: "jq", - Path: "slices/mydir/jq.yaml", + RealName: "jq", + Name: "jq", + Path: "slices/mydir/jq.yaml", Slices: map[string]*setup.Slice{ "bins": { Package: "jq", @@ -1465,8 +1478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1538,8 +1552,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1560,8 +1575,9 @@ var setupTests = []setupTest{{ }, }, "myotherpkg": { - Name: "myotherpkg", - Path: "slices/mydir/myotherpkg.yaml", + RealName: "myotherpkg", + Name: "myotherpkg", + Path: "slices/mydir/myotherpkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "myotherpkg", @@ -1721,8 +1737,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1774,8 +1791,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1869,8 +1887,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1882,8 +1901,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg2", @@ -2040,9 +2060,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2166,9 +2187,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2233,9 +2255,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2328,9 +2351,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2454,8 +2478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg1": { - Name: "mypkg1", - Path: "slices/mydir/mypkg1.yaml", + RealName: "mypkg1", + Name: "mypkg1", + Path: "slices/mydir/mypkg1.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg1", @@ -2475,8 +2500,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg2", @@ -2489,8 +2515,9 @@ var setupTests = []setupTest{{ }, }, "mypkg3": { - Name: "mypkg3", - Path: "slices/mydir/mypkg3.yaml", + RealName: "mypkg3", + Name: "mypkg3", + Path: "slices/mydir/mypkg3.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg3", @@ -2889,9 +2916,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2939,9 +2967,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3128,9 +3157,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3250,9 +3280,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3372,9 +3403,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3494,9 +3526,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3614,9 +3647,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3657,8 +3691,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -3896,6 +3931,291 @@ var setupTests = []setupTest{{ `, }, relerror: `package "mypkg" slices defined more than once: slices/dir1/mypkg.yaml and slices/dir2/mypkg.yaml`, +}, { + summary: "Store package is parsed correctly", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: "bin", + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "bin-mypkg": { + RealName: "mypkg", + Name: "bin-mypkg", + Path: "slices/bin/mypkg.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{}, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, +}, { + summary: "Store used with older format (v1/v2) is not allowed", + input: map[string]string{ + "chisel.yaml": strings.ReplaceAll(testutil.DefaultChiselYaml, "format: v1", "format: v2"), + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'store' and 'default-track' are unsupported before format v3`, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + archive: ubuntu + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + archive: ubuntu + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, +}, { + summary: "Store package missing default-track (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + `, + }, + relerror: `cannot parse package "mypkg": 'store' requires 'default-track'`, +}, { + summary: "default-track without store (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' requires 'store'`, +}, { + summary: "default-track must not contain / (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0/stable" + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' must not contain /`, +}, { + summary: "Package store references undefined store (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: non-existing + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": store "non-existing" not defined in release`, +}, { + summary: "Store missing version", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing version field`, +}, { + summary: "Store missing kind", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + version: 26.10 + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing kind field`, +}, { + summary: "Store missing default-prefix", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + `, + }, + relerror: `chisel.yaml: store "bin" missing default-prefix field`, +}, { + summary: "Same-named package in archive and store", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/curl.yaml": ` + package: curl + slices: + libs: + contents: + /usr/lib/libcurl.so: + bins: + contents: + /usr/bin/curl: + `, + "slices/bin/curl.yaml": ` + package: curl + store: bin + default-track: "3.0" + slices: + bins: + contents: + /usr/bin/curl-bin: + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: "bin", + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "curl": { + RealName: "curl", + Name: "curl", + Path: "slices/curl.yaml", + Slices: map[string]*setup.Slice{ + "libs": { + Package: "curl", + Name: "libs", + Contents: map[string]setup.PathInfo{ + "/usr/lib/libcurl.so": {Kind: setup.CopyPath}, + }, + }, + "bins": { + Package: "curl", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl": {Kind: setup.CopyPath}, + }, + }, + }, + }, + "bin-curl": { + RealName: "curl", + Name: "bin-curl", + Path: "slices/bin/curl.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{ + "bins": { + Package: "bin-curl", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl-bin": {Kind: setup.CopyPath}, + }, + }, + }, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, }} func (s *S) TestParseRelease(c *C) { @@ -3984,6 +4304,48 @@ func (s *S) TestParseRelease(c *C) { runParseReleaseTests(c, v3FormatTests) } +// TestSelectStoreUnknownKind is a dedicated test because the unknown store kind +// is only reported at selection time and only makes sense for the v3 format. +func (s *S) TestSelectStoreUnknownKind(c *C) { + runParseReleaseTests(c, []setupTest{{ + summary: "Store unknown kind", + selslices: []setup.SliceKey{{Package: "bin-mypkg", Slice: "myslice"}}, + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + stores: + bin: + kind: unknown + version: 26.10 + default-prefix: "bin-" + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /dir/file: {} + `, + }, + selerror: `slice bin-mypkg_myslice refers to store "bin" with unknown kind "unknown"`, + }}) +} + func runParseReleaseTests(c *C, tests []setupTest) { for _, test := range tests { c.Logf("Summary: %s", test.summary) @@ -4232,6 +4594,40 @@ func (s *S) TestPackageYAMLFormat(c *C) { mypkg_three: {arch: i386} `, }, + }, { + summary: "Store package fields", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /usr/bin/mypkg: {} + `, + }, }} for _, test := range tests { diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index cefe86cb4..763ee781d 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -35,6 +35,7 @@ type yamlRelease struct { // fields that break said compatibility (e.g. "pro" archives) and merged // together with "archives". V2Archives map[string]yamlArchive `yaml:"v2-archives"` + Stores map[string]yamlStore `yaml:"stores"` } const ( @@ -52,9 +53,17 @@ type yamlArchive struct { PubKeys []string `yaml:"public-keys"` } +type yamlStore struct { + Kind string `yaml:"kind"` + Version string `yaml:"version"` + DefaultPrefix string `yaml:"default-prefix"` +} + type yamlPackage struct { - Name string `yaml:"package"` - Archive string `yaml:"archive,omitempty"` + Name string `yaml:"package"` + Archive string `yaml:"archive,omitempty"` + Store string `yaml:"store,omitempty"` + DefaultTrack string `yaml:"default-track,omitempty"` // For backwards-compatibility reasons with v1 and v2, essential needs // custom logic to be parsed. See [yamlEssentialListMap]. Essential yamlEssentialListMap `yaml:"essential,omitempty"` @@ -433,14 +442,39 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.Archives[archiveName] = details } + // Parse stores. + if len(yamlVar.Stores) > 0 && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("%s: 'stores' is unsupported before format v3", fileName) + } + if len(yamlVar.Stores) > 0 { + release.Stores = make(map[string]*Store, len(yamlVar.Stores)) + } + for storeName, details := range yamlVar.Stores { + if details.Kind == "" { + return nil, fmt.Errorf("%s: store %q missing kind field", fileName, storeName) + } + if details.Version == "" { + return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) + } + if details.DefaultPrefix == "" { + return nil, fmt.Errorf("%s: store %q missing default-prefix field", fileName, storeName) + } + release.Stores[storeName] = &Store{ + Name: storeName, + Kind: details.Kind, + Version: details.Version, + DefaultPrefix: details.DefaultPrefix, + } + } + return release, err } -func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error) { +func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Package, error) { pkg := Package{ - Name: pkgName, - Path: pkgPath, - Slices: make(map[string]*Slice), + RealName: pkgName, + Path: pkgPath, + Slices: make(map[string]*Slice), } yamlPkg := yamlPackage{} @@ -450,32 +484,64 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error if err != nil { return nil, fmt.Errorf("cannot parse package %q slice definitions: %v", pkgName, err) } - if yamlPkg.Name != pkg.Name { + if yamlPkg.Name != pkg.RealName { return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } - if format == "v1" || format == "v2" { + if (yamlPkg.Store != "" || yamlPkg.DefaultTrack != "") && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("cannot parse package %q: 'store' and 'default-track' are unsupported before format v3", pkgName) + } + if yamlPkg.Store != "" && yamlPkg.Archive != "" { + return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) + } + if yamlPkg.Store != "" { + if yamlPkg.DefaultTrack == "" { + return nil, fmt.Errorf("cannot parse package %q: 'store' requires 'default-track'", pkgName) + } + if strings.Contains(yamlPkg.DefaultTrack, "/") { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' must not contain /", pkgName) + } + pkg.Store = yamlPkg.Store + pkg.DefaultTrack = yamlPkg.DefaultTrack + } else { + if yamlPkg.DefaultTrack != "" { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' requires 'store'", pkgName) + } + } + + // Derive the package unique name from its store prefix if applicable. + var prefix string + if pkg.Store != "" { + store, ok := release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("cannot parse package %q: store %q not defined in release", pkgName, pkg.Store) + } + prefix = store.DefaultPrefix + } + pkg.Name = prefix + pkgName + + if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkg.Name, sliceName}) } } } else { if yamlPkg.V3Essential != nil { - return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkgName) + return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkg.Name) } if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkg.Name, sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkg.Name, sliceName}) } } } @@ -491,10 +557,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkgName, sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkg.Name, sliceName}, yamlSlice.Hint) } slice := &Slice{ - Package: pkgName, + Package: pkg.Name, Name: sliceName, Hint: yamlSlice.Hint, Scripts: SliceScripts{ @@ -531,17 +597,17 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error zeroPathGenerate.Generate = yamlPath.Generate if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Prefer != "" || yamlPath.Until != UntilNone { return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } if _, err := validateGeneratePath(contPath); err != nil { - return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkg.Name, sliceName, err) } kinds = append(kinds, GeneratePath) } else if strings.ContainsAny(contPath, "*?") { if yamlPath != nil { if !yamlPath.SameContent(&zeroPath) || yamlPath.Prefer != "" { return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } } kinds = append(kinds, GlobPath) @@ -554,7 +620,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error if yamlPath.Dir { if !strings.HasSuffix(contPath, "/") { return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } kinds = append(kinds, DirPath) } @@ -577,17 +643,17 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error switch until { case UntilNone, UntilMutate: default: - return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) + return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkg.Name, sliceName, contPath, until) } arch = yamlPath.Arch.List for _, s := range arch { if deb.ValidateArch(s) != nil { - return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) + return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkg.Name, sliceName, contPath, s) } } } - if prefer == pkgName { - return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkgName, sliceName, contPath) + if prefer == pkg.Name { + return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkg.Name, sliceName, contPath) } if len(kinds) == 0 { kinds = append(kinds, CopyPath) @@ -597,10 +663,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error for i, s := range kinds { list[i] = string(s) } - return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkgName, sliceName, contPath, strings.Join(list, ", ")) + return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkg.Name, sliceName, contPath, strings.Join(list, ", ")) } if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { - return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) + return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkg.Name, sliceName, contPath) } slice.Contents[contPath] = PathInfo{ Kind: kinds[0], @@ -690,9 +756,11 @@ func sliceToYAML(s *Slice) (*yamlSlice, error) { // packageToYAML converts a Package object to a yamlPackage object. func packageToYAML(p *Package) (*yamlPackage, error) { pkg := &yamlPackage{ - Name: p.Name, - Archive: p.Archive, - Slices: make(map[string]yamlSlice, len(p.Slices)), + Name: p.RealName, + Archive: p.Archive, + Store: p.Store, + DefaultTrack: p.DefaultTrack, + Slices: make(map[string]yamlSlice, len(p.Slices)), } for name, slice := range p.Slices { yamlSlice, err := sliceToYAML(slice) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb2..b7432e083 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -21,6 +21,8 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/store" + "github.com/canonical/chisel/internal/tarball" ) const manifestMode fs.FileMode = 0644 @@ -28,6 +30,7 @@ const manifestMode fs.FileMode = 0644 type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive + Stores map[string]store.Store TargetDir string } @@ -37,6 +40,23 @@ type pathData struct { hardLink bool } +type sourceKind int + +const ( + sourceArchive sourceKind = iota + sourceStore +) + +// pkgSourceInfo records the resolved source for a package in the selection. +// It abstracts over archive and store packages. +type pkgSourceInfo struct { + arch string + kind sourceKind + archive archive.Archive + store store.Store + pkg *setup.Package +} + type contentChecker struct { knownPaths map[string]pathData } @@ -90,7 +110,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + pkgSources, err := resolvePkgSources(options.Archives, options.Stores, options.Selection) if err != nil { return err } @@ -101,19 +121,19 @@ func Run(options *RunOptions) error { } // Build information to process the selection. - extract := make(map[string]map[string][]deb.ExtractInfo) + extract := make(map[string]map[string][]tarball.ExtractInfo) for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - extractPackage = make(map[string][]deb.ExtractInfo) + extractPackage = make(map[string][]tarball.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgArchive[slice.Package].Options().Arch + src := pkgSources[slice.Package] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue } - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if preferredPkg, ok := prefers[targetPath]; ok && preferredPkg.Name != slice.Package { @@ -125,7 +145,7 @@ func Run(options *RunOptions) error { if sourcePath == "" { sourcePath = targetPath } - extractPackage[sourcePath] = append(extractPackage[sourcePath], deb.ExtractInfo{ + extractPackage[sourcePath] = append(extractPackage[sourcePath], tarball.ExtractInfo{ Path: targetPath, Context: slice, }) @@ -137,7 +157,7 @@ func Run(options *RunOptions) error { if targetDir == "" || targetDir == "/" { continue } - extractPackage[targetDir] = append(extractPackage[targetDir], deb.ExtractInfo{ + extractPackage[targetDir] = append(extractPackage[targetDir], tarball.ExtractInfo{ Path: targetDir, Optional: true, }) @@ -152,13 +172,28 @@ func Run(options *RunOptions) error { if packages[slice.Package] != nil { continue } - reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) - if err != nil { - return err + src := pkgSources[slice.Package] + var reader io.ReadSeekCloser + if src.kind == sourceStore { + // The store channel track is "-", + // e.g. "3.1-26.10". The version pins the release series. + // Risk is left unspecified for now; the store applies its default. + // In the future the risk will optionnaly come from the CLI. + track := src.pkg.DefaultTrack + "-" + src.store.Options().Version + reader, _, err = src.store.Fetch(src.pkg.RealName, track, "") + if err != nil { + return err + } + } else { + var info *archive.PackageInfo + reader, info, err = src.archive.Fetch(src.pkg.RealName) + if err != nil { + return err + } + pkgInfos = append(pkgInfos, info) } defer reader.Close() packages[slice.Package] = reader - pkgInfos = append(pkgInfos, info) } // When creating content, record if a path is known and whether they are @@ -177,7 +212,7 @@ func Run(options *RunOptions) error { var implicitConflicts []string // Creates the filesystem entry and adds it to the report. It also updates // knownPaths with the files created. - create := func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + create := func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { entry, err := fsutil.Create(o) if err != nil { return err @@ -238,10 +273,23 @@ func Run(options *RunOptions) error { if reader == nil { continue } - err := deb.Extract(reader, &deb.ExtractOptions{ + src := pkgSources[slice.Package] + var openData func(io.ReadSeeker) (io.ReadCloser, error) + if src.kind == sourceStore { + switch store.StoreKind(src.store.Options().Kind) { + case store.StoreKindBin: + openData = tarball.XZDataReader + default: + return fmt.Errorf("cannot extract from store of kind %q: unsupported artefact format", src.store.Options().Kind) + } + } else { + openData = deb.DataReader + } + err := tarball.Extract(reader, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, + OpenData: openData, Create: create, }) reader.Close() @@ -269,9 +317,9 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArchive[slice.Package].Options().Arch + src := pkgSources[slice.Package] for relPath, pathInfo := range slice.Contents { - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath || @@ -489,10 +537,12 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent }) } -// selectPkgArchives selects the highest priority archive containing the package -// unless a particular archive is pinned within the slice definition file. It -// returns a map of archives indexed by package names. -func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { +// resolvePkgSources determines the source for each package in the selection. +// For archive packages it selects the highest priority archive containing the +// package unless a particular archive is pinned within the slice definition +// file. For store packages it records the opened store handle. It returns a map +// of pkgSourceInfo indexed by package names. +func resolvePkgSources(archives map[string]archive.Archive, stores map[string]store.Store, selection *setup.Selection) (map[string]*pkgSourceInfo, error) { sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) for _, archive := range selection.Release.Archives { if archive.Priority < 0 { @@ -506,12 +556,25 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return b.Priority - a.Priority }) - pkgArchive := make(map[string]archive.Archive) + pkgSources := make(map[string]*pkgSourceInfo) for _, s := range selection.Slices { - if _, ok := pkgArchive[s.Package]; ok { + if _, ok := pkgSources[s.Package]; ok { continue } pkg := selection.Release.Packages[s.Package] + if pkg.Store != "" { + storeHandle := stores[pkg.Store] + if storeHandle == nil { + return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store) + } + pkgSources[pkg.Name] = &pkgSourceInfo{ + arch: storeHandle.Options().Arch, + kind: sourceStore, + store: storeHandle, + pkg: pkg, + } + continue + } var candidates []*setup.Archive if pkg.Archive == "" { @@ -525,15 +588,21 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel var chosen archive.Archive for _, archiveInfo := range candidates { archive := archives[archiveInfo.Name] - if archive != nil && archive.Exists(pkg.Name) { + if archive != nil && archive.Exists(pkg.RealName) { chosen = archive break } } if chosen == nil { - return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) + } + pkgSources[pkg.Name] = &pkgSourceInfo{ + arch: chosen.Options().Arch, + kind: sourceArchive, + archive: chosen, + pkg: pkg, } - pkgArchive[pkg.Name] = chosen } - return pkgArchive, nil + + return pkgSources, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8d3cf775c..f13eb08ab 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -18,6 +18,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/testutil" "github.com/canonical/chisel/public/manifest" ) @@ -1978,6 +1979,64 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, +}, { + summary: "Store package extracts, manifest reports missing package", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }, { + Name: "store-pkg", + Store: "bin", + Data: testutil.MustMakeBin([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Dir(0o755, "./dir/"), + testutil.Reg(0o644, "./dir/store-file", "store content"), + }), + }}, + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: 3.1 + slices: + myslice: + contents: + /dir/store-file: + `, + }, + error: `internal error: invalid manifest: slice bin-store-pkg_myslice refers to missing package "bin-store-pkg"`, +}, { + summary: "Store package with invalid (non-xz) data fails extraction", + slices: []setup.SliceKey{{"bin-store-pkg", "myslice"}}, + arch: "amd64", + pkgs: []*testutil.TestPackage{{ + Name: "store-pkg", + Store: "bin", + Data: []byte("this is not an xz archive"), + }}, + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: 3.1 + slices: + myslice: + contents: + /dir/store-file: + `, + }, + error: `cannot extract from package "bin-store-pkg": xz: invalid header magic bytes`, }} func (s *S) TestRun(c *C) { @@ -1989,7 +2048,7 @@ func (s *S) TestRun(c *C) { for _, t := range slicerTests { m := make(map[string]string) for k, v := range t.release { - if !strings.Contains(v, "v2-archives:") { + if !strings.Contains(v, "v2-archives:") && strings.Contains(v, "format: v1") { v = strings.ReplaceAll(v, "archives:", "v2-archives:") } m[k] = v @@ -2085,6 +2144,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { for name, setupArchive := range release.Archives { pkgs := make(map[string]*testutil.TestPackage) for _, pkg := range test.pkgs { + if pkg.Store != "" { + continue + } if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { pkgs[pkg.Name] = pkg } @@ -2103,9 +2165,27 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { archives[name] = archive } + stores := map[string]store.Store{} + for name, relStore := range release.Stores { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if pkg.Store == name { + pkgs[pkg.Name] = pkg + } + } + stores[name] = &testutil.TestStore{ + Packages: pkgs, + Opts: store.Options{ + Kind: relStore.Kind, + Version: relStore.Version, + }, + } + } + options := slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: c.MkDir(), } if test.hackopt != nil { diff --git a/internal/store/export_test.go b/internal/store/export_test.go new file mode 100644 index 000000000..cdaf5d1d6 --- /dev/null +++ b/internal/store/export_test.go @@ -0,0 +1,19 @@ +package store + +import "net/http" + +var ( + ValidateDownloadURL = validateDownloadURL + BinStagingEnvVar = binStagingEnvVar +) + +func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) { + _httpDo := httpDo + _bulkDo := bulkDo + httpDo = do + bulkDo = do + return func() { + httpDo = _httpDo + bulkDo = _bulkDo + } +} diff --git a/internal/store/log.go b/internal/store/log.go new file mode 100644 index 000000000..d8c11a443 --- /dev/null +++ b/internal/store/log.go @@ -0,0 +1,53 @@ +package store + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 000000000..c0bf61ec7 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,323 @@ +package store + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/deb" +) + +// Store provides access to packages from the Store API. +type Store interface { + Options() *Options + Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) +} + +// StorePackageInfo holds metadata about a package. +type StorePackageInfo struct { + Name string + Version string + Revision int + SHA384 string +} + +type Options struct { + Arch string + CacheDir string + Kind string + Version string +} + +type StoreKind string + +const StoreKindBin StoreKind = "bin" + +const defaultRisk = "stable" + +type binStore struct { + options Options + cache *cache.Cache + apiURL string + downloadHost string +} + +const ( + binAPIBase = "https://api.snapcraft.io/v2" + binAPIBaseStaging = "https://api.staging.snapcraft.io/v2" + binDownloadHost = "api.snapcraft.io" + binDownloadHostStaging = "api.staging.snapcraft.io" + binStagingEnvVar = "CHISEL_BIN_STAGING" +) + +var httpClient = &http.Client{ + Timeout: 30 * time.Second, +} + +var httpDo = httpClient.Do + +var bulkClient = &http.Client{ + Timeout: 5 * time.Minute, +} + +var bulkDo = bulkClient.Do + +// nameExp matches a valid package name. It is used to validate the name +// before sending it to the store API. Unlike other payload values (track, +// risk, arch) which are validated upstream, the package name comes directly +// from the release YAML without format validation. +var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) + +type UnknownStoreKindError struct { + kind string +} + +func (e *UnknownStoreKindError) Error() string { + return fmt.Sprintf("unsupported store kind %q", e.kind) +} + +func Open(options *Options) (Store, error) { + var err error + if options.Arch == "" { + options.Arch, err = deb.InferArch() + } else { + err = deb.ValidateArch(options.Arch) + } + if err != nil { + return nil, err + } + + switch StoreKind(options.Kind) { + case StoreKindBin: + apiURL := binAPIBase + downloadHost := binDownloadHost + if os.Getenv(binStagingEnvVar) != "" { + apiURL = binAPIBaseStaging + downloadHost = binDownloadHostStaging + } + return &binStore{ + options: *options, + cache: &cache.Cache{Dir: options.CacheDir}, + apiURL: apiURL, + downloadHost: downloadHost, + }, nil + default: + return nil, &UnknownStoreKindError{kind: options.Kind} + } +} + +// resolveRequest is the body sent to the revisions/resolve endpoint. +type resolveRequest struct { + Packages []resolvePackage `json:"packages"` +} + +type resolvePackage struct { + InstanceKey string `json:"instance-key"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Channel string `json:"channel"` + Platform binPlatform `json:"platform"` +} + +type resolveResponse struct { + PackageResults []resolveResult `json:"package-results"` +} + +type resolveResult struct { + InstanceKey string `json:"instance-key"` + Status string `json:"status"` + Error *resolveError `json:"error"` + Result *resolveEntry `json:"result"` +} + +type resolveError struct { + Message string `json:"message"` +} + +type resolveEntry struct { + Revision binRevision `json:"revision"` +} + +type binPlatform struct { + Architecture string `json:"architecture"` +} + +type binRevision struct { + Version string `json:"version"` + Revision int `json:"revision"` + Download binDownload `json:"download"` +} + +type binDownload struct { + URL string `json:"url"` + SHA384 string `json:"sha3-384"` +} + +// resolveRevision resolves a single package revision via the store API. It +// returns the matching revision or an error if the package is not found or has +// no release for the requested channel and architecture. +func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, error) { + if !nameExp.MatchString(name) { + return nil, fmt.Errorf("invalid package name %q", name) + } + u, err := url.Parse(s.apiURL) + if err != nil { + return nil, fmt.Errorf("internal error: cannot parse bin store URL: %v", err) + } + u = u.JoinPath("revisions", "resolve") + + reqBody, err := json.Marshal(resolveRequest{ + Packages: []resolvePackage{{ + InstanceKey: name, + Namespace: string(StoreKindBin), + Name: name, + Channel: track + "/" + risk, + Platform: binPlatform{Architecture: s.options.Arch}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("internal error: cannot encode resolve request: %v", err) + } + + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := httpDo(req) + if err != nil { + return nil, fmt.Errorf("cannot talk to store: %v", err) + } + defer resp.Body.Close() + + // The resolve endpoint returns 200 for both success and per-package errors + // (e.g. package not found). Request-level errors (e.g. malformed channel) + // return 4xx. A simple status check is sufficient here because per-package + // errors are handled below via the result status field. + if resp.StatusCode != 200 { + return nil, fmt.Errorf("cannot fetch from store: %v", resp.Status) + } + + var res resolveResponse + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + return nil, fmt.Errorf("cannot decode store response: %v", err) + } + + if len(res.PackageResults) == 0 { + return nil, fmt.Errorf("package %q not found", name) + } + if len(res.PackageResults) > 1 { + return nil, fmt.Errorf("internal error: store returned %d results for package %q", len(res.PackageResults), name) + } + result := &res.PackageResults[0] + if result.Status != "ok" || result.Result == nil { + if result.Error != nil { + return nil, fmt.Errorf("package %q not found: %s", name, result.Error.Message) + } + return nil, fmt.Errorf("package %q not found", name) + } + rev := &result.Result.Revision + if rev.Download.SHA384 == "" { + return nil, fmt.Errorf("package %q has no download digest", name) + } + return rev, nil +} + +func (s *binStore) Options() *Options { + return &s.options +} + +func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { + if risk == "" { + risk = defaultRisk + } + logf("Fetching package %s %s/%s...", name, track, risk) + + rev, err := s.resolveRevision(name, track, risk) + if err != nil { + return nil, nil, err + } + + digest := rev.Download.SHA384 + info := &StorePackageInfo{ + Name: name, + Version: rev.Version, + Revision: rev.Revision, + SHA384: rev.Download.SHA384, + } + + const digestKind = cache.SHA384 + // Check cache first. + reader, err := s.cache.Open(digestKind, digest) + if err == nil { + logf("Using cached package %s", name) + return reader, info, nil + } else if err != cache.ErrMiss { + return nil, nil, err + } + + // Download the package. + err = validateDownloadURL(rev.Download.URL, s.downloadHost) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequest("GET", rev.Download.URL, nil) + if err != nil { + return nil, nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + + httpResp, err := bulkDo(req) + if err != nil { + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode != 200 { + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, httpResp.Status) + } + + writer := s.cache.Create(digestKind, digest) + defer writer.Close() + + _, err = io.Copy(writer, httpResp.Body) + if err == nil { + err = writer.Close() + } + if err != nil { + return nil, nil, fmt.Errorf("cannot fetch package %q: %v", name, err) + } + + reader, err = s.cache.Open(digestKind, writer.Digest()) + if err != nil { + return nil, nil, err + } + return reader, info, nil +} + +// validateDownloadURL checks that the download URL is HTTPS and from the +// allowed host. +func validateDownloadURL(downloadURL, allowedHost string) error { + u, err := url.Parse(downloadURL) + if err != nil { + return fmt.Errorf("cannot parse download URL: %v", err) + } + if u.Scheme != "https" { + return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) + } + host := strings.ToLower(u.Hostname()) + if host == allowedHost { + return nil + } + return fmt.Errorf("download URL has untrusted host %q", host) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 000000000..76d016110 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,527 @@ +package store_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/sha3" + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/store" +) + +type storeSuite struct { + tempDir string + cacheDir string + fakeDoFunc func(req *http.Request) (*http.Response, error) + restore func() + envRestore func() +} + +var _ = Suite(&storeSuite{}) + +func (s *storeSuite) SetUpTest(c *C) { + s.tempDir = c.MkDir() + s.cacheDir = filepath.Join(s.tempDir, "cache") + c.Assert(os.MkdirAll(s.cacheDir, 0o755), IsNil) + + s.envRestore = fakeEnv(store.BinStagingEnvVar, "") + s.restore = store.FakeDo(s.doRequest) + s.fakeDoFunc = nil +} + +func (s *storeSuite) TearDownTest(c *C) { + s.restore() + s.envRestore() +} + +func (s *storeSuite) doRequest(req *http.Request) (*http.Response, error) { + if s.fakeDoFunc != nil { + return s.fakeDoFunc(req) + } + return nil, fmt.Errorf("unexpected HTTP request: %s", req.URL.String()) +} + +func fakeEnv(name, value string) (restore func()) { + origValue, origSet := os.LookupEnv(name) + os.Setenv(name, value) + return func() { + if origSet { + os.Setenv(name, origValue) + } else { + os.Unsetenv(name) + } + } +} + +func sha384Hash(data []byte) string { + h := sha3.New384() + h.Write(data) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +type resolveBodyOptions struct { + name string + track string + risk string + arch string + version string + revision int + sha384 string + downloadURL string +} + +func makeResolveBody(opts resolveBodyOptions) []byte { + if opts.downloadURL == "" { + opts.downloadURL = "https://api.snapcraft.io/api/v1/bins/download/" + opts.name + ".bin" + } + return []byte(fmt.Sprintf(`{ + "craft-results": [], + "package-results": [ + { + "instance-key": %q, + "status": "ok", + "result": { + "channel": { + "name": %q, + "risk": %q, + "track": %q, + "platform": {"architecture": %q} + }, + "revision": { + "version": %q, + "revision": %d, + "download": { + "url": %q, + "sha3-384": %q, + "size": 1024 + } + } + } + } + ] + }`, opts.name, opts.track+"/"+opts.risk, opts.risk, opts.track, opts.arch, opts.version, opts.revision, opts.downloadURL, opts.sha384)) +} + +func makeResolveErrorBody(name, code, message string) []byte { + return []byte(fmt.Sprintf(`{ + "craft-results": [], + "package-results": [ + { + "instance-key": %q, + "status": "error", + "error": {"code": %q, "message": %q}, + "result": null + } + ] + }`, name, code, message)) +} + +func (s *storeSuite) TestValidateDownloadURL(c *C) { + tests := []struct { + summary string + url string + allowedHost string + error string + }{ + {"Valid production host", "https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, + { + "HTTP rejected", + "http://api.snapcraft.io/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, + }, + { + "Untrusted host", + "https://evil.example.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "evil.example.com"`, + }, + { + "Untrusted host case-insensitive", + "https://Evil.Example.Com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "evil.example.com"`, + }, + { + "Host suffix attack", + "https://api.snapcraft.io.evil.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `download URL has untrusted host "api.snapcraft.io.evil.com"`, + }, + {"Invalid URL", "://invalid-url", "api.snapcraft.io", `cannot parse download URL: .*`}, + } + for _, test := range tests { + c.Logf("Summary: %s", test.summary) + err := store.ValidateDownloadURL(test.url, test.allowedHost) + if test.error == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, ErrorMatches, test.error) + } + } +} + +func (s *storeSuite) TestOpenOptionErrors(c *C) { + tests := []struct { + summary string + options store.Options + error string + }{{ + summary: "Valid amd64", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "bin"}, + }, { + summary: "Invalid architecture", + options: store.Options{Arch: "invalid", CacheDir: s.cacheDir, Kind: "bin"}, + error: "invalid package architecture: invalid", + }, { + summary: "Unsupported store kind", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "snap"}, + error: `unsupported store kind "snap"`, + }} + for _, test := range tests { + c.Logf("Summary: %s", test.summary) + _, err := store.Open(&test.options) + if test.error == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, ErrorMatches, test.error) + } + } +} + +type fetchTest struct { + summary string + risk string + status int + statusText string + body string + error string +} + +var fetchTests = []fetchTest{{ + summary: "Defaults to stable risk when unspecified", + risk: "", + status: 200, + // Uses a real digest so the cache verification passes. + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + })), +}, { + summary: "Package not found", + risk: "stable", + status: 200, + body: string(makeResolveErrorBody("curl", "package-not-found", "Package not found")), + error: `package "curl" not found: Package not found`, +}, { + summary: "Server error", + risk: "stable", + status: 500, + statusText: "500 Internal Server Error", + body: "boom", + error: "cannot fetch from store: 500 Internal Server Error", +}, { + summary: "Malformed response body", + risk: "stable", + status: 200, + body: "not json", + error: "cannot decode store response: .*", +}, { + summary: "Missing download digest", + risk: "stable", + status: 200, + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + })), + error: `package "curl" has no download digest`, +}} + +func (s *storeSuite) TestFetch(c *C) { + tarData := []byte("fake tar.xz content") + for _, test := range fetchTests { + c.Logf("Summary: %s", test.summary) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: test.status, + Status: test.statusText, + Body: io.NopCloser(strings.NewReader(test.body)), + }, nil + } + // Download URL. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", test.risk) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + } else { + c.Assert(err, IsNil) + } + } +} + +func (s *storeSuite) TestResolveRequest(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/v2/revisions/resolve" { + // Download request. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + c.Assert(req.Method, Equals, "POST") + c.Assert(req.Header.Get("Content-Type"), Equals, "application/json") + c.Assert(req.Header.Get("Accept"), Equals, "application/json") + + var body map[string]any + err := json.NewDecoder(req.Body).Decode(&body) + c.Assert(err, IsNil) + pkgs := body["packages"].([]any) + c.Assert(pkgs, HasLen, 1) + pkg := pkgs[0].(map[string]any) + c.Assert(pkg["instance-key"], Equals, "curl") + c.Assert(pkg["namespace"], Equals, "bin") + c.Assert(pkg["name"], Equals, "curl") + c.Assert(pkg["channel"], Equals, "latest/stable") + c.Assert(pkg["platform"].(map[string]any)["architecture"], Equals, "amd64") + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) +} + +func (s *storeSuite) TestFetchCacheMiss(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + callCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + callCount++ + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + // Download URL. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.Version, Equals, "8.5.0") + c.Assert(info.SHA384, Equals, digest) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) + c.Assert(callCount, Equals, 2) + + // Verify it's in the cache. + cc := cache.Cache{Dir: s.cacheDir} + cached, err := cc.Read(cache.SHA384, digest) + c.Assert(err, IsNil) + c.Assert(cached, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchCacheHit(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + // Pre-populate the cache. + cc := cache.Cache{Dir: s.cacheDir} + err := cc.Write(cache.SHA384, digest, tarData) + c.Assert(err, IsNil) + + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) + + resolveCallCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + resolveCallCount++ + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return nil, fmt.Errorf("download should not be called for cache hit") + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.SHA384, Equals, digest) + c.Assert(resolveCallCount, Equals, 1) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { + // Override the download URL to an invalid one. + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + downloadURL: "http://evil.example.com/bins/curl.tar.xz", + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) +} + +func (s *storeSuite) TestStagingEnvVar(c *C) { + s.envRestore() + s.envRestore = fakeEnv(store.BinStagingEnvVar, "1") + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + downloadURL: "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin", + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + // Verify staging URL is used. + c.Assert(req.URL.Host, Equals, "api.staging.snapcraft.io") + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) +} + +func (s *storeSuite) TestFetchDownloadError(c *C) { + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + }) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(resolveBody)), + }, nil + } + return &http.Response{ + StatusCode: 500, + Status: "500 Internal Server Error", + Body: io.NopCloser(strings.NewReader("boom")), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `cannot download package "curl": 500 Internal Server Error`) +} diff --git a/internal/store/suite_test.go b/internal/store/suite_test.go new file mode 100644 index 000000000..2eac508f2 --- /dev/null +++ b/internal/store/suite_test.go @@ -0,0 +1,24 @@ +package store_test + +import ( + "testing" + + "github.com/canonical/chisel/internal/store" + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + store.SetDebug(true) + store.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + store.SetDebug(false) + store.SetLogger(nil) +} diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go new file mode 100644 index 000000000..ce1fda5b2 --- /dev/null +++ b/internal/tarball/extract.go @@ -0,0 +1,405 @@ +package tarball + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/strdist" +) + +type ExtractOptions struct { + Package string + TargetDir string + Extract map[string][]ExtractInfo + // OpenData opens the uncompressed tar data stream of the package. It + // abstracts over the package format (e.g. a deb archive opener or a plain + // tarball opener), allowing Extract to operate on any package whose data + // payload is a tar stream. + OpenData func(io.ReadSeeker) (io.ReadCloser, error) + // Create can optionally be set to control the creation of extracted entries. + // extractInfos is set to the matching entries in Extract, and is nil in cases where + // the created entry is implicit and unlisted (for example, parent directories). + Create func(extractInfos []ExtractInfo, options *fsutil.CreateOptions) error +} + +type ExtractInfo struct { + Path string + Mode uint + Optional bool + Context any +} + +func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { + if options.OpenData == nil { + return nil, fmt.Errorf("internal error: ExtractOptions.OpenData is unset") + } + for extractPath, extractInfos := range options.Extract { + isGlob := strings.ContainsAny(extractPath, "*?") + if isGlob { + for _, extractInfo := range extractInfos { + if extractInfo.Path != extractPath || extractInfo.Mode != 0 { + return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) + } + } + } + } + + if options.Create == nil { + validOpts := *options + validOpts.Create = func(_ []ExtractInfo, o *fsutil.CreateOptions) error { + _, err := fsutil.Create(o) + return err + } + return &validOpts, nil + } + + return options, nil +} + +func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) + } + }() + + logf("Extracting files from package %q...", options.Package) + + validOpts, err := getValidOptions(options) + if err != nil { + return err + } + + _, err = os.Stat(validOpts.TargetDir) + if os.IsNotExist(err) { + return fmt.Errorf("target directory does not exist") + } else if err != nil { + return err + } + + return extractData(pkgReader, validOpts) +} + +func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { + dataReader, err := options.OpenData(pkgReader) + if err != nil { + return err + } + defer dataReader.Close() + + oldUmask := syscall.Umask(0) + defer func() { + syscall.Umask(oldUmask) + }() + + pendingPaths := make(map[string]bool) + for extractPath, extractInfos := range options.Extract { + for _, extractInfo := range extractInfos { + if !extractInfo.Optional { + pendingPaths[extractPath] = true + break + } + } + } + + // Store the hard links that we cannot extract when we first iterate over + // the tarball. + // + // This happens because the tarball only stores the contents once in the + // first entry and the rest of them point to the first one. Therefore, we + // cannot tell whether we need to extract the content until after we get to + // a hard link. In this case, we need a second pass. + pendingHardLinks := make(map[string][]pendingHardLink) + + // When creating a file we will iterate through its parent directories and + // create them with the permissions defined in the tarball. + // + // The assumption is that the tar entries of the parent directories appear + // before the entry for the file itself. This is the case for .deb files but + // not for all tarballs. + tarDirMode := make(map[string]fs.FileMode) + tarReader := tar.NewReader(dataReader) + for { + tarHeader, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + sourcePath, ok := sanitizeTarPath(tarHeader.Name) + if !ok { + continue + } + + sourceIsDir := sourcePath[len(sourcePath)-1] == '/' + if sourceIsDir { + tarDirMode[sourcePath] = tarHeader.FileInfo().Mode() + } + + // Find all globs and copies that require this source, and map them by + // their target paths on disk. + targetPaths := map[string][]ExtractInfo{} + for extractPath, extractInfos := range options.Extract { + if extractPath == "" { + continue + } + if strings.ContainsAny(extractPath, "*?") { + if strdist.GlobPath(extractPath, sourcePath) { + targetPaths[sourcePath] = append(targetPaths[sourcePath], extractInfos...) + delete(pendingPaths, extractPath) + } + } else if extractPath == sourcePath { + for _, extractInfo := range extractInfos { + targetPaths[extractInfo.Path] = append(targetPaths[extractInfo.Path], extractInfo) + } + delete(pendingPaths, extractPath) + } + } + if len(targetPaths) == 0 { + // Nothing to do. + continue + } + + var contentCache []byte + var contentIsCached = len(targetPaths) > 1 && !sourceIsDir + if contentIsCached { + // Read and cache the content so it may be reused. + // As an alternative, to avoid having an entire file in + // memory at once this logic might open the first file + // written and copy it every time. For now, the choice + // is speed over memory efficiency. + data, err := io.ReadAll(tarReader) + if err != nil { + return err + } + contentCache = data + } + + var pathReader io.Reader = tarReader + for targetPath, extractInfos := range targetPaths { + if contentIsCached { + pathReader = bytes.NewReader(contentCache) + } + mode := extractInfos[0].Mode + for _, extractInfo := range extractInfos { + if extractInfo.Mode != mode { + if mode < extractInfo.Mode { + mode, extractInfo.Mode = extractInfo.Mode, mode + } + return fmt.Errorf("path %s requested twice with diverging mode: 0%03o != 0%03o", targetPath, mode, extractInfo.Mode) + } + } + if mode != 0 { + tarHeader.Mode = int64(mode) + } + // Create the parent directories using the permissions from the tarball. + parents := parentDirs(targetPath) + for _, path := range parents { + if path == "/" { + continue + } + mode, ok := tarDirMode[path] + if !ok { + continue + } + delete(tarDirMode, path) + + createOptions := &fsutil.CreateOptions{ + Root: options.TargetDir, + Path: path, + Mode: mode, + MakeParents: true, + } + err := options.Create(nil, createOptions) + if err != nil { + return err + } + } + link := tarHeader.Linkname + if tarHeader.Typeflag == tar.TypeLink { + // A hard link requires the real path of the target file. + link = filepath.Join(options.TargetDir, link) + } + + // Create the entry itself. + createOptions := &fsutil.CreateOptions{ + Root: options.TargetDir, + Path: targetPath, + Mode: tarHeader.FileInfo().Mode(), + Data: pathReader, + Link: link, + MakeParents: true, + OverrideMode: true, + } + err := options.Create(extractInfos, createOptions) + if err != nil && os.IsNotExist(err) && tarHeader.Typeflag == tar.TypeLink { + // The hard link could not be created because the content + // was not extracted previously. Add this hard link entry + // to the pending list to extract later. + relLinkPath, ok := sanitizeTarPath(tarHeader.Linkname) + if !ok { + return fmt.Errorf("invalid link target %s", tarHeader.Linkname) + } + info := pendingHardLink{ + path: targetPath, + extractInfos: extractInfos, + } + pendingHardLinks[relLinkPath] = append(pendingHardLinks[relLinkPath], info) + } else if err != nil { + return err + } + } + } + + if len(pendingHardLinks) > 0 { + // Go over the tarball again to textract the pending hard links. + extractHardLinkOptions := &extractHardLinkOptions{ + ExtractOptions: options, + pendingLinks: pendingHardLinks, + } + _, err := pkgReader.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = extractHardLinks(pkgReader, extractHardLinkOptions) + if err != nil { + return err + } + } + + if len(pendingPaths) > 0 { + pendingList := make([]string, 0, len(pendingPaths)) + for pendingPath := range pendingPaths { + pendingList = append(pendingList, pendingPath) + } + if len(pendingList) == 1 { + return fmt.Errorf("no content at %s", pendingList[0]) + } else { + sort.Strings(pendingList) + return fmt.Errorf("no content at:\n- %s", strings.Join(pendingList, "\n- ")) + } + } + + return nil +} + +type pendingHardLink struct { + path string + extractInfos []ExtractInfo +} + +type extractHardLinkOptions struct { + *ExtractOptions + pendingLinks map[string][]pendingHardLink +} + +// extractHardLinks iterates through the tarball a second time to extract the +// hard links that were not extracted in the first pass. +func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { + dataReader, err := opts.OpenData(pkgReader) + if err != nil { + return err + } + defer dataReader.Close() + + tarReader := tar.NewReader(dataReader) + for { + tarHeader, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + sourcePath, ok := sanitizeTarPath(tarHeader.Name) + if !ok { + continue + } + + links := opts.pendingLinks[sourcePath] + if len(links) == 0 { + continue + } + + // For a target path, the first hard link will be created as a file with + // the content of the target path. If there are more pending hard links, + // the remaining ones will be created as hard links with the newly + // created file as their target. + absLink := filepath.Join(opts.TargetDir, links[0].path) + // Extract the content to the first hard link path. + createOptions := &fsutil.CreateOptions{ + Root: opts.TargetDir, + Path: links[0].path, + Mode: tarHeader.FileInfo().Mode(), + Data: tarReader, + } + err = opts.Create(links[0].extractInfos, createOptions) + if err != nil { + return err + } + + // Create the remaining hard links. + for _, link := range links[1:] { + createOptions := &fsutil.CreateOptions{ + Root: opts.TargetDir, + Path: link.path, + Mode: tarHeader.FileInfo().Mode(), + // Link to the first file extracted for the hard links. + Link: absLink, + } + err := opts.Create(link.extractInfos, createOptions) + if err != nil { + return err + } + } + delete(opts.pendingLinks, sourcePath) + } + + // If there are pending links, that means the link targets do not come from + // this package. + if len(opts.pendingLinks) > 0 { + var targets []string + for target := range opts.pendingLinks { + targets = append(targets, target) + } + sort.Strings(targets) + link := opts.pendingLinks[targets[0]][0] + return fmt.Errorf("cannot create hard link %s: no content at %s", link.path, targets[0]) + } + + return nil +} + +func parentDirs(path string) []string { + path = filepath.Clean(path) + parents := make([]string, strings.Count(path, "/")) + count := 0 + for i, c := range path { + if c == '/' { + parents[count] = path[:i+1] + count++ + } + } + return parents +} + +// sanitizeTarPath removes the leading "./" from the source path in the tarball, +// and verifies that the path is not empty. +func sanitizeTarPath(path string) (string, bool) { + if len(path) < 3 || path[0] != '.' || path[1] != '/' { + return "", false + } + return path[1:], true +} diff --git a/internal/deb/extract_test.go b/internal/tarball/extract_test.go similarity index 69% rename from internal/deb/extract_test.go rename to internal/tarball/extract_test.go index 1ec0b8a5f..f855d9bc4 100644 --- a/internal/deb/extract_test.go +++ b/internal/tarball/extract_test.go @@ -1,4 +1,4 @@ -package deb_test +package tarball_test import ( "bytes" @@ -12,14 +12,15 @@ import ( "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) type extractTest struct { summary string pkgdata []byte - options deb.ExtractOptions - hackopt func(c *C, o *deb.ExtractOptions) + options tarball.ExtractOptions + hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string // paths which the extractor did not create explicitly. notCreated []string @@ -29,28 +30,28 @@ type extractTest struct { var extractTests = []extractTest{{ summary: "Extract nothing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ + options: tarball.ExtractOptions{ Extract: nil, }, result: map[string]string{}, }, { summary: "Extract a few entries", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }}, - "/dir/other-file": []deb.ExtractInfo{{ + "/dir/other-file": []tarball.ExtractInfo{{ Path: "/dir/other-file", }}, - "/dir/several/levels/deep/file": []deb.ExtractInfo{{ + "/dir/several/levels/deep/file": []tarball.ExtractInfo{{ Path: "/dir/several/levels/deep/file", }}, - "/dir/nested/": []deb.ExtractInfo{{ + "/dir/nested/": []tarball.ExtractInfo{{ Path: "/dir/nested/", }}, - "/other-dir/": []deb.ExtractInfo{{ + "/other-dir/": []tarball.ExtractInfo{{ Path: "/other-dir/", }}, }, @@ -70,21 +71,21 @@ var extractTests = []extractTest{{ }, { summary: "Extract a few entries, nil Create closure", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }}, - "/dir/other-file": []deb.ExtractInfo{{ + "/dir/other-file": []tarball.ExtractInfo{{ Path: "/dir/other-file", }}, - "/dir/several/levels/deep/file": []deb.ExtractInfo{{ + "/dir/several/levels/deep/file": []tarball.ExtractInfo{{ Path: "/dir/several/levels/deep/file", }}, - "/dir/nested/": []deb.ExtractInfo{{ + "/dir/nested/": []tarball.ExtractInfo{{ Path: "/dir/nested/", }}, - "/other-dir/": []deb.ExtractInfo{{ + "/other-dir/": []tarball.ExtractInfo{{ Path: "/other-dir/", }}, }, @@ -100,19 +101,19 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/file": "file 0644 6bc26dff", "/other-dir/": "dir 0755", }, - hackopt: func(c *C, o *deb.ExtractOptions) { + hackopt: func(c *C, o *tarball.ExtractOptions) { o.Create = nil }, }, { summary: "Copy a couple of entries elsewhere", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/foo/file-copy", Mode: 0600, }}, - "/dir/several/levels/deep/": []deb.ExtractInfo{{ + "/dir/several/levels/deep/": []tarball.ExtractInfo{{ Path: "/foo/bar/dir-copy", Mode: 0700, }}, @@ -128,9 +129,9 @@ var extractTests = []extractTest{{ }, { summary: "Copy same file twice", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/foo/file-copy-1", }, { Path: "/dir/bar/file-copy-2", @@ -148,9 +149,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing a single dir level", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s*/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s*/": []tarball.ExtractInfo{{ Path: "/dir/s*/", }}, }, @@ -163,9 +164,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing for files with multiple levels at once", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s**": []tarball.ExtractInfo{{ Path: "/dir/s**", }}, }, @@ -181,12 +182,12 @@ var extractTests = []extractTest{{ }, { summary: "Globbing multiple paths", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s**": []tarball.ExtractInfo{{ Path: "/dir/s**", }}, - "/dir/n*/": []deb.ExtractInfo{{ + "/dir/n*/": []tarball.ExtractInfo{{ Path: "/dir/n*/", }}, }, @@ -203,9 +204,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing must have matching source and target", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/foo/b**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/foo/b**": []tarball.ExtractInfo{{ Path: "/foo/g**", }}, }, @@ -214,9 +215,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing must also have a single target", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/foo/b**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/foo/b**": []tarball.ExtractInfo{{ Path: "/foo/b**", }, { Path: "/foo/g**", @@ -227,9 +228,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing cannot change modes", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/n**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/n**": []tarball.ExtractInfo{{ Path: "/dir/n**", Mode: 0777, }}, @@ -239,9 +240,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing file", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-file": []tarball.ExtractInfo{{ Path: "/missing-file", }}, }, @@ -250,9 +251,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing directory", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-dir/": []tarball.ExtractInfo{{ Path: "/missing-dir/", }}, }, @@ -261,9 +262,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing glob", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-dir/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-dir/**": []tarball.ExtractInfo{{ Path: "/missing-dir/**", }}, }, @@ -272,12 +273,12 @@ var extractTests = []extractTest{{ }, { summary: "Missing multiple entries", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-file": []tarball.ExtractInfo{{ Path: "missing-file", }}, - "/missing-dir/": []deb.ExtractInfo{{ + "/missing-dir/": []tarball.ExtractInfo{{ Path: "/missing-dir/", }}, }, @@ -286,16 +287,16 @@ var extractTests = []extractTest{{ }, { summary: "Optional entries may be missing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", }}, - "/dir/optional": []deb.ExtractInfo{{ + "/dir/optional": []tarball.ExtractInfo{{ Path: "/other-dir/foo", Optional: true, }}, - "/optional-dir/": []deb.ExtractInfo{{ + "/optional-dir/": []tarball.ExtractInfo{{ Path: "/foo/optional-dir/", Optional: true, }}, @@ -308,9 +309,9 @@ var extractTests = []extractTest{{ }, { summary: "Optional entries mixed in cannot be missing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/missing-file": []tarball.ExtractInfo{{ Path: "/dir/optional", Optional: true, }, { @@ -327,9 +328,9 @@ var extractTests = []extractTest{{ testutil.Dir(0766, "./日本/"), testutil.Reg(0644, "./日本/語", "whatever"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/日本/語": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/日本/語": []tarball.ExtractInfo{{ Path: "/日本/語", }}, }, @@ -342,13 +343,13 @@ var extractTests = []extractTest{{ }, { summary: "Entries for same destination must have the same mode", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", Mode: 0777, }}, - "/d**": []deb.ExtractInfo{{ + "/d**": []tarball.ExtractInfo{{ Path: "/d**", }}, }, @@ -361,9 +362,9 @@ var extractTests = []extractTest{{ testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -380,9 +381,9 @@ var extractTests = []extractTest{{ testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/hardlink": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/hardlink": []tarball.ExtractInfo{{ Path: "/hardlink", }}, }, @@ -397,9 +398,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "./non-existing-target"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/hardlink": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/hardlink": []tarball.ExtractInfo{{ Path: "/hardlink", }}, }, @@ -412,9 +413,9 @@ var extractTests = []extractTest{{ testutil.Hrd(0644, "./hardlink1", "./non-existing-target"), testutil.Hrd(0644, "./hardlink2", "./non-existing-target"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -427,9 +428,9 @@ var extractTests = []extractTest{{ testutil.Lnk(0644, "./symlink", "./file"), testutil.Hrd(0644, "./hardlink", "./symlink"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -442,15 +443,15 @@ var extractTests = []extractTest{{ }, { summary: "Explicit extraction overrides existing file", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", Mode: 0777, }}, }, }, - hackopt: func(c *C, o *deb.ExtractOptions) { + hackopt: func(c *C, o *tarball.ExtractOptions) { err := os.Mkdir(path.Join(o.TargetDir, "/dir"), 0666) c.Assert(err, IsNil) }, @@ -464,9 +465,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "/etc/group"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -478,9 +479,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Reg(0644, "./../file", "hijacking system file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -496,8 +497,10 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader createdPaths := make(map[string]bool) - options.Create = func(_ []deb.ExtractInfo, o *fsutil.CreateOptions) error { + options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) if o.Mode.IsDir() { relPath = relPath + "/" @@ -511,7 +514,7 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - err := deb.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -539,8 +542,8 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - options deb.ExtractOptions - calls map[string][]deb.ExtractInfo + options tarball.ExtractOptions + calls map[string][]tarball.ExtractInfo }{{ summary: "Create is called with the set of ExtractInfo(s) that match the file", pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -548,50 +551,50 @@ var extractCreateCallbackTests = []struct { testutil.Dir(0766, "./dir/"), testutil.Reg(0644, "./dir/file", "whatever"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", }}, - "/d**": []deb.ExtractInfo{{ + "/d**": []tarball.ExtractInfo{{ Path: "/d**", }}, - "/d?r/": []deb.ExtractInfo{{ + "/d?r/": []tarball.ExtractInfo{{ Path: "/d?r/", }}, - "/dir/file": []deb.ExtractInfo{{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }, { Path: "/dir/file-cpy", }}, - "/foo/": []deb.ExtractInfo{{ + "/foo/": []tarball.ExtractInfo{{ Path: "/foo/", Optional: true, }}, }, }, - calls: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{ - deb.ExtractInfo{ + calls: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d**", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d?r/", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/", }, }, - "/dir/file": []deb.ExtractInfo{ - deb.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d**", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/file", }, }, - "/dir/file-cpy": []deb.ExtractInfo{ - deb.ExtractInfo{ + "/dir/file-cpy": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/file-cpy", }, }, @@ -605,8 +608,10 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir - createExtractInfos := map[string][]deb.ExtractInfo{} - options.Create = func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader + createExtractInfos := map[string][]tarball.ExtractInfo{} + options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { // Creating implicit parent directories, we don't care about those. return nil @@ -622,9 +627,21 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - err := deb.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) } } + +func (s *S) TestExtractMissingOpenData(c *C) { + options := tarball.ExtractOptions{ + Package: "test-package", + TargetDir: c.MkDir(), + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": {{Path: "/dir/file"}}, + }, + } + err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), &options) + c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: ExtractOptions.OpenData is unset`) +} diff --git a/internal/tarball/log.go b/internal/tarball/log.go new file mode 100644 index 000000000..903bf279d --- /dev/null +++ b/internal/tarball/log.go @@ -0,0 +1,53 @@ +package tarball + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/tarball/suite_test.go b/internal/tarball/suite_test.go new file mode 100644 index 000000000..e99ce3e92 --- /dev/null +++ b/internal/tarball/suite_test.go @@ -0,0 +1,20 @@ +package tarball_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/tarball" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + tarball.SetDebug(true) + tarball.SetLogger(c) +} diff --git a/internal/tarball/xz.go b/internal/tarball/xz.go new file mode 100644 index 000000000..700215a09 --- /dev/null +++ b/internal/tarball/xz.go @@ -0,0 +1,15 @@ +package tarball + +import ( + "io" + + "github.com/ulikunitz/xz" +) + +func XZDataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..ccac36890 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -13,15 +13,6 @@ type TestArchive struct { Packages map[string]*TestPackage } -type TestPackage struct { - Name string - Version string - Hash string - Arch string - Data []byte - Archives []string -} - func (a *TestArchive) Options() *archive.Options { return &a.Opts } diff --git a/internal/testutil/defaults.go b/internal/testutil/defaults.go index fa08b4e63..046d46360 100644 --- a/internal/testutil/defaults.go +++ b/internal/testutil/defaults.go @@ -1,5 +1,7 @@ package testutil +import "strings" + var testKey = PGPKeys["key1"] var DefaultChiselYaml = ` @@ -17,3 +19,11 @@ var DefaultChiselYaml = ` test-key: id: ` + testKey.ID + ` armor: |` + "\n" + PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + +var DefaultChiselYamlWithStores = strings.ReplaceAll(DefaultChiselYaml, "format: v1", "format: v3") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" +` diff --git a/internal/testutil/package.go b/internal/testutil/package.go new file mode 100644 index 000000000..561eb105c --- /dev/null +++ b/internal/testutil/package.go @@ -0,0 +1,17 @@ +package testutil + +// TestPackage is a source-agnostic package fixture that can be served by either +// a TestArchive or a TestStore. +type TestPackage struct { + Name string + Version string + Hash string + Arch string + Data []byte + // Archives lists the archives the package belongs to. It is only relevant + // for archive-backed packages. + Archives []string + // Store names the store the package is served from. When set, the package + // is store-backed rather than archive-backed. + Store string +} diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index f901873d3..8fb8fb8b2 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,6 +8,7 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -160,6 +161,42 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } +// compressBytesXz compresses the input using XZ, the compression format used by +// store (bin) packages. +func compressBytesXz(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MakeBin builds a store (bin) package from the given tar entries: an +// XZ-compressed plain tarball. Unlike MakeDeb, there is no ar container. +func MakeBin(entries []TarEntry) ([]byte, error) { + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + return compressBytesXz(tarData) +} + +// MustMakeBin is the panicking variant of MakeBin. +func MustMakeBin(entries []TarEntry) []byte { + data, err := MakeBin(entries) + if err != nil { + panic(err) + } + return data +} + // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry { diff --git a/internal/testutil/store.go b/internal/testutil/store.go new file mode 100644 index 000000000..fe91fdd09 --- /dev/null +++ b/internal/testutil/store.go @@ -0,0 +1,31 @@ +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/canonical/chisel/internal/store" +) + +type TestStore struct { + Opts store.Options + Packages map[string]*TestPackage +} + +func (s *TestStore) Options() *store.Options { + return &s.Opts +} + +func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *store.StorePackageInfo, error) { + pkg, ok := s.Packages[name] + if !ok { + return nil, nil, fmt.Errorf("cannot find package %q in store", name) + } + info := &store.StorePackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA384: pkg.Hash, + } + return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil +}