From df532f73521ffaa40762fbd5d5b23ed7b442a1ec Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 27 May 2026 16:06:36 +0200 Subject: [PATCH 01/61] feat: support bin packages parsing --- internal/setup/setup.go | 34 ++++- internal/setup/setup_test.go | 261 ++++++++++++++++++++++++++++++++++- internal/setup/yaml.go | 65 ++++++++- internal/slicer/slicer.go | 16 ++- 4 files changed, 363 insertions(+), 13 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index aaa8d1e55..f304c6832 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,6 +16,19 @@ import ( "github.com/canonical/chisel/internal/strdist" ) +// StoreKind identifies the backend type of a store. +type StoreKind string + +const StoreBin StoreKind = "bin" + +// Store is the location from which binary packages are obtained via a store API. +type Store struct { + Name string + Kind StoreKind + Version string + DefaultPrefix string +} + // Release is a collection of package slices targeting a particular // distribution version. type Release struct { @@ -23,6 +36,7 @@ type Release struct { Path string Packages map[string]*Package Archives map[string]*Archive + Stores map[string]*Store Maintenance *Maintenance } @@ -51,10 +65,12 @@ type Archive struct { // Package holds a collection of slices that represent parts of themselves. type Package struct { - Name string - Path string - Archive string - Slices map[string]*Slice + Name string + Path string + Archive string + Store string + DefaultTrack string + Slices map[string]*Slice } // Slice holds the details about a package slice. @@ -337,6 +353,16 @@ func (r *Release) validate() error { } } + // Check that stores referenced in packages are defined. + for _, pkg := range r.Packages { + if pkg.Store == "" { + continue + } + if _, ok := r.Stores[pkg.Store]; !ok { + return fmt.Errorf("%s: package refers to undefined store %q", pkg.Path, pkg.Store) + } + } + return nil } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d669943f6..3c5792134 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -3896,6 +3896,195 @@ var setupTests = []setupTest{{ `, }, relerror: `package "mypkg" slices defined more than once: slices/dir1/mypkg.yaml and slices/dir2/mypkg.yaml`, +}, { + summary: "Store package is parsed correctly", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "26.10", + Suites: []string{"stonking"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: setup.StoreBin, + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/bin/mypkg.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{}, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "slices/bin/mypkg.yaml": ` + package: mypkg + archive: ubuntu + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, +}, { + summary: "Store package missing default-track", + input: map[string]string{ + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' is required when 'store' is set`, +}, { + summary: "default-track without store", + input: map[string]string{ + "slices/bin/mypkg.yaml": ` + package: mypkg + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'store' is required when 'default-track' is set`, +}, { + summary: "default-track must not contain /", + input: map[string]string{ + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0/stable" + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' must be a track name without /`, +}, { + summary: "Package store references undefined store", + input: map[string]string{ + "slices/bin/mypkg.yaml": ` + package: mypkg + store: no-such-store + default-track: "3.0" + `, + }, + relerror: `slices/bin/mypkg.yaml: package refers to undefined store "no-such-store"`, +}, { + summary: "Store with invalid kind", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + mystore: + kind: unknown-kind + version: 26.10 + default-prefix: "pfx-" + `, + }, + relerror: `chisel.yaml: store "mystore" has invalid kind "unknown-kind"`, +}, { + summary: "Store missing version", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing version field`, +}, { + summary: "Store missing default-prefix", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + `, + }, + relerror: `chisel.yaml: store "bin" missing default-prefix field`, }} func (s *S) TestParseRelease(c *C) { @@ -3998,9 +4187,9 @@ func runParseReleaseTests(c *C, tests []setupTest) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) c.Assert(err, IsNil) } @@ -4045,6 +4234,40 @@ func runParseReleaseTests(c *C, tests []setupTest) { } } +func (s *S) TestStoresNotSupportedInOldFormats(c *C) { + for _, format := range []string{"v1", "v2"} { + c.Logf("Format: %s", format) + chiselYaml := ` + format: ` + format + ` + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 22.04 + components: [main, universe] + suites: [jammy] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 22.04 + default-prefix: "bin-" + ` + dir := c.MkDir() + err := os.WriteFile(filepath.Join(dir, "chisel.yaml"), testutil.Reindent(chiselYaml), 0o644) + c.Assert(err, IsNil) + err = os.MkdirAll(filepath.Join(dir, "slices"), 0o755) + c.Assert(err, IsNil) + _, err = setup.ReadRelease(dir) + c.Assert(err, ErrorMatches, `chisel.yaml: stores is not supported in format "`+format+`"`) + } +} + func (s *S) TestPackageMarshalYAML(c *C) { for _, test := range setupTests { c.Logf("Summary: %s", test.summary) @@ -4232,6 +4455,40 @@ func (s *S) TestPackageYAMLFormat(c *C) { mypkg_three: {arch: i386} `, }, + }, { + summary: "Store package fields", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /usr/bin/mypkg: {} + `, + }, }} for _, test := range tests { diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index cefe86cb4..b9b6d2fd9 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -35,6 +35,7 @@ type yamlRelease struct { // fields that break said compatibility (e.g. "pro" archives) and merged // together with "archives". V2Archives map[string]yamlArchive `yaml:"v2-archives"` + Stores map[string]yamlStore `yaml:"stores"` } const ( @@ -52,9 +53,17 @@ type yamlArchive struct { PubKeys []string `yaml:"public-keys"` } +type yamlStore struct { + Kind string `yaml:"kind"` + Version string `yaml:"version"` + DefaultPrefix string `yaml:"default-prefix"` +} + type yamlPackage struct { - Name string `yaml:"package"` - Archive string `yaml:"archive,omitempty"` + Name string `yaml:"package"` + Archive string `yaml:"archive,omitempty"` + Store string `yaml:"store,omitempty"` + DefaultTrack string `yaml:"default-track,omitempty"` // For backwards-compatibility reasons with v1 and v2, essential needs // custom logic to be parsed. See [yamlEssentialListMap]. Essential yamlEssentialListMap `yaml:"essential,omitempty"` @@ -433,6 +442,33 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.Archives[archiveName] = details } + // Parse stores. + if len(yamlVar.Stores) > 0 && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("%s: stores is not supported in format %q", fileName, release.Format) + } + if len(yamlVar.Stores) > 0 { + release.Stores = make(map[string]*Store, len(yamlVar.Stores)) + } + for storeName, details := range yamlVar.Stores { + switch StoreKind(details.Kind) { + case StoreBin: + default: + return nil, fmt.Errorf("%s: store %q has invalid kind %q", fileName, storeName, details.Kind) + } + if details.Version == "" { + return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) + } + if details.DefaultPrefix == "" { + return nil, fmt.Errorf("%s: store %q missing default-prefix field", fileName, storeName) + } + release.Stores[storeName] = &Store{ + Name: storeName, + Kind: StoreKind(details.Kind), + Version: details.Version, + DefaultPrefix: details.DefaultPrefix, + } + } + return release, err } @@ -480,6 +516,23 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } } + if yamlPkg.Store != "" && yamlPkg.Archive != "" { + return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) + } + if yamlPkg.Store != "" { + if yamlPkg.DefaultTrack == "" { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' is required when 'store' is set", pkgName) + } + if strings.Contains(yamlPkg.DefaultTrack, "/") { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' must be a track name without /", pkgName) + } + pkg.Store = yamlPkg.Store + pkg.DefaultTrack = yamlPkg.DefaultTrack + } else { + if yamlPkg.DefaultTrack != "" { + return nil, fmt.Errorf("cannot parse package %q: 'store' is required when 'default-track' is set", pkgName) + } + } pkg.Archive = yamlPkg.Archive zeroPath := yamlPath{} for sliceName, yamlSlice := range yamlPkg.Slices { @@ -690,9 +743,11 @@ func sliceToYAML(s *Slice) (*yamlSlice, error) { // packageToYAML converts a Package object to a yamlPackage object. func packageToYAML(p *Package) (*yamlPackage, error) { pkg := &yamlPackage{ - Name: p.Name, - Archive: p.Archive, - Slices: make(map[string]yamlSlice, len(p.Slices)), + Name: p.Name, + Archive: p.Archive, + Store: p.Store, + DefaultTrack: p.DefaultTrack, + Slices: make(map[string]yamlSlice, len(p.Slices)), } for name, slice := range p.Slices { yamlSlice, err := sliceToYAML(slice) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb2..ce54bb278 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -95,6 +95,13 @@ func Run(options *RunOptions) error { return err } + // Build a map from package name to architecture. + pkgArch := make(map[string]string) + for pkg, a := range pkgArchive { + pkgArch[pkg] = a.Options().Arch + } + // TODO Handle packages coming from a store as well when we support them. + prefers, err := options.Selection.Prefers() if err != nil { return err @@ -108,7 +115,7 @@ func Run(options *RunOptions) error { extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgArchive[slice.Package].Options().Arch + arch := pkgArch[slice.Package] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -269,7 +276,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArchive[slice.Package].Options().Arch + arch := pkgArch[slice.Package] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -512,6 +519,11 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel continue } pkg := selection.Release.Packages[s.Package] + if pkg.Store != "" { + // Packages coming from a store are not fetched from an archive, + // so we skip them here. + continue + } var candidates []*setup.Archive if pkg.Archive == "" { From f3a58a64a674d2e38f07323b6d5002d86f212e32 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 27 May 2026 16:19:07 +0200 Subject: [PATCH 02/61] fix: correct style fix Signed-off-by: Paul Mars --- internal/setup/setup_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 3c5792134..8328651a7 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -4187,9 +4187,9 @@ func runParseReleaseTests(c *C, tests []setupTest) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0o755) + err := os.MkdirAll(filepath.Dir(fpath), 0755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) c.Assert(err, IsNil) } From daf5a7185d97ac4524be75789aaaf7eb558eea50 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 27 May 2026 16:21:10 +0200 Subject: [PATCH 03/61] fix: lint Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index ce54bb278..86562b8b6 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -101,7 +101,7 @@ func Run(options *RunOptions) error { pkgArch[pkg] = a.Options().Arch } // TODO Handle packages coming from a store as well when we support them. - + prefers, err := options.Selection.Prefers() if err != nil { return err From 633b0e1b976c012182f0b0b1209bd6d86cab854e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 28 May 2026 08:31:02 +0200 Subject: [PATCH 04/61] ci: rerun From 6fd3d1c1abbbf1139a152b1336145a4311ba3895 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 29 May 2026 15:30:47 +0200 Subject: [PATCH 05/61] fix: fully qualified slice names Co-authored-by: Copilot Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 2 +- internal/apacheutil/util.go | 22 +++- internal/apacheutil/util_test.go | 52 +++++++++ internal/setup/setup.go | 110 +++++++++++++------ internal/setup/setup_test.go | 175 +++++++++++++++++++++++++------ internal/setup/yaml.go | 8 +- internal/slicer/slicer.go | 36 ++++--- internal/slicer/slicer_test.go | 160 ++++++++++++++-------------- 8 files changed, 395 insertions(+), 170 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index fa5485c6c..228d8f1b0 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -83,7 +83,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* allPkgSlices := make(map[string]bool) sliceExists := func(key setup.SliceKey) bool { - pkg, ok := release.Packages[key.Package] + pkg, ok := release.Packages[key.MapKey()] if !ok { return false } diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go index 7e0f8e6d8..2ad9a364b 100644 --- a/internal/apacheutil/util.go +++ b/internal/apacheutil/util.go @@ -9,10 +9,24 @@ import ( type SliceKey struct { Package string + Kind string Slice string } -func (s SliceKey) String() string { return s.Package + "_" + s.Slice } +func (s SliceKey) String() string { + if s.Kind != "" { + return s.Kind + "/" + s.Package + "_" + s.Slice + } + return s.Package + "_" + s.Slice +} + +// MapKey returns the qualified package key used for Release.Packages lookups. +func (s SliceKey) MapKey() string { + if s.Kind != "" { + return s.Kind + "/" + s.Package + } + return s.Package +} // FnameExp matches the slice definition file basename. var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) @@ -20,13 +34,13 @@ var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) // SnameExp matches only the slice name, without the leading package name. var SnameExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})$`) -// knameExp matches the slice full name in pkg_slice format. -var knameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) +// knameExp matches the slice full name in pkg_slice or kind/pkg_slice format. +var knameExp = regexp.MustCompile(`^(?:([a-z0-9](?:-?[.a-z0-9+]){0,})/)?([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) func ParseSliceKey(sliceKey string) (SliceKey, error) { match := knameExp.FindStringSubmatch(sliceKey) if match == nil { return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) } - return SliceKey{match[1], match[2]}, nil + return SliceKey{Package: match[2], Kind: match[1], Slice: match[3]}, nil } diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go index 6289473cc..752af2ebf 100644 --- a/internal/apacheutil/util_test.go +++ b/internal/apacheutil/util_test.go @@ -78,6 +78,15 @@ var sliceKeyTests = []struct { }, { input: "..._bar", err: `invalid slice reference: "\.\.\._bar"`, +}, { + input: "bin/curl_bins", + expected: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, +}, { + input: "bin/g++_bins", + expected: apacheutil.SliceKey{Package: "g++", Kind: "bin", Slice: "bins"}, +}, { + input: "b/curl_bins", + expected: apacheutil.SliceKey{Package: "curl", Kind: "b", Slice: "bins"}, }, { input: "white space_no-whitespace", err: `invalid slice reference: "white space_no-whitespace"`, @@ -94,3 +103,46 @@ func (s *S) TestParseSliceKey(c *C) { c.Assert(key, DeepEquals, test.expected) } } + +var sliceKeyStringTests = []struct { + key apacheutil.SliceKey + expected string +}{{ + key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, + expected: "curl_bins", +}, { + key: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, + expected: "bin/curl_bins", +}, { + key: apacheutil.SliceKey{Package: "g++", Kind: "bin", Slice: "bins"}, + expected: "bin/g++_bins", +}} + +func (s *S) TestSliceKeyString(c *C) { + for _, test := range sliceKeyStringTests { + c.Assert(test.key.String(), Equals, test.expected) + } +} + +var sliceKeyMapKeyTests = []struct { + key apacheutil.SliceKey + expected string +}{{ + key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, + expected: "curl", +}, { + key: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, + expected: "bin/curl", +}, { + key: apacheutil.SliceKey{Package: "curl", Kind: "bin"}, + expected: "bin/curl", +}, { + key: apacheutil.SliceKey{Package: "curl"}, + expected: "curl", +}} + +func (s *S) TestSliceKeyMapKey(c *C) { + for _, test := range sliceKeyMapKeyTests { + c.Assert(test.key.MapKey(), Equals, test.expected) + } +} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index f304c6832..4794351ca 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -76,6 +76,7 @@ type Package struct { // Slice holds the details about a package slice. type Slice struct { Package string + Kind string Name string Hint string Essential map[SliceKey]EssentialInfo @@ -149,7 +150,34 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { return apacheutil.ParseSliceKey(sliceKey) } -func (s *Slice) String() string { return s.Package + "_" + s.Name } +// pkgKind returns the Kind value for a package based on its store reference. +func pkgKind(release *Release, pkg *Package) string { + if pkg.Store == "" { + return "" + } + store := release.Stores[pkg.Store] + if store == nil { + return "" + } + return string(store.Kind) +} + +// pkgMapKey returns the qualified key for a package used as the +// Release.Packages map key. +func pkgMapKey(name, kind string) string { + if kind != "" { + return kind + "/" + name + } + return name +} + +func (s *Slice) String() string { + return SliceKey{Package: s.Package, Kind: s.Kind, Slice: s.Name}.String() +} + +func (s *Slice) MapKey() string { + return SliceKey{Package: s.Package, Kind: s.Kind}.MapKey() +} // Selection holds the required configuration to create a Build for a selection // of slices from a Release. It's still an abstract proposal in the sense that @@ -176,16 +204,18 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } + sliceKey := pkgMapKey(slice.Package, slice.Kind) old, ok := pathPreferredPkg[path] if !ok { - pathPreferredPkg[path] = s.Release.Packages[slice.Package] + pathPreferredPkg[path] = s.Release.Packages[sliceKey] continue } - if old.Name == slice.Package { + oldKey := pkgMapKey(old.Name, pkgKind(s.Release, old)) + if oldKey == sliceKey { // Skip if the package was already recorded. continue } - preferred, err := preferredPathPackage(path, old.Name, slice.Package, prefers) + preferred, err := preferredPathPackage(path, oldKey, sliceKey, prefers) if err != nil { // Note: we have checked above that the path has prefers and // they are different packages so the error cannot be @@ -238,13 +268,16 @@ func (r *Release) validate() error { // cannot validate that they are the same without downloading the package. paths := make(map[string][]*Slice) for _, pkg := range r.Packages { + kind := pkgKind(r, pkg) for _, new := range pkg.Slices { - keys = append(keys, SliceKey{pkg.Name, new.Name}) + keys = append(keys, SliceKey{Package: pkg.Name, Kind: kind, Slice: new.Name}) + newKey := pkgMapKey(new.Package, new.Kind) for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - if new.Package != old.Package { - _, err := preferredPathPackage(newPath, new.Package, old.Package, prefers) + oldKey := pkgMapKey(old.Package, old.Kind) + if newKey != oldKey { + _, err := preferredPathPackage(newPath, newKey, oldKey, prefers) if err == nil { continue } else if err != errPreferNone { @@ -253,8 +286,8 @@ func (r *Release) validate() error { } oldInfo := old.Contents[newPath] - if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && new.Package != old.Package { - if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name { + if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && newKey != oldKey { + if oldKey > newKey || oldKey == newKey && old.Name > new.Name { old, new = new, old } return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath) @@ -277,7 +310,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if slice.Package == skey.pkg { + if pkgMapKey(slice.Package, slice.Kind) == skey.pkg { found = true break } @@ -302,14 +335,16 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] + newKey := pkgMapKey(new.Package, new.Kind) + oldKey := pkgMapKey(old.Package, old.Kind) if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { - if new.Package == old.Package { + if newKey == oldKey { continue } } if strdist.GlobPath(newPath, oldPath) { - if (old.Package > new.Package) || (old.Package == new.Package && old.Name > new.Name) || - (old.Package == new.Package && old.Name == new.Name && oldPath > newPath) { + if (oldKey > newKey) || (oldKey == newKey && old.Name > new.Name) || + (oldKey == newKey && old.Name == new.Name && oldPath > newPath) { old, new = new, old oldPath, newPath = newPath, oldPath } @@ -372,10 +407,9 @@ func (r *Release) validate() error { // If arch is supplied, essential(s) not specific to that arch are not // considered. func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, error) { - // Preprocess the list to improve error messages. for _, key := range keys { - if pkg, ok := pkgs[key.Package]; !ok { + if pkg, ok := pkgs[key.MapKey()]; !ok { return nil, fmt.Errorf("slices of package %q not found", key.Package) } else if _, ok := pkg.Slices[key.Slice]; !ok { return nil, fmt.Errorf("slice %s not found", key) @@ -393,7 +427,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } seen[key] = true - pkg := pkgs[key.Package] + pkg := pkgs[key.MapKey()] slice := pkg.Slices[key.Slice] fqslice := slice.String() predecessors := successors[fqslice] @@ -402,7 +436,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } fqreq := req.String() - if reqpkg, ok := pkgs[req.Package]; !ok || reqpkg.Slices[req.Slice] == nil { + if reqpkg, ok := pkgs[req.MapKey()]; !ok || reqpkg.Slices[req.Slice] == nil { return nil, fmt.Errorf("%s requires %s, but slice is missing", fqslice, fqreq) } predecessors = append(predecessors, fqreq) @@ -417,9 +451,11 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, if len(names) > 1 { return nil, fmt.Errorf("essential loop detected: %s", strings.Join(names, ", ")) } - name := names[0] - dot := strings.IndexByte(name, '_') - order = append(order, SliceKey{name[:dot], name[dot+1:]}) + key, err := ParseSliceKey(names[0]) + if err != nil { + return nil, fmt.Errorf("internal error: cannot parse slice key %q", names[0]) + } + order = append(order, key) } return order, nil @@ -467,9 +503,6 @@ func readSlices(release *Release, baseDir, dirName string) error { pkgName := match[1] pkgPath := filepath.Join(dirName, entry.Name()) - if pkg, ok := release.Packages[pkgName]; ok { - return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, pkg.Path, stripBase(baseDir, pkgPath)) - } data, err := os.ReadFile(pkgPath) if err != nil { // Errors from package os generally include the path. @@ -481,7 +514,20 @@ func readSlices(release *Release, baseDir, dirName string) error { return err } - release.Packages[pkg.Name] = pkg + // Derive the package Kind from its store reference. + kind := pkgKind(release, pkg) + + // Set Kind on all slices in the package. + for _, slice := range pkg.Slices { + slice.Kind = kind + } + + // Use the qualified key for the Packages map. + mapKey := pkgMapKey(pkg.Name, kind) + if existing, ok := release.Packages[mapKey]; ok { + return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, existing.Path, stripBase(baseDir, pkgPath)) + } + release.Packages[mapKey] = pkg } return nil } @@ -514,7 +560,7 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error } selection.Slices = make([]*Slice, len(sorted)) for i, key := range sorted { - selection.Slices[i] = release.Packages[key.Package].Slices[key.Slice] + selection.Slices[i] = release.Packages[key.MapKey()].Slices[key.Slice] } for _, new := range selection.Slices { @@ -546,32 +592,32 @@ type preferKey struct { func (r *Release) prefers() (map[preferKey]string, error) { prefers := make(map[preferKey]string) - for _, pkg := range r.Packages { + for mapKey, pkg := range r.Packages { for _, slice := range pkg.Slices { for path, info := range slice.Contents { if info.Prefer != "" { if _, ok := r.Packages[info.Prefer]; !ok { return nil, fmt.Errorf("slice %s path %s 'prefer' refers to undefined package %q", slice, path, info.Prefer) } - tkey := preferKey{preferTarget, path, pkg.Name} + tkey := preferKey{preferTarget, path, mapKey} skey := preferKey{preferSource, path, info.Prefer} if target, ok := prefers[tkey]; ok { if target != info.Prefer { pkg1, pkg2 := sortPair(target, info.Prefer) return nil, fmt.Errorf("package %q has conflicting prefers for %s: %s != %s", - pkg.Name, path, pkg1, pkg2) + mapKey, path, pkg1, pkg2) } } else if source, ok := prefers[skey]; ok { - if source != pkg.Name { - pkg1, pkg2 := sortPair(source, pkg.Name) + if source != mapKey { + pkg1, pkg2 := sortPair(source, mapKey) return nil, fmt.Errorf("packages %q and %q cannot both prefer %q for %s", pkg1, pkg2, info.Prefer, path) } } else { prefers[tkey] = info.Prefer - prefers[skey] = pkg.Name + prefers[skey] = mapKey // Sample package that requires this path to be in a prefer relationship. - prefers[preferKey{preferSource, path, ""}] = pkg.Name + prefers[preferKey{preferSource, path, ""}] = mapKey } } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 8328651a7..a3b48b872 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -150,7 +150,7 @@ var setupTests = []setupTest{{ "/file/path2": {Kind: "copy", Info: "/other/path"}, "/file/path3": {Kind: "symlink", Info: "/other/path"}, "/file/path4": {Kind: "text", Info: "content", Until: "mutate"}, - "/file/path5": {Kind: "copy", Mode: 0755, Mutable: true}, + "/file/path5": {Kind: "copy", Mode: 0o755, Mutable: true}, "/file/path6/": {Kind: "dir"}, }, }, @@ -158,7 +158,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "myslice1"}: {}, + {Package: "mypkg", Slice: "myslice1"}: {}, }, Contents: map[string]setup.PathInfo{ "/another/path": {Kind: "copy"}, @@ -421,7 +421,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{"mypkg1", "myslice1"}}, + selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -444,7 +444,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{"mypkg2", "myslice2"}}, + selslices: []setup.SliceKey{{Package: "mypkg2", Slice: "myslice2"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -453,7 +453,7 @@ var setupTests = []setupTest{{ Package: "mypkg2", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg1", "myslice1"}: {}, + {Package: "mypkg1", Slice: "myslice1"}: {}, }, }}, }, @@ -483,7 +483,7 @@ var setupTests = []setupTest{{ /path3: {symlink: /link} `, }, - selslices: []setup.SliceKey{{"mypkg1", "myslice1"}, {"mypkg1", "myslice2"}, {"mypkg2", "myslice1"}}, + selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}, {Package: "mypkg1", Slice: "myslice2"}, {Package: "mypkg2", Slice: "myslice1"}}, }, { summary: "Conflicting paths across slices", input: map[string]string{ @@ -1472,7 +1472,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, }, }, "slice2": { @@ -1483,16 +1483,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "slice2"}: {}, - {"mypkg", "slice1"}: {}, - {"mypkg", "slice4"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice1"}: {}, + {Package: "mypkg", Slice: "slice4"}: {}, }, }, "slice4": { Package: "mypkg", Name: "slice4", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, }, }, }, @@ -1545,16 +1545,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"myotherpkg", "slice2"}: {}, - {"mypkg", "slice2"}: {}, - {"myotherpkg", "slice1"}: {}, + {Package: "myotherpkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, + {Package: "myotherpkg", Slice: "slice1"}: {}, }, }, "slice2": { Package: "mypkg", Name: "slice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"myotherpkg", "slice2"}: {}, + {Package: "myotherpkg", Slice: "slice2"}: {}, }, }, }, @@ -1739,7 +1739,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{"mypkg", "myslice"}}, + selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg", @@ -1792,7 +1792,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{"mypkg", "myslice"}}, + selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, selerror: `slice mypkg_myslice has invalid 'generate' for path /dir/\*\*: "foo"`, }, { summary: "Paths with generate: manifest must have trailing /**", @@ -2407,10 +2407,10 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer'", selslices: []setup.SliceKey{ - {"mypkg1", "myslice1"}, - {"mypkg1", "myslice2"}, - {"mypkg2", "myslice1"}, - {"mypkg3", "myslice1"}, + {Package: "mypkg1", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice2"}, + {Package: "mypkg2", Slice: "myslice1"}, + {Package: "mypkg3", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -2514,9 +2514,9 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer' depends on selection", selslices: []setup.SliceKey{ - {"mypkg1", "myslice1"}, - {"mypkg1", "myslice2"}, - {"mypkg2", "myslice1"}, + {Package: "mypkg1", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice2"}, + {Package: "mypkg2", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -3664,24 +3664,24 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "myslice2"}: {Arch: []string{"amd64"}}, - {"mypkg", "myslice3"}: {Arch: []string{"amd64", "arm64"}}, - {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, - {"mypkg", "myslice5"}: {Arch: nil}, + {Package: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, + {Package: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice5"}: {Arch: nil}, }, }, "myslice2": { Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice3": { Package: "mypkg", Name: "myslice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice4": { @@ -3693,7 +3693,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice5", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, }, @@ -3947,7 +3947,7 @@ var setupTests = []setupTest{{ }, }, Packages: map[string]*setup.Package{ - "mypkg": { + "bin/mypkg": { Name: "mypkg", Path: "slices/bin/mypkg.yaml", Store: "bin", @@ -4085,6 +4085,115 @@ var setupTests = []setupTest{{ `, }, relerror: `chisel.yaml: store "bin" missing default-prefix field`, +}, { + summary: "Same-named package in archive and store", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, + "slices/curl.yaml": ` + package: curl + slices: + libs: + contents: + /usr/lib/libcurl.so: + bins: + contents: + /usr/bin/curl: + `, + "slices/bin/curl.yaml": ` + package: curl + store: bin + default-track: "3.0" + slices: + bins: + contents: + /usr/bin/curl-bin: + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "26.10", + Suites: []string{"stonking"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: setup.StoreBin, + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "curl": { + Name: "curl", + Path: "slices/curl.yaml", + Slices: map[string]*setup.Slice{ + "libs": { + Package: "curl", + Kind: "", + Name: "libs", + Contents: map[string]setup.PathInfo{ + "/usr/lib/libcurl.so": {Kind: setup.CopyPath}, + }, + }, + "bins": { + Package: "curl", + Kind: "", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl": {Kind: setup.CopyPath}, + }, + }, + }, + }, + "bin/curl": { + Name: "curl", + Path: "slices/bin/curl.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{ + "bins": { + Package: "curl", + Kind: "bin", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl-bin": {Kind: setup.CopyPath}, + }, + }, + }, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, }} func (s *S) TestParseRelease(c *C) { @@ -4589,7 +4698,7 @@ func (s *S) TestSelectEmptyArch(c *C) { release, err := setup.ReadRelease(dir) c.Assert(err, IsNil) - selslice := []setup.SliceKey{{"mypkg", "myslice"}} + selslice := []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}} selection, err := setup.Select(release, selslice, "") c.Assert(err, IsNil) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index b9b6d2fd9..36335b190 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -496,7 +496,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkgName, Slice: sliceName}) } } } else { @@ -508,10 +508,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgName, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgName, Slice: sliceName}) } } } @@ -544,7 +544,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkgName, sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgName, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ Package: pkgName, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 86562b8b6..8fefa4b5b 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -95,10 +95,10 @@ func Run(options *RunOptions) error { return err } - // Build a map from package name to architecture. + // Build a map from package map key to architecture. pkgArch := make(map[string]string) - for pkg, a := range pkgArchive { - pkgArch[pkg] = a.Options().Arch + for pkgKey, a := range pkgArchive { + pkgArch[pkgKey] = a.Options().Arch } // TODO Handle packages coming from a store as well when we support them. @@ -110,12 +110,13 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - extractPackage := extract[slice.Package] + sliceKey := slice.MapKey() + extractPackage := extract[sliceKey] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) - extract[slice.Package] = extractPackage + extract[sliceKey] = extractPackage } - arch := pkgArch[slice.Package] + arch := pkgArch[sliceKey] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -156,15 +157,16 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - if packages[slice.Package] != nil { + sliceKey := slice.MapKey() + if packages[sliceKey] != nil { continue } - reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) + reader, info, err := pkgArchive[sliceKey].Fetch(slice.Package) if err != nil { return err } defer reader.Close() - packages[slice.Package] = reader + packages[sliceKey] = reader pkgInfos = append(pkgInfos, info) } @@ -241,18 +243,19 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - reader := packages[slice.Package] + sliceKey := slice.MapKey() + reader := packages[sliceKey] if reader == nil { continue } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, - Extract: extract[slice.Package], + Extract: extract[sliceKey], TargetDir: targetDir, Create: create, }) reader.Close() - packages[slice.Package] = nil + packages[sliceKey] = nil if err != nil { return err } @@ -276,7 +279,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.Package] + arch := pkgArch[slice.MapKey()] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -515,10 +518,11 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - if _, ok := pkgArchive[s.Package]; ok { + sliceKey := s.MapKey() + if _, ok := pkgArchive[sliceKey]; ok { continue } - pkg := selection.Release.Packages[s.Package] + pkg := selection.Release.Packages[sliceKey] if pkg.Store != "" { // Packages coming from a store are not fetched from an archive, // so we skip them here. @@ -545,7 +549,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgArchive[pkg.Name] = chosen + pkgArchive[sliceKey] = chosen } return pkgArchive, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8d3cf775c..8935014fb 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -78,7 +78,7 @@ var testPackageCopyrightEntries = []testutil.TarEntry{ var slicerTests = []slicerTest{{ summary: "Basic slicing", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -111,7 +111,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Glob extraction", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -133,7 +133,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -153,7 +153,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new nested file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -174,7 +174,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new directory under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -194,7 +194,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file using glob and preserve parent directory permissions", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -218,7 +218,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Conditional architecture", arch: "amd64", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -249,7 +249,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Copyright is not installed implicitly", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", // Add the copyright entries to the package. @@ -274,8 +274,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages", slices: []setup.SliceKey{ - {"test-package", "myslice"}, - {"other-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -317,9 +317,9 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {"a-implicit-parent", "myslice"}, - {"b-explicit-dir", "myslice"}, - {"c-implicit-parent", "myslice"}}, + {Package: "a-implicit-parent", Slice: "myslice"}, + {Package: "b-explicit-dir", Slice: "myslice"}, + {Package: "c-implicit-parent", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "a-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -376,8 +376,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid same file in two slices in different packages", slices: []setup.SliceKey{ - {"test-package", "myslice"}, - {"other-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -409,7 +409,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: write a file", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -430,7 +430,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: read a file", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -456,7 +456,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove file after mutate", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -480,7 +480,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove wildcard after mutate", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -498,7 +498,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -519,7 +519,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: writing same contents to existing file does not set the final hash in report", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -540,7 +540,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: cannot write non-mutable files", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -555,7 +555,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to unlisted file", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -569,7 +569,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -584,7 +584,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/`, }, { summary: "Script: cannot read unlisted content", - slices: []setup.SliceKey{{"test-package", "myslice2"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -600,7 +600,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice2: cannot read file which is not selected: /dir/text-file`, }, { summary: "Script: can read globbed content", - slices: []setup.SliceKey{{"test-package", "myslice1"}, {"test-package", "myslice2"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice1"}, {Package: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -615,7 +615,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative content root directory must not error", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -636,7 +636,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list parent directories of normal paths", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -655,7 +655,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list unselected directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -670,7 +670,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /a/d/`, }, { summary: "Cannot list file path as a directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -685,7 +685,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a directory: /a/b/c`, }, { summary: "Can list parent directories of globs", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -699,7 +699,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list directories not matched by glob", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -714,7 +714,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /other-dir/`, }, { summary: "Duplicate copyright symlink is ignored", - slices: []setup.SliceKey{{"copyright-symlink-openssl", "bins"}}, + slices: []setup.SliceKey{{Package: "copyright-symlink-openssl", Slice: "bins"}}, pkgs: []*testutil.TestPackage{{ Name: "copyright-symlink-openssl", Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), @@ -746,7 +746,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list unclean directory paths", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -765,7 +765,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot read directories", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -780,7 +780,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a file: /x/y`, }, { summary: "Multiple archives with priority", - slices: []setup.SliceKey{{"test-package", "myslice"}, {"other-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}, {Package: "other-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -864,7 +864,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive bypasses higher priority", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -932,7 +932,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive does not have the package", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -978,7 +978,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "No archives have the package", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{}, release: map[string]string{ "chisel.yaml": ` @@ -1015,7 +1015,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archives are ignored when not explicitly pinned in package", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1052,7 +1052,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archive explicitly pinned in package", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -1102,8 +1102,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Multiple slices of same package", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1140,8 +1140,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Same glob in several entries with until:mutate and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1186,8 +1186,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping globs, until:mutate and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice2"}, - {"test-package", "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1232,8 +1232,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on entry and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1278,8 +1278,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on glob and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1307,8 +1307,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on both and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1331,8 +1331,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Content not created in packages with until:mutate on one and reading from script", slices: []setup.SliceKey{ - {"test-package", "myslice1"}, - {"test-package", "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1355,8 +1355,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, both are recorded", slices: []setup.SliceKey{ - {"test-package", "myslice"}, - {"other-package", "myslice"}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1392,7 +1392,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Two packages, only one is selected and recorded", slices: []setup.SliceKey{ - {"test-package", "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1426,7 +1426,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative paths are properly trimmed during extraction", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1457,7 +1457,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Producing a manifest is not mandatory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, hackopt: func(c *C, opts *slicer.RunOptions) { // Remove the manifest slice that the tests add automatically. var index int @@ -1479,7 +1479,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "No valid archives defined due to invalid pro value", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "chisel.yaml": ` format: v1 @@ -1506,8 +1506,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid hard link in two slices in the same package", slices: []setup.SliceKey{ - {"test-package", "slice1"}, - {"test-package", "slice2"}}, + {Package: "test-package", Slice: "slice1"}, + {Package: "test-package", Slice: "slice2"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1540,7 +1540,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link entries can be extracted without extracting the regular file", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1570,7 +1570,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifier for different groups", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1605,7 +1605,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Single hard link entry can be extracted without regular file and no hard links are created", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1632,7 +1632,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link to symlink does not follow symlink", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1664,8 +1664,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifiers are unique across packages", slices: []setup.SliceKey{ - {"test-package1", "myslice"}, - {"test-package2", "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1715,7 +1715,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Mutations for hard links are forbidden", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1740,7 +1740,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard links can be marked as mutable, but not mutated", slices: []setup.SliceKey{ - {"test-package", "myslice"}}, + {Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1769,7 +1769,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Hard links cannot escape the target directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1789,7 +1789,7 @@ var slicerTests = []slicerTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1810,8 +1810,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ - {"test-package1", "myslice"}, - {"test-package2", "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1871,8 +1871,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Warning when implicit parent directories conflict", slices: []setup.SliceKey{ - {"test-package1", "myslice"}, - {"test-package2", "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1910,7 +1910,7 @@ var slicerTests = []slicerTest{{ logOutput: `(?s).*Warning: Path "/parent/" has diverging modes in different packages\. Please report\..*`, }, { summary: "Arch specific slice is not installed when it does not match requested arch", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, arch: "amd64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1929,7 +1929,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Arch specific slice is installed when it matches requested arch", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, arch: "arm64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1953,7 +1953,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Transitive essential", - slices: []setup.SliceKey{{"test-package", "first"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "first"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package From b68da534098c0a4029327ec83b42424fa7d465ec Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 1 Jun 2026 16:26:29 +0200 Subject: [PATCH 06/61] style: rename to avoid confusion Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 2 +- internal/apacheutil/util.go | 4 +-- internal/apacheutil/util_test.go | 8 +++--- internal/setup/setup.go | 42 ++++++++++++++++---------------- internal/slicer/slicer.go | 34 +++++++++++++------------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index 228d8f1b0..534dc6df6 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -83,7 +83,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* allPkgSlices := make(map[string]bool) sliceExists := func(key setup.SliceKey) bool { - pkg, ok := release.Packages[key.MapKey()] + pkg, ok := release.Packages[key.PkgKey()] if !ok { return false } diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go index 2ad9a364b..37e1e4e4a 100644 --- a/internal/apacheutil/util.go +++ b/internal/apacheutil/util.go @@ -20,8 +20,8 @@ func (s SliceKey) String() string { return s.Package + "_" + s.Slice } -// MapKey returns the qualified package key used for Release.Packages lookups. -func (s SliceKey) MapKey() string { +// PkgKey returns the qualified package key used for Release.Packages lookups. +func (s SliceKey) PkgKey() string { if s.Kind != "" { return s.Kind + "/" + s.Package } diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go index 752af2ebf..db4ef7cb7 100644 --- a/internal/apacheutil/util_test.go +++ b/internal/apacheutil/util_test.go @@ -124,7 +124,7 @@ func (s *S) TestSliceKeyString(c *C) { } } -var sliceKeyMapKeyTests = []struct { +var sliceKeyPkgKeyTests = []struct { key apacheutil.SliceKey expected string }{{ @@ -141,8 +141,8 @@ var sliceKeyMapKeyTests = []struct { expected: "curl", }} -func (s *S) TestSliceKeyMapKey(c *C) { - for _, test := range sliceKeyMapKeyTests { - c.Assert(test.key.MapKey(), Equals, test.expected) +func (s *S) TestSliceKeyPkgKey(c *C) { + for _, test := range sliceKeyPkgKeyTests { + c.Assert(test.key.PkgKey(), Equals, test.expected) } } diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 4794351ca..2bdb80b77 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -162,9 +162,9 @@ func pkgKind(release *Release, pkg *Package) string { return string(store.Kind) } -// pkgMapKey returns the qualified key for a package used as the +// pkgKey returns the qualified key for a package used as the // Release.Packages map key. -func pkgMapKey(name, kind string) string { +func pkgKey(name, kind string) string { if kind != "" { return kind + "/" + name } @@ -175,8 +175,8 @@ func (s *Slice) String() string { return SliceKey{Package: s.Package, Kind: s.Kind, Slice: s.Name}.String() } -func (s *Slice) MapKey() string { - return SliceKey{Package: s.Package, Kind: s.Kind}.MapKey() +func (s *Slice) PkgKey() string { + return SliceKey{Package: s.Package, Kind: s.Kind}.PkgKey() } // Selection holds the required configuration to create a Build for a selection @@ -204,18 +204,18 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sliceKey := pkgMapKey(slice.Package, slice.Kind) + sPkgKey := pkgKey(slice.Package, slice.Kind) old, ok := pathPreferredPkg[path] if !ok { - pathPreferredPkg[path] = s.Release.Packages[sliceKey] + pathPreferredPkg[path] = s.Release.Packages[sPkgKey] continue } - oldKey := pkgMapKey(old.Name, pkgKind(s.Release, old)) - if oldKey == sliceKey { + oldKey := pkgKey(old.Name, pkgKind(s.Release, old)) + if oldKey == sPkgKey { // Skip if the package was already recorded. continue } - preferred, err := preferredPathPackage(path, oldKey, sliceKey, prefers) + preferred, err := preferredPathPackage(path, oldKey, sPkgKey, prefers) if err != nil { // Note: we have checked above that the path has prefers and // they are different packages so the error cannot be @@ -271,11 +271,11 @@ func (r *Release) validate() error { kind := pkgKind(r, pkg) for _, new := range pkg.Slices { keys = append(keys, SliceKey{Package: pkg.Name, Kind: kind, Slice: new.Name}) - newKey := pkgMapKey(new.Package, new.Kind) + newKey := pkgKey(new.Package, new.Kind) for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldKey := pkgMapKey(old.Package, old.Kind) + oldKey := pkgKey(old.Package, old.Kind) if newKey != oldKey { _, err := preferredPathPackage(newPath, newKey, oldKey, prefers) if err == nil { @@ -310,7 +310,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if pkgMapKey(slice.Package, slice.Kind) == skey.pkg { + if pkgKey(slice.Package, slice.Kind) == skey.pkg { found = true break } @@ -335,8 +335,8 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newKey := pkgMapKey(new.Package, new.Kind) - oldKey := pkgMapKey(old.Package, old.Kind) + newKey := pkgKey(new.Package, new.Kind) + oldKey := pkgKey(old.Package, old.Kind) if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { if newKey == oldKey { continue @@ -409,7 +409,7 @@ func (r *Release) validate() error { func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, error) { // Preprocess the list to improve error messages. for _, key := range keys { - if pkg, ok := pkgs[key.MapKey()]; !ok { + if pkg, ok := pkgs[key.PkgKey()]; !ok { return nil, fmt.Errorf("slices of package %q not found", key.Package) } else if _, ok := pkg.Slices[key.Slice]; !ok { return nil, fmt.Errorf("slice %s not found", key) @@ -427,7 +427,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } seen[key] = true - pkg := pkgs[key.MapKey()] + pkg := pkgs[key.PkgKey()] slice := pkg.Slices[key.Slice] fqslice := slice.String() predecessors := successors[fqslice] @@ -436,7 +436,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } fqreq := req.String() - if reqpkg, ok := pkgs[req.MapKey()]; !ok || reqpkg.Slices[req.Slice] == nil { + if reqpkg, ok := pkgs[req.PkgKey()]; !ok || reqpkg.Slices[req.Slice] == nil { return nil, fmt.Errorf("%s requires %s, but slice is missing", fqslice, fqreq) } predecessors = append(predecessors, fqreq) @@ -523,11 +523,11 @@ func readSlices(release *Release, baseDir, dirName string) error { } // Use the qualified key for the Packages map. - mapKey := pkgMapKey(pkg.Name, kind) - if existing, ok := release.Packages[mapKey]; ok { + pkgMapKey := pkgKey(pkg.Name, kind) + if existing, ok := release.Packages[pkgMapKey]; ok { return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, existing.Path, stripBase(baseDir, pkgPath)) } - release.Packages[mapKey] = pkg + release.Packages[pkgMapKey] = pkg } return nil } @@ -560,7 +560,7 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error } selection.Slices = make([]*Slice, len(sorted)) for i, key := range sorted { - selection.Slices[i] = release.Packages[key.MapKey()].Slices[key.Slice] + selection.Slices[i] = release.Packages[key.PkgKey()].Slices[key.Slice] } for _, new := range selection.Slices { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 8fefa4b5b..2a7711694 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -110,13 +110,13 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - sliceKey := slice.MapKey() - extractPackage := extract[sliceKey] + pkgKey := slice.PkgKey() + extractPackage := extract[pkgKey] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) - extract[sliceKey] = extractPackage + extract[pkgKey] = extractPackage } - arch := pkgArch[sliceKey] + arch := pkgArch[pkgKey] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -157,16 +157,16 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - sliceKey := slice.MapKey() - if packages[sliceKey] != nil { + pkgKey := slice.PkgKey() + if packages[pkgKey] != nil { continue } - reader, info, err := pkgArchive[sliceKey].Fetch(slice.Package) + reader, info, err := pkgArchive[pkgKey].Fetch(slice.Package) if err != nil { return err } defer reader.Close() - packages[sliceKey] = reader + packages[pkgKey] = reader pkgInfos = append(pkgInfos, info) } @@ -243,19 +243,19 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - sliceKey := slice.MapKey() - reader := packages[sliceKey] + pkgKey := slice.PkgKey() + reader := packages[pkgKey] if reader == nil { continue } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, - Extract: extract[sliceKey], + Extract: extract[pkgKey], TargetDir: targetDir, Create: create, }) reader.Close() - packages[sliceKey] = nil + packages[pkgKey] = nil if err != nil { return err } @@ -279,7 +279,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.MapKey()] + arch := pkgArch[slice.PkgKey()] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -518,11 +518,11 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - sliceKey := s.MapKey() - if _, ok := pkgArchive[sliceKey]; ok { + pkgKey := s.PkgKey() + if _, ok := pkgArchive[pkgKey]; ok { continue } - pkg := selection.Release.Packages[sliceKey] + pkg := selection.Release.Packages[pkgKey] if pkg.Store != "" { // Packages coming from a store are not fetched from an archive, // so we skip them here. @@ -549,7 +549,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgArchive[sliceKey] = chosen + pkgArchive[pkgKey] = chosen } return pkgArchive, nil } From 84948766e91259551a3cc98820b6b5b2eb984b81 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 13:36:26 +0200 Subject: [PATCH 07/61] fix: start renaming to realname Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 4 +- internal/apacheutil/util.go | 24 +- internal/apacheutil/util_test.go | 59 ++--- internal/manifestutil/manifestutil.go | 6 +- internal/setup/setup.go | 106 ++++----- internal/setup/setup_test.go | 130 ++++------ internal/setup/yaml.go | 28 +-- internal/slicer/slicer.go | 49 ++-- internal/slicer/slicer_test.go | 326 +++++++++++++------------- 9 files changed, 343 insertions(+), 389 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index 534dc6df6..4a08d9693 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -83,7 +83,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* allPkgSlices := make(map[string]bool) sliceExists := func(key setup.SliceKey) bool { - pkg, ok := release.Packages[key.PkgKey()] + pkg, ok := release.Packages[key.RealName] if !ok { return false } @@ -98,7 +98,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* notFound = append(notFound, query) continue } - pkg, slice = key.Package, key.Slice + pkg, slice = key.RealName, key.Slice } else { if _, ok := release.Packages[query]; !ok { notFound = append(notFound, query) diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go index 37e1e4e4a..8beaf2bef 100644 --- a/internal/apacheutil/util.go +++ b/internal/apacheutil/util.go @@ -8,24 +8,12 @@ import ( ) type SliceKey struct { - Package string - Kind string - Slice string + RealName string + Slice string } func (s SliceKey) String() string { - if s.Kind != "" { - return s.Kind + "/" + s.Package + "_" + s.Slice - } - return s.Package + "_" + s.Slice -} - -// PkgKey returns the qualified package key used for Release.Packages lookups. -func (s SliceKey) PkgKey() string { - if s.Kind != "" { - return s.Kind + "/" + s.Package - } - return s.Package + return s.RealName + "_" + s.Slice } // FnameExp matches the slice definition file basename. @@ -34,13 +22,13 @@ var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) // SnameExp matches only the slice name, without the leading package name. var SnameExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})$`) -// knameExp matches the slice full name in pkg_slice or kind/pkg_slice format. -var knameExp = regexp.MustCompile(`^(?:([a-z0-9](?:-?[.a-z0-9+]){0,})/)?([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) +// knameExp matches the slice full name in pkg_slice format. +var knameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})_([a-z](?:-?[a-z0-9]){2,})$`) func ParseSliceKey(sliceKey string) (SliceKey, error) { match := knameExp.FindStringSubmatch(sliceKey) if match == nil { return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) } - return SliceKey{Package: match[2], Kind: match[1], Slice: match[3]}, nil + return SliceKey{RealName: match[1], Slice: match[2]}, nil } diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go index db4ef7cb7..4acab2cb5 100644 --- a/internal/apacheutil/util_test.go +++ b/internal/apacheutil/util_test.go @@ -14,34 +14,34 @@ var sliceKeyTests = []struct { err string }{{ input: "foo_bar", - expected: apacheutil.SliceKey{Package: "foo", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "foo", Slice: "bar"}, }, { input: "fo_bar", - expected: apacheutil.SliceKey{Package: "fo", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "fo", Slice: "bar"}, }, { input: "1234_bar", - expected: apacheutil.SliceKey{Package: "1234", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "1234", Slice: "bar"}, }, { input: "foo1.1-2-3_bar", - expected: apacheutil.SliceKey{Package: "foo1.1-2-3", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "foo1.1-2-3", Slice: "bar"}, }, { input: "foo-pkg_dashed-slice-name", - expected: apacheutil.SliceKey{Package: "foo-pkg", Slice: "dashed-slice-name"}, + expected: apacheutil.SliceKey{RealName: "foo-pkg", Slice: "dashed-slice-name"}, }, { input: "foo+_bar", - expected: apacheutil.SliceKey{Package: "foo+", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "foo+", Slice: "bar"}, }, { input: "foo_slice123", - expected: apacheutil.SliceKey{Package: "foo", Slice: "slice123"}, + expected: apacheutil.SliceKey{RealName: "foo", Slice: "slice123"}, }, { input: "g++_bins", - expected: apacheutil.SliceKey{Package: "g++", Slice: "bins"}, + expected: apacheutil.SliceKey{RealName: "g++", Slice: "bins"}, }, { input: "a+_bar", - expected: apacheutil.SliceKey{Package: "a+", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "a+", Slice: "bar"}, }, { input: "a._bar", - expected: apacheutil.SliceKey{Package: "a.", Slice: "bar"}, + expected: apacheutil.SliceKey{RealName: "a.", Slice: "bar"}, }, { input: "foo_ba", err: `invalid slice reference: "foo_ba"`, @@ -78,15 +78,6 @@ var sliceKeyTests = []struct { }, { input: "..._bar", err: `invalid slice reference: "\.\.\._bar"`, -}, { - input: "bin/curl_bins", - expected: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, -}, { - input: "bin/g++_bins", - expected: apacheutil.SliceKey{Package: "g++", Kind: "bin", Slice: "bins"}, -}, { - input: "b/curl_bins", - expected: apacheutil.SliceKey{Package: "curl", Kind: "b", Slice: "bins"}, }, { input: "white space_no-whitespace", err: `invalid slice reference: "white space_no-whitespace"`, @@ -108,14 +99,14 @@ var sliceKeyStringTests = []struct { key apacheutil.SliceKey expected string }{{ - key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, + key: apacheutil.SliceKey{RealName: "curl", Slice: "bins"}, expected: "curl_bins", }, { - key: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, - expected: "bin/curl_bins", + key: apacheutil.SliceKey{RealName: "bin-curl", Slice: "bins"}, + expected: "bin-curl_bins", }, { - key: apacheutil.SliceKey{Package: "g++", Kind: "bin", Slice: "bins"}, - expected: "bin/g++_bins", + key: apacheutil.SliceKey{RealName: "bin-g++", Slice: "bins"}, + expected: "bin-g++_bins", }} func (s *S) TestSliceKeyString(c *C) { @@ -124,25 +115,25 @@ func (s *S) TestSliceKeyString(c *C) { } } -var sliceKeyPkgKeyTests = []struct { +var sliceKeyRealNameTests = []struct { key apacheutil.SliceKey expected string }{{ - key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, + key: apacheutil.SliceKey{RealName: "curl", Slice: "bins"}, expected: "curl", }, { - key: apacheutil.SliceKey{Package: "curl", Kind: "bin", Slice: "bins"}, - expected: "bin/curl", + key: apacheutil.SliceKey{RealName: "bin-curl", Slice: "bins"}, + expected: "bin-curl", }, { - key: apacheutil.SliceKey{Package: "curl", Kind: "bin"}, - expected: "bin/curl", + key: apacheutil.SliceKey{RealName: "bin-curl"}, + expected: "bin-curl", }, { - key: apacheutil.SliceKey{Package: "curl"}, + key: apacheutil.SliceKey{RealName: "curl"}, expected: "curl", }} -func (s *S) TestSliceKeyPkgKey(c *C) { - for _, test := range sliceKeyPkgKeyTests { - c.Assert(test.key.PkgKey(), Equals, test.expected) +func (s *S) TestSliceKeyRealName(c *C) { + for _, test := range sliceKeyRealNameTests { + c.Assert(test.key.RealName, Equals, test.expected) } } diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b054022..edc3616b1 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -134,7 +134,7 @@ func manifestAddReport(dbw *jsonwall.DBWriter, report *Report) error { func unixPerm(mode fs.FileMode) (perm uint32) { perm = uint32(mode.Perm()) if mode&fs.ModeSticky != 0 { - perm |= 01000 + perm |= 0o1000 } return perm } @@ -291,8 +291,8 @@ func Validate(mfest *manifest.Manifest) (err error) { if err != nil { return err } - if !pkgExist[sk.Package] { - return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.Package) + if !pkgExist[sk.RealName] { + return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.RealName) } sliceExist[slice.Name] = true return nil diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 2bdb80b77..97c502366 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,15 +16,10 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// StoreKind identifies the backend type of a store. -type StoreKind string - -const StoreBin StoreKind = "bin" - // Store is the location from which binary packages are obtained via a store API. type Store struct { Name string - Kind StoreKind + Kind string Version string DefaultPrefix string } @@ -75,13 +70,13 @@ type Package struct { // Slice holds the details about a package slice. type Slice struct { - Package string - Kind string - Name string - Hint string - Essential map[SliceKey]EssentialInfo - Contents map[string]PathInfo - Scripts SliceScripts + Package string + DefaultPrefix string + Name string + Hint string + Essential map[SliceKey]EssentialInfo + Contents map[string]PathInfo + Scripts SliceScripts } type EssentialInfo struct { @@ -150,8 +145,8 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { return apacheutil.ParseSliceKey(sliceKey) } -// pkgKind returns the Kind value for a package based on its store reference. -func pkgKind(release *Release, pkg *Package) string { +// pkgDefaultPrefix returns the DefaultPrefix for a package based on its store reference. +func pkgDefaultPrefix(release *Release, pkg *Package) string { if pkg.Store == "" { return "" } @@ -159,24 +154,21 @@ func pkgKind(release *Release, pkg *Package) string { if store == nil { return "" } - return string(store.Kind) + return store.DefaultPrefix } -// pkgKey returns the qualified key for a package used as the +// realName returns the real name for a package used as the // Release.Packages map key. -func pkgKey(name, kind string) string { - if kind != "" { - return kind + "/" + name - } - return name +func realName(name, defaultPrefix string) string { + return defaultPrefix + name } func (s *Slice) String() string { - return SliceKey{Package: s.Package, Kind: s.Kind, Slice: s.Name}.String() + return SliceKey{RealName: s.RealName(), Slice: s.Name}.String() } -func (s *Slice) PkgKey() string { - return SliceKey{Package: s.Package, Kind: s.Kind}.PkgKey() +func (s *Slice) RealName() string { + return s.DefaultPrefix + s.Package } // Selection holds the required configuration to create a Build for a selection @@ -204,18 +196,18 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sPkgKey := pkgKey(slice.Package, slice.Kind) + sRealName := slice.RealName() old, ok := pathPreferredPkg[path] if !ok { - pathPreferredPkg[path] = s.Release.Packages[sPkgKey] + pathPreferredPkg[path] = s.Release.Packages[sRealName] continue } - oldKey := pkgKey(old.Name, pkgKind(s.Release, old)) - if oldKey == sPkgKey { + oldRealName := realName(old.Name, pkgDefaultPrefix(s.Release, old)) + if oldRealName == sRealName { // Skip if the package was already recorded. continue } - preferred, err := preferredPathPackage(path, oldKey, sPkgKey, prefers) + preferred, err := preferredPathPackage(path, oldRealName, sRealName, prefers) if err != nil { // Note: we have checked above that the path has prefers and // they are different packages so the error cannot be @@ -268,16 +260,16 @@ func (r *Release) validate() error { // cannot validate that they are the same without downloading the package. paths := make(map[string][]*Slice) for _, pkg := range r.Packages { - kind := pkgKind(r, pkg) + prefix := pkgDefaultPrefix(r, pkg) for _, new := range pkg.Slices { - keys = append(keys, SliceKey{Package: pkg.Name, Kind: kind, Slice: new.Name}) - newKey := pkgKey(new.Package, new.Kind) + keys = append(keys, SliceKey{RealName: realName(pkg.Name, prefix), Slice: new.Name}) + newRealName := new.RealName() for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldKey := pkgKey(old.Package, old.Kind) - if newKey != oldKey { - _, err := preferredPathPackage(newPath, newKey, oldKey, prefers) + oldRealName := old.RealName() + if newRealName != oldRealName { + _, err := preferredPathPackage(newPath, newRealName, oldRealName, prefers) if err == nil { continue } else if err != errPreferNone { @@ -286,8 +278,8 @@ func (r *Release) validate() error { } oldInfo := old.Contents[newPath] - if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && newKey != oldKey { - if oldKey > newKey || oldKey == newKey && old.Name > new.Name { + if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && newRealName != oldRealName { + if oldRealName > newRealName || oldRealName == newRealName && old.Name > new.Name { old, new = new, old } return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath) @@ -310,7 +302,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if pkgKey(slice.Package, slice.Kind) == skey.pkg { + if slice.RealName() == skey.pkg { found = true break } @@ -335,16 +327,16 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newKey := pkgKey(new.Package, new.Kind) - oldKey := pkgKey(old.Package, old.Kind) + newRealName := new.RealName() + oldRealName := old.RealName() if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { - if newKey == oldKey { + if newRealName == oldRealName { continue } } if strdist.GlobPath(newPath, oldPath) { - if (oldKey > newKey) || (oldKey == newKey && old.Name > new.Name) || - (oldKey == newKey && old.Name == new.Name && oldPath > newPath) { + if (oldRealName > newRealName) || (oldRealName == newRealName && old.Name > new.Name) || + (oldRealName == newRealName && old.Name == new.Name && oldPath > newPath) { old, new = new, old oldPath, newPath = newPath, oldPath } @@ -409,8 +401,8 @@ func (r *Release) validate() error { func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, error) { // Preprocess the list to improve error messages. for _, key := range keys { - if pkg, ok := pkgs[key.PkgKey()]; !ok { - return nil, fmt.Errorf("slices of package %q not found", key.Package) + if pkg, ok := pkgs[key.RealName]; !ok { + return nil, fmt.Errorf("slices of package %q not found", key.RealName) } else if _, ok := pkg.Slices[key.Slice]; !ok { return nil, fmt.Errorf("slice %s not found", key) } @@ -427,7 +419,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } seen[key] = true - pkg := pkgs[key.PkgKey()] + pkg := pkgs[key.RealName] slice := pkg.Slices[key.Slice] fqslice := slice.String() predecessors := successors[fqslice] @@ -436,7 +428,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } fqreq := req.String() - if reqpkg, ok := pkgs[req.PkgKey()]; !ok || reqpkg.Slices[req.Slice] == nil { + if reqpkg, ok := pkgs[req.RealName]; !ok || reqpkg.Slices[req.Slice] == nil { return nil, fmt.Errorf("%s requires %s, but slice is missing", fqslice, fqreq) } predecessors = append(predecessors, fqreq) @@ -514,20 +506,20 @@ func readSlices(release *Release, baseDir, dirName string) error { return err } - // Derive the package Kind from its store reference. - kind := pkgKind(release, pkg) + // Derive the package DefaultPrefix from its store reference. + prefix := pkgDefaultPrefix(release, pkg) - // Set Kind on all slices in the package. + // Set DefaultPrefix on all slices in the package. for _, slice := range pkg.Slices { - slice.Kind = kind + slice.DefaultPrefix = prefix } - // Use the qualified key for the Packages map. - pkgMapKey := pkgKey(pkg.Name, kind) - if existing, ok := release.Packages[pkgMapKey]; ok { + // Use the real name for the Packages map. + pkgRealName := realName(pkg.Name, prefix) + if existing, ok := release.Packages[pkgRealName]; ok { return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, existing.Path, stripBase(baseDir, pkgPath)) } - release.Packages[pkgMapKey] = pkg + release.Packages[pkgRealName] = pkg } return nil } @@ -560,7 +552,7 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error } selection.Slices = make([]*Slice, len(sorted)) for i, key := range sorted { - selection.Slices[i] = release.Packages[key.PkgKey()].Slices[key.Slice] + selection.Slices[i] = release.Packages[key.RealName].Slices[key.Slice] } for _, new := range selection.Slices { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index a3b48b872..e8538213e 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -158,7 +158,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice1"}: {}, + {RealName: "mypkg", Slice: "myslice1"}: {}, }, Contents: map[string]setup.PathInfo{ "/another/path": {Kind: "copy"}, @@ -421,7 +421,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{RealName: "mypkg1", Slice: "myslice1"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -444,7 +444,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{Package: "mypkg2", Slice: "myslice2"}}, + selslices: []setup.SliceKey{{RealName: "mypkg2", Slice: "myslice2"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -453,7 +453,7 @@ var setupTests = []setupTest{{ Package: "mypkg2", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg1", Slice: "myslice1"}: {}, + {RealName: "mypkg1", Slice: "myslice1"}: {}, }, }}, }, @@ -483,7 +483,7 @@ var setupTests = []setupTest{{ /path3: {symlink: /link} `, }, - selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}, {Package: "mypkg1", Slice: "myslice2"}, {Package: "mypkg2", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{RealName: "mypkg1", Slice: "myslice1"}, {RealName: "mypkg1", Slice: "myslice2"}, {RealName: "mypkg2", Slice: "myslice1"}}, }, { summary: "Conflicting paths across slices", input: map[string]string{ @@ -1472,7 +1472,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, + {RealName: "mypkg", Slice: "slice2"}: {}, }, }, "slice2": { @@ -1483,16 +1483,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, - {Package: "mypkg", Slice: "slice1"}: {}, - {Package: "mypkg", Slice: "slice4"}: {}, + {RealName: "mypkg", Slice: "slice2"}: {}, + {RealName: "mypkg", Slice: "slice1"}: {}, + {RealName: "mypkg", Slice: "slice4"}: {}, }, }, "slice4": { Package: "mypkg", Name: "slice4", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, + {RealName: "mypkg", Slice: "slice2"}: {}, }, }, }, @@ -1545,16 +1545,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "myotherpkg", Slice: "slice2"}: {}, - {Package: "mypkg", Slice: "slice2"}: {}, - {Package: "myotherpkg", Slice: "slice1"}: {}, + {RealName: "myotherpkg", Slice: "slice2"}: {}, + {RealName: "mypkg", Slice: "slice2"}: {}, + {RealName: "myotherpkg", Slice: "slice1"}: {}, }, }, "slice2": { Package: "mypkg", Name: "slice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "myotherpkg", Slice: "slice2"}: {}, + {RealName: "myotherpkg", Slice: "slice2"}: {}, }, }, }, @@ -1739,7 +1739,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg", @@ -1792,7 +1792,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}}, selerror: `slice mypkg_myslice has invalid 'generate' for path /dir/\*\*: "foo"`, }, { summary: "Paths with generate: manifest must have trailing /**", @@ -2407,10 +2407,10 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer'", selslices: []setup.SliceKey{ - {Package: "mypkg1", Slice: "myslice1"}, - {Package: "mypkg1", Slice: "myslice2"}, - {Package: "mypkg2", Slice: "myslice1"}, - {Package: "mypkg3", Slice: "myslice1"}, + {RealName: "mypkg1", Slice: "myslice1"}, + {RealName: "mypkg1", Slice: "myslice2"}, + {RealName: "mypkg2", Slice: "myslice1"}, + {RealName: "mypkg3", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -2514,9 +2514,9 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer' depends on selection", selslices: []setup.SliceKey{ - {Package: "mypkg1", Slice: "myslice1"}, - {Package: "mypkg1", Slice: "myslice2"}, - {Package: "mypkg2", Slice: "myslice1"}, + {RealName: "mypkg1", Slice: "myslice1"}, + {RealName: "mypkg1", Slice: "myslice2"}, + {RealName: "mypkg2", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -3664,24 +3664,24 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, - {Package: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, - {Package: "mypkg", Slice: "myslice5"}: {Arch: nil}, + {RealName: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, + {RealName: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, + {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {RealName: "mypkg", Slice: "myslice5"}: {Arch: nil}, }, }, "myslice2": { Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice3": { Package: "mypkg", Name: "myslice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice4": { @@ -3693,7 +3693,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice5", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, }, @@ -3941,13 +3941,13 @@ var setupTests = []setupTest{{ Stores: map[string]*setup.Store{ "bin": { Name: "bin", - Kind: setup.StoreBin, + Kind: "bin", Version: "26.10", DefaultPrefix: "bin-", }, }, Packages: map[string]*setup.Package{ - "bin/mypkg": { + "bin-mypkg": { Name: "mypkg", Path: "slices/bin/mypkg.yaml", Store: "bin", @@ -4009,32 +4009,6 @@ var setupTests = []setupTest{{ `, }, relerror: `slices/bin/mypkg.yaml: package refers to undefined store "no-such-store"`, -}, { - summary: "Store with invalid kind", - input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - mystore: - kind: unknown-kind - version: 26.10 - default-prefix: "pfx-" - `, - }, - relerror: `chisel.yaml: store "mystore" has invalid kind "unknown-kind"`, }, { summary: "Store missing version", input: map[string]string{ @@ -4144,7 +4118,7 @@ var setupTests = []setupTest{{ Stores: map[string]*setup.Store{ "bin": { Name: "bin", - Kind: setup.StoreBin, + Kind: "bin", Version: "26.10", DefaultPrefix: "bin-", }, @@ -4155,33 +4129,33 @@ var setupTests = []setupTest{{ Path: "slices/curl.yaml", Slices: map[string]*setup.Slice{ "libs": { - Package: "curl", - Kind: "", - Name: "libs", + Package: "curl", + DefaultPrefix: "", + Name: "libs", Contents: map[string]setup.PathInfo{ "/usr/lib/libcurl.so": {Kind: setup.CopyPath}, }, }, "bins": { - Package: "curl", - Kind: "", - Name: "bins", + Package: "curl", + DefaultPrefix: "", + Name: "bins", Contents: map[string]setup.PathInfo{ "/usr/bin/curl": {Kind: setup.CopyPath}, }, }, }, }, - "bin/curl": { + "bin-curl": { Name: "curl", Path: "slices/bin/curl.yaml", Store: "bin", DefaultTrack: "3.0", Slices: map[string]*setup.Slice{ "bins": { - Package: "curl", - Kind: "bin", - Name: "bins", + Package: "curl", + DefaultPrefix: "bin-", + Name: "bins", Contents: map[string]setup.PathInfo{ "/usr/bin/curl-bin": {Kind: setup.CopyPath}, }, @@ -4296,9 +4270,9 @@ func runParseReleaseTests(c *C, tests []setupTest) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) c.Assert(err, IsNil) } @@ -4393,16 +4367,16 @@ func (s *S) TestPackageMarshalYAML(c *C) { dir := c.MkDir() // Write chisel.yaml. fpath := filepath.Join(dir, "chisel.yaml") - err := os.WriteFile(fpath, testutil.Reindent(data), 0644) + err := os.WriteFile(fpath, testutil.Reindent(data), 0o644) c.Assert(err, IsNil) // Write the packages YAML. for _, pkg := range test.release.Packages { fpath = filepath.Join(dir, pkg.Path) - err = os.MkdirAll(filepath.Dir(fpath), 0755) + err = os.MkdirAll(filepath.Dir(fpath), 0o755) c.Assert(err, IsNil) pkgData, err := yaml.Marshal(pkg) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(string(pkgData)), 0644) + err = os.WriteFile(fpath, testutil.Reindent(string(pkgData)), 0o644) c.Assert(err, IsNil) } @@ -4415,7 +4389,7 @@ func (s *S) TestPackageMarshalYAML(c *C) { } func (s *S) TestPackageYAMLFormat(c *C) { - var tests = []struct { + tests := []struct { summary string input map[string]string expected map[string]string @@ -4610,9 +4584,9 @@ func (s *S) TestPackageYAMLFormat(c *C) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) c.Assert(err, IsNil) } @@ -4698,7 +4672,7 @@ func (s *S) TestSelectEmptyArch(c *C) { release, err := setup.ReadRelease(dir) c.Assert(err, IsNil) - selslice := []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}} + selslice := []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}} selection, err := setup.Select(release, selslice, "") c.Assert(err, IsNil) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 36335b190..a08f4911f 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -127,8 +127,10 @@ func (es yamlEssentialListMap) MarshalYAML() (any, error) { return es.Values, nil } -var _ yaml.Marshaler = yamlEssentialListMap{} -var _ yaml.Unmarshaler = (*yamlEssentialListMap)(nil) +var ( + _ yaml.Marshaler = yamlEssentialListMap{} + _ yaml.Unmarshaler = (*yamlEssentialListMap)(nil) +) type yamlPath struct { Dir bool `yaml:"make,omitempty"` @@ -450,10 +452,8 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.Stores = make(map[string]*Store, len(yamlVar.Stores)) } for storeName, details := range yamlVar.Stores { - switch StoreKind(details.Kind) { - case StoreBin: - default: - return nil, fmt.Errorf("%s: store %q has invalid kind %q", fileName, storeName, details.Kind) + if details.Kind == "" { + return nil, fmt.Errorf("%s: store %q missing kind field", fileName, storeName) } if details.Version == "" { return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) @@ -463,7 +463,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { } release.Stores[storeName] = &Store{ Name: storeName, - Kind: StoreKind(details.Kind), + Kind: details.Kind, Version: details.Version, DefaultPrefix: details.DefaultPrefix, } @@ -496,7 +496,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{RealName: pkgName, Slice: sliceName}) } } } else { @@ -508,10 +508,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{RealName: pkgName, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{RealName: pkgName, Slice: sliceName}) } } } @@ -544,7 +544,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgName, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{RealName: pkgName, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ Package: pkgName, @@ -571,7 +571,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error if !path.IsAbs(contPath) || path.Clean(contPath) != comparePath { return nil, fmt.Errorf("slice %s_%s has invalid content path: %s", pkgName, sliceName, contPath) } - var kinds = make([]PathKind, 0, 3) + kinds := make([]PathKind, 0, 3) var info string var mode uint var mutable bool @@ -856,7 +856,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { + if sliceKey.RealName == slice.RealName() && sliceKey.Slice == slice.Name { // Do not add the slice to its own essentials list. return nil } @@ -878,7 +878,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { + if sliceKey.RealName == slice.RealName() && sliceKey.Slice == slice.Name { return fmt.Errorf("cannot add slice to itself as essential %s in %s", refName, pkgPath) } if _, ok := slice.Essential[sliceKey]; ok { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2a7711694..f0ac9b6c0 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -23,7 +23,7 @@ import ( "github.com/canonical/chisel/internal/setup" ) -const manifestMode fs.FileMode = 0644 +const manifestMode fs.FileMode = 0o644 type RunOptions struct { Selection *setup.Selection @@ -95,10 +95,10 @@ func Run(options *RunOptions) error { return err } - // Build a map from package map key to architecture. + // Build a map from package real name to architecture. pkgArch := make(map[string]string) - for pkgKey, a := range pkgArchive { - pkgArch[pkgKey] = a.Options().Arch + for realName, a := range pkgArchive { + pkgArch[realName] = a.Options().Arch } // TODO Handle packages coming from a store as well when we support them. @@ -110,13 +110,13 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - pkgKey := slice.PkgKey() - extractPackage := extract[pkgKey] + realName := slice.RealName() + extractPackage := extract[realName] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) - extract[pkgKey] = extractPackage + extract[realName] = extractPackage } - arch := pkgArch[pkgKey] + arch := pkgArch[realName] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -157,16 +157,16 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - pkgKey := slice.PkgKey() - if packages[pkgKey] != nil { + realName := slice.RealName() + if packages[realName] != nil { continue } - reader, info, err := pkgArchive[pkgKey].Fetch(slice.Package) + reader, info, err := pkgArchive[realName].Fetch(slice.Package) if err != nil { return err } defer reader.Close() - packages[pkgKey] = reader + packages[realName] = reader pkgInfos = append(pkgInfos, info) } @@ -243,19 +243,19 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - pkgKey := slice.PkgKey() - reader := packages[pkgKey] + realName := slice.RealName() + reader := packages[realName] if reader == nil { continue } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, - Extract: extract[pkgKey], + Extract: extract[realName], TargetDir: targetDir, Create: create, }) reader.Close() - packages[pkgKey] = nil + packages[realName] = nil if err != nil { return err } @@ -279,7 +279,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.PkgKey()] + arch := pkgArch[slice.RealName()] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -361,7 +361,8 @@ func Run(options *RunOptions) error { } func generateManifests(targetDir string, selection *setup.Selection, - report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { + report *manifestutil.Report, pkgInfos []*archive.PackageInfo, +) error { manifestSlices := manifestutil.FindPaths(selection.Slices) if len(manifestSlices) == 0 { // Nothing to do. @@ -466,9 +467,9 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent targetMode := pathInfo.Mode if targetMode == 0 { if pathInfo.Kind == setup.DirPath { - targetMode = 0755 + targetMode = 0o755 } else { - targetMode = 0644 + targetMode = 0o644 } } @@ -518,11 +519,11 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - pkgKey := s.PkgKey() - if _, ok := pkgArchive[pkgKey]; ok { + realName := s.RealName() + if _, ok := pkgArchive[realName]; ok { continue } - pkg := selection.Release.Packages[pkgKey] + pkg := selection.Release.Packages[realName] if pkg.Store != "" { // Packages coming from a store are not fetched from an archive, // so we skip them here. @@ -549,7 +550,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgArchive[pkgKey] = chosen + pkgArchive[realName] = chosen } return pkgArchive, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8935014fb..c6cdc92d2 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -22,9 +22,7 @@ import ( "github.com/canonical/chisel/public/manifest" ) -var ( - testKey = testutil.PGPKeys["key1"] -) +var testKey = testutil.PGPKeys["key1"] type slicerTest struct { summary string @@ -46,7 +44,7 @@ var packageEntries = map[string][]testutil.TarEntry{ {Header: tar.Header{Name: "./usr/"}}, {Header: tar.Header{Name: "./usr/lib/"}}, {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/"}}, - {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/libssl.so.3", Mode: 00755}}, + {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/libssl.so.3", Mode: 0o0755}}, {Header: tar.Header{Name: "./usr/share/"}}, {Header: tar.Header{Name: "./usr/share/doc/"}}, {Header: tar.Header{Name: "./usr/share/doc/copyright-symlink-libssl3/"}}, @@ -59,7 +57,7 @@ var packageEntries = map[string][]testutil.TarEntry{ {Header: tar.Header{Name: "./etc/ssl/openssl.cnf"}}, {Header: tar.Header{Name: "./usr/"}}, {Header: tar.Header{Name: "./usr/bin/"}}, - {Header: tar.Header{Name: "./usr/bin/openssl", Mode: 00755}}, + {Header: tar.Header{Name: "./usr/bin/openssl", Mode: 0o0755}}, {Header: tar.Header{Name: "./usr/share/"}}, {Header: tar.Header{Name: "./usr/share/doc/"}}, {Header: tar.Header{Name: "./usr/share/doc/copyright-symlink-openssl/"}}, @@ -69,16 +67,16 @@ var packageEntries = map[string][]testutil.TarEntry{ var testPackageCopyrightEntries = []testutil.TarEntry{ // Hardcoded copyright paths. - testutil.Dir(0755, "./usr/"), - testutil.Dir(0755, "./usr/share/"), - testutil.Dir(0755, "./usr/share/doc/"), - testutil.Dir(0755, "./usr/share/doc/test-package/"), - testutil.Reg(0644, "./usr/share/doc/test-package/copyright", "copyright"), + testutil.Dir(0o755, "./usr/"), + testutil.Dir(0o755, "./usr/share/"), + testutil.Dir(0o755, "./usr/share/doc/"), + testutil.Dir(0o755, "./usr/share/doc/test-package/"), + testutil.Reg(0o644, "./usr/share/doc/test-package/copyright", "copyright"), } var slicerTests = []slicerTest{{ summary: "Basic slicing", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -111,7 +109,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Glob extraction", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -133,7 +131,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -153,7 +151,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new nested file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -174,7 +172,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new directory under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -194,7 +192,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file using glob and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -218,7 +216,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Conditional architecture", arch: "amd64", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -249,7 +247,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Copyright is not installed implicitly", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", // Add the copyright entries to the package. @@ -274,8 +272,9 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + {RealName: "other-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -317,25 +316,26 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {Package: "a-implicit-parent", Slice: "myslice"}, - {Package: "b-explicit-dir", Slice: "myslice"}, - {Package: "c-implicit-parent", Slice: "myslice"}}, + {RealName: "a-implicit-parent", Slice: "myslice"}, + {RealName: "b-explicit-dir", Slice: "myslice"}, + {RealName: "c-implicit-parent", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "a-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file-1", "random"), + testutil.Dir(0o755, "./dir/"), + testutil.Reg(0o644, "./dir/file-1", "random"), }), }, { Name: "b-explicit-dir", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(01777, "./dir/"), + testutil.Dir(0o1777, "./dir/"), }), }, { Name: "c-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0766, "./dir/"), - testutil.Reg(0644, "./dir/file-2", "random"), + testutil.Dir(0o766, "./dir/"), + testutil.Reg(0o644, "./dir/file-2", "random"), }), }}, release: map[string]string{ @@ -376,8 +376,9 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid same file in two slices in different packages", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + {RealName: "other-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -409,7 +410,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: write a file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -430,7 +431,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: read a file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -456,7 +457,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove file after mutate", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -480,7 +481,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove wildcard after mutate", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -498,7 +499,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -519,7 +520,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: writing same contents to existing file does not set the final hash in report", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -540,7 +541,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: cannot write non-mutable files", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -555,7 +556,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to unlisted file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -569,7 +570,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -584,7 +585,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/`, }, { summary: "Script: cannot read unlisted content", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -600,7 +601,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice2: cannot read file which is not selected: /dir/text-file`, }, { summary: "Script: can read globbed content", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice1"}, {Package: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice1"}, {RealName: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -615,7 +616,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative content root directory must not error", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -636,7 +637,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list parent directories of normal paths", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -655,7 +656,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list unselected directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -670,7 +671,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /a/d/`, }, { summary: "Cannot list file path as a directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -685,7 +686,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a directory: /a/b/c`, }, { summary: "Can list parent directories of globs", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -699,7 +700,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list directories not matched by glob", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -714,7 +715,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /other-dir/`, }, { summary: "Duplicate copyright symlink is ignored", - slices: []setup.SliceKey{{Package: "copyright-symlink-openssl", Slice: "bins"}}, + slices: []setup.SliceKey{{RealName: "copyright-symlink-openssl", Slice: "bins"}}, pkgs: []*testutil.TestPackage{{ Name: "copyright-symlink-openssl", Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), @@ -746,7 +747,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list unclean directory paths", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -765,7 +766,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot read directories", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -780,14 +781,14 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a file: /x/y`, }, { summary: "Multiple archives with priority", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}, {Package: "other-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}, {RealName: "other-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { @@ -796,7 +797,7 @@ var slicerTests = []slicerTest{{ Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from bar"), + testutil.Reg(0o644, "./file", "from bar"), }), Archives: []string{"bar"}, }, { @@ -805,7 +806,7 @@ var slicerTests = []slicerTest{{ Version: "v3", Arch: "a3", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./other-file", "from bar"), + testutil.Reg(0o644, "./other-file", "from bar"), }), Archives: []string{"bar"}, }}, @@ -864,14 +865,14 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive bypasses higher priority", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { @@ -880,7 +881,7 @@ var slicerTests = []slicerTest{{ Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from bar"), + testutil.Reg(0o644, "./file", "from bar"), }), Archives: []string{"bar"}, }}, @@ -932,11 +933,11 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive does not have the package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -978,7 +979,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "No archives have the package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{}, release: map[string]string{ "chisel.yaml": ` @@ -1015,11 +1016,11 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archives are ignored when not explicitly pinned in package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -1052,14 +1053,14 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archive explicitly pinned in package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0644, "./file", "from foo"), + testutil.Reg(0o644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -1102,8 +1103,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Multiple slices of same package", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1140,8 +1141,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Same glob in several entries with until:mutate and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1186,8 +1187,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping globs, until:mutate and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice2"}, - {Package: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1232,8 +1233,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on entry and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1278,8 +1279,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on glob and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1307,8 +1308,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on both and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1331,8 +1332,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Content not created in packages with until:mutate on one and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {RealName: "test-package", Slice: "myslice1"}, + {RealName: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1355,8 +1356,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, both are recorded", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}, + {RealName: "test-package", Slice: "myslice"}, + {RealName: "other-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1392,7 +1393,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Two packages, only one is selected and recorded", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, + {RealName: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1426,7 +1427,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative paths are properly trimmed during extraction", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1436,12 +1437,12 @@ var slicerTests = []slicerTest{{ // relative path. Since TrimLeft takes in a cutset instead of a // prefix, the desired relative path was not produced. // See https://github.com/canonical/chisel/pull/145. - testutil.Dir(0755, "./foo-bar/"), + testutil.Dir(0o755, "./foo-bar/"), }), }}, hackopt: func(c *C, opts *slicer.RunOptions) { opts.TargetDir = filepath.Join(filepath.Clean(opts.TargetDir), "foo") - err := os.Mkdir(opts.TargetDir, 0755) + err := os.Mkdir(opts.TargetDir, 0o755) c.Assert(err, IsNil) }, release: map[string]string{ @@ -1457,7 +1458,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Producing a manifest is not mandatory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, hackopt: func(c *C, opts *slicer.RunOptions) { // Remove the manifest slice that the tests add automatically. var index int @@ -1479,7 +1480,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "No valid archives defined due to invalid pro value", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, release: map[string]string{ "chisel.yaml": ` format: v1 @@ -1506,14 +1507,15 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid hard link in two slices in the same package", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "slice1"}, - {Package: "test-package", Slice: "slice2"}}, + {RealName: "test-package", Slice: "slice1"}, + {RealName: "test-package", Slice: "slice2"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Hrd(0644, "./hardlink", "./file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Hrd(0o644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1540,14 +1542,15 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link entries can be extracted without extracting the regular file", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Hrd(0644, "./hardlink1", "./file"), - testutil.Hrd(0644, "./hardlink2", "./file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Hrd(0o644, "./hardlink1", "./file"), + testutil.Hrd(0o644, "./hardlink2", "./file"), }), }}, release: map[string]string{ @@ -1570,15 +1573,16 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifier for different groups", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file1", "text for file1"), - testutil.Reg(0644, "./file2", "text for file2"), - testutil.Hrd(0644, "./hardlink1", "./file1"), - testutil.Hrd(0644, "./hardlink2", "./file2"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file1", "text for file1"), + testutil.Reg(0o644, "./file2", "text for file2"), + testutil.Hrd(0o644, "./hardlink1", "./file1"), + testutil.Hrd(0o644, "./hardlink2", "./file2"), }), }}, release: map[string]string{ @@ -1605,13 +1609,14 @@ var slicerTests = []slicerTest{{ }, { summary: "Single hard link entry can be extracted without regular file and no hard links are created", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Hrd(0644, "./hardlink", "./file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Hrd(0o644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1632,15 +1637,16 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link to symlink does not follow symlink", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Lnk(0644, "./symlink", "./file"), - testutil.Hrd(0644, "./hardlink", "./symlink"), + testutil.Dir(0o755, "./"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Lnk(0o644, "./symlink", "./file"), + testutil.Hrd(0o644, "./hardlink", "./symlink"), }), }}, release: map[string]string{ @@ -1664,22 +1670,22 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifiers are unique across packages", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {RealName: "test-package1", Slice: "myslice"}, + {RealName: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file1", "foo"), - testutil.Hrd(0644, "./hardlink1", "./file1"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file1", "foo"), + testutil.Hrd(0o644, "./hardlink1", "./file1"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file2", "foo"), - testutil.Hrd(0644, "./hardlink2", "./file2"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file2", "foo"), + testutil.Hrd(0o644, "./hardlink2", "./file2"), }), }}, release: map[string]string{ @@ -1715,13 +1721,14 @@ var slicerTests = []slicerTest{{ }, { summary: "Mutations for hard links are forbidden", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Hrd(0644, "./hardlink", "./file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Hrd(0o644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1740,13 +1747,14 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard links can be marked as mutable, but not mutated", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}}, + {RealName: "test-package", Slice: "myslice"}, + }, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), - testutil.Hrd(0644, "./hardlink", "./file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), + testutil.Hrd(0o644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1769,12 +1777,12 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Hard links cannot escape the target directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Hrd(0644, "./hardlink", "/etc/group"), + testutil.Dir(0o755, "./"), + testutil.Hrd(0o644, "./hardlink", "/etc/group"), }), }}, release: map[string]string{ @@ -1789,12 +1797,12 @@ var slicerTests = []slicerTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./../file", "hijacking system file"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./../file", "hijacking system file"), }), }}, release: map[string]string{ @@ -1810,25 +1818,25 @@ var slicerTests = []slicerTest{{ }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {RealName: "test-package1", Slice: "myslice"}, + {RealName: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "foo"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "foo"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), - testutil.Reg(0644, "./file", "bar"), + testutil.Dir(0o755, "./"), + testutil.Reg(0o644, "./file", "bar"), }), }, { Name: "test-package3", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), + testutil.Dir(0o755, "./"), }), }}, release: map[string]string{ @@ -1871,24 +1879,24 @@ var slicerTests = []slicerTest{{ }, { summary: "Warning when implicit parent directories conflict", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {RealName: "test-package1", Slice: "myslice"}, + {RealName: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), + testutil.Dir(0o755, "./"), // Note that both implicit parents have different permissions. - testutil.Dir(0766, "./parent/"), - testutil.Reg(0644, "./parent/foo", "whatever"), + testutil.Dir(0o766, "./parent/"), + testutil.Reg(0o644, "./parent/foo", "whatever"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./"), + testutil.Dir(0o755, "./"), // And here. - testutil.Dir(0755, "./parent/"), - testutil.Reg(0644, "./parent/bar", "whatever"), + testutil.Dir(0o755, "./parent/"), + testutil.Reg(0o644, "./parent/bar", "whatever"), }), }}, release: map[string]string{ @@ -1910,7 +1918,7 @@ var slicerTests = []slicerTest{{ logOutput: `(?s).*Warning: Path "/parent/" has diverging modes in different packages\. Please report\..*`, }, { summary: "Arch specific slice is not installed when it does not match requested arch", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, arch: "amd64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1929,7 +1937,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Arch specific slice is installed when it matches requested arch", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, arch: "arm64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1953,7 +1961,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Transitive essential", - slices: []setup.SliceKey{{Package: "test-package", Slice: "first"}}, + slices: []setup.SliceKey{{RealName: "test-package", Slice: "first"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2049,9 +2057,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { releaseDir := c.MkDir() for path, data := range test.release { fpath := filepath.Join(releaseDir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) c.Assert(err, IsNil) } @@ -2059,7 +2067,7 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { c.Assert(err, IsNil) // Create a manifest slice and add it to the selection. - manifestPackage := test.slices[0].Package + manifestPackage := test.slices[0].RealName manifestPath := "/chisel-data/manifest.wall" release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ Package: manifestPackage, @@ -2074,8 +2082,8 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { Scripts: setup.SliceScripts{}, } testSlices = append(testSlices, setup.SliceKey{ - Package: manifestPackage, - Slice: "manifest", + RealName: manifestPackage, + Slice: "manifest", }) selection, err := setup.Select(release, testSlices, test.arch) @@ -2229,9 +2237,9 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { // in the manifest itself. s, err := os.Stat(path.Join(targetDir, manifestPath)) c.Assert(err, IsNil) - c.Assert(s.Mode(), Equals, fs.FileMode(0644)) + c.Assert(s.Mode(), Equals, fs.FileMode(0o644)) err = mfest.IteratePaths(manifestPath, func(p *manifest.Path) error { - c.Assert(p.Mode, Equals, fmt.Sprintf("%#o", fs.FileMode(0644))) + c.Assert(p.Mode, Equals, fmt.Sprintf("%#o", fs.FileMode(0o644))) return nil }) c.Assert(err, IsNil) From 510d4cdfce912e5fd83268d7a7c30e4366e49eff Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 14:10:19 +0200 Subject: [PATCH 08/61] fix: more renaming/refactoring Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 10 +- internal/apacheutil/util.go | 8 +- internal/apacheutil/util_test.go | 42 +++---- internal/manifestutil/manifestutil.go | 4 +- internal/setup/setup.go | 16 +-- internal/setup/setup_test.go | 62 +++++----- internal/setup/yaml.go | 12 +- internal/slicer/slicer_test.go | 166 +++++++++++++------------- 8 files changed, 161 insertions(+), 159 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index 4a08d9693..a9fa826e6 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -11,8 +11,9 @@ import ( "github.com/canonical/chisel/internal/setup" ) -var shortInfoHelp = "Show information about package slices" -var longInfoHelp = ` +var ( + shortInfoHelp = "Show information about package slices" + longInfoHelp = ` The info command shows detailed information about package slices. It accepts a whitespace-separated list of strings. The list can be @@ -23,6 +24,7 @@ the output is a list of YAML documents separated by a "---" line. Slice definitions are shown verbatim according to their definition in the selected release. For example, globs are not expanded. ` +) var infoDescs = map[string]string{ "release": "Chisel release name or directory (e.g. ubuntu-22.04)", @@ -83,7 +85,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* allPkgSlices := make(map[string]bool) sliceExists := func(key setup.SliceKey) bool { - pkg, ok := release.Packages[key.RealName] + pkg, ok := release.Packages[key.Package] if !ok { return false } @@ -98,7 +100,7 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* notFound = append(notFound, query) continue } - pkg, slice = key.RealName, key.Slice + pkg, slice = key.Package, key.Slice } else { if _, ok := release.Packages[query]; !ok { notFound = append(notFound, query) diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go index 8beaf2bef..ee98e55ba 100644 --- a/internal/apacheutil/util.go +++ b/internal/apacheutil/util.go @@ -8,12 +8,12 @@ import ( ) type SliceKey struct { - RealName string - Slice string + Package string + Slice string } func (s SliceKey) String() string { - return s.RealName + "_" + s.Slice + return s.Package + "_" + s.Slice } // FnameExp matches the slice definition file basename. @@ -30,5 +30,5 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { if match == nil { return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) } - return SliceKey{RealName: match[1], Slice: match[2]}, nil + return SliceKey{Package: match[1], Slice: match[2]}, nil } diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go index 4acab2cb5..05f091c95 100644 --- a/internal/apacheutil/util_test.go +++ b/internal/apacheutil/util_test.go @@ -14,34 +14,34 @@ var sliceKeyTests = []struct { err string }{{ input: "foo_bar", - expected: apacheutil.SliceKey{RealName: "foo", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "foo", Slice: "bar"}, }, { input: "fo_bar", - expected: apacheutil.SliceKey{RealName: "fo", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "fo", Slice: "bar"}, }, { input: "1234_bar", - expected: apacheutil.SliceKey{RealName: "1234", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "1234", Slice: "bar"}, }, { input: "foo1.1-2-3_bar", - expected: apacheutil.SliceKey{RealName: "foo1.1-2-3", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "foo1.1-2-3", Slice: "bar"}, }, { input: "foo-pkg_dashed-slice-name", - expected: apacheutil.SliceKey{RealName: "foo-pkg", Slice: "dashed-slice-name"}, + expected: apacheutil.SliceKey{Package: "foo-pkg", Slice: "dashed-slice-name"}, }, { input: "foo+_bar", - expected: apacheutil.SliceKey{RealName: "foo+", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "foo+", Slice: "bar"}, }, { input: "foo_slice123", - expected: apacheutil.SliceKey{RealName: "foo", Slice: "slice123"}, + expected: apacheutil.SliceKey{Package: "foo", Slice: "slice123"}, }, { input: "g++_bins", - expected: apacheutil.SliceKey{RealName: "g++", Slice: "bins"}, + expected: apacheutil.SliceKey{Package: "g++", Slice: "bins"}, }, { input: "a+_bar", - expected: apacheutil.SliceKey{RealName: "a+", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "a+", Slice: "bar"}, }, { input: "a._bar", - expected: apacheutil.SliceKey{RealName: "a.", Slice: "bar"}, + expected: apacheutil.SliceKey{Package: "a.", Slice: "bar"}, }, { input: "foo_ba", err: `invalid slice reference: "foo_ba"`, @@ -99,13 +99,13 @@ var sliceKeyStringTests = []struct { key apacheutil.SliceKey expected string }{{ - key: apacheutil.SliceKey{RealName: "curl", Slice: "bins"}, + key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, expected: "curl_bins", }, { - key: apacheutil.SliceKey{RealName: "bin-curl", Slice: "bins"}, + key: apacheutil.SliceKey{Package: "bin-curl", Slice: "bins"}, expected: "bin-curl_bins", }, { - key: apacheutil.SliceKey{RealName: "bin-g++", Slice: "bins"}, + key: apacheutil.SliceKey{Package: "bin-g++", Slice: "bins"}, expected: "bin-g++_bins", }} @@ -115,25 +115,25 @@ func (s *S) TestSliceKeyString(c *C) { } } -var sliceKeyRealNameTests = []struct { +var sliceKeyPackageTests = []struct { key apacheutil.SliceKey expected string }{{ - key: apacheutil.SliceKey{RealName: "curl", Slice: "bins"}, + key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, expected: "curl", }, { - key: apacheutil.SliceKey{RealName: "bin-curl", Slice: "bins"}, + key: apacheutil.SliceKey{Package: "bin-curl", Slice: "bins"}, expected: "bin-curl", }, { - key: apacheutil.SliceKey{RealName: "bin-curl"}, + key: apacheutil.SliceKey{Package: "bin-curl"}, expected: "bin-curl", }, { - key: apacheutil.SliceKey{RealName: "curl"}, + key: apacheutil.SliceKey{Package: "curl"}, expected: "curl", }} -func (s *S) TestSliceKeyRealName(c *C) { - for _, test := range sliceKeyRealNameTests { - c.Assert(test.key.RealName, Equals, test.expected) +func (s *S) TestSliceKeyPackage(c *C) { + for _, test := range sliceKeyPackageTests { + c.Assert(test.key.Package, Equals, test.expected) } } diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index edc3616b1..da9cc8b0a 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -291,8 +291,8 @@ func Validate(mfest *manifest.Manifest) (err error) { if err != nil { return err } - if !pkgExist[sk.RealName] { - return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.RealName) + if !pkgExist[sk.Package] { + return fmt.Errorf("slice %s refers to missing package %q", slice.Name, sk.Package) } sliceExist[slice.Name] = true return nil diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 97c502366..5bea0109a 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -98,7 +98,7 @@ const ( GeneratePath PathKind = "generate" // TODO Maybe in the future, for binary support. - //Base64Path PathKind = "base64" + // Base64Path PathKind = "base64" ) type PathUntil string @@ -164,7 +164,7 @@ func realName(name, defaultPrefix string) string { } func (s *Slice) String() string { - return SliceKey{RealName: s.RealName(), Slice: s.Name}.String() + return SliceKey{Package: s.RealName(), Slice: s.Name}.String() } func (s *Slice) RealName() string { @@ -262,7 +262,7 @@ func (r *Release) validate() error { for _, pkg := range r.Packages { prefix := pkgDefaultPrefix(r, pkg) for _, new := range pkg.Slices { - keys = append(keys, SliceKey{RealName: realName(pkg.Name, prefix), Slice: new.Name}) + keys = append(keys, SliceKey{Package: realName(pkg.Name, prefix), Slice: new.Name}) newRealName := new.RealName() for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { @@ -401,8 +401,8 @@ func (r *Release) validate() error { func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, error) { // Preprocess the list to improve error messages. for _, key := range keys { - if pkg, ok := pkgs[key.RealName]; !ok { - return nil, fmt.Errorf("slices of package %q not found", key.RealName) + if pkg, ok := pkgs[key.Package]; !ok { + return nil, fmt.Errorf("slices of package %q not found", key.Package) } else if _, ok := pkg.Slices[key.Slice]; !ok { return nil, fmt.Errorf("slice %s not found", key) } @@ -419,7 +419,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } seen[key] = true - pkg := pkgs[key.RealName] + pkg := pkgs[key.Package] slice := pkg.Slices[key.Slice] fqslice := slice.String() predecessors := successors[fqslice] @@ -428,7 +428,7 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, continue } fqreq := req.String() - if reqpkg, ok := pkgs[req.RealName]; !ok || reqpkg.Slices[req.Slice] == nil { + if reqpkg, ok := pkgs[req.Package]; !ok || reqpkg.Slices[req.Slice] == nil { return nil, fmt.Errorf("%s requires %s, but slice is missing", fqslice, fqreq) } predecessors = append(predecessors, fqreq) @@ -552,7 +552,7 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error } selection.Slices = make([]*Slice, len(sorted)) for i, key := range sorted { - selection.Slices[i] = release.Packages[key.RealName].Slices[key.Slice] + selection.Slices[i] = release.Packages[key.Package].Slices[key.Slice] } for _, new := range selection.Slices { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index e8538213e..7a5ee21de 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -158,7 +158,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "myslice1"}: {}, + {Package: "mypkg", Slice: "myslice1"}: {}, }, Contents: map[string]setup.PathInfo{ "/another/path": {Kind: "copy"}, @@ -421,7 +421,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{RealName: "mypkg1", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -444,7 +444,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{RealName: "mypkg2", Slice: "myslice2"}}, + selslices: []setup.SliceKey{{Package: "mypkg2", Slice: "myslice2"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -453,7 +453,7 @@ var setupTests = []setupTest{{ Package: "mypkg2", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg1", Slice: "myslice1"}: {}, + {Package: "mypkg1", Slice: "myslice1"}: {}, }, }}, }, @@ -483,7 +483,7 @@ var setupTests = []setupTest{{ /path3: {symlink: /link} `, }, - selslices: []setup.SliceKey{{RealName: "mypkg1", Slice: "myslice1"}, {RealName: "mypkg1", Slice: "myslice2"}, {RealName: "mypkg2", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}, {Package: "mypkg1", Slice: "myslice2"}, {Package: "mypkg2", Slice: "myslice1"}}, }, { summary: "Conflicting paths across slices", input: map[string]string{ @@ -1472,7 +1472,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, }, }, "slice2": { @@ -1483,16 +1483,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "slice2"}: {}, - {RealName: "mypkg", Slice: "slice1"}: {}, - {RealName: "mypkg", Slice: "slice4"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice1"}: {}, + {Package: "mypkg", Slice: "slice4"}: {}, }, }, "slice4": { Package: "mypkg", Name: "slice4", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, }, }, }, @@ -1545,16 +1545,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "myotherpkg", Slice: "slice2"}: {}, - {RealName: "mypkg", Slice: "slice2"}: {}, - {RealName: "myotherpkg", Slice: "slice1"}: {}, + {Package: "myotherpkg", Slice: "slice2"}: {}, + {Package: "mypkg", Slice: "slice2"}: {}, + {Package: "myotherpkg", Slice: "slice1"}: {}, }, }, "slice2": { Package: "mypkg", Name: "slice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "myotherpkg", Slice: "slice2"}: {}, + {Package: "myotherpkg", Slice: "slice2"}: {}, }, }, }, @@ -1739,7 +1739,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg", @@ -1792,7 +1792,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, selerror: `slice mypkg_myslice has invalid 'generate' for path /dir/\*\*: "foo"`, }, { summary: "Paths with generate: manifest must have trailing /**", @@ -2407,10 +2407,10 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer'", selslices: []setup.SliceKey{ - {RealName: "mypkg1", Slice: "myslice1"}, - {RealName: "mypkg1", Slice: "myslice2"}, - {RealName: "mypkg2", Slice: "myslice1"}, - {RealName: "mypkg3", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice2"}, + {Package: "mypkg2", Slice: "myslice1"}, + {Package: "mypkg3", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -2514,9 +2514,9 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer' depends on selection", selslices: []setup.SliceKey{ - {RealName: "mypkg1", Slice: "myslice1"}, - {RealName: "mypkg1", Slice: "myslice2"}, - {RealName: "mypkg2", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice1"}, + {Package: "mypkg1", Slice: "myslice2"}, + {Package: "mypkg2", Slice: "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -3664,24 +3664,24 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, - {RealName: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, - {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, - {RealName: "mypkg", Slice: "myslice5"}: {Arch: nil}, + {Package: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, + {Package: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice5"}: {Arch: nil}, }, }, "myslice2": { Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice3": { Package: "mypkg", Name: "myslice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice4": { @@ -3693,7 +3693,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice5", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {RealName: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, }, @@ -4672,7 +4672,7 @@ func (s *S) TestSelectEmptyArch(c *C) { release, err := setup.ReadRelease(dir) c.Assert(err, IsNil) - selslice := []setup.SliceKey{{RealName: "mypkg", Slice: "myslice"}} + selslice := []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}} selection, err := setup.Select(release, selslice, "") c.Assert(err, IsNil) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index a08f4911f..b1e2b6fe9 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -496,7 +496,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{RealName: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkgName, Slice: sliceName}) } } } else { @@ -508,10 +508,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{RealName: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgName, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{RealName: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgName, Slice: sliceName}) } } } @@ -544,7 +544,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{RealName: pkgName, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgName, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ Package: pkgName, @@ -856,7 +856,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.RealName == slice.RealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.RealName() && sliceKey.Slice == slice.Name { // Do not add the slice to its own essentials list. return nil } @@ -878,7 +878,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.RealName == slice.RealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.RealName() && sliceKey.Slice == slice.Name { return fmt.Errorf("cannot add slice to itself as essential %s in %s", refName, pkgPath) } if _, ok := slice.Essential[sliceKey]; ok { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index c6cdc92d2..d11d8ca07 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -76,7 +76,7 @@ var testPackageCopyrightEntries = []testutil.TarEntry{ var slicerTests = []slicerTest{{ summary: "Basic slicing", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -109,7 +109,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Glob extraction", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -131,7 +131,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -151,7 +151,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new nested file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -172,7 +172,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new directory under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -192,7 +192,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file using glob and preserve parent directory permissions", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -216,7 +216,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Conditional architecture", arch: "amd64", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -247,7 +247,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Copyright is not installed implicitly", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", // Add the copyright entries to the package. @@ -272,8 +272,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, - {RealName: "other-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -316,9 +316,9 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {RealName: "a-implicit-parent", Slice: "myslice"}, - {RealName: "b-explicit-dir", Slice: "myslice"}, - {RealName: "c-implicit-parent", Slice: "myslice"}, + {Package: "a-implicit-parent", Slice: "myslice"}, + {Package: "b-explicit-dir", Slice: "myslice"}, + {Package: "c-implicit-parent", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "a-implicit-parent", @@ -376,8 +376,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid same file in two slices in different packages", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, - {RealName: "other-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -410,7 +410,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: write a file", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -431,7 +431,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: read a file", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -457,7 +457,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove file after mutate", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -481,7 +481,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove wildcard after mutate", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -499,7 +499,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -520,7 +520,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: writing same contents to existing file does not set the final hash in report", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -541,7 +541,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: cannot write non-mutable files", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -556,7 +556,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to unlisted file", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -570,7 +570,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to directory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -585,7 +585,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/`, }, { summary: "Script: cannot read unlisted content", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -601,7 +601,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice2: cannot read file which is not selected: /dir/text-file`, }, { summary: "Script: can read globbed content", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice1"}, {RealName: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice1"}, {Package: "test-package", Slice: "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -616,7 +616,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative content root directory must not error", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -637,7 +637,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list parent directories of normal paths", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -656,7 +656,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list unselected directory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -671,7 +671,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /a/d/`, }, { summary: "Cannot list file path as a directory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -686,7 +686,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a directory: /a/b/c`, }, { summary: "Can list parent directories of globs", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -700,7 +700,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list directories not matched by glob", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -715,7 +715,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /other-dir/`, }, { summary: "Duplicate copyright symlink is ignored", - slices: []setup.SliceKey{{RealName: "copyright-symlink-openssl", Slice: "bins"}}, + slices: []setup.SliceKey{{Package: "copyright-symlink-openssl", Slice: "bins"}}, pkgs: []*testutil.TestPackage{{ Name: "copyright-symlink-openssl", Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), @@ -747,7 +747,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list unclean directory paths", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -766,7 +766,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot read directories", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -781,7 +781,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a file: /x/y`, }, { summary: "Multiple archives with priority", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}, {RealName: "other-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}, {Package: "other-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -865,7 +865,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive bypasses higher priority", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -933,7 +933,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive does not have the package", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -979,7 +979,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "No archives have the package", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{}, release: map[string]string{ "chisel.yaml": ` @@ -1016,7 +1016,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archives are ignored when not explicitly pinned in package", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1053,7 +1053,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archive explicitly pinned in package", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", @@ -1103,8 +1103,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Multiple slices of same package", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1141,8 +1141,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Same glob in several entries with until:mutate and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1187,8 +1187,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping globs, until:mutate and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice2"}, - {RealName: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1233,8 +1233,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on entry and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1279,8 +1279,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on glob and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1308,8 +1308,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on both and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1332,8 +1332,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Content not created in packages with until:mutate on one and reading from script", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice1"}, - {RealName: "test-package", Slice: "myslice2"}, + {Package: "test-package", Slice: "myslice1"}, + {Package: "test-package", Slice: "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1356,8 +1356,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, both are recorded", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, - {RealName: "other-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, + {Package: "other-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1393,7 +1393,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Two packages, only one is selected and recorded", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1427,7 +1427,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative paths are properly trimmed during extraction", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1458,7 +1458,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Producing a manifest is not mandatory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, hackopt: func(c *C, opts *slicer.RunOptions) { // Remove the manifest slice that the tests add automatically. var index int @@ -1480,7 +1480,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "No valid archives defined due to invalid pro value", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, release: map[string]string{ "chisel.yaml": ` format: v1 @@ -1507,8 +1507,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid hard link in two slices in the same package", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "slice1"}, - {RealName: "test-package", Slice: "slice2"}, + {Package: "test-package", Slice: "slice1"}, + {Package: "test-package", Slice: "slice2"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1542,7 +1542,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link entries can be extracted without extracting the regular file", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1573,7 +1573,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifier for different groups", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1609,7 +1609,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Single hard link entry can be extracted without regular file and no hard links are created", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1637,7 +1637,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link to symlink does not follow symlink", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1670,8 +1670,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifiers are unique across packages", slices: []setup.SliceKey{ - {RealName: "test-package1", Slice: "myslice"}, - {RealName: "test-package2", Slice: "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1721,7 +1721,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Mutations for hard links are forbidden", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1747,7 +1747,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard links can be marked as mutable, but not mutated", slices: []setup.SliceKey{ - {RealName: "test-package", Slice: "myslice"}, + {Package: "test-package", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1777,7 +1777,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Hard links cannot escape the target directory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1797,7 +1797,7 @@ var slicerTests = []slicerTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1818,8 +1818,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ - {RealName: "test-package1", Slice: "myslice"}, - {RealName: "test-package2", Slice: "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1879,8 +1879,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Warning when implicit parent directories conflict", slices: []setup.SliceKey{ - {RealName: "test-package1", Slice: "myslice"}, - {RealName: "test-package2", Slice: "myslice"}, + {Package: "test-package1", Slice: "myslice"}, + {Package: "test-package2", Slice: "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", @@ -1918,7 +1918,7 @@ var slicerTests = []slicerTest{{ logOutput: `(?s).*Warning: Path "/parent/" has diverging modes in different packages\. Please report\..*`, }, { summary: "Arch specific slice is not installed when it does not match requested arch", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, arch: "amd64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1937,7 +1937,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Arch specific slice is installed when it matches requested arch", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, arch: "arm64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1961,7 +1961,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Transitive essential", - slices: []setup.SliceKey{{RealName: "test-package", Slice: "first"}}, + slices: []setup.SliceKey{{Package: "test-package", Slice: "first"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2067,7 +2067,7 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { c.Assert(err, IsNil) // Create a manifest slice and add it to the selection. - manifestPackage := test.slices[0].RealName + manifestPackage := test.slices[0].Package manifestPath := "/chisel-data/manifest.wall" release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ Package: manifestPackage, @@ -2082,8 +2082,8 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { Scripts: setup.SliceScripts{}, } testSlices = append(testSlices, setup.SliceKey{ - RealName: manifestPackage, - Slice: "manifest", + Package: manifestPackage, + Slice: "manifest", }) selection, err := setup.Select(release, testSlices, test.arch) From 2eb88e0ca0c8bbcfdb0af57636df0ea77f7b2cbe Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 14:15:23 +0200 Subject: [PATCH 09/61] fix: more renaming Signed-off-by: Paul Mars --- internal/setup/setup.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 5bea0109a..e6b1f2dac 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -584,32 +584,32 @@ type preferKey struct { func (r *Release) prefers() (map[preferKey]string, error) { prefers := make(map[preferKey]string) - for mapKey, pkg := range r.Packages { + for realName, pkg := range r.Packages { for _, slice := range pkg.Slices { for path, info := range slice.Contents { if info.Prefer != "" { if _, ok := r.Packages[info.Prefer]; !ok { return nil, fmt.Errorf("slice %s path %s 'prefer' refers to undefined package %q", slice, path, info.Prefer) } - tkey := preferKey{preferTarget, path, mapKey} + tkey := preferKey{preferTarget, path, realName} skey := preferKey{preferSource, path, info.Prefer} if target, ok := prefers[tkey]; ok { if target != info.Prefer { pkg1, pkg2 := sortPair(target, info.Prefer) return nil, fmt.Errorf("package %q has conflicting prefers for %s: %s != %s", - mapKey, path, pkg1, pkg2) + realName, path, pkg1, pkg2) } } else if source, ok := prefers[skey]; ok { - if source != mapKey { - pkg1, pkg2 := sortPair(source, mapKey) + if source != realName { + pkg1, pkg2 := sortPair(source, realName) return nil, fmt.Errorf("packages %q and %q cannot both prefer %q for %s", pkg1, pkg2, info.Prefer, path) } } else { prefers[tkey] = info.Prefer - prefers[skey] = mapKey + prefers[skey] = realName // Sample package that requires this path to be in a prefer relationship. - prefers[preferKey{preferSource, path, ""}] = mapKey + prefers[preferKey{preferSource, path, ""}] = realName } } } From 74ee5856ac963c54cc1a2550a400812082e427dc Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 14:51:39 +0200 Subject: [PATCH 10/61] fix: improve name displayed Signed-off-by: Paul Mars --- internal/setup/setup.go | 11 ++------ internal/setup/yaml.go | 62 ++++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index e6b1f2dac..5e063919a 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -501,20 +501,13 @@ func readSlices(release *Release, baseDir, dirName string) error { return fmt.Errorf("cannot read slice definition file: %v", err) } - pkg, err := parsePackage(release.Format, pkgName, stripBase(baseDir, pkgPath), data) + pkg, err := parsePackage(release, pkgName, stripBase(baseDir, pkgPath), data) if err != nil { return err } - // Derive the package DefaultPrefix from its store reference. - prefix := pkgDefaultPrefix(release, pkg) - - // Set DefaultPrefix on all slices in the package. - for _, slice := range pkg.Slices { - slice.DefaultPrefix = prefix - } - // Use the real name for the Packages map. + prefix := pkgDefaultPrefix(release, pkg) pkgRealName := realName(pkg.Name, prefix) if existing, ok := release.Packages[pkgRealName]; ok { return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, existing.Path, stripBase(baseDir, pkgPath)) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index b1e2b6fe9..edb24981b 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -472,7 +472,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { return release, err } -func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error) { +func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Package, error) { pkg := Package{ Name: pkgName, Path: pkgPath, @@ -490,7 +490,30 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } - if format == "v1" || format == "v2" { + if yamlPkg.Store != "" && yamlPkg.Archive != "" { + return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) + } + if yamlPkg.Store != "" { + if yamlPkg.DefaultTrack == "" { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' is required when 'store' is set", pkgName) + } + if strings.Contains(yamlPkg.DefaultTrack, "/") { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' must be a track name without /", pkgName) + } + pkg.Store = yamlPkg.Store + pkg.DefaultTrack = yamlPkg.DefaultTrack + } else { + if yamlPkg.DefaultTrack != "" { + return nil, fmt.Errorf("cannot parse package %q: 'store' is required when 'default-track' is set", pkgName) + } + } + pkg.Archive = yamlPkg.Archive + + // Derive the package DefaultPrefix from its store reference. + prefix := pkgDefaultPrefix(release, &pkg) + pkgRealName := realName(pkgName, prefix) + + if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkgName) } @@ -501,39 +524,21 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error } } else { if yamlPkg.V3Essential != nil { - return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkgName) + return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkgRealName) } if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkgRealName) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgRealName, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgRealName, Slice: sliceName}) } } } - if yamlPkg.Store != "" && yamlPkg.Archive != "" { - return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) - } - if yamlPkg.Store != "" { - if yamlPkg.DefaultTrack == "" { - return nil, fmt.Errorf("cannot parse package %q: 'default-track' is required when 'store' is set", pkgName) - } - if strings.Contains(yamlPkg.DefaultTrack, "/") { - return nil, fmt.Errorf("cannot parse package %q: 'default-track' must be a track name without /", pkgName) - } - pkg.Store = yamlPkg.Store - pkg.DefaultTrack = yamlPkg.DefaultTrack - } else { - if yamlPkg.DefaultTrack != "" { - return nil, fmt.Errorf("cannot parse package %q: 'store' is required when 'default-track' is set", pkgName) - } - } - pkg.Archive = yamlPkg.Archive zeroPath := yamlPath{} for sliceName, yamlSlice := range yamlPkg.Slices { match := apacheutil.SnameExp.FindStringSubmatch(sliceName) @@ -544,12 +549,13 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgName, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgRealName, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ - Package: pkgName, - Name: sliceName, - Hint: yamlSlice.Hint, + Package: pkgName, + DefaultPrefix: prefix, + Name: sliceName, + Hint: yamlSlice.Hint, Scripts: SliceScripts{ Mutate: yamlSlice.Mutate, }, From 3cab6a339ec21b34a0edc6477edd963b847e9113 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 15:23:20 +0200 Subject: [PATCH 11/61] fix: refine naming Signed-off-by: Paul Mars --- internal/apacheutil/util.go | 6 +- internal/apacheutil/util_test.go | 43 ------- internal/setup/setup.go | 37 ++---- internal/setup/setup_test.go | 214 ++++++++++++++++++------------- internal/setup/yaml.go | 18 +-- internal/slicer/slicer.go | 10 +- 6 files changed, 156 insertions(+), 172 deletions(-) diff --git a/internal/apacheutil/util.go b/internal/apacheutil/util.go index ee98e55ba..7e0f8e6d8 100644 --- a/internal/apacheutil/util.go +++ b/internal/apacheutil/util.go @@ -12,9 +12,7 @@ type SliceKey struct { Slice string } -func (s SliceKey) String() string { - return s.Package + "_" + s.Slice -} +func (s SliceKey) String() string { return s.Package + "_" + s.Slice } // FnameExp matches the slice definition file basename. var FnameExp = regexp.MustCompile(`^([a-z0-9](?:-?[.a-z0-9+]){1,})\.yaml$`) @@ -30,5 +28,5 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { if match == nil { return SliceKey{}, fmt.Errorf("invalid slice reference: %q", sliceKey) } - return SliceKey{Package: match[1], Slice: match[2]}, nil + return SliceKey{match[1], match[2]}, nil } diff --git a/internal/apacheutil/util_test.go b/internal/apacheutil/util_test.go index 05f091c95..6289473cc 100644 --- a/internal/apacheutil/util_test.go +++ b/internal/apacheutil/util_test.go @@ -94,46 +94,3 @@ func (s *S) TestParseSliceKey(c *C) { c.Assert(key, DeepEquals, test.expected) } } - -var sliceKeyStringTests = []struct { - key apacheutil.SliceKey - expected string -}{{ - key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, - expected: "curl_bins", -}, { - key: apacheutil.SliceKey{Package: "bin-curl", Slice: "bins"}, - expected: "bin-curl_bins", -}, { - key: apacheutil.SliceKey{Package: "bin-g++", Slice: "bins"}, - expected: "bin-g++_bins", -}} - -func (s *S) TestSliceKeyString(c *C) { - for _, test := range sliceKeyStringTests { - c.Assert(test.key.String(), Equals, test.expected) - } -} - -var sliceKeyPackageTests = []struct { - key apacheutil.SliceKey - expected string -}{{ - key: apacheutil.SliceKey{Package: "curl", Slice: "bins"}, - expected: "curl", -}, { - key: apacheutil.SliceKey{Package: "bin-curl", Slice: "bins"}, - expected: "bin-curl", -}, { - key: apacheutil.SliceKey{Package: "bin-curl"}, - expected: "bin-curl", -}, { - key: apacheutil.SliceKey{Package: "curl"}, - expected: "curl", -}} - -func (s *S) TestSliceKeyPackage(c *C) { - for _, test := range sliceKeyPackageTests { - c.Assert(test.key.Package, Equals, test.expected) - } -} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 5e063919a..7e40490c1 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -60,6 +60,7 @@ type Archive struct { // Package holds a collection of slices that represent parts of themselves. type Package struct { + RealName string Name string Path string Archive string @@ -157,17 +158,11 @@ func pkgDefaultPrefix(release *Release, pkg *Package) string { return store.DefaultPrefix } -// realName returns the real name for a package used as the -// Release.Packages map key. -func realName(name, defaultPrefix string) string { - return defaultPrefix + name -} - func (s *Slice) String() string { - return SliceKey{Package: s.RealName(), Slice: s.Name}.String() + return SliceKey{Package: s.RealPkgName(), Slice: s.Name}.String() } -func (s *Slice) RealName() string { +func (s *Slice) RealPkgName() string { return s.DefaultPrefix + s.Package } @@ -196,13 +191,13 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sRealName := slice.RealName() + sRealName := slice.RealPkgName() old, ok := pathPreferredPkg[path] if !ok { pathPreferredPkg[path] = s.Release.Packages[sRealName] continue } - oldRealName := realName(old.Name, pkgDefaultPrefix(s.Release, old)) + oldRealName := old.RealName if oldRealName == sRealName { // Skip if the package was already recorded. continue @@ -260,14 +255,13 @@ func (r *Release) validate() error { // cannot validate that they are the same without downloading the package. paths := make(map[string][]*Slice) for _, pkg := range r.Packages { - prefix := pkgDefaultPrefix(r, pkg) for _, new := range pkg.Slices { - keys = append(keys, SliceKey{Package: realName(pkg.Name, prefix), Slice: new.Name}) - newRealName := new.RealName() + keys = append(keys, SliceKey{Package: pkg.RealName, Slice: new.Name}) + newRealName := new.RealPkgName() for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldRealName := old.RealName() + oldRealName := old.RealPkgName() if newRealName != oldRealName { _, err := preferredPathPackage(newPath, newRealName, oldRealName, prefers) if err == nil { @@ -302,7 +296,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if slice.RealName() == skey.pkg { + if slice.RealPkgName() == skey.pkg { found = true break } @@ -327,8 +321,8 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newRealName := new.RealName() - oldRealName := old.RealName() + newRealName := new.RealPkgName() + oldRealName := old.RealPkgName() if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { if newRealName == oldRealName { continue @@ -506,13 +500,10 @@ func readSlices(release *Release, baseDir, dirName string) error { return err } - // Use the real name for the Packages map. - prefix := pkgDefaultPrefix(release, pkg) - pkgRealName := realName(pkg.Name, prefix) - if existing, ok := release.Packages[pkgRealName]; ok { - return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, existing.Path, stripBase(baseDir, pkgPath)) + if existing, ok := release.Packages[pkg.RealName]; ok { + return fmt.Errorf("package %q slices defined more than once: %s and %s", pkg.RealName, existing.Path, stripBase(baseDir, pkgPath)) } - release.Packages[pkgRealName] = pkg + release.Packages[pkg.RealName] = pkg } return nil } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 7a5ee21de..165fc1dc6 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -92,9 +92,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -139,8 +140,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -204,8 +206,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -270,9 +273,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -327,9 +331,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -582,8 +587,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -800,8 +806,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -843,8 +850,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -887,8 +895,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -960,9 +969,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1166,8 +1176,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1241,9 +1252,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1354,8 +1366,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "jq": { - Name: "jq", - Path: "slices/mydir/jq.yaml", + RealName: "jq", + Name: "jq", + Path: "slices/mydir/jq.yaml", Slices: map[string]*setup.Slice{ "bins": { Package: "jq", @@ -1465,8 +1478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1538,8 +1552,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1560,8 +1575,9 @@ var setupTests = []setupTest{{ }, }, "myotherpkg": { - Name: "myotherpkg", - Path: "slices/mydir/myotherpkg.yaml", + RealName: "myotherpkg", + Name: "myotherpkg", + Path: "slices/mydir/myotherpkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "myotherpkg", @@ -1721,8 +1737,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1774,8 +1791,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1869,8 +1887,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1882,8 +1901,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg2", @@ -2040,9 +2060,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2166,9 +2187,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2233,9 +2255,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2328,9 +2351,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2454,8 +2478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg1": { - Name: "mypkg1", - Path: "slices/mydir/mypkg1.yaml", + RealName: "mypkg1", + Name: "mypkg1", + Path: "slices/mydir/mypkg1.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg1", @@ -2475,8 +2500,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg2", @@ -2489,8 +2515,9 @@ var setupTests = []setupTest{{ }, }, "mypkg3": { - Name: "mypkg3", - Path: "slices/mydir/mypkg3.yaml", + RealName: "mypkg3", + Name: "mypkg3", + Path: "slices/mydir/mypkg3.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg3", @@ -2889,9 +2916,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2939,9 +2967,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3128,9 +3157,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3250,9 +3280,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3372,9 +3403,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3494,9 +3526,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3614,9 +3647,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3657,8 +3691,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -3948,6 +3983,7 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "bin-mypkg": { + RealName: "bin-mypkg", Name: "mypkg", Path: "slices/bin/mypkg.yaml", Store: "bin", @@ -4125,8 +4161,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "curl": { - Name: "curl", - Path: "slices/curl.yaml", + RealName: "curl", + Name: "curl", + Path: "slices/curl.yaml", Slices: map[string]*setup.Slice{ "libs": { Package: "curl", @@ -4147,6 +4184,7 @@ var setupTests = []setupTest{{ }, }, "bin-curl": { + RealName: "bin-curl", Name: "curl", Path: "slices/bin/curl.yaml", Store: "bin", diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index edb24981b..31ead7603 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -511,7 +511,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack // Derive the package DefaultPrefix from its store reference. prefix := pkgDefaultPrefix(release, &pkg) - pkgRealName := realName(pkgName, prefix) + pkg.RealName = prefix + pkgName if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { @@ -519,22 +519,22 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkgName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkg.RealName, Slice: sliceName}) } } } else { if yamlPkg.V3Essential != nil { - return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkgRealName) + return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkg.RealName) } if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkgRealName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkg.RealName) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkgRealName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkg.RealName, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkgRealName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkg.RealName, Slice: sliceName}) } } } @@ -549,7 +549,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkgRealName, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkg.RealName, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ Package: pkgName, @@ -862,7 +862,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.RealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.RealPkgName() && sliceKey.Slice == slice.Name { // Do not add the slice to its own essentials list. return nil } @@ -884,7 +884,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.RealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.RealPkgName() && sliceKey.Slice == slice.Name { return fmt.Errorf("cannot add slice to itself as essential %s in %s", refName, pkgPath) } if _, ok := slice.Essential[sliceKey]; ok { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index f0ac9b6c0..887a577e2 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -110,7 +110,7 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - realName := slice.RealName() + realName := slice.RealPkgName() extractPackage := extract[realName] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) @@ -157,7 +157,7 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - realName := slice.RealName() + realName := slice.RealPkgName() if packages[realName] != nil { continue } @@ -243,7 +243,7 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - realName := slice.RealName() + realName := slice.RealPkgName() reader := packages[realName] if reader == nil { continue @@ -279,7 +279,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.RealName()] + arch := pkgArch[slice.RealPkgName()] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -519,7 +519,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - realName := s.RealName() + realName := s.RealPkgName() if _, ok := pkgArchive[realName]; ok { continue } From d86b9f9ca7c39935c15cc4a2e3b49a570f0bacc3 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 15:28:44 +0200 Subject: [PATCH 12/61] fix: add missing check on format Signed-off-by: Paul Mars --- internal/setup/yaml.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 31ead7603..b0dd52e16 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -490,6 +490,9 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } + if (yamlPkg.Store != "" || yamlPkg.DefaultTrack != "") && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("cannot parse package %q: 'store' and 'default-track' are not supported in format %q", pkgName, release.Format) + } if yamlPkg.Store != "" && yamlPkg.Archive != "" { return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) } From 1cda0ff23e8d2fde16b50eeff4e8f9387f4f4196 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 15:44:13 +0200 Subject: [PATCH 13/61] fix: remove pkgDefaultPrefix Signed-off-by: Paul Mars --- internal/setup/setup.go | 12 ---- internal/setup/setup_test.go | 113 +++++++++++++++++++++++++++++++++-- internal/setup/yaml.go | 9 ++- 3 files changed, 116 insertions(+), 18 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 7e40490c1..ed994476a 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -146,18 +146,6 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { return apacheutil.ParseSliceKey(sliceKey) } -// pkgDefaultPrefix returns the DefaultPrefix for a package based on its store reference. -func pkgDefaultPrefix(release *Release, pkg *Package) string { - if pkg.Store == "" { - return "" - } - store := release.Stores[pkg.Store] - if store == nil { - return "" - } - return store.DefaultPrefix -} - func (s *Slice) String() string { return SliceKey{Package: s.RealPkgName(), Slice: s.Name}.String() } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 165fc1dc6..17586b2b8 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -3999,6 +3999,27 @@ var setupTests = []setupTest{{ }, { summary: "Store and archive are mutually exclusive", input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, "slices/bin/mypkg.yaml": ` package: mypkg archive: ubuntu @@ -4008,8 +4029,29 @@ var setupTests = []setupTest{{ }, relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, }, { - summary: "Store package missing default-track", + summary: "Store package missing default-track (v3)", input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, "slices/bin/mypkg.yaml": ` package: mypkg store: bin @@ -4017,8 +4059,29 @@ var setupTests = []setupTest{{ }, relerror: `cannot parse package "mypkg": 'default-track' is required when 'store' is set`, }, { - summary: "default-track without store", + summary: "default-track without store (v3)", input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, "slices/bin/mypkg.yaml": ` package: mypkg default-track: "3.0" @@ -4026,8 +4089,29 @@ var setupTests = []setupTest{{ }, relerror: `cannot parse package "mypkg": 'store' is required when 'default-track' is set`, }, { - summary: "default-track must not contain /", + summary: "default-track must not contain / (v3)", input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, "slices/bin/mypkg.yaml": ` package: mypkg store: bin @@ -4036,8 +4120,29 @@ var setupTests = []setupTest{{ }, relerror: `cannot parse package "mypkg": 'default-track' must be a track name without /`, }, { - summary: "Package store references undefined store", + summary: "Package store references undefined store (v3)", input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + `, "slices/bin/mypkg.yaml": ` package: mypkg store: no-such-store diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index b0dd52e16..1418e0b82 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -510,10 +510,14 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack return nil, fmt.Errorf("cannot parse package %q: 'store' is required when 'default-track' is set", pkgName) } } - pkg.Archive = yamlPkg.Archive // Derive the package DefaultPrefix from its store reference. - prefix := pkgDefaultPrefix(release, &pkg) + var prefix string + if pkg.Store != "" { + if store := release.Stores[pkg.Store]; store != nil { + prefix = store.DefaultPrefix + } + } pkg.RealName = prefix + pkgName if release.Format == "v1" || release.Format == "v2" { @@ -542,6 +546,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } } + pkg.Archive = yamlPkg.Archive zeroPath := yamlPath{} for sliceName, yamlSlice := range yamlPkg.Slices { match := apacheutil.SnameExp.FindStringSubmatch(sliceName) From 267c6a32c2748d8ed9bcb974a4bd2565bee9e9d8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 15:48:18 +0200 Subject: [PATCH 14/61] fix: consistent use of pkg.RealName Signed-off-by: Paul Mars --- internal/setup/yaml.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 1418e0b82..f13232eef 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -522,7 +522,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkg.RealName) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { @@ -598,17 +598,17 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack zeroPathGenerate.Generate = yamlPath.Generate if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Prefer != "" || yamlPath.Until != UntilNone { return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", - pkgName, sliceName, contPath) + pkg.RealName, sliceName, contPath) } if _, err := validateGeneratePath(contPath); err != nil { - return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkg.RealName, sliceName, err) } kinds = append(kinds, GeneratePath) } else if strings.ContainsAny(contPath, "*?") { if yamlPath != nil { if !yamlPath.SameContent(&zeroPath) || yamlPath.Prefer != "" { return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", - pkgName, sliceName, contPath) + pkg.RealName, sliceName, contPath) } } kinds = append(kinds, GlobPath) @@ -621,7 +621,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack if yamlPath.Dir { if !strings.HasSuffix(contPath, "/") { return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", - pkgName, sliceName, contPath) + pkg.RealName, sliceName, contPath) } kinds = append(kinds, DirPath) } @@ -644,17 +644,17 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack switch until { case UntilNone, UntilMutate: default: - return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) + return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkg.RealName, sliceName, contPath, until) } arch = yamlPath.Arch.List for _, s := range arch { if deb.ValidateArch(s) != nil { - return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) + return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkg.RealName, sliceName, contPath, s) } } } - if prefer == pkgName { - return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkgName, sliceName, contPath) + if prefer == pkg.RealName { + return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkg.RealName, sliceName, contPath) } if len(kinds) == 0 { kinds = append(kinds, CopyPath) @@ -664,10 +664,10 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack for i, s := range kinds { list[i] = string(s) } - return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkgName, sliceName, contPath, strings.Join(list, ", ")) + return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkg.RealName, sliceName, contPath, strings.Join(list, ", ")) } if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { - return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) + return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkg.RealName, sliceName, contPath) } slice.Contents[contPath] = PathInfo{ Kind: kinds[0], From 619e0ecb174654412781873c1583dfb69438ddc8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 15:58:11 +0200 Subject: [PATCH 15/61] fix: more name tweaking Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 887a577e2..dc3728623 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -243,19 +243,19 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - realName := slice.RealPkgName() - reader := packages[realName] + realPkgName := slice.RealPkgName() + reader := packages[realPkgName] if reader == nil { continue } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, - Extract: extract[realName], + Extract: extract[realPkgName], TargetDir: targetDir, Create: create, }) reader.Close() - packages[realName] = nil + packages[realPkgName] = nil if err != nil { return err } @@ -519,11 +519,11 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - realName := s.RealPkgName() - if _, ok := pkgArchive[realName]; ok { + realPkgName := s.RealPkgName() + if _, ok := pkgArchive[realPkgName]; ok { continue } - pkg := selection.Release.Packages[realName] + pkg := selection.Release.Packages[realPkgName] if pkg.Store != "" { // Packages coming from a store are not fetched from an archive, // so we skip them here. @@ -550,7 +550,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgArchive[realName] = chosen + pkgArchive[realPkgName] = chosen } return pkgArchive, nil } From 952e929fbac8ebfeb1e4674d6cbee7862bbdeeda Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 16:08:35 +0200 Subject: [PATCH 16/61] fix: correct involontary lint fixes Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 6 ++---- internal/manifestutil/manifestutil.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index a9fa826e6..fa5485c6c 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -11,9 +11,8 @@ import ( "github.com/canonical/chisel/internal/setup" ) -var ( - shortInfoHelp = "Show information about package slices" - longInfoHelp = ` +var shortInfoHelp = "Show information about package slices" +var longInfoHelp = ` The info command shows detailed information about package slices. It accepts a whitespace-separated list of strings. The list can be @@ -24,7 +23,6 @@ the output is a list of YAML documents separated by a "---" line. Slice definitions are shown verbatim according to their definition in the selected release. For example, globs are not expanded. ` -) var infoDescs = map[string]string{ "release": "Chisel release name or directory (e.g. ubuntu-22.04)", diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index da9cc8b0a..16b054022 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -134,7 +134,7 @@ func manifestAddReport(dbw *jsonwall.DBWriter, report *Report) error { func unixPerm(mode fs.FileMode) (perm uint32) { perm = uint32(mode.Perm()) if mode&fs.ModeSticky != 0 { - perm |= 0o1000 + perm |= 01000 } return perm } From fb4d6e5024fb22d2225d2b7427b7efc4dfc7006b Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 15 Jun 2026 14:39:39 +0200 Subject: [PATCH 17/61] fix: revert accidental linting fixes Signed-off-by: Paul Mars --- internal/setup/setup.go | 2 +- internal/setup/yaml.go | 8 +++----- internal/slicer/slicer.go | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ed994476a..9ac34ea3c 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -99,7 +99,7 @@ const ( GeneratePath PathKind = "generate" // TODO Maybe in the future, for binary support. - // Base64Path PathKind = "base64" + //Base64Path PathKind = "base64" ) type PathUntil string diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index f13232eef..cb392e918 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -127,10 +127,8 @@ func (es yamlEssentialListMap) MarshalYAML() (any, error) { return es.Values, nil } -var ( - _ yaml.Marshaler = yamlEssentialListMap{} - _ yaml.Unmarshaler = (*yamlEssentialListMap)(nil) -) +var _ yaml.Marshaler = yamlEssentialListMap{} +var _ yaml.Unmarshaler = (*yamlEssentialListMap)(nil) type yamlPath struct { Dir bool `yaml:"make,omitempty"` @@ -585,7 +583,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack if !path.IsAbs(contPath) || path.Clean(contPath) != comparePath { return nil, fmt.Errorf("slice %s_%s has invalid content path: %s", pkgName, sliceName, contPath) } - kinds := make([]PathKind, 0, 3) + var kinds = make([]PathKind, 0, 3) var info string var mode uint var mutable bool diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index dc3728623..581efebe4 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -23,7 +23,7 @@ import ( "github.com/canonical/chisel/internal/setup" ) -const manifestMode fs.FileMode = 0o644 +const manifestMode fs.FileMode = 0644 type RunOptions struct { Selection *setup.Selection @@ -467,9 +467,9 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent targetMode := pathInfo.Mode if targetMode == 0 { if pathInfo.Kind == setup.DirPath { - targetMode = 0o755 + targetMode = 0755 } else { - targetMode = 0o644 + targetMode = 0644 } } From ffc5aaf016b3070e459ba8d9a243a130a66ad217 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 15 Jun 2026 14:44:50 +0200 Subject: [PATCH 18/61] style: harmonize names Signed-off-by: Paul Mars --- internal/setup/setup.go | 16 ++++++++-------- internal/setup/yaml.go | 4 ++-- internal/slicer/slicer.go | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 9ac34ea3c..015a7c47d 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -147,10 +147,10 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { } func (s *Slice) String() string { - return SliceKey{Package: s.RealPkgName(), Slice: s.Name}.String() + return SliceKey{Package: s.PkgRealName(), Slice: s.Name}.String() } -func (s *Slice) RealPkgName() string { +func (s *Slice) PkgRealName() string { return s.DefaultPrefix + s.Package } @@ -179,7 +179,7 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sRealName := slice.RealPkgName() + sRealName := slice.PkgRealName() old, ok := pathPreferredPkg[path] if !ok { pathPreferredPkg[path] = s.Release.Packages[sRealName] @@ -245,11 +245,11 @@ func (r *Release) validate() error { for _, pkg := range r.Packages { for _, new := range pkg.Slices { keys = append(keys, SliceKey{Package: pkg.RealName, Slice: new.Name}) - newRealName := new.RealPkgName() + newRealName := new.PkgRealName() for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldRealName := old.RealPkgName() + oldRealName := old.PkgRealName() if newRealName != oldRealName { _, err := preferredPathPackage(newPath, newRealName, oldRealName, prefers) if err == nil { @@ -284,7 +284,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if slice.RealPkgName() == skey.pkg { + if slice.PkgRealName() == skey.pkg { found = true break } @@ -309,8 +309,8 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newRealName := new.RealPkgName() - oldRealName := old.RealPkgName() + newRealName := new.PkgRealName() + oldRealName := old.PkgRealName() if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { if newRealName == oldRealName { continue diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index cb392e918..4a5616236 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -868,7 +868,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.RealPkgName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.PkgRealName() && sliceKey.Slice == slice.Name { // Do not add the slice to its own essentials list. return nil } @@ -890,7 +890,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.RealPkgName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.PkgRealName() && sliceKey.Slice == slice.Name { return fmt.Errorf("cannot add slice to itself as essential %s in %s", refName, pkgPath) } if _, ok := slice.Essential[sliceKey]; ok { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 581efebe4..747ad900f 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -110,7 +110,7 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - realName := slice.RealPkgName() + realName := slice.PkgRealName() extractPackage := extract[realName] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) @@ -157,7 +157,7 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - realName := slice.RealPkgName() + realName := slice.PkgRealName() if packages[realName] != nil { continue } @@ -243,7 +243,7 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - realPkgName := slice.RealPkgName() + realPkgName := slice.PkgRealName() reader := packages[realPkgName] if reader == nil { continue @@ -279,7 +279,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.RealPkgName()] + arch := pkgArch[slice.PkgRealName()] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -519,7 +519,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - realPkgName := s.RealPkgName() + realPkgName := s.PkgRealName() if _, ok := pkgArchive[realPkgName]; ok { continue } From d2140005c1e6873150eb3b6a65ccb2ee0ded982f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 16 Jun 2026 12:02:54 +0200 Subject: [PATCH 19/61] feat: revert approach Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 7 ++-- internal/manifestutil/manifestutil.go | 31 +++++++++++--- internal/setup/setup.go | 47 +++++++++++---------- internal/setup/setup_test.go | 23 +++++------ internal/setup/yaml.go | 59 +++++++++++++-------------- internal/slicer/slicer.go | 18 ++++---- 6 files changed, 102 insertions(+), 83 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index fa5485c6c..da4d5a18b 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -123,9 +123,10 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* } else { releasePkg := release.Packages[pkgName] pkg = &setup.Package{ - Name: releasePkg.Name, - Archive: releasePkg.Archive, - Slices: make(map[string]*setup.Slice), + Name: releasePkg.Name, + RealName: releasePkg.RealName, + Archive: releasePkg.Archive, + Slices: make(map[string]*setup.Slice), } for _, sliceName := range pkgSlices[pkgName] { pkg.Slices[sliceName] = releasePkg.Slices[sliceName] diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b054022..3c9a06aec 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -37,6 +37,7 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice + Packages map[string]*setup.Package Report *Report } @@ -50,7 +51,7 @@ func Write(options *WriteOptions, writer io.Writer) error { return err } - err = manifestAddPackages(dbw, options.PackageInfo) + err = manifestAddPackages(dbw, options.PackageInfo, options.Packages) if err != nil { return err } @@ -69,11 +70,21 @@ func Write(options *WriteOptions, writer io.Writer) error { return err } -func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { +func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo, packages map[string]*setup.Package) error { for _, info := range infos { + pkgName := info.Name + if packages != nil { + // Find the unique package name for this bare archive name. + for uniqueName, pkg := range packages { + if pkg.RealName == info.Name { + pkgName = uniqueName + break + } + } + } err := dbw.Add(&manifest.Package{ Kind: "package", - Name: info.Name, + Name: pkgName, Version: info.Version, Digest: info.SHA256, Arch: info.Arch, @@ -159,8 +170,18 @@ func fastValidate(options *WriteOptions) (err error) { } sliceExist := map[string]bool{} for _, slice := range options.Selection { - if _, ok := pkgExist[slice.Package]; !ok { - return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) + if options.Packages != nil { + pkg := options.Packages[slice.Package] + if pkg == nil { + return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) + } + if _, ok := pkgExist[pkg.RealName]; !ok { + return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) + } + } else { + if _, ok := pkgExist[slice.Package]; !ok { + return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) + } } sliceExist[slice.String()] = true } diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 015a7c47d..ffba1d9a2 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -59,9 +59,12 @@ type Archive struct { } // Package holds a collection of slices that represent parts of themselves. +// Name is the unique package identifier (e.g. "bin-curl" for store packages, +// "curl" for archive packages). RealName is the bare name visible in the +// Debian archive (e.g. "curl"). type Package struct { - RealName string Name string + RealName string Path string Archive string Store string @@ -70,14 +73,14 @@ type Package struct { } // Slice holds the details about a package slice. +// Package is the unique package identifier matching Package.Name. type Slice struct { - Package string - DefaultPrefix string - Name string - Hint string - Essential map[SliceKey]EssentialInfo - Contents map[string]PathInfo - Scripts SliceScripts + Package string + Name string + Hint string + Essential map[SliceKey]EssentialInfo + Contents map[string]PathInfo + Scripts SliceScripts } type EssentialInfo struct { @@ -147,11 +150,7 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { } func (s *Slice) String() string { - return SliceKey{Package: s.PkgRealName(), Slice: s.Name}.String() -} - -func (s *Slice) PkgRealName() string { - return s.DefaultPrefix + s.Package + return SliceKey{Package: s.Package, Slice: s.Name}.String() } // Selection holds the required configuration to create a Build for a selection @@ -179,13 +178,13 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sRealName := slice.PkgRealName() + sRealName := slice.Package old, ok := pathPreferredPkg[path] if !ok { pathPreferredPkg[path] = s.Release.Packages[sRealName] continue } - oldRealName := old.RealName + oldRealName := old.Name if oldRealName == sRealName { // Skip if the package was already recorded. continue @@ -244,12 +243,12 @@ func (r *Release) validate() error { paths := make(map[string][]*Slice) for _, pkg := range r.Packages { for _, new := range pkg.Slices { - keys = append(keys, SliceKey{Package: pkg.RealName, Slice: new.Name}) - newRealName := new.PkgRealName() + keys = append(keys, SliceKey{Package: pkg.Name, Slice: new.Name}) + newRealName := new.Package for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldRealName := old.PkgRealName() + oldRealName := old.Package if newRealName != oldRealName { _, err := preferredPathPackage(newPath, newRealName, oldRealName, prefers) if err == nil { @@ -284,7 +283,7 @@ func (r *Release) validate() error { found := false for _, slice := range paths[skey.path] { - if slice.PkgRealName() == skey.pkg { + if slice.Package == skey.pkg { found = true break } @@ -309,8 +308,8 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newRealName := new.PkgRealName() - oldRealName := old.PkgRealName() + newRealName := new.Package + oldRealName := old.Package if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { if newRealName == oldRealName { continue @@ -488,10 +487,10 @@ func readSlices(release *Release, baseDir, dirName string) error { return err } - if existing, ok := release.Packages[pkg.RealName]; ok { - return fmt.Errorf("package %q slices defined more than once: %s and %s", pkg.RealName, existing.Path, stripBase(baseDir, pkgPath)) + if existing, ok := release.Packages[pkg.Name]; ok { + return fmt.Errorf("package %q slices defined more than once: %s and %s", pkg.Name, existing.Path, stripBase(baseDir, pkgPath)) } - release.Packages[pkg.RealName] = pkg + release.Packages[pkg.Name] = pkg } return nil } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 17586b2b8..e44114da1 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -3983,8 +3983,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "bin-mypkg": { - RealName: "bin-mypkg", - Name: "mypkg", + RealName: "mypkg", + Name: "bin-mypkg", Path: "slices/bin/mypkg.yaml", Store: "bin", DefaultTrack: "3.0", @@ -4271,17 +4271,15 @@ var setupTests = []setupTest{{ Path: "slices/curl.yaml", Slices: map[string]*setup.Slice{ "libs": { - Package: "curl", - DefaultPrefix: "", - Name: "libs", + Package: "curl", + Name: "libs", Contents: map[string]setup.PathInfo{ "/usr/lib/libcurl.so": {Kind: setup.CopyPath}, }, }, "bins": { - Package: "curl", - DefaultPrefix: "", - Name: "bins", + Package: "curl", + Name: "bins", Contents: map[string]setup.PathInfo{ "/usr/bin/curl": {Kind: setup.CopyPath}, }, @@ -4289,16 +4287,15 @@ var setupTests = []setupTest{{ }, }, "bin-curl": { - RealName: "bin-curl", - Name: "curl", + RealName: "curl", + Name: "bin-curl", Path: "slices/bin/curl.yaml", Store: "bin", DefaultTrack: "3.0", Slices: map[string]*setup.Slice{ "bins": { - Package: "curl", - DefaultPrefix: "bin-", - Name: "bins", + Package: "bin-curl", + Name: "bins", Contents: map[string]setup.PathInfo{ "/usr/bin/curl-bin": {Kind: setup.CopyPath}, }, diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 4a5616236..12be94504 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -472,9 +472,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Package, error) { pkg := Package{ - Name: pkgName, - Path: pkgPath, - Slices: make(map[string]*Slice), + RealName: pkgName, + Path: pkgPath, + Slices: make(map[string]*Slice), } yamlPkg := yamlPackage{} @@ -484,7 +484,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack if err != nil { return nil, fmt.Errorf("cannot parse package %q slice definitions: %v", pkgName, err) } - if yamlPkg.Name != pkg.Name { + if yamlPkg.Name != pkg.RealName { return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } @@ -509,37 +509,37 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } } - // Derive the package DefaultPrefix from its store reference. + // Derive the package unique name from its store prefix. var prefix string if pkg.Store != "" { if store := release.Stores[pkg.Store]; store != nil { prefix = store.DefaultPrefix } } - pkg.RealName = prefix + pkgName + pkg.Name = prefix + pkgName if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkg.RealName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkg.RealName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkg.Name, Slice: sliceName}) } } } else { if yamlPkg.V3Essential != nil { - return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkg.RealName) + return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkg.Name) } if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkg.RealName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkg.RealName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkg.Name, Slice: sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkg.RealName, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkg.Name, Slice: sliceName}) } } } @@ -555,13 +555,12 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkg.RealName, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkg.Name, Slice: sliceName}, yamlSlice.Hint) } slice := &Slice{ - Package: pkgName, - DefaultPrefix: prefix, - Name: sliceName, - Hint: yamlSlice.Hint, + Package: pkg.Name, + Name: sliceName, + Hint: yamlSlice.Hint, Scripts: SliceScripts{ Mutate: yamlSlice.Mutate, }, @@ -596,17 +595,17 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack zeroPathGenerate.Generate = yamlPath.Generate if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Prefer != "" || yamlPath.Until != UntilNone { return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", - pkg.RealName, sliceName, contPath) + pkg.Name, sliceName, contPath) } if _, err := validateGeneratePath(contPath); err != nil { - return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkg.RealName, sliceName, err) + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkg.Name, sliceName, err) } kinds = append(kinds, GeneratePath) } else if strings.ContainsAny(contPath, "*?") { if yamlPath != nil { if !yamlPath.SameContent(&zeroPath) || yamlPath.Prefer != "" { return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", - pkg.RealName, sliceName, contPath) + pkg.Name, sliceName, contPath) } } kinds = append(kinds, GlobPath) @@ -619,7 +618,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack if yamlPath.Dir { if !strings.HasSuffix(contPath, "/") { return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", - pkg.RealName, sliceName, contPath) + pkg.Name, sliceName, contPath) } kinds = append(kinds, DirPath) } @@ -642,17 +641,17 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack switch until { case UntilNone, UntilMutate: default: - return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkg.RealName, sliceName, contPath, until) + return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkg.Name, sliceName, contPath, until) } arch = yamlPath.Arch.List for _, s := range arch { if deb.ValidateArch(s) != nil { - return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkg.RealName, sliceName, contPath, s) + return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkg.Name, sliceName, contPath, s) } } } - if prefer == pkg.RealName { - return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkg.RealName, sliceName, contPath) + if prefer == pkg.Name { + return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkg.Name, sliceName, contPath) } if len(kinds) == 0 { kinds = append(kinds, CopyPath) @@ -662,10 +661,10 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack for i, s := range kinds { list[i] = string(s) } - return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkg.RealName, sliceName, contPath, strings.Join(list, ", ")) + return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkg.Name, sliceName, contPath, strings.Join(list, ", ")) } if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { - return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkg.RealName, sliceName, contPath) + return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkg.Name, sliceName, contPath) } slice.Contents[contPath] = PathInfo{ Kind: kinds[0], @@ -755,7 +754,7 @@ func sliceToYAML(s *Slice) (*yamlSlice, error) { // packageToYAML converts a Package object to a yamlPackage object. func packageToYAML(p *Package) (*yamlPackage, error) { pkg := &yamlPackage{ - Name: p.Name, + Name: p.RealName, Archive: p.Archive, Store: p.Store, DefaultTrack: p.DefaultTrack, @@ -868,7 +867,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.PkgRealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { // Do not add the slice to its own essentials list. return nil } @@ -890,7 +889,7 @@ func parseEssentials(yamlPkg *yamlPackage, yamlSlice *yamlSlice, pkgPath string, if err != nil { return fmt.Errorf("package %q has invalid essential slice reference: %q", yamlPkg.Name, refName) } - if sliceKey.Package == slice.PkgRealName() && sliceKey.Slice == slice.Name { + if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { return fmt.Errorf("cannot add slice to itself as essential %s in %s", refName, pkgPath) } if _, ok := slice.Essential[sliceKey]; ok { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 747ad900f..c5ef75fc1 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -110,7 +110,7 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - realName := slice.PkgRealName() + realName := slice.Package extractPackage := extract[realName] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) @@ -157,11 +157,12 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - realName := slice.PkgRealName() + realName := slice.Package if packages[realName] != nil { continue } - reader, info, err := pkgArchive[realName].Fetch(slice.Package) + pkg := options.Selection.Release.Packages[realName] + reader, info, err := pkgArchive[realName].Fetch(pkg.RealName) if err != nil { return err } @@ -243,7 +244,7 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - realPkgName := slice.PkgRealName() + realPkgName := slice.Package reader := packages[realPkgName] if reader == nil { continue @@ -279,7 +280,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.PkgRealName()] + arch := pkgArch[slice.Package] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -398,6 +399,7 @@ func generateManifests(targetDir string, selection *setup.Selection, writeOptions := &manifestutil.WriteOptions{ PackageInfo: pkgInfos, Selection: selection.Slices, + Packages: selection.Release.Packages, Report: report, } err = manifestutil.Write(writeOptions, w) @@ -519,7 +521,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - realPkgName := s.PkgRealName() + realPkgName := s.Package if _, ok := pkgArchive[realPkgName]; ok { continue } @@ -542,13 +544,13 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel var chosen archive.Archive for _, archiveInfo := range candidates { archive := archives[archiveInfo.Name] - if archive != nil && archive.Exists(pkg.Name) { + if archive != nil && archive.Exists(pkg.RealName) { chosen = archive break } } if chosen == nil { - return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) } pkgArchive[realPkgName] = chosen } From 8e8aef0eab9dfddbe44b097c301a88760c04c0af Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 16 Jun 2026 13:57:35 +0200 Subject: [PATCH 20/61] refactor: more cleaning Signed-off-by: Paul Mars --- internal/manifestutil/manifestutil.go | 31 +-- internal/setup/setup.go | 66 +++--- internal/setup/setup_test.go | 74 +++--- internal/setup/yaml.go | 8 +- internal/slicer/slicer.go | 43 ++-- internal/slicer/slicer_test.go | 320 +++++++++++++------------- 6 files changed, 245 insertions(+), 297 deletions(-) diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 3c9a06aec..16b054022 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -37,7 +37,6 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice - Packages map[string]*setup.Package Report *Report } @@ -51,7 +50,7 @@ func Write(options *WriteOptions, writer io.Writer) error { return err } - err = manifestAddPackages(dbw, options.PackageInfo, options.Packages) + err = manifestAddPackages(dbw, options.PackageInfo) if err != nil { return err } @@ -70,21 +69,11 @@ func Write(options *WriteOptions, writer io.Writer) error { return err } -func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo, packages map[string]*setup.Package) error { +func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { for _, info := range infos { - pkgName := info.Name - if packages != nil { - // Find the unique package name for this bare archive name. - for uniqueName, pkg := range packages { - if pkg.RealName == info.Name { - pkgName = uniqueName - break - } - } - } err := dbw.Add(&manifest.Package{ Kind: "package", - Name: pkgName, + Name: info.Name, Version: info.Version, Digest: info.SHA256, Arch: info.Arch, @@ -170,18 +159,8 @@ func fastValidate(options *WriteOptions) (err error) { } sliceExist := map[string]bool{} for _, slice := range options.Selection { - if options.Packages != nil { - pkg := options.Packages[slice.Package] - if pkg == nil { - return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) - } - if _, ok := pkgExist[pkg.RealName]; !ok { - return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) - } - } else { - if _, ok := pkgExist[slice.Package]; !ok { - return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) - } + if _, ok := pkgExist[slice.Package]; !ok { + return fmt.Errorf("slice %s refers to missing package %q", slice.String(), slice.Package) } sliceExist[slice.String()] = true } diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ffba1d9a2..4fab6b282 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,7 +16,7 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// Store is the location from which binary packages are obtained via a store API. +// Store is the location from which bin packages are obtained via a store API. type Store struct { Name string Kind string @@ -59,11 +59,11 @@ type Archive struct { } // Package holds a collection of slices that represent parts of themselves. -// Name is the unique package identifier (e.g. "bin-curl" for store packages, -// "curl" for archive packages). RealName is the bare name visible in the -// Debian archive (e.g. "curl"). type Package struct { - Name string + // Name is the unique package identifier (e.g. "bin-curl" for store packages, + // "curl" for archive packages). + Name string + // RealName is the bare name visible in the Debian archive (e.g. "curl"). RealName string Path string Archive string @@ -73,7 +73,6 @@ type Package struct { } // Slice holds the details about a package slice. -// Package is the unique package identifier matching Package.Name. type Slice struct { Package string Name string @@ -149,9 +148,7 @@ func ParseSliceKey(sliceKey string) (SliceKey, error) { return apacheutil.ParseSliceKey(sliceKey) } -func (s *Slice) String() string { - return SliceKey{Package: s.Package, Slice: s.Name}.String() -} +func (s *Slice) String() string { return s.Package + "_" + s.Name } // Selection holds the required configuration to create a Build for a selection // of slices from a Release. It's still an abstract proposal in the sense that @@ -178,18 +175,16 @@ func (s *Selection) Prefers() (map[string]*Package, error) { if !hasPrefers { continue } - sRealName := slice.Package old, ok := pathPreferredPkg[path] if !ok { - pathPreferredPkg[path] = s.Release.Packages[sRealName] + pathPreferredPkg[path] = s.Release.Packages[slice.Package] continue } - oldRealName := old.Name - if oldRealName == sRealName { + if old.Name == slice.Package { // Skip if the package was already recorded. continue } - preferred, err := preferredPathPackage(path, oldRealName, sRealName, prefers) + preferred, err := preferredPathPackage(path, old.Name, slice.Package, prefers) if err != nil { // Note: we have checked above that the path has prefers and // they are different packages so the error cannot be @@ -243,14 +238,12 @@ func (r *Release) validate() error { paths := make(map[string][]*Slice) for _, pkg := range r.Packages { for _, new := range pkg.Slices { - keys = append(keys, SliceKey{Package: pkg.Name, Slice: new.Name}) - newRealName := new.Package + keys = append(keys, SliceKey{pkg.Name, new.Name}) for newPath, newInfo := range new.Contents { if oldSlices, ok := paths[newPath]; ok { for _, old := range oldSlices { - oldRealName := old.Package - if newRealName != oldRealName { - _, err := preferredPathPackage(newPath, newRealName, oldRealName, prefers) + if new.Package != old.Package { + _, err := preferredPathPackage(newPath, new.Package, old.Package, prefers) if err == nil { continue } else if err != errPreferNone { @@ -259,8 +252,8 @@ func (r *Release) validate() error { } oldInfo := old.Contents[newPath] - if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && newRealName != oldRealName { - if oldRealName > newRealName || oldRealName == newRealName && old.Name > new.Name { + if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && new.Package != old.Package { + if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name { old, new = new, old } return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath) @@ -308,16 +301,14 @@ func (r *Release) validate() error { } for _, new := range newSlices { newInfo := new.Contents[newPath] - newRealName := new.Package - oldRealName := old.Package if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { - if newRealName == oldRealName { + if new.Package == old.Package { continue } } if strdist.GlobPath(newPath, oldPath) { - if (oldRealName > newRealName) || (oldRealName == newRealName && old.Name > new.Name) || - (oldRealName == newRealName && old.Name == new.Name && oldPath > newPath) { + if (old.Package > new.Package) || (old.Package == new.Package && old.Name > new.Name) || + (old.Package == new.Package && old.Name == new.Name && oldPath > newPath) { old, new = new, old oldPath, newPath = newPath, oldPath } @@ -380,6 +371,7 @@ func (r *Release) validate() error { // If arch is supplied, essential(s) not specific to that arch are not // considered. func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, error) { + // Preprocess the list to improve error messages. for _, key := range keys { if pkg, ok := pkgs[key.Package]; !ok { @@ -424,11 +416,9 @@ func order(pkgs map[string]*Package, keys []SliceKey, arch string) ([]SliceKey, if len(names) > 1 { return nil, fmt.Errorf("essential loop detected: %s", strings.Join(names, ", ")) } - key, err := ParseSliceKey(names[0]) - if err != nil { - return nil, fmt.Errorf("internal error: cannot parse slice key %q", names[0]) - } - order = append(order, key) + name := names[0] + dot := strings.IndexByte(name, '_') + order = append(order, SliceKey{name[:dot], name[dot+1:]}) } return order, nil @@ -555,32 +545,32 @@ type preferKey struct { func (r *Release) prefers() (map[preferKey]string, error) { prefers := make(map[preferKey]string) - for realName, pkg := range r.Packages { + for _, pkg := range r.Packages { for _, slice := range pkg.Slices { for path, info := range slice.Contents { if info.Prefer != "" { if _, ok := r.Packages[info.Prefer]; !ok { return nil, fmt.Errorf("slice %s path %s 'prefer' refers to undefined package %q", slice, path, info.Prefer) } - tkey := preferKey{preferTarget, path, realName} + tkey := preferKey{preferTarget, path, pkg.Name} skey := preferKey{preferSource, path, info.Prefer} if target, ok := prefers[tkey]; ok { if target != info.Prefer { pkg1, pkg2 := sortPair(target, info.Prefer) return nil, fmt.Errorf("package %q has conflicting prefers for %s: %s != %s", - realName, path, pkg1, pkg2) + pkg.Name, path, pkg1, pkg2) } } else if source, ok := prefers[skey]; ok { - if source != realName { - pkg1, pkg2 := sortPair(source, realName) + if source != pkg.Name { + pkg1, pkg2 := sortPair(source, pkg.Name) return nil, fmt.Errorf("packages %q and %q cannot both prefer %q for %s", pkg1, pkg2, info.Prefer, path) } } else { prefers[tkey] = info.Prefer - prefers[skey] = realName + prefers[skey] = pkg.Name // Sample package that requires this path to be in a prefer relationship. - prefers[preferKey{preferSource, path, ""}] = realName + prefers[preferKey{preferSource, path, ""}] = pkg.Name } } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index e44114da1..f384674b0 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -160,7 +160,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice1"}: {}, + {"mypkg", "myslice1"}: {}, }, Contents: map[string]setup.PathInfo{ "/another/path": {Kind: "copy"}, @@ -426,7 +426,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{"mypkg1", "myslice1"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -449,7 +449,7 @@ var setupTests = []setupTest{{ myslice2: {essential: [mypkg1_myslice1]} `, }, - selslices: []setup.SliceKey{{Package: "mypkg2", Slice: "myslice2"}}, + selslices: []setup.SliceKey{{"mypkg2", "myslice2"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg1", @@ -458,7 +458,7 @@ var setupTests = []setupTest{{ Package: "mypkg2", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg1", Slice: "myslice1"}: {}, + {"mypkg1", "myslice1"}: {}, }, }}, }, @@ -488,7 +488,7 @@ var setupTests = []setupTest{{ /path3: {symlink: /link} `, }, - selslices: []setup.SliceKey{{Package: "mypkg1", Slice: "myslice1"}, {Package: "mypkg1", Slice: "myslice2"}, {Package: "mypkg2", Slice: "myslice1"}}, + selslices: []setup.SliceKey{{"mypkg1", "myslice1"}, {"mypkg1", "myslice2"}, {"mypkg2", "myslice1"}}, }, { summary: "Conflicting paths across slices", input: map[string]string{ @@ -1486,7 +1486,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, + {"mypkg", "slice2"}: {}, }, }, "slice2": { @@ -1497,16 +1497,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, - {Package: "mypkg", Slice: "slice1"}: {}, - {Package: "mypkg", Slice: "slice4"}: {}, + {"mypkg", "slice2"}: {}, + {"mypkg", "slice1"}: {}, + {"mypkg", "slice4"}: {}, }, }, "slice4": { Package: "mypkg", Name: "slice4", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "slice2"}: {}, + {"mypkg", "slice2"}: {}, }, }, }, @@ -1560,16 +1560,16 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "slice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "myotherpkg", Slice: "slice2"}: {}, - {Package: "mypkg", Slice: "slice2"}: {}, - {Package: "myotherpkg", Slice: "slice1"}: {}, + {"myotherpkg", "slice2"}: {}, + {"mypkg", "slice2"}: {}, + {"myotherpkg", "slice1"}: {}, }, }, "slice2": { Package: "mypkg", Name: "slice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "myotherpkg", Slice: "slice2"}: {}, + {"myotherpkg", "slice2"}: {}, }, }, }, @@ -1756,7 +1756,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{"mypkg", "myslice"}}, selection: &setup.Selection{ Slices: []*setup.Slice{{ Package: "mypkg", @@ -1810,7 +1810,7 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, - selslices: []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}}, + selslices: []setup.SliceKey{{"mypkg", "myslice"}}, selerror: `slice mypkg_myslice has invalid 'generate' for path /dir/\*\*: "foo"`, }, { summary: "Paths with generate: manifest must have trailing /**", @@ -2431,10 +2431,10 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer'", selslices: []setup.SliceKey{ - {Package: "mypkg1", Slice: "myslice1"}, - {Package: "mypkg1", Slice: "myslice2"}, - {Package: "mypkg2", Slice: "myslice1"}, - {Package: "mypkg3", Slice: "myslice1"}, + {"mypkg1", "myslice1"}, + {"mypkg1", "myslice2"}, + {"mypkg2", "myslice1"}, + {"mypkg3", "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -2541,9 +2541,9 @@ var setupTests = []setupTest{{ }, { summary: "Path conflicts with 'prefer' depends on selection", selslices: []setup.SliceKey{ - {Package: "mypkg1", Slice: "myslice1"}, - {Package: "mypkg1", Slice: "myslice2"}, - {Package: "mypkg2", Slice: "myslice1"}, + {"mypkg1", "myslice1"}, + {"mypkg1", "myslice2"}, + {"mypkg2", "myslice1"}, }, input: map[string]string{ "slices/mydir/mypkg1.yaml": ` @@ -3699,24 +3699,24 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice1", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice2"}: {Arch: []string{"amd64"}}, - {Package: "mypkg", Slice: "myslice3"}: {Arch: []string{"amd64", "arm64"}}, - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, - {Package: "mypkg", Slice: "myslice5"}: {Arch: nil}, + {"mypkg", "myslice2"}: {Arch: []string{"amd64"}}, + {"mypkg", "myslice3"}: {Arch: []string{"amd64", "arm64"}}, + {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {"mypkg", "myslice5"}: {Arch: nil}, }, }, "myslice2": { Package: "mypkg", Name: "myslice2", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice3": { Package: "mypkg", Name: "myslice3", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, "myslice4": { @@ -3728,7 +3728,7 @@ var setupTests = []setupTest{{ Package: "mypkg", Name: "myslice5", Essential: map[setup.SliceKey]setup.EssentialInfo{ - {Package: "mypkg", Slice: "myslice4"}: {Arch: []string{"amd64", "i386"}}, + {"mypkg", "myslice4"}: {Arch: []string{"amd64", "i386"}}, }, }, }, @@ -4507,16 +4507,16 @@ func (s *S) TestPackageMarshalYAML(c *C) { dir := c.MkDir() // Write chisel.yaml. fpath := filepath.Join(dir, "chisel.yaml") - err := os.WriteFile(fpath, testutil.Reindent(data), 0o644) + err := os.WriteFile(fpath, testutil.Reindent(data), 0644) c.Assert(err, IsNil) // Write the packages YAML. for _, pkg := range test.release.Packages { fpath = filepath.Join(dir, pkg.Path) - err = os.MkdirAll(filepath.Dir(fpath), 0o755) + err = os.MkdirAll(filepath.Dir(fpath), 0755) c.Assert(err, IsNil) pkgData, err := yaml.Marshal(pkg) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(string(pkgData)), 0o644) + err = os.WriteFile(fpath, testutil.Reindent(string(pkgData)), 0644) c.Assert(err, IsNil) } @@ -4529,7 +4529,7 @@ func (s *S) TestPackageMarshalYAML(c *C) { } func (s *S) TestPackageYAMLFormat(c *C) { - tests := []struct { + var tests = []struct { summary string input map[string]string expected map[string]string @@ -4724,9 +4724,9 @@ func (s *S) TestPackageYAMLFormat(c *C) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0o755) + err := os.MkdirAll(filepath.Dir(fpath), 0755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) c.Assert(err, IsNil) } @@ -4812,7 +4812,7 @@ func (s *S) TestSelectEmptyArch(c *C) { release, err := setup.ReadRelease(dir) c.Assert(err, IsNil) - selslice := []setup.SliceKey{{Package: "mypkg", Slice: "myslice"}} + selslice := []setup.SliceKey{{"mypkg", "myslice"}} selection, err := setup.Select(release, selslice, "") c.Assert(err, IsNil) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 12be94504..8b8e86d31 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -524,7 +524,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{Package: pkg.Name, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkg.Name, sliceName}) } } } else { @@ -536,10 +536,10 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{Package: pkg.Name, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkg.Name, sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{Package: pkg.Name, Slice: sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkg.Name, sliceName}) } } } @@ -555,7 +555,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{Package: pkg.Name, Slice: sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkg.Name, sliceName}, yamlSlice.Hint) } slice := &Slice{ Package: pkg.Name, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index c5ef75fc1..39c89771c 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -95,13 +95,6 @@ func Run(options *RunOptions) error { return err } - // Build a map from package real name to architecture. - pkgArch := make(map[string]string) - for realName, a := range pkgArchive { - pkgArch[realName] = a.Options().Arch - } - // TODO Handle packages coming from a store as well when we support them. - prefers, err := options.Selection.Prefers() if err != nil { return err @@ -110,13 +103,12 @@ func Run(options *RunOptions) error { // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { - realName := slice.Package - extractPackage := extract[realName] + extractPackage := extract[slice.Package] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) - extract[realName] = extractPackage + extract[slice.Package] = extractPackage } - arch := pkgArch[realName] + arch := pkgArchive[slice.Package].Options().Arch for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -157,17 +149,16 @@ func Run(options *RunOptions) error { packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { - realName := slice.Package - if packages[realName] != nil { + if packages[slice.Package] != nil { continue } - pkg := options.Selection.Release.Packages[realName] - reader, info, err := pkgArchive[realName].Fetch(pkg.RealName) + pkg := options.Selection.Release.Packages[slice.Package] + reader, info, err := pkgArchive[slice.Package].Fetch(pkg.RealName) if err != nil { return err } defer reader.Close() - packages[realName] = reader + packages[slice.Package] = reader pkgInfos = append(pkgInfos, info) } @@ -244,19 +235,18 @@ func Run(options *RunOptions) error { // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { - realPkgName := slice.Package - reader := packages[realPkgName] + reader := packages[slice.Package] if reader == nil { continue } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, - Extract: extract[realPkgName], + Extract: extract[slice.Package], TargetDir: targetDir, Create: create, }) reader.Close() - packages[realPkgName] = nil + packages[slice.Package] = nil if err != nil { return err } @@ -280,7 +270,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArch[slice.Package] + arch := pkgArchive[slice.Package].Options().Arch for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -362,8 +352,7 @@ func Run(options *RunOptions) error { } func generateManifests(targetDir string, selection *setup.Selection, - report *manifestutil.Report, pkgInfos []*archive.PackageInfo, -) error { + report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { manifestSlices := manifestutil.FindPaths(selection.Slices) if len(manifestSlices) == 0 { // Nothing to do. @@ -399,7 +388,6 @@ func generateManifests(targetDir string, selection *setup.Selection, writeOptions := &manifestutil.WriteOptions{ PackageInfo: pkgInfos, Selection: selection.Slices, - Packages: selection.Release.Packages, Report: report, } err = manifestutil.Write(writeOptions, w) @@ -521,11 +509,10 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - realPkgName := s.Package - if _, ok := pkgArchive[realPkgName]; ok { + if _, ok := pkgArchive[s.Package]; ok { continue } - pkg := selection.Release.Packages[realPkgName] + pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { // Packages coming from a store are not fetched from an archive, // so we skip them here. @@ -552,7 +539,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) } - pkgArchive[realPkgName] = chosen + pkgArchive[pkg.Name] = chosen } return pkgArchive, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d11d8ca07..8d3cf775c 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -22,7 +22,9 @@ import ( "github.com/canonical/chisel/public/manifest" ) -var testKey = testutil.PGPKeys["key1"] +var ( + testKey = testutil.PGPKeys["key1"] +) type slicerTest struct { summary string @@ -44,7 +46,7 @@ var packageEntries = map[string][]testutil.TarEntry{ {Header: tar.Header{Name: "./usr/"}}, {Header: tar.Header{Name: "./usr/lib/"}}, {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/"}}, - {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/libssl.so.3", Mode: 0o0755}}, + {Header: tar.Header{Name: "./usr/lib/x86_64-linux-gnu/libssl.so.3", Mode: 00755}}, {Header: tar.Header{Name: "./usr/share/"}}, {Header: tar.Header{Name: "./usr/share/doc/"}}, {Header: tar.Header{Name: "./usr/share/doc/copyright-symlink-libssl3/"}}, @@ -57,7 +59,7 @@ var packageEntries = map[string][]testutil.TarEntry{ {Header: tar.Header{Name: "./etc/ssl/openssl.cnf"}}, {Header: tar.Header{Name: "./usr/"}}, {Header: tar.Header{Name: "./usr/bin/"}}, - {Header: tar.Header{Name: "./usr/bin/openssl", Mode: 0o0755}}, + {Header: tar.Header{Name: "./usr/bin/openssl", Mode: 00755}}, {Header: tar.Header{Name: "./usr/share/"}}, {Header: tar.Header{Name: "./usr/share/doc/"}}, {Header: tar.Header{Name: "./usr/share/doc/copyright-symlink-openssl/"}}, @@ -67,16 +69,16 @@ var packageEntries = map[string][]testutil.TarEntry{ var testPackageCopyrightEntries = []testutil.TarEntry{ // Hardcoded copyright paths. - testutil.Dir(0o755, "./usr/"), - testutil.Dir(0o755, "./usr/share/"), - testutil.Dir(0o755, "./usr/share/doc/"), - testutil.Dir(0o755, "./usr/share/doc/test-package/"), - testutil.Reg(0o644, "./usr/share/doc/test-package/copyright", "copyright"), + testutil.Dir(0755, "./usr/"), + testutil.Dir(0755, "./usr/share/"), + testutil.Dir(0755, "./usr/share/doc/"), + testutil.Dir(0755, "./usr/share/doc/test-package/"), + testutil.Reg(0644, "./usr/share/doc/test-package/copyright", "copyright"), } var slicerTests = []slicerTest{{ summary: "Basic slicing", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -109,7 +111,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Glob extraction", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -131,7 +133,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -151,7 +153,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new nested file under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -172,7 +174,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new directory under extracted directory and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -192,7 +194,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Create new file using glob and preserve parent directory permissions", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -216,7 +218,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Conditional architecture", arch: "amd64", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -247,7 +249,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Copyright is not installed implicitly", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", // Add the copyright entries to the package. @@ -272,9 +274,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}, + {"other-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -316,26 +317,25 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, explicit path has preference over implicit parent", slices: []setup.SliceKey{ - {Package: "a-implicit-parent", Slice: "myslice"}, - {Package: "b-explicit-dir", Slice: "myslice"}, - {Package: "c-implicit-parent", Slice: "myslice"}, - }, + {"a-implicit-parent", "myslice"}, + {"b-explicit-dir", "myslice"}, + {"c-implicit-parent", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "a-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./dir/"), - testutil.Reg(0o644, "./dir/file-1", "random"), + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file-1", "random"), }), }, { Name: "b-explicit-dir", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o1777, "./dir/"), + testutil.Dir(01777, "./dir/"), }), }, { Name: "c-implicit-parent", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o766, "./dir/"), - testutil.Reg(0o644, "./dir/file-2", "random"), + testutil.Dir(0766, "./dir/"), + testutil.Reg(0644, "./dir/file-2", "random"), }), }}, release: map[string]string{ @@ -376,9 +376,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid same file in two slices in different packages", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}, + {"other-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.PackageData["test-package"], @@ -410,7 +409,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: write a file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -431,7 +430,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: read a file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -457,7 +456,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove file after mutate", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -481,7 +480,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: use 'until' to remove wildcard after mutate", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -499,7 +498,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -520,7 +519,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: writing same contents to existing file does not set the final hash in report", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -541,7 +540,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Script: cannot write non-mutable files", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -556,7 +555,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to unlisted file", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -570,7 +569,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, }, { summary: "Script: cannot write to directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -585,7 +584,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot write file which is not mutable: /dir/`, }, { summary: "Script: cannot read unlisted content", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{"test-package", "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -601,7 +600,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice2: cannot read file which is not selected: /dir/text-file`, }, { summary: "Script: can read globbed content", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice1"}, {Package: "test-package", Slice: "myslice2"}}, + slices: []setup.SliceKey{{"test-package", "myslice1"}, {"test-package", "myslice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -616,7 +615,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative content root directory must not error", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -637,7 +636,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list parent directories of normal paths", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -656,7 +655,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list unselected directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -671,7 +670,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /a/d/`, }, { summary: "Cannot list file path as a directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -686,7 +685,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a directory: /a/b/c`, }, { summary: "Can list parent directories of globs", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -700,7 +699,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot list directories not matched by glob", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -715,7 +714,7 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: cannot list directory which is not selected: /other-dir/`, }, { summary: "Duplicate copyright symlink is ignored", - slices: []setup.SliceKey{{Package: "copyright-symlink-openssl", Slice: "bins"}}, + slices: []setup.SliceKey{{"copyright-symlink-openssl", "bins"}}, pkgs: []*testutil.TestPackage{{ Name: "copyright-symlink-openssl", Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), @@ -747,7 +746,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Can list unclean directory paths", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -766,7 +765,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Cannot read directories", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -781,14 +780,14 @@ var slicerTests = []slicerTest{{ error: `slice test-package_myslice: content is not a file: /x/y`, }, { summary: "Multiple archives with priority", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}, {Package: "other-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}, {"other-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from foo"), + testutil.Reg(0644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { @@ -797,7 +796,7 @@ var slicerTests = []slicerTest{{ Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from bar"), + testutil.Reg(0644, "./file", "from bar"), }), Archives: []string{"bar"}, }, { @@ -806,7 +805,7 @@ var slicerTests = []slicerTest{{ Version: "v3", Arch: "a3", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./other-file", "from bar"), + testutil.Reg(0644, "./other-file", "from bar"), }), Archives: []string{"bar"}, }}, @@ -865,14 +864,14 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive bypasses higher priority", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from foo"), + testutil.Reg(0644, "./file", "from foo"), }), Archives: []string{"foo"}, }, { @@ -881,7 +880,7 @@ var slicerTests = []slicerTest{{ Version: "v2", Arch: "a2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from bar"), + testutil.Reg(0644, "./file", "from bar"), }), Archives: []string{"bar"}, }}, @@ -933,11 +932,11 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Pinned archive does not have the package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from foo"), + testutil.Reg(0644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -979,7 +978,7 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "No archives have the package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{}, release: map[string]string{ "chisel.yaml": ` @@ -1016,11 +1015,11 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archives are ignored when not explicitly pinned in package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from foo"), + testutil.Reg(0644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -1053,14 +1052,14 @@ var slicerTests = []slicerTest{{ error: `cannot find package "test-package" in archive\(s\)`, }, { summary: "Negative priority archive explicitly pinned in package", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Hash: "h1", Version: "v1", Arch: "a1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Reg(0o644, "./file", "from foo"), + testutil.Reg(0644, "./file", "from foo"), }), Archives: []string{"foo"}, }}, @@ -1103,8 +1102,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Multiple slices of same package", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1141,8 +1140,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Same glob in several entries with until:mutate and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1187,8 +1186,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping globs, until:mutate and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice2"}, - {Package: "test-package", Slice: "myslice1"}, + {"test-package", "myslice2"}, + {"test-package", "myslice1"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1233,8 +1232,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on entry and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1279,8 +1278,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on glob and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1308,8 +1307,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Overlapping glob and single entry, until:mutate on both and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1332,8 +1331,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Content not created in packages with until:mutate on one and reading from script", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice1"}, - {Package: "test-package", Slice: "myslice2"}, + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1356,8 +1355,8 @@ var slicerTests = []slicerTest{{ }, { summary: "Install two packages, both are recorded", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - {Package: "other-package", Slice: "myslice"}, + {"test-package", "myslice"}, + {"other-package", "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1393,7 +1392,7 @@ var slicerTests = []slicerTest{{ }, { summary: "Two packages, only one is selected and recorded", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, + {"test-package", "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package", @@ -1427,7 +1426,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Relative paths are properly trimmed during extraction", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -1437,12 +1436,12 @@ var slicerTests = []slicerTest{{ // relative path. Since TrimLeft takes in a cutset instead of a // prefix, the desired relative path was not produced. // See https://github.com/canonical/chisel/pull/145. - testutil.Dir(0o755, "./foo-bar/"), + testutil.Dir(0755, "./foo-bar/"), }), }}, hackopt: func(c *C, opts *slicer.RunOptions) { opts.TargetDir = filepath.Join(filepath.Clean(opts.TargetDir), "foo") - err := os.Mkdir(opts.TargetDir, 0o755) + err := os.Mkdir(opts.TargetDir, 0755) c.Assert(err, IsNil) }, release: map[string]string{ @@ -1458,7 +1457,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Producing a manifest is not mandatory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, hackopt: func(c *C, opts *slicer.RunOptions) { // Remove the manifest slice that the tests add automatically. var index int @@ -1480,7 +1479,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "No valid archives defined due to invalid pro value", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "chisel.yaml": ` format: v1 @@ -1507,15 +1506,14 @@ var slicerTests = []slicerTest{{ }, { summary: "Valid hard link in two slices in the same package", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "slice1"}, - {Package: "test-package", Slice: "slice2"}, - }, + {"test-package", "slice1"}, + {"test-package", "slice2"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Hrd(0o644, "./hardlink", "./file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Hrd(0644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1542,15 +1540,14 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link entries can be extracted without extracting the regular file", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Hrd(0o644, "./hardlink1", "./file"), - testutil.Hrd(0o644, "./hardlink2", "./file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Hrd(0644, "./hardlink1", "./file"), + testutil.Hrd(0644, "./hardlink2", "./file"), }), }}, release: map[string]string{ @@ -1573,16 +1570,15 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifier for different groups", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file1", "text for file1"), - testutil.Reg(0o644, "./file2", "text for file2"), - testutil.Hrd(0o644, "./hardlink1", "./file1"), - testutil.Hrd(0o644, "./hardlink2", "./file2"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file1", "text for file1"), + testutil.Reg(0644, "./file2", "text for file2"), + testutil.Hrd(0644, "./hardlink1", "./file1"), + testutil.Hrd(0644, "./hardlink2", "./file2"), }), }}, release: map[string]string{ @@ -1609,14 +1605,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Single hard link entry can be extracted without regular file and no hard links are created", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Hrd(0o644, "./hardlink", "./file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Hrd(0644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1637,16 +1632,15 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link to symlink does not follow symlink", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Lnk(0o644, "./symlink", "./file"), - testutil.Hrd(0o644, "./hardlink", "./symlink"), + testutil.Dir(0755, "./"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Lnk(0644, "./symlink", "./file"), + testutil.Hrd(0644, "./hardlink", "./symlink"), }), }}, release: map[string]string{ @@ -1670,22 +1664,22 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard link identifiers are unique across packages", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {"test-package1", "myslice"}, + {"test-package2", "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file1", "foo"), - testutil.Hrd(0o644, "./hardlink1", "./file1"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file1", "foo"), + testutil.Hrd(0644, "./hardlink1", "./file1"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file2", "foo"), - testutil.Hrd(0o644, "./hardlink2", "./file2"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file2", "foo"), + testutil.Hrd(0644, "./hardlink2", "./file2"), }), }}, release: map[string]string{ @@ -1721,14 +1715,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Mutations for hard links are forbidden", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Hrd(0o644, "./hardlink", "./file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Hrd(0644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1747,14 +1740,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Hard links can be marked as mutable, but not mutated", slices: []setup.SliceKey{ - {Package: "test-package", Slice: "myslice"}, - }, + {"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), - testutil.Hrd(0o644, "./hardlink", "./file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), + testutil.Hrd(0644, "./hardlink", "./file"), }), }}, release: map[string]string{ @@ -1777,12 +1769,12 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Hard links cannot escape the target directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Hrd(0o644, "./hardlink", "/etc/group"), + testutil.Dir(0755, "./"), + testutil.Hrd(0644, "./hardlink", "/etc/group"), }), }}, release: map[string]string{ @@ -1797,12 +1789,12 @@ var slicerTests = []slicerTest{{ error: `cannot extract from package "test-package": invalid link target /etc/group`, }, { summary: "Cannot extract outside of target directory", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, pkgs: []*testutil.TestPackage{{ Name: "test-package", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./../file", "hijacking system file"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./../file", "hijacking system file"), }), }}, release: map[string]string{ @@ -1818,25 +1810,25 @@ var slicerTests = []slicerTest{{ }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {"test-package1", "myslice"}, + {"test-package2", "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "foo"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "foo"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), - testutil.Reg(0o644, "./file", "bar"), + testutil.Dir(0755, "./"), + testutil.Reg(0644, "./file", "bar"), }), }, { Name: "test-package3", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), + testutil.Dir(0755, "./"), }), }}, release: map[string]string{ @@ -1879,24 +1871,24 @@ var slicerTests = []slicerTest{{ }, { summary: "Warning when implicit parent directories conflict", slices: []setup.SliceKey{ - {Package: "test-package1", Slice: "myslice"}, - {Package: "test-package2", Slice: "myslice"}, + {"test-package1", "myslice"}, + {"test-package2", "myslice"}, }, pkgs: []*testutil.TestPackage{{ Name: "test-package1", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), + testutil.Dir(0755, "./"), // Note that both implicit parents have different permissions. - testutil.Dir(0o766, "./parent/"), - testutil.Reg(0o644, "./parent/foo", "whatever"), + testutil.Dir(0766, "./parent/"), + testutil.Reg(0644, "./parent/foo", "whatever"), }), }, { Name: "test-package2", Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0o755, "./"), + testutil.Dir(0755, "./"), // And here. - testutil.Dir(0o755, "./parent/"), - testutil.Reg(0o644, "./parent/bar", "whatever"), + testutil.Dir(0755, "./parent/"), + testutil.Reg(0644, "./parent/bar", "whatever"), }), }}, release: map[string]string{ @@ -1918,7 +1910,7 @@ var slicerTests = []slicerTest{{ logOutput: `(?s).*Warning: Path "/parent/" has diverging modes in different packages\. Please report\..*`, }, { summary: "Arch specific slice is not installed when it does not match requested arch", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, arch: "amd64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1937,7 +1929,7 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{}, }, { summary: "Arch specific slice is installed when it matches requested arch", - slices: []setup.SliceKey{{Package: "test-package", Slice: "myslice"}}, + slices: []setup.SliceKey{{"test-package", "myslice"}}, arch: "arm64", release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -1961,7 +1953,7 @@ var slicerTests = []slicerTest{{ }, }, { summary: "Transitive essential", - slices: []setup.SliceKey{{Package: "test-package", Slice: "first"}}, + slices: []setup.SliceKey{{"test-package", "first"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2057,9 +2049,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { releaseDir := c.MkDir() for path, data := range test.release { fpath := filepath.Join(releaseDir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0o755) + err := os.MkdirAll(filepath.Dir(fpath), 0755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) c.Assert(err, IsNil) } @@ -2237,9 +2229,9 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { // in the manifest itself. s, err := os.Stat(path.Join(targetDir, manifestPath)) c.Assert(err, IsNil) - c.Assert(s.Mode(), Equals, fs.FileMode(0o644)) + c.Assert(s.Mode(), Equals, fs.FileMode(0644)) err = mfest.IteratePaths(manifestPath, func(p *manifest.Path) error { - c.Assert(p.Mode, Equals, fmt.Sprintf("%#o", fs.FileMode(0o644))) + c.Assert(p.Mode, Equals, fmt.Sprintf("%#o", fs.FileMode(0644))) return nil }) c.Assert(err, IsNil) From e7357d3ad0b1229c64d56b43ca2e9120157b960a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 16 Jun 2026 15:39:02 +0200 Subject: [PATCH 21/61] fix: review Signed-off-by: Paul Mars --- internal/setup/setup.go | 2 +- internal/setup/setup_test.go | 8 ++++---- internal/setup/yaml.go | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 4fab6b282..ac780f948 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,7 +16,7 @@ import ( "github.com/canonical/chisel/internal/strdist" ) -// Store is the location from which bin packages are obtained via a store API. +// Store is the location from which packages are obtained via a store API. type Store struct { Name string Kind string diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index f384674b0..0e14a374f 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -4057,7 +4057,7 @@ var setupTests = []setupTest{{ store: bin `, }, - relerror: `cannot parse package "mypkg": 'default-track' is required when 'store' is set`, + relerror: `cannot parse package "mypkg": 'store' requires 'default-track'`, }, { summary: "default-track without store (v3)", input: map[string]string{ @@ -4087,7 +4087,7 @@ var setupTests = []setupTest{{ default-track: "3.0" `, }, - relerror: `cannot parse package "mypkg": 'store' is required when 'default-track' is set`, + relerror: `cannot parse package "mypkg": 'default-track' requires 'store'`, }, { summary: "default-track must not contain / (v3)", input: map[string]string{ @@ -4118,7 +4118,7 @@ var setupTests = []setupTest{{ default-track: "3.0/stable" `, }, - relerror: `cannot parse package "mypkg": 'default-track' must be a track name without /`, + relerror: `cannot parse package "mypkg": 'default-track' must not contain /`, }, { summary: "Package store references undefined store (v3)", input: map[string]string{ @@ -4487,7 +4487,7 @@ func (s *S) TestStoresNotSupportedInOldFormats(c *C) { err = os.MkdirAll(filepath.Join(dir, "slices"), 0o755) c.Assert(err, IsNil) _, err = setup.ReadRelease(dir) - c.Assert(err, ErrorMatches, `chisel.yaml: stores is not supported in format "`+format+`"`) + c.Assert(err, ErrorMatches, `chisel.yaml: cannot use stores in format "`+format+`"`) } } diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 8b8e86d31..33177bd79 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -444,7 +444,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { // Parse stores. if len(yamlVar.Stores) > 0 && (release.Format == "v1" || release.Format == "v2") { - return nil, fmt.Errorf("%s: stores is not supported in format %q", fileName, release.Format) + return nil, fmt.Errorf("%s: cannot use stores in format %q", fileName, release.Format) } if len(yamlVar.Stores) > 0 { release.Stores = make(map[string]*Store, len(yamlVar.Stores)) @@ -496,16 +496,16 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } if yamlPkg.Store != "" { if yamlPkg.DefaultTrack == "" { - return nil, fmt.Errorf("cannot parse package %q: 'default-track' is required when 'store' is set", pkgName) + return nil, fmt.Errorf("cannot parse package %q: 'store' requires 'default-track'", pkgName) } if strings.Contains(yamlPkg.DefaultTrack, "/") { - return nil, fmt.Errorf("cannot parse package %q: 'default-track' must be a track name without /", pkgName) + return nil, fmt.Errorf("cannot parse package %q: 'default-track' must not contain /", pkgName) } pkg.Store = yamlPkg.Store pkg.DefaultTrack = yamlPkg.DefaultTrack } else { if yamlPkg.DefaultTrack != "" { - return nil, fmt.Errorf("cannot parse package %q: 'store' is required when 'default-track' is set", pkgName) + return nil, fmt.Errorf("cannot parse package %q: 'default-track' requires 'store'", pkgName) } } From 27e27097bc013fa7b3cfd514bcc8758bf6c7f2d9 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 08:13:59 +0200 Subject: [PATCH 22/61] fix: ensure store is defined when parsing Signed-off-by: Paul Mars --- internal/setup/setup.go | 10 ---------- internal/setup/setup_test.go | 2 +- internal/setup/yaml.go | 10 ++++++---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ac780f948..7f6f76e71 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -352,16 +352,6 @@ func (r *Release) validate() error { } } - // Check that stores referenced in packages are defined. - for _, pkg := range r.Packages { - if pkg.Store == "" { - continue - } - if _, ok := r.Stores[pkg.Store]; !ok { - return fmt.Errorf("%s: package refers to undefined store %q", pkg.Path, pkg.Store) - } - } - return nil } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 0e14a374f..1c22e1ad8 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -4149,7 +4149,7 @@ var setupTests = []setupTest{{ default-track: "3.0" `, }, - relerror: `slices/bin/mypkg.yaml: package refers to undefined store "no-such-store"`, + relerror: `cannot parse package "mypkg": store "no-such-store" not defined in release`, }, { summary: "Store missing version", input: map[string]string{ diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 33177bd79..8be95cfeb 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -509,12 +509,14 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } } - // Derive the package unique name from its store prefix. + // Derive the package unique name from its store prefix if applicable. var prefix string if pkg.Store != "" { - if store := release.Stores[pkg.Store]; store != nil { - prefix = store.DefaultPrefix - } + store, ok := release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("cannot parse package %q: store %q not defined in release", pkgName, pkg.Store) + } + prefix = store.DefaultPrefix } pkg.Name = prefix + pkgName From 6d19749f662d1159cbf569bb2be792333d0b45dc Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 08:16:57 +0200 Subject: [PATCH 23/61] fix: properly fill returned pkgs when searching Signed-off-by: Paul Mars --- cmd/chisel/cmd_info.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index da4d5a18b..67ede89a8 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -123,10 +123,12 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* } else { releasePkg := release.Packages[pkgName] pkg = &setup.Package{ - Name: releasePkg.Name, - RealName: releasePkg.RealName, - Archive: releasePkg.Archive, - Slices: make(map[string]*setup.Slice), + Name: releasePkg.Name, + RealName: releasePkg.RealName, + Archive: releasePkg.Archive, + Store: releasePkg.Store, + DefaultTrack: releasePkg.DefaultTrack, + Slices: make(map[string]*setup.Slice), } for _, sliceName := range pkgSlices[pkgName] { pkg.Slices[sliceName] = releasePkg.Slices[sliceName] From 824c002df3731d384154332f29d95f79edec768a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 08:32:46 +0200 Subject: [PATCH 24/61] style: lint Signed-off-by: Paul Mars --- internal/setup/yaml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 8be95cfeb..4e5246238 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -515,7 +515,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack store, ok := release.Stores[pkg.Store] if !ok { return nil, fmt.Errorf("cannot parse package %q: store %q not defined in release", pkgName, pkg.Store) - } + } prefix = store.DefaultPrefix } pkg.Name = prefix + pkgName From 7358c94818e97f1057d00ff4330f1aa19f7466a8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 11:17:01 +0200 Subject: [PATCH 25/61] fix: filter store pkgs out of the selection for now Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 106 +++++++++++++++++++++++++-------- internal/slicer/slicer_test.go | 56 +++++++++++++++++ 2 files changed, 138 insertions(+), 24 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 39c89771c..ba04b1814 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -37,6 +37,23 @@ type pathData struct { hardLink bool } +type sourceKind int + +const ( + sourceArchive sourceKind = iota + sourceStore +) + +// pkgSourceInfo records the resolved source for a package in the selection. +// It abstracts over archive and store packages +type pkgSourceInfo struct { + arch string + kind sourceKind + archive archive.Archive + store *setup.Store + pkg *setup.Package +} + type contentChecker struct { knownPaths map[string]pathData } @@ -90,30 +107,43 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + pkgSources, err := resolvePkgSources(options.Archives, options.Selection) if err != nil { return err } - prefers, err := options.Selection.Prefers() + // Store package processing is not yet implemented, so store slices + // are filtered out from the selection for now. + var archiveSlices []*setup.Slice + for _, slice := range options.Selection.Slices { + if pkgSources[slice.Package].kind != sourceStore { + archiveSlices = append(archiveSlices, slice) + } + } + selection := &setup.Selection{ + Release: options.Selection.Release, + Slices: archiveSlices, + } + + prefers, err := selection.Prefers() if err != nil { return err } // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) - for _, slice := range options.Selection.Slices { + for _, slice := range selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgArchive[slice.Package].Options().Arch + src := pkgSources[slice.Package] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue } - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if preferredPkg, ok := prefers[targetPath]; ok && preferredPkg.Name != slice.Package { @@ -148,12 +178,12 @@ func Run(options *RunOptions) error { // Fetch all packages, using the selection order. packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo - for _, slice := range options.Selection.Slices { + for _, slice := range selection.Slices { if packages[slice.Package] != nil { continue } - pkg := options.Selection.Release.Packages[slice.Package] - reader, info, err := pkgArchive[slice.Package].Fetch(pkg.RealName) + src := pkgSources[slice.Package] + reader, info, err := src.archive.Fetch(src.pkg.RealName) if err != nil { return err } @@ -234,7 +264,7 @@ func Run(options *RunOptions) error { } // Extract all packages, also using the selection order. - for _, slice := range options.Selection.Slices { + for _, slice := range selection.Slices { reader := packages[slice.Package] if reader == nil { continue @@ -269,10 +299,10 @@ func Run(options *RunOptions) error { // First group them by their relative path. Then create them and attribute // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} - for _, slice := range options.Selection.Slices { - arch := pkgArchive[slice.Package].Options().Arch + for _, slice := range selection.Slices { + src := pkgSources[slice.Package] for relPath, pathInfo := range slice.Contents { - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath || @@ -329,7 +359,7 @@ func Run(options *RunOptions) error { CheckRead: checker.checkKnown, OnWrite: report.Mutate, } - for _, slice := range options.Selection.Slices { + for _, slice := range selection.Slices { opts := scripts.RunOptions{ Label: "mutate", Script: slice.Scripts.Mutate, @@ -348,7 +378,7 @@ func Run(options *RunOptions) error { return err } - return generateManifests(targetDir, options.Selection, report, pkgInfos) + return generateManifests(targetDir, selection, report, pkgInfos) } func generateManifests(targetDir string, selection *setup.Selection, @@ -490,10 +520,12 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent }) } -// selectPkgArchives selects the highest priority archive containing the package -// unless a particular archive is pinned within the slice definition file. It -// returns a map of archives indexed by package names. -func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { +// resolvePkgSources determines the source for each package in the selection. +// For archive packages it selects the highest priority archive containing the +// package unless a particular archive is pinned within the slice definition +// file. For store packages it records the store reference. It returns a map +// of pkgSourceInfo indexed by package names. +func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Selection) (map[string]*pkgSourceInfo, error) { sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) for _, archive := range selection.Release.Archives { if archive.Priority < 0 { @@ -507,15 +539,19 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return b.Priority - a.Priority }) - pkgArchive := make(map[string]archive.Archive) + pkgSources := make(map[string]*pkgSourceInfo) for _, s := range selection.Slices { - if _, ok := pkgArchive[s.Package]; ok { + if _, ok := pkgSources[s.Package]; ok { continue } pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { - // Packages coming from a store are not fetched from an archive, - // so we skip them here. + store := selection.Release.Stores[pkg.Store] + pkgSources[pkg.Name] = &pkgSourceInfo{ + kind: sourceStore, + store: store, + pkg: pkg, + } continue } @@ -539,7 +575,29 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) } - pkgArchive[pkg.Name] = chosen + pkgSources[pkg.Name] = &pkgSourceInfo{ + arch: chosen.Options().Arch, + kind: sourceArchive, + archive: chosen, + pkg: pkg, + } + } + + // Store packages do not have an archive to derive the architecture from. + // All packages in a selection share the same architecture, so we can + // borrow it from any archive package that was already resolved. + var arch string + for _, src := range pkgSources { + if src.kind == sourceArchive { + arch = src.arch + break + } } - return pkgArchive, nil + for _, src := range pkgSources { + if src.kind == sourceStore { + src.arch = arch + } + } + + return pkgSources, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8d3cf775c..91b4656aa 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2017,6 +2017,62 @@ func (s *S) TestRun(c *C) { runSlicerTests(s, c, v2FormatTests) } +var storeSlicerTests = []slicerTest{{ + summary: "Store package is skipped when fetching is not implemented", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 22.04 + components: [main, universe] + suites: [jammy] + public-keys: [test-key] + stores: + test-store: + kind: test + version: 1.0 + default-prefix: bin- + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: test-store + default-track: stable + slices: + myslice: + contents: + /dir/store-file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", + }, +}} + +func (s *S) TestRunStorePackage(c *C) { + runSlicerTests(s, c, storeSlicerTests) +} + func runSlicerTests(s *S, c *C, tests []slicerTest) { for _, test := range tests { for _, testSlices := range testutil.Permutations(test.slices) { From be67dbde7917ab8546084b14be4906ce3a9fef34 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 11:34:21 +0200 Subject: [PATCH 26/61] fix: check store kind is known Signed-off-by: Paul Mars --- internal/setup/setup_test.go | 26 ++++++++++++++++++++++++++ internal/setup/yaml.go | 3 +++ 2 files changed, 29 insertions(+) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 1c22e1ad8..3ce298438 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -4175,6 +4175,32 @@ var setupTests = []setupTest{{ `, }, relerror: `chisel.yaml: store "bin" missing version field`, +}, { + summary: "Store unknown kind", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: unknown + version: 26.10 + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: unknown store kind "unknown" for store "bin"`, }, { summary: "Store missing default-prefix", input: map[string]string{ diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 4e5246238..3f53c849a 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -453,6 +453,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if details.Kind == "" { return nil, fmt.Errorf("%s: store %q missing kind field", fileName, storeName) } + if details.Kind != "bin" { + return nil, fmt.Errorf("%s: unknown store kind %q for store %q", fileName, details.Kind, storeName) + } if details.Version == "" { return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) } From 22050d42c9e6cc8b0cd9f4ccbc543ca904bc0254 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 11:42:03 +0200 Subject: [PATCH 27/61] fix: use bin value for kind in tests Signed-off-by: Paul Mars --- internal/slicer/slicer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 91b4656aa..774ac2658 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2035,7 +2035,7 @@ var storeSlicerTests = []slicerTest{{ public-keys: [test-key] stores: test-store: - kind: test + kind: bin version: 1.0 default-prefix: bin- public-keys: From 2e69156c8b8a47fa8e5a72d1254a6bebeb893a4c Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 17 Jun 2026 11:46:07 +0200 Subject: [PATCH 28/61] style: add missing period Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index ba04b1814..77e6fcbf4 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -45,7 +45,7 @@ const ( ) // pkgSourceInfo records the resolved source for a package in the selection. -// It abstracts over archive and store packages +// It abstracts over archive and store packages. type pkgSourceInfo struct { arch string kind sourceKind From ed9e9234389b48e9607a092d38117ce5c7b07f43 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 10:55:51 +0200 Subject: [PATCH 29/61] fix: review Signed-off-by: Paul Mars --- internal/setup/setup_test.go | 90 ++++++++++++++++++++-------------- internal/setup/yaml.go | 4 +- internal/slicer/slicer_test.go | 24 +-------- internal/testutil/defaults.go | 10 ++++ 4 files changed, 66 insertions(+), 62 deletions(-) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 3ce298438..ab07320ca 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -3996,6 +3996,29 @@ var setupTests = []setupTest{{ EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), }, }, +}, { + summary: "Store used with older format (v1/v2) is not allowed", + input: map[string]string{ + "chisel.yaml": strings.ReplaceAll(testutil.DefaultChiselYaml, "format: v1", "format: v2"), + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'store' and 'default-track' are unsupported before format v3`, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + archive: ubuntu + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, }, { summary: "Store and archive are mutually exclusive", input: map[string]string{ @@ -4145,11 +4168,11 @@ var setupTests = []setupTest{{ `, "slices/bin/mypkg.yaml": ` package: mypkg - store: no-such-store + store: non-existing default-track: "3.0" `, }, - relerror: `cannot parse package "mypkg": store "no-such-store" not defined in release`, + relerror: `cannot parse package "mypkg": store "non-existing" not defined in release`, }, { summary: "Store missing version", input: map[string]string{ @@ -4175,6 +4198,31 @@ var setupTests = []setupTest{{ `, }, relerror: `chisel.yaml: store "bin" missing version field`, +}, { + summary: "Store missing kind", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + version: 26.10 + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing kind field`, }, { summary: "Store unknown kind", input: map[string]string{ @@ -4436,9 +4484,9 @@ func runParseReleaseTests(c *C, tests []setupTest) { dir := c.MkDir() for path, data := range test.input { fpath := filepath.Join(dir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0o755) + err := os.MkdirAll(filepath.Dir(fpath), 0755) c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) c.Assert(err, IsNil) } @@ -4483,40 +4531,6 @@ func runParseReleaseTests(c *C, tests []setupTest) { } } -func (s *S) TestStoresNotSupportedInOldFormats(c *C) { - for _, format := range []string{"v1", "v2"} { - c.Logf("Format: %s", format) - chiselYaml := ` - format: ` + format + ` - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 22.04 - components: [main, universe] - suites: [jammy] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 22.04 - default-prefix: "bin-" - ` - dir := c.MkDir() - err := os.WriteFile(filepath.Join(dir, "chisel.yaml"), testutil.Reindent(chiselYaml), 0o644) - c.Assert(err, IsNil) - err = os.MkdirAll(filepath.Join(dir, "slices"), 0o755) - c.Assert(err, IsNil) - _, err = setup.ReadRelease(dir) - c.Assert(err, ErrorMatches, `chisel.yaml: cannot use stores in format "`+format+`"`) - } -} - func (s *S) TestPackageMarshalYAML(c *C) { for _, test := range setupTests { c.Logf("Summary: %s", test.summary) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 3f53c849a..53122a356 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -444,7 +444,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { // Parse stores. if len(yamlVar.Stores) > 0 && (release.Format == "v1" || release.Format == "v2") { - return nil, fmt.Errorf("%s: cannot use stores in format %q", fileName, release.Format) + return nil, fmt.Errorf("%s: 'stores' is unsupported before format v3", fileName) } if len(yamlVar.Stores) > 0 { release.Stores = make(map[string]*Store, len(yamlVar.Stores)) @@ -492,7 +492,7 @@ func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Pack } if (yamlPkg.Store != "" || yamlPkg.DefaultTrack != "") && (release.Format == "v1" || release.Format == "v2") { - return nil, fmt.Errorf("cannot parse package %q: 'store' and 'default-track' are not supported in format %q", pkgName, release.Format) + return nil, fmt.Errorf("cannot parse package %q: 'store' and 'default-track' are unsupported before format v3", pkgName) } if yamlPkg.Store != "" && yamlPkg.Archive != "" { return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 774ac2658..40840c672 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2022,27 +2022,7 @@ var storeSlicerTests = []slicerTest{{ slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, arch: "amd64", release: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 22.04 - components: [main, universe] - suites: [jammy] - public-keys: [test-key] - stores: - test-store: - kind: bin - version: 1.0 - default-prefix: bin- - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/mydir/test-package.yaml": ` package: test-package slices: @@ -2052,7 +2032,7 @@ var storeSlicerTests = []slicerTest{{ `, "slices/mydir/store-pkg.yaml": ` package: store-pkg - store: test-store + store: bin default-track: stable slices: myslice: diff --git a/internal/testutil/defaults.go b/internal/testutil/defaults.go index fa08b4e63..046d46360 100644 --- a/internal/testutil/defaults.go +++ b/internal/testutil/defaults.go @@ -1,5 +1,7 @@ package testutil +import "strings" + var testKey = PGPKeys["key1"] var DefaultChiselYaml = ` @@ -17,3 +19,11 @@ var DefaultChiselYaml = ` test-key: id: ` + testKey.ID + ` armor: |` + "\n" + PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + +var DefaultChiselYamlWithStores = strings.ReplaceAll(DefaultChiselYaml, "format: v1", "format: v3") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" +` From 65771cbcc2d60798637f19baf175d62a7dd8bff6 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 13:46:24 +0200 Subject: [PATCH 30/61] tests: integrate v3 tests to common table Signed-off-by: Paul Mars --- internal/slicer/slicer_test.go | 68 ++++++++++++++++------------------ 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 40840c672..d91bd4aa9 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1978,6 +1978,36 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, +}, { + summary: "Store package is skipped when fetching is not implemented", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: stable + slices: + myslice: + contents: + /dir/store-file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", + }, }} func (s *S) TestRun(c *C) { @@ -1989,7 +2019,7 @@ func (s *S) TestRun(c *C) { for _, t := range slicerTests { m := make(map[string]string) for k, v := range t.release { - if !strings.Contains(v, "v2-archives:") { + if !strings.Contains(v, "v2-archives:") && strings.Contains(v, "format: v1") { v = strings.ReplaceAll(v, "archives:", "v2-archives:") } m[k] = v @@ -2017,42 +2047,6 @@ func (s *S) TestRun(c *C) { runSlicerTests(s, c, v2FormatTests) } -var storeSlicerTests = []slicerTest{{ - summary: "Store package is skipped when fetching is not implemented", - slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, - arch: "amd64", - release: map[string]string{ - "chisel.yaml": testutil.DefaultChiselYamlWithStores, - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - myslice: - contents: - /dir/file: - `, - "slices/mydir/store-pkg.yaml": ` - package: store-pkg - store: bin - default-track: stable - slices: - myslice: - contents: - /dir/store-file: - `, - }, - filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/file": "file 0644 cc55e2ec", - }, - manifestPaths: map[string]string{ - "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", - }, -}} - -func (s *S) TestRunStorePackage(c *C) { - runSlicerTests(s, c, storeSlicerTests) -} - func runSlicerTests(s *S, c *C, tests []slicerTest) { for _, test := range tests { for _, testSlices := range testutil.Permutations(test.slices) { From d929e92bc2803bd1e1fe30bab888a1099b04cae7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 15:23:59 +0200 Subject: [PATCH 31/61] fix: check store kind when slices used Signed-off-by: Paul Mars --- internal/setup/setup.go | 15 ++++++++ internal/setup/setup_test.go | 68 ++++++++++++++++++++++-------------- internal/setup/yaml.go | 3 -- 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 7f6f76e71..67c478f62 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -517,6 +517,21 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error new, newPath, newInfo.Generate) } } + // An invalid store kind should only throw an error if a slice references it. + // Hence, the check is here. + pkg := release.Packages[new.Package] + if pkg.Store == "" { + continue + } + store, ok := selection.Release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("internal error: slice %s refers to missing store %q", new, pkg.Store) + } + switch store.Kind { + case "bin": + default: + return nil, fmt.Errorf("slice %s refers to store %q with unknown kind %q", new, pkg.Store, store.Kind) + } } return selection, nil diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index ab07320ca..8f63fccfb 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -4223,32 +4223,6 @@ var setupTests = []setupTest{{ `, }, relerror: `chisel.yaml: store "bin" missing kind field`, -}, { - summary: "Store unknown kind", - input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: unknown - version: 26.10 - default-prefix: "bin-" - `, - }, - relerror: `chisel.yaml: unknown store kind "unknown" for store "bin"`, }, { summary: "Store missing default-prefix", input: map[string]string{ @@ -4470,6 +4444,48 @@ func (s *S) TestParseRelease(c *C) { runParseReleaseTests(c, v3FormatTests) } +// TestSelectStoreUnknownKind is a dedicated test because the unknown store kind +// is only reported at selection time and only makes sense for the v3 format. +func (s *S) TestSelectStoreUnknownKind(c *C) { + runParseReleaseTests(c, []setupTest{{ + summary: "Store unknown kind", + selslices: []setup.SliceKey{{Package: "bin-mypkg", Slice: "myslice"}}, + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + stores: + bin: + kind: unknown + version: 26.10 + default-prefix: "bin-" + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /dir/file: {} + `, + }, + selerror: `slice bin-mypkg_myslice refers to store "bin" with unknown kind "unknown"`, + }}) +} + func runParseReleaseTests(c *C, tests []setupTest) { for _, test := range tests { c.Logf("Summary: %s", test.summary) diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index 53122a356..763ee781d 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -453,9 +453,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if details.Kind == "" { return nil, fmt.Errorf("%s: store %q missing kind field", fileName, storeName) } - if details.Kind != "bin" { - return nil, fmt.Errorf("%s: unknown store kind %q for store %q", fileName, details.Kind, storeName) - } if details.Version == "" { return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) } From 867fcff8361693b4237174477f040086c72eabf4 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 15:34:43 +0200 Subject: [PATCH 32/61] docs: clarify reason for temp arch filling Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 77e6fcbf4..5f23fe987 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -583,9 +583,10 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel } } - // Store packages do not have an archive to derive the architecture from. - // All packages in a selection share the same architecture, so we can - // borrow it from any archive package that was already resolved. + // Until a store is implemented as a package source there is no proper way to + // determine the architecture for store packages. + // So relying on the fact that all packages in a selection share the same architecture, + // we can borrow it from any archive package that was already resolved. var arch string for _, src := range pkgSources { if src.kind == sourceArchive { From 33fc4cacfaad07c2cc3bd01f68770babe59ae18c Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 11:27:22 +0200 Subject: [PATCH 33/61] fix: apply review comments Signed-off-by: Paul Mars --- internal/setup/setup_test.go | 164 +++-------------------------------- 1 file changed, 12 insertions(+), 152 deletions(-) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 8f63fccfb..0ba3fd814 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -152,7 +152,7 @@ var setupTests = []setupTest{{ "/file/path2": {Kind: "copy", Info: "/other/path"}, "/file/path3": {Kind: "symlink", Info: "/other/path"}, "/file/path4": {Kind: "text", Info: "content", Until: "mutate"}, - "/file/path5": {Kind: "copy", Mode: 0o755, Mutable: true}, + "/file/path5": {Kind: "copy", Mode: 0755, Mutable: true}, "/file/path6/": {Kind: "dir"}, }, }, @@ -3934,27 +3934,7 @@ var setupTests = []setupTest{{ }, { summary: "Store package is parsed correctly", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg store: bin @@ -3966,8 +3946,8 @@ var setupTests = []setupTest{{ Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", - Version: "26.10", - Suites: []string{"stonking"}, + Version: "22.04", + Suites: []string{"jammy"}, Components: []string{"main", "universe"}, PubKeys: []*packet.PublicKey{testKey.PubKey}, Maintained: true, @@ -4022,27 +4002,7 @@ var setupTests = []setupTest{{ }, { summary: "Store and archive are mutually exclusive", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg archive: ubuntu @@ -4054,27 +4014,7 @@ var setupTests = []setupTest{{ }, { summary: "Store package missing default-track (v3)", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg store: bin @@ -4084,27 +4024,7 @@ var setupTests = []setupTest{{ }, { summary: "default-track without store (v3)", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg default-track: "3.0" @@ -4114,27 +4034,7 @@ var setupTests = []setupTest{{ }, { summary: "default-track must not contain / (v3)", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg store: bin @@ -4145,27 +4045,7 @@ var setupTests = []setupTest{{ }, { summary: "Package store references undefined store (v3)", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/bin/mypkg.yaml": ` package: mypkg store: non-existing @@ -4251,27 +4131,7 @@ var setupTests = []setupTest{{ }, { summary: "Same-named package in archive and store", input: map[string]string{ - "chisel.yaml": ` - format: v3 - maintenance: - standard: 2025-01-01 - end-of-life: 2100-01-01 - archives: - ubuntu: - version: 26.10 - components: [main, universe] - suites: [stonking] - public-keys: [test-key] - public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - stores: - bin: - kind: bin - version: 26.10 - default-prefix: "bin-" - `, + "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/curl.yaml": ` package: curl slices: @@ -4297,8 +4157,8 @@ var setupTests = []setupTest{{ Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", - Version: "26.10", - Suites: []string{"stonking"}, + Version: "22.04", + Suites: []string{"jammy"}, Components: []string{"main", "universe"}, PubKeys: []*packet.PublicKey{testKey.PubKey}, Maintained: true, From c2be41a6dd465fcc9b2c9538f156ec7d98e49537 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 13:22:28 +0200 Subject: [PATCH 34/61] fx: clearly fail if store packages are found for now Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 33 +++++++++++++-------------------- internal/slicer/slicer_test.go | 10 ++-------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 5f23fe987..55ae4428c 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -112,27 +112,14 @@ func Run(options *RunOptions) error { return err } - // Store package processing is not yet implemented, so store slices - // are filtered out from the selection for now. - var archiveSlices []*setup.Slice - for _, slice := range options.Selection.Slices { - if pkgSources[slice.Package].kind != sourceStore { - archiveSlices = append(archiveSlices, slice) - } - } - selection := &setup.Selection{ - Release: options.Selection.Release, - Slices: archiveSlices, - } - - prefers, err := selection.Prefers() + prefers, err := options.Selection.Prefers() if err != nil { return err } // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) - for _, slice := range selection.Slices { + for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { extractPackage = make(map[string][]deb.ExtractInfo) @@ -178,11 +165,17 @@ func Run(options *RunOptions) error { // Fetch all packages, using the selection order. packages := make(map[string]io.ReadSeekCloser) var pkgInfos []*archive.PackageInfo - for _, slice := range selection.Slices { + for _, slice := range options.Selection.Slices { if packages[slice.Package] != nil { continue } src := pkgSources[slice.Package] + // Store packages are distributed as "ar" archives, whose extraction is + // not yet implemented. Fail until store handling and the "ar" format + // support are in place. + if src.kind == sourceStore { + return fmt.Errorf("cannot fetch package %q from store: store packages are not yet supported", src.pkg.Name) + } reader, info, err := src.archive.Fetch(src.pkg.RealName) if err != nil { return err @@ -264,7 +257,7 @@ func Run(options *RunOptions) error { } // Extract all packages, also using the selection order. - for _, slice := range selection.Slices { + for _, slice := range options.Selection.Slices { reader := packages[slice.Package] if reader == nil { continue @@ -299,7 +292,7 @@ func Run(options *RunOptions) error { // First group them by their relative path. Then create them and attribute // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} - for _, slice := range selection.Slices { + for _, slice := range options.Selection.Slices { src := pkgSources[slice.Package] for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { @@ -359,7 +352,7 @@ func Run(options *RunOptions) error { CheckRead: checker.checkKnown, OnWrite: report.Mutate, } - for _, slice := range selection.Slices { + for _, slice := range options.Selection.Slices { opts := scripts.RunOptions{ Label: "mutate", Script: slice.Scripts.Mutate, @@ -378,7 +371,7 @@ func Run(options *RunOptions) error { return err } - return generateManifests(targetDir, selection, report, pkgInfos) + return generateManifests(targetDir, options.Selection, report, pkgInfos) } func generateManifests(targetDir string, selection *setup.Selection, diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d91bd4aa9..ddfea6057 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1979,7 +1979,7 @@ var slicerTests = []slicerTest{{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, }, { - summary: "Store package is skipped when fetching is not implemented", + summary: "Store package fails as it is not yet supported", slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, arch: "amd64", release: map[string]string{ @@ -2001,13 +2001,7 @@ var slicerTests = []slicerTest{{ /dir/store-file: `, }, - filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/file": "file 0644 cc55e2ec", - }, - manifestPaths: map[string]string{ - "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", - }, + error: `cannot fetch package "bin-store-pkg" from store: store packages are not yet supported`, }} func (s *S) TestRun(c *C) { From a816047b7d75bf8b18356ccbd8e604933d92bc41 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 16:08:08 +0200 Subject: [PATCH 35/61] fix: remove wrong field Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 55ae4428c..3c5b11485 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -50,7 +50,7 @@ type pkgSourceInfo struct { arch string kind sourceKind archive archive.Archive - store *setup.Store + // TODO: add store handle when store support is implemented. pkg *setup.Package } @@ -539,10 +539,9 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel } pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { - store := selection.Release.Stores[pkg.Store] pkgSources[pkg.Name] = &pkgSourceInfo{ + // TODO: Fill with the live store handle when store support is implemented. kind: sourceStore, - store: store, pkg: pkg, } continue From 9d86eba1248f912cd086e3f5c99aa033e4a5c68c Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 24 Jun 2026 10:32:33 +0200 Subject: [PATCH 36/61] style: lint error Signed-off-by: Paul Mars --- internal/slicer/slicer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 3c5b11485..a5911e9e8 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -51,7 +51,7 @@ type pkgSourceInfo struct { kind sourceKind archive archive.Archive // TODO: add store handle when store support is implemented. - pkg *setup.Package + pkg *setup.Package } type contentChecker struct { @@ -541,8 +541,8 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel if pkg.Store != "" { pkgSources[pkg.Name] = &pkgSourceInfo{ // TODO: Fill with the live store handle when store support is implemented. - kind: sourceStore, - pkg: pkg, + kind: sourceStore, + pkg: pkg, } continue } From 2c7d66f3ef453a5f4856e568c815da7375ee9d7a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 11:21:49 +0200 Subject: [PATCH 37/61] ci: rerun From 643a52fbd2460e9f1f163f98d11ec73760d6a880 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 16:13:49 +0200 Subject: [PATCH 38/61] feat: draft accessing bin store --- cmd/chisel/cmd_cut.go | 15 ++ cmd/chisel/main.go | 2 + internal/slicer/slicer.go | 2 + internal/store/export_test.go | 20 ++ internal/store/log.go | 53 +++++ internal/store/store.go | 274 ++++++++++++++++++++++ internal/store/store_test.go | 422 ++++++++++++++++++++++++++++++++++ internal/store/suite_test.go | 13 ++ 8 files changed, 801 insertions(+) create mode 100644 internal/store/export_test.go create mode 100644 internal/store/log.go create mode 100644 internal/store/store.go create mode 100644 internal/store/store_test.go create mode 100644 internal/store/suite_test.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79a..a37967312 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -8,6 +8,7 @@ import ( "github.com/jessevdk/go-flags" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" @@ -121,9 +122,23 @@ func (cmd *cmdCut) Execute(args []string) error { } } + stores := make(map[string]store.Store) + for _, storeInfo := range release.Stores { + openStore, err := store.Open(&store.Options{ + Arch: cmd.Arch, + CacheDir: cache.DefaultDir("chisel"), + }) + if err != nil { + return err + } + stores[storeInfo.Name] = openStore + break + } + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: cmd.RootDir, }) return err diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index 441d1a543..9ab8a510a 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -13,6 +13,7 @@ import ( "golang.org/x/term" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" @@ -327,6 +328,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + store.SetLogger(log.Default()) SetLogger(log.Default()) parser := Parser() diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index a5911e9e8..07b4ac7d2 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,6 +16,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" @@ -28,6 +29,7 @@ const manifestMode fs.FileMode = 0644 type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive + Stores map[string]store.Store TargetDir string } diff --git a/internal/store/export_test.go b/internal/store/export_test.go new file mode 100644 index 000000000..b4c64aadb --- /dev/null +++ b/internal/store/export_test.go @@ -0,0 +1,20 @@ +package store + +import "net/http" + +var ( + ValidateDownloadURL = validateDownloadURL + StagingEnvVar = stagingEnvVar +) + +func SetHTTPDo(fn func(req *http.Request) (*http.Response, error)) (restore func()) { + saved := httpDo + httpDo = fn + return func() { httpDo = saved } +} + +func SetBulkDo(fn func(req *http.Request) (*http.Response, error)) (restore func()) { + saved := bulkDo + bulkDo = fn + return func() { bulkDo = saved } +} diff --git a/internal/store/log.go b/internal/store/log.go new file mode 100644 index 000000000..d8c11a443 --- /dev/null +++ b/internal/store/log.go @@ -0,0 +1,53 @@ +package store + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 000000000..c169ed3c2 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,274 @@ +package store + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/deb" +) + +// Store provides access to bin packages from the Snapcraft store API. +type Store interface { + Fetch(name, track, risk string) (io.ReadSeekCloser, *BinPackageInfo, error) + Exists(name, track, risk string) bool + Info(name, track, risk string) (*BinPackageInfo, error) +} + +// BinPackageInfo holds metadata about a bin package. +type BinPackageInfo struct { + Name string + Version string + Revision int + SHA384 string +} + +// Options configures a bin source. +type Options struct { + Arch string + CacheDir string +} + +type binStore struct { + options Options + cache *cache.Cache + apiURL string +} + +const ( + binsAPIBase = "https://api.snapcraft.io/v2/bins" + binsAPIStaging = "https://api.staging.snapcraft.io/v2/bins" + stagingEnvVar = "CHISEL_BINS_STAGING" +) + +var httpClient = &http.Client{ + Timeout: 30 * time.Second, +} + +var httpDo = httpClient.Do + +var bulkClient = &http.Client{ + Timeout: 5 * time.Minute, +} + +var bulkDo = bulkClient.Do + +// Open creates a new bin source with the given options. +func Open(options *Options) (Store, error) { + var err error + if options.Arch == "" { + options.Arch, err = deb.InferArch() + } else { + err = deb.ValidateArch(options.Arch) + } + if err != nil { + return nil, err + } + apiURL := binsAPIBase + if os.Getenv(stagingEnvVar) != "" { + apiURL = binsAPIStaging + } + return &binStore{ + options: *options, + cache: &cache.Cache{Dir: options.CacheDir}, + apiURL: apiURL, + }, nil +} + +// binInfoResponse represents the JSON response from the bins info endpoint. +type binInfoResponse struct { + Name string `json:"name"` + PackageID string `json:"package-id"` + ChannelMap []struct { + Channel struct { + Name string `json:"name"` + Risk string `json:"risk"` + Track string `json:"track"` + Platform struct { + Architecture string `json:"architecture"` + } `json:"platform"` + } `json:"channel"` + Revision struct { + Version string `json:"version"` + Revision int `json:"revision"` + Download struct { + URL string `json:"url"` + SHA3384 string `json:"sha3-384"` + Size int64 `json:"size"` + } `json:"download"` + Platforms []struct { + Architecture string `json:"architecture"` + } `json:"platforms"` + } `json:"revision"` + } `json:"channel-map"` +} + +func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { + infoURL, err := url.JoinPath(s.apiURL, "info", name) + if err != nil { + return nil, fmt.Errorf("internal error: cannot construct bins API URL: %v", err) + } + infoURL += "?fields=download,version,revision,channel-map" + + req, err := http.NewRequest("GET", infoURL, nil) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + + resp, err := httpDo(req) + if err != nil { + return nil, fmt.Errorf("cannot talk to bins API: %v", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case 200: + // ok + case 404: + return nil, fmt.Errorf("bin %q not found", name) + default: + return nil, fmt.Errorf("cannot fetch from bins API: %v", resp.Status) + } + + var info binInfoResponse + err = json.NewDecoder(resp.Body).Decode(&info) + if err != nil { + return nil, fmt.Errorf("cannot decode bins API response: %v", err) + } + return &info, nil +} + +// selectRevision finds the channel-map entry matching the requested track, +// risk, and architecture. +func selectRevision(info *binInfoResponse, arch, track, risk string) (downloadURL, sha3384, version string, revision int, err error) { + for _, entry := range info.ChannelMap { + if entry.Channel.Track != track || entry.Channel.Risk != risk { + continue + } + if entry.Channel.Platform.Architecture != arch { + continue + } + return entry.Revision.Download.URL, entry.Revision.Download.SHA3384, + entry.Revision.Version, entry.Revision.Revision, nil + } + return "", "", "", 0, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) +} + +// allowedDownloadHosts lists the hosts from which bin downloads are permitted. +var allowedDownloadHosts = []string{ + "api.snapcraft.io", + "api.staging.snapcraft.io", + "storage.snapcraftcontent.com", +} + +// validateDownloadURL checks that the download URL is HTTPS and from an +// allowed host. +func validateDownloadURL(downloadURL string) error { + u, err := url.Parse(downloadURL) + if err != nil { + return fmt.Errorf("cannot parse bin download URL: %v", err) + } + if u.Scheme != "https" { + return fmt.Errorf("bin download URL must use HTTPS: %q", downloadURL) + } + for _, host := range allowedDownloadHosts { + if u.Host == host || strings.HasSuffix(u.Host, "."+host) { + return nil + } + } + return fmt.Errorf("bin download URL has untrusted host %q", u.Host) +} + +func (s *binStore) Info(name, track, risk string) (*BinPackageInfo, error) { + resp, err := s.fetchBinInfo(name) + if err != nil { + return nil, err + } + _, sha384, version, revision, err := selectRevision(resp, s.options.Arch, track, risk) + if err != nil { + return nil, err + } + return &BinPackageInfo{ + Name: name, + Version: version, + Revision: revision, + SHA384: sha384, + }, nil +} + +func (s *binStore) Exists(name, track, risk string) bool { + _, err := s.Info(name, track, risk) + return err == nil +} + +func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *BinPackageInfo, error) { + logf("Fetching bin %s %s/%s ...", name, track, risk) + + resp, err := s.fetchBinInfo(name) + if err != nil { + return nil, nil, err + } + downloadURL, sha384, version, revision, err := selectRevision(resp, s.options.Arch, track, risk) + if err != nil { + return nil, nil, err + } + + info := &BinPackageInfo{ + Name: name, + Version: version, + Revision: revision, + SHA384: sha384, + } + + // Check cache first. + reader, err := s.cache.Open(cache.SHA384, sha384) + if err == nil { + logf("Using cached bin %s", name) + return reader, info, nil + } else if err != cache.ErrMiss { + return nil, nil, err + } + + // Download the bin. + err = validateDownloadURL(downloadURL) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequest("GET", downloadURL, nil) + if err != nil { + return nil, nil, fmt.Errorf("cannot create HTTP request: %v", err) + } + + httpResp, err := bulkDo(req) + if err != nil { + return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode != 200 { + return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, httpResp.Status) + } + + writer := s.cache.Create(cache.SHA384, sha384) + defer writer.Close() + + _, err = io.Copy(writer, httpResp.Body) + if err == nil { + err = writer.Close() + } + if err != nil { + return nil, nil, fmt.Errorf("cannot fetch bin %q: %v", name, err) + } + + reader, err = s.cache.Open(cache.SHA384, sha384) + if err != nil { + return nil, nil, err + } + return reader, info, nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 000000000..5291c3df2 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,422 @@ +package store_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/sha3" + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/store" + "github.com/canonical/chisel/internal/testutil" +) + +type storeSuite struct { + tempDir string + cacheDir string + fakeDoFunc func(req *http.Request) (*http.Response, error) + restoreDo func() + restoreBulk func() + envRestore func() +} + +var _ = Suite(&storeSuite{}) + +func (s *storeSuite) SetUpTest(c *C) { + s.tempDir = c.MkDir() + s.cacheDir = filepath.Join(s.tempDir, "cache") + c.Assert(os.MkdirAll(s.cacheDir, 0o755), IsNil) + + s.envRestore = fakeEnv("") + s.restoreDo = store.SetHTTPDo(s.doRequest) + s.restoreBulk = store.SetBulkDo(s.doRequest) + s.fakeDoFunc = nil +} + +func (s *storeSuite) TearDownTest(c *C) { + s.restoreDo() + s.restoreBulk() + s.envRestore() +} + +func (s *storeSuite) doRequest(req *http.Request) (*http.Response, error) { + if s.fakeDoFunc != nil { + return s.fakeDoFunc(req) + } + return nil, fmt.Errorf("unexpected HTTP request: %s", req.URL.String()) +} + +func fakeEnv(staging string) func() { + oldStaging := os.Getenv(store.StagingEnvVar) + if staging != "" { + os.Setenv(store.StagingEnvVar, staging) + } else { + os.Unsetenv(store.StagingEnvVar) + } + return func() { + if oldStaging != "" { + os.Setenv(store.StagingEnvVar, oldStaging) + } else { + os.Unsetenv(store.StagingEnvVar) + } + } +} + +func sha384Hash(data []byte) string { + h := sha3.New384() + h.Write(data) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func makeBinInfoResponse(name, track, risk, arch, version string, revision int, sha384 string) *binInfoResponseJSON { + return &binInfoResponseJSON{ + Name: name, + ChannelMap: []binChannelMapJSON{ + { + Channel: binChannelJSON{ + Name: track + "/" + risk, + Risk: risk, + Track: track, + Platform: binPlatformJSON{ + Architecture: arch, + }, + }, + Revision: binRevisionJSON{ + Version: version, + Revision: revision, + Download: binDownloadJSON{ + URL: "https://storage.snapcraftcontent.com/bins/" + name + ".tar.xz", + SHA384: sha384, + Size: 1024, + }, + }, + }, + }, + } +} + +// JSON structures matching the internal binInfoResponse for test construction. +type binInfoResponseJSON struct { + Name string `json:"name"` + ChannelMap []binChannelMapJSON `json:"channel-map"` +} + +type binChannelMapJSON struct { + Channel binChannelJSON `json:"channel"` + Revision binRevisionJSON `json:"revision"` +} + +type binChannelJSON struct { + Name string `json:"name"` + Risk string `json:"risk"` + Track string `json:"track"` + Platform binPlatformJSON `json:"platform"` +} + +type binPlatformJSON struct { + Architecture string `json:"architecture"` +} + +type binRevisionJSON struct { + Version string `json:"version"` + Revision int `json:"revision"` + Download binDownloadJSON `json:"download"` +} + +type binDownloadJSON struct { + URL string `json:"url"` + SHA384 string `json:"sha3-384"` + Size int64 `json:"size"` +} + +func (s *storeSuite) TestValidateDownloadURL(c *C) { + tests := []struct { + url string + errStr string + }{ + {"https://storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, + {"https://api.snapcraft.io/v2/bins/foo", ""}, + {"https://api.staging.snapcraft.io/v2/bins/foo", ""}, + {"https://sub.storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, + {"http://storage.snapcraftcontent.com/bins/foo.tar.xz", "must use HTTPS"}, + {"https://evil.example.com/bins/foo.tar.xz", "untrusted host"}, + {"https://storage.snapcraftcontent.com.evil.com/bins/foo.tar.xz", "untrusted host"}, + {"://invalid-url", "cannot parse"}, + } + for _, test := range tests { + err := store.ValidateDownloadURL(test.url) + if test.errStr == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, NotNil) + c.Assert(err.Error(), testutil.Contains, test.errStr) + } + } +} + +func (s *storeSuite) TestOpenArchValidation(c *C) { + tests := []struct { + arch string + errStr string + }{ + {"amd64", ""}, + {"arm64", ""}, + {"invalid", "invalid package architecture"}, + } + for _, test := range tests { + _, err := store.Open(&store.Options{ + Arch: test.arch, + CacheDir: s.cacheDir, + }) + if test.errStr == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, NotNil) + c.Assert(err.Error(), testutil.Contains, test.errStr) + } + } +} + +func (s *storeSuite) TestInfoSuccess(c *C) { + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody, _ := json.Marshal(infoResp) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + c.Assert(req.URL.Path, Equals, "/v2/bins/info/curl") + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + info, err := src.Info("curl", "latest", "stable") + c.Assert(err, IsNil) + c.Assert(info.Name, Equals, "curl") + c.Assert(info.Version, Equals, "8.5.0") + c.Assert(info.Revision, Equals, 42) + c.Assert(info.SHA384, Equals, "abc123") +} + +func (s *storeSuite) TestInfoNotFound(c *C) { + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 404, + Body: io.NopCloser(strings.NewReader("not found")), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + _, err = src.Info("nonexistent", "latest", "stable") + c.Assert(err, NotNil) + c.Assert(err.Error(), testutil.Contains, "not found") +} + +func (s *storeSuite) TestInfoNoMatchingChannel(c *C) { + infoResp := makeBinInfoResponse("curl", "latest", "stable", "arm64", "8.5.0", 42, "abc123") + infoBody, _ := json.Marshal(infoResp) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + // The response only has arm64, but we're asking for amd64. + _, err = src.Info("curl", "latest", "stable") + c.Assert(err, NotNil) + c.Assert(err.Error(), testutil.Contains, "has no") +} + +func (s *storeSuite) TestExists(c *C) { + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody, _ := json.Marshal(infoResp) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if strings.Contains(req.URL.Path, "/info/curl") { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + return &http.Response{ + StatusCode: 404, + Body: io.NopCloser(strings.NewReader("not found")), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + c.Assert(src.Exists("curl", "latest", "stable"), Equals, true) + c.Assert(src.Exists("nonexistent", "latest", "stable"), Equals, false) +} + +func (s *storeSuite) TestFetchCacheMiss(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + infoBody, _ := json.Marshal(infoResp) + + callCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + callCount++ + if strings.Contains(req.URL.Path, "/info/") { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + // Download URL + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.Version, Equals, "8.5.0") + c.Assert(info.SHA384, Equals, digest) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) + c.Assert(callCount, Equals, 2) + + // Verify it's in the cache. + cc := cache.Cache{Dir: s.cacheDir} + cached, err := cc.Read(cache.SHA384, digest) + c.Assert(err, IsNil) + c.Assert(cached, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchCacheHit(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + + // Pre-populate the cache. + cc := cache.Cache{Dir: s.cacheDir} + err := cc.Write(cache.SHA384, digest, tarData) + c.Assert(err, IsNil) + + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + infoBody, _ := json.Marshal(infoResp) + + infoCallCount := 0 + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if strings.Contains(req.URL.Path, "/info/") { + infoCallCount++ + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + return nil, fmt.Errorf("download should not be called for cache hit") + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + reader, info, err := src.Fetch("curl", "latest", "stable") + c.Assert(err, IsNil) + defer reader.Close() + + c.Assert(info.Name, Equals, "curl") + c.Assert(info.SHA384, Equals, digest) + c.Assert(infoCallCount, Equals, 1) + + data, err := io.ReadAll(reader) + c.Assert(err, IsNil) + c.Assert(data, DeepEquals, tarData) +} + +func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + // Override the download URL to an invalid one. + infoResp.ChannelMap[0].Revision.Download.URL = "http://evil.example.com/bins/curl.tar.xz" + infoBody, _ := json.Marshal(infoResp) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, NotNil) + c.Assert(err.Error(), testutil.Contains, "must use HTTPS") +} + +func (s *storeSuite) TestStagingEnvVar(c *C) { + s.envRestore() + s.envRestore = fakeEnv("1") + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + }) + c.Assert(err, IsNil) + + infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody, _ := json.Marshal(infoResp) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + // Verify staging URL is used. + c.Assert(req.URL.Host, Equals, "api.staging.snapcraft.io") + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + + _, err = src.Info("curl", "latest", "stable") + c.Assert(err, IsNil) +} diff --git a/internal/store/suite_test.go b/internal/store/suite_test.go new file mode 100644 index 000000000..552ab1cc8 --- /dev/null +++ b/internal/store/suite_test.go @@ -0,0 +1,13 @@ +package store_test + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) From 46a353ac8a78f772d93f489d4e063ac620dfd946 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 16:30:59 +0200 Subject: [PATCH 39/61] fix: adjust naming --- cmd/chisel/cmd_cut.go | 1 + internal/store/store.go | 53 ++++++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index a37967312..db56124a0 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -127,6 +127,7 @@ func (cmd *cmdCut) Execute(args []string) error { openStore, err := store.Open(&store.Options{ Arch: cmd.Arch, CacheDir: cache.DefaultDir("chisel"), + Kind: storeInfo.Kind, }) if err != nil { return err diff --git a/internal/store/store.go b/internal/store/store.go index c169ed3c2..25dd68996 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -14,27 +14,31 @@ import ( "github.com/canonical/chisel/internal/deb" ) -// Store provides access to bin packages from the Snapcraft store API. +// Store provides access to packages from the Snapcraft store API. type Store interface { - Fetch(name, track, risk string) (io.ReadSeekCloser, *BinPackageInfo, error) + Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) Exists(name, track, risk string) bool - Info(name, track, risk string) (*BinPackageInfo, error) + Info(name, track, risk string) (*StorePackageInfo, error) } -// BinPackageInfo holds metadata about a bin package. -type BinPackageInfo struct { +// StorePackageInfo holds metadata about a package. +type StorePackageInfo struct { Name string Version string Revision int SHA384 string } -// Options configures a bin source. type Options struct { Arch string CacheDir string + Kind string } +type storeKind string + +const storeKindBin storeKind = "bin" + type binStore struct { options Options cache *cache.Cache @@ -42,9 +46,9 @@ type binStore struct { } const ( - binsAPIBase = "https://api.snapcraft.io/v2/bins" - binsAPIStaging = "https://api.staging.snapcraft.io/v2/bins" - stagingEnvVar = "CHISEL_BINS_STAGING" + binAPIBase = "https://api.snapcraft.io/v2/bins" + binAPIStaging = "https://api.staging.snapcraft.io/v2/bins" + stagingEnvVar = "CHISEL_BIN_STAGING" ) var httpClient = &http.Client{ @@ -59,7 +63,6 @@ var bulkClient = &http.Client{ var bulkDo = bulkClient.Do -// Open creates a new bin source with the given options. func Open(options *Options) (Store, error) { var err error if options.Arch == "" { @@ -70,15 +73,21 @@ func Open(options *Options) (Store, error) { if err != nil { return nil, err } - apiURL := binsAPIBase - if os.Getenv(stagingEnvVar) != "" { - apiURL = binsAPIStaging + + switch storeKind(options.Kind) { + case storeKindBin: + apiURL := binAPIBase + if os.Getenv(stagingEnvVar) != "" { + apiURL = binAPIStaging + } + return &binStore{ + options: *options, + cache: &cache.Cache{Dir: options.CacheDir}, + apiURL: apiURL, + }, nil + default: + return nil, fmt.Errorf("unsupported store kind %q", options.Kind) } - return &binStore{ - options: *options, - cache: &cache.Cache{Dir: options.CacheDir}, - apiURL: apiURL, - }, nil } // binInfoResponse represents the JSON response from the bins info endpoint. @@ -185,7 +194,7 @@ func validateDownloadURL(downloadURL string) error { return fmt.Errorf("bin download URL has untrusted host %q", u.Host) } -func (s *binStore) Info(name, track, risk string) (*BinPackageInfo, error) { +func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { resp, err := s.fetchBinInfo(name) if err != nil { return nil, err @@ -194,7 +203,7 @@ func (s *binStore) Info(name, track, risk string) (*BinPackageInfo, error) { if err != nil { return nil, err } - return &BinPackageInfo{ + return &StorePackageInfo{ Name: name, Version: version, Revision: revision, @@ -207,7 +216,7 @@ func (s *binStore) Exists(name, track, risk string) bool { return err == nil } -func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *BinPackageInfo, error) { +func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { logf("Fetching bin %s %s/%s ...", name, track, risk) resp, err := s.fetchBinInfo(name) @@ -219,7 +228,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *BinPacka return nil, nil, err } - info := &BinPackageInfo{ + info := &StorePackageInfo{ Name: name, Version: version, Revision: revision, From dac1ec7afcb75addab4659fc20ae62481c0d8bf2 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 19 Jun 2026 16:45:57 +0200 Subject: [PATCH 40/61] style: linting errors --- cmd/chisel/cmd_cut.go | 2 +- cmd/chisel/main.go | 2 +- internal/slicer/slicer.go | 2 +- internal/store/store.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index db56124a0..6e4db2255 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -8,10 +8,10 @@ import ( "github.com/jessevdk/go-flags" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" ) var shortCutHelp = "Cut a tree with selected slices" diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index 9ab8a510a..a982e3ef3 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -13,10 +13,10 @@ import ( "golang.org/x/term" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" //"github.com/canonical/chisel/internal/logger" ) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 07b4ac7d2..2ac4cd083 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,12 +16,12 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/store" ) const manifestMode fs.FileMode = 0644 diff --git a/internal/store/store.go b/internal/store/store.go index 25dd68996..03ff28eef 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -32,7 +32,7 @@ type StorePackageInfo struct { type Options struct { Arch string CacheDir string - Kind string + Kind string } type storeKind string @@ -48,7 +48,7 @@ type binStore struct { const ( binAPIBase = "https://api.snapcraft.io/v2/bins" binAPIStaging = "https://api.staging.snapcraft.io/v2/bins" - stagingEnvVar = "CHISEL_BIN_STAGING" + stagingEnvVar = "CHISEL_BIN_STAGING" ) var httpClient = &http.Client{ From 97bfd5bcbf7bde930a90ea2664d6d4e669356938 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 14:18:44 +0200 Subject: [PATCH 41/61] feat: refining --- internal/store/export_test.go | 2 +- internal/store/store.go | 27 ++- internal/store/store_test.go | 323 ++++++++++++++++++++-------------- 3 files changed, 205 insertions(+), 147 deletions(-) diff --git a/internal/store/export_test.go b/internal/store/export_test.go index b4c64aadb..f69322be8 100644 --- a/internal/store/export_test.go +++ b/internal/store/export_test.go @@ -4,7 +4,7 @@ import "net/http" var ( ValidateDownloadURL = validateDownloadURL - StagingEnvVar = stagingEnvVar + BinStagingEnvVar = binStagingEnvVar ) func SetHTTPDo(fn func(req *http.Request) (*http.Response, error)) (restore func()) { diff --git a/internal/store/store.go b/internal/store/store.go index 03ff28eef..ec39cd45e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "regexp" "strings" "time" @@ -46,9 +47,9 @@ type binStore struct { } const ( - binAPIBase = "https://api.snapcraft.io/v2/bins" - binAPIStaging = "https://api.staging.snapcraft.io/v2/bins" - stagingEnvVar = "CHISEL_BIN_STAGING" + binAPIBase = "https://api.snapcraft.io/v2/bins" + binAPIStaging = "https://api.staging.snapcraft.io/v2/bins" + binStagingEnvVar = "CHISEL_BIN_STAGING" ) var httpClient = &http.Client{ @@ -77,7 +78,7 @@ func Open(options *Options) (Store, error) { switch storeKind(options.Kind) { case storeKindBin: apiURL := binAPIBase - if os.Getenv(stagingEnvVar) != "" { + if os.Getenv(binStagingEnvVar) != "" { apiURL = binAPIStaging } return &binStore{ @@ -90,7 +91,7 @@ func Open(options *Options) (Store, error) { } } -// binInfoResponse represents the JSON response from the bins info endpoint. +// binInfoResponse represents the JSON response from the bin store info endpoint. type binInfoResponse struct { Name string `json:"name"` PackageID string `json:"package-id"` @@ -119,9 +120,12 @@ type binInfoResponse struct { } func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { + if !nameExp.MatchString(name) { + return nil, fmt.Errorf("invalid package name %q", name) + } infoURL, err := url.JoinPath(s.apiURL, "info", name) if err != nil { - return nil, fmt.Errorf("internal error: cannot construct bins API URL: %v", err) + return nil, fmt.Errorf("internal error: cannot construct bin store URL: %v", err) } infoURL += "?fields=download,version,revision,channel-map" @@ -132,7 +136,7 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { resp, err := httpDo(req) if err != nil { - return nil, fmt.Errorf("cannot talk to bins API: %v", err) + return nil, fmt.Errorf("cannot talk to bin store: %v", err) } defer resp.Body.Close() @@ -142,13 +146,13 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { case 404: return nil, fmt.Errorf("bin %q not found", name) default: - return nil, fmt.Errorf("cannot fetch from bins API: %v", resp.Status) + return nil, fmt.Errorf("cannot fetch from bin store: %v", resp.Status) } var info binInfoResponse err = json.NewDecoder(resp.Body).Decode(&info) if err != nil { - return nil, fmt.Errorf("cannot decode bins API response: %v", err) + return nil, fmt.Errorf("cannot decode bin store response: %v", err) } return &info, nil } @@ -169,6 +173,11 @@ func selectRevision(info *binInfoResponse, arch, track, risk string) (downloadUR return "", "", "", 0, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) } +// nameExp matches a valid package name. It deliberately forbids "/" and any +// leading "." so that a name cannot be used to traverse or otherwise alter the +// store API URL path when interpolated into it. +var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) + // allowedDownloadHosts lists the hosts from which bin downloads are permitted. var allowedDownloadHosts = []string{ "api.snapcraft.io", diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5291c3df2..e1525c0bd 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,7 +2,6 @@ package store_test import ( "bytes" - "encoding/json" "fmt" "io" "net/http" @@ -15,7 +14,6 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/store" - "github.com/canonical/chisel/internal/testutil" ) type storeSuite struct { @@ -54,17 +52,17 @@ func (s *storeSuite) doRequest(req *http.Request) (*http.Response, error) { } func fakeEnv(staging string) func() { - oldStaging := os.Getenv(store.StagingEnvVar) + oldStaging := os.Getenv(store.BinStagingEnvVar) if staging != "" { - os.Setenv(store.StagingEnvVar, staging) + os.Setenv(store.BinStagingEnvVar, staging) } else { - os.Unsetenv(store.StagingEnvVar) + os.Unsetenv(store.BinStagingEnvVar) } return func() { if oldStaging != "" { - os.Setenv(store.StagingEnvVar, oldStaging) + os.Setenv(store.BinStagingEnvVar, oldStaging) } else { - os.Unsetenv(store.StagingEnvVar) + os.Unsetenv(store.BinStagingEnvVar) } } } @@ -75,165 +73,181 @@ func sha384Hash(data []byte) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func makeBinInfoResponse(name, track, risk, arch, version string, revision int, sha384 string) *binInfoResponseJSON { - return &binInfoResponseJSON{ - Name: name, - ChannelMap: []binChannelMapJSON{ - { - Channel: binChannelJSON{ - Name: track + "/" + risk, - Risk: risk, - Track: track, - Platform: binPlatformJSON{ - Architecture: arch, - }, - }, - Revision: binRevisionJSON{ - Version: version, - Revision: revision, - Download: binDownloadJSON{ - URL: "https://storage.snapcraftcontent.com/bins/" + name + ".tar.xz", - SHA384: sha384, - Size: 1024, - }, - }, - }, - }, - } -} - -// JSON structures matching the internal binInfoResponse for test construction. -type binInfoResponseJSON struct { - Name string `json:"name"` - ChannelMap []binChannelMapJSON `json:"channel-map"` -} - -type binChannelMapJSON struct { - Channel binChannelJSON `json:"channel"` - Revision binRevisionJSON `json:"revision"` -} - -type binChannelJSON struct { - Name string `json:"name"` - Risk string `json:"risk"` - Track string `json:"track"` - Platform binPlatformJSON `json:"platform"` -} - -type binPlatformJSON struct { - Architecture string `json:"architecture"` +func makeBinInfoBody(name, track, risk, arch, version string, revision int, sha384 string) []byte { + downloadURL := "https://storage.snapcraftcontent.com/bins/" + name + ".tar.xz" + return makeBinInfoBodyWithURL(name, track, risk, arch, version, revision, sha384, downloadURL) } -type binRevisionJSON struct { - Version string `json:"version"` - Revision int `json:"revision"` - Download binDownloadJSON `json:"download"` -} - -type binDownloadJSON struct { - URL string `json:"url"` - SHA384 string `json:"sha3-384"` - Size int64 `json:"size"` +func makeBinInfoBodyWithURL(name, track, risk, arch, version string, revision int, sha384, downloadURL string) []byte { + return []byte(fmt.Sprintf(`{ + "name": %q, + "channel-map": [ + { + "channel": { + "name": %q, + "risk": %q, + "track": %q, + "platform": {"architecture": %q} + }, + "revision": { + "version": %q, + "revision": %d, + "download": { + "url": %q, + "sha3-384": %q, + "size": 1024 + } + } + } + ] + }`, name, track+"/"+risk, risk, track, arch, version, revision, downloadURL, sha384)) } func (s *storeSuite) TestValidateDownloadURL(c *C) { tests := []struct { - url string - errStr string + url string + error string }{ {"https://storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, {"https://api.snapcraft.io/v2/bins/foo", ""}, {"https://api.staging.snapcraft.io/v2/bins/foo", ""}, {"https://sub.storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, - {"http://storage.snapcraftcontent.com/bins/foo.tar.xz", "must use HTTPS"}, - {"https://evil.example.com/bins/foo.tar.xz", "untrusted host"}, - {"https://storage.snapcraftcontent.com.evil.com/bins/foo.tar.xz", "untrusted host"}, - {"://invalid-url", "cannot parse"}, + { + "http://storage.snapcraftcontent.com/bins/foo.tar.xz", + `bin download URL must use HTTPS: "http://storage.snapcraftcontent.com/bins/foo.tar.xz"`, + }, + { + "https://evil.example.com/bins/foo.tar.xz", + `bin download URL has untrusted host "evil.example.com"`, + }, + { + "https://storage.snapcraftcontent.com.evil.com/bins/foo.tar.xz", + `bin download URL has untrusted host "storage.snapcraftcontent.com.evil.com"`, + }, + {"://invalid-url", `cannot parse bin download URL: .*`}, } for _, test := range tests { err := store.ValidateDownloadURL(test.url) - if test.errStr == "" { + if test.error == "" { c.Assert(err, IsNil) } else { - c.Assert(err, NotNil) - c.Assert(err.Error(), testutil.Contains, test.errStr) + c.Assert(err, ErrorMatches, test.error) } } } func (s *storeSuite) TestOpenArchValidation(c *C) { tests := []struct { - arch string - errStr string + arch string + error string }{ {"amd64", ""}, {"arm64", ""}, - {"invalid", "invalid package architecture"}, + {"invalid", "invalid package architecture: invalid"}, } for _, test := range tests { _, err := store.Open(&store.Options{ Arch: test.arch, CacheDir: s.cacheDir, + Kind: "bin", }) - if test.errStr == "" { + if test.error == "" { c.Assert(err, IsNil) } else { - c.Assert(err, NotNil) - c.Assert(err.Error(), testutil.Contains, test.errStr) + c.Assert(err, ErrorMatches, test.error) } } } -func (s *storeSuite) TestInfoSuccess(c *C) { - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") - infoBody, _ := json.Marshal(infoResp) - - s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - c.Assert(req.URL.Path, Equals, "/v2/bins/info/curl") - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), - }, nil - } - - src, err := store.Open(&store.Options{ - Arch: "amd64", - CacheDir: s.cacheDir, - }) - c.Assert(err, IsNil) - - info, err := src.Info("curl", "latest", "stable") - c.Assert(err, IsNil) - c.Assert(info.Name, Equals, "curl") - c.Assert(info.Version, Equals, "8.5.0") - c.Assert(info.Revision, Equals, 42) - c.Assert(info.SHA384, Equals, "abc123") +type infoTest struct { + summary string + status int + statusText string + body string + info *store.StorePackageInfo + error string } -func (s *storeSuite) TestInfoNotFound(c *C) { - s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 404, - Body: io.NopCloser(strings.NewReader("not found")), - }, nil - } +var infoTests = []infoTest{{ + summary: "Successful info", + status: 200, + body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), + info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, +}, { + summary: "Package not found", + status: 404, + body: "not found", + error: `bin "curl" not found`, +}, { + summary: "No release for the requested architecture", + status: 200, + body: string(makeBinInfoBody("curl", "latest", "stable", "arm64", "8.5.0", 42, "abc123")), + error: `bin "curl" has no latest/stable release for architecture "amd64"`, +}, { + summary: "Server error", + status: 500, + statusText: "500 Internal Server Error", + body: "boom", + error: "cannot fetch from bin store: 500 Internal Server Error", +}, { + summary: "Malformed response body", + status: 200, + body: "not json", + error: "cannot decode bin store response: .*", +}, { + summary: "Selects the entry matching the requested architecture", + status: 200, + body: `{ + "name": "curl", + "channel-map": [ + { + "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "arm64"}}, + "revision": {"version": "8.0.0", "revision": 1, "download": {"url": "https://storage.snapcraftcontent.com/bins/curl.tar.xz", "sha3-384": "arm64hash", "size": 1024}} + }, + { + "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "amd64"}}, + "revision": {"version": "8.5.0", "revision": 42, "download": {"url": "https://storage.snapcraftcontent.com/bins/curl.tar.xz", "sha3-384": "amd64hash", "size": 1024}} + } + ] + }`, + info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "amd64hash"}, +}} + +func (s *storeSuite) TestInfo(c *C) { + for _, test := range infoTests { + c.Logf("Summary: %s", test.summary) + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: test.status, + Status: test.statusText, + Body: io.NopCloser(strings.NewReader(test.body)), + }, nil + } - src, err := store.Open(&store.Options{ - Arch: "amd64", - CacheDir: s.cacheDir, - }) - c.Assert(err, IsNil) + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) - _, err = src.Info("nonexistent", "latest", "stable") - c.Assert(err, NotNil) - c.Assert(err.Error(), testutil.Contains, "not found") + info, err := src.Info("curl", "latest", "stable") + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + c.Assert(info, DeepEquals, test.info) + } } -func (s *storeSuite) TestInfoNoMatchingChannel(c *C) { - infoResp := makeBinInfoResponse("curl", "latest", "stable", "arm64", "8.5.0", 42, "abc123") - infoBody, _ := json.Marshal(infoResp) +func (s *storeSuite) TestInfoRequest(c *C) { + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + c.Assert(req.URL.Path, Equals, "/v2/bins/info/curl") + c.Assert(req.URL.Query().Get("fields"), Equals, "download,version,revision,channel-map") return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(infoBody)), @@ -243,18 +257,16 @@ func (s *storeSuite) TestInfoNoMatchingChannel(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) - // The response only has arm64, but we're asking for amd64. _, err = src.Info("curl", "latest", "stable") - c.Assert(err, NotNil) - c.Assert(err.Error(), testutil.Contains, "has no") + c.Assert(err, IsNil) } func (s *storeSuite) TestExists(c *C) { - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") - infoBody, _ := json.Marshal(infoResp) + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if strings.Contains(req.URL.Path, "/info/curl") { @@ -272,6 +284,7 @@ func (s *storeSuite) TestExists(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) @@ -283,8 +296,7 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) - infoBody, _ := json.Marshal(infoResp) + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) callCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -305,6 +317,7 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) @@ -337,8 +350,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { err := cc.Write(cache.SHA384, digest, tarData) c.Assert(err, IsNil) - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) - infoBody, _ := json.Marshal(infoResp) + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) infoCallCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -355,6 +367,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) @@ -372,10 +385,9 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { } func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") // Override the download URL to an invalid one. - infoResp.ChannelMap[0].Revision.Download.URL = "http://evil.example.com/bins/curl.tar.xz" - infoBody, _ := json.Marshal(infoResp) + infoBody := makeBinInfoBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", + "http://evil.example.com/bins/curl.tar.xz") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { return &http.Response{ @@ -387,12 +399,12 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) _, _, err = src.Fetch("curl", "latest", "stable") - c.Assert(err, NotNil) - c.Assert(err.Error(), testutil.Contains, "must use HTTPS") + c.Assert(err, ErrorMatches, `bin download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) } func (s *storeSuite) TestStagingEnvVar(c *C) { @@ -402,11 +414,11 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { src, err := store.Open(&store.Options{ Arch: "amd64", CacheDir: s.cacheDir, + Kind: "bin", }) c.Assert(err, IsNil) - infoResp := makeBinInfoResponse("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") - infoBody, _ := json.Marshal(infoResp) + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { // Verify staging URL is used. @@ -420,3 +432,40 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { _, err = src.Info("curl", "latest", "stable") c.Assert(err, IsNil) } + +func (s *storeSuite) TestOpenUnsupportedKind(c *C) { + _, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "snap", + }) + c.Assert(err, ErrorMatches, `unsupported store kind "snap"`) +} + +func (s *storeSuite) TestFetchDownloadError(c *C) { + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if strings.Contains(req.URL.Path, "/info/") { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + return &http.Response{ + StatusCode: 500, + Status: "500 Internal Server Error", + Body: io.NopCloser(strings.NewReader("boom")), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `cannot download bin "curl": 500 Internal Server Error`) +} From 88d0cc325cf66a876e98b8b00210362fb4546b03 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 14:31:22 +0200 Subject: [PATCH 42/61] fix: open all stores --- cmd/chisel/cmd_cut.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 6e4db2255..1ba1f2033 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -133,7 +133,6 @@ func (cmd *cmdCut) Execute(args []string) error { return err } stores[storeInfo.Name] = openStore - break } err = slicer.Run(&slicer.RunOptions{ From bbea434a7a9ad9b60cf6898fb260496f88a18f32 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 14:39:54 +0200 Subject: [PATCH 43/61] fix: reapply --- internal/store/store.go | 104 ++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index ec39cd45e..b842fb086 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -93,43 +93,53 @@ func Open(options *Options) (Store, error) { // binInfoResponse represents the JSON response from the bin store info endpoint. type binInfoResponse struct { - Name string `json:"name"` - PackageID string `json:"package-id"` - ChannelMap []struct { - Channel struct { - Name string `json:"name"` - Risk string `json:"risk"` - Track string `json:"track"` - Platform struct { - Architecture string `json:"architecture"` - } `json:"platform"` - } `json:"channel"` - Revision struct { - Version string `json:"version"` - Revision int `json:"revision"` - Download struct { - URL string `json:"url"` - SHA3384 string `json:"sha3-384"` - Size int64 `json:"size"` - } `json:"download"` - Platforms []struct { - Architecture string `json:"architecture"` - } `json:"platforms"` - } `json:"revision"` - } `json:"channel-map"` + Name string `json:"name"` + PackageID string `json:"package-id"` + ChannelMap []binChannelMap `json:"channel-map"` +} + +type binChannelMap struct { + Channel binChannel `json:"channel"` + Revision binRevision `json:"revision"` +} + +type binChannel struct { + Name string `json:"name"` + Risk string `json:"risk"` + Track string `json:"track"` + Platform binPlatform `json:"platform"` +} + +type binPlatform struct { + Architecture string `json:"architecture"` +} + +type binRevision struct { + Version string `json:"version"` + Revision int `json:"revision"` + Download binDownload `json:"download"` +} + +type binDownload struct { + URL string `json:"url"` + SHA3384 string `json:"sha3-384"` + Size int64 `json:"size"` } func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { if !nameExp.MatchString(name) { return nil, fmt.Errorf("invalid package name %q", name) } - infoURL, err := url.JoinPath(s.apiURL, "info", name) + u, err := url.Parse(s.apiURL) if err != nil { - return nil, fmt.Errorf("internal error: cannot construct bin store URL: %v", err) + return nil, fmt.Errorf("internal error: cannot parse bin store URL: %v", err) } - infoURL += "?fields=download,version,revision,channel-map" + u = u.JoinPath("info", name) + u.RawQuery = url.Values{ + "fields": {"download,version,revision,channel-map"}, + }.Encode() - req, err := http.NewRequest("GET", infoURL, nil) + req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } @@ -158,19 +168,19 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { } // selectRevision finds the channel-map entry matching the requested track, -// risk, and architecture. -func selectRevision(info *binInfoResponse, arch, track, risk string) (downloadURL, sha3384, version string, revision int, err error) { - for _, entry := range info.ChannelMap { +// risk, and architecture, returning its revision. +func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevision, error) { + for i := range info.ChannelMap { + entry := &info.ChannelMap[i] if entry.Channel.Track != track || entry.Channel.Risk != risk { continue } if entry.Channel.Platform.Architecture != arch { continue } - return entry.Revision.Download.URL, entry.Revision.Download.SHA3384, - entry.Revision.Version, entry.Revision.Revision, nil + return &entry.Revision, nil } - return "", "", "", 0, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) + return nil, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) } // nameExp matches a valid package name. It deliberately forbids "/" and any @@ -208,15 +218,15 @@ func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { if err != nil { return nil, err } - _, sha384, version, revision, err := selectRevision(resp, s.options.Arch, track, risk) + rev, err := selectRevision(resp, s.options.Arch, track, risk) if err != nil { return nil, err } return &StorePackageInfo{ Name: name, - Version: version, - Revision: revision, - SHA384: sha384, + Version: rev.Version, + Revision: rev.Revision, + SHA384: rev.Download.SHA3384, }, nil } @@ -232,20 +242,20 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac if err != nil { return nil, nil, err } - downloadURL, sha384, version, revision, err := selectRevision(resp, s.options.Arch, track, risk) + rev, err := selectRevision(resp, s.options.Arch, track, risk) if err != nil { return nil, nil, err } info := &StorePackageInfo{ Name: name, - Version: version, - Revision: revision, - SHA384: sha384, + Version: rev.Version, + Revision: rev.Revision, + SHA384: rev.Download.SHA3384, } // Check cache first. - reader, err := s.cache.Open(cache.SHA384, sha384) + reader, err := s.cache.Open(cache.SHA384, rev.Download.SHA3384) if err == nil { logf("Using cached bin %s", name) return reader, info, nil @@ -254,11 +264,11 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac } // Download the bin. - err = validateDownloadURL(downloadURL) + err = validateDownloadURL(rev.Download.URL) if err != nil { return nil, nil, err } - req, err := http.NewRequest("GET", downloadURL, nil) + req, err := http.NewRequest("GET", rev.Download.URL, nil) if err != nil { return nil, nil, fmt.Errorf("cannot create HTTP request: %v", err) } @@ -273,7 +283,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, httpResp.Status) } - writer := s.cache.Create(cache.SHA384, sha384) + writer := s.cache.Create(cache.SHA384, rev.Download.SHA3384) defer writer.Close() _, err = io.Copy(writer, httpResp.Body) @@ -284,7 +294,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot fetch bin %q: %v", name, err) } - reader, err = s.cache.Open(cache.SHA384, sha384) + reader, err = s.cache.Open(cache.SHA384, rev.Download.SHA3384) if err != nil { return nil, nil, err } From 0f2a6a2c56f2564e091c874321177594dd6f7218 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 17:38:28 +0200 Subject: [PATCH 44/61] feat: wire to Run --- internal/slicer/slicer.go | 46 +++++++++++++++++++++------------- internal/slicer/slicer_test.go | 26 ++++++++++++++++++- internal/store/store.go | 10 +++++++- internal/testutil/archive.go | 9 ------- internal/testutil/package.go | 17 +++++++++++++ internal/testutil/store.go | 43 +++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 28 deletions(-) create mode 100644 internal/testutil/package.go create mode 100644 internal/testutil/store.go diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2ac4cd083..c1a608369 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -52,8 +52,8 @@ type pkgSourceInfo struct { arch string kind sourceKind archive archive.Archive - // TODO: add store handle when store support is implemented. - pkg *setup.Package + store store.Store + pkg *setup.Package } type contentChecker struct { @@ -109,7 +109,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgSources, err := resolvePkgSources(options.Archives, options.Selection) + pkgSources, err := resolvePkgSources(options.Archives, options.Stores, options.Selection) if err != nil { return err } @@ -172,19 +172,25 @@ func Run(options *RunOptions) error { continue } src := pkgSources[slice.Package] - // Store packages are distributed as "ar" archives, whose extraction is - // not yet implemented. Fail until store handling and the "ar" format - // support are in place. + var reader io.ReadSeekCloser if src.kind == sourceStore { - return fmt.Errorf("cannot fetch package %q from store: store packages are not yet supported", src.pkg.Name) - } - reader, info, err := src.archive.Fetch(src.pkg.RealName) - if err != nil { - return err + // The package metadata returned by the store is not yet recorded + // in the manifest; this is handled in a subsequent change. + // Risk is left unspecified for now; the store applies its default. + reader, _, err = src.store.Fetch(src.pkg.RealName, src.pkg.DefaultTrack, "") + if err != nil { + return err + } + } else { + var info *archive.PackageInfo + reader, info, err = src.archive.Fetch(src.pkg.RealName) + if err != nil { + return err + } + pkgInfos = append(pkgInfos, info) } defer reader.Close() packages[slice.Package] = reader - pkgInfos = append(pkgInfos, info) } // When creating content, record if a path is known and whether they are @@ -264,6 +270,12 @@ func Run(options *RunOptions) error { if reader == nil { continue } + src := pkgSources[slice.Package] + // Store packages are distributed as plain tarballs, whose extraction + // is not yet implemented. Fail until the format support is in place. + if src.kind == sourceStore { + return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.Name) + } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], @@ -518,9 +530,9 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent // resolvePkgSources determines the source for each package in the selection. // For archive packages it selects the highest priority archive containing the // package unless a particular archive is pinned within the slice definition -// file. For store packages it records the store reference. It returns a map +// file. For store packages it records the opened store handle. It returns a map // of pkgSourceInfo indexed by package names. -func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Selection) (map[string]*pkgSourceInfo, error) { +func resolvePkgSources(archives map[string]archive.Archive, stores map[string]store.Store, selection *setup.Selection) (map[string]*pkgSourceInfo, error) { sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) for _, archive := range selection.Release.Archives { if archive.Priority < 0 { @@ -542,9 +554,9 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { pkgSources[pkg.Name] = &pkgSourceInfo{ - // TODO: Fill with the live store handle when store support is implemented. - kind: sourceStore, - pkg: pkg, + kind: sourceStore, + store: stores[pkg.Store], + pkg: pkg, } continue } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index ddfea6057..01c985eb3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -18,6 +18,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/internal/store" "github.com/canonical/chisel/internal/testutil" "github.com/canonical/chisel/public/manifest" ) @@ -1982,6 +1983,14 @@ var slicerTests = []slicerTest{{ summary: "Store package fails as it is not yet supported", slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, arch: "amd64", + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }, { + Name: "store-pkg", + Store: "bin", + Data: testutil.PackageData["test-package"], + }}, release: map[string]string{ "chisel.yaml": testutil.DefaultChiselYamlWithStores, "slices/mydir/test-package.yaml": ` @@ -2001,7 +2010,7 @@ var slicerTests = []slicerTest{{ /dir/store-file: `, }, - error: `cannot fetch package "bin-store-pkg" from store: store packages are not yet supported`, + error: `cannot extract package "bin-store-pkg" from store: store packages are not yet supported`, }} func (s *S) TestRun(c *C) { @@ -2109,6 +2118,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { for name, setupArchive := range release.Archives { pkgs := make(map[string]*testutil.TestPackage) for _, pkg := range test.pkgs { + if pkg.Store != "" { + continue + } if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { pkgs[pkg.Name] = pkg } @@ -2127,9 +2139,21 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { archives[name] = archive } + stores := map[string]store.Store{} + for name := range release.Stores { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if pkg.Store == name { + pkgs[pkg.Name] = pkg + } + } + stores[name] = &testutil.TestStore{Packages: pkgs} + } + options := slicer.RunOptions{ Selection: selection, Archives: archives, + Stores: stores, TargetDir: c.MkDir(), } if test.hackopt != nil { diff --git a/internal/store/store.go b/internal/store/store.go index b842fb086..076a3a8c6 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -40,6 +40,10 @@ type storeKind string const storeKindBin storeKind = "bin" +// defaultRisk is the channel risk used when a specific risk has not been +// requested. +const defaultRisk = "stable" + type binStore struct { options Options cache *cache.Cache @@ -168,8 +172,12 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { } // selectRevision finds the channel-map entry matching the requested track, -// risk, and architecture, returning its revision. +// risk, and architecture, returning its revision. An empty risk falls back to +// defaultRisk. func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevision, error) { + if risk == "" { + risk = defaultRisk + } for i := range info.ChannelMap { entry := &info.ChannelMap[i] if entry.Channel.Track != track || entry.Channel.Risk != risk { diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..ccac36890 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -13,15 +13,6 @@ type TestArchive struct { Packages map[string]*TestPackage } -type TestPackage struct { - Name string - Version string - Hash string - Arch string - Data []byte - Archives []string -} - func (a *TestArchive) Options() *archive.Options { return &a.Opts } diff --git a/internal/testutil/package.go b/internal/testutil/package.go new file mode 100644 index 000000000..561eb105c --- /dev/null +++ b/internal/testutil/package.go @@ -0,0 +1,17 @@ +package testutil + +// TestPackage is a source-agnostic package fixture that can be served by either +// a TestArchive or a TestStore. +type TestPackage struct { + Name string + Version string + Hash string + Arch string + Data []byte + // Archives lists the archives the package belongs to. It is only relevant + // for archive-backed packages. + Archives []string + // Store names the store the package is served from. When set, the package + // is store-backed rather than archive-backed. + Store string +} diff --git a/internal/testutil/store.go b/internal/testutil/store.go new file mode 100644 index 000000000..f3356d7e8 --- /dev/null +++ b/internal/testutil/store.go @@ -0,0 +1,43 @@ +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/canonical/chisel/internal/store" +) + +type TestStore struct { + Packages map[string]*TestPackage +} + +func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *store.StorePackageInfo, error) { + pkg, ok := s.Packages[name] + if !ok { + return nil, nil, fmt.Errorf("cannot find package %q in store", name) + } + info := &store.StorePackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA384: pkg.Hash, + } + return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil +} + +func (s *TestStore) Exists(name, track, risk string) bool { + _, ok := s.Packages[name] + return ok +} + +func (s *TestStore) Info(name, track, risk string) (*store.StorePackageInfo, error) { + pkg, ok := s.Packages[name] + if !ok { + return nil, fmt.Errorf("cannot find package %q in store", name) + } + return &store.StorePackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA384: pkg.Hash, + }, nil +} From 2ee8aa961315ec38ca69216bef1c205ad7b1b19a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 22 Jun 2026 18:22:03 +0200 Subject: [PATCH 45/61] fix: review --- internal/slicer/slicer.go | 6 +++++- internal/store/store.go | 33 ++++++++++++++++++++++----------- internal/store/store_test.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index c1a608369..bd9f9aeaf 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -553,9 +553,13 @@ func resolvePkgSources(archives map[string]archive.Archive, stores map[string]st } pkg := selection.Release.Packages[s.Package] if pkg.Store != "" { + storeHandle := stores[pkg.Store] + if storeHandle == nil { + return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store) + } pkgSources[pkg.Name] = &pkgSourceInfo{ kind: sourceStore, - store: stores[pkg.Store], + store: storeHandle, pkg: pkg, } continue diff --git a/internal/store/store.go b/internal/store/store.go index 076a3a8c6..131ff9344 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -171,13 +171,17 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { return &info, nil } -// selectRevision finds the channel-map entry matching the requested track, -// risk, and architecture, returning its revision. An empty risk falls back to -// defaultRisk. -func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevision, error) { +func resolveRisk(risk string) string { if risk == "" { - risk = defaultRisk + return defaultRisk } + return risk +} + +// selectRevision finds the channel-map entry matching the requested track, +// risk, and architecture, returning its revision. It fails if the matched +// revision has no download digest. +func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevision, error) { for i := range info.ChannelMap { entry := &info.ChannelMap[i] if entry.Channel.Track != track || entry.Channel.Risk != risk { @@ -186,6 +190,9 @@ func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevisi if entry.Channel.Platform.Architecture != arch { continue } + if entry.Revision.Download.SHA3384 == "" { + return nil, fmt.Errorf("bin %q has no download digest", info.Name) + } return &entry.Revision, nil } return nil, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) @@ -213,15 +220,17 @@ func validateDownloadURL(downloadURL string) error { if u.Scheme != "https" { return fmt.Errorf("bin download URL must use HTTPS: %q", downloadURL) } - for _, host := range allowedDownloadHosts { - if u.Host == host || strings.HasSuffix(u.Host, "."+host) { + host := strings.ToLower(u.Hostname()) + for _, allowed := range allowedDownloadHosts { + if host == allowed || strings.HasSuffix(host, "."+allowed) { return nil } } - return fmt.Errorf("bin download URL has untrusted host %q", u.Host) + return fmt.Errorf("bin download URL has untrusted host %q", host) } func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { + risk = resolveRisk(risk) resp, err := s.fetchBinInfo(name) if err != nil { return nil, err @@ -244,6 +253,7 @@ func (s *binStore) Exists(name, track, risk string) bool { } func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { + risk = resolveRisk(risk) logf("Fetching bin %s %s/%s ...", name, track, risk) resp, err := s.fetchBinInfo(name) @@ -255,6 +265,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, err } + digest := rev.Download.SHA3384 info := &StorePackageInfo{ Name: name, Version: rev.Version, @@ -263,7 +274,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac } // Check cache first. - reader, err := s.cache.Open(cache.SHA384, rev.Download.SHA3384) + reader, err := s.cache.Open(cache.SHA384, digest) if err == nil { logf("Using cached bin %s", name) return reader, info, nil @@ -291,7 +302,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, httpResp.Status) } - writer := s.cache.Create(cache.SHA384, rev.Download.SHA3384) + writer := s.cache.Create(cache.SHA384, digest) defer writer.Close() _, err = io.Copy(writer, httpResp.Body) @@ -302,7 +313,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot fetch bin %q: %v", name, err) } - reader, err = s.cache.Open(cache.SHA384, rev.Download.SHA3384) + reader, err = s.cache.Open(cache.SHA384, digest) if err != nil { return nil, nil, err } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index e1525c0bd..e6ba5590a 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -112,6 +112,8 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { {"https://api.snapcraft.io/v2/bins/foo", ""}, {"https://api.staging.snapcraft.io/v2/bins/foo", ""}, {"https://sub.storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, + {"https://storage.snapcraftcontent.com:443/bins/foo.tar.xz", ""}, + {"https://Storage.SnapcraftContent.Com/bins/foo.tar.xz", ""}, { "http://storage.snapcraftcontent.com/bins/foo.tar.xz", `bin download URL must use HTTPS: "http://storage.snapcraftcontent.com/bins/foo.tar.xz"`, @@ -120,6 +122,10 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { "https://evil.example.com/bins/foo.tar.xz", `bin download URL has untrusted host "evil.example.com"`, }, + { + "https://Evil.Example.Com/bins/foo.tar.xz", + `bin download URL has untrusted host "evil.example.com"`, + }, { "https://storage.snapcraftcontent.com.evil.com/bins/foo.tar.xz", `bin download URL has untrusted host "storage.snapcraftcontent.com.evil.com"`, @@ -211,6 +217,11 @@ var infoTests = []infoTest{{ ] }`, info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "amd64hash"}, +}, { + summary: "Missing download digest", + status: 200, + body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), + error: `bin "curl" has no download digest`, }} func (s *storeSuite) TestInfo(c *C) { @@ -407,6 +418,28 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { c.Assert(err, ErrorMatches, `bin download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) } +func (s *storeSuite) TestFetchMissingDigest(c *C) { + // The store omits the sha3-384 digest from the response. + infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "") + + s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } + + src, err := store.Open(&store.Options{ + Arch: "amd64", + CacheDir: s.cacheDir, + Kind: "bin", + }) + c.Assert(err, IsNil) + + _, _, err = src.Fetch("curl", "latest", "stable") + c.Assert(err, ErrorMatches, `bin "curl" has no download digest`) +} + func (s *storeSuite) TestStagingEnvVar(c *C) { s.envRestore() s.envRestore = fakeEnv("1") From fdce477e1c66db00fc53e6898d7e3dd7265ac44e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 09:14:26 +0200 Subject: [PATCH 46/61] fix: improve naming and add more tests --- internal/store/store.go | 14 +++++++------- internal/store/store_test.go | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 131ff9344..348a81bc5 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -125,9 +125,9 @@ type binRevision struct { } type binDownload struct { - URL string `json:"url"` - SHA3384 string `json:"sha3-384"` - Size int64 `json:"size"` + URL string `json:"url"` + SHA384 string `json:"sha3-384"` + Size int64 `json:"size"` } func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { @@ -190,7 +190,7 @@ func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevisi if entry.Channel.Platform.Architecture != arch { continue } - if entry.Revision.Download.SHA3384 == "" { + if entry.Revision.Download.SHA384 == "" { return nil, fmt.Errorf("bin %q has no download digest", info.Name) } return &entry.Revision, nil @@ -243,7 +243,7 @@ func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { Name: name, Version: rev.Version, Revision: rev.Revision, - SHA384: rev.Download.SHA3384, + SHA384: rev.Download.SHA384, }, nil } @@ -265,12 +265,12 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, err } - digest := rev.Download.SHA3384 + digest := rev.Download.SHA384 info := &StorePackageInfo{ Name: name, Version: rev.Version, Revision: rev.Revision, - SHA384: rev.Download.SHA3384, + SHA384: rev.Download.SHA384, } // Check cache first. diff --git a/internal/store/store_test.go b/internal/store/store_test.go index e6ba5590a..2131f4b78 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -167,6 +167,7 @@ func (s *storeSuite) TestOpenArchValidation(c *C) { type infoTest struct { summary string + risk string status int statusText string body string @@ -176,32 +177,44 @@ type infoTest struct { var infoTests = []infoTest{{ summary: "Successful info", + risk: "stable", + status: 200, + body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), + info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, +}, { + summary: "Defaults to stable risk when unspecified", + risk: "", status: 200, body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, }, { summary: "Package not found", + risk: "stable", status: 404, body: "not found", error: `bin "curl" not found`, }, { summary: "No release for the requested architecture", + risk: "stable", status: 200, body: string(makeBinInfoBody("curl", "latest", "stable", "arm64", "8.5.0", 42, "abc123")), error: `bin "curl" has no latest/stable release for architecture "amd64"`, }, { summary: "Server error", + risk: "stable", status: 500, statusText: "500 Internal Server Error", body: "boom", error: "cannot fetch from bin store: 500 Internal Server Error", }, { summary: "Malformed response body", + risk: "stable", status: 200, body: "not json", error: "cannot decode bin store response: .*", }, { summary: "Selects the entry matching the requested architecture", + risk: "stable", status: 200, body: `{ "name": "curl", @@ -219,6 +232,7 @@ var infoTests = []infoTest{{ info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "amd64hash"}, }, { summary: "Missing download digest", + risk: "stable", status: 200, body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), error: `bin "curl" has no download digest`, @@ -243,7 +257,7 @@ func (s *storeSuite) TestInfo(c *C) { }) c.Assert(err, IsNil) - info, err := src.Info("curl", "latest", "stable") + info, err := src.Info("curl", "latest", test.risk) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue From c728291e8f79f86d2c8df26404fa8aa7c294800e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 10:54:01 +0200 Subject: [PATCH 47/61] fix: properly build track value with store version --- cmd/chisel/cmd_cut.go | 1 + internal/slicer/slicer.go | 3 ++- internal/slicer/slicer_test.go | 11 ++++++++--- internal/store/store.go | 6 ++++++ internal/testutil/store.go | 5 +++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 1ba1f2033..8cdd11ce6 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -128,6 +128,7 @@ func (cmd *cmdCut) Execute(args []string) error { Arch: cmd.Arch, CacheDir: cache.DefaultDir("chisel"), Kind: storeInfo.Kind, + Version: storeInfo.Version, }) if err != nil { return err diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index bd9f9aeaf..711cae433 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -177,7 +177,8 @@ func Run(options *RunOptions) error { // The package metadata returned by the store is not yet recorded // in the manifest; this is handled in a subsequent change. // Risk is left unspecified for now; the store applies its default. - reader, _, err = src.store.Fetch(src.pkg.RealName, src.pkg.DefaultTrack, "") + track := src.pkg.DefaultTrack + "-" + src.store.Options().Version + reader, _, err = src.store.Fetch(src.pkg.RealName, track, "") if err != nil { return err } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 01c985eb3..c5a0f0053 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2003,7 +2003,7 @@ var slicerTests = []slicerTest{{ "slices/mydir/store-pkg.yaml": ` package: store-pkg store: bin - default-track: stable + default-track: 3.1 slices: myslice: contents: @@ -2140,14 +2140,19 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { } stores := map[string]store.Store{} - for name := range release.Stores { + for name, relStore := range release.Stores { pkgs := make(map[string]*testutil.TestPackage) for _, pkg := range test.pkgs { if pkg.Store == name { pkgs[pkg.Name] = pkg } } - stores[name] = &testutil.TestStore{Packages: pkgs} + stores[name] = &testutil.TestStore{ + Packages: pkgs, + Opts: store.Options{ + Version: relStore.Version, + }, + } } options := slicer.RunOptions{ diff --git a/internal/store/store.go b/internal/store/store.go index 348a81bc5..ab54fa54c 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,6 +17,7 @@ import ( // Store provides access to packages from the Snapcraft store API. type Store interface { + Options() *Options Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) Exists(name, track, risk string) bool Info(name, track, risk string) (*StorePackageInfo, error) @@ -34,6 +35,7 @@ type Options struct { Arch string CacheDir string Kind string + Version string } type storeKind string @@ -229,6 +231,10 @@ func validateDownloadURL(downloadURL string) error { return fmt.Errorf("bin download URL has untrusted host %q", host) } +func (s *binStore) Options() *Options { + return &s.options +} + func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { risk = resolveRisk(risk) resp, err := s.fetchBinInfo(name) diff --git a/internal/testutil/store.go b/internal/testutil/store.go index f3356d7e8..cd0650580 100644 --- a/internal/testutil/store.go +++ b/internal/testutil/store.go @@ -9,9 +9,14 @@ import ( ) type TestStore struct { + Opts store.Options Packages map[string]*TestPackage } +func (s *TestStore) Options() *store.Options { + return &s.Opts +} + func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *store.StorePackageInfo, error) { pkg, ok := s.Packages[name] if !ok { From fd0534eb30a5a26ffe8bdea095437c285824613f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 14:02:32 +0200 Subject: [PATCH 48/61] feat: refine download host validation --- internal/store/store.go | 56 +++++++++++++++++------------------- internal/store/store_test.go | 52 +++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index ab54fa54c..5b2b746c6 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -15,7 +15,7 @@ import ( "github.com/canonical/chisel/internal/deb" ) -// Store provides access to packages from the Snapcraft store API. +// Store provides access to packages from the Store API. type Store interface { Options() *Options Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) @@ -42,20 +42,21 @@ type storeKind string const storeKindBin storeKind = "bin" -// defaultRisk is the channel risk used when a specific risk has not been -// requested. const defaultRisk = "stable" type binStore struct { - options Options - cache *cache.Cache - apiURL string + options Options + cache *cache.Cache + apiURL string + downloadHost string } const ( - binAPIBase = "https://api.snapcraft.io/v2/bins" - binAPIStaging = "https://api.staging.snapcraft.io/v2/bins" - binStagingEnvVar = "CHISEL_BIN_STAGING" + binAPIBase = "https://api.snapcraft.io/v2/bins" + binAPIBaseStaging = "https://api.staging.snapcraft.io/v2/bins" + binDownloadHost = "api.snapcraft.io" + binDownloadHostStaging = "api.staging.snapcraft.io" + binStagingEnvVar = "CHISEL_BIN_STAGING" ) var httpClient = &http.Client{ @@ -84,13 +85,16 @@ func Open(options *Options) (Store, error) { switch storeKind(options.Kind) { case storeKindBin: apiURL := binAPIBase + downloadHost := binDownloadHost if os.Getenv(binStagingEnvVar) != "" { - apiURL = binAPIStaging + apiURL = binAPIBaseStaging + downloadHost = binDownloadHostStaging } return &binStore{ - options: *options, - cache: &cache.Cache{Dir: options.CacheDir}, - apiURL: apiURL, + options: *options, + cache: &cache.Cache{Dir: options.CacheDir}, + apiURL: apiURL, + downloadHost: downloadHost, }, nil default: return nil, fmt.Errorf("unsupported store kind %q", options.Kind) @@ -205,16 +209,9 @@ func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevisi // store API URL path when interpolated into it. var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) -// allowedDownloadHosts lists the hosts from which bin downloads are permitted. -var allowedDownloadHosts = []string{ - "api.snapcraft.io", - "api.staging.snapcraft.io", - "storage.snapcraftcontent.com", -} - -// validateDownloadURL checks that the download URL is HTTPS and from an +// validateDownloadURL checks that the download URL is HTTPS and from the // allowed host. -func validateDownloadURL(downloadURL string) error { +func validateDownloadURL(downloadURL, allowedHost string) error { u, err := url.Parse(downloadURL) if err != nil { return fmt.Errorf("cannot parse bin download URL: %v", err) @@ -223,10 +220,8 @@ func validateDownloadURL(downloadURL string) error { return fmt.Errorf("bin download URL must use HTTPS: %q", downloadURL) } host := strings.ToLower(u.Hostname()) - for _, allowed := range allowedDownloadHosts { - if host == allowed || strings.HasSuffix(host, "."+allowed) { - return nil - } + if host == allowedHost || strings.HasSuffix(host, "."+allowedHost) { + return nil } return fmt.Errorf("bin download URL has untrusted host %q", host) } @@ -279,8 +274,9 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac SHA384: rev.Download.SHA384, } + const digestKind = cache.SHA256 // Check cache first. - reader, err := s.cache.Open(cache.SHA384, digest) + reader, err := s.cache.Open(digestKind, digest) if err == nil { logf("Using cached bin %s", name) return reader, info, nil @@ -289,7 +285,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac } // Download the bin. - err = validateDownloadURL(rev.Download.URL) + err = validateDownloadURL(rev.Download.URL, s.downloadHost) if err != nil { return nil, nil, err } @@ -308,7 +304,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, httpResp.Status) } - writer := s.cache.Create(cache.SHA384, digest) + writer := s.cache.Create(digestKind, digest) defer writer.Close() _, err = io.Copy(writer, httpResp.Body) @@ -319,7 +315,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac return nil, nil, fmt.Errorf("cannot fetch bin %q: %v", name, err) } - reader, err = s.cache.Open(cache.SHA384, digest) + reader, err = s.cache.Open(digestKind, writer.Digest()) if err != nil { return nil, nil, err } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 2131f4b78..6d29fb167 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -74,7 +74,7 @@ func sha384Hash(data []byte) string { } func makeBinInfoBody(name, track, risk, arch, version string, revision int, sha384 string) []byte { - downloadURL := "https://storage.snapcraftcontent.com/bins/" + name + ".tar.xz" + downloadURL := "https://api.snapcraft.io/api/v1/bins/download/" + name + ".bin" return makeBinInfoBodyWithURL(name, track, risk, arch, version, revision, sha384, downloadURL) } @@ -105,35 +105,49 @@ func makeBinInfoBodyWithURL(name, track, risk, arch, version string, revision in func (s *storeSuite) TestValidateDownloadURL(c *C) { tests := []struct { - url string - error string + url string + allowedHost string + error string }{ - {"https://storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, - {"https://api.snapcraft.io/v2/bins/foo", ""}, - {"https://api.staging.snapcraft.io/v2/bins/foo", ""}, - {"https://sub.storage.snapcraftcontent.com/bins/foo.tar.xz", ""}, - {"https://storage.snapcraftcontent.com:443/bins/foo.tar.xz", ""}, - {"https://Storage.SnapcraftContent.Com/bins/foo.tar.xz", ""}, + {"https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, + // Staging host is rejected when the production host is allowed. + { + "https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `bin download URL has untrusted host "api.staging.snapcraft.io"`, + }, + // Staging host is allowed when it is the allowed host. + {"https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", "api.staging.snapcraft.io", ""}, + // Production API host is rejected when the staging host is allowed. + { + "https://api.snapcraft.io/api/v1/bins/download/foo.bin", + "api.staging.snapcraft.io", + `bin download URL has untrusted host "api.snapcraft.io"`, + }, { - "http://storage.snapcraftcontent.com/bins/foo.tar.xz", - `bin download URL must use HTTPS: "http://storage.snapcraftcontent.com/bins/foo.tar.xz"`, + "http://api.snapcraft.io/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `bin download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, }, { - "https://evil.example.com/bins/foo.tar.xz", + "https://evil.example.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", `bin download URL has untrusted host "evil.example.com"`, }, { - "https://Evil.Example.Com/bins/foo.tar.xz", + "https://Evil.Example.Com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", `bin download URL has untrusted host "evil.example.com"`, }, { - "https://storage.snapcraftcontent.com.evil.com/bins/foo.tar.xz", - `bin download URL has untrusted host "storage.snapcraftcontent.com.evil.com"`, + "https://api.snapcraft.io.evil.com/api/v1/bins/download/foo.bin", + "api.snapcraft.io", + `bin download URL has untrusted host "api.snapcraft.io.evil.com"`, }, - {"://invalid-url", `cannot parse bin download URL: .*`}, + {"://invalid-url", "api.snapcraft.io", `cannot parse bin download URL: .*`}, } for _, test := range tests { - err := store.ValidateDownloadURL(test.url) + err := store.ValidateDownloadURL(test.url, test.allowedHost) if test.error == "" { c.Assert(err, IsNil) } else { @@ -221,11 +235,11 @@ var infoTests = []infoTest{{ "channel-map": [ { "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "arm64"}}, - "revision": {"version": "8.0.0", "revision": 1, "download": {"url": "https://storage.snapcraftcontent.com/bins/curl.tar.xz", "sha3-384": "arm64hash", "size": 1024}} + "revision": {"version": "8.0.0", "revision": 1, "download": {"url": "https://api.snapcraft.io/api/v1/bins/download/curl.bin", "sha3-384": "arm64hash", "size": 1024}} }, { "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "amd64"}}, - "revision": {"version": "8.5.0", "revision": 42, "download": {"url": "https://storage.snapcraftcontent.com/bins/curl.tar.xz", "sha3-384": "amd64hash", "size": 1024}} + "revision": {"version": "8.5.0", "revision": 42, "download": {"url": "https://api.snapcraft.io/api/v1/bins/download/curl.bin", "sha3-384": "amd64hash", "size": 1024}} } ] }`, From 885c5d3cbc4bba816de70e3bda7037d586cb46ee Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 15:18:29 +0200 Subject: [PATCH 49/61] feat: use revisions/resolve endpoint --- internal/store/store.go | 139 +++++++++++++++++--------------- internal/store/store_test.go | 150 ++++++++++++++++++++--------------- 2 files changed, 160 insertions(+), 129 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 5b2b746c6..f6fa8e827 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "encoding/json" "fmt" "io" @@ -52,8 +53,8 @@ type binStore struct { } const ( - binAPIBase = "https://api.snapcraft.io/v2/bins" - binAPIBaseStaging = "https://api.staging.snapcraft.io/v2/bins" + binAPIBase = "https://api.snapcraft.io/v2" + binAPIBaseStaging = "https://api.staging.snapcraft.io/v2" binDownloadHost = "api.snapcraft.io" binDownloadHostStaging = "api.staging.snapcraft.io" binStagingEnvVar = "CHISEL_BIN_STAGING" @@ -101,23 +102,37 @@ func Open(options *Options) (Store, error) { } } -// binInfoResponse represents the JSON response from the bin store info endpoint. -type binInfoResponse struct { - Name string `json:"name"` - PackageID string `json:"package-id"` - ChannelMap []binChannelMap `json:"channel-map"` +// resolveRequest is the body sent to the revisions/resolve endpoint. +type resolveRequest struct { + Packages []resolvePackage `json:"packages"` } -type binChannelMap struct { - Channel binChannel `json:"channel"` - Revision binRevision `json:"revision"` +type resolvePackage struct { + InstanceKey string `json:"instance-key"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Channel string `json:"channel"` + Platform binPlatform `json:"platform"` +} + +type resolveResponse struct { + PackageResults []resolveResult `json:"package-results"` +} + +type resolveResult struct { + InstanceKey string `json:"instance-key"` + Status string `json:"status"` + Error *resolveError `json:"error"` + Result *resolveEntry `json:"result"` +} + +type resolveError struct { + Code string `json:"code"` + Message string `json:"message"` } -type binChannel struct { - Name string `json:"name"` - Risk string `json:"risk"` - Track string `json:"track"` - Platform binPlatform `json:"platform"` +type resolveEntry struct { + Revision binRevision `json:"revision"` } type binPlatform struct { @@ -136,7 +151,10 @@ type binDownload struct { Size int64 `json:"size"` } -func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { +// resolveRevision resolves a single package revision via the store API. It +// returns the matching revision or an error if the package is not found or has +// no release for the requested channel and architecture. +func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, error) { if !nameExp.MatchString(name) { return nil, fmt.Errorf("invalid package name %q", name) } @@ -144,15 +162,27 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { if err != nil { return nil, fmt.Errorf("internal error: cannot parse bin store URL: %v", err) } - u = u.JoinPath("info", name) - u.RawQuery = url.Values{ - "fields": {"download,version,revision,channel-map"}, - }.Encode() + u = u.JoinPath("revisions", "resolve") + + reqBody, err := json.Marshal(resolveRequest{ + Packages: []resolvePackage{{ + InstanceKey: name, + Namespace: string(storeKindBin), + Name: name, + Channel: track + "/" + risk, + Platform: binPlatform{Architecture: s.options.Arch}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("internal error: cannot encode resolve request: %v", err) + } - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(reqBody)) if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") resp, err := httpDo(req) if err != nil { @@ -160,48 +190,31 @@ func (s *binStore) fetchBinInfo(name string) (*binInfoResponse, error) { } defer resp.Body.Close() - switch resp.StatusCode { - case 200: - // ok - case 404: - return nil, fmt.Errorf("bin %q not found", name) - default: + if resp.StatusCode != 200 { return nil, fmt.Errorf("cannot fetch from bin store: %v", resp.Status) } - var info binInfoResponse - err = json.NewDecoder(resp.Body).Decode(&info) + var res resolveResponse + err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { return nil, fmt.Errorf("cannot decode bin store response: %v", err) } - return &info, nil -} -func resolveRisk(risk string) string { - if risk == "" { - return defaultRisk + if len(res.PackageResults) == 0 { + return nil, fmt.Errorf("bin %q not found", name) } - return risk -} - -// selectRevision finds the channel-map entry matching the requested track, -// risk, and architecture, returning its revision. It fails if the matched -// revision has no download digest. -func selectRevision(info *binInfoResponse, arch, track, risk string) (*binRevision, error) { - for i := range info.ChannelMap { - entry := &info.ChannelMap[i] - if entry.Channel.Track != track || entry.Channel.Risk != risk { - continue - } - if entry.Channel.Platform.Architecture != arch { - continue - } - if entry.Revision.Download.SHA384 == "" { - return nil, fmt.Errorf("bin %q has no download digest", info.Name) + result := &res.PackageResults[0] + if result.Status != "ok" || result.Result == nil { + if result.Error != nil { + return nil, fmt.Errorf("bin %q not found: %s", name, result.Error.Message) } - return &entry.Revision, nil + return nil, fmt.Errorf("bin %q not found", name) + } + rev := &result.Result.Revision + if rev.Download.SHA384 == "" { + return nil, fmt.Errorf("bin %q has no download digest", name) } - return nil, fmt.Errorf("bin %q has no %s/%s release for architecture %q", info.Name, track, risk, arch) + return rev, nil } // nameExp matches a valid package name. It deliberately forbids "/" and any @@ -231,12 +244,10 @@ func (s *binStore) Options() *Options { } func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { - risk = resolveRisk(risk) - resp, err := s.fetchBinInfo(name) - if err != nil { - return nil, err + if risk == "" { + risk = defaultRisk } - rev, err := selectRevision(resp, s.options.Arch, track, risk) + rev, err := s.resolveRevision(name, track, risk) if err != nil { return nil, err } @@ -254,14 +265,12 @@ func (s *binStore) Exists(name, track, risk string) bool { } func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { - risk = resolveRisk(risk) + if risk == "" { + risk = defaultRisk + } logf("Fetching bin %s %s/%s ...", name, track, risk) - resp, err := s.fetchBinInfo(name) - if err != nil { - return nil, nil, err - } - rev, err := selectRevision(resp, s.options.Arch, track, risk) + rev, err := s.resolveRevision(name, track, risk) if err != nil { return nil, nil, err } @@ -274,7 +283,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac SHA384: rev.Download.SHA384, } - const digestKind = cache.SHA256 + const digestKind = cache.SHA384 // Check cache first. reader, err := s.cache.Open(digestKind, digest) if err == nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 6d29fb167..e162aaef1 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store_test import ( "bytes" + "encoding/json" "fmt" "io" "net/http" @@ -73,29 +74,33 @@ func sha384Hash(data []byte) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func makeBinInfoBody(name, track, risk, arch, version string, revision int, sha384 string) []byte { +func makeResolveBody(name, track, risk, arch, version string, revision int, sha384 string) []byte { downloadURL := "https://api.snapcraft.io/api/v1/bins/download/" + name + ".bin" - return makeBinInfoBodyWithURL(name, track, risk, arch, version, revision, sha384, downloadURL) + return makeResolveBodyWithURL(name, track, risk, arch, version, revision, sha384, downloadURL) } -func makeBinInfoBodyWithURL(name, track, risk, arch, version string, revision int, sha384, downloadURL string) []byte { +func makeResolveBodyWithURL(name, track, risk, arch, version string, revision int, sha384, downloadURL string) []byte { return []byte(fmt.Sprintf(`{ - "name": %q, - "channel-map": [ + "craft-results": [], + "package-results": [ { - "channel": { - "name": %q, - "risk": %q, - "track": %q, - "platform": {"architecture": %q} - }, - "revision": { - "version": %q, - "revision": %d, - "download": { - "url": %q, - "sha3-384": %q, - "size": 1024 + "instance-key": %q, + "status": "ok", + "result": { + "channel": { + "name": %q, + "risk": %q, + "track": %q, + "platform": {"architecture": %q} + }, + "revision": { + "version": %q, + "revision": %d, + "download": { + "url": %q, + "sha3-384": %q, + "size": 1024 + } } } } @@ -103,6 +108,20 @@ func makeBinInfoBodyWithURL(name, track, risk, arch, version string, revision in }`, name, track+"/"+risk, risk, track, arch, version, revision, downloadURL, sha384)) } +func makeResolveErrorBody(name, code, message string) []byte { + return []byte(fmt.Sprintf(`{ + "craft-results": [], + "package-results": [ + { + "instance-key": %q, + "status": "error", + "error": {"code": %q, "message": %q}, + "result": null + } + ] + }`, name, code, message)) +} + func (s *storeSuite) TestValidateDownloadURL(c *C) { tests := []struct { url string @@ -193,26 +212,20 @@ var infoTests = []infoTest{{ summary: "Successful info", risk: "stable", status: 200, - body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), + body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, }, { summary: "Defaults to stable risk when unspecified", risk: "", status: 200, - body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), + body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, }, { summary: "Package not found", risk: "stable", - status: 404, - body: "not found", - error: `bin "curl" not found`, -}, { - summary: "No release for the requested architecture", - risk: "stable", status: 200, - body: string(makeBinInfoBody("curl", "latest", "stable", "arm64", "8.5.0", 42, "abc123")), - error: `bin "curl" has no latest/stable release for architecture "amd64"`, + body: string(makeResolveErrorBody("curl", "package-not-found", "Package not found")), + error: `bin "curl" not found: Package not found`, }, { summary: "Server error", risk: "stable", @@ -226,29 +239,11 @@ var infoTests = []infoTest{{ status: 200, body: "not json", error: "cannot decode bin store response: .*", -}, { - summary: "Selects the entry matching the requested architecture", - risk: "stable", - status: 200, - body: `{ - "name": "curl", - "channel-map": [ - { - "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "arm64"}}, - "revision": {"version": "8.0.0", "revision": 1, "download": {"url": "https://api.snapcraft.io/api/v1/bins/download/curl.bin", "sha3-384": "arm64hash", "size": 1024}} - }, - { - "channel": {"name": "latest/stable", "risk": "stable", "track": "latest", "platform": {"architecture": "amd64"}}, - "revision": {"version": "8.5.0", "revision": 42, "download": {"url": "https://api.snapcraft.io/api/v1/bins/download/curl.bin", "sha3-384": "amd64hash", "size": 1024}} - } - ] - }`, - info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "amd64hash"}, }, { summary: "Missing download digest", risk: "stable", status: 200, - body: string(makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), + body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), error: `bin "curl" has no download digest`, }} @@ -282,11 +277,26 @@ func (s *storeSuite) TestInfo(c *C) { } func (s *storeSuite) TestInfoRequest(c *C) { - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - c.Assert(req.URL.Path, Equals, "/v2/bins/info/curl") - c.Assert(req.URL.Query().Get("fields"), Equals, "download,version,revision,channel-map") + c.Assert(req.Method, Equals, "POST") + c.Assert(req.URL.Path, Equals, "/v2/revisions/resolve") + c.Assert(req.Header.Get("Content-Type"), Equals, "application/json") + c.Assert(req.Header.Get("Accept"), Equals, "application/json") + + var body map[string]any + err := json.NewDecoder(req.Body).Decode(&body) + c.Assert(err, IsNil) + pkgs := body["packages"].([]any) + c.Assert(pkgs, HasLen, 1) + pkg := pkgs[0].(map[string]any) + c.Assert(pkg["instance-key"], Equals, "curl") + c.Assert(pkg["namespace"], Equals, "bin") + c.Assert(pkg["name"], Equals, "curl") + c.Assert(pkg["channel"], Equals, "latest/stable") + c.Assert(pkg["platform"].(map[string]any)["architecture"], Equals, "amd64") + return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(infoBody)), @@ -305,18 +315,30 @@ func (s *storeSuite) TestInfoRequest(c *C) { } func (s *storeSuite) TestExists(c *C) { - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + okBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + notFoundBody := makeResolveErrorBody("nonexistent", "package-not-found", "Package not found") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - if strings.Contains(req.URL.Path, "/info/curl") { + var body struct { + Packages []struct { + Name string `json:"name"` + } `json:"packages"` + } + err := json.NewDecoder(req.Body).Decode(&body) + c.Assert(err, IsNil) + pkgName := "" + if len(body.Packages) > 0 { + pkgName = body.Packages[0].Name + } + if pkgName == "curl" { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(okBody)), }, nil } return &http.Response{ - StatusCode: 404, - Body: io.NopCloser(strings.NewReader("not found")), + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(notFoundBody)), }, nil } @@ -335,12 +357,12 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) callCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { callCount++ - if strings.Contains(req.URL.Path, "/info/") { + if req.URL.Path == "/v2/revisions/resolve" { return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(infoBody)), @@ -389,11 +411,11 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { err := cc.Write(cache.SHA384, digest, tarData) c.Assert(err, IsNil) - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) infoCallCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - if strings.Contains(req.URL.Path, "/info/") { + if req.URL.Path == "/v2/revisions/resolve" { infoCallCount++ return &http.Response{ StatusCode: 200, @@ -425,7 +447,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { // Override the download URL to an invalid one. - infoBody := makeBinInfoBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", + infoBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", "http://evil.example.com/bins/curl.tar.xz") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -448,7 +470,7 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { func (s *storeSuite) TestFetchMissingDigest(c *C) { // The store omits the sha3-384 digest from the response. - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "") + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { return &http.Response{ @@ -479,7 +501,7 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { }) c.Assert(err, IsNil) - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { // Verify staging URL is used. @@ -504,10 +526,10 @@ func (s *storeSuite) TestOpenUnsupportedKind(c *C) { } func (s *storeSuite) TestFetchDownloadError(c *C) { - infoBody := makeBinInfoBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - if strings.Contains(req.URL.Path, "/info/") { + if req.URL.Path == "/v2/revisions/resolve" { return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(infoBody)), From ea50c1963db600e46e689b1bdd7ee3f5be404742 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 15:42:57 +0200 Subject: [PATCH 50/61] fix: raise consistency --- cmd/chisel/cmd_cut.go | 4 ++-- internal/slicer/slicer.go | 20 +++------------- internal/store/export_test.go | 19 +++++++-------- internal/store/store.go | 33 +++++++++++++------------ internal/store/store_test.go | 45 ++++++++++++++++------------------- 5 files changed, 51 insertions(+), 70 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 8cdd11ce6..8e97454ab 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -123,7 +123,7 @@ func (cmd *cmdCut) Execute(args []string) error { } stores := make(map[string]store.Store) - for _, storeInfo := range release.Stores { + for storeName, storeInfo := range release.Stores { openStore, err := store.Open(&store.Options{ Arch: cmd.Arch, CacheDir: cache.DefaultDir("chisel"), @@ -133,7 +133,7 @@ func (cmd *cmdCut) Execute(args []string) error { if err != nil { return err } - stores[storeInfo.Name] = openStore + stores[storeName] = openStore } err = slicer.Run(&slicer.RunOptions{ diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 711cae433..05f823210 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -177,6 +177,8 @@ func Run(options *RunOptions) error { // The package metadata returned by the store is not yet recorded // in the manifest; this is handled in a subsequent change. // Risk is left unspecified for now; the store applies its default. + // The store channel track is "-", + // e.g. "3.1-26.10". The version pins the release series. track := src.pkg.DefaultTrack + "-" + src.store.Options().Version reader, _, err = src.store.Fetch(src.pkg.RealName, track, "") if err != nil { @@ -559,6 +561,7 @@ func resolvePkgSources(archives map[string]archive.Archive, stores map[string]st return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store) } pkgSources[pkg.Name] = &pkgSourceInfo{ + arch: storeHandle.Options().Arch, kind: sourceStore, store: storeHandle, pkg: pkg, @@ -594,22 +597,5 @@ func resolvePkgSources(archives map[string]archive.Archive, stores map[string]st } } - // Until a store is implemented as a package source there is no proper way to - // determine the architecture for store packages. - // So relying on the fact that all packages in a selection share the same architecture, - // we can borrow it from any archive package that was already resolved. - var arch string - for _, src := range pkgSources { - if src.kind == sourceArchive { - arch = src.arch - break - } - } - for _, src := range pkgSources { - if src.kind == sourceStore { - src.arch = arch - } - } - return pkgSources, nil } diff --git a/internal/store/export_test.go b/internal/store/export_test.go index f69322be8..cdaf5d1d6 100644 --- a/internal/store/export_test.go +++ b/internal/store/export_test.go @@ -7,14 +7,13 @@ var ( BinStagingEnvVar = binStagingEnvVar ) -func SetHTTPDo(fn func(req *http.Request) (*http.Response, error)) (restore func()) { - saved := httpDo - httpDo = fn - return func() { httpDo = saved } -} - -func SetBulkDo(fn func(req *http.Request) (*http.Response, error)) (restore func()) { - saved := bulkDo - bulkDo = fn - return func() { bulkDo = saved } +func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) { + _httpDo := httpDo + _bulkDo := bulkDo + httpDo = do + bulkDo = do + return func() { + httpDo = _httpDo + bulkDo = _bulkDo + } } diff --git a/internal/store/store.go b/internal/store/store.go index f6fa8e827..c3829ae7d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -148,7 +148,6 @@ type binRevision struct { type binDownload struct { URL string `json:"url"` SHA384 string `json:"sha3-384"` - Size int64 `json:"size"` } // resolveRevision resolves a single package revision via the store API. It @@ -186,33 +185,33 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro resp, err := httpDo(req) if err != nil { - return nil, fmt.Errorf("cannot talk to bin store: %v", err) + return nil, fmt.Errorf("cannot talk to store: %v", err) } defer resp.Body.Close() if resp.StatusCode != 200 { - return nil, fmt.Errorf("cannot fetch from bin store: %v", resp.Status) + return nil, fmt.Errorf("cannot fetch from store: %v", resp.Status) } var res resolveResponse err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { - return nil, fmt.Errorf("cannot decode bin store response: %v", err) + return nil, fmt.Errorf("cannot decode store response: %v", err) } if len(res.PackageResults) == 0 { - return nil, fmt.Errorf("bin %q not found", name) + return nil, fmt.Errorf("package %q not found", name) } result := &res.PackageResults[0] if result.Status != "ok" || result.Result == nil { if result.Error != nil { - return nil, fmt.Errorf("bin %q not found: %s", name, result.Error.Message) + return nil, fmt.Errorf("package %q not found: %s", name, result.Error.Message) } - return nil, fmt.Errorf("bin %q not found", name) + return nil, fmt.Errorf("package %q not found", name) } rev := &result.Result.Revision if rev.Download.SHA384 == "" { - return nil, fmt.Errorf("bin %q has no download digest", name) + return nil, fmt.Errorf("package %q has no download digest", name) } return rev, nil } @@ -227,16 +226,16 @@ var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) func validateDownloadURL(downloadURL, allowedHost string) error { u, err := url.Parse(downloadURL) if err != nil { - return fmt.Errorf("cannot parse bin download URL: %v", err) + return fmt.Errorf("cannot parse download URL: %v", err) } if u.Scheme != "https" { - return fmt.Errorf("bin download URL must use HTTPS: %q", downloadURL) + return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) } host := strings.ToLower(u.Hostname()) if host == allowedHost || strings.HasSuffix(host, "."+allowedHost) { return nil } - return fmt.Errorf("bin download URL has untrusted host %q", host) + return fmt.Errorf("download URL has untrusted host %q", host) } func (s *binStore) Options() *Options { @@ -268,7 +267,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac if risk == "" { risk = defaultRisk } - logf("Fetching bin %s %s/%s ...", name, track, risk) + logf("Fetching package %s %s/%s...", name, track, risk) rev, err := s.resolveRevision(name, track, risk) if err != nil { @@ -287,13 +286,13 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac // Check cache first. reader, err := s.cache.Open(digestKind, digest) if err == nil { - logf("Using cached bin %s", name) + logf("Using cached package %s", name) return reader, info, nil } else if err != cache.ErrMiss { return nil, nil, err } - // Download the bin. + // Download the package. err = validateDownloadURL(rev.Download.URL, s.downloadHost) if err != nil { return nil, nil, err @@ -305,12 +304,12 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac httpResp, err := bulkDo(req) if err != nil { - return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, err) + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, err) } defer httpResp.Body.Close() if httpResp.StatusCode != 200 { - return nil, nil, fmt.Errorf("cannot download bin %q: %v", name, httpResp.Status) + return nil, nil, fmt.Errorf("cannot download package %q: %v", name, httpResp.Status) } writer := s.cache.Create(digestKind, digest) @@ -321,7 +320,7 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac err = writer.Close() } if err != nil { - return nil, nil, fmt.Errorf("cannot fetch bin %q: %v", name, err) + return nil, nil, fmt.Errorf("cannot fetch package %q: %v", name, err) } reader, err = s.cache.Open(digestKind, writer.Digest()) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index e162aaef1..381632d49 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -18,12 +18,11 @@ import ( ) type storeSuite struct { - tempDir string - cacheDir string - fakeDoFunc func(req *http.Request) (*http.Response, error) - restoreDo func() - restoreBulk func() - envRestore func() + tempDir string + cacheDir string + fakeDoFunc func(req *http.Request) (*http.Response, error) + restore func() + envRestore func() } var _ = Suite(&storeSuite{}) @@ -34,14 +33,12 @@ func (s *storeSuite) SetUpTest(c *C) { c.Assert(os.MkdirAll(s.cacheDir, 0o755), IsNil) s.envRestore = fakeEnv("") - s.restoreDo = store.SetHTTPDo(s.doRequest) - s.restoreBulk = store.SetBulkDo(s.doRequest) + s.restore = store.FakeDo(s.doRequest) s.fakeDoFunc = nil } func (s *storeSuite) TearDownTest(c *C) { - s.restoreDo() - s.restoreBulk() + s.restore() s.envRestore() } @@ -133,7 +130,7 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { { "https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", - `bin download URL has untrusted host "api.staging.snapcraft.io"`, + `download URL has untrusted host "api.staging.snapcraft.io"`, }, // Staging host is allowed when it is the allowed host. {"https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", "api.staging.snapcraft.io", ""}, @@ -141,29 +138,29 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { { "https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.staging.snapcraft.io", - `bin download URL has untrusted host "api.snapcraft.io"`, + `download URL has untrusted host "api.snapcraft.io"`, }, { "http://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", - `bin download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, + `download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, }, { "https://evil.example.com/api/v1/bins/download/foo.bin", "api.snapcraft.io", - `bin download URL has untrusted host "evil.example.com"`, + `download URL has untrusted host "evil.example.com"`, }, { "https://Evil.Example.Com/api/v1/bins/download/foo.bin", "api.snapcraft.io", - `bin download URL has untrusted host "evil.example.com"`, + `download URL has untrusted host "evil.example.com"`, }, { "https://api.snapcraft.io.evil.com/api/v1/bins/download/foo.bin", "api.snapcraft.io", - `bin download URL has untrusted host "api.snapcraft.io.evil.com"`, + `download URL has untrusted host "api.snapcraft.io.evil.com"`, }, - {"://invalid-url", "api.snapcraft.io", `cannot parse bin download URL: .*`}, + {"://invalid-url", "api.snapcraft.io", `cannot parse download URL: .*`}, } for _, test := range tests { err := store.ValidateDownloadURL(test.url, test.allowedHost) @@ -225,26 +222,26 @@ var infoTests = []infoTest{{ risk: "stable", status: 200, body: string(makeResolveErrorBody("curl", "package-not-found", "Package not found")), - error: `bin "curl" not found: Package not found`, + error: `package "curl" not found: Package not found`, }, { summary: "Server error", risk: "stable", status: 500, statusText: "500 Internal Server Error", body: "boom", - error: "cannot fetch from bin store: 500 Internal Server Error", + error: "cannot fetch from store: 500 Internal Server Error", }, { summary: "Malformed response body", risk: "stable", status: 200, body: "not json", - error: "cannot decode bin store response: .*", + error: "cannot decode store response: .*", }, { summary: "Missing download digest", risk: "stable", status: 200, body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), - error: `bin "curl" has no download digest`, + error: `package "curl" has no download digest`, }} func (s *storeSuite) TestInfo(c *C) { @@ -465,7 +462,7 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { c.Assert(err, IsNil) _, _, err = src.Fetch("curl", "latest", "stable") - c.Assert(err, ErrorMatches, `bin download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) + c.Assert(err, ErrorMatches, `download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) } func (s *storeSuite) TestFetchMissingDigest(c *C) { @@ -487,7 +484,7 @@ func (s *storeSuite) TestFetchMissingDigest(c *C) { c.Assert(err, IsNil) _, _, err = src.Fetch("curl", "latest", "stable") - c.Assert(err, ErrorMatches, `bin "curl" has no download digest`) + c.Assert(err, ErrorMatches, `package "curl" has no download digest`) } func (s *storeSuite) TestStagingEnvVar(c *C) { @@ -550,5 +547,5 @@ func (s *storeSuite) TestFetchDownloadError(c *C) { c.Assert(err, IsNil) _, _, err = src.Fetch("curl", "latest", "stable") - c.Assert(err, ErrorMatches, `cannot download bin "curl": 500 Internal Server Error`) + c.Assert(err, ErrorMatches, `cannot download package "curl": 500 Internal Server Error`) } From 0da89bbe8195a35b62af1ecb8843a2c9556dad6a Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 15:46:18 +0200 Subject: [PATCH 51/61] fix: use RealName in extract error --- internal/slicer/slicer.go | 2 +- internal/slicer/slicer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 05f823210..3fbe31f13 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -277,7 +277,7 @@ func Run(options *RunOptions) error { // Store packages are distributed as plain tarballs, whose extraction // is not yet implemented. Fail until the format support is in place. if src.kind == sourceStore { - return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.Name) + return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName) } err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index c5a0f0053..806bcaa84 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2010,7 +2010,7 @@ var slicerTests = []slicerTest{{ /dir/store-file: `, }, - error: `cannot extract package "bin-store-pkg" from store: store packages are not yet supported`, + error: `cannot extract package "store-pkg" from store: store packages are not yet supported`, }} func (s *S) TestRun(c *C) { From 4fe25b106a2b335f422559afde5433305185ccd6 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 15:54:23 +0200 Subject: [PATCH 52/61] fix: clean model and add guardrail --- internal/store/store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/store/store.go b/internal/store/store.go index c3829ae7d..c039a70ad 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -127,7 +127,6 @@ type resolveResult struct { } type resolveError struct { - Code string `json:"code"` Message string `json:"message"` } @@ -202,6 +201,9 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro if len(res.PackageResults) == 0 { return nil, fmt.Errorf("package %q not found", name) } + if len(res.PackageResults) > 1 { + return nil, fmt.Errorf("internal error: store returned %d results for package %q", len(res.PackageResults), name) + } result := &res.PackageResults[0] if result.Status != "ok" || result.Result == nil { if result.Error != nil { From 2bb1b3ecb678e8d274367d02c96d0b988713ea54 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 23 Jun 2026 17:27:54 +0200 Subject: [PATCH 53/61] refactor: clean useless method --- internal/store/store.go | 19 +++++++++--------- internal/store/store_test.go | 39 ------------------------------------ internal/testutil/store.go | 5 ----- 3 files changed, 9 insertions(+), 54 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index c039a70ad..111fc33f4 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -20,7 +20,6 @@ import ( type Store interface { Options() *Options Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) - Exists(name, track, risk string) bool Info(name, track, risk string) (*StorePackageInfo, error) } @@ -188,6 +187,10 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro } defer resp.Body.Close() + // The resolve endpoint returns 200 for both success and per-package errors + // (e.g. package not found). Request-level errors (e.g. malformed channel) + // return 4xx. A simple status check is sufficient here because per-package + // errors are handled below via the result status field. if resp.StatusCode != 200 { return nil, fmt.Errorf("cannot fetch from store: %v", resp.Status) } @@ -218,9 +221,10 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro return rev, nil } -// nameExp matches a valid package name. It deliberately forbids "/" and any -// leading "." so that a name cannot be used to traverse or otherwise alter the -// store API URL path when interpolated into it. +// nameExp matches a valid package name. It is used to validate the name +// before sending it to the store API. Unlike other payload values (track, +// risk, arch) which are validated upstream, the package name comes directly +// from the release YAML without format validation. var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) // validateDownloadURL checks that the download URL is HTTPS and from the @@ -234,7 +238,7 @@ func validateDownloadURL(downloadURL, allowedHost string) error { return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) } host := strings.ToLower(u.Hostname()) - if host == allowedHost || strings.HasSuffix(host, "."+allowedHost) { + if host == allowedHost { return nil } return fmt.Errorf("download URL has untrusted host %q", host) @@ -260,11 +264,6 @@ func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { }, nil } -func (s *binStore) Exists(name, track, risk string) bool { - _, err := s.Info(name, track, risk) - return err == nil -} - func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { if risk == "" { risk = defaultRisk diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 381632d49..817287f2b 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -311,45 +311,6 @@ func (s *storeSuite) TestInfoRequest(c *C) { c.Assert(err, IsNil) } -func (s *storeSuite) TestExists(c *C) { - okBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") - notFoundBody := makeResolveErrorBody("nonexistent", "package-not-found", "Package not found") - - s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - var body struct { - Packages []struct { - Name string `json:"name"` - } `json:"packages"` - } - err := json.NewDecoder(req.Body).Decode(&body) - c.Assert(err, IsNil) - pkgName := "" - if len(body.Packages) > 0 { - pkgName = body.Packages[0].Name - } - if pkgName == "curl" { - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(okBody)), - }, nil - } - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(notFoundBody)), - }, nil - } - - src, err := store.Open(&store.Options{ - Arch: "amd64", - CacheDir: s.cacheDir, - Kind: "bin", - }) - c.Assert(err, IsNil) - - c.Assert(src.Exists("curl", "latest", "stable"), Equals, true) - c.Assert(src.Exists("nonexistent", "latest", "stable"), Equals, false) -} - func (s *storeSuite) TestFetchCacheMiss(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) diff --git a/internal/testutil/store.go b/internal/testutil/store.go index cd0650580..e8bf07ba1 100644 --- a/internal/testutil/store.go +++ b/internal/testutil/store.go @@ -30,11 +30,6 @@ func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *store.S return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil } -func (s *TestStore) Exists(name, track, risk string) bool { - _, ok := s.Packages[name] - return ok -} - func (s *TestStore) Info(name, track, risk string) (*store.StorePackageInfo, error) { pkg, ok := s.Packages[name] if !ok { From e299985f28214a0d885aa83ff285504adfface7e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 26 Jun 2026 09:24:20 +0200 Subject: [PATCH 54/61] fix: review --- cmd/chisel/cmd_cut.go | 6 ++++++ internal/slicer/slicer.go | 7 +++---- internal/store/store.go | 10 +++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 8e97454ab..fd82fcd29 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "slices" "time" @@ -131,6 +132,11 @@ func (cmd *cmdCut) Execute(args []string) error { Version: storeInfo.Version, }) if err != nil { + var unknownStoreKindError *store.UnknownStoreKindError + if errors.As(err, &unknownStoreKindError) { + logf("Store %q ignored: %v", storeName, err) + continue + } return err } stores[storeName] = openStore diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 3fbe31f13..a0aae7519 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -174,11 +174,10 @@ func Run(options *RunOptions) error { src := pkgSources[slice.Package] var reader io.ReadSeekCloser if src.kind == sourceStore { - // The package metadata returned by the store is not yet recorded - // in the manifest; this is handled in a subsequent change. - // Risk is left unspecified for now; the store applies its default. // The store channel track is "-", // e.g. "3.1-26.10". The version pins the release series. + // Risk is left unspecified for now; the store applies its default. + // In the future the risk will optionnaly come from the CLI. track := src.pkg.DefaultTrack + "-" + src.store.Options().Version reader, _, err = src.store.Fetch(src.pkg.RealName, track, "") if err != nil { @@ -276,7 +275,7 @@ func Run(options *RunOptions) error { src := pkgSources[slice.Package] // Store packages are distributed as plain tarballs, whose extraction // is not yet implemented. Fail until the format support is in place. - if src.kind == sourceStore { + if src.kind != sourceArchive { return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName) } err := deb.Extract(reader, &deb.ExtractOptions{ diff --git a/internal/store/store.go b/internal/store/store.go index 111fc33f4..479c9a4e2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -71,6 +71,14 @@ var bulkClient = &http.Client{ var bulkDo = bulkClient.Do +type UnknownStoreKindError struct { + kind string +} + +func (e *UnknownStoreKindError) Error() string { + return fmt.Sprintf("unsupported store kind %q", e.kind) +} + func Open(options *Options) (Store, error) { var err error if options.Arch == "" { @@ -97,7 +105,7 @@ func Open(options *Options) (Store, error) { downloadHost: downloadHost, }, nil default: - return nil, fmt.Errorf("unsupported store kind %q", options.Kind) + return nil, &UnknownStoreKindError{kind: options.Kind} } } From a03516171843b88c6fb8e7eab68f55d5a83bc290 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 29 Jun 2026 10:30:39 +0200 Subject: [PATCH 55/61] fix: review --- internal/store/store.go | 46 ++++++++++++++++++------------------ internal/store/store_test.go | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 479c9a4e2..6216bb2d8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -71,6 +71,12 @@ var bulkClient = &http.Client{ var bulkDo = bulkClient.Do +// nameExp matches a valid package name. It is used to validate the name +// before sending it to the store API. Unlike other payload values (track, +// risk, arch) which are validated upstream, the package name comes directly +// from the release YAML without format validation. +var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) + type UnknownStoreKindError struct { kind string } @@ -229,29 +235,6 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro return rev, nil } -// nameExp matches a valid package name. It is used to validate the name -// before sending it to the store API. Unlike other payload values (track, -// risk, arch) which are validated upstream, the package name comes directly -// from the release YAML without format validation. -var nameExp = regexp.MustCompile(`^[a-z0-9][a-z0-9+.-]*$`) - -// validateDownloadURL checks that the download URL is HTTPS and from the -// allowed host. -func validateDownloadURL(downloadURL, allowedHost string) error { - u, err := url.Parse(downloadURL) - if err != nil { - return fmt.Errorf("cannot parse download URL: %v", err) - } - if u.Scheme != "https" { - return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) - } - host := strings.ToLower(u.Hostname()) - if host == allowedHost { - return nil - } - return fmt.Errorf("download URL has untrusted host %q", host) -} - func (s *binStore) Options() *Options { return &s.options } @@ -338,3 +321,20 @@ func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePac } return reader, info, nil } + +// validateDownloadURL checks that the download URL is HTTPS and from the +// allowed host. +func validateDownloadURL(downloadURL, allowedHost string) error { + u, err := url.Parse(downloadURL) + if err != nil { + return fmt.Errorf("cannot parse download URL: %v", err) + } + if u.Scheme != "https" { + return fmt.Errorf("download URL must use HTTPS: %q", downloadURL) + } + host := strings.ToLower(u.Hostname()) + if host == allowedHost { + return nil + } + return fmt.Errorf("download URL has untrusted host %q", host) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 817287f2b..caad3e998 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -326,7 +326,7 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { Body: io.NopCloser(bytes.NewReader(infoBody)), }, nil } - // Download URL + // Download URL. return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(tarData)), From 3cdb869fed9077ecde1e1a4c3bbdf176b911f738 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 29 Jun 2026 14:53:26 +0200 Subject: [PATCH 56/61] fix: review --- internal/store/store.go | 17 ------ internal/store/store_test.go | 108 +++++++++++++++-------------------- internal/testutil/store.go | 12 ---- 3 files changed, 45 insertions(+), 92 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index 6216bb2d8..9aacb543b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -20,7 +20,6 @@ import ( type Store interface { Options() *Options Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) - Info(name, track, risk string) (*StorePackageInfo, error) } // StorePackageInfo holds metadata about a package. @@ -239,22 +238,6 @@ func (s *binStore) Options() *Options { return &s.options } -func (s *binStore) Info(name, track, risk string) (*StorePackageInfo, error) { - if risk == "" { - risk = defaultRisk - } - rev, err := s.resolveRevision(name, track, risk) - if err != nil { - return nil, err - } - return &StorePackageInfo{ - Name: name, - Version: rev.Version, - Revision: rev.Revision, - SHA384: rev.Download.SHA384, - }, nil -} - func (s *binStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *StorePackageInfo, error) { if risk == "" { risk = defaultRisk diff --git a/internal/store/store_test.go b/internal/store/store_test.go index caad3e998..d6c7d7a7f 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -126,20 +126,6 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { error string }{ {"https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, - // Staging host is rejected when the production host is allowed. - { - "https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", - "api.snapcraft.io", - `download URL has untrusted host "api.staging.snapcraft.io"`, - }, - // Staging host is allowed when it is the allowed host. - {"https://api.staging.snapcraft.io/api/v1/bins/download/foo.bin", "api.staging.snapcraft.io", ""}, - // Production API host is rejected when the staging host is allowed. - { - "https://api.snapcraft.io/api/v1/bins/download/foo.bin", - "api.staging.snapcraft.io", - `download URL has untrusted host "api.snapcraft.io"`, - }, { "http://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", @@ -195,28 +181,22 @@ func (s *storeSuite) TestOpenArchValidation(c *C) { } } -type infoTest struct { +type fetchTest struct { summary string risk string status int statusText string body string - info *store.StorePackageInfo error string } -var infoTests = []infoTest{{ - summary: "Successful info", - risk: "stable", - status: 200, - body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), - info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, -}, { +var fetchTests = []fetchTest{{ summary: "Defaults to stable risk when unspecified", risk: "", status: 200, - body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123")), - info: &store.StorePackageInfo{Name: "curl", Version: "8.5.0", Revision: 42, SHA384: "abc123"}, + // Uses a real digest so the cache verification passes. + body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, + sha384Hash([]byte("fake tar.xz content")))), }, { summary: "Package not found", risk: "stable", @@ -244,15 +224,23 @@ var infoTests = []infoTest{{ error: `package "curl" has no download digest`, }} -func (s *storeSuite) TestInfo(c *C) { - for _, test := range infoTests { +func (s *storeSuite) TestFetch(c *C) { + tarData := []byte("fake tar.xz content") + for _, test := range fetchTests { c.Logf("Summary: %s", test.summary) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: test.status, + Status: test.statusText, + Body: io.NopCloser(strings.NewReader(test.body)), + }, nil + } + // Download URL. return &http.Response{ - StatusCode: test.status, - Status: test.statusText, - Body: io.NopCloser(strings.NewReader(test.body)), + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), }, nil } @@ -263,22 +251,29 @@ func (s *storeSuite) TestInfo(c *C) { }) c.Assert(err, IsNil) - info, err := src.Info("curl", "latest", test.risk) + _, _, err = src.Fetch("curl", "latest", test.risk) if test.error != "" { c.Assert(err, ErrorMatches, test.error) - continue + } else { + c.Assert(err, IsNil) } - c.Assert(err, IsNil) - c.Assert(info, DeepEquals, test.info) } } -func (s *storeSuite) TestInfoRequest(c *C) { - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") +func (s *storeSuite) TestResolveRequest(c *C) { + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { + if req.URL.Path != "/v2/revisions/resolve" { + // Download request. + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(tarData)), + }, nil + } c.Assert(req.Method, Equals, "POST") - c.Assert(req.URL.Path, Equals, "/v2/revisions/resolve") c.Assert(req.Header.Get("Content-Type"), Equals, "application/json") c.Assert(req.Header.Get("Accept"), Equals, "application/json") @@ -307,7 +302,7 @@ func (s *storeSuite) TestInfoRequest(c *C) { }) c.Assert(err, IsNil) - _, err = src.Info("curl", "latest", "stable") + _, _, err = src.Fetch("curl", "latest", "stable") c.Assert(err, IsNil) } @@ -426,28 +421,6 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { c.Assert(err, ErrorMatches, `download URL must use HTTPS: "http://evil.example.com/bins/curl.tar.xz"`) } -func (s *storeSuite) TestFetchMissingDigest(c *C) { - // The store omits the sha3-384 digest from the response. - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "") - - s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), - }, nil - } - - src, err := store.Open(&store.Options{ - Arch: "amd64", - CacheDir: s.cacheDir, - Kind: "bin", - }) - c.Assert(err, IsNil) - - _, _, err = src.Fetch("curl", "latest", "stable") - c.Assert(err, ErrorMatches, `package "curl" has no download digest`) -} - func (s *storeSuite) TestStagingEnvVar(c *C) { s.envRestore() s.envRestore = fakeEnv("1") @@ -459,18 +432,27 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { }) c.Assert(err, IsNil) - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + tarData := []byte("fake tar.xz content") + digest := sha384Hash(tarData) + infoBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, digest, + "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { // Verify staging URL is used. c.Assert(req.URL.Host, Equals, "api.staging.snapcraft.io") + if req.URL.Path == "/v2/revisions/resolve" { + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(infoBody)), + }, nil + } return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(tarData)), }, nil } - _, err = src.Info("curl", "latest", "stable") + _, _, err = src.Fetch("curl", "latest", "stable") c.Assert(err, IsNil) } diff --git a/internal/testutil/store.go b/internal/testutil/store.go index e8bf07ba1..fe91fdd09 100644 --- a/internal/testutil/store.go +++ b/internal/testutil/store.go @@ -29,15 +29,3 @@ func (s *TestStore) Fetch(name, track, risk string) (io.ReadSeekCloser, *store.S } return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil } - -func (s *TestStore) Info(name, track, risk string) (*store.StorePackageInfo, error) { - pkg, ok := s.Packages[name] - if !ok { - return nil, fmt.Errorf("cannot find package %q in store", name) - } - return &store.StorePackageInfo{ - Name: pkg.Name, - Version: pkg.Version, - SHA384: pkg.Hash, - }, nil -} From 79c437efdde7c30df3adcf46f79c4a21052ee3ef Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 29 Jun 2026 15:33:33 +0200 Subject: [PATCH 57/61] fix: review --- internal/store/store_test.go | 66 +++++++++++++++++++----------------- internal/store/suite_test.go | 9 +++++ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index d6c7d7a7f..ca9f8805b 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -32,7 +32,7 @@ func (s *storeSuite) SetUpTest(c *C) { s.cacheDir = filepath.Join(s.tempDir, "cache") c.Assert(os.MkdirAll(s.cacheDir, 0o755), IsNil) - s.envRestore = fakeEnv("") + s.envRestore = fakeEnv(store.BinStagingEnvVar, "") s.restore = store.FakeDo(s.doRequest) s.fakeDoFunc = nil } @@ -49,18 +49,14 @@ func (s *storeSuite) doRequest(req *http.Request) (*http.Response, error) { return nil, fmt.Errorf("unexpected HTTP request: %s", req.URL.String()) } -func fakeEnv(staging string) func() { - oldStaging := os.Getenv(store.BinStagingEnvVar) - if staging != "" { - os.Setenv(store.BinStagingEnvVar, staging) - } else { - os.Unsetenv(store.BinStagingEnvVar) - } +func fakeEnv(name, value string) (restore func()) { + origValue, origSet := os.LookupEnv(name) + os.Setenv(name, value) return func() { - if oldStaging != "" { - os.Setenv(store.BinStagingEnvVar, oldStaging) + if origSet { + os.Setenv(name, origValue) } else { - os.Unsetenv(store.BinStagingEnvVar) + os.Unsetenv(name) } } } @@ -121,34 +117,40 @@ func makeResolveErrorBody(name, code, message string) []byte { func (s *storeSuite) TestValidateDownloadURL(c *C) { tests := []struct { + summary string url string allowedHost string error string }{ - {"https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, + {"Valid production host", "https://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", ""}, { + "HTTP rejected", "http://api.snapcraft.io/api/v1/bins/download/foo.bin", "api.snapcraft.io", `download URL must use HTTPS: "http://api.snapcraft.io/api/v1/bins/download/foo.bin"`, }, { + "Untrusted host", "https://evil.example.com/api/v1/bins/download/foo.bin", "api.snapcraft.io", `download URL has untrusted host "evil.example.com"`, }, { + "Untrusted host case-insensitive", "https://Evil.Example.Com/api/v1/bins/download/foo.bin", "api.snapcraft.io", `download URL has untrusted host "evil.example.com"`, }, { + "Host suffix attack", "https://api.snapcraft.io.evil.com/api/v1/bins/download/foo.bin", "api.snapcraft.io", `download URL has untrusted host "api.snapcraft.io.evil.com"`, }, - {"://invalid-url", "api.snapcraft.io", `cannot parse download URL: .*`}, + {"Invalid URL", "://invalid-url", "api.snapcraft.io", `cannot parse download URL: .*`}, } for _, test := range tests { + c.Logf("Summary: %s", test.summary) err := store.ValidateDownloadURL(test.url, test.allowedHost) if test.error == "" { c.Assert(err, IsNil) @@ -160,14 +162,16 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { func (s *storeSuite) TestOpenArchValidation(c *C) { tests := []struct { - arch string - error string + summary string + arch string + error string }{ - {"amd64", ""}, - {"arm64", ""}, - {"invalid", "invalid package architecture: invalid"}, + {"Valid amd64", "amd64", ""}, + {"Valid arm64", "arm64", ""}, + {"Invalid architecture", "invalid", "invalid package architecture: invalid"}, } for _, test := range tests { + c.Logf("Summary: %s", test.summary) _, err := store.Open(&store.Options{ Arch: test.arch, CacheDir: s.cacheDir, @@ -263,7 +267,7 @@ func (s *storeSuite) TestFetch(c *C) { func (s *storeSuite) TestResolveRequest(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if req.URL.Path != "/v2/revisions/resolve" { @@ -291,7 +295,7 @@ func (s *storeSuite) TestResolveRequest(c *C) { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } @@ -310,7 +314,7 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) callCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -318,7 +322,7 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { if req.URL.Path == "/v2/revisions/resolve" { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } // Download URL. @@ -364,7 +368,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { err := cc.Write(cache.SHA384, digest, tarData) c.Assert(err, IsNil) - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) infoCallCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -372,7 +376,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { infoCallCount++ return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } return nil, fmt.Errorf("download should not be called for cache hit") @@ -400,13 +404,13 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { // Override the download URL to an invalid one. - infoBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", + resolveBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", "http://evil.example.com/bins/curl.tar.xz") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } @@ -423,7 +427,7 @@ func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { func (s *storeSuite) TestStagingEnvVar(c *C) { s.envRestore() - s.envRestore = fakeEnv("1") + s.envRestore = fakeEnv(store.BinStagingEnvVar, "1") src, err := store.Open(&store.Options{ Arch: "amd64", @@ -434,7 +438,7 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - infoBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, digest, + resolveBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, digest, "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -443,7 +447,7 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { if req.URL.Path == "/v2/revisions/resolve" { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } return &http.Response{ @@ -466,13 +470,13 @@ func (s *storeSuite) TestOpenUnsupportedKind(c *C) { } func (s *storeSuite) TestFetchDownloadError(c *C) { - infoBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if req.URL.Path == "/v2/revisions/resolve" { return &http.Response{ StatusCode: 200, - Body: io.NopCloser(bytes.NewReader(infoBody)), + Body: io.NopCloser(bytes.NewReader(resolveBody)), }, nil } return &http.Response{ diff --git a/internal/store/suite_test.go b/internal/store/suite_test.go index 552ab1cc8..4eb91ecb0 100644 --- a/internal/store/suite_test.go +++ b/internal/store/suite_test.go @@ -3,6 +3,7 @@ package store_test import ( "testing" + "github.com/canonical/chisel/internal/store" . "gopkg.in/check.v1" ) @@ -11,3 +12,11 @@ func Test(t *testing.T) { TestingT(t) } type S struct{} var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + store.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + store.SetLogger(nil) +} From 684f809b226b60a2c7f96fb72345d053db0a3be2 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 11:40:29 +0200 Subject: [PATCH 58/61] test: fix inconsistencies --- internal/store/store_test.go | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index ca9f8805b..c2c0c7cc7 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -160,23 +160,26 @@ func (s *storeSuite) TestValidateDownloadURL(c *C) { } } -func (s *storeSuite) TestOpenArchValidation(c *C) { +func (s *storeSuite) TestOpenOptionErrors(c *C) { tests := []struct { summary string - arch string + options store.Options error string - }{ - {"Valid amd64", "amd64", ""}, - {"Valid arm64", "arm64", ""}, - {"Invalid architecture", "invalid", "invalid package architecture: invalid"}, - } + }{{ + summary: "Valid amd64", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "bin"}, + }, { + summary: "Invalid architecture", + options: store.Options{Arch: "invalid", CacheDir: s.cacheDir, Kind: "bin"}, + error: "invalid package architecture: invalid", + }, { + summary: "Unsupported store kind", + options: store.Options{Arch: "amd64", CacheDir: s.cacheDir, Kind: "snap"}, + error: `unsupported store kind "snap"`, + }} for _, test := range tests { c.Logf("Summary: %s", test.summary) - _, err := store.Open(&store.Options{ - Arch: test.arch, - CacheDir: s.cacheDir, - Kind: "bin", - }) + _, err := store.Open(&test.options) if test.error == "" { c.Assert(err, IsNil) } else { @@ -460,15 +463,6 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { c.Assert(err, IsNil) } -func (s *storeSuite) TestOpenUnsupportedKind(c *C) { - _, err := store.Open(&store.Options{ - Arch: "amd64", - CacheDir: s.cacheDir, - Kind: "snap", - }) - c.Assert(err, ErrorMatches, `unsupported store kind "snap"`) -} - func (s *storeSuite) TestFetchDownloadError(c *C) { resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") From 4a4472cf002f66b3d032d5e8e61cbe1280a82880 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 13:59:16 +0200 Subject: [PATCH 59/61] test: refine --- internal/store/store_test.go | 75 ++++++++++++++++++++++++++---------- internal/store/suite_test.go | 2 + 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index c2c0c7cc7..76d016110 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -67,12 +67,21 @@ func sha384Hash(data []byte) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func makeResolveBody(name, track, risk, arch, version string, revision int, sha384 string) []byte { - downloadURL := "https://api.snapcraft.io/api/v1/bins/download/" + name + ".bin" - return makeResolveBodyWithURL(name, track, risk, arch, version, revision, sha384, downloadURL) +type resolveBodyOptions struct { + name string + track string + risk string + arch string + version string + revision int + sha384 string + downloadURL string } -func makeResolveBodyWithURL(name, track, risk, arch, version string, revision int, sha384, downloadURL string) []byte { +func makeResolveBody(opts resolveBodyOptions) []byte { + if opts.downloadURL == "" { + opts.downloadURL = "https://api.snapcraft.io/api/v1/bins/download/" + opts.name + ".bin" + } return []byte(fmt.Sprintf(`{ "craft-results": [], "package-results": [ @@ -98,7 +107,7 @@ func makeResolveBodyWithURL(name, track, risk, arch, version string, revision in } } ] - }`, name, track+"/"+risk, risk, track, arch, version, revision, downloadURL, sha384)) + }`, opts.name, opts.track+"/"+opts.risk, opts.risk, opts.track, opts.arch, opts.version, opts.revision, opts.downloadURL, opts.sha384)) } func makeResolveErrorBody(name, code, message string) []byte { @@ -202,8 +211,11 @@ var fetchTests = []fetchTest{{ risk: "", status: 200, // Uses a real digest so the cache verification passes. - body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, - sha384Hash([]byte("fake tar.xz content")))), + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + })), }, { summary: "Package not found", risk: "stable", @@ -227,8 +239,11 @@ var fetchTests = []fetchTest{{ summary: "Missing download digest", risk: "stable", status: 200, - body: string(makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "")), - error: `package "curl" has no download digest`, + body: string(makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + })), + error: `package "curl" has no download digest`, }} func (s *storeSuite) TestFetch(c *C) { @@ -270,7 +285,10 @@ func (s *storeSuite) TestFetch(c *C) { func (s *storeSuite) TestResolveRequest(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if req.URL.Path != "/v2/revisions/resolve" { @@ -317,7 +335,10 @@ func (s *storeSuite) TestFetchCacheMiss(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) callCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { @@ -371,12 +392,15 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { err := cc.Write(cache.SHA384, digest, tarData) c.Assert(err, IsNil) - resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, digest) + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + }) - infoCallCount := 0 + resolveCallCount := 0 s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if req.URL.Path == "/v2/revisions/resolve" { - infoCallCount++ + resolveCallCount++ return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewReader(resolveBody)), @@ -398,7 +422,7 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { c.Assert(info.Name, Equals, "curl") c.Assert(info.SHA384, Equals, digest) - c.Assert(infoCallCount, Equals, 1) + c.Assert(resolveCallCount, Equals, 1) data, err := io.ReadAll(reader) c.Assert(err, IsNil) @@ -407,8 +431,12 @@ func (s *storeSuite) TestFetchCacheHit(c *C) { func (s *storeSuite) TestFetchInvalidDownloadURL(c *C) { // Override the download URL to an invalid one. - resolveBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123", - "http://evil.example.com/bins/curl.tar.xz") + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + downloadURL: "http://evil.example.com/bins/curl.tar.xz", + }) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { return &http.Response{ @@ -441,8 +469,11 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { tarData := []byte("fake tar.xz content") digest := sha384Hash(tarData) - resolveBody := makeResolveBodyWithURL("curl", "latest", "stable", "amd64", "8.5.0", 42, digest, - "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin") + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, sha384: digest, + downloadURL: "https://api.staging.snapcraft.io/api/v1/bins/download/curl.bin", + }) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { // Verify staging URL is used. @@ -464,7 +495,11 @@ func (s *storeSuite) TestStagingEnvVar(c *C) { } func (s *storeSuite) TestFetchDownloadError(c *C) { - resolveBody := makeResolveBody("curl", "latest", "stable", "amd64", "8.5.0", 42, "abc123") + resolveBody := makeResolveBody(resolveBodyOptions{ + name: "curl", track: "latest", risk: "stable", arch: "amd64", + version: "8.5.0", revision: 42, + sha384: sha384Hash([]byte("fake tar.xz content")), + }) s.fakeDoFunc = func(req *http.Request) (*http.Response, error) { if req.URL.Path == "/v2/revisions/resolve" { diff --git a/internal/store/suite_test.go b/internal/store/suite_test.go index 4eb91ecb0..2eac508f2 100644 --- a/internal/store/suite_test.go +++ b/internal/store/suite_test.go @@ -14,9 +14,11 @@ type S struct{} var _ = Suite(&S{}) func (s *S) SetUpTest(c *C) { + store.SetDebug(true) store.SetLogger(c) } func (s *S) TearDownTest(c *C) { + store.SetDebug(false) store.SetLogger(nil) } From 98a0ec950f3586eb2b0e6cfe3d10b046e5cc7e4f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 10:59:55 +0200 Subject: [PATCH 60/61] refactor: move tarball extract to dedicated pkg --- cmd/chisel/main.go | 2 + internal/archive/archive_test.go | 6 +- internal/deb/extract.go | 391 --------------------- internal/slicer/slicer.go | 14 +- internal/tarball/extract.go | 398 ++++++++++++++++++++++ internal/{deb => tarball}/extract_test.go | 246 ++++++------- internal/tarball/log.go | 53 +++ internal/tarball/suite_test.go | 20 ++ 8 files changed, 606 insertions(+), 524 deletions(-) create mode 100644 internal/tarball/extract.go rename internal/{deb => tarball}/extract_test.go (71%) create mode 100644 internal/tarball/log.go create mode 100644 internal/tarball/suite_test.go diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index a982e3ef3..0450e901c 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -17,6 +17,7 @@ import ( "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/store" + "github.com/canonical/chisel/internal/tarball" //"github.com/canonical/chisel/internal/logger" ) @@ -329,6 +330,7 @@ func run() error { setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) store.SetLogger(log.Default()) + tarball.SetLogger(log.Default()) SetLogger(log.Default()) parser := Parser() diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 8ca840e59..c21e48161 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -18,7 +18,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" - "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -856,10 +856,10 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) - err = deb.Extract(pkg, &deb.ExtractOptions{ + err = tarball.Extract(pkg, &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, - Extract: map[string][]deb.ExtractInfo{ + Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 67bee3f3d..bf543c2d8 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -1,384 +1,15 @@ package deb import ( - "archive/tar" - "bytes" "compress/gzip" "fmt" "io" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - "syscall" "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" - - "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/strdist" ) -type ExtractOptions struct { - Package string - TargetDir string - Extract map[string][]ExtractInfo - // Create can optionally be set to control the creation of extracted entries. - // extractInfos is set to the matching entries in Extract, and is nil in cases where - // the created entry is implicit and unlisted (for example, parent directories). - Create func(extractInfos []ExtractInfo, options *fsutil.CreateOptions) error -} - -type ExtractInfo struct { - Path string - Mode uint - Optional bool - Context any -} - -func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { - for extractPath, extractInfos := range options.Extract { - isGlob := strings.ContainsAny(extractPath, "*?") - if isGlob { - for _, extractInfo := range extractInfos { - if extractInfo.Path != extractPath || extractInfo.Mode != 0 { - return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) - } - } - } - } - - if options.Create == nil { - validOpts := *options - validOpts.Create = func(_ []ExtractInfo, o *fsutil.CreateOptions) error { - _, err := fsutil.Create(o) - return err - } - return &validOpts, nil - } - - return options, nil -} - -func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) - } - }() - - logf("Extracting files from package %q...", options.Package) - - validOpts, err := getValidOptions(options) - if err != nil { - return err - } - - _, err = os.Stat(validOpts.TargetDir) - if os.IsNotExist(err) { - return fmt.Errorf("target directory does not exist") - } else if err != nil { - return err - } - - return extractData(pkgReader, validOpts) -} - -func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := DataReader(pkgReader) - if err != nil { - return err - } - defer dataReader.Close() - - oldUmask := syscall.Umask(0) - defer func() { - syscall.Umask(oldUmask) - }() - - pendingPaths := make(map[string]bool) - for extractPath, extractInfos := range options.Extract { - for _, extractInfo := range extractInfos { - if !extractInfo.Optional { - pendingPaths[extractPath] = true - break - } - } - } - - // Store the hard links that we cannot extract when we first iterate over - // the tarball. - // - // This happens because the tarball only stores the contents once in the - // first entry and the rest of them point to the first one. Therefore, we - // cannot tell whether we need to extract the content until after we get to - // a hard link. In this case, we need a second pass. - pendingHardLinks := make(map[string][]pendingHardLink) - - // When creating a file we will iterate through its parent directories and - // create them with the permissions defined in the tarball. - // - // The assumption is that the tar entries of the parent directories appear - // before the entry for the file itself. This is the case for .deb files but - // not for all tarballs. - tarDirMode := make(map[string]fs.FileMode) - tarReader := tar.NewReader(dataReader) - for { - tarHeader, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - sourcePath, ok := sanitizeTarPath(tarHeader.Name) - if !ok { - continue - } - - sourceIsDir := sourcePath[len(sourcePath)-1] == '/' - if sourceIsDir { - tarDirMode[sourcePath] = tarHeader.FileInfo().Mode() - } - - // Find all globs and copies that require this source, and map them by - // their target paths on disk. - targetPaths := map[string][]ExtractInfo{} - for extractPath, extractInfos := range options.Extract { - if extractPath == "" { - continue - } - if strings.ContainsAny(extractPath, "*?") { - if strdist.GlobPath(extractPath, sourcePath) { - targetPaths[sourcePath] = append(targetPaths[sourcePath], extractInfos...) - delete(pendingPaths, extractPath) - } - } else if extractPath == sourcePath { - for _, extractInfo := range extractInfos { - targetPaths[extractInfo.Path] = append(targetPaths[extractInfo.Path], extractInfo) - } - delete(pendingPaths, extractPath) - } - } - if len(targetPaths) == 0 { - // Nothing to do. - continue - } - - var contentCache []byte - var contentIsCached = len(targetPaths) > 1 && !sourceIsDir - if contentIsCached { - // Read and cache the content so it may be reused. - // As an alternative, to avoid having an entire file in - // memory at once this logic might open the first file - // written and copy it every time. For now, the choice - // is speed over memory efficiency. - data, err := io.ReadAll(tarReader) - if err != nil { - return err - } - contentCache = data - } - - var pathReader io.Reader = tarReader - for targetPath, extractInfos := range targetPaths { - if contentIsCached { - pathReader = bytes.NewReader(contentCache) - } - mode := extractInfos[0].Mode - for _, extractInfo := range extractInfos { - if extractInfo.Mode != mode { - if mode < extractInfo.Mode { - mode, extractInfo.Mode = extractInfo.Mode, mode - } - return fmt.Errorf("path %s requested twice with diverging mode: 0%03o != 0%03o", targetPath, mode, extractInfo.Mode) - } - } - if mode != 0 { - tarHeader.Mode = int64(mode) - } - // Create the parent directories using the permissions from the tarball. - parents := parentDirs(targetPath) - for _, path := range parents { - if path == "/" { - continue - } - mode, ok := tarDirMode[path] - if !ok { - continue - } - delete(tarDirMode, path) - - createOptions := &fsutil.CreateOptions{ - Root: options.TargetDir, - Path: path, - Mode: mode, - MakeParents: true, - } - err := options.Create(nil, createOptions) - if err != nil { - return err - } - } - link := tarHeader.Linkname - if tarHeader.Typeflag == tar.TypeLink { - // A hard link requires the real path of the target file. - link = filepath.Join(options.TargetDir, link) - } - - // Create the entry itself. - createOptions := &fsutil.CreateOptions{ - Root: options.TargetDir, - Path: targetPath, - Mode: tarHeader.FileInfo().Mode(), - Data: pathReader, - Link: link, - MakeParents: true, - OverrideMode: true, - } - err := options.Create(extractInfos, createOptions) - if err != nil && os.IsNotExist(err) && tarHeader.Typeflag == tar.TypeLink { - // The hard link could not be created because the content - // was not extracted previously. Add this hard link entry - // to the pending list to extract later. - relLinkPath, ok := sanitizeTarPath(tarHeader.Linkname) - if !ok { - return fmt.Errorf("invalid link target %s", tarHeader.Linkname) - } - info := pendingHardLink{ - path: targetPath, - extractInfos: extractInfos, - } - pendingHardLinks[relLinkPath] = append(pendingHardLinks[relLinkPath], info) - } else if err != nil { - return err - } - } - } - - if len(pendingHardLinks) > 0 { - // Go over the tarball again to textract the pending hard links. - extractHardLinkOptions := &extractHardLinkOptions{ - ExtractOptions: options, - pendingLinks: pendingHardLinks, - } - _, err := pkgReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = extractHardLinks(pkgReader, extractHardLinkOptions) - if err != nil { - return err - } - } - - if len(pendingPaths) > 0 { - pendingList := make([]string, 0, len(pendingPaths)) - for pendingPath := range pendingPaths { - pendingList = append(pendingList, pendingPath) - } - if len(pendingList) == 1 { - return fmt.Errorf("no content at %s", pendingList[0]) - } else { - sort.Strings(pendingList) - return fmt.Errorf("no content at:\n- %s", strings.Join(pendingList, "\n- ")) - } - } - - return nil -} - -type pendingHardLink struct { - path string - extractInfos []ExtractInfo -} - -type extractHardLinkOptions struct { - *ExtractOptions - pendingLinks map[string][]pendingHardLink -} - -// extractHardLinks iterates through the tarball a second time to extract the -// hard links that were not extracted in the first pass. -func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := DataReader(pkgReader) - if err != nil { - return err - } - defer dataReader.Close() - - tarReader := tar.NewReader(dataReader) - for { - tarHeader, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - sourcePath, ok := sanitizeTarPath(tarHeader.Name) - if !ok { - continue - } - - links := opts.pendingLinks[sourcePath] - if len(links) == 0 { - continue - } - - // For a target path, the first hard link will be created as a file with - // the content of the target path. If there are more pending hard links, - // the remaining ones will be created as hard links with the newly - // created file as their target. - absLink := filepath.Join(opts.TargetDir, links[0].path) - // Extract the content to the first hard link path. - createOptions := &fsutil.CreateOptions{ - Root: opts.TargetDir, - Path: links[0].path, - Mode: tarHeader.FileInfo().Mode(), - Data: tarReader, - } - err = opts.Create(links[0].extractInfos, createOptions) - if err != nil { - return err - } - - // Create the remaining hard links. - for _, link := range links[1:] { - createOptions := &fsutil.CreateOptions{ - Root: opts.TargetDir, - Path: link.path, - Mode: tarHeader.FileInfo().Mode(), - // Link to the first file extracted for the hard links. - Link: absLink, - } - err := opts.Create(link.extractInfos, createOptions) - if err != nil { - return err - } - } - delete(opts.pendingLinks, sourcePath) - } - - // If there are pending links, that means the link targets do not come from - // this package. - if len(opts.pendingLinks) > 0 { - var targets []string - for target := range opts.pendingLinks { - targets = append(targets, target) - } - sort.Strings(targets) - link := opts.pendingLinks[targets[0]][0] - return fmt.Errorf("cannot create hard link %s: no content at %s", link.path, targets[0]) - } - - return nil -} - // DataReader takes a Reader for the ar file belonging to a Debian package and // returns a Reader to the inner tarball. func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { @@ -416,25 +47,3 @@ func DataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { return dataReader, nil } - -func parentDirs(path string) []string { - path = filepath.Clean(path) - parents := make([]string, strings.Count(path, "/")) - count := 0 - for i, c := range path { - if c == '/' { - parents[count] = path[:i+1] - count++ - } - } - return parents -} - -// sanitizeTarPath removes the leading "./" from the source path in the tarball, -// and verifies that the path is not empty. -func sanitizeTarPath(path string) (string, bool) { - if len(path) < 3 || path[0] != '.' || path[1] != '/' { - return "", false - } - return path[1:], true -} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index a0aae7519..b62266795 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,12 +16,12 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/store" + "github.com/canonical/chisel/internal/tarball" ) const manifestMode fs.FileMode = 0644 @@ -120,11 +120,11 @@ func Run(options *RunOptions) error { } // Build information to process the selection. - extract := make(map[string]map[string][]deb.ExtractInfo) + extract := make(map[string]map[string][]tarball.ExtractInfo) for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - extractPackage = make(map[string][]deb.ExtractInfo) + extractPackage = make(map[string][]tarball.ExtractInfo) extract[slice.Package] = extractPackage } src := pkgSources[slice.Package] @@ -144,7 +144,7 @@ func Run(options *RunOptions) error { if sourcePath == "" { sourcePath = targetPath } - extractPackage[sourcePath] = append(extractPackage[sourcePath], deb.ExtractInfo{ + extractPackage[sourcePath] = append(extractPackage[sourcePath], tarball.ExtractInfo{ Path: targetPath, Context: slice, }) @@ -156,7 +156,7 @@ func Run(options *RunOptions) error { if targetDir == "" || targetDir == "/" { continue } - extractPackage[targetDir] = append(extractPackage[targetDir], deb.ExtractInfo{ + extractPackage[targetDir] = append(extractPackage[targetDir], tarball.ExtractInfo{ Path: targetDir, Optional: true, }) @@ -211,7 +211,7 @@ func Run(options *RunOptions) error { var implicitConflicts []string // Creates the filesystem entry and adds it to the report. It also updates // knownPaths with the files created. - create := func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + create := func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { entry, err := fsutil.Create(o) if err != nil { return err @@ -278,7 +278,7 @@ func Run(options *RunOptions) error { if src.kind != sourceArchive { return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName) } - err := deb.Extract(reader, &deb.ExtractOptions{ + err := tarball.Extract(reader, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go new file mode 100644 index 000000000..40db2cbac --- /dev/null +++ b/internal/tarball/extract.go @@ -0,0 +1,398 @@ +package tarball + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/canonical/chisel/internal/deb" + "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/strdist" +) + +type ExtractOptions struct { + Package string + TargetDir string + Extract map[string][]ExtractInfo + // Create can optionally be set to control the creation of extracted entries. + // extractInfos is set to the matching entries in Extract, and is nil in cases where + // the created entry is implicit and unlisted (for example, parent directories). + Create func(extractInfos []ExtractInfo, options *fsutil.CreateOptions) error +} + +type ExtractInfo struct { + Path string + Mode uint + Optional bool + Context any +} + +func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { + for extractPath, extractInfos := range options.Extract { + isGlob := strings.ContainsAny(extractPath, "*?") + if isGlob { + for _, extractInfo := range extractInfos { + if extractInfo.Path != extractPath || extractInfo.Mode != 0 { + return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) + } + } + } + } + + if options.Create == nil { + validOpts := *options + validOpts.Create = func(_ []ExtractInfo, o *fsutil.CreateOptions) error { + _, err := fsutil.Create(o) + return err + } + return &validOpts, nil + } + + return options, nil +} + +func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("cannot extract from package %q: %w", options.Package, err) + } + }() + + logf("Extracting files from package %q...", options.Package) + + validOpts, err := getValidOptions(options) + if err != nil { + return err + } + + _, err = os.Stat(validOpts.TargetDir) + if os.IsNotExist(err) { + return fmt.Errorf("target directory does not exist") + } else if err != nil { + return err + } + + return extractData(pkgReader, validOpts) +} + +func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { + dataReader, err := deb.DataReader(pkgReader) + if err != nil { + return err + } + defer dataReader.Close() + + oldUmask := syscall.Umask(0) + defer func() { + syscall.Umask(oldUmask) + }() + + pendingPaths := make(map[string]bool) + for extractPath, extractInfos := range options.Extract { + for _, extractInfo := range extractInfos { + if !extractInfo.Optional { + pendingPaths[extractPath] = true + break + } + } + } + + // Store the hard links that we cannot extract when we first iterate over + // the tarball. + // + // This happens because the tarball only stores the contents once in the + // first entry and the rest of them point to the first one. Therefore, we + // cannot tell whether we need to extract the content until after we get to + // a hard link. In this case, we need a second pass. + pendingHardLinks := make(map[string][]pendingHardLink) + + // When creating a file we will iterate through its parent directories and + // create them with the permissions defined in the tarball. + // + // The assumption is that the tar entries of the parent directories appear + // before the entry for the file itself. This is the case for .deb files but + // not for all tarballs. + tarDirMode := make(map[string]fs.FileMode) + tarReader := tar.NewReader(dataReader) + for { + tarHeader, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + sourcePath, ok := sanitizeTarPath(tarHeader.Name) + if !ok { + continue + } + + sourceIsDir := sourcePath[len(sourcePath)-1] == '/' + if sourceIsDir { + tarDirMode[sourcePath] = tarHeader.FileInfo().Mode() + } + + // Find all globs and copies that require this source, and map them by + // their target paths on disk. + targetPaths := map[string][]ExtractInfo{} + for extractPath, extractInfos := range options.Extract { + if extractPath == "" { + continue + } + if strings.ContainsAny(extractPath, "*?") { + if strdist.GlobPath(extractPath, sourcePath) { + targetPaths[sourcePath] = append(targetPaths[sourcePath], extractInfos...) + delete(pendingPaths, extractPath) + } + } else if extractPath == sourcePath { + for _, extractInfo := range extractInfos { + targetPaths[extractInfo.Path] = append(targetPaths[extractInfo.Path], extractInfo) + } + delete(pendingPaths, extractPath) + } + } + if len(targetPaths) == 0 { + // Nothing to do. + continue + } + + var contentCache []byte + var contentIsCached = len(targetPaths) > 1 && !sourceIsDir + if contentIsCached { + // Read and cache the content so it may be reused. + // As an alternative, to avoid having an entire file in + // memory at once this logic might open the first file + // written and copy it every time. For now, the choice + // is speed over memory efficiency. + data, err := io.ReadAll(tarReader) + if err != nil { + return err + } + contentCache = data + } + + var pathReader io.Reader = tarReader + for targetPath, extractInfos := range targetPaths { + if contentIsCached { + pathReader = bytes.NewReader(contentCache) + } + mode := extractInfos[0].Mode + for _, extractInfo := range extractInfos { + if extractInfo.Mode != mode { + if mode < extractInfo.Mode { + mode, extractInfo.Mode = extractInfo.Mode, mode + } + return fmt.Errorf("path %s requested twice with diverging mode: 0%03o != 0%03o", targetPath, mode, extractInfo.Mode) + } + } + if mode != 0 { + tarHeader.Mode = int64(mode) + } + // Create the parent directories using the permissions from the tarball. + parents := parentDirs(targetPath) + for _, path := range parents { + if path == "/" { + continue + } + mode, ok := tarDirMode[path] + if !ok { + continue + } + delete(tarDirMode, path) + + createOptions := &fsutil.CreateOptions{ + Root: options.TargetDir, + Path: path, + Mode: mode, + MakeParents: true, + } + err := options.Create(nil, createOptions) + if err != nil { + return err + } + } + link := tarHeader.Linkname + if tarHeader.Typeflag == tar.TypeLink { + // A hard link requires the real path of the target file. + link = filepath.Join(options.TargetDir, link) + } + + // Create the entry itself. + createOptions := &fsutil.CreateOptions{ + Root: options.TargetDir, + Path: targetPath, + Mode: tarHeader.FileInfo().Mode(), + Data: pathReader, + Link: link, + MakeParents: true, + OverrideMode: true, + } + err := options.Create(extractInfos, createOptions) + if err != nil && os.IsNotExist(err) && tarHeader.Typeflag == tar.TypeLink { + // The hard link could not be created because the content + // was not extracted previously. Add this hard link entry + // to the pending list to extract later. + relLinkPath, ok := sanitizeTarPath(tarHeader.Linkname) + if !ok { + return fmt.Errorf("invalid link target %s", tarHeader.Linkname) + } + info := pendingHardLink{ + path: targetPath, + extractInfos: extractInfos, + } + pendingHardLinks[relLinkPath] = append(pendingHardLinks[relLinkPath], info) + } else if err != nil { + return err + } + } + } + + if len(pendingHardLinks) > 0 { + // Go over the tarball again to textract the pending hard links. + extractHardLinkOptions := &extractHardLinkOptions{ + ExtractOptions: options, + pendingLinks: pendingHardLinks, + } + _, err := pkgReader.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = extractHardLinks(pkgReader, extractHardLinkOptions) + if err != nil { + return err + } + } + + if len(pendingPaths) > 0 { + pendingList := make([]string, 0, len(pendingPaths)) + for pendingPath := range pendingPaths { + pendingList = append(pendingList, pendingPath) + } + if len(pendingList) == 1 { + return fmt.Errorf("no content at %s", pendingList[0]) + } else { + sort.Strings(pendingList) + return fmt.Errorf("no content at:\n- %s", strings.Join(pendingList, "\n- ")) + } + } + + return nil +} + +type pendingHardLink struct { + path string + extractInfos []ExtractInfo +} + +type extractHardLinkOptions struct { + *ExtractOptions + pendingLinks map[string][]pendingHardLink +} + +// extractHardLinks iterates through the tarball a second time to extract the +// hard links that were not extracted in the first pass. +func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { + dataReader, err := deb.DataReader(pkgReader) + if err != nil { + return err + } + defer dataReader.Close() + + tarReader := tar.NewReader(dataReader) + for { + tarHeader, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + sourcePath, ok := sanitizeTarPath(tarHeader.Name) + if !ok { + continue + } + + links := opts.pendingLinks[sourcePath] + if len(links) == 0 { + continue + } + + // For a target path, the first hard link will be created as a file with + // the content of the target path. If there are more pending hard links, + // the remaining ones will be created as hard links with the newly + // created file as their target. + absLink := filepath.Join(opts.TargetDir, links[0].path) + // Extract the content to the first hard link path. + createOptions := &fsutil.CreateOptions{ + Root: opts.TargetDir, + Path: links[0].path, + Mode: tarHeader.FileInfo().Mode(), + Data: tarReader, + } + err = opts.Create(links[0].extractInfos, createOptions) + if err != nil { + return err + } + + // Create the remaining hard links. + for _, link := range links[1:] { + createOptions := &fsutil.CreateOptions{ + Root: opts.TargetDir, + Path: link.path, + Mode: tarHeader.FileInfo().Mode(), + // Link to the first file extracted for the hard links. + Link: absLink, + } + err := opts.Create(link.extractInfos, createOptions) + if err != nil { + return err + } + } + delete(opts.pendingLinks, sourcePath) + } + + // If there are pending links, that means the link targets do not come from + // this package. + if len(opts.pendingLinks) > 0 { + var targets []string + for target := range opts.pendingLinks { + targets = append(targets, target) + } + sort.Strings(targets) + link := opts.pendingLinks[targets[0]][0] + return fmt.Errorf("cannot create hard link %s: no content at %s", link.path, targets[0]) + } + + return nil +} + +func parentDirs(path string) []string { + path = filepath.Clean(path) + parents := make([]string, strings.Count(path, "/")) + count := 0 + for i, c := range path { + if c == '/' { + parents[count] = path[:i+1] + count++ + } + } + return parents +} + +// sanitizeTarPath removes the leading "./" from the source path in the tarball, +// and verifies that the path is not empty. +func sanitizeTarPath(path string) (string, bool) { + if len(path) < 3 || path[0] != '.' || path[1] != '/' { + return "", false + } + return path[1:], true +} diff --git a/internal/deb/extract_test.go b/internal/tarball/extract_test.go similarity index 71% rename from internal/deb/extract_test.go rename to internal/tarball/extract_test.go index 1ec0b8a5f..fd5eb6147 100644 --- a/internal/deb/extract_test.go +++ b/internal/tarball/extract_test.go @@ -1,4 +1,4 @@ -package deb_test +package tarball_test import ( "bytes" @@ -10,16 +10,16 @@ import ( . "gopkg.in/check.v1" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) type extractTest struct { summary string pkgdata []byte - options deb.ExtractOptions - hackopt func(c *C, o *deb.ExtractOptions) + options tarball.ExtractOptions + hackopt func(c *C, o *tarball.ExtractOptions) result map[string]string // paths which the extractor did not create explicitly. notCreated []string @@ -29,28 +29,28 @@ type extractTest struct { var extractTests = []extractTest{{ summary: "Extract nothing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ + options: tarball.ExtractOptions{ Extract: nil, }, result: map[string]string{}, }, { summary: "Extract a few entries", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }}, - "/dir/other-file": []deb.ExtractInfo{{ + "/dir/other-file": []tarball.ExtractInfo{{ Path: "/dir/other-file", }}, - "/dir/several/levels/deep/file": []deb.ExtractInfo{{ + "/dir/several/levels/deep/file": []tarball.ExtractInfo{{ Path: "/dir/several/levels/deep/file", }}, - "/dir/nested/": []deb.ExtractInfo{{ + "/dir/nested/": []tarball.ExtractInfo{{ Path: "/dir/nested/", }}, - "/other-dir/": []deb.ExtractInfo{{ + "/other-dir/": []tarball.ExtractInfo{{ Path: "/other-dir/", }}, }, @@ -70,21 +70,21 @@ var extractTests = []extractTest{{ }, { summary: "Extract a few entries, nil Create closure", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }}, - "/dir/other-file": []deb.ExtractInfo{{ + "/dir/other-file": []tarball.ExtractInfo{{ Path: "/dir/other-file", }}, - "/dir/several/levels/deep/file": []deb.ExtractInfo{{ + "/dir/several/levels/deep/file": []tarball.ExtractInfo{{ Path: "/dir/several/levels/deep/file", }}, - "/dir/nested/": []deb.ExtractInfo{{ + "/dir/nested/": []tarball.ExtractInfo{{ Path: "/dir/nested/", }}, - "/other-dir/": []deb.ExtractInfo{{ + "/other-dir/": []tarball.ExtractInfo{{ Path: "/other-dir/", }}, }, @@ -100,19 +100,19 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/file": "file 0644 6bc26dff", "/other-dir/": "dir 0755", }, - hackopt: func(c *C, o *deb.ExtractOptions) { + hackopt: func(c *C, o *tarball.ExtractOptions) { o.Create = nil }, }, { summary: "Copy a couple of entries elsewhere", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/foo/file-copy", Mode: 0600, }}, - "/dir/several/levels/deep/": []deb.ExtractInfo{{ + "/dir/several/levels/deep/": []tarball.ExtractInfo{{ Path: "/foo/bar/dir-copy", Mode: 0700, }}, @@ -128,9 +128,9 @@ var extractTests = []extractTest{{ }, { summary: "Copy same file twice", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/foo/file-copy-1", }, { Path: "/dir/bar/file-copy-2", @@ -148,9 +148,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing a single dir level", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s*/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s*/": []tarball.ExtractInfo{{ Path: "/dir/s*/", }}, }, @@ -163,9 +163,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing for files with multiple levels at once", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s**": []tarball.ExtractInfo{{ Path: "/dir/s**", }}, }, @@ -181,12 +181,12 @@ var extractTests = []extractTest{{ }, { summary: "Globbing multiple paths", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/s**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/s**": []tarball.ExtractInfo{{ Path: "/dir/s**", }}, - "/dir/n*/": []deb.ExtractInfo{{ + "/dir/n*/": []tarball.ExtractInfo{{ Path: "/dir/n*/", }}, }, @@ -203,9 +203,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing must have matching source and target", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/foo/b**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/foo/b**": []tarball.ExtractInfo{{ Path: "/foo/g**", }}, }, @@ -214,9 +214,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing must also have a single target", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/foo/b**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/foo/b**": []tarball.ExtractInfo{{ Path: "/foo/b**", }, { Path: "/foo/g**", @@ -227,9 +227,9 @@ var extractTests = []extractTest{{ }, { summary: "Globbing cannot change modes", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/n**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/n**": []tarball.ExtractInfo{{ Path: "/dir/n**", Mode: 0777, }}, @@ -239,9 +239,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing file", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-file": []tarball.ExtractInfo{{ Path: "/missing-file", }}, }, @@ -250,9 +250,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing directory", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-dir/": []tarball.ExtractInfo{{ Path: "/missing-dir/", }}, }, @@ -261,9 +261,9 @@ var extractTests = []extractTest{{ }, { summary: "Missing glob", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-dir/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-dir/**": []tarball.ExtractInfo{{ Path: "/missing-dir/**", }}, }, @@ -272,12 +272,12 @@ var extractTests = []extractTest{{ }, { summary: "Missing multiple entries", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/missing-file": []tarball.ExtractInfo{{ Path: "missing-file", }}, - "/missing-dir/": []deb.ExtractInfo{{ + "/missing-dir/": []tarball.ExtractInfo{{ Path: "/missing-dir/", }}, }, @@ -286,16 +286,16 @@ var extractTests = []extractTest{{ }, { summary: "Optional entries may be missing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", }}, - "/dir/optional": []deb.ExtractInfo{{ + "/dir/optional": []tarball.ExtractInfo{{ Path: "/other-dir/foo", Optional: true, }}, - "/optional-dir/": []deb.ExtractInfo{{ + "/optional-dir/": []tarball.ExtractInfo{{ Path: "/foo/optional-dir/", Optional: true, }}, @@ -308,9 +308,9 @@ var extractTests = []extractTest{{ }, { summary: "Optional entries mixed in cannot be missing", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/missing-file": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/missing-file": []tarball.ExtractInfo{{ Path: "/dir/optional", Optional: true, }, { @@ -327,9 +327,9 @@ var extractTests = []extractTest{{ testutil.Dir(0766, "./日本/"), testutil.Reg(0644, "./日本/語", "whatever"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/日本/語": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/日本/語": []tarball.ExtractInfo{{ Path: "/日本/語", }}, }, @@ -342,13 +342,13 @@ var extractTests = []extractTest{{ }, { summary: "Entries for same destination must have the same mode", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", Mode: 0777, }}, - "/d**": []deb.ExtractInfo{{ + "/d**": []tarball.ExtractInfo{{ Path: "/d**", }}, }, @@ -361,9 +361,9 @@ var extractTests = []extractTest{{ testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -380,9 +380,9 @@ var extractTests = []extractTest{{ testutil.Reg(0644, "./file", "text for file"), testutil.Hrd(0644, "./hardlink", "./file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/hardlink": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/hardlink": []tarball.ExtractInfo{{ Path: "/hardlink", }}, }, @@ -397,9 +397,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "./non-existing-target"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/hardlink": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/hardlink": []tarball.ExtractInfo{{ Path: "/hardlink", }}, }, @@ -412,9 +412,9 @@ var extractTests = []extractTest{{ testutil.Hrd(0644, "./hardlink1", "./non-existing-target"), testutil.Hrd(0644, "./hardlink2", "./non-existing-target"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -427,9 +427,9 @@ var extractTests = []extractTest{{ testutil.Lnk(0644, "./symlink", "./file"), testutil.Hrd(0644, "./hardlink", "./symlink"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -442,15 +442,15 @@ var extractTests = []extractTest{{ }, { summary: "Explicit extraction overrides existing file", pkgdata: testutil.PackageData["test-package"], - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", Mode: 0777, }}, }, }, - hackopt: func(c *C, o *deb.ExtractOptions) { + hackopt: func(c *C, o *tarball.ExtractOptions) { err := os.Mkdir(path.Join(o.TargetDir, "/dir"), 0666) c.Assert(err, IsNil) }, @@ -464,9 +464,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Hrd(0644, "./hardlink", "/etc/group"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -478,9 +478,9 @@ var extractTests = []extractTest{{ testutil.Dir(0755, "./"), testutil.Reg(0644, "./../file", "hijacking system file"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/**": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/**": []tarball.ExtractInfo{{ Path: "/**", }}, }, @@ -497,7 +497,7 @@ func (s *S) TestExtract(c *C) { options.Package = "test-package" options.TargetDir = dir createdPaths := make(map[string]bool) - options.Create = func(_ []deb.ExtractInfo, o *fsutil.CreateOptions) error { + options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) if o.Mode.IsDir() { relPath = relPath + "/" @@ -511,7 +511,7 @@ func (s *S) TestExtract(c *C) { test.hackopt(c, &options) } - err := deb.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -539,8 +539,8 @@ func (s *S) TestExtract(c *C) { var extractCreateCallbackTests = []struct { summary string pkgdata []byte - options deb.ExtractOptions - calls map[string][]deb.ExtractInfo + options tarball.ExtractOptions + calls map[string][]tarball.ExtractInfo }{{ summary: "Create is called with the set of ExtractInfo(s) that match the file", pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ @@ -548,50 +548,50 @@ var extractCreateCallbackTests = []struct { testutil.Dir(0766, "./dir/"), testutil.Reg(0644, "./dir/file", "whatever"), }), - options: deb.ExtractOptions{ - Extract: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{{ + options: tarball.ExtractOptions{ + Extract: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{{ Path: "/dir/", }}, - "/d**": []deb.ExtractInfo{{ + "/d**": []tarball.ExtractInfo{{ Path: "/d**", }}, - "/d?r/": []deb.ExtractInfo{{ + "/d?r/": []tarball.ExtractInfo{{ Path: "/d?r/", }}, - "/dir/file": []deb.ExtractInfo{{ + "/dir/file": []tarball.ExtractInfo{{ Path: "/dir/file", }, { Path: "/dir/file-cpy", }}, - "/foo/": []deb.ExtractInfo{{ + "/foo/": []tarball.ExtractInfo{{ Path: "/foo/", Optional: true, }}, }, }, - calls: map[string][]deb.ExtractInfo{ - "/dir/": []deb.ExtractInfo{ - deb.ExtractInfo{ + calls: map[string][]tarball.ExtractInfo{ + "/dir/": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d**", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d?r/", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/", }, }, - "/dir/file": []deb.ExtractInfo{ - deb.ExtractInfo{ + "/dir/file": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/d**", }, - deb.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/file", }, }, - "/dir/file-cpy": []deb.ExtractInfo{ - deb.ExtractInfo{ + "/dir/file-cpy": []tarball.ExtractInfo{ + tarball.ExtractInfo{ Path: "/dir/file-cpy", }, }, @@ -605,8 +605,8 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir - createExtractInfos := map[string][]deb.ExtractInfo{} - options.Create = func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + createExtractInfos := map[string][]tarball.ExtractInfo{} + options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { // Creating implicit parent directories, we don't care about those. return nil @@ -622,7 +622,7 @@ func (s *S) TestExtractCreateCallback(c *C) { return nil } - err := deb.Extract(bytes.NewReader(test.pkgdata), &options) + err := tarball.Extract(bytes.NewReader(test.pkgdata), &options) c.Assert(err, IsNil) c.Assert(createExtractInfos, DeepEquals, test.calls) diff --git a/internal/tarball/log.go b/internal/tarball/log.go new file mode 100644 index 000000000..903bf279d --- /dev/null +++ b/internal/tarball/log.go @@ -0,0 +1,53 @@ +package tarball + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...any) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/tarball/suite_test.go b/internal/tarball/suite_test.go new file mode 100644 index 000000000..e99ce3e92 --- /dev/null +++ b/internal/tarball/suite_test.go @@ -0,0 +1,20 @@ +package tarball_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/tarball" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + tarball.SetDebug(true) + tarball.SetLogger(c) +} From 9512354b932ff40ff554f28cfe154d620465b83d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 30 Jun 2026 09:15:13 +0200 Subject: [PATCH 61/61] feat: extract bins --- internal/archive/archive_test.go | 2 ++ internal/slicer/slicer.go | 16 ++++++++++---- internal/slicer/slicer_test.go | 33 +++++++++++++++++++++++++--- internal/store/store.go | 10 ++++----- internal/tarball/extract.go | 13 ++++++++--- internal/tarball/extract_test.go | 17 +++++++++++++++ internal/tarball/xz.go | 15 +++++++++++++ internal/testutil/pkgdata.go | 37 ++++++++++++++++++++++++++++++++ 8 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 internal/tarball/xz.go diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index c21e48161..30d65a481 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/archive/testarchive" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" @@ -859,6 +860,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { err = tarball.Extract(pkg, &tarball.ExtractOptions{ Package: test.pkg, TargetDir: extractDir, + OpenData: deb.DataReader, Extract: map[string][]tarball.ExtractInfo{ fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index b62266795..b7432e083 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -16,6 +16,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" @@ -273,15 +274,22 @@ func Run(options *RunOptions) error { continue } src := pkgSources[slice.Package] - // Store packages are distributed as plain tarballs, whose extraction - // is not yet implemented. Fail until the format support is in place. - if src.kind != sourceArchive { - return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName) + var openData func(io.ReadSeeker) (io.ReadCloser, error) + if src.kind == sourceStore { + switch store.StoreKind(src.store.Options().Kind) { + case store.StoreKindBin: + openData = tarball.XZDataReader + default: + return fmt.Errorf("cannot extract from store of kind %q: unsupported artefact format", src.store.Options().Kind) + } + } else { + openData = deb.DataReader } err := tarball.Extract(reader, &tarball.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, + OpenData: openData, Create: create, }) reader.Close() diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 806bcaa84..f13eb08ab 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1980,7 +1980,7 @@ var slicerTests = []slicerTest{{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, }, { - summary: "Store package fails as it is not yet supported", + summary: "Store package extracts, manifest reports missing package", slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, arch: "amd64", pkgs: []*testutil.TestPackage{{ @@ -1989,7 +1989,11 @@ var slicerTests = []slicerTest{{ }, { Name: "store-pkg", Store: "bin", - Data: testutil.PackageData["test-package"], + Data: testutil.MustMakeBin([]testutil.TarEntry{ + testutil.Dir(0o755, "./"), + testutil.Dir(0o755, "./dir/"), + testutil.Reg(0o644, "./dir/store-file", "store content"), + }), }}, release: map[string]string{ "chisel.yaml": testutil.DefaultChiselYamlWithStores, @@ -2010,7 +2014,29 @@ var slicerTests = []slicerTest{{ /dir/store-file: `, }, - error: `cannot extract package "store-pkg" from store: store packages are not yet supported`, + error: `internal error: invalid manifest: slice bin-store-pkg_myslice refers to missing package "bin-store-pkg"`, +}, { + summary: "Store package with invalid (non-xz) data fails extraction", + slices: []setup.SliceKey{{"bin-store-pkg", "myslice"}}, + arch: "amd64", + pkgs: []*testutil.TestPackage{{ + Name: "store-pkg", + Store: "bin", + Data: []byte("this is not an xz archive"), + }}, + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: 3.1 + slices: + myslice: + contents: + /dir/store-file: + `, + }, + error: `cannot extract from package "bin-store-pkg": xz: invalid header magic bytes`, }} func (s *S) TestRun(c *C) { @@ -2150,6 +2176,7 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { stores[name] = &testutil.TestStore{ Packages: pkgs, Opts: store.Options{ + Kind: relStore.Kind, Version: relStore.Version, }, } diff --git a/internal/store/store.go b/internal/store/store.go index 9aacb543b..c0bf61ec7 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -37,9 +37,9 @@ type Options struct { Version string } -type storeKind string +type StoreKind string -const storeKindBin storeKind = "bin" +const StoreKindBin StoreKind = "bin" const defaultRisk = "stable" @@ -95,8 +95,8 @@ func Open(options *Options) (Store, error) { return nil, err } - switch storeKind(options.Kind) { - case storeKindBin: + switch StoreKind(options.Kind) { + case StoreKindBin: apiURL := binAPIBase downloadHost := binDownloadHost if os.Getenv(binStagingEnvVar) != "" { @@ -177,7 +177,7 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro reqBody, err := json.Marshal(resolveRequest{ Packages: []resolvePackage{{ InstanceKey: name, - Namespace: string(storeKindBin), + Namespace: string(StoreKindBin), Name: name, Channel: track + "/" + risk, Platform: binPlatform{Architecture: s.options.Arch}, diff --git a/internal/tarball/extract.go b/internal/tarball/extract.go index 40db2cbac..ce1fda5b2 100644 --- a/internal/tarball/extract.go +++ b/internal/tarball/extract.go @@ -12,7 +12,6 @@ import ( "strings" "syscall" - "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/strdist" ) @@ -21,6 +20,11 @@ type ExtractOptions struct { Package string TargetDir string Extract map[string][]ExtractInfo + // OpenData opens the uncompressed tar data stream of the package. It + // abstracts over the package format (e.g. a deb archive opener or a plain + // tarball opener), allowing Extract to operate on any package whose data + // payload is a tar stream. + OpenData func(io.ReadSeeker) (io.ReadCloser, error) // Create can optionally be set to control the creation of extracted entries. // extractInfos is set to the matching entries in Extract, and is nil in cases where // the created entry is implicit and unlisted (for example, parent directories). @@ -35,6 +39,9 @@ type ExtractInfo struct { } func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { + if options.OpenData == nil { + return nil, fmt.Errorf("internal error: ExtractOptions.OpenData is unset") + } for extractPath, extractInfos := range options.Extract { isGlob := strings.ContainsAny(extractPath, "*?") if isGlob { @@ -83,7 +90,7 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) { } func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error { - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := options.OpenData(pkgReader) if err != nil { return err } @@ -300,7 +307,7 @@ type extractHardLinkOptions struct { // extractHardLinks iterates through the tarball a second time to extract the // hard links that were not extracted in the first pass. func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error { - dataReader, err := deb.DataReader(pkgReader) + dataReader, err := opts.OpenData(pkgReader) if err != nil { return err } diff --git a/internal/tarball/extract_test.go b/internal/tarball/extract_test.go index fd5eb6147..f855d9bc4 100644 --- a/internal/tarball/extract_test.go +++ b/internal/tarball/extract_test.go @@ -10,6 +10,7 @@ import ( . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" @@ -496,6 +497,8 @@ func (s *S) TestExtract(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader createdPaths := make(map[string]bool) options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) @@ -605,6 +608,8 @@ func (s *S) TestExtractCreateCallback(c *C) { options := test.options options.Package = "test-package" options.TargetDir = dir + // The test fixtures are .deb archives, so use the deb data opener. + options.OpenData = deb.DataReader createExtractInfos := map[string][]tarball.ExtractInfo{} options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error { if extractInfos == nil { @@ -628,3 +633,15 @@ func (s *S) TestExtractCreateCallback(c *C) { c.Assert(createExtractInfos, DeepEquals, test.calls) } } + +func (s *S) TestExtractMissingOpenData(c *C) { + options := tarball.ExtractOptions{ + Package: "test-package", + TargetDir: c.MkDir(), + Extract: map[string][]tarball.ExtractInfo{ + "/dir/file": {{Path: "/dir/file"}}, + }, + } + err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), &options) + c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: ExtractOptions.OpenData is unset`) +} diff --git a/internal/tarball/xz.go b/internal/tarball/xz.go new file mode 100644 index 000000000..700215a09 --- /dev/null +++ b/internal/tarball/xz.go @@ -0,0 +1,15 @@ +package tarball + +import ( + "io" + + "github.com/ulikunitz/xz" +) + +func XZDataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) { + xzReader, err := xz.NewReader(pkgReader) + if err != nil { + return nil, err + } + return io.NopCloser(xzReader), nil +} diff --git a/internal/testutil/pkgdata.go b/internal/testutil/pkgdata.go index f901873d3..8fb8fb8b2 100644 --- a/internal/testutil/pkgdata.go +++ b/internal/testutil/pkgdata.go @@ -8,6 +8,7 @@ import ( "github.com/blakesmith/ar" "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" ) var PackageData = map[string][]byte{} @@ -160,6 +161,42 @@ func MustMakeDeb(entries []TarEntry) []byte { return data } +// compressBytesXz compresses the input using XZ, the compression format used by +// store (bin) packages. +func compressBytesXz(input []byte) ([]byte, error) { + var buf bytes.Buffer + writer, err := xz.NewWriter(&buf) + if err != nil { + return nil, err + } + if _, err = writer.Write(input); err != nil { + return nil, err + } + if err = writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MakeBin builds a store (bin) package from the given tar entries: an +// XZ-compressed plain tarball. Unlike MakeDeb, there is no ar container. +func MakeBin(entries []TarEntry) ([]byte, error) { + tarData, err := makeTar(entries) + if err != nil { + return nil, err + } + return compressBytesXz(tarData) +} + +// MustMakeBin is the panicking variant of MakeBin. +func MustMakeBin(entries []TarEntry) []byte { + data, err := MakeBin(entries) + if err != nil { + panic(err) + } + return data +} + // Reg is a shortcut for creating a regular file TarEntry structure (with // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file". func Reg(mode int64, path, content string) TarEntry {