feat: store package support

cmd/chisel/cmd_cut.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"slices"
 	"time"
@@ -11,6 +12,7 @@ import (
 	"github.com/canonical/chisel/internal/cache"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/internal/slicer"
+	"github.com/canonical/chisel/internal/store"
 )
 
 var shortCutHelp = "Cut a tree with selected slices"
@@ -121,9 +123,29 @@ func (cmd *cmdCut) Execute(args []string) error {
 		}
 	}
 
+	stores := make(map[string]store.Store)
+	for storeName, storeInfo := range release.Stores {
+		openStore, err := store.Open(&store.Options{
+			Arch:     cmd.Arch,
+			CacheDir: cache.DefaultDir("chisel"),
+			Kind:     storeInfo.Kind,
+			Version:  storeInfo.Version,
+		})
+		if err != nil {
+			var unknownStoreKindError *store.UnknownStoreKindError
+			if errors.As(err, &unknownStoreKindError) {
+				logf("Store %q ignored: %v", storeName, err)
+				continue
+			}
+			return err
+		}
+		stores[storeName] = openStore
+	}
+
 	err = slicer.Run(&slicer.RunOptions{
 		Selection: selection,
 		Archives:  archives,
+		Stores:    stores,
 		TargetDir: cmd.RootDir,
 	})
 	return err

cmd/chisel/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/internal/slicer"
+	"github.com/canonical/chisel/internal/store"
 	"github.com/canonical/chisel/internal/tarball"
 	//"github.com/canonical/chisel/internal/logger"
 )
@@ -328,6 +329,7 @@ func run() error {
 	deb.SetLogger(log.Default())
 	setup.SetLogger(log.Default())
 	slicer.SetLogger(log.Default())
+	store.SetLogger(log.Default())
 	tarball.SetLogger(log.Default())
 	SetLogger(log.Default())
 

internal/archive/archive.go

@@ -26,10 +26,12 @@ type Archive interface {
 }
 
 type PackageInfo struct {
-	Name    string
-	Version string
-	Arch    string
-	SHA256  string
+	Name     string
+	Version  string
+	Arch     string
+	SHA256   string
+	Revision int
+	SHA384   string
 }
 
 type Options struct {

internal/slicer/slicer.go

@@ -20,14 +20,16 @@ import (
 	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/scripts"
 	"github.com/canonical/chisel/internal/setup"
+	"github.com/canonical/chisel/internal/store"
 	"github.com/canonical/chisel/internal/tarball"
 )
 
-const manifestMode fs.FileMode = 0644
+const manifestMode fs.FileMode = 0o644
 
 type RunOptions struct {
 	Selection *setup.Selection
 	Archives  map[string]archive.Archive
+	Stores    map[string]store.Store
 	TargetDir string
 }
 
@@ -99,7 +101,7 @@ func Run(options *RunOptions) error {
 		targetDir = filepath.Join(dir, targetDir)
 	}
 
-	pkgSources, err := resolvePkgSources(options.Archives, options.Selection)
+	pkgSources, err := resolvePkgSources(options.Archives, options.Stores, options.Selection)
 	if err != nil {
 		return err
 	}
@@ -248,6 +250,12 @@ func Run(options *RunOptions) error {
 		if reader == nil {
 			continue
 		}
+		// src := pkgSources[slice.Package]
+		// Store packages are distributed as plain tarballs, whose extraction
+		// is not yet implemented. Fail until the format support is in place.
+		// if src.kind != sourceArchive {
+		// 	return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName)
+		// }
 		err := tarball.Extract(reader, &tarball.ExtractOptions{
 			Package:   slice.Package,
 			Extract:   extract[slice.Package],
@@ -360,8 +368,7 @@ func Run(options *RunOptions) error {
 	return generateManifests(targetDir, options.Selection, report, pkgInfos)
 }
 
-func generateManifests(targetDir string, selection *setup.Selection,
-	report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error {
+func generateManifests(targetDir string, selection *setup.Selection, report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error {
 	manifestSlices := manifestutil.FindPaths(selection.Slices)
 	if len(manifestSlices) == 0 {
 		// Nothing to do.
@@ -502,10 +509,9 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent
 // resolvePkgSources determines the source for each package in the selection.
 // For archive packages it selects the highest priority archive containing the
 // package unless a particular archive is pinned within the slice definition
-// file. For store packages it records a fetch function that returns an error
-// until store support is implemented. It returns a map of pkgSource indexed by
-// package names.
-func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Selection) (map[string]pkgSource, error) {
+// file. For store packages it records the opened store handle. It returns a map
+// of pkgSourceInfo indexed by package names.
+func resolvePkgSources(archives map[string]archive.Archive, stores map[string]store.Store, selection *setup.Selection) (map[string]pkgSource, error) {
 	sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives))
 	for _, archive := range selection.Release.Archives {
 		if archive.Priority < 0 {
@@ -526,10 +532,19 @@ func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Sel
 		}
 		pkg := selection.Release.Packages[s.Package]
 		if pkg.Store != "" {
+			storeHandle := stores[pkg.Store]
+			if storeHandle == nil {
+				return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store)
+			}
+			// The store channel track is "<default-track>-<store version>",
+			// e.g. "3.1-26.10". The version pins the release series.
+			// Risk is left unspecified for now; the store applies its default.
+			// In the future the risk will optionnaly come from the CLI.
+			track := pkg.DefaultTrack + "-" + storeHandle.Options().Version
 			pkgSources[pkg.Name] = pkgSource{
-				// TODO: set the arch when implementing fetching from the store.
+				arch: storeHandle.Options().Arch,
 				fetch: func() (io.ReadSeekCloser, *archive.PackageInfo, error) {
-					return nil, nil, fmt.Errorf("cannot fetch package %q from store %q: not implemented", pkg.Name, pkg.Store)
+					return storeHandle.Fetch(pkg.RealName, track, "")
 				},
 			}
 			continue

internal/slicer/slicer_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/internal/slicer"
+	"github.com/canonical/chisel/internal/store"
 	"github.com/canonical/chisel/internal/testutil"
 	"github.com/canonical/chisel/public/manifest"
 )
@@ -1982,6 +1983,14 @@ var slicerTests = []slicerTest{{
 	summary: "Store package fetching not yet implemented",
 	slices:  []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}},
 	arch:    "amd64",
+	pkgs: []*testutil.TestPackage{{
+		Name: "test-package",
+		Data: testutil.PackageData["test-package"],
+	}, {
+		Name:  "store-pkg",
+		Store: "bin",
+		Data:  testutil.PackageData["test-package"],
+	}},
 	release: map[string]string{
 		"chisel.yaml": testutil.DefaultChiselYamlWithStores,
 		"slices/mydir/test-package.yaml": `
@@ -2001,7 +2010,7 @@ var slicerTests = []slicerTest{{
 						/dir/store-file:
 		`,
 	},
-	error: `cannot fetch package "bin-store-pkg" from store "bin": not implemented`,
+	error: `cannot extract package "store-pkg" from store: store packages are not yet supported`,
 }}
 
 func (s *S) TestRun(c *C) {
@@ -2109,6 +2118,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) {
 			for name, setupArchive := range release.Archives {
 				pkgs := make(map[string]*testutil.TestPackage)
 				for _, pkg := range test.pkgs {
+					if pkg.Store != "" {
+						continue
+					}
 					if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) {
 						pkgs[pkg.Name] = pkg
 					}
@@ -2127,9 +2139,26 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) {
 				archives[name] = archive
 			}
 
+			stores := map[string]store.Store{}
+			for name, relStore := range release.Stores {
+				pkgs := make(map[string]*testutil.TestPackage)
+				for _, pkg := range test.pkgs {
+					if pkg.Store == name {
+						pkgs[pkg.Name] = pkg
+					}
+				}
+				stores[name] = &testutil.TestStore{
+					Packages: pkgs,
+					Opts: store.Options{
+						Version: relStore.Version,
+					},
+				}
+			}
+
 			options := slicer.RunOptions{
 				Selection: selection,
 				Archives:  archives,
+				Stores:    stores,
 				TargetDir: c.MkDir(),
 			}
 			if test.hackopt != nil {

internal/store/export_test.go

@@ -0,0 +1,19 @@
+package store
+
+import "net/http"
+
+var (
+	ValidateDownloadURL = validateDownloadURL
+	BinStagingEnvVar    = binStagingEnvVar
+)
+
+func FakeDo(do func(req *http.Request) (*http.Response, error)) (restore func()) {
+	_httpDo := httpDo
+	_bulkDo := bulkDo
+	httpDo = do
+	bulkDo = do
+	return func() {
+		httpDo = _httpDo
+		bulkDo = _bulkDo
+	}
+}

internal/store/log.go

@@ -0,0 +1,53 @@
+package store
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Avoid importing the log type information unnecessarily.  There's a small cost
+// associated with using an interface rather than the type.  Depending on how
+// often the logger is plugged in, it would be worth using the type instead.
+type log_Logger interface {
+	Output(calldepth int, s string) error
+}
+
+var globalLoggerLock sync.Mutex
+var globalLogger log_Logger
+var globalDebug bool
+
+// Specify the *log.Logger object where log messages should be sent to.
+func SetLogger(logger log_Logger) {
+	globalLoggerLock.Lock()
+	globalLogger = logger
+	globalLoggerLock.Unlock()
+}
+
+// Enable the delivery of debug messages to the logger.  Only meaningful
+// if a logger is also set.
+func SetDebug(debug bool) {
+	globalLoggerLock.Lock()
+	globalDebug = debug
+	globalLoggerLock.Unlock()
+}
+
+// logf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf.
+func logf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}
+
+// debugf sends to the logger registered via SetLogger the string resulting
+// from running format and args through Sprintf, but only if debugging was
+// enabled via SetDebug.
+func debugf(format string, args ...any) {
+	globalLoggerLock.Lock()
+	defer globalLoggerLock.Unlock()
+	if globalDebug && globalLogger != nil {
+		globalLogger.Output(2, fmt.Sprintf(format, args...))
+	}
+}