From f67ce660fb4c8758b3622fab424770f1d753c413 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 27 Aug 2026 17:20:18 +0100 Subject: [PATCH] Add zstd and conda archive inspection --- README.md | 2 +- cmd/brief/inspect.go | 27 +++++-- cmd/brief/inspect_test.go | 146 ++++++++++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 6 +- 5 files changed, 175 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a01cf2f..2a4ba60 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Use `--verbose` to include homepage, docs, and repo links for each detected tool ## Inspect -`brief inspect` reports the format, architecture, linked libraries, and build metadata of native objects in binaries and package archives. It accepts ELF, Mach-O, PE, zip, tar, gzip, bzip2, and xz inputs, including wheels, gems, and JARs. +`brief inspect` reports the format, architecture, linked libraries, and build metadata of native objects in binaries and package archives. It accepts ELF, Mach-O, PE, zip, tar, gzip, bzip2, xz, and zstd inputs, including wheels, gems, JARs, and conda packages. ``` brief inspect build/tool diff --git a/cmd/brief/inspect.go b/cmd/brief/inspect.go index a040c5c..79aea86 100644 --- a/cmd/brief/inspect.go +++ b/cmd/brief/inspect.go @@ -23,6 +23,7 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/magic" + "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" "github.com/git-pkgs/brief" @@ -243,6 +244,17 @@ func preflightArtifactArchive(f *os.File, filePath, format string, maxEntries in return preflightTAR(gz, maxEntries, maxBytes) case magic.FormatBZIP2: return preflightTAR(bzip2.NewReader(newArchiveInputLimitReader(f, maxArchiveInputBytes)), maxEntries, maxBytes) + case magic.FormatZstd: + zstdReader, err := zstd.NewReader( + newArchiveInputLimitReader(f, maxArchiveInputBytes), + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderMaxMemory(uint64(maxArchiveExtractedBytes)), + ) + if err != nil { + return fmt.Errorf("opening zstd: %w", err) + } + defer zstdReader.Close() + return preflightTAR(zstdReader, maxEntries, maxBytes) case magic.FormatXZ: if err := preflightXZ(newArchiveInputLimitReader(f, maxArchiveInputBytes), maxXZDictionaryBytes); err != nil { return err @@ -492,15 +504,20 @@ func readZIP64DirectoryEnd(r io.ReaderAt, end zipDirectoryEnd) (zipDirectoryEnd, } // openArtifactArchive uses the sniffed physical format instead of trusting a -// possibly misleading filename extension. Gems need their filename for the -// archives package to unwrap data.tar.gz; malformed gems fall back to plain -// tar inspection. +// possibly misleading filename extension. Gems and conda packages need their +// filename to select the nested archive reader. Invalid nested structures fall +// back to inspection using the physical format. func openArtifactArchive(f *os.File, path, format string, caseInsensitive bool) (*archiveLimitReader, error) { var r archives.Reader var err error - if format == magic.FormatTAR && strings.EqualFold(filepath.Ext(path), ".gem") { + ext := filepath.Ext(path) + if format == magic.FormatTAR && strings.EqualFold(ext, ".gem") || + format == magic.FormatZIP && strings.EqualFold(ext, ".conda") { r, err = archives.Open(filepath.Base(path), newArchiveInputLimitReader(f, maxArchiveInputBytes)) if err != nil { + if errors.Is(err, archives.ErrEntryLimit) || errors.Is(err, archives.ErrDecompressLimit) { + return nil, fmt.Errorf("%w: %w", errArchiveLimit, err) + } r = nil if _, seekErr := f.Seek(0, io.SeekStart); seekErr != nil { return nil, seekErr @@ -847,7 +864,7 @@ func isNativeObject(format string) bool { func isArchive(format string) bool { switch format { case magic.FormatZIP, magic.FormatTAR, magic.FormatGZIP, - magic.FormatBZIP2, magic.FormatXZ: + magic.FormatBZIP2, magic.FormatXZ, magic.FormatZstd: return true } return false diff --git a/cmd/brief/inspect_test.go b/cmd/brief/inspect_test.go index 349dda9..9b0a8f9 100644 --- a/cmd/brief/inspect_test.go +++ b/cmd/brief/inspect_test.go @@ -22,6 +22,7 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/magic" + "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" "github.com/git-pkgs/brief" @@ -29,6 +30,8 @@ import ( "github.com/git-pkgs/brief/report" ) +const inspectCondaHelperEnv = "BRIEF_INSPECT_CONDA_HELPER" + func TestInspectPathBareObject(t *testing.T) { bin := buildHello(t, runtime.GOOS, runtime.GOARCH) @@ -143,6 +146,64 @@ func TestInspectPathArchive(t *testing.T) { } } +func TestInspectPathTarZstd(t *testing.T) { + bin := buildHello(t, runtime.GOOS, runtime.GOARCH) + archive := writeTarZstd(t, map[string]string{ + "bin/hello": bin, + "README.txt": "plain text\n", + }) + + if !shouldAutoInspect(archive) { + t.Fatal("zstd tar should auto-inspect") + } + art, err := inspectPath(archive) + if err != nil { + t.Fatal(err) + } + if art.Format != magic.FormatZstd { + t.Fatalf("Format = %q, want %q", art.Format, magic.FormatZstd) + } + if art.Entries != 2 { + t.Fatalf("Entries = %d, want 2", art.Entries) + } + if len(art.NativeObjects) != 1 || art.NativeObjects[0].Path != "bin/hello" { + t.Fatalf("NativeObjects = %+v, want bin/hello", art.NativeObjects) + } +} + +func TestInspectCondaCommand(t *testing.T) { + if archive := os.Getenv(inspectCondaHelperEnv); archive != "" { + cmdInspect([]string{"-json", archive}) + os.Exit(0) + } + + bin := buildHello(t, runtime.GOOS, runtime.GOARCH) + archive := writeConda(t, + map[string]string{"bin/hello": bin}, + map[string]string{"info/index.json": `{"name":"hello"}`}, + ) + + cmd := exec.Command(os.Args[0], "-test.run=^TestInspectCondaCommand$") + cmd.Env = append(os.Environ(), inspectCondaHelperEnv+"="+archive) + out, err := cmd.Output() + if err != nil { + t.Fatalf("inspect command failed: %v", err) + } + var art brief.Artifact + if err := json.Unmarshal(out, &art); err != nil { + t.Fatalf("parsing inspect output: %v\n%s", err, out) + } + if art.Format != magic.FormatZIP { + t.Fatalf("Format = %q, want %q", art.Format, magic.FormatZIP) + } + if art.Entries != 2 { + t.Fatalf("Entries = %d, want 2 merged conda entries", art.Entries) + } + if len(art.NativeObjects) != 1 || art.NativeObjects[0].Path != "bin/hello" { + t.Fatalf("NativeObjects = %+v, want bin/hello", art.NativeObjects) + } +} + func TestInspectPathUsesDetectedArchiveFormat(t *testing.T) { archive := writeZip(t, map[string]string{ "README.txt": "zip content\n", @@ -290,6 +351,18 @@ func TestArchivePreflightLimits(t *testing.T) { } }) + t.Run("zstd tar entry preflight", func(t *testing.T) { + archive := writeTarZstd(t, map[string]string{"a": "one", "b": "two"}) + f, err := os.Open(archive) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + if err := preflightArtifactArchive(f, archive, magic.FormatZstd, 1, 10); !errors.Is(err, errArchiveLimit) { + t.Fatalf("preflightArtifactArchive error = %v, want errArchiveLimit", err) + } + }) + t.Run("gem nested tar preflight", func(t *testing.T) { archive := writeGem(t, map[string]string{"a": "one", "b": "two"}) f, err := os.Open(archive) @@ -651,6 +724,79 @@ func writeTar(tb testing.TB, entries map[string]string) string { return path } +func writeTarZstd(tb testing.TB, entries map[string]string) string { + tb.Helper() + path := filepath.Join(tb.TempDir(), "fixture.tar.zst") + if err := os.WriteFile(path, tarZstdData(tb, entries), 0o644); err != nil { + tb.Fatal(err) + } + return path +} + +func tarZstdData(tb testing.TB, entries map[string]string) []byte { + tb.Helper() + var data bytes.Buffer + zw, err := zstd.NewWriter(&data) + if err != nil { + tb.Fatal(err) + } + tw := tar.NewWriter(zw) + for name, val := range entries { + content := []byte(val) + if fileData, err := os.ReadFile(val); err == nil { + content = fileData + } + header := &tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))} + if err := tw.WriteHeader(header); err != nil { + tb.Fatal(err) + } + if _, err := tw.Write(content); err != nil { + tb.Fatal(err) + } + } + if err := tw.Close(); err != nil { + tb.Fatal(err) + } + if err := zw.Close(); err != nil { + tb.Fatal(err) + } + return data.Bytes() +} + +func writeConda(tb testing.TB, packageEntries, infoEntries map[string]string) string { + tb.Helper() + path := filepath.Join(tb.TempDir(), "fixture.conda") + f, err := os.Create(path) + if err != nil { + tb.Fatal(err) + } + zw := zip.NewWriter(f) + members := []struct { + name string + data []byte + }{ + {name: "metadata.json", data: []byte(`{"conda_pkg_format_version":2}`)}, + {name: "pkg-hello-1.0-0.tar.zst", data: tarZstdData(tb, packageEntries)}, + {name: "info-hello-1.0-0.tar.zst", data: tarZstdData(tb, infoEntries)}, + } + for _, member := range members { + w, err := zw.CreateHeader(&zip.FileHeader{Name: member.name, Method: zip.Store}) + if err != nil { + tb.Fatal(err) + } + if _, err := w.Write(member.data); err != nil { + tb.Fatal(err) + } + } + if err := zw.Close(); err != nil { + tb.Fatal(err) + } + if err := f.Close(); err != nil { + tb.Fatal(err) + } + return path +} + func writeGem(tb testing.TB, entries map[string]string) string { tb.Helper() diff --git a/go.mod b/go.mod index 66bdb49..62730fc 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.26.6 require ( github.com/BurntSushi/toml v1.6.0 - github.com/git-pkgs/archives v0.5.1 + github.com/git-pkgs/archives v0.6.0 github.com/git-pkgs/clone v0.7.1 github.com/git-pkgs/enrichment v0.7.0 github.com/git-pkgs/forge v0.9.0 @@ -17,6 +17,7 @@ require ( github.com/git-pkgs/purl v0.1.17 github.com/git-pkgs/registries v0.8.0 github.com/git-pkgs/spdx v0.3.1 + github.com/klauspost/compress v1.19.2 github.com/ulikunitz/xz v0.5.16 golang.org/x/term v0.45.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index b9c5f62..20af7fc 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= -github.com/git-pkgs/archives v0.5.1 h1:qwu/vsoerQZF1iysRtfcxpy1KIUSJJSpXJ5JNxzNoQw= -github.com/git-pkgs/archives v0.5.1/go.mod h1:AKpkxnts49R9uAt1mL2ULYcHrmYujCDVu24IsFvW9so= +github.com/git-pkgs/archives v0.6.0 h1:AIVuWH9Wra2YrgRzSTECxVjTv3jKzka7woVc8JAnPD4= +github.com/git-pkgs/archives v0.6.0/go.mod h1:h7jYt1y3XH04D1W0NjX1BWbSzsWd2tqV0K1viq+7y0U= github.com/git-pkgs/clone v0.7.1 h1:ai1sI2EypbgJemWEwSIW7v+SndnHHusGxOKD4+bLT2Y= github.com/git-pkgs/clone v0.7.1/go.mod h1:Dk6k+HGvI8+OR27ijxGtCcHhSIyhGBWShE6S6ie5v0s= github.com/git-pkgs/enrichment v0.7.0 h1:LfIzlVArc2p0MONO08ybC5jiHlysfzS3YyZ5ry2d6Lw= @@ -51,6 +51,8 @@ github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWo github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=