Add optional submodule scanning

README.md

@@ -70,7 +70,7 @@ brief crate:serde
 brief pypi:requests
 ```
 
-Local scans inspect up to eight directory levels and 10,000 filesystem entries by default. Use `--scan-depth N` or `--scan-limit N` to change those bounds. External line counters have a two-second limit, configurable with `--line-count-timeout D`, and reports mark truncated scans. Set any of these three values to `0` to remove that bound. Use `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration.
+Local scans inspect up to eight directory levels and 10,000 filesystem entries by default. Use `--scan-depth N` or `--scan-limit N` to change those bounds. External line counters have a two-second limit, configurable with `--line-count-timeout D`, and reports mark truncated scans. Set any of these three values to `0` to remove that bound. Use `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. `--include-submodules` scans initialized Git submodules recursively, including one below an otherwise skipped directory such as `vendor/`; neighboring vendored files and missing submodule worktrees stay excluded. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration.
 
 Remote sources are shallow-cloned by default. Use `--depth 0` for a full clone, `--keep` to preserve the clone, or `--dir ./somewhere` to clone into a specific directory. Use `--cache ./cache` to keep one shallow checkout per HTTPS URL and reuse it across runs. Cache mode cannot be combined with `--depth 0` or `--dir`.
 

cmd/brief/diff.go

@@ -25,6 +25,7 @@ func cmdDiff(args []string) {
 	scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)")
 	scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)")
 	lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)")
+	includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents")
 	_ = fs.Parse(args)
 
 	// Determine the project root (git toplevel).
@@ -90,6 +91,7 @@ func cmdDiff(args []string) {
 	engine.ScanDepth = *scanDepth
 	engine.ScanLimit = *scanLimit
 	engine.LineCountTimeout = *lineCountTimeout
+	engine.IncludeSubmodules = *includeSubmodules
 	r, err := engine.Run()
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err)
@@ -101,7 +103,8 @@ func cmdDiff(args []string) {
 	r.DiffCommits = commits
 	r.ChangedFiles = changedFiles
 
-	r = detect.FilterByChangedFiles(r, knowledgeBase, changedFiles)
+	filterFiles := engine.ExpandSubmoduleChanges(changedFiles)
+	r = detect.FilterByChangedFiles(r, knowledgeBase, filterFiles)
 
 	if *category != "" {
 		r = filterCategory(r, *category)

cmd/brief/enrich.go

@@ -36,6 +36,7 @@ func cmdEnrich(args []string) {
 	scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)")
 	lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)")
 	skip := fs.String("skip", "", "Additional directories to skip, comma-separated")
+	includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents")
 	_ = fs.Parse(args)
 
 	path := "."
@@ -56,7 +57,7 @@ func cmdEnrich(args []string) {
 
 	code := runEnrich(
 		src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip,
-		*jsonFlag, *humanFlag, *markdownFlag, *verbose,
+		*includeSubmodules, *jsonFlag, *humanFlag, *markdownFlag, *verbose,
 	)
 	src.Cleanup()
 	os.Exit(code)
@@ -67,7 +68,7 @@ func runEnrich(
 	scanDepth, scanLimit int,
 	lineCountTimeout time.Duration,
 	skip string,
-	jsonFlag, humanFlag, markdownFlag, verbose bool,
+	includeSubmodules, jsonFlag, humanFlag, markdownFlag, verbose bool,
 ) int {
 	knowledgeBase, err := kb.Load(brief.KnowledgeFS)
 	if err != nil {
@@ -79,6 +80,7 @@ func runEnrich(
 	engine.ScanDepth = scanDepth
 	engine.ScanLimit = scanLimit
 	engine.LineCountTimeout = lineCountTimeout
+	engine.IncludeSubmodules = includeSubmodules
 	if skip != "" {
 		engine.SkipDirs = strings.Split(skip, ",")
 	}

cmd/brief/main.go

@@ -82,6 +82,7 @@ func cmdScan(args []string) {
 	lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)")
 	skip := fs.String("skip", "", "Additional directories to skip, comma-separated")
 	tracked := fs.Bool("tracked", false, "Only consider files tracked by git")
+	includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents")
 	version := fs.Bool("version", false, "Print version and exit")
 	_ = fs.Parse(args)
 
@@ -119,7 +120,7 @@ func cmdScan(args []string) {
 
 	code := runScan(
 		src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip, *category,
-		*tracked, *jsonFlag, *humanFlag, *markdownFlag, *verbose,
+		*tracked, *includeSubmodules, *jsonFlag, *humanFlag, *markdownFlag, *verbose,
 	)
 	src.Cleanup()
 	os.Exit(code)
@@ -130,7 +131,7 @@ func runScan(
 	scanDepth, scanLimit int,
 	lineCountTimeout time.Duration,
 	skip, category string,
-	tracked, jsonFlag, humanFlag, markdownFlag, verbose bool,
+	tracked, includeSubmodules, jsonFlag, humanFlag, markdownFlag, verbose bool,
 ) int {
 	knowledgeBase, err := kb.Load(brief.KnowledgeFS)
 	if err != nil {
@@ -143,6 +144,7 @@ func runScan(
 	engine.ScanLimit = scanLimit
 	engine.LineCountTimeout = lineCountTimeout
 	engine.TrackedOnly = tracked
+	engine.IncludeSubmodules = includeSubmodules
 	if skip != "" {
 		engine.SkipDirs = strings.Split(skip, ",")
 	}

cmd/brief/main_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
@@ -13,6 +14,8 @@ import (
 
 const scanHelperRootEnv = "BRIEF_SCAN_HELPER_ROOT"
 const diffHelperEnv = "BRIEF_DIFF_HELPER"
+const submoduleHelperRootEnv = "BRIEF_SUBMODULE_HELPER_ROOT"
+const submoduleDiffHelperEnv = "BRIEF_SUBMODULE_DIFF_HELPER"
 
 func TestScanDefaultsBoundRecursiveDetection(t *testing.T) {
 	if root := os.Getenv(scanHelperRootEnv); root != "" {
@@ -76,6 +79,111 @@ func TestDiffAppliesScanOverrides(t *testing.T) {
 	}
 }
 
+func TestScanIncludeSubmodulesFlag(t *testing.T) {
+	if root := os.Getenv(submoduleHelperRootEnv); root != "" {
+		cmdScan([]string{"-json", "-include-submodules", root})
+		return
+	}
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not installed")
+	}
+
+	native := t.TempDir()
+	initGitScanFixture(t, native)
+	writeScanFixture(t, native, "native.c", "int native(void) { return 0; }\n")
+	runGitFixture(t, native, "add", "native.c")
+	runGitFixture(t, native, "commit", "-q", "-m", "add native source")
+
+	parent := t.TempDir()
+	initGitScanFixture(t, parent)
+	writeScanFixture(t, parent, "main.py", "print('example')\n")
+	runGitFixture(t, parent, "add", "main.py")
+	runGitFixture(t, parent, "commit", "-q", "-m", "add parent source")
+	runGitFixture(t, parent, "-c", "protocol.file.allow=always", "submodule", "add", "-q", native, "vendor/native")
+	runGitFixture(t, parent, "commit", "-q", "-m", "add submodule")
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestScanIncludeSubmodulesFlag$")
+	cmd.Env = append(os.Environ(), submoduleHelperRootEnv+"="+parent)
+	out, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("scan command failed: %v", err)
+	}
+
+	var report brief.Report
+	if err := json.Unmarshal(out, &report); err != nil {
+		t.Fatalf("parsing scan output: %v\n%s", err, out)
+	}
+	if !slices.ContainsFunc(report.Languages, func(language brief.Detection) bool {
+		return language.Name == "C"
+	}) {
+		t.Errorf("languages = %+v, want C from initialized submodule", report.Languages)
+	}
+}
+
+func TestDiffIncludeSubmodulesFlag(t *testing.T) {
+	if os.Getenv(submoduleDiffHelperEnv) != "" {
+		cmdDiff([]string{"-json", "-include-submodules", "HEAD"})
+		os.Exit(0)
+	}
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not installed")
+	}
+
+	native := t.TempDir()
+	initGitScanFixture(t, native)
+	writeScanFixture(t, native, "go.mod", "module example.com/native\n\ngo 1.22\n")
+	writeScanFixture(t, native, "native.c", "int native(void) { return 0; }\n")
+	runGitFixture(t, native, "add", "go.mod", "native.c")
+	runGitFixture(t, native, "commit", "-q", "-m", "add native source")
+
+	parent := t.TempDir()
+	initGitScanFixture(t, parent)
+	writeScanFixture(t, parent, "main.py", "print('example')\n")
+	runGitFixture(t, parent, "add", "main.py")
+	runGitFixture(t, parent, "commit", "-q", "-m", "add parent source")
+	runGitFixture(t, parent, "-c", "protocol.file.allow=always", "submodule", "add", "-q", native, "modules/native")
+	runGitFixture(t, parent, "commit", "-q", "-m", "add submodule")
+
+	checkout := filepath.Join(parent, "modules/native")
+	writeScanFixture(t, checkout, "version.txt", "2\n")
+	runGitFixture(t, checkout, "add", "version.txt")
+	runGitFixture(
+		t, checkout, "-c", "user.name=Test", "-c", "user.email=test@example.com",
+		"commit", "-q", "-m", "update native source",
+	)
+	runGitFixture(t, parent, "add", "modules/native")
+
+	cmd := exec.Command(os.Args[0], "-test.run=^TestDiffIncludeSubmodulesFlag$")
+	cmd.Dir = parent
+	cmd.Env = append(os.Environ(), submoduleDiffHelperEnv+"=1")
+	out, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("diff command failed: %v", err)
+	}
+
+	var report brief.Report
+	if err := json.Unmarshal(out, &report); err != nil {
+		t.Fatalf("parsing diff output: %v\n%s", err, out)
+	}
+	if !slices.ContainsFunc(report.Languages, func(language brief.Detection) bool {
+		return language.Name == "C"
+	}) {
+		t.Errorf("languages = %+v, want C from changed submodule", report.Languages)
+	}
+	if !slices.ContainsFunc(report.Manifests, func(manifest brief.ManifestInfo) bool {
+		return manifest.Path == "modules/native/go.mod"
+	}) {
+		t.Errorf("manifests = %+v, want modules/native/go.mod", report.Manifests)
+	}
+}
+
+func initGitScanFixture(t *testing.T, dir string) {
+	t.Helper()
+	runGitFixture(t, dir, "init", "-q")
+	runGitFixture(t, dir, "config", "user.name", "Test")
+	runGitFixture(t, dir, "config", "user.email", "test@example.com")
+}
+
 func runGitFixture(t *testing.T, dir string, args ...string) {
 	t.Helper()
 	cmd := exec.Command("git", args...)

cmd/brief/threat.go

@@ -34,6 +34,7 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou
 	scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)")
 	lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)")
 	skip := fs.String("skip", "", "Additional directories to skip, comma-separated")
+	includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents")
 	_ = fs.Parse(args)
 
 	path := "."
@@ -51,6 +52,7 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou
 	engine.ScanDepth = *scanDepth
 	engine.ScanLimit = *scanLimit
 	engine.LineCountTimeout = *lineCountTimeout
+	engine.IncludeSubmodules = *includeSubmodules
 	if *skip != "" {
 		engine.SkipDirs = strings.Split(*skip, ",")
 	}