diff --git a/README.md b/README.md index 139b610..a01cf2f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ brief crate:serde brief pypi:requests ``` -Local scans inspect up to eight directory levels and 10,000 filesystem entries by default. Use `--scan-depth N` or `--scan-limit N` to change those bounds. External line counters have a two-second limit, configurable with `--line-count-timeout D`, and reports mark truncated scans. Set any of these three values to `0` to remove that bound. Use `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration. +Local scans inspect up to eight directory levels and 10,000 filesystem entries by default. Use `--scan-depth N` or `--scan-limit N` to change those bounds. External line counters have a two-second limit, configurable with `--line-count-timeout D`, and reports mark truncated scans. Set any of these three values to `0` to remove that bound. Use `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. `--include-submodules` scans initialized Git submodules recursively, including one below an otherwise skipped directory such as `vendor/`; neighboring vendored files and missing submodule worktrees stay excluded. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration. Remote sources are shallow-cloned by default. Use `--depth 0` for a full clone, `--keep` to preserve the clone, or `--dir ./somewhere` to clone into a specific directory. Use `--cache ./cache` to keep one shallow checkout per HTTPS URL and reuse it across runs. Cache mode cannot be combined with `--depth 0` or `--dir`. diff --git a/cmd/brief/diff.go b/cmd/brief/diff.go index e08829d..ba07730 100644 --- a/cmd/brief/diff.go +++ b/cmd/brief/diff.go @@ -25,6 +25,7 @@ func cmdDiff(args []string) { scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)") scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") + includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents") _ = fs.Parse(args) // Determine the project root (git toplevel). @@ -90,6 +91,7 @@ func cmdDiff(args []string) { engine.ScanDepth = *scanDepth engine.ScanLimit = *scanLimit engine.LineCountTimeout = *lineCountTimeout + engine.IncludeSubmodules = *includeSubmodules r, err := engine.Run() if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) @@ -101,7 +103,8 @@ func cmdDiff(args []string) { r.DiffCommits = commits r.ChangedFiles = changedFiles - r = detect.FilterByChangedFiles(r, knowledgeBase, changedFiles) + filterFiles := engine.ExpandSubmoduleChanges(changedFiles) + r = detect.FilterByChangedFiles(r, knowledgeBase, filterFiles) if *category != "" { r = filterCategory(r, *category) diff --git a/cmd/brief/enrich.go b/cmd/brief/enrich.go index 454f592..dff7d33 100644 --- a/cmd/brief/enrich.go +++ b/cmd/brief/enrich.go @@ -36,6 +36,7 @@ func cmdEnrich(args []string) { scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") + includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents") _ = fs.Parse(args) path := "." @@ -56,7 +57,7 @@ func cmdEnrich(args []string) { code := runEnrich( src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip, - *jsonFlag, *humanFlag, *markdownFlag, *verbose, + *includeSubmodules, *jsonFlag, *humanFlag, *markdownFlag, *verbose, ) src.Cleanup() os.Exit(code) @@ -67,7 +68,7 @@ func runEnrich( scanDepth, scanLimit int, lineCountTimeout time.Duration, skip string, - jsonFlag, humanFlag, markdownFlag, verbose bool, + includeSubmodules, jsonFlag, humanFlag, markdownFlag, verbose bool, ) int { knowledgeBase, err := kb.Load(brief.KnowledgeFS) if err != nil { @@ -79,6 +80,7 @@ func runEnrich( engine.ScanDepth = scanDepth engine.ScanLimit = scanLimit engine.LineCountTimeout = lineCountTimeout + engine.IncludeSubmodules = includeSubmodules if skip != "" { engine.SkipDirs = strings.Split(skip, ",") } diff --git a/cmd/brief/main.go b/cmd/brief/main.go index f76cbc0..82e0cd9 100644 --- a/cmd/brief/main.go +++ b/cmd/brief/main.go @@ -82,6 +82,7 @@ func cmdScan(args []string) { lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") tracked := fs.Bool("tracked", false, "Only consider files tracked by git") + includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents") version := fs.Bool("version", false, "Print version and exit") _ = fs.Parse(args) @@ -119,7 +120,7 @@ func cmdScan(args []string) { code := runScan( src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip, *category, - *tracked, *jsonFlag, *humanFlag, *markdownFlag, *verbose, + *tracked, *includeSubmodules, *jsonFlag, *humanFlag, *markdownFlag, *verbose, ) src.Cleanup() os.Exit(code) @@ -130,7 +131,7 @@ func runScan( scanDepth, scanLimit int, lineCountTimeout time.Duration, skip, category string, - tracked, jsonFlag, humanFlag, markdownFlag, verbose bool, + tracked, includeSubmodules, jsonFlag, humanFlag, markdownFlag, verbose bool, ) int { knowledgeBase, err := kb.Load(brief.KnowledgeFS) if err != nil { @@ -143,6 +144,7 @@ func runScan( engine.ScanLimit = scanLimit engine.LineCountTimeout = lineCountTimeout engine.TrackedOnly = tracked + engine.IncludeSubmodules = includeSubmodules if skip != "" { engine.SkipDirs = strings.Split(skip, ",") } diff --git a/cmd/brief/main_test.go b/cmd/brief/main_test.go index 35662d6..cd438cc 100644 --- a/cmd/brief/main_test.go +++ b/cmd/brief/main_test.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "testing" @@ -13,6 +14,8 @@ import ( const scanHelperRootEnv = "BRIEF_SCAN_HELPER_ROOT" const diffHelperEnv = "BRIEF_DIFF_HELPER" +const submoduleHelperRootEnv = "BRIEF_SUBMODULE_HELPER_ROOT" +const submoduleDiffHelperEnv = "BRIEF_SUBMODULE_DIFF_HELPER" func TestScanDefaultsBoundRecursiveDetection(t *testing.T) { if root := os.Getenv(scanHelperRootEnv); root != "" { @@ -76,6 +79,111 @@ func TestDiffAppliesScanOverrides(t *testing.T) { } } +func TestScanIncludeSubmodulesFlag(t *testing.T) { + if root := os.Getenv(submoduleHelperRootEnv); root != "" { + cmdScan([]string{"-json", "-include-submodules", root}) + return + } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + native := t.TempDir() + initGitScanFixture(t, native) + writeScanFixture(t, native, "native.c", "int native(void) { return 0; }\n") + runGitFixture(t, native, "add", "native.c") + runGitFixture(t, native, "commit", "-q", "-m", "add native source") + + parent := t.TempDir() + initGitScanFixture(t, parent) + writeScanFixture(t, parent, "main.py", "print('example')\n") + runGitFixture(t, parent, "add", "main.py") + runGitFixture(t, parent, "commit", "-q", "-m", "add parent source") + runGitFixture(t, parent, "-c", "protocol.file.allow=always", "submodule", "add", "-q", native, "vendor/native") + runGitFixture(t, parent, "commit", "-q", "-m", "add submodule") + + cmd := exec.Command(os.Args[0], "-test.run=^TestScanIncludeSubmodulesFlag$") + cmd.Env = append(os.Environ(), submoduleHelperRootEnv+"="+parent) + out, err := cmd.Output() + if err != nil { + t.Fatalf("scan command failed: %v", err) + } + + var report brief.Report + if err := json.Unmarshal(out, &report); err != nil { + t.Fatalf("parsing scan output: %v\n%s", err, out) + } + if !slices.ContainsFunc(report.Languages, func(language brief.Detection) bool { + return language.Name == "C" + }) { + t.Errorf("languages = %+v, want C from initialized submodule", report.Languages) + } +} + +func TestDiffIncludeSubmodulesFlag(t *testing.T) { + if os.Getenv(submoduleDiffHelperEnv) != "" { + cmdDiff([]string{"-json", "-include-submodules", "HEAD"}) + os.Exit(0) + } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + native := t.TempDir() + initGitScanFixture(t, native) + writeScanFixture(t, native, "go.mod", "module example.com/native\n\ngo 1.22\n") + writeScanFixture(t, native, "native.c", "int native(void) { return 0; }\n") + runGitFixture(t, native, "add", "go.mod", "native.c") + runGitFixture(t, native, "commit", "-q", "-m", "add native source") + + parent := t.TempDir() + initGitScanFixture(t, parent) + writeScanFixture(t, parent, "main.py", "print('example')\n") + runGitFixture(t, parent, "add", "main.py") + runGitFixture(t, parent, "commit", "-q", "-m", "add parent source") + runGitFixture(t, parent, "-c", "protocol.file.allow=always", "submodule", "add", "-q", native, "modules/native") + runGitFixture(t, parent, "commit", "-q", "-m", "add submodule") + + checkout := filepath.Join(parent, "modules/native") + writeScanFixture(t, checkout, "version.txt", "2\n") + runGitFixture(t, checkout, "add", "version.txt") + runGitFixture( + t, checkout, "-c", "user.name=Test", "-c", "user.email=test@example.com", + "commit", "-q", "-m", "update native source", + ) + runGitFixture(t, parent, "add", "modules/native") + + cmd := exec.Command(os.Args[0], "-test.run=^TestDiffIncludeSubmodulesFlag$") + cmd.Dir = parent + cmd.Env = append(os.Environ(), submoduleDiffHelperEnv+"=1") + out, err := cmd.Output() + if err != nil { + t.Fatalf("diff command failed: %v", err) + } + + var report brief.Report + if err := json.Unmarshal(out, &report); err != nil { + t.Fatalf("parsing diff output: %v\n%s", err, out) + } + if !slices.ContainsFunc(report.Languages, func(language brief.Detection) bool { + return language.Name == "C" + }) { + t.Errorf("languages = %+v, want C from changed submodule", report.Languages) + } + if !slices.ContainsFunc(report.Manifests, func(manifest brief.ManifestInfo) bool { + return manifest.Path == "modules/native/go.mod" + }) { + t.Errorf("manifests = %+v, want modules/native/go.mod", report.Manifests) + } +} + +func initGitScanFixture(t *testing.T, dir string) { + t.Helper() + runGitFixture(t, dir, "init", "-q") + runGitFixture(t, dir, "config", "user.name", "Test") + runGitFixture(t, dir, "config", "user.email", "test@example.com") +} + func runGitFixture(t *testing.T, dir string, args ...string) { t.Helper() cmd := exec.Command("git", args...) diff --git a/cmd/brief/threat.go b/cmd/brief/threat.go index 20fe131..eba6b44 100644 --- a/cmd/brief/threat.go +++ b/cmd/brief/threat.go @@ -34,6 +34,7 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") + includeSubmodules := fs.Bool("include-submodules", false, "Include initialized Git submodule contents") _ = fs.Parse(args) path := "." @@ -51,6 +52,7 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou engine.ScanDepth = *scanDepth engine.ScanLimit = *scanLimit engine.LineCountTimeout = *lineCountTimeout + engine.IncludeSubmodules = *includeSubmodules if *skip != "" { engine.SkipDirs = strings.Split(*skip, ",") } diff --git a/detect/detect.go b/detect/detect.go index ef39a76..28999f6 100644 --- a/detect/detect.go +++ b/detect/detect.go @@ -14,6 +14,7 @@ import ( "path" "path/filepath" "regexp" + "slices" "sort" "strings" "sync" @@ -46,6 +47,7 @@ const ( categoryLint = "lint" categoryTest = "test" categoryTypecheck = "typecheck" + lineCounterSCC = "scc" rankHigh = 3 rankMedium = 2 @@ -54,44 +56,49 @@ const ( // Engine runs detection against a project directory. type Engine struct { - KB *kb.KnowledgeBase - Root string - ScanDepth int // optional max directory depth for recursive detection (0 = unlimited) - ScanLimit int // optional max filesystem entries per scan (0 = unlimited) - LineCountTimeout time.Duration // optional timeout for external line counters (0 = unlimited) - SkipDirs []string // additional directories to skip during walks - TrackedOnly bool // only consider files tracked by git - filesChecked int - toolsChecked int - toolsMatched int + KB *kb.KnowledgeBase + Root string + ScanDepth int // optional max directory depth for recursive detection (0 = unlimited) + ScanLimit int // optional max filesystem entries per scan (0 = unlimited) + LineCountTimeout time.Duration // optional timeout for external line counters (0 = unlimited) + IncludeSubmodules bool // include initialized Git submodule contents + SkipDirs []string // additional directories to skip during walks + TrackedOnly bool // only consider files tracked by git + filesChecked int + toolsChecked int + toolsMatched int detectedEcosystems map[string]bool // ecosystems whose language was detected // Lazily populated caches - tracked map[string]bool // git-tracked files relative to Root, nil when TrackedOnly is off - trackedDirs map[string]bool // directories that contain at least one tracked file - trackedDeps map[string]bool // whether a deps directory contains git-tracked files - fileExts map[string]int // cached file extension counts in the project - dirCache map[string][]string - depsLoaded bool - runtimeDeps map[string]bool // all runtime/unscoped dependency names - devDeps map[string]bool // development/test/build dependency names - allDeps map[string]bool // union of both - parsedDeps []brief.DepInfo // direct dependencies with PURLs - manifests []brief.ManifestInfo - manifestPathsCache []string - manifestPathsLoaded bool - cargoRoot string // relative directory containing the primary Cargo.toml - cargoFound bool - cargoLoaded bool - projectFiles []string // broad detection candidates - projectDirs []string - indexedFiles []string // includes routed hidden roots - indexedDirs []string - projectFilesLoaded bool - scanTruncated bool - scanDepthTruncated bool - scanEntries int + tracked map[string]bool // git-tracked files relative to Root, nil when TrackedOnly is off + trackedDirs map[string]bool // directories that contain at least one tracked file + trackedDeps map[string]bool // whether a deps directory contains git-tracked files + fileExts map[string]int // cached file extension counts in the project + dirCache map[string][]string + depsLoaded bool + runtimeDeps map[string]bool // all runtime/unscoped dependency names + devDeps map[string]bool // development/test/build dependency names + allDeps map[string]bool // union of both + parsedDeps []brief.DepInfo // direct dependencies with PURLs + manifests []brief.ManifestInfo + manifestPathsCache []string + manifestPathsLoaded bool + projectFiles []string // broad detection candidates + projectDirs []string + indexedFiles []string // includes routed hidden roots + indexedDirs []string + projectFilesLoaded bool + submodules []submoduleInfo + submodulesLoaded bool + submoduleEntries int + submoduleByPath map[string]submoduleInfo + submoduleRoutes map[string]bool + includedSubmodules []string + includedSubmoduleSet map[string]bool + scanTruncated bool + scanDepthTruncated bool + scanEntries int } // sortLanguagesByFileCount reorders detected languages so the one with @@ -166,11 +173,34 @@ func (e *Engine) loadTracked(abs string) error { } e.tracked = make(map[string]bool) e.trackedDirs = make(map[string]bool) - for p := range strings.SplitSeq(string(out), "\x00") { + e.addTrackedFiles("", out) + if !e.IncludeSubmodules { + return nil + } + e.loadSubmodules() + for _, submodule := range e.submodules { + if !submodule.Initialized { + continue + } + if e.ScanDepth > 0 && pathDepth(submodule.Path) > e.ScanDepth { + continue + } + submoduleRoot := filepath.Join(abs, submodule.Path) + out, err := e.git(submoduleRoot, "ls-files", "-z") + if err != nil { + continue + } + e.addTrackedFiles(submodule.Path, out) + } + return nil +} + +func (e *Engine) addTrackedFiles(prefix string, output []byte) { + for p := range strings.SplitSeq(string(output), "\x00") { if p == "" { continue } - p = filepath.FromSlash(p) + p = filepath.Join(prefix, filepath.FromSlash(p)) e.tracked[p] = true for d := filepath.Dir(p); d != "."; d = filepath.Dir(d) { if e.trackedDirs[d] { @@ -179,7 +209,6 @@ func (e *Engine) loadTracked(abs string) error { e.trackedDirs[d] = true } } - return nil } // isTracked reports whether a path relative to Root should be considered. @@ -199,6 +228,11 @@ func (e *Engine) shouldSkipDirPath(dirPath string) bool { if strings.HasPrefix(name, ".") { return true } + if rel, err := filepath.Rel(e.Root, dirPath); err == nil { + if _, ok := e.submoduleForPath(rel); ok { + return true + } + } if defaultSkipDirs[name] { return true } @@ -218,7 +252,7 @@ func (e *Engine) shouldSkipDirPath(dirPath string) bool { func (e *Engine) shouldIndexHiddenRoot(dirPath string) bool { name := filepath.Base(dirPath) - if !indexedHiddenRootDirs[name] || filepath.Clean(filepath.Dir(dirPath)) != filepath.Clean(e.Root) { + if !indexedHiddenRootDirs[name] || !e.isAnalysisRootPath(filepath.Dir(dirPath)) { return false } for _, dir := range e.SkipDirs { @@ -249,7 +283,16 @@ func (e *Engine) depsDirHasTrackedFiles(dirPath string) bool { if hasTracked, ok := e.trackedDeps[rel]; ok { return hasTracked } - out, err := e.git(e.Root, "ls-files", "-z", "--", filepath.ToSlash(rel)) + gitRoot := e.Root + gitRel := rel + if analysisRoot := e.analysisRootFor(rel); analysisRoot != "" { + gitRoot = filepath.Join(e.Root, analysisRoot) + gitRel, err = filepath.Rel(analysisRoot, rel) + if err != nil { + return true + } + } + out, err := e.git(gitRoot, "ls-files", "-z", "--", filepath.ToSlash(gitRel)) hasTracked := err != nil || len(out) > 0 e.trackedDeps[rel] = hasTracked return hasTracked @@ -596,8 +639,14 @@ func (e *Engine) exists(pattern string) bool { if kb.HasGlobPattern(dir) { return e.globMatches(dir, true) } - info, err := os.Stat(filepath.Join(e.Root, dir)) - return err == nil && info.IsDir() && e.isTracked(filepath.FromSlash(dir)) + for _, root := range e.analysisRoots() { + candidate := filepath.Join(root, filepath.FromSlash(dir)) + info, err := os.Stat(filepath.Join(e.Root, candidate)) + if err == nil && info.IsDir() && e.isTracked(candidate) { + return true + } + } + return false } // Handle recursive glob patterns like "**/*.py" @@ -624,17 +673,29 @@ func (e *Engine) exactFileExists(file string) bool { func (e *Engine) rootCandidates(file string) []string { file = filepath.ToSlash(filepath.Clean(file)) - candidates := []string{file} + var candidates []string + seen := make(map[string]bool) + add := func(candidate string) { + candidate = filepath.ToSlash(filepath.Clean(candidate)) + if !seen[candidate] { + seen[candidate] = true + candidates = append(candidates, candidate) + } + } + for _, root := range e.analysisRoots() { + add(path.Join(filepath.ToSlash(root), file)) + } switch file { case cargoManifestFile, cargoLockFile, ".cargo/config.toml": default: return candidates } - root, found := e.cargoManifestRoot() - if !found || root == "" { - return candidates + for _, analysisRoot := range e.analysisRoots() { + root, found := e.cargoManifestRootFrom(analysisRoot) + if found { + add(path.Join(filepath.ToSlash(root), file)) + } } - candidates = append(candidates, path.Join(root, file)) return candidates } @@ -647,7 +708,7 @@ func (e *Engine) globMatches(pattern string, wantDir bool) bool { candidates = e.projectDirs } for _, rel := range candidates { - if matchPathPattern(pattern, filepath.ToSlash(rel)) { + if e.matchesProjectPattern(pattern, rel) { return true } } @@ -658,7 +719,7 @@ func (e *Engine) globMatches(pattern string, wantDir bool) bool { func (e *Engine) recursiveGlob(pattern string) bool { e.loadProjectFiles() for _, rel := range e.projectFiles { - if matchPathPattern(pattern, filepath.ToSlash(rel)) { + if e.matchesProjectPattern(pattern, rel) { return true } } @@ -671,15 +732,16 @@ func (e *Engine) loadProjectFiles() { } e.projectFilesLoaded = true visited := 0 - e.scanProjectDir(e.Root, "", false, &visited) + e.scanProjectDir(e.Root, "", &visited, false) e.scanEntries = visited sort.Strings(e.indexedDirs) sort.Strings(e.indexedFiles) + sort.Strings(e.includedSubmodules) sort.Strings(e.projectDirs) sort.Strings(e.projectFiles) } -func (e *Engine) scanProjectDir(dirPath, relDir string, routeOnly bool, visited *int) bool { +func (e *Engine) scanProjectDir(dirPath, relDir string, visited *int, routeOnly bool) bool { dir, err := os.Open(dirPath) if err != nil { e.scanTruncated = true @@ -694,7 +756,7 @@ func (e *Engine) scanProjectDir(dirPath, relDir string, routeOnly bool, visited return false } for _, entry := range entries { - if e.scanProjectEntry(dirPath, relDir, entry, routeOnly, visited) { + if e.scanProjectEntry(dirPath, relDir, entry, visited, routeOnly) { return true } } @@ -707,8 +769,8 @@ func (e *Engine) scanProjectDir(dirPath, relDir string, routeOnly bool, visited func (e *Engine) scanProjectEntry( dirPath, relDir string, entry os.DirEntry, - routeOnly bool, visited *int, + routeOnly bool, ) bool { if e.ScanLimit > 0 && *visited >= e.ScanLimit { e.scanTruncated = true @@ -716,6 +778,14 @@ func (e *Engine) scanProjectEntry( } *visited++ rel := filepath.Join(relDir, entry.Name()) + submoduleRoute := e.IncludeSubmodules && e.initializedSubmoduleRoute(rel) + hiddenRoute := e.indexedHiddenRoute(rel) + if e.scanTruncated { + return true + } + if routeOnly && !submoduleRoute && !hiddenRoute { + return false + } info, err := entry.Info() if err != nil { e.scanTruncated = true @@ -732,20 +802,46 @@ func (e *Engine) scanProjectEntry( } filePath := filepath.Join(dirPath, entry.Name()) - route := e.shouldIndexHiddenRoot(filePath) - if (e.shouldSkipDirPath(filePath) && !route) || !e.isTracked(rel) { + nextRouteOnly := routeOnly + if slices.Contains(e.SkipDirs, entry.Name()) { + return false + } + skipDir := e.shouldSkipDirPath(filePath) + if e.scanTruncated { + return true + } + if skipDir { + if !e.shouldIndexHiddenRoot(filePath) && !submoduleRoute { + return false + } + nextRouteOnly = true + } + if !e.isTracked(rel) { return false } if e.ScanDepth > 0 && pathDepth(rel) > e.ScanDepth { e.scanDepthTruncated = true return false } - nextRouteOnly := routeOnly || route + submodule, isSubmodule := e.submoduleForPath(rel) + if isSubmodule && submodule.Initialized && e.IncludeSubmodules { + e.addIncludedSubmodule(rel) + nextRouteOnly = false + } e.indexedDirs = append(e.indexedDirs, rel) if !nextRouteOnly { e.projectDirs = append(e.projectDirs, rel) } - return e.scanProjectDir(filePath, rel, nextRouteOnly, visited) + return e.scanProjectDir(filePath, rel, visited, nextRouteOnly) +} + +func (e *Engine) indexedHiddenRoute(rel string) bool { + local := filepath.Clean(e.pathAtAnalysisRoot(rel)) + if local == "." || local == "" { + return false + } + name, _, _ := strings.Cut(local, string(filepath.Separator)) + return indexedHiddenRootDirs[name] } func pathDepth(rel string) int { @@ -816,10 +912,10 @@ func (e *Engine) contains(file string, patterns []string) bool { return e.globContains(file, patterns) } - files := []string{file} + files := e.rootCandidates(file) if filepath.ToSlash(file) == cargoManifestFile { for _, manifest := range e.manifestPaths() { - if manifest != file && path.Base(filepath.ToSlash(manifest)) == cargoManifestFile { + if !slices.Contains(files, manifest) && path.Base(filepath.ToSlash(manifest)) == cargoManifestFile { files = append(files, manifest) } } @@ -837,7 +933,7 @@ func (e *Engine) contains(file string, patterns []string) bool { func (e *Engine) globContains(pattern string, contentPatterns []string) bool { e.loadProjectFiles() for _, rel := range e.projectFiles { - if !matchPathPattern(pattern, filepath.ToSlash(rel)) { + if !e.matchesProjectPattern(pattern, rel) { continue } data, err := e.safeReadFile(rel) @@ -970,37 +1066,39 @@ func (e *Engine) manifestPaths() []string { paths = append(paths, p) } - for _, mf := range e.KB.ManifestFiles { - add(mf) - } - if cargoRoot, found := e.cargoManifestRoot(); found && cargoRoot != "" { - add(path.Join(cargoRoot, cargoManifestFile)) - add(path.Join(cargoRoot, cargoLockFile)) + roots := e.analysisRoots() + for _, root := range roots { + for _, mf := range e.KB.ManifestFiles { + add(path.Join(filepath.ToSlash(root), mf)) + } } e.loadProjectFiles() for _, rel := range e.indexedFiles { slashRel := filepath.ToSlash(rel) - if matchPathPattern(".github/workflows/*.yml", slashRel) || - matchPathPattern(".github/workflows/*.yaml", slashRel) { + if e.matchesProjectPattern(".github/workflows/*.yml", rel) || + e.matchesProjectPattern(".github/workflows/*.yaml", rel) { add(slashRel) } } - e.addCargoWorkspaceManifests(add) - e.addGoWorkspaceManifests(add) - e.addPackageWorkspaceManifests(add) - e.addPnpmWorkspaceManifests(add) + for _, root := range roots { + cargoRoot, found := e.cargoManifestRootFrom(root) + if found { + add(path.Join(cargoRoot, cargoManifestFile)) + add(path.Join(cargoRoot, cargoLockFile)) + e.addCargoWorkspaceManifestsFrom(cargoRoot, add) + } + e.addGoWorkspaceManifestsFrom(root, add) + e.addPackageWorkspaceManifestsFrom(root, add) + e.addPnpmWorkspaceManifestsFrom(root, add) + } e.manifestPathsCache = paths return paths } -func (e *Engine) addCargoWorkspaceManifests(add func(string)) { - cargoRoot, found := e.cargoManifestRoot() - if !found { - return - } +func (e *Engine) addCargoWorkspaceManifestsFrom(cargoRoot string, add func(string)) { data, err := e.safeReadFile(path.Join(cargoRoot, cargoManifestFile)) if err != nil { return @@ -1031,60 +1129,76 @@ func (e *Engine) addCargoWorkspaceManifests(add func(string)) { } } -// cargoManifestRoot returns the root Cargo manifest directory. When the -// repository root has no Cargo.toml but contains Rust source, it finds the -// shallowest Cargo.toml visible to the configured scan. -func (e *Engine) cargoManifestRoot() (string, bool) { - if e.cargoLoaded { - return e.cargoRoot, e.cargoFound +// cargoManifestRootFrom returns the root Cargo manifest directory below base. +// When base has no Cargo.toml but contains Rust source, it finds the shallowest +// Cargo.toml visible to the configured scan. +func (e *Engine) cargoManifestRootFrom(base string) (string, bool) { + base = filepath.Clean(base) + if base == "." { + base = "" } - e.cargoLoaded = true - - if e.exactFileExists(cargoManifestFile) { - e.cargoFound = true - return "", true + manifest := filepath.Join(base, cargoManifestFile) + if e.exactFileExists(manifest) { + return filepath.ToSlash(base), true } - e.loadFileExts() - if e.fileExts[".rs"] == 0 { + e.loadProjectFiles() + hasRust := false + for _, rel := range e.projectFiles { + if e.analysisRootFor(rel) == base && filepath.Ext(rel) == ".rs" { + hasRust = true + break + } + } + if !hasRust { return "", false } + best := "" bestDepth := 0 - e.loadProjectFiles() for _, rel := range e.projectFiles { - if filepath.Base(rel) != cargoManifestFile { + if e.analysisRootFor(rel) != base || filepath.Base(rel) != cargoManifestFile { continue } - dir := filepath.ToSlash(filepath.Dir(rel)) - depth := strings.Count(dir, "/") + 1 - if !e.cargoFound || depth < bestDepth || (depth == bestDepth && dir < e.cargoRoot) { - e.cargoRoot = dir - e.cargoFound = true + dir := filepath.Dir(rel) + localDir, err := filepath.Rel(baseOrDot(base), dir) + if err != nil { + continue + } + depth := pathDepth(localDir) + slashDir := filepath.ToSlash(dir) + if best == "" || depth < bestDepth || (depth == bestDepth && slashDir < best) { + best = slashDir bestDepth = depth if depth == 1 { break } } } + return best, best != "" +} - return e.cargoRoot, e.cargoFound +func baseOrDot(base string) string { + if base == "" { + return "." + } + return base } -func (e *Engine) addGoWorkspaceManifests(add func(string)) { - data, err := e.safeReadFile("go.work") +func (e *Engine) addGoWorkspaceManifestsFrom(base string, add func(string)) { + data, err := e.safeReadFile(path.Join(filepath.ToSlash(base), "go.work")) if err != nil { return } for _, member := range parseGoWorkUsePaths(string(data)) { - for _, dir := range e.expandWorkspacePattern(member) { + for _, dir := range e.expandWorkspacePatternFrom(filepath.ToSlash(base), member) { add(path.Join(dir, "go.mod")) } } } -func (e *Engine) addPackageWorkspaceManifests(add func(string)) { - data, err := e.safeReadFile("package.json") +func (e *Engine) addPackageWorkspaceManifestsFrom(base string, add func(string)) { + data, err := e.safeReadFile(path.Join(filepath.ToSlash(base), "package.json")) if err != nil { return } @@ -1096,14 +1210,14 @@ func (e *Engine) addPackageWorkspaceManifests(add func(string)) { return } for _, pattern := range packageWorkspacePatterns(root.Workspaces) { - for _, dir := range e.expandWorkspacePattern(pattern) { + for _, dir := range e.expandWorkspacePatternFrom(filepath.ToSlash(base), pattern) { add(path.Join(dir, "package.json")) } } } -func (e *Engine) addPnpmWorkspaceManifests(add func(string)) { - data, err := e.safeReadFile("pnpm-workspace.yaml") +func (e *Engine) addPnpmWorkspaceManifestsFrom(base string, add func(string)) { + data, err := e.safeReadFile(path.Join(filepath.ToSlash(base), "pnpm-workspace.yaml")) if err != nil { return } @@ -1124,9 +1238,9 @@ func (e *Engine) addPnpmWorkspaceManifests(add func(string)) { } includes = append(includes, pattern) } - excluded := e.workspacePatternSet(excludes) + excluded := e.workspacePatternSetFrom(filepath.ToSlash(base), excludes) for _, pattern := range includes { - for _, dir := range e.expandWorkspacePattern(pattern) { + for _, dir := range e.expandWorkspacePatternFrom(filepath.ToSlash(base), pattern) { if excluded[dir] { continue } @@ -1220,10 +1334,6 @@ func cleanWorkspaceMember(member string) string { return filepath.ToSlash(filepath.Clean(member)) } -func (e *Engine) workspacePatternSet(patterns []string) map[string]bool { - return e.workspacePatternSetFrom("", patterns) -} - func (e *Engine) workspacePatternSetFrom(base string, patterns []string) map[string]bool { set := make(map[string]bool) for _, pattern := range patterns { @@ -1234,10 +1344,6 @@ func (e *Engine) workspacePatternSetFrom(base string, patterns []string) map[str return set } -func (e *Engine) expandWorkspacePattern(pattern string) []string { - return e.expandWorkspacePatternFrom("", pattern) -} - func (e *Engine) expandWorkspacePatternFrom(base, pattern string) []string { pattern = cleanWorkspaceMember(pattern) if pattern == "." || pattern == "" || strings.HasPrefix(pattern, "../") || filepath.IsAbs(pattern) { @@ -1588,15 +1694,11 @@ func (e *Engine) detectLayout(languages []brief.Detection) *brief.LayoutInfo { layout := &brief.LayoutInfo{} for _, dir := range e.KB.Layouts.Layout.SourceDirs { - if e.exists(dir + "/") { - layout.SourceDirs = append(layout.SourceDirs, dir) - } + layout.SourceDirs = append(layout.SourceDirs, e.layoutDirsNamed(dir)...) } for _, dir := range e.KB.Layouts.Layout.TestDirs { - if e.exists(dir + "/") { - layout.TestDirs = append(layout.TestDirs, dir) - } + layout.TestDirs = append(layout.TestDirs, e.layoutDirsNamed(dir)...) } if len(layout.SourceDirs) == 0 { @@ -1635,13 +1737,30 @@ func (e *Engine) inferFlatLayout(languages []brief.Detection, testDirs []string) skip[d] = true } + e.loadProjectFiles() var found []string - for _, name := range e.dirDirs(".") { - if e.shouldSkipDirPath(filepath.Join(e.Root, name)) || skip[name] { + for _, rel := range e.projectDirs { + local := e.pathAtAnalysisRoot(rel) + if local == "." || strings.Contains(local, string(filepath.Separator)) { continue } - if e.dirHasExtension(name, exts) { - found = append(found, name) + name := filepath.Base(local) + if skip[name] { + continue + } + if e.projectDirHasExtension(rel, exts) { + found = append(found, filepath.ToSlash(rel)) + } + } + return found +} + +func (e *Engine) layoutDirsNamed(name string) []string { + e.loadProjectFiles() + var found []string + for _, rel := range e.projectDirs { + if filepath.ToSlash(e.pathAtAnalysisRoot(rel)) == name { + found = append(found, filepath.ToSlash(rel)) } } return found @@ -1668,13 +1787,13 @@ func (e *Engine) languageExtensions(name string) []string { return exts } -// dirHasExtension reports whether dir directly contains a file with one of the -// given extensions. -func (e *Engine) dirHasExtension(dir string, exts []string) bool { - for _, name := range e.dirFiles(dir) { - ext := filepath.Ext(name) +func (e *Engine) projectDirHasExtension(dir string, exts []string) bool { + for _, file := range e.projectFiles { + if filepath.Dir(file) != dir { + continue + } for _, want := range exts { - if ext == want { + if filepath.Ext(file) == want { return true } } @@ -1828,11 +1947,10 @@ func (e *Engine) detectSkills() []brief.Skill { e.loadProjectFiles() for _, glob := range []string{"skills/*/SKILL.md", ".claude/skills/*/SKILL.md"} { for _, rel := range e.indexedFiles { - rel = filepath.ToSlash(rel) - if !matchPathPattern(glob, rel) { + if !e.matchesProjectPattern(glob, rel) { continue } - skills = append(skills, e.parseSkill(rel)) + skills = append(skills, e.parseSkill(filepath.ToSlash(rel))) } } return skills @@ -2190,38 +2308,82 @@ func (e *Engine) detectLineCount(absPath string) *brief.LineCount { if e.scanTruncated || e.scanDepthTruncated { return nil } + roots := e.analysisRoots() + ctx := context.Background() + cancel := func() {} + if e.LineCountTimeout > 0 { + ctx, cancel = context.WithTimeout(ctx, e.LineCountTimeout) + } + defer cancel() - // Try scc first - if _, err := exec.LookPath("scc"); err == nil { - if out, timedOut, err := e.lineCounterOutput("scc", e.sccArgs(absPath)...); err == nil { - return parseSCCOutput(out) + if _, err := exec.LookPath(lineCounterSCC); err == nil { + if count, timedOut, ok := e.countRoots(ctx, lineCounterSCC, absPath, roots); ok { + return count } else if timedOut { return nil } } - // Try tokei if _, err := exec.LookPath("tokei"); err == nil { - if out, _, err := e.lineCounterOutput("tokei", "--output", "json", absPath); err == nil { - return parseTokeiOutput(out) + if count, _, ok := e.countRoots(ctx, "tokei", absPath, roots); ok { + return count } } return nil } -func (e *Engine) lineCounterOutput(name string, args ...string) ([]byte, bool, error) { - if e.LineCountTimeout == 0 { - out, err := exec.Command(name, args...).Output() - return out, false, err +func (e *Engine) countRoots( + ctx context.Context, + name, absPath string, + roots []string, +) (*brief.LineCount, bool, bool) { + var total *brief.LineCount + for _, root := range roots { + rootPath := filepath.Join(absPath, root) + var args []string + switch name { + case lineCounterSCC: + args = e.sccArgs(rootPath, root) + case "tokei": + args = e.tokeiArgs(rootPath, root) + } + out, err := exec.CommandContext(ctx, name, args...).Output() + if err != nil { + return nil, ctx.Err() != nil, false + } + var count *brief.LineCount + if name == lineCounterSCC { + count = parseSCCOutput(out) + } else { + count = parseTokeiOutput(out) + } + if count == nil { + return nil, false, false + } + total = mergeLineCounts(total, count) + } + return total, false, true +} + +func mergeLineCounts(total, count *brief.LineCount) *brief.LineCount { + if total == nil { + total = &brief.LineCount{ByLanguage: make(map[string]int), Source: count.Source} } - ctx, cancel := context.WithTimeout(context.Background(), e.LineCountTimeout) - defer cancel() - out, err := exec.CommandContext(ctx, name, args...).Output() - return out, ctx.Err() != nil, err + total.TotalFiles += count.TotalFiles + total.TotalLines += count.TotalLines + for language, lines := range count.ByLanguage { + total.ByLanguage[language] += lines + } + return total +} + +func (e *Engine) sccArgs(absPath, root string) []string { + dirs := e.lineCountExcludedDirs(absPath, root) + return []string{"--format", "json", "--exclude-dir", strings.Join(dirs, ","), absPath} } -func (e *Engine) sccArgs(absPath string) []string { +func (e *Engine) lineCountExcludedDirs(absPath, root string) []string { excluded := make(map[string]bool) for dir := range defaultSkipDirs { excluded[dir] = true @@ -2234,8 +2396,9 @@ func (e *Engine) sccArgs(absPath string) []string { for _, dir := range []string{".git", ".hg", ".svn"} { excluded[dir] = true } - depsPath := filepath.Join(e.Root, "deps") - if info, err := os.Stat(depsPath); err == nil && info.IsDir() && e.shouldSkipDirPath(depsPath) { + depsPath := filepath.Join(absPath, "deps") + logicalDepsPath := filepath.Join(e.Root, root, "deps") + if info, err := os.Stat(depsPath); err == nil && info.IsDir() && e.shouldSkipDirPath(logicalDepsPath) { excluded["deps"] = true } @@ -2244,7 +2407,27 @@ func (e *Engine) sccArgs(absPath string) []string { dirs = append(dirs, dir) } sort.Strings(dirs) - return []string{"--format", "json", "--exclude-dir", strings.Join(dirs, ","), absPath} + return dirs +} + +func (e *Engine) tokeiArgs(absPath, root string) []string { + args := []string{"--output", "json"} + excluded := make(map[string]bool) + for _, dir := range e.lineCountExcludedDirs(absPath, root) { + excluded[filepath.ToSlash(dir)+"/"] = true + } + for _, submodule := range e.directSubmodulePaths(root) { + excluded[submodule+"/"] = true + } + patterns := make([]string, 0, len(excluded)) + for pattern := range excluded { + patterns = append(patterns, pattern) + } + sort.Strings(patterns) + for _, pattern := range patterns { + args = append(args, "--exclude", pattern) + } + return append(args, absPath) } // parseSCCOutput parses scc --format json output. @@ -2261,7 +2444,7 @@ func parseSCCOutput(data []byte) *brief.LineCount { lc := &brief.LineCount{ ByLanguage: make(map[string]int), - Source: "scc", + Source: lineCounterSCC, } for _, r := range results { lc.TotalFiles += r.Count diff --git a/detect/detect_test.go b/detect/detect_test.go index e34b737..d53e755 100644 --- a/detect/detect_test.go +++ b/detect/detect_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strings" "testing" @@ -13,6 +14,21 @@ import ( "github.com/git-pkgs/brief/kb" ) +const lineCounterHelperEnv = "BRIEF_TEST_LINE_COUNTER" + +func TestMain(m *testing.M) { + if os.Getenv(lineCounterHelperEnv) != "" { + target := filepath.ToSlash(os.Args[len(os.Args)-1]) + if strings.HasSuffix(target, "/vendor/native") { + _, _ = os.Stdout.WriteString("[{\"Name\":\"C\",\"Lines\":1,\"Code\":1,\"Count\":1}]\n") + } else { + _, _ = os.Stdout.WriteString("[]\n") + } + os.Exit(0) + } + os.Exit(m.Run()) +} + func loadKB(t *testing.T) *kb.KnowledgeBase { t.Helper() knowledgeBase, err := kb.Load(brief.KnowledgeFS) @@ -821,10 +837,17 @@ func TestCargoManifestRootPrefersShallowest(t *testing.T) { writeFile(t, dir, "a/deep/src/lib.rs", "pub fn deep() {}\n") writeFile(t, dir, "z/Cargo.toml", "[package]\nname = \"shallow\"\nversion = \"0.1.0\"\n") - root, found := New(loadKB(t), dir).cargoManifestRoot() - if !found || root != "z" { - t.Fatalf("cargoManifestRoot = %q, %v, want z, true", root, found) + report := runOn(t, dir) + for _, manager := range report.PackageManagers { + if manager.Name != "Cargo" { + continue + } + if !slices.Contains(manager.ConfigFiles, "z/Cargo.toml") { + t.Fatalf("Cargo config files = %v, want z/Cargo.toml", manager.ConfigFiles) + } + return } + t.Fatalf("package managers = %v, want Cargo", packageManagerNames(report)) } func TestCargoLockfileDependencies(t *testing.T) { @@ -1633,6 +1656,312 @@ func TestRecursiveGlobHonorsScanLimit(t *testing.T) { } } +func TestIncludeSubmodules(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + deep := initGitProject(t, map[string]string{ + "Cargo.toml": "[package]\nname = \"deep\"\nversion = \"0.1.0\"\n", + "src/lib.rs": "pub fn deep() {}\n", + }) + native := initGitProject(t, map[string]string{ + "CMakeLists.txt": "cmake_minimum_required(VERSION 3.20)\nproject(native C)\n", + "deps/local/source.jl": "module LocalDependency\nend\n", + "go.mod": "module example.com/native\n\ngo 1.22\n\nrequire github.com/google/uuid v1.6.0\n", + "native.go": "package native\n", + "src/native.c": "int native(void) { return 0; }\n", + }) + addGitSubmodule(t, native, deep, "third_party/deep") + + parent := initGitProject(t, map[string]string{ + "main.py": "print('example')\n", + "pyproject.toml": "[project]\nname = \"example\"\nversion = \"1.0.0\"\n", + }) + addGitSubmodule(t, parent, native, "vendor/native") + gitFixtureCommand(t, parent, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--recursive") + writeProjectFile(t, parent, "vendor/unrelated/noise.f90", "program noise\nend program noise\n") + gitFixtureCommand(t, parent, "add", "vendor/unrelated/noise.f90") + gitFixtureCommand(t, parent, "commit", "-q", "-m", "add unrelated vendor file") + installSubmoduleSCC(t) + + withoutEngine := New(loadKB(t), parent) + withoutEngine.LineCountTimeout = 0 + without, err := withoutEngine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + assertToolDetected(t, without, "dependency_bot", "Git Submodules") + for _, name := range []string{"C", "Go", "Julia", "Rust", "Fortran"} { + if slices.Contains(languageNames(without), name) { + t.Errorf("default scan included %s from a submodule or vendor directory", name) + } + } + if without.Lines == nil || without.Lines.Source != lineCounterSCC || without.Lines.ByLanguage["C"] != 0 { + t.Errorf("default line counts should exclude submodule C source, got %+v", without.Lines) + } + + tooShallow := New(loadKB(t), parent) + tooShallow.IncludeSubmodules = true + tooShallow.ScanDepth = 1 + tooShallow.LineCountTimeout = 0 + shallowReport, err := tooShallow.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + for _, name := range []string{"C", "Go", "Julia", "Rust"} { + if slices.Contains(languageNames(shallowReport), name) { + t.Errorf("ScanDepth=1 included %s below vendor/native", name) + } + } + + engine := New(loadKB(t), parent) + engine.IncludeSubmodules = true + engine.ScanDepth = 5 + engine.LineCountTimeout = 0 + with, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + for _, name := range []string{"C", "Go", "Julia", "Rust"} { + if !slices.Contains(languageNames(with), name) { + t.Errorf("included submodule languages = %v, want %s", languageNames(with), name) + } + } + if slices.Contains(languageNames(with), "Fortran") { + t.Errorf("submodule scan included unrelated vendored source: %v", languageNames(with)) + } + assertToolDetected(t, with, "build", "CMake") + if !slices.Contains(packageManagerNames(with), "Go Modules") { + t.Errorf("package managers = %v, want Go Modules", packageManagerNames(with)) + } + if !slices.ContainsFunc(with.Manifests, func(manifest brief.ManifestInfo) bool { + return manifest.Path == "vendor/native/go.mod" && manifest.Ecosystem == "golang" + }) { + t.Errorf("manifests should include vendor/native/go.mod, got %+v", with.Manifests) + } + if !slices.ContainsFunc(with.Manifests, func(manifest brief.ManifestInfo) bool { + return manifest.Path == "vendor/native/third_party/deep/Cargo.toml" && manifest.Ecosystem == "cargo" + }) { + t.Errorf("manifests should include the recursive submodule Cargo.toml, got %+v", with.Manifests) + } + if !slices.ContainsFunc(with.Dependencies, func(dependency brief.DepInfo) bool { + return dependency.Name == "github.com/google/uuid" + }) { + t.Errorf("dependencies should include the submodule Go dependency, got %+v", with.Dependencies) + } + for _, want := range []string{"vendor/native/src", "vendor/native/third_party/deep/src"} { + if with.Layout == nil || !slices.Contains(with.Layout.SourceDirs, want) { + t.Errorf("source directories = %+v, want %q", with.Layout, want) + } + } + if with.Lines == nil || with.Lines.Source != lineCounterSCC || with.Lines.ByLanguage["C"] == 0 { + t.Errorf("line counts should include submodule C source, got %+v", with.Lines) + } +} + +func TestIncludeSubmodulesHonorsExplicitSkip(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + native := initGitProject(t, map[string]string{"native.c": "int native(void) { return 0; }\n"}) + parent := initGitProject(t, map[string]string{"main.py": "print('example')\n"}) + addGitSubmodule(t, parent, native, "vendor/native") + + engine := New(loadKB(t), parent) + engine.IncludeSubmodules = true + engine.SkipDirs = []string{"vendor"} + report, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if slices.Contains(languageNames(report), "C") { + t.Errorf("explicit vendor skip included C: %v", languageNames(report)) + } +} + +func installSubmoduleSCC(t *testing.T) { + t.Helper() + dir := t.TempDir() + helperName := lineCounterSCC + if runtime.GOOS == "windows" { + helperName += ".exe" + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + helperPath := filepath.Join(dir, helperName) + if runtime.GOOS == "windows" { + data, err := os.ReadFile(executable) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(helperPath, data, 0o755); err != nil { + t.Fatal(err) + } + } else if err := os.Link(executable, helperPath); err != nil { + t.Fatal(err) + } + t.Setenv(lineCounterHelperEnv, "1") + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestIncludeSubmodulesSkipsUninitializedCheckout(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + native := initGitProject(t, map[string]string{"native.c": "int native(void) { return 0; }\n"}) + parent := initGitProject(t, map[string]string{"main.py": "print('example')\n"}) + addGitSubmodule(t, parent, native, "modules/native") + checkout := filepath.Join(t.TempDir(), "checkout") + gitFixtureCommand(t, ".", "clone", "-q", parent, checkout) + + engine := New(loadKB(t), checkout) + engine.IncludeSubmodules = true + report, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + assertToolDetected(t, report, "dependency_bot", "Git Submodules") + if slices.Contains(languageNames(report), "C") { + t.Errorf("uninitialized submodule contributed a language: %v", languageNames(report)) + } +} + +func TestIncludeSubmodulesHonorsScanLimitWhileReadingGitmodules(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + dir := t.TempDir() + writeProjectFile(t, dir, ".gitmodules", `[submodule "one"] + path = modules/one + url = ../one +[submodule "two"] + path = modules/two + url = ../two +[submodule "three"] + path = modules/three + url = ../three +`) + engine := New(loadKB(t), dir) + engine.IncludeSubmodules = true + engine.ScanLimit = 2 + report, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if !report.Stats.ScanTruncated { + t.Fatal("expected .gitmodules entries to respect the scan limit") + } +} + +func TestDefaultScanIgnoresSubmoduleEntryBudget(t *testing.T) { + dir := t.TempDir() + writeProjectFile(t, dir, ".gitmodules", `[submodule "one"] + path = modules/one + url = ../one +[submodule "two"] + path = modules/two + url = ../two +[submodule "three"] + path = modules/three + url = ../three +[submodule "four"] + path = modules/four + url = ../four +[submodule "five"] + path = modules/five + url = ../five +`) + writeProjectFile(t, dir, "README.md", "example\n") + writeProjectFile(t, dir, "src/main.go", "package example\n") + + engine := New(loadKB(t), dir) + engine.ScanLimit = 4 + report, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if report.Stats.ScanTruncated { + t.Fatal("default scan was truncated by .gitmodules entries") + } + if !slices.Contains(languageNames(report), "Go") { + t.Errorf("languages = %v, want Go", languageNames(report)) + } +} + +func TestTrackedSubmodulesHonorScanDepth(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + native := initGitProject(t, map[string]string{"native.c": "int native(void) { return 0; }\n"}) + parent := initGitProject(t, map[string]string{"main.py": "print('example')\n"}) + addGitSubmodule(t, parent, native, "one/two/native") + + engine := New(loadKB(t), parent) + engine.IncludeSubmodules = true + engine.TrackedOnly = true + engine.ScanDepth = 1 + if _, err := engine.Run(); err != nil { + t.Fatalf("Run: %v", err) + } + submoduleFile := filepath.Join("one", "two", "native", "native.c") + if engine.tracked[submoduleFile] { + t.Errorf("tracked files include out-of-depth submodule file %q", submoduleFile) + } +} + +func TestIncludeSubmodulesRejectsUnavailableWorktrees(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + tests := []struct { + name string + configure func(*testing.T, string) + }{ + { + name: "stale gitdir", + configure: func(t *testing.T, root string) { + writeProjectFile(t, root, "modules/native/.git", "gitdir: ../missing\n") + }, + }, + { + name: "standalone repository", + configure: func(t *testing.T, root string) { + gitFixtureCommand(t, filepath.Join(root, "modules/native"), "init", "-q") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + writeProjectFile(t, dir, ".gitmodules", `[submodule "native"] + path = modules/native + url = ../native +`) + writeProjectFile(t, dir, "modules/native/native.c", "int native(void) { return 0; }\n") + tt.configure(t, dir) + + engine := New(loadKB(t), dir) + engine.IncludeSubmodules = true + report, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if slices.Contains(languageNames(report), "C") { + t.Errorf("unavailable submodule contributed a language: %v", languageNames(report)) + } + }) + } +} + func TestTrackedOnlyIgnoresDepthOfUntrackedDirectories(t *testing.T) { dir := t.TempDir() writeProjectFile(t, dir, "one/tracked.go", "package example\n") @@ -1668,6 +1997,35 @@ func writeProjectFile(t *testing.T, dir, path, content string) { } } +func initGitProject(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + gitFixtureCommand(t, dir, "init", "-q") + gitFixtureCommand(t, dir, "config", "user.name", "Test") + gitFixtureCommand(t, dir, "config", "user.email", "test@example.com") + for name, content := range files { + writeProjectFile(t, dir, name, content) + } + gitFixtureCommand(t, dir, "add", ".") + gitFixtureCommand(t, dir, "commit", "-q", "-m", "initial") + return dir +} + +func addGitSubmodule(t *testing.T, parent, child, path string) { + t.Helper() + gitFixtureCommand(t, parent, "-c", "protocol.file.allow=always", "submodule", "add", "-q", child, path) + gitFixtureCommand(t, parent, "commit", "-q", "-m", "add submodule") +} + +func gitFixtureCommand(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + func runOn(t *testing.T, dir string) *brief.Report { t.Helper() r, err := New(loadKB(t), dir).Run() @@ -2021,7 +2379,7 @@ func TestSCCArgsIncludeResolvedSkipDirs(t *testing.T) { dir := t.TempDir() engine := New(loadKB(t), dir) engine.SkipDirs = []string{"generated"} - args := engine.sccArgs(dir) + args := engine.sccArgs(dir, "") excludeIndex := slices.Index(args, "--exclude-dir") if excludeIndex == -1 || excludeIndex+1 >= len(args) { @@ -2042,7 +2400,7 @@ func TestSCCArgsIncludeResolvedSkipDirs(t *testing.T) { t.Fatal(err) } engine = New(loadKB(t), dir) - args = engine.sccArgs(dir) + args = engine.sccArgs(dir, "") excludeIndex = slices.Index(args, "--exclude-dir") excluded = strings.Split(args[excludeIndex+1], ",") if !slices.Contains(excluded, "deps") { @@ -2050,6 +2408,68 @@ func TestSCCArgsIncludeResolvedSkipDirs(t *testing.T) { } } +func TestSCCArgsResolveDepsFromRelativeRoot(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp(cwd, "brief-relative-root-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("RemoveAll: %v", err) + } + }) + gitFixtureCommand(t, dir, "init", "-q") + writeProjectFile(t, dir, "deps/example/lib.ex", "defmodule Example do\nend\n") + relativeRoot, err := filepath.Rel(cwd, dir) + if err != nil { + t.Fatal(err) + } + absRoot, err := filepath.Abs(relativeRoot) + if err != nil { + t.Fatal(err) + } + + engine := New(loadKB(t), relativeRoot) + args := engine.sccArgs(absRoot, "") + excludeIndex := slices.Index(args, "--exclude-dir") + if excludeIndex == -1 || excludeIndex+1 >= len(args) { + t.Fatalf("scc args missing --exclude-dir: %v", args) + } + excluded := strings.Split(args[excludeIndex+1], ",") + if !slices.Contains(excluded, "deps") { + t.Errorf("scc exclusions should contain deps for a relative root, got %v", excluded) + } +} + +func TestTokeiArgsIncludeResolvedSkipDirsAndSubmodules(t *testing.T) { + dir := t.TempDir() + engine := New(loadKB(t), dir) + engine.SkipDirs = []string{"generated"} + engine.submodulesLoaded = true + engine.submodules = []submoduleInfo{{Path: "modules/native"}} + args := engine.tokeiArgs(dir, "") + + var excluded []string + for i, arg := range args { + if arg == "--exclude" && i+1 < len(args) { + excluded = append(excluded, args[i+1]) + } + } + for _, want := range []string{"generated/", "modules/native/", "vendor/"} { + if !slices.Contains(excluded, want) { + t.Errorf("tokei exclusions should contain %q, got %v", want, excluded) + } + } +} + func TestDetectSelf(t *testing.T) { dir := t.TempDir() gomod := "module github.com/git-pkgs/brief\n\ngo 1.22.0\n" diff --git a/detect/submodules.go b/detect/submodules.go new file mode 100644 index 0000000..86719a5 --- /dev/null +++ b/detect/submodules.go @@ -0,0 +1,233 @@ +package detect + +import ( + "os" + "os/exec" + "path/filepath" + "sort" + "strings" +) + +type submoduleInfo struct { + Path string + Parent string + Initialized bool +} + +func (e *Engine) loadSubmodules() { + if e.submodulesLoaded { + return + } + e.submodulesLoaded = true + e.loadSubmodulesFrom(e.Root, "", make(map[string]bool)) + sort.Slice(e.submodules, func(i, j int) bool { + return e.submodules[i].Path < e.submodules[j].Path + }) + e.submoduleByPath = make(map[string]submoduleInfo, len(e.submodules)) + e.submoduleRoutes = make(map[string]bool) + for _, submodule := range e.submodules { + e.submoduleByPath[submodule.Path] = submodule + if !submodule.Initialized { + continue + } + for route := submodule.Path; route != "."; route = filepath.Dir(route) { + e.submoduleRoutes[route] = true + } + } +} + +func (e *Engine) loadSubmodulesFrom(root, parent string, seen map[string]bool) { + modulesFile := filepath.Join(root, ".gitmodules") + info, err := os.Lstat(modulesFile) + if err != nil || !info.Mode().IsRegular() { + return + } + absRoot, err := filepath.Abs(root) + if err != nil || seen[absRoot] { + return + } + seen[absRoot] = true + + cmd := exec.Command("git", "config", "--null", "--file", ".gitmodules", "--get-regexp", `^submodule\..*\.path$`) + cmd.Dir = root + out, err := cmd.Output() + if err != nil { + return + } + for record := range strings.SplitSeq(string(out), "\x00") { + if record == "" { + continue + } + if e.IncludeSubmodules && e.ScanLimit > 0 && e.submoduleEntries >= e.ScanLimit { + e.scanTruncated = true + return + } + e.submoduleEntries++ + _, configuredPath, ok := strings.Cut(record, "\n") + if !ok { + continue + } + child, ok := cleanSubmodulePath(configuredPath) + if !ok { + continue + } + fullPath := filepath.Clean(filepath.Join(parent, child)) + childRoot := filepath.Join(root, child) + initialized := e.IncludeSubmodules && initializedSubmoduleDir(childRoot, root) + e.submodules = append(e.submodules, submoduleInfo{ + Path: fullPath, + Parent: parent, + Initialized: initialized, + }) + if initialized && e.IncludeSubmodules && + (e.ScanDepth == 0 || pathDepth(fullPath) <= e.ScanDepth) { + e.loadSubmodulesFrom(childRoot, fullPath, seen) + } + } +} + +func cleanSubmodulePath(value string) (string, bool) { + value = filepath.FromSlash(strings.TrimSpace(value)) + if value == "" || filepath.IsAbs(value) { + return "", false + } + cleaned := filepath.Clean(value) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) { + return "", false + } + return cleaned, true +} + +func initializedSubmoduleDir(root, parent string) bool { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return false + } + out, err := exec.Command("git", "-C", root, "rev-parse", "--show-superproject-working-tree").Output() + if err != nil || strings.TrimSpace(string(out)) == "" { + return false + } + superproject, err := os.Stat(filepath.FromSlash(strings.TrimSpace(string(out)))) + if err != nil { + return false + } + parentInfo, err := os.Stat(parent) + return err == nil && os.SameFile(superproject, parentInfo) +} + +func (e *Engine) submoduleForPath(rel string) (submoduleInfo, bool) { + e.loadSubmodules() + rel = filepath.Clean(rel) + submodule, ok := e.submoduleByPath[rel] + return submodule, ok +} + +func (e *Engine) initializedSubmoduleRoute(rel string) bool { + e.loadSubmodules() + return e.submoduleRoutes[filepath.Clean(rel)] +} + +func (e *Engine) addIncludedSubmodule(rel string) { + if e.includedSubmoduleSet == nil { + e.includedSubmoduleSet = make(map[string]bool) + } + if e.includedSubmoduleSet[rel] { + return + } + e.includedSubmoduleSet[rel] = true + e.includedSubmodules = append(e.includedSubmodules, rel) +} + +func (e *Engine) analysisRoots() []string { + e.loadProjectFiles() + roots := make([]string, 1, len(e.includedSubmodules)+1) + copyRoots := append([]string(nil), e.includedSubmodules...) + sort.Strings(copyRoots) + return append(roots, copyRoots...) +} + +func (e *Engine) analysisRootFor(rel string) string { + for candidate := filepath.Clean(rel); candidate != "."; candidate = filepath.Dir(candidate) { + if e.includedSubmoduleSet[candidate] { + return candidate + } + } + return "" +} + +func (e *Engine) pathAtAnalysisRoot(rel string) string { + root := e.analysisRootFor(rel) + if root == "" { + return filepath.Clean(rel) + } + local, err := filepath.Rel(root, rel) + if err != nil { + return filepath.Clean(rel) + } + return local +} + +func (e *Engine) matchesProjectPattern(pattern, rel string) bool { + slashRel := filepath.ToSlash(rel) + if matchPathPattern(pattern, slashRel) { + return true + } + local := e.pathAtAnalysisRoot(rel) + return local != rel && matchPathPattern(pattern, filepath.ToSlash(local)) +} + +func (e *Engine) isAnalysisRootPath(root string) bool { + if filepath.Clean(root) == filepath.Clean(e.Root) { + return true + } + rel, err := filepath.Rel(e.Root, root) + if err != nil { + return false + } + submodule, ok := e.submoduleForPath(rel) + return ok && submodule.Initialized +} + +func (e *Engine) directSubmodulePaths(parent string) []string { + e.loadSubmodules() + var paths []string + for _, submodule := range e.submodules { + if submodule.Parent != parent { + continue + } + rel, err := filepath.Rel(baseOrDot(parent), submodule.Path) + if err == nil { + paths = append(paths, filepath.ToSlash(rel)) + } + } + sort.Strings(paths) + return paths +} + +// ExpandSubmoduleChanges adds scanned files beneath changed submodule roots. +func (e *Engine) ExpandSubmoduleChanges(changedFiles []string) []string { + e.loadProjectFiles() + expanded := append([]string(nil), changedFiles...) + seen := make(map[string]bool, len(changedFiles)) + for _, file := range changedFiles { + seen[filepath.ToSlash(filepath.Clean(filepath.FromSlash(file)))] = true + } + for _, changed := range changedFiles { + root := filepath.Clean(filepath.FromSlash(changed)) + if !e.includedSubmoduleSet[root] { + continue + } + prefix := root + string(filepath.Separator) + for _, file := range e.projectFiles { + if !strings.HasPrefix(file, prefix) { + continue + } + slashFile := filepath.ToSlash(file) + if !seen[slashFile] { + seen[slashFile] = true + expanded = append(expanded, slashFile) + } + } + } + return expanded +}