diff --git a/README.md b/README.md index fa0e456..139b610 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ brief crate:serde brief pypi:requests ``` -Local scans recurse through the project tree. Use `--scan-depth N` to limit recursion, `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration. +Local scans inspect up to eight directory levels and 10,000 filesystem entries by default. Use `--scan-depth N` or `--scan-limit N` to change those bounds. External line counters have a two-second limit, configurable with `--line-count-timeout D`, and reports mark truncated scans. Set any of these three values to `0` to remove that bound. Use `--skip dir1,dir2` to add directory exclusions, or `--tracked` to consider only files tracked by Git. When Rust source exists without a root `Cargo.toml`, brief uses the shallowest `Cargo.toml` within the scan depth as an additional manifest root. Cargo workspace member manifests are also checked for tool configuration. Remote sources are shallow-cloned by default. Use `--depth 0` for a full clone, `--keep` to preserve the clone, or `--dir ./somewhere` to clone into a specific directory. Use `--cache ./cache` to keep one shallow checkout per HTTPS URL and reuse it across runs. Cache mode cannot be combined with `--depth 0` or `--dir`. diff --git a/brief.go b/brief.go index 8cd254b..89c10ba 100644 --- a/brief.go +++ b/brief.go @@ -200,11 +200,13 @@ type LineCount struct { // Stats holds performance and coverage metrics from the detection run. type Stats struct { - Duration time.Duration `json:"-"` - DurationMS float64 `json:"duration_ms"` - FilesChecked int `json:"files_checked"` - ToolsMatched int `json:"tools_matched"` - ToolsChecked int `json:"tools_checked"` + Duration time.Duration `json:"-"` + DurationMS float64 `json:"duration_ms"` + FilesChecked int `json:"files_checked"` + ToolsMatched int `json:"tools_matched"` + ToolsChecked int `json:"tools_checked"` + ScanEntries int `json:"scan_entries"` + ScanTruncated bool `json:"scan_truncated,omitempty"` } // DepInfo is a parsed dependency from a manifest file. diff --git a/cmd/brief/diff.go b/cmd/brief/diff.go index c39ca9d..e08829d 100644 --- a/cmd/brief/diff.go +++ b/cmd/brief/diff.go @@ -22,6 +22,9 @@ func cmdDiff(args []string) { markdownFlag := fs.Bool("markdown", false, "Force markdown output") verbose := fs.Bool("verbose", false, "Include breadcrumb/reference information") category := fs.String("category", "", "Only report on specific category") + scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)") + scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") + lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") _ = fs.Parse(args) // Determine the project root (git toplevel). @@ -84,6 +87,9 @@ func cmdDiff(args []string) { }() engine := detect.New(knowledgeBase, root) + engine.ScanDepth = *scanDepth + engine.ScanLimit = *scanLimit + engine.LineCountTimeout = *lineCountTimeout r, err := engine.Run() if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) diff --git a/cmd/brief/enrich.go b/cmd/brief/enrich.go index cfa5dc9..454f592 100644 --- a/cmd/brief/enrich.go +++ b/cmd/brief/enrich.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/BurntSushi/toml" "github.com/git-pkgs/brief" @@ -31,7 +32,9 @@ func cmdEnrich(args []string) { depth := fs.Int("depth", -1, "Git clone depth (0 = full clone, default shallow)") dir := fs.String("dir", "", "Directory to clone remote source into") cache := fs.String("cache", "", "Persistent shallow cache for HTTPS remotes (incompatible with -depth 0 and -dir)") - scanDepth := fs.Int("scan-depth", 0, "Max directory depth for language detection (0 = unlimited)") + scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)") + scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") + lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") _ = fs.Parse(args) @@ -51,12 +54,21 @@ func cmdEnrich(args []string) { os.Exit(1) } - code := runEnrich(src.Dir, *scanDepth, *skip, *jsonFlag, *humanFlag, *markdownFlag, *verbose) + code := runEnrich( + src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip, + *jsonFlag, *humanFlag, *markdownFlag, *verbose, + ) src.Cleanup() os.Exit(code) } -func runEnrich(dir string, scanDepth int, skip string, jsonFlag, humanFlag, markdownFlag, verbose bool) int { +func runEnrich( + dir string, + scanDepth, scanLimit int, + lineCountTimeout time.Duration, + skip string, + jsonFlag, humanFlag, markdownFlag, verbose bool, +) int { knowledgeBase, err := kb.Load(brief.KnowledgeFS) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error loading knowledge base: %v\n", err) @@ -65,6 +77,8 @@ func runEnrich(dir string, scanDepth int, skip string, jsonFlag, humanFlag, mark engine := detect.New(knowledgeBase, dir) engine.ScanDepth = scanDepth + engine.ScanLimit = scanLimit + engine.LineCountTimeout = lineCountTimeout if skip != "" { engine.SkipDirs = strings.Split(skip, ",") } diff --git a/cmd/brief/main.go b/cmd/brief/main.go index 1ffea09..f76cbc0 100644 --- a/cmd/brief/main.go +++ b/cmd/brief/main.go @@ -77,7 +77,9 @@ func cmdScan(args []string) { depth := fs.Int("depth", -1, "Git clone depth (0 = full clone, default shallow)") dir := fs.String("dir", "", "Directory to clone remote source into") cache := fs.String("cache", "", "Persistent shallow cache for HTTPS remotes (incompatible with -depth 0 and -dir)") - scanDepth := fs.Int("scan-depth", 0, "Max directory depth for language detection (0 = unlimited)") + scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)") + scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") + lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") tracked := fs.Bool("tracked", false, "Only consider files tracked by git") version := fs.Bool("version", false, "Print version and exit") @@ -115,12 +117,21 @@ func cmdScan(args []string) { os.Exit(1) } - code := runScan(src.Dir, *scanDepth, *skip, *category, *tracked, *jsonFlag, *humanFlag, *markdownFlag, *verbose) + code := runScan( + src.Dir, *scanDepth, *scanLimit, *lineCountTimeout, *skip, *category, + *tracked, *jsonFlag, *humanFlag, *markdownFlag, *verbose, + ) src.Cleanup() os.Exit(code) } -func runScan(dir string, scanDepth int, skip, category string, tracked, jsonFlag, humanFlag, markdownFlag, verbose bool) int { +func runScan( + dir string, + scanDepth, scanLimit int, + lineCountTimeout time.Duration, + skip, category string, + tracked, jsonFlag, humanFlag, markdownFlag, verbose bool, +) int { knowledgeBase, err := kb.Load(brief.KnowledgeFS) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error loading knowledge base: %v\n", err) @@ -129,6 +140,8 @@ func runScan(dir string, scanDepth int, skip, category string, tracked, jsonFlag engine := detect.New(knowledgeBase, dir) engine.ScanDepth = scanDepth + engine.ScanLimit = scanLimit + engine.LineCountTimeout = lineCountTimeout engine.TrackedOnly = tracked if skip != "" { engine.SkipDirs = strings.Split(skip, ",") diff --git a/cmd/brief/main_test.go b/cmd/brief/main_test.go new file mode 100644 index 0000000..35662d6 --- /dev/null +++ b/cmd/brief/main_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/brief" +) + +const scanHelperRootEnv = "BRIEF_SCAN_HELPER_ROOT" +const diffHelperEnv = "BRIEF_DIFF_HELPER" + +func TestScanDefaultsBoundRecursiveDetection(t *testing.T) { + if root := os.Getenv(scanHelperRootEnv); root != "" { + cmdScan([]string{"-json", root}) + return + } + + root := t.TempDir() + writeScanFixture(t, root, "pyproject.toml", "[project]\nname = \"example\"\nversion = \"1.0.0\"\n") + writeScanFixture(t, root, "pipelines/example/Snakefile", "rule all:\n") + deep := filepath.Join( + "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "asv.conf.json", + ) + writeScanFixture(t, root, deep, "{}\n") + + cmd := exec.Command(os.Args[0], "-test.run=^TestScanDefaultsBoundRecursiveDetection$") + cmd.Env = append(os.Environ(), scanHelperRootEnv+"="+root, "PATH=") + out, err := cmd.Output() + if err != nil { + t.Fatalf("scan command failed: %v", err) + } + + var report brief.Report + if err := json.Unmarshal(out, &report); err != nil { + t.Fatalf("parsing scan output: %v\n%s", err, out) + } + if !reportHasTool(&report, "build", "Snakemake") { + t.Fatal("expected nested Snakemake project within the default depth") + } + if reportHasTool(&report, "test", "ASV") { + t.Fatal("did not expect ASV beyond the default depth") + } + if !report.Stats.ScanTruncated { + t.Fatal("expected scan output to report the depth truncation") + } +} + +func TestDiffAppliesScanOverrides(t *testing.T) { + if os.Getenv(diffHelperEnv) != "" { + cmdDiff([]string{"-scan-limit=-1", "HEAD"}) + return + } + + dir := t.TempDir() + writeScanFixture(t, dir, "go.mod", "module example.com/project\n\ngo 1.22\n") + writeScanFixture(t, dir, "main.go", "package main\n") + runGitFixture(t, dir, "init", "-q") + runGitFixture(t, dir, "add", "go.mod", "main.go") + runGitFixture(t, dir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-q", "-m", "initial") + writeScanFixture(t, dir, "main.go", "package main\n\nfunc main() {}\n") + + cmd := exec.Command(os.Args[0], "-test.run=^TestDiffAppliesScanOverrides$") + cmd.Dir = dir + cmd.Env = append(os.Environ(), diffHelperEnv+"=1", "PATH="+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("diff command succeeded with a negative scan limit\n%s", out) + } + if !strings.Contains(string(out), "scan limit must not be negative") { + t.Fatalf("diff command output = %q, want scan limit validation error", out) + } +} + +func runGitFixture(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +func writeScanFixture(t *testing.T, root, name, content string) { + t.Helper() + full := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func reportHasTool(report *brief.Report, category, name string) bool { + for _, tool := range report.Tools[category] { + if tool.Name == name { + return true + } + } + return false +} diff --git a/cmd/brief/threat.go b/cmd/brief/threat.go index 1dbe839..20fe131 100644 --- a/cmd/brief/threat.go +++ b/cmd/brief/threat.go @@ -30,7 +30,9 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou jsonFlag := fs.Bool("json", false, "Force JSON output") humanFlag := fs.Bool("human", false, "Force human-readable output") markdownFlag := fs.Bool("markdown", false, "Force markdown output") - scanDepth := fs.Int("scan-depth", 0, "Max directory depth for language detection (0 = unlimited)") + scanDepth := fs.Int("scan-depth", detect.DefaultScanDepth, "Max directory depth for recursive detection (0 = unlimited)") + scanLimit := fs.Int("scan-limit", detect.DefaultScanLimit, "Max filesystem entries to scan (0 = unlimited)") + lineCountTimeout := fs.Duration("line-count-timeout", detect.DefaultLineCountTimeout, "Max time for line counting (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") _ = fs.Parse(args) @@ -47,6 +49,8 @@ func runDetection(name string, args []string) (*detect.Engine, *brief.Report, ou engine := detect.New(knowledgeBase, path) engine.ScanDepth = *scanDepth + engine.ScanLimit = *scanLimit + engine.LineCountTimeout = *lineCountTimeout if *skip != "" { engine.SkipDirs = strings.Split(*skip, ",") } diff --git a/detect/detect.go b/detect/detect.go index 811fe7a..ef39a76 100644 --- a/detect/detect.go +++ b/detect/detect.go @@ -3,6 +3,7 @@ package detect import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -28,9 +29,15 @@ import ( ) const ( - extScanFileLimit = 10000 // max files to visit when collecting extensions - microsPerMS = 1000.0 // microseconds per millisecond - globSplitParts = 2 // expected parts when splitting "**/" patterns + // DefaultScanDepth bounds recursive detection unless the caller overrides it. + DefaultScanDepth = 8 + // DefaultScanLimit bounds the filesystem entries visited by one detection run. + DefaultScanLimit = 10000 + // DefaultLineCountTimeout bounds external line counters. + DefaultLineCountTimeout = 2 * time.Second + + microsPerMS = 1000.0 + scanReadBatchSize = 128 cargoManifestFile = "Cargo.toml" cargoLockFile = "Cargo.lock" categoryBuild = "build" @@ -47,14 +54,16 @@ const ( // Engine runs detection against a project directory. type Engine struct { - KB *kb.KnowledgeBase - Root string - ScanDepth int // optional max directory depth for recursive detection (0 = unlimited) - SkipDirs []string // additional directories to skip during walks - TrackedOnly bool // only consider files tracked by git - filesChecked int - toolsChecked int - toolsMatched int + KB *kb.KnowledgeBase + Root string + ScanDepth int // optional max directory depth for recursive detection (0 = unlimited) + ScanLimit int // optional max filesystem entries per scan (0 = unlimited) + LineCountTimeout time.Duration // optional timeout for external line counters (0 = unlimited) + SkipDirs []string // additional directories to skip during walks + TrackedOnly bool // only consider files tracked by git + filesChecked int + toolsChecked int + toolsMatched int detectedEcosystems map[string]bool // ecosystems whose language was detected @@ -75,6 +84,14 @@ type Engine struct { cargoRoot string // relative directory containing the primary Cargo.toml cargoFound bool cargoLoaded bool + projectFiles []string // broad detection candidates + projectDirs []string + indexedFiles []string // includes routed hidden roots + indexedDirs []string + projectFilesLoaded bool + scanTruncated bool + scanDepthTruncated bool + scanEntries int } // sortLanguagesByFileCount reorders detected languages so the one with @@ -130,6 +147,16 @@ var defaultSkipDirs = map[string]bool{ "coverage": true, } +var indexedHiddenRootDirs = map[string]bool{ + ".claude": true, + ".cursor": true, + ".forgejo": true, + ".gitea": true, + ".github": true, + ".gitlab": true, + ".junie": true, +} + // loadTracked populates the set of git-tracked files under Root by running // git ls-files once. Paths are stored relative to Root using the OS separator. func (e *Engine) loadTracked(abs string) error { @@ -189,6 +216,19 @@ func (e *Engine) shouldSkipDirPath(dirPath string) bool { return false } +func (e *Engine) shouldIndexHiddenRoot(dirPath string) bool { + name := filepath.Base(dirPath) + if !indexedHiddenRootDirs[name] || filepath.Clean(filepath.Dir(dirPath)) != filepath.Clean(e.Root) { + return false + } + for _, dir := range e.SkipDirs { + if name == dir { + return false + } + } + return true +} + func (e *Engine) exactFileAt(filePath string) bool { info, err := os.Stat(filePath) return err == nil && !info.IsDir() @@ -217,12 +257,21 @@ func (e *Engine) depsDirHasTrackedFiles(dirPath string) bool { // New creates a detection engine for the given project root. func New(knowledgeBase *kb.KnowledgeBase, root string) *Engine { - return &Engine{KB: knowledgeBase, Root: root} + return &Engine{ + KB: knowledgeBase, + Root: root, + ScanDepth: DefaultScanDepth, + ScanLimit: DefaultScanLimit, + LineCountTimeout: DefaultLineCountTimeout, + } } // Run performs full detection and returns a Report. func (e *Engine) Run() (*brief.Report, error) { start := time.Now() + if err := e.validateOptions(); err != nil { + return nil, err + } abs, err := filepath.Abs(e.Root) if err != nil { @@ -289,16 +338,31 @@ func (e *Engine) Run() (*brief.Report, error) { elapsed := time.Since(start) report.Stats = brief.Stats{ - Duration: elapsed, - DurationMS: float64(elapsed.Microseconds()) / microsPerMS, - FilesChecked: e.filesChecked, - ToolsMatched: e.toolsMatched, - ToolsChecked: e.toolsChecked, + Duration: elapsed, + DurationMS: float64(elapsed.Microseconds()) / microsPerMS, + FilesChecked: e.filesChecked, + ToolsMatched: e.toolsMatched, + ToolsChecked: e.toolsChecked, + ScanEntries: e.scanEntries, + ScanTruncated: e.scanTruncated || e.scanDepthTruncated, } return report, nil } +func (e *Engine) validateOptions() error { + if e.ScanDepth < 0 { + return fmt.Errorf("scan depth must not be negative") + } + if e.ScanLimit < 0 { + return fmt.Errorf("scan limit must not be negative") + } + if e.LineCountTimeout < 0 { + return fmt.Errorf("line count timeout must not be negative") + } + return nil +} + const selfModulePath = "github.com/git-pkgs/brief" func (e *Engine) detectSelf(root string, report *brief.Report) { @@ -555,7 +619,7 @@ func (e *Engine) exists(pattern string) bool { func (e *Engine) exactFileExists(file string) bool { info, err := os.Stat(filepath.Join(e.Root, filepath.FromSlash(file))) - return err == nil && !info.IsDir() && e.isTracked(filepath.FromSlash(file)) + return err == nil && info.Mode().IsRegular() && e.isTracked(filepath.FromSlash(file)) } func (e *Engine) rootCandidates(file string) []string { @@ -577,126 +641,131 @@ func (e *Engine) rootCandidates(file string) []string { // globMatches reports whether a root-level glob pattern matches at least one // entry of the requested kind. func (e *Engine) globMatches(pattern string, wantDir bool) bool { - matches, err := filepath.Glob(filepath.Join(e.Root, pattern)) - if err != nil { - return false + e.loadProjectFiles() + candidates := e.projectFiles + if wantDir { + candidates = e.projectDirs } - for _, m := range matches { - info, err := os.Stat(m) - if err != nil || info.IsDir() != wantDir { - continue - } - rel, err := filepath.Rel(e.Root, m) - if err != nil { - continue - } - if e.isTracked(rel) { + for _, rel := range candidates { + if matchPathPattern(pattern, filepath.ToSlash(rel)) { return true } } return false } -// recursiveGlob handles ** patterns by checking against the cached file extension set. -// Falls back to a bounded walk if the cache isn't populated. +// recursiveGlob matches recursive patterns against the bounded project file index. func (e *Engine) recursiveGlob(pattern string) bool { - parts := strings.SplitN(pattern, "**/", globSplitParts) - if len(parts) != globSplitParts { - return false + e.loadProjectFiles() + for _, rel := range e.projectFiles { + if matchPathPattern(pattern, filepath.ToSlash(rel)) { + return true + } + } + return false +} + +func (e *Engine) loadProjectFiles() { + if e.projectFilesLoaded { + return } - suffix := parts[1] // e.g. "*.py" + e.projectFilesLoaded = true + visited := 0 + e.scanProjectDir(e.Root, "", false, &visited) + e.scanEntries = visited + sort.Strings(e.indexedDirs) + sort.Strings(e.indexedFiles) + sort.Strings(e.projectDirs) + sort.Strings(e.projectFiles) +} - // Use the cached extension set for simple "**/*.ext" patterns - if strings.HasPrefix(suffix, "*.") && strings.Count(suffix, ".") == 1 { - ext := suffix[1:] // ".py" - e.loadFileExts() - return e.fileExts[ext] > 0 +func (e *Engine) scanProjectDir(dirPath, relDir string, routeOnly bool, visited *int) bool { + dir, err := os.Open(dirPath) + if err != nil { + e.scanTruncated = true + return false } + defer func() { _ = dir.Close() }() - // Fall back to walk for complex patterns. - // Uses WalkDir to avoid following symlinks into directories. - root := filepath.Join(e.Root, parts[0]) - found := false - errDone := errors.New("found") - _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil + for { + entries, readErr := dir.ReadDir(scanReadBatchSize) + if readErr != nil && !errors.Is(readErr, io.EOF) { + e.scanTruncated = true + return false } - rel, _ := filepath.Rel(e.Root, path) - if d.IsDir() { - name := d.Name() - if name != "." && e.shouldSkipDirPath(path) { - return filepath.SkipDir - } - if !e.isTracked(rel) { - return filepath.SkipDir + for _, entry := range entries { + if e.scanProjectEntry(dirPath, relDir, entry, routeOnly, visited) { + return true } - return nil } - if d.Type()&os.ModeSymlink != 0 { - return nil - } - if !e.isTracked(rel) { - return nil + if errors.Is(readErr, io.EOF) { + return false } - matched, _ := filepath.Match(suffix, d.Name()) - if matched { - found = true - return errDone + } +} + +func (e *Engine) scanProjectEntry( + dirPath, relDir string, + entry os.DirEntry, + routeOnly bool, + visited *int, +) bool { + if e.ScanLimit > 0 && *visited >= e.ScanLimit { + e.scanTruncated = true + return true + } + *visited++ + rel := filepath.Join(relDir, entry.Name()) + info, err := entry.Info() + if err != nil { + e.scanTruncated = true + return false + } + if !info.IsDir() { + if info.Mode().IsRegular() && e.isTracked(rel) { + e.indexedFiles = append(e.indexedFiles, rel) + if !routeOnly { + e.projectFiles = append(e.projectFiles, rel) + } } - return nil - }) - return found + return false + } + + filePath := filepath.Join(dirPath, entry.Name()) + route := e.shouldIndexHiddenRoot(filePath) + if (e.shouldSkipDirPath(filePath) && !route) || !e.isTracked(rel) { + return false + } + if e.ScanDepth > 0 && pathDepth(rel) > e.ScanDepth { + e.scanDepthTruncated = true + return false + } + nextRouteOnly := routeOnly || route + e.indexedDirs = append(e.indexedDirs, rel) + if !nextRouteOnly { + e.projectDirs = append(e.projectDirs, rel) + } + return e.scanProjectDir(filePath, rel, nextRouteOnly, visited) +} + +func pathDepth(rel string) int { + if rel == "" || rel == "." { + return 0 + } + return strings.Count(filepath.Clean(rel), string(filepath.Separator)) + 1 } -// loadFileExts walks the project to collect file extension counts. Cached for -// the lifetime of the engine. The walk is bounded by extScanFileLimit rather -// than directory depth so that deep source layouts such as -// app/src/main/java// are reached; directory skips already prune the -// expensive vendor/build directories. -// Uses WalkDir instead of Walk to avoid following symlinks into directories. func (e *Engine) loadFileExts() { if e.fileExts != nil { return } + e.loadProjectFiles() e.fileExts = make(map[string]int) - rootLen := len(e.Root) - seen := 0 - errDone := errors.New("done") - _ = filepath.WalkDir(e.Root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - rel := strings.TrimPrefix(path[rootLen:], string(filepath.Separator)) - if d.IsDir() { - name := d.Name() - if name != "." && e.shouldSkipDirPath(path) { - return filepath.SkipDir - } - if e.ScanDepth > 0 && strings.Count(rel, string(filepath.Separator))+1 > e.ScanDepth { - return filepath.SkipDir - } - if !e.isTracked(rel) { - return filepath.SkipDir - } - return nil - } - if d.Type()&os.ModeSymlink != 0 { - return nil - } - if !e.isTracked(rel) { - return nil - } - ext := filepath.Ext(d.Name()) - if ext != "" { + for _, rel := range e.projectFiles { + if ext := filepath.Ext(rel); ext != "" { e.fileExts[ext]++ } - seen++ - if seen >= extScanFileLimit { - return errDone - } - return nil - }) + } } // safeReadFile reads a file within the project root, rejecting symlinks @@ -717,9 +786,19 @@ func (e *Engine) safeReadFile(file string) ([]byte, error) { if !strings.HasPrefix(target, absRoot+string(filepath.Separator)) { return nil, fmt.Errorf("symlink escapes project root: %s -> %s", file, target) } + targetInfo, err := os.Stat(target) + if err != nil { + return nil, err + } + if !targetInfo.Mode().IsRegular() { + return nil, fmt.Errorf("path is not a regular file: %s", file) + } // Safe symlink within root: read the resolved target directly. return os.ReadFile(target) } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("path is not a regular file: %s", file) + } // Not a symlink: open with O_NOFOLLOW so a symlink swap between // the Lstat and Open is rejected by the kernel. f, err := openNoFollow(path) @@ -756,47 +835,17 @@ func (e *Engine) contains(file string, patterns []string) bool { } func (e *Engine) globContains(pattern string, contentPatterns []string) bool { - found := false - errFound := errors.New("found") - _ = filepath.WalkDir(e.Root, func(filePath string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - - rel, err := filepath.Rel(e.Root, filePath) - if err != nil { - return nil - } - if d.IsDir() { - if rel != "." && e.shouldSkipDirPath(filePath) { - return filepath.SkipDir - } - if e.ScanDepth > 0 && rel != "." && - strings.Count(rel, string(filepath.Separator))+1 > e.ScanDepth { - return filepath.SkipDir - } - if !e.isTracked(rel) { - return filepath.SkipDir - } - return nil - } - - if !e.isTracked(rel) || !matchPathPattern(pattern, filepath.ToSlash(rel)) { - return nil - } - info, err := d.Info() - if err != nil || !info.Mode().IsRegular() { - return nil + e.loadProjectFiles() + for _, rel := range e.projectFiles { + if !matchPathPattern(pattern, filepath.ToSlash(rel)) { + continue } - data, err := e.safeReadFile(rel) - if err != nil || !containsAny(string(data), contentPatterns) { - return nil + if err == nil && containsAny(string(data), contentPatterns) { + return true } - found = true - return errFound - }) - return found + } + return false } func containsAny(content string, patterns []string) bool { @@ -929,13 +978,12 @@ func (e *Engine) manifestPaths() []string { add(path.Join(cargoRoot, cargoLockFile)) } - // GitHub Actions workflow files - wfMatches, _ := filepath.Glob(filepath.Join(e.Root, ".github/workflows/*.yml")) - wfMatchesYAML, _ := filepath.Glob(filepath.Join(e.Root, ".github/workflows/*.yaml")) - for _, m := range append(wfMatches, wfMatchesYAML...) { - rel, err := filepath.Rel(e.Root, m) - if err == nil { - add(rel) + e.loadProjectFiles() + for _, rel := range e.indexedFiles { + slashRel := filepath.ToSlash(rel) + if matchPathPattern(".github/workflows/*.yml", slashRel) || + matchPathPattern(".github/workflows/*.yaml", slashRel) { + add(slashRel) } } @@ -1003,30 +1051,10 @@ func (e *Engine) cargoManifestRoot() (string, bool) { } bestDepth := 0 - errShallowest := errors.New("found shallowest Cargo manifest") - _ = filepath.WalkDir(e.Root, func(filePath string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - rel, err := filepath.Rel(e.Root, filePath) - if err != nil { - return nil - } - if d.IsDir() { - if rel != "." && e.shouldSkipDirPath(filePath) { - return filepath.SkipDir - } - if e.ScanDepth > 0 && rel != "." && - strings.Count(rel, string(filepath.Separator))+1 > e.ScanDepth { - return filepath.SkipDir - } - if !e.isTracked(rel) { - return filepath.SkipDir - } - return nil - } - if d.Name() != cargoManifestFile || d.Type()&os.ModeSymlink != 0 || !e.isTracked(rel) { - return nil + e.loadProjectFiles() + for _, rel := range e.projectFiles { + if filepath.Base(rel) != cargoManifestFile { + continue } dir := filepath.ToSlash(filepath.Dir(rel)) depth := strings.Count(dir, "/") + 1 @@ -1035,11 +1063,10 @@ func (e *Engine) cargoManifestRoot() (string, bool) { e.cargoFound = true bestDepth = depth if depth == 1 { - return errShallowest + break } } - return nil - }) + } return e.cargoRoot, e.cargoFound } @@ -1050,7 +1077,9 @@ func (e *Engine) addGoWorkspaceManifests(add func(string)) { return } for _, member := range parseGoWorkUsePaths(string(data)) { - add(path.Join(member, "go.mod")) + for _, dir := range e.expandWorkspacePattern(member) { + add(path.Join(dir, "go.mod")) + } } } @@ -1217,30 +1246,14 @@ func (e *Engine) expandWorkspacePatternFrom(base, pattern string) []string { if base != "" { pattern = path.Join(base, pattern) } - // filepath.Glob does not implement recursive doublestar matching: `**` - // behaves like `*`, so workspace patterns such as packages/** only match - // one directory segment. - matches, err := filepath.Glob(filepath.Join(e.Root, filepath.FromSlash(pattern))) - if err != nil || len(matches) == 0 { - return []string{pattern} - } + e.loadProjectFiles() var dirs []string - for _, match := range matches { - info, err := os.Stat(match) - if err != nil || !info.IsDir() { - continue - } - rel, err := filepath.Rel(e.Root, match) - if err != nil { - continue - } - rel = filepath.ToSlash(filepath.Clean(rel)) - if rel == "." || strings.HasPrefix(rel, "../") { - continue + for _, rel := range e.projectDirs { + slashRel := filepath.ToSlash(rel) + if matchPathPattern(pattern, slashRel) { + dirs = append(dirs, slashRel) } - dirs = append(dirs, rel) } - sort.Strings(dirs) return dirs } @@ -1532,7 +1545,6 @@ func (sc *styleCounts) toStyleInfo() *brief.StyleInfo { } // inferStyle samples source files to detect indentation style. -// Uses WalkDir to avoid following symlinks, and reads via safeReadFile. func (e *Engine) inferStyle() *brief.StyleInfo { if e.KB.StyleConfig == nil { return nil @@ -1549,38 +1561,20 @@ func (e *Engine) inferStyle() *brief.StyleInfo { } var sc styleCounts - errDone := errors.New("done") - _ = filepath.WalkDir(e.Root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - if d.IsDir() { - name := d.Name() - if name != "." && e.shouldSkipDirPath(path) { - return filepath.SkipDir - } - return nil - } - if d.Type()&os.ModeSymlink != 0 { - return nil - } + e.loadProjectFiles() + for _, rel := range e.projectFiles { if sc.sampled >= limit { - return errDone + break } - if !exts[filepath.Ext(path)] { - return nil - } - rel, err := filepath.Rel(e.Root, path) - if err != nil { - return nil + if !exts[filepath.Ext(rel)] { + continue } data, err := e.safeReadFile(rel) if err != nil { - return nil + continue } sc.addFile(data) - return nil - }) + } return sc.toStyleInfo() } @@ -1641,17 +1635,8 @@ func (e *Engine) inferFlatLayout(languages []brief.Detection, testDirs []string) skip[d] = true } - entries, err := os.ReadDir(e.Root) - if err != nil { - return nil - } - var found []string - for _, ent := range entries { - if !ent.IsDir() { - continue - } - name := ent.Name() + for _, name := range e.dirDirs(".") { if e.shouldSkipDirPath(filepath.Join(e.Root, name)) || skip[name] { continue } @@ -1748,29 +1733,28 @@ var ( func (e *Engine) detectTemplates() *brief.TemplateInfo { t := &brief.TemplateInfo{} for _, base := range templateBaseDirs { - entries, err := os.ReadDir(filepath.Join(e.Root, filepath.FromSlash(base))) - if err != nil { - continue - } - for _, ent := range entries { - name := ent.Name() + for _, name := range e.dirDirs(base) { lower := strings.ToLower(name) rel := name if base != "." { rel = path.Join(base, name) } - if ent.IsDir() { - switch lower { - case "issue_template", "issue_templates": - e.collectTemplates(rel, &t.Issue, &t.Config) - case "pull_request_template", "merge_request_templates": - e.collectTemplates(rel, &t.PullRequest, nil) - } - continue + switch lower { + case "issue_template", "issue_templates": + e.collectTemplates(rel, &t.Issue, &t.Config) + case "pull_request_template", "merge_request_templates": + e.collectTemplates(rel, &t.PullRequest, nil) } - if !templateExts[path.Ext(lower)] || !e.isTracked(rel) { + } + for _, name := range e.dirFiles(base) { + lower := strings.ToLower(name) + if !templateExts[path.Ext(lower)] { continue } + rel := name + if base != "." { + rel = path.Join(base, name) + } switch strings.TrimSuffix(lower, path.Ext(lower)) { case "issue_template": t.Issue = append(t.Issue, rel) @@ -1790,23 +1774,12 @@ func (e *Engine) detectTemplates() *brief.TemplateInfo { // collectTemplates lists template files in dir, separating the issue chooser // config.yml from actual templates. func (e *Engine) collectTemplates(dir string, into *[]string, config *string) { - entries, err := os.ReadDir(filepath.Join(e.Root, filepath.FromSlash(dir))) - if err != nil { - return - } - for _, ent := range entries { - if ent.IsDir() { - continue - } - name := ent.Name() + for _, name := range e.dirFiles(dir) { lower := strings.ToLower(name) if !templateExts[path.Ext(lower)] { continue } rel := path.Join(dir, name) - if !e.isTracked(rel) { - continue - } if config != nil && (lower == "config.yml" || lower == "config.yaml") { if *config == "" { *config = rel @@ -1852,18 +1825,13 @@ type skillFrontmatter struct { // detectSkills looks for agent skill definitions the project provides. func (e *Engine) detectSkills() []brief.Skill { var skills []brief.Skill + e.loadProjectFiles() for _, glob := range []string{"skills/*/SKILL.md", ".claude/skills/*/SKILL.md"} { - matches, err := filepath.Glob(filepath.Join(e.Root, filepath.FromSlash(glob))) - if err != nil { - continue - } - sort.Strings(matches) - for _, abs := range matches { - rel, err := filepath.Rel(e.Root, abs) - if err != nil { + for _, rel := range e.indexedFiles { + rel = filepath.ToSlash(rel) + if !matchPathPattern(glob, rel) { continue } - rel = filepath.ToSlash(rel) skills = append(skills, e.parseSkill(rel)) } } @@ -1910,16 +1878,26 @@ func (e *Engine) dirFiles(dir string) []string { if cached, ok := e.dirCache[dir]; ok { return cached } + e.loadProjectFiles() + names := directProjectNames(e.indexedFiles, dir) + e.dirCache[dir] = names + return names +} + +func (e *Engine) dirDirs(dir string) []string { + e.loadProjectFiles() + return directProjectNames(e.indexedDirs, dir) +} + +func directProjectNames(entries []string, dir string) []string { + dir = path.Clean(filepath.ToSlash(dir)) var names []string - entries, err := os.ReadDir(filepath.Join(e.Root, filepath.FromSlash(dir))) - if err == nil { - for _, ent := range entries { - if !ent.IsDir() { - names = append(names, ent.Name()) - } + for _, entry := range entries { + entry = filepath.ToSlash(entry) + if path.Dir(entry) == dir { + names = append(names, path.Base(entry)) } } - e.dirCache[dir] = names return names } @@ -1961,13 +1939,13 @@ func (e *Engine) detectPlatforms() *brief.PlatformInfo { func (e *Engine) parseCIMatrices(platforms *brief.PlatformInfo) { ci := e.KB.CIConfig.CI + e.loadProjectFiles() for _, fp := range ci.Files { - matches, err := filepath.Glob(filepath.Join(e.Root, fp.Pattern)) - if err != nil { - continue - } - for _, path := range matches { - e.parseCIWorkflow(path, ci.MatrixKeys, platforms) + for _, rel := range e.indexedFiles { + rel = filepath.ToSlash(rel) + if matchPathPattern(fp.Pattern, rel) { + e.parseCIWorkflow(filepath.Join(e.Root, filepath.FromSlash(rel)), ci.MatrixKeys, platforms) + } } } } @@ -2208,18 +2186,23 @@ func (e *Engine) git(dir string, args ...string) ([]byte, error) { // detectLineCount gets line counts using scc or tokei if available. func (e *Engine) detectLineCount(absPath string) *brief.LineCount { + e.loadProjectFiles() + if e.scanTruncated || e.scanDepthTruncated { + return nil + } + // Try scc first if _, err := exec.LookPath("scc"); err == nil { - cmd := exec.Command("scc", e.sccArgs(absPath)...) - if out, err := cmd.Output(); err == nil { + if out, timedOut, err := e.lineCounterOutput("scc", e.sccArgs(absPath)...); err == nil { return parseSCCOutput(out) + } else if timedOut { + return nil } } // Try tokei if _, err := exec.LookPath("tokei"); err == nil { - cmd := exec.Command("tokei", "--output", "json", absPath) - if out, err := cmd.Output(); err == nil { + if out, _, err := e.lineCounterOutput("tokei", "--output", "json", absPath); err == nil { return parseTokeiOutput(out) } } @@ -2227,6 +2210,17 @@ func (e *Engine) detectLineCount(absPath string) *brief.LineCount { return nil } +func (e *Engine) lineCounterOutput(name string, args ...string) ([]byte, bool, error) { + if e.LineCountTimeout == 0 { + out, err := exec.Command(name, args...).Output() + return out, false, err + } + ctx, cancel := context.WithTimeout(context.Background(), e.LineCountTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).Output() + return out, ctx.Err() != nil, err +} + func (e *Engine) sccArgs(absPath string) []string { excluded := make(map[string]bool) for dir := range defaultSkipDirs { diff --git a/detect/detect_test.go b/detect/detect_test.go index d851b2d..e34b737 100644 --- a/detect/detect_test.go +++ b/detect/detect_test.go @@ -7,6 +7,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/git-pkgs/brief" "github.com/git-pkgs/brief/kb" @@ -310,6 +311,35 @@ func TestResourceGroups(t *testing.T) { } } +func TestResourceDetectionHonorsScanLimit(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "README.md", "readme") + writeFile(t, dir, "LICENSE", "license") + + engine := New(loadKB(t), dir) + engine.ScanLimit = 1 + r, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if !r.Stats.ScanTruncated { + t.Fatal("expected scan truncation") + } + if r.Resources == nil { + t.Fatal("expected one indexed resource") + } + + indexed := make(map[string]bool, len(engine.indexedFiles)) + for _, file := range engine.indexedFiles { + indexed[filepath.ToSlash(file)] = true + } + for _, resource := range []string{r.Resources.Readme, r.Resources.License} { + if resource != "" && !indexed[resource] { + t.Errorf("resource %q was detected outside the bounded project index", resource) + } + } +} + func TestResourceCaseInsensitive(t *testing.T) { dir := t.TempDir() for _, p := range []string{"ReadMe.rst", "Security.MD", ".github/Code_Of_Conduct.md"} { @@ -528,6 +558,7 @@ name: excel-tools description: Generate spreadsheets --- `) + writeFile(t, dir, ".claude/skills/excel/helper.py", "print('helper')\n") writeFile(t, dir, "skills/empty/SKILL.md", "no frontmatter\n") engine := New(loadKB(t), dir) @@ -538,6 +569,9 @@ description: Generate spreadsheets if len(r.Skills) != 3 { t.Fatalf("expected 3 skills, got %d: %+v", len(r.Skills), r.Skills) } + if slices.Contains(languageNames(r), "Python") { + t.Errorf("hidden skill support file affected language detection: %v", languageNames(r)) + } byPath := map[string]brief.Skill{} for _, s := range r.Skills { @@ -590,6 +624,21 @@ func TestDetectSkillsNone(t *testing.T) { } } +func TestDetectSkillsHonorsHiddenDirectorySkip(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, ".claude/skills/example/SKILL.md", "skill\n") + + engine := New(loadKB(t), dir) + engine.SkipDirs = []string{".claude"} + r, err := engine.Run() + if err != nil { + t.Fatalf("Run: %v", err) + } + if r.Skills != nil { + t.Errorf("expected skipped skills to be omitted, got %+v", r.Skills) + } +} + func TestRubyPlatforms(t *testing.T) { r := rubyReport(t) if r.Platforms == nil { @@ -883,6 +932,28 @@ func TestPackageWorkspaceMemberDependencies(t *testing.T) { assertToolDetected(t, r, "library", "axios") } +func TestPackageWorkspaceHonorsScanDepth(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "package.json", `{ + "private": true, + "workspaces": ["packages/*"] +}`) + writeFile(t, dir, "packages/web/package.json", `{ + "dependencies": { + "axios": "^1.7.0" + } +}`) + + engine := New(loadKB(t), dir) + engine.ScanDepth = 1 + r, err := engine.Run() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + assertToolNotDetected(t, r, "library", "axios") +} + func TestPnpmWorkspaceMemberDependencies(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "package.json", `{"private": true}`) @@ -1477,6 +1548,107 @@ func TestScanDepthOverride(t *testing.T) { } } +func TestNewUsesDefaultScanBounds(t *testing.T) { + engine := New(loadKB(t), t.TempDir()) + if engine.ScanDepth != DefaultScanDepth { + t.Errorf("ScanDepth = %d, want %d", engine.ScanDepth, DefaultScanDepth) + } + if engine.ScanLimit != DefaultScanLimit { + t.Errorf("ScanLimit = %d, want %d", engine.ScanLimit, DefaultScanLimit) + } + if engine.LineCountTimeout != DefaultLineCountTimeout { + t.Errorf("LineCountTimeout = %s, want %s", engine.LineCountTimeout, DefaultLineCountTimeout) + } +} + +func TestRunRejectsNegativeScanBounds(t *testing.T) { + tests := []struct { + name string + set func(*Engine) + want string + }{ + {name: "depth", set: func(engine *Engine) { engine.ScanDepth = -1 }, want: "scan depth"}, + {name: "limit", set: func(engine *Engine) { engine.ScanLimit = -1 }, want: "scan limit"}, + { + name: "line count timeout", + set: func(engine *Engine) { engine.LineCountTimeout = -time.Second }, + want: "line count timeout", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + engine := New(loadKB(t), t.TempDir()) + tt.set(engine) + if _, err := engine.Run(); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("Run() error = %v, want an error containing %q", err, tt.want) + } + }) + } +} + +func TestRecursiveGlobHonorsDefaultDepth(t *testing.T) { + dir := t.TempDir() + writeProjectFile(t, dir, "pipelines/example/Snakefile", "rule all:\n") + deep := filepath.Join( + "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "asv.conf.json", + ) + writeProjectFile(t, dir, deep, "{}\n") + + engine := New(loadKB(t), dir) + if !engine.recursiveGlob("**/Snakefile") { + t.Fatal("expected nested Snakefile within the default depth") + } + if engine.recursiveGlob("**/asv.conf.json") { + t.Fatal("did not expect a file beyond the default depth") + } + + unlimited := New(loadKB(t), dir) + unlimited.ScanDepth = 0 + if !unlimited.recursiveGlob("**/asv.conf.json") { + t.Fatal("expected ScanDepth=0 to remove the depth bound") + } +} + +func TestRecursiveGlobHonorsScanLimit(t *testing.T) { + dir := t.TempDir() + writeProjectFile(t, dir, "nested/Snakefile", "rule all:\n") + + engine := New(loadKB(t), dir) + engine.ScanLimit = 1 + if engine.recursiveGlob("**/Snakefile") { + t.Fatal("did not expect a file beyond the scan entry limit") + } + if !engine.scanTruncated { + t.Fatal("expected the scan to report truncation") + } + if engine.scanEntries != engine.ScanLimit { + t.Fatalf("scan entries = %d, want %d", engine.scanEntries, engine.ScanLimit) + } + + unlimited := New(loadKB(t), dir) + unlimited.ScanLimit = 0 + if !unlimited.recursiveGlob("**/Snakefile") { + t.Fatal("expected ScanLimit=0 to remove the entry bound") + } +} + +func TestTrackedOnlyIgnoresDepthOfUntrackedDirectories(t *testing.T) { + dir := t.TempDir() + writeProjectFile(t, dir, "one/tracked.go", "package example\n") + writeProjectFile(t, dir, "one/two/untracked.go", "package example\n") + + engine := New(loadKB(t), dir) + engine.ScanDepth = 1 + engine.tracked = map[string]bool{filepath.Join("one", "tracked.go"): true} + engine.trackedDirs = map[string]bool{"one": true} + engine.loadProjectFiles() + + if engine.scanDepthTruncated { + t.Fatal("untracked directories must not mark a tracked-only scan as depth-truncated") + } +} + func languageNames(r *brief.Report) []string { names := make([]string, 0, len(r.Languages)) for _, l := range r.Languages { diff --git a/detect/special_file_unix_test.go b/detect/special_file_unix_test.go new file mode 100644 index 0000000..947c6d7 --- /dev/null +++ b/detect/special_file_unix_test.go @@ -0,0 +1,28 @@ +//go:build unix + +package detect + +import ( + "path/filepath" + "slices" + "syscall" + "testing" +) + +func TestProjectFileIndexExcludesNamedPipes(t *testing.T) { + dir := t.TempDir() + pipe := filepath.Join(dir, "blocked.toml") + if err := syscall.Mkfifo(pipe, 0o600); err != nil { + t.Fatal(err) + } + + engine := New(loadKB(t), dir) + engine.loadProjectFiles() + + if slices.Contains(engine.projectFiles, "blocked.toml") { + t.Fatal("named pipe was included in the project file index") + } + if engine.exactFileExists("blocked.toml") { + t.Fatal("named pipe was treated as a regular file") + } +} diff --git a/kb/kb.go b/kb/kb.go index 6b98178..6fa7bcd 100644 --- a/kb/kb.go +++ b/kb/kb.go @@ -5,12 +5,15 @@ import ( "embed" "fmt" "io/fs" + "path" "path/filepath" "strings" "github.com/BurntSushi/toml" ) +const maxToolPathSignals = 64 + // ToolDef is the parsed representation of a tool TOML file. type ToolDef struct { Tool ToolInfo `toml:"tool"` @@ -425,6 +428,9 @@ func (base *KnowledgeBase) Validate() error { } } for _, tool := range base.Tools { + if err := validateToolPaths(tool); err != nil { + return err + } for _, id := range tool.Security.Threats { if _, ok := base.Threats[id]; !ok { return fmt.Errorf("%s: [security].threats references unknown threat id %q", tool.Source, id) @@ -439,6 +445,113 @@ func (base *KnowledgeBase) Validate() error { return nil } +func validateToolPaths(tool *ToolDef) error { + total := len(tool.Detect.Files) + len(tool.Detect.ExcludeFiles) + + len(tool.Detect.FileContains) + len(tool.Detect.ExcludeFileContains) + + len(tool.Detect.KeyExists) + len(tool.Config.Files) + if tool.Config.Lockfile != "" { + total++ + } + if total > maxToolPathSignals { + return fmt.Errorf("%s: %d path signals exceeds limit of %d", tool.Source, total, maxToolPathSignals) + } + + checkAll := func(field string, patterns []string) error { + for _, pattern := range patterns { + if err := validatePathPattern(pattern); err != nil { + return fmt.Errorf("%s: %s pattern %q: %w", tool.Source, field, pattern, err) + } + } + return nil + } + if err := checkAll("detect.files", tool.Detect.Files); err != nil { + return err + } + if err := checkAll("detect.exclude_files", tool.Detect.ExcludeFiles); err != nil { + return err + } + if err := checkAll("config.files", tool.Config.Files); err != nil { + return err + } + if tool.Config.Lockfile != "" { + if err := checkAll("config.lockfile", []string{tool.Config.Lockfile}); err != nil { + return err + } + } + for pattern := range tool.Detect.KeyExists { + if HasGlobPattern(pattern) { + return fmt.Errorf("%s: detect.key_exists pattern %q must name a specific file", tool.Source, pattern) + } + if err := checkAll("detect.key_exists", []string{pattern}); err != nil { + return err + } + } + for pattern := range tool.Detect.FileContains { + if err := validateContentPath(tool.Source, "detect.file_contains", pattern); err != nil { + return err + } + } + for pattern := range tool.Detect.ExcludeFileContains { + if err := validateContentPath(tool.Source, "detect.exclude_file_contains", pattern); err != nil { + return err + } + } + return nil +} + +func validateContentPath(source, field, pattern string) error { + if err := validatePathPattern(pattern); err != nil { + return fmt.Errorf("%s: %s pattern %q: %w", source, field, pattern, err) + } + if strings.HasSuffix(pattern, "/") { + return fmt.Errorf("%s: %s pattern %q must name a specific file", source, field, pattern) + } + if !HasGlobPattern(pattern) { + return nil + } + if HasGlobPattern(path.Base(pattern)) { + return fmt.Errorf("%s: %s pattern %q must name a specific file", source, field, pattern) + } + return nil +} + +func validatePathPattern(pattern string) error { + if pattern == "" { + return fmt.Errorf("must not be empty") + } + if strings.Contains(pattern, "\\") { + return fmt.Errorf("must use forward slashes") + } + if path.IsAbs(pattern) { + return fmt.Errorf("must stay within the project root") + } + trimmed := strings.TrimSuffix(pattern, "/") + if trimmed == "" { + return fmt.Errorf("must not be empty") + } + cleaned := path.Clean(trimmed) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return fmt.Errorf("must stay within the project root") + } + doublestar := 0 + for _, segment := range strings.Split(trimmed, "/") { + if strings.Contains(segment, "**") { + if segment != "**" { + return fmt.Errorf("** must occupy a complete path segment") + } + doublestar++ + continue + } + if _, err := path.Match(segment, segment); err != nil { + return fmt.Errorf("invalid glob: %w", err) + } + } + if doublestar > 1 { + return fmt.Errorf("must not contain more than one ** segment") + } + return nil +} + // ToolsForCategory returns all tools matching a category. func (base *KnowledgeBase) ToolsForCategory(category string) []*ToolDef { return base.ByCategory[category] diff --git a/kb/kb_test.go b/kb/kb_test.go index 501666f..84937e6 100644 --- a/kb/kb_test.go +++ b/kb/kb_test.go @@ -155,6 +155,84 @@ func TestValidateRejectsUnknownExplicitThreat(t *testing.T) { } } +func TestValidateRejectsUnsafePathPatterns(t *testing.T) { + tests := []struct { + name string + pattern string + }{ + {name: "parent traversal", pattern: "../outside.toml"}, + {name: "partial doublestar", pattern: "nested/**file.toml"}, + {name: "multiple doublestars", pattern: "one/**/two/**/file.toml"}, + {name: "invalid glob", pattern: "nested/[file.toml"}, + {name: "backslash", pattern: `nested\file.toml`}, + {name: "root", pattern: "/"}, + {name: "relative root", pattern: "./"}, + {name: "normalized root", pattern: "nested/../"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + base := &kb.KnowledgeBase{Tools: []*kb.ToolDef{{ + Source: "knowledge/example.toml", + Detect: kb.DetectInfo{Files: []string{tt.pattern}}, + }}} + if err := base.Validate(); err == nil { + t.Fatalf("expected validation error for %q", tt.pattern) + } + }) + } +} + +func TestValidateRejectsBroadContentGlob(t *testing.T) { + base := &kb.KnowledgeBase{Tools: []*kb.ToolDef{{ + Source: "knowledge/example.toml", + Detect: kb.DetectInfo{ + FileContains: map[string][]string{"**/*": {"marker"}}, + }, + }}} + if err := base.Validate(); err == nil { + t.Fatal("expected validation error for broad content glob") + } +} + +func TestValidateRejectsContentDirectory(t *testing.T) { + base := &kb.KnowledgeBase{Tools: []*kb.ToolDef{{ + Source: "knowledge/example.toml", + Detect: kb.DetectInfo{ + FileContains: map[string][]string{"config/": {"marker"}}, + }, + }}} + if err := base.Validate(); err == nil { + t.Fatal("expected validation error for file_contains directory") + } +} + +func TestValidateRejectsKeyExistsGlob(t *testing.T) { + base := &kb.KnowledgeBase{Tools: []*kb.ToolDef{{ + Source: "knowledge/example.toml", + Detect: kb.DetectInfo{ + KeyExists: map[string][]string{"**/*.json": {"scripts.test"}}, + }, + }}} + if err := base.Validate(); err == nil { + t.Fatal("expected validation error for key_exists glob") + } +} + +func TestValidateRejectsExcessivePathSignals(t *testing.T) { + patterns := make([]string, 65) + for i := range patterns { + patterns[i] = "file.toml" + } + base := &kb.KnowledgeBase{Tools: []*kb.ToolDef{{ + Source: "knowledge/example.toml", + Detect: kb.DetectInfo{Files: patterns}, + }}} + if err := base.Validate(); err == nil { + t.Fatal("expected validation error for excessive path signals") + } +} + func TestTaxonomyEmpty(t *testing.T) { var empty kb.Taxonomy if !empty.Empty() { diff --git a/report/markdown.go b/report/markdown.go index dd086de..9518497 100644 --- a/report/markdown.go +++ b/report/markdown.go @@ -32,8 +32,8 @@ func Markdown(w io.Writer, r *brief.Report, verbose bool) { mdLines(w, r.Lines) mdEnrichment(w, r.Enrichment) - _, _ = fmt.Fprintf(w, "---\n\n%.1fms | %d files checked | %d/%d tools matched\n", - r.Stats.DurationMS, r.Stats.FilesChecked, r.Stats.ToolsMatched, r.Stats.ToolsChecked) + _, _ = fmt.Fprintf(w, "---\n\n%.1fms | %d files checked | %d/%d tools matched%s\n", + r.Stats.DurationMS, r.Stats.FilesChecked, r.Stats.ToolsMatched, r.Stats.ToolsChecked, scanStatus(r.Stats, " | ")) } func mdTruncatedList(w io.Writer, header string, items []string) { diff --git a/report/report.go b/report/report.go index fd163e7..64b3bf5 100644 --- a/report/report.go +++ b/report/report.go @@ -108,8 +108,15 @@ func Human(w io.Writer, r *brief.Report, verbose bool) { printLines(w, r.Lines) printEnrichment(w, r.Enrichment) - _, _ = fmt.Fprintf(w, "\n%.1fms %d files checked %d/%d tools matched\n", - r.Stats.DurationMS, r.Stats.FilesChecked, r.Stats.ToolsMatched, r.Stats.ToolsChecked) + _, _ = fmt.Fprintf(w, "\n%.1fms %d files checked %d/%d tools matched%s\n", + r.Stats.DurationMS, r.Stats.FilesChecked, r.Stats.ToolsMatched, r.Stats.ToolsChecked, scanStatus(r.Stats, " ")) +} + +func scanStatus(stats brief.Stats, prefix string) string { + if stats.ScanTruncated { + return prefix + "scan truncated" + } + return "" } func printTruncatedList(w io.Writer, header string, items []string) { diff --git a/report/report_test.go b/report/report_test.go index 93330de..2cf8621 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -30,6 +30,22 @@ func TestHumanLayout(t *testing.T) { } } +func TestScanTruncatedStatus(t *testing.T) { + r := &brief.Report{Stats: brief.Stats{ScanTruncated: true}} + + var human bytes.Buffer + Human(&human, r, false) + if !strings.Contains(human.String(), "scan truncated") { + t.Errorf("human report missing scan status\ngot:\n%s", human.String()) + } + + var markdown bytes.Buffer + Markdown(&markdown, r, false) + if !strings.Contains(markdown.String(), "scan truncated") { + t.Errorf("markdown report missing scan status\ngot:\n%s", markdown.String()) + } +} + func TestMarkdownLayout(t *testing.T) { r := &brief.Report{ Version: "dev",