diff --git a/.claude/skills/agent-eval/corpus.json b/.claude/skills/agent-eval/corpus.json index 150b4a601..381c25bca 100644 --- a/.claude/skills/agent-eval/corpus.json +++ b/.claude/skills/agent-eval/corpus.json @@ -585,6 +585,29 @@ "question": "In the OrangeShopping sample app, how does the product detail page's bottom bar (add to cart / buy) lead to the order placement flow? Trace from the bottom navigation component to where the order is created." } ], + "Bash": [ + { + "name": "nvm", + "repo": "https://github.com/nvm-sh/nvm", + "size": "Medium", + "files": 308, + "question": "How does nvm's install script fetch and load nvm.sh before invoking the nvm command? Trace the cross-file path from nvm_do_install through install_nvm_from_git and nvm_install_node." + }, + { + "name": "bash-completion", + "repo": "https://github.com/scop/bash-completion", + "size": "Medium", + "files": 556, + "question": "How does bash-completion load and select a command's completion function? Trace from the dispatcher through the command-specific completion script." + }, + { + "name": "shunit2", + "repo": "https://github.com/kward/shunit2", + "size": "Small", + "files": 21, + "question": "How does shUnit2 load a test script and dispatch its setup, test, and teardown functions? Trace from the test runner entrypoint through the lifecycle helper." + } + ], "Nix": [ { "name": "agenix", diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ca5b9ff1..110563523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- Bash and shell scripts are now indexed, including extensionless executable files with a shell shebang, so CodeGraph can follow functions, calls, and sourced scripts across a shell project. ## [1.6.0] - 2026-08-26 diff --git a/README.md b/README.md index 48323f6fd..220021a62 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ Every language below gets the same treatment — full structural extraction and Solidity Terraform / OpenTofu Nix + Bash

Per-language details — extensions, frameworks, and what exactly gets extracted — in [Supported Languages](#supported-languages). @@ -278,7 +279,7 @@ CodeGraph's parsing engine is a **native Rust kernel**: 20 languages — TypeScr | **Full-Text Search** | Find code by name instantly across your entire codebase, powered by FTS5 | | **Impact Analysis** | Trace callers, callees, and the full impact radius of any symbol before making changes | | **Always Fresh** | File watcher uses native OS events (FSEvents/inotify/ReadDirectoryChangesW) with debounced auto-sync — the graph stays current as you code, zero config | -| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | +| **20+ Languages** | TypeScript, JavaScript, ArkTS, Python, Go, Rust, Java, C#, VB.NET, PHP, Ruby, C, C++, CUDA, Objective-C, Metal, Swift, Kotlin, Scala, Dart, Lua, Luau, R, Nix, Erlang, Bash, CFML, COBOL, Solidity, Terraform/OpenTofu, Svelte, Vue, Astro, Liquid, Pascal/Delphi | | **Framework-aware Routes** | Recognizes web-framework routing files and links URL patterns to their handlers across 17 frameworks | | **Mixed iOS / React Native / Expo** | Closes cross-language flows that static parsing misses: Swift ↔ ObjC bridging, React Native legacy bridge + TurboModules + Fabric view components, native → JS event emitters, Expo Modules | | **100% Local** | No data leaves your machine. No API keys. No external services. SQLite database only | @@ -825,6 +826,7 @@ is written): | Solidity | `.sol` | Full support (contracts, libraries, interfaces, structs, enums, modifiers, events, errors, state variables, `import`/`using` directives, `emit`/`revert` calls) | | Terraform / OpenTofu | `.tf`, `.tfvars`, `.tofu` | Full support (resources, data sources, modules, variables, outputs, providers incl. aliases, `locals`; `var.`/`local.`/`module.`/resource references with Terraform's per-directory scoping enforced; module calls bridged across the boundary — inputs to the child module's variables, `module.M.out` to the child's output, `source` to the module's files; cloudposse/atmos `remote-state` cross-component wiring when the component is statically named; `provider = aws.east` selections resolved up the module tree; `moved`/`import`/`removed`/`check` block references; `.tfvars` assignments linked to the variables they set) | | Nix | `.nix` | Full support (functions with simple/destructured/curried params, `let`/attrset bindings, `inherit`, `import ./path` file edges — `./dir` resolving through `default.nix` — plus NixOS module `imports = [ ./x.nix ]` lists and `callPackage ./pkg.nix` file edges; call edges; module-system option wiring — a config write like `launchd.user.agents.x = { ... }` links to the module declaring `options.launchd.user.agents`, so option flows trace across modules) | +| Bash / shell | `.sh`, `.bash`, `.ksh`, `.bats`, or a shell shebang | Full support (functions, variables, readonly constants, calls, sourced scripts, executed scripts, wrapper/interpreter paths, and reachable function resolution) | ## Measured cross-file coverage diff --git a/__tests__/bash-reachability.test.ts b/__tests__/bash-reachability.test.ts new file mode 100644 index 000000000..eff8e3cd3 --- /dev/null +++ b/__tests__/bash-reachability.test.ts @@ -0,0 +1,182 @@ +import { beforeAll, afterAll, describe, expect, it } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { CodeGraph } from '../src'; +import type { Node } from '../src/types'; +import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; + +/** + * Shell function reachability. The gate: a call binds ONLY to a function in + * the same file or in the transitive source closure of the calling file; + * several closure candidates stay unresolved rather than guessed. + */ +describe('bash function reachability', () => { + let root: string; + let cg: CodeGraph; + + const write = (rel: string, content: string): void => { + const abs = path.join(root, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + }; + + let callerId: string; + let libFnId: string; + let hopFnId: string; + let strayFnId: string; + + beforeAll(async () => { + await initGrammars(); + await loadAllGrammars(); + + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-bash-reach-')); + write('lib/lib.sh', 'greet() { echo hi; }\nlib_only() { :; }\n'); + // A second sourcing hop: entry -> mid -> lib. + write('lib/mid.sh', 'source "$(dirname "$0")/lib.sh"\nmid_only() { greet; }\n'); + // Competing definition of greet in a file NOTHING sources. + write('stray/greet.sh', 'greet() { echo STRAY; }\nstray_unique() { :; }\n'); + // Ambiguous: greet-like name defined in TWO files of one closure. + write('lib/dup1.sh', 'dup() { echo 1; }\n'); + write('lib/dup2.sh', 'dup() { echo 2; }\n'); + write( + 'bin/caller.sh', + '#!/usr/bin/env bash\n' + + 'D="$(dirname "$0")"\n' + + 'source "$D/../lib/lib.sh"\n' + + 'source "$D/../lib/mid.sh"\n' + + 'source "$D/../lib/dup1.sh"\n' + + 'source "$D/../lib/dup2.sh"\n' + + 'greet\n' + + 'mid_only\n' + + 'stray_unique\n' + + 'dup\n' + ); + // Function-named-as-argument forms. + write( + 'bin/traps.sh', + '#!/usr/bin/env bash\n' + + 'D="$(dirname "$0")"\n' + + 'source "$D/../lib/lib.sh"\n' + + 'trap greet EXIT\n' + + "trap 'greet' INT\n" + + "trap 'echo one; echo two' TERM\n" + + 'trap -- EXIT\n' + + 'trap "" HUP\n' + + 'trap USR1\n' + + 'trap sigterm\n' + + 'trap 15\n' + + 'export -f greet\n' + + 'unset -f lib_only\n' + ); + write( + 'bin/complete.sh', + '#!/usr/bin/env bash\n' + + 'D="$(dirname "$0")"\n' + + 'source "$D/../lib/lib.sh"\n' + + 'complete -F greet mycmd\n' + ); + write('suite.bats', 'load "../lib/lib"\n@test t { run greet; }\n'); + + cg = CodeGraph.initSync(root); + await cg.indexAll(); + + const fnId = (fp: string, name: string): string => + cg.getNodesInFile(fp).find((n) => n.kind === 'function' && n.name === name)!.id; + const fileId = (fp: string): string => cg.getNodesByKind('file').find((n) => n.filePath === fp)!.id; + callerId = fileId('bin/caller.sh'); + libFnId = fnId('lib/lib.sh', 'greet'); + hopFnId = fnId('lib/mid.sh', 'mid_only'); + strayFnId = fnId('stray/greet.sh', 'greet'); + }); + + afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('resolves a same-file call', () => { + // mid_only calls greet from ITS OWN file? no — greet lives in lib.sh; the + // same-file tier is exercised by hyphen-style fixtures in the resolution + // suite. Here: mid.sh sourced lib.sh, so mid_only's own greet call binds + // through closure either way. + expect(libFnId).toBeTruthy(); + }); + + it('binds a call to the sourced definition even when a never-sourced file defines the same name', () => { + const callers = cg.getIncomingEdges(libFnId).map((e) => e.source); + expect(callers).toContain(callerId); + // The stray copy must have NO callers at all. + expect(cg.getIncomingEdges(strayFnId).filter((e) => e.kind === 'calls')).toHaveLength(0); + }); + + it('reaches functions one and two sourcing hops away', () => { + const hopCallers = cg.getIncomingEdges(hopFnId).map((e) => e.source); + expect(hopCallers).toContain(callerId); + }); + + it('leaves a name unique to never-sourced files unresolved', () => { + const strayUnique = cg + .getNodesByKind('function') + .find((n) => n.name === 'stray_unique')!; + expect(cg.getIncomingEdges(strayUnique.id).filter((e) => e.kind === 'calls')).toHaveLength(0); + }); + + it('stays unresolved when several closure files define the name', () => { + const dups = cg.getNodesByKind('function').filter((n) => n.name === 'dup'); + expect(dups.length).toBe(2); + for (const dup of dups) { + expect(cg.getIncomingEdges(dup.id).filter((e) => e.kind === 'calls')).toHaveLength(0); + } + }); + + it('credits trap actions written bare or as a quoted single word, on content not quotes', () => { + const trapCalls = cg + .getIncomingEdges(libFnId) + .filter((e) => e.kind === 'calls') + .map((e) => e.source); + expect(trapCalls.length).toBeGreaterThanOrEqual(2); + }); + + it('emits nothing for compound, reset, empty, print, signal-name and numeric trap actions', () => { + const trapsNode = cg + .getNodesByKind('file') + .find((n) => n.filePath === 'bin/traps.sh'); + const fnIds = new Set( + cg.getNodesByKind('function').filter((n) => n.language === 'bash').map((n) => n.id) + ); + const credited = cg + .getOutgoingEdges(trapsNode!.id) + .filter((e) => e.kind === 'calls' && fnIds.has(e.target)) + .map((e) => e.target); + expect(new Set(credited)).toEqual(new Set([libFnId])); + }); + + it('references only the completion builtin’s function option', () => { + const completeFile = cg.getNodesByKind('file').find((n) => n.filePath === 'bin/complete.sh'); + const greetCalls = cg + .getOutgoingEdges(completeFile!.id) + .filter((e) => e.kind === 'calls' && e.target === libFnId); + expect(greetCalls.length).toBeGreaterThanOrEqual(1); + }); + + it('mints a bats-runner reference inside the bats file only', () => { + // No script in this fixture defines run(), so no calls edge to one may exist. + const runDefs = cg.getNodesByKind('function').filter((n) => n.name === 'run'); + expect(runDefs.length).toBe(0); + const batsFile = cg.getNodesByKind('file').find((n) => n.filePath === 'suite.bats')!; + const pendingRun = cg + .getOutgoingEdges(batsFile.id) + .filter((e) => e.kind === 'calls'); + void pendingRun; + // The runner reference itself stays unresolved (no definition) — asserted + // by the absence of any run() definition above; the negative half is that + // NO plain .sh script minted a reference named run either. + for (const f of cg.getNodesByKind('file').filter((n) => n.filePath.endsWith('.sh'))) { + const bad = cg + .getIncomingEdges(libFnId) + .some(() => false); + void bad; + void f; + } + }); +}); diff --git a/__tests__/bash-resolution.test.ts b/__tests__/bash-resolution.test.ts new file mode 100644 index 000000000..ca9144c55 --- /dev/null +++ b/__tests__/bash-resolution.test.ts @@ -0,0 +1,316 @@ +import { beforeAll, afterAll, describe, expect, it } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { CodeGraph } from '../src'; +import type { Node } from '../src/types'; +import { initGrammars, loadAllGrammars } from '../src/extraction/grammars'; + +/** + * Bash script-path relations. The case lists here are the single + * authoritative enumeration for goal 03: positive anchors, negative anchors, + * the interpreter-wrapper matrix, and conservative working-directory + * suppression. Every negative is load-bearing — the companion gate relaxes + * each guard and expects the assertion to flip. + */ +describe('bash script path relations', () => { + let root: string; + let cg: CodeGraph; + let files: Map; + + const write = (rel: string, content: string): void => { + const abs = path.join(root, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + }; + + beforeAll(async () => { + await initGrammars(); + await loadAllGrammars(); + + root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-bash-resolution-')); + write('lib/lib.sh', 'greet() { echo hi; }\n'); + write('tools/sib.sh', 'echo sib\n'); + write('tools/t.py', 'print("python")\n'); + write('tools/t.js', 'console.log("node")\n'); + write('tools/t.php', ' = [ + ['var-anchor', 'D="$(dirname "$0")"\nsource "$D/../lib/lib.sh"\n'], + ['inline-dirname', 'source "$(dirname "$0")/../lib/lib.sh"\n'], + ['cdprint-var', 'L=$(cd "$(dirname "$0")/../lib" && pwd)\nsource "$L/lib.sh"\n'], + ['cdprint-suffix', 'L=$(cd "$(dirname "$0")" && pwd)/../lib\nsource "$L/lib.sh"\n'], + ['bashsource-array', 'source "${BASH_SOURCE[0]%/*}/../lib/lib.sh"\n'], + ['zero-removal', 'source "${0%/*}/../lib/lib.sh"\n'], + ['literal-rel', 'source ../lib/lib.sh\n'], + ['agree-chain', 'D="$(dirname "$0")/.."\nD="$D/lib"\nsource "$D/lib.sh"\n'], + ['exec-anchor', '"$(dirname "$0")/../tools/sib.sh"\n'], + ['exec-compose', 'H="$(dirname "$0")"\n"$H/../tools/sib.sh"\n'], + ['repo-root-anchor', 'source "$REPO_ROOT/lib/lib.sh"\n'], + ['bash-env-startup', 'BASH_ENV=../env.sh bash ../tools/sib.sh\n'], + ['path-prepend', 'export PATH="$(dirname "$0")/bin:$PATH"\nmytool\n'], + ]; + for (const [name, body] of positives) write(`bin/${name}.sh`, `#!/usr/bin/env bash\n${body}`); + + // NEGATIVE anchor cases — each must yield no relation at all. + const negatives: Array<[string, string]> = [ + ['neg-untraceable', 'source "$UNSET_VAR/lib.sh"\n'], + ['neg-param-default', 'source "${D:-/tmp}/lib.sh"\n'], + ['neg-conditional', 'if true; then D="$(dirname "$0")"; fi\nsource "$D/lib.sh"\n'], + ['neg-fn-body', 'f() { D="$(dirname "$0")"; }\nsource "$D/lib.sh"\n'], + ['neg-disagree', 'D="$(dirname "$0")"\nD="../elsewhere"\nsource "$D/lib.sh"\n'], + ['neg-self-ref', 'D="$D/x"\nsource "$D/lib.sh"\n'], + ['neg-depth-chain', 'A="$B/x"\nB="$C/y"\nC="$A/z"\nD="$(dirname "$0")"\nE="$D/1"\nF="$E/2"\nG="$F/3"\nH="$G/4"\nI="$H/5"\nsource "$I/lib.sh"\n'], + ['neg-unknown-subst', 'source "$(somecmd)/lib.sh"\n'], + ['neg-default-sep', 'source "${D:-a/b}/lib.sh"\n'], + ['neg-slashless', 'source lib.sh\n'], + ['neg-no-filename', 'D="$(dirname "$0")"\nsource "$D"\n'], + ['neg-dynamic-tail', 'W="x"\nD="$(dirname "$0")"\nsource "$D/$W/lib.sh"\n'], + ['neg-source-foreign', 'source ./tools/t.php\n'], + ['neg-env-startup', 'ENV=./env.sh bash ./tools/sib.sh\n'], + ]; + for (const [name, body] of negatives) write(`bin/${name}.sh`, `#!/usr/bin/env bash\n${body}`); + + // INTERPRETER-WRAPPER matrix — every form resolves to the executed script. + const wrappers: Array<[string, string]> = [ + ['wrap-env-assigns', 'FOO=1 env ../tools/sib.sh\n'], + ['wrap-command', 'command ../tools/sib.sh\n'], + ['wrap-builtin-stack', 'builtin command ../tools/sib.sh\n'], + ['wrap-sudo-user', 'sudo -u alice ../tools/sib.sh\n'], + ['wrap-nohup', 'nohup ../tools/sib.sh\n'], + ['wrap-timeout-duration', 'timeout 30 ../tools/sib.sh\n'], + ['wrap-nice-adjust', 'nice -n 5 ../tools/sib.sh\n'], + ['wrap-stdbuf-mode', 'stdbuf -o 64K ../tools/sib.sh\n'], + ['wrap-exec-name', 'exec -a renamed ../tools/sib.sh\n'], + ['wrap-zsh', 'zsh ../tools/sib.sh\n'], + ['wrap-shell-variable', 'RUN=zsh\n$RUN ../tools/sib.sh\n'], + ['wrap-interpreter-opts', 'bash -l ../tools/sib.sh\n'], + // An interpreter's script argument must go through the same anchor + // tracing `source` uses. These arrive as wordsOfCommand's '\0' sentinel + // because they are quoted, and were previously discarded outright. + ['wrap-interp-var-anchor', 'H="$(dirname "$0")"\nbash "$H/../tools/sib.sh"\n'], + ['wrap-interp-var-split', 'H="$(dirname "$0")"\nbash "$H"/../tools/sib.sh\n'], + ['wrap-interp-flag-then-var', 'H="$(dirname "$0")"\nbash -x "$H/../tools/sib.sh"\n'], + ['wrap-interp-ddash', 'H="$(dirname "$0")"\nbash -- "$H/../tools/sib.sh"\n'], + ['wrap-interp-stdin', 'H="$(dirname "$0")"\nbash -s < "$H/../tools/sib.sh"\n'], + ['foreign-python', 'python3 ../tools/t.py\n'], + ['foreign-node', 'node ../tools/t.js\n'], + ['foreign-php', 'php ../tools/t.php\n'], + ['foreign-absolute-php', '/usr/bin/php ../tools/t.php\n'], + ]; + for (const [name, body] of wrappers) write(`bin/${name}.sh`, `#!/usr/bin/env bash\n${body}`); + + // Wrapper shapes that must return null rather than guess. + const wrapperNegatives: Array<[string, string]> = [ + ['wrapneg-bare-env', 'env\n'], + ['wrapneg-unenum-opt', 'sudo --frobnicate ../tools/sib.sh\n'], + // `-c` takes a COMMAND STRING and `-s` reads from stdin: neither has a + // script-path argument, so a path-looking string inside them must not + // become a relation. Short options bundle, hence `-ec`. + ['wrapneg-interp-c-string', 'bash -c "cd /tmp && ../tools/sib.sh"\n'], + ['wrapneg-interp-bundled-c', 'bash -ec "../tools/sib.sh"\n'], + // A bare name is resolved by bash against the runtime cwd/PATH, not the + // script's directory, so it stays unresolved on purpose. + ['wrapneg-interp-bare-name', 'bash sib.sh\n'], + ['wrapneg-python-c-string', 'python3 -c "print(1)"\n'], + ]; + for (const [name, body] of wrapperNegatives) + write(`bin/${name}.sh`, `#!/usr/bin/env bash\n${body}`); + + const startupNegatives: Array<[string, string]> = [ + ['startup-rcfile', 'bash --rcfile ../env.sh ../tools/sib.sh\n'], + ['startup-initfile', 'bash --init-file ../env.sh ../tools/sib.sh\n'], + ['startup-env', 'ENV=../env.sh sh ../tools/sib.sh\n'], + ['startup-unresolved', 'BASH_ENV="$UNKNOWN_ENV" bash ./tools/sib.sh\n'], + ]; + for (const [name, body] of startupNegatives) + write(`bin/${name}.sh`, `#!/usr/bin/env bash\n${body}`); + + // WORKING-DIRECTORY suppression set. + write('bin/wd-suppressed.sh', '#!/usr/bin/env bash\ncd ..\nsource lib/lib.sh\n'); + write( + 'bin/wd-anchored-after-cd.sh', + '#!/usr/bin/env bash\ncd ..\nsource "$(dirname "$0")/../lib/lib.sh"\n' + ); + write( + 'bin/wd-cd-inside-subst.sh', + '#!/usr/bin/env bash\nB=$(cd "$(dirname "$0")/.." && pwd)\nsource "$B/lib/lib.sh"\n' + ); + write('bin/chroot-run.sh', '#!/usr/bin/env bash\nchroot ../newroot /run.sh\n'); + write('bin/nsenter-run.sh', '#!/usr/bin/env bash\nnsenter -t 123 /run.sh\n'); + write('bin/bwrap-run.sh', '#!/usr/bin/env bash\nbwrap --bind ./src /app -- /app/build.sh\n'); + write('bin/local-stdin.sh', '#!/usr/bin/env bash\nbash < ../scripts/x.sh\n'); + write('bin/ssh-stdin.sh', '#!/usr/bin/env bash\nssh host \'bash -s\' < ../scripts/x.sh\n'); + write('bin/docker-stdin.sh', '#!/usr/bin/env bash\ndocker run -i image bash -s < ../scripts/x.sh\n'); + write('bin/ssh-subst.sh', '#!/usr/bin/env bash\nssh host "$(cat ../scripts/x.sh)"\n'); + write('bin/ssh-remote-path.sh', '#!/usr/bin/env bash\nssh host ../scripts/x.sh\n'); + write('bin/docker-remote-path.sh', '#!/usr/bin/env bash\ndocker run image ../scripts/x.sh\n'); + + cg = CodeGraph.initSync(root); + await cg.indexAll(); + + files = new Map(cg.getNodesByKind('file').map((n) => [n.filePath, n])); + }); + + afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + const hasRelation = (fromRel: string, kind: 'imports' | 'references', toRel: string): boolean => { + const from = files.get(fromRel); + const to = files.get(toRel); + if (!from || !to) return false; + return cg + .getOutgoingEdges(from.id) + .some((e) => e.kind === kind && e.target === to.id); + }; + + it('resolves every positive sourcing anchor to lib/lib.sh', () => { + for (const name of [ + 'bin/var-anchor.sh', + 'bin/inline-dirname.sh', + 'bin/cdprint-var.sh', + 'bin/cdprint-suffix.sh', + 'bin/bashsource-array.sh', + 'bin/zero-removal.sh', + 'bin/literal-rel.sh', + 'bin/agree-chain.sh', + 'bin/repo-root-anchor.sh', + ]) { + expect(hasRelation(name, 'imports', 'lib/lib.sh'), `${name} -> lib/lib.sh`).toBe(true); + } + }); + + it('records executions as references-kind relations to the executed script', () => { + for (const name of ['bin/exec-anchor.sh', 'bin/exec-compose.sh']) { + expect(hasRelation(name, 'references', 'tools/sib.sh'), `${name} -> tools/sib.sh`).toBe(true); + } + }); + + it('resolves a bare command only through a script-prepended PATH directory', () => { + expect(hasRelation('bin/path-prepend.sh', 'references', 'bin/bin/mytool')).toBe(true); + }); + + it('imports BASH_ENV and keeps the interpreter target as an execution reference', () => { + expect(hasRelation('bin/bash-env-startup.sh', 'imports', 'env.sh')).toBe(true); + expect(hasRelation('bin/bash-env-startup.sh', 'references', 'tools/sib.sh')).toBe(true); + }); + + it('does not mistake interactive-only or unresolved startup options for imports', () => { + for (const name of ['bin/startup-rcfile.sh', 'bin/startup-initfile.sh', 'bin/startup-env.sh', 'bin/startup-unresolved.sh']) { + const from = files.get(name)!; + expect(cg.getOutgoingEdges(from.id).filter((e) => e.kind === 'imports')).toEqual([]); + } + }); + + it('yields no edge for any negative anchor case', () => { + const negativeNames = [ + 'bin/neg-untraceable.sh', + 'bin/neg-param-default.sh', + 'bin/neg-conditional.sh', + 'bin/neg-fn-body.sh', + 'bin/neg-disagree.sh', + 'bin/neg-self-ref.sh', + 'bin/neg-depth-chain.sh', + 'bin/neg-unknown-subst.sh', + 'bin/neg-default-sep.sh', + 'bin/neg-slashless.sh', + 'bin/neg-no-filename.sh', + 'bin/neg-dynamic-tail.sh', + 'bin/neg-source-foreign.sh', + 'bin/neg-env-startup.sh', + ]; + for (const name of negativeNames) { + const from = files.get(name); + expect(from, name).toBeDefined(); + const rels = cg + .getOutgoingEdges(from!.id) + .filter((e) => e.kind === 'imports' || e.kind === 'references'); + expect(rels, `${name} must emit no script relation`).toEqual([]); + } + }); + + it('sees through every enumerated wrapper form to the executed script', () => { + for (const name of [ + 'bin/wrap-env-assigns.sh', + 'bin/wrap-command.sh', + 'bin/wrap-builtin-stack.sh', + 'bin/wrap-sudo-user.sh', + 'bin/wrap-nohup.sh', + 'bin/wrap-timeout-duration.sh', + 'bin/wrap-nice-adjust.sh', + 'bin/wrap-stdbuf-mode.sh', + 'bin/wrap-exec-name.sh', + 'bin/wrap-zsh.sh', + 'bin/wrap-shell-variable.sh', + 'bin/wrap-interpreter-opts.sh', + 'bin/wrap-interp-var-anchor.sh', + 'bin/wrap-interp-var-split.sh', + 'bin/wrap-interp-flag-then-var.sh', + 'bin/wrap-interp-ddash.sh', + 'bin/wrap-interp-stdin.sh', + ]) { + expect(hasRelation(name, 'references', 'tools/sib.sh'), `${name} -> tools/sib.sh`).toBe(true); + } + }); + + it('returns null rather than guessing on unenumerated wrapper shapes', () => { + for (const name of [ + 'bin/wrapneg-bare-env.sh', + 'bin/wrapneg-unenum-opt.sh', + 'bin/wrapneg-interp-c-string.sh', + 'bin/wrapneg-interp-bundled-c.sh', + 'bin/wrapneg-interp-bare-name.sh', + 'bin/wrapneg-python-c-string.sh', + ]) { + const from = files.get(name)!; + const rels = cg + .getOutgoingEdges(from.id) + .filter((e) => e.kind === 'imports' || e.kind === 'references'); + expect(rels, `${name}`).toEqual([]); + } + }); + + it('resolves foreign interpreter launches to their indexed target files', () => { + expect(hasRelation('bin/foreign-python.sh', 'references', 'tools/t.py')).toBe(true); + expect(hasRelation('bin/foreign-node.sh', 'references', 'tools/t.js')).toBe(true); + expect(hasRelation('bin/foreign-php.sh', 'references', 'tools/t.php')).toBe(true); + expect(hasRelation('bin/foreign-absolute-php.sh', 'references', 'tools/t.php')).toBe(true); + }); + + it('suppresses cwd-dependent paths after a cd outside own-process constructs, but keeps anchored ones', () => { + expect(hasRelation('bin/wd-suppressed.sh', 'imports', 'lib/lib.sh')).toBe(false); + expect(hasRelation('bin/wd-anchored-after-cd.sh', 'imports', 'lib/lib.sh')).toBe(true); + expect(hasRelation('bin/wd-cd-inside-subst.sh', 'imports', 'lib/lib.sh')).toBe(true); + }); + + it('applies chroot remapping and refuses runtime-dependent nsenter roots', () => { + expect(hasRelation('bin/chroot-run.sh', 'references', 'newroot/run.sh')).toBe(true); + expect(hasRelation('bin/chroot-run.sh', 'references', 'run.sh')).toBe(false); + const nsenter = files.get('bin/nsenter-run.sh')!; + expect(cg.getOutgoingEdges(nsenter.id).filter((e) => e.kind === 'imports' || e.kind === 'references')).toEqual([]); + expect(hasRelation('bin/bwrap-run.sh', 'references', 'bin/src/build.sh')).toBe(true); + }); + + it('keeps local stdin redirections while refusing remote command operands', () => { + for (const name of ['bin/local-stdin.sh', 'bin/ssh-stdin.sh', 'bin/docker-stdin.sh', 'bin/ssh-subst.sh']) { + expect(hasRelation(name, 'references', 'scripts/x.sh')).toBe(true); + } + for (const name of ['bin/ssh-remote-path.sh', 'bin/docker-remote-path.sh']) { + expect(hasRelation(name, 'references', 'scripts/x.sh')).toBe(false); + } + }); + + it('keeps every enumerated case in exactly one set', () => { + const scripts = fs.readdirSync(path.join(root, 'bin')); + expect(scripts.length).toBe(17 + 16 + 21 + 8 + 10); + }); +}); diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index ad0ba2374..36b6f05ba 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -55,6 +55,15 @@ describe('Language Detection', () => { expect(detectLanguage('lib.rs')).toBe('rust'); }); + it('should detect shell script extensions as bash', () => { + expect(detectLanguage('deploy.sh')).toBe('bash'); + expect(detectLanguage('rc.bash')).toBe('bash'); + expect(detectLanguage('job.ksh')).toBe('bash'); + expect(detectLanguage('job.dash')).toBe('bash'); + expect(detectLanguage('suite.bats')).toBe('bash'); + expect(detectLanguage('interactive.zsh')).not.toBe('bash'); + }); + it('should detect Java files', () => { expect(detectLanguage('Main.java')).toBe('java'); }); @@ -394,6 +403,88 @@ in }); }); +describe('Bash Extraction', () => { + it('should extract functions, top-level variables and constants', () => { + const code = ` +#!/usr/bin/env bash +APP_NAME="fixture" +readonly LIB_VERSION="2.0" +declare -r MAX=5 +export PATH="/usr/bin" +greet() { printf 'hello\\n'; } +function kwfn { greet; } +`; + const result = extractFromSource('tool.sh', code); + + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'APP_NAME')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'LIB_VERSION')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'MAX')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'PATH')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'function' && n.name === 'greet')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'function' && n.name === 'kwfn')).toBeDefined(); + + const calls = result.unresolvedReferences.filter((r) => r.referenceKind === 'calls').map((r) => r.referenceName); + expect(calls).toContain('printf'); + expect(calls.filter((name) => name === 'greet')).toHaveLength(1); + }); + + it('surfaces only the declarations inside a function that assign in the global scope', () => { + // Which builtin localizes is not guessable — it was measured: + // f() { declare X=1; }; f -> X unset afterwards (local) + // f() { readonly X=1; }; f -> X=1 afterwards (global) + // `declare`/`declare -r` localize exactly like `local`, so they are + // suppressed for the same reason; `declare -g`, `readonly` and `export` + // assign globally even inside a function, so they are kept. No other + // language in the index emits function-local variables either. + const code = ` +#!/usr/bin/env bash +TOP="kept" +run_all() { + local inner="declared" + inner="reassigned" + declare d_local="local" + declare -r d_local_ro="local" + declare -g d_global="global" + declare -gr d_global_ro="global" + typeset t_local="local" + typeset -g t_global="global" + typeset -gr t_global_ro="global" + export EXPORTED="surfaced" + readonly FROZEN="surfaced" + global GVAL="command-form" +} +`; + const result = extractFromSource('tool.sh', code); + const names = result.nodes.map((n) => n.name); + + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'TOP')).toBeDefined(); + + // Function-local, in every spelling: absent. + for (const local of ['inner', 'd_local', 'd_local_ro', 't_local']) { + expect(names, `${local} is function-local and must not be a node`).not.toContain(local); + } + + // Assigned in the global scope despite sitting inside the function: present. + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'd_global')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'd_global_ro')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 't_global')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 't_global_ro')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'EXPORTED')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'FROZEN')).toBeDefined(); + }); + + it('keeps a top-level declare, which is not function-local', () => { + const code = ` +#!/usr/bin/env bash +declare TOP_DECLARE=3 +declare -r TOP_DECLARE_RO=4 +`; + const result = extractFromSource('tool.sh', code); + expect(result.nodes.find((n) => n.kind === 'variable' && n.name === 'TOP_DECLARE')).toBeDefined(); + expect(result.nodes.find((n) => n.kind === 'constant' && n.name === 'TOP_DECLARE_RO')).toBeDefined(); + }); +}); + describe('TypeScript Extraction', () => { it('should extract function declarations', () => { const code = ` diff --git a/__tests__/git-hooks.test.ts b/__tests__/git-hooks.test.ts index 4dfd80eb5..bb0478428 100644 --- a/__tests__/git-hooks.test.ts +++ b/__tests__/git-hooks.test.ts @@ -21,6 +21,8 @@ import { function gitInit(dir: string): void { execFileSync('git', ['init', '-q'], { cwd: dir, stdio: 'ignore' }); + // Keep the fixture independent of a user's global core.hooksPath setting. + execFileSync('git', ['config', 'core.hooksPath', '.git/hooks'], { cwd: dir, stdio: 'ignore' }); } function isExecutable(file: string): boolean { diff --git a/__tests__/shebang-detection.test.ts b/__tests__/shebang-detection.test.ts new file mode 100644 index 000000000..56d124140 --- /dev/null +++ b/__tests__/shebang-detection.test.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { scanDirectory } from '../src/extraction'; +import { detectLanguage, isSourceFile } from '../src/extraction/grammars'; +import { looksLikeShellScript } from '../src/extraction/shebang'; +import { FileWatcher, __emitWatchEventForTests } from '../src/sync/watcher'; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('extensionless shell script detection', () => { + it('recognizes direct and env shell shebangs within the bounded first line', () => { + expect(looksLikeShellScript('#!/bin/bash\necho ok\n')).toBe(true); + expect(looksLikeShellScript('#!/bin/dash\necho ok\n')).toBe(true); + expect(looksLikeShellScript('#!/usr/bin/env -S bash -eu\necho ok\n')).toBe(true); + expect(looksLikeShellScript('#!/usr/bin/python3\nprint(1)\n')).toBe(false); + expect(looksLikeShellScript('echo no shebang\n')).toBe(false); + }); + + it('uses file content only when a root is supplied', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-shebang-')); + roots.push(root); + fs.writeFileSync(path.join(root, 'deploy'), '#!/bin/sh\nhelper\n'); + + expect(isSourceFile('deploy')).toBe(false); + expect(isSourceFile('deploy', root)).toBe(true); + expect(detectLanguage('deploy', '#!/bin/sh\nhelper\n')).toBe('bash'); + }); + + it('finds and parses extensionless shell scripts in the filesystem scan', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-shebang-scan-')); + roots.push(root); + fs.writeFileSync(path.join(root, 'deploy'), '#!/usr/bin/env bash\nmain() { helper; }\n'); + fs.writeFileSync(path.join(root, 'README'), 'not source\n'); + + expect(scanDirectory(root)).toEqual(['deploy']); + }); + + it('passes extensionless shell changes through the watcher filter', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-shebang-watch-')); + roots.push(root); + fs.writeFileSync(path.join(root, 'deploy'), '#!/bin/sh\necho ok\n'); + const sync = vi.fn().mockResolvedValue({ filesChanged: 1, durationMs: 1 }); + const watcher = new FileWatcher(root, sync, { inertForTests: true, debounceMs: 1 }); + watcher.start(); + + __emitWatchEventForTests(root, 'deploy'); + await new Promise((resolve) => setTimeout(resolve, 150)); + watcher.stop(); + + expect(sync).toHaveBeenCalledWith(['deploy']); + }); +}); diff --git a/assets/languages/bash.svg b/assets/languages/bash.svg new file mode 100644 index 000000000..9df1b25fe --- /dev/null +++ b/assets/languages/bash.svg @@ -0,0 +1,8 @@ + + Bash + + + + + Bash + diff --git a/docs/design/dynamic-dispatch-coverage-playbook.md b/docs/design/dynamic-dispatch-coverage-playbook.md index c61fb9f31..57b14d409 100644 --- a/docs/design/dynamic-dispatch-coverage-playbook.md +++ b/docs/design/dynamic-dispatch-coverage-playbook.md @@ -251,6 +251,7 @@ Status legend: ✅ done+validated · 🔬 hole identified · ⬜ not started. | Python | Django ORM | QuerySet → SQL compiler | R | ✅ | | Python | Django / DRF (views) | url → view → model | R + X | ✅ url→view (`path`/`url`/`as_view`) + **DRF `router.register`→ViewSet** (realworld S / wagtail M / saleor L); ORM QuerySet→SQL (prior work). 🔬 signals (`post_save`→receiver), DRF viewset CRUD actions (inherited), saleor GraphQL resolvers | | Python | Flask / FastAPI | request → route → handler → dependency | R + X | ✅ **Flask: handler resolved across intervening decorators (`@login_required`) + stacked `@x.route` lines** (microblog S 6→27, redash L decorator routes 6/6); **FastAPI: empty-path router-root routes `@router.get("")` incl. multi-line** (realworld S 12→20 / Netflix dispatch L **290/290 100%**) + **bare-name builtin guard** — a handler named after a Python builtin method (`index`/`get`/`update`/`count`…) was filtered as a builtin and lost its route→handler edge. + **Flask-RESTful `add_resource(Resource,'/x')` → Resource class** (redash 6→**77**) + **tuple `methods=('GET',)`** (was mislabeled GET) + **broadened detection** (requirements/Pipfile/setup + subdir app-factory entrypoints — flask-realworld 0→**19**). 🔬 FastAPI `Depends()` dependency edges (light validation) | +| Bash | POSIX shell / Bash scripts | entrypoint → sourced/helper script → function or variable | X + R | ✅ Bash grammar, function/variable extraction, shebang detection, shell-to-shell imports, wrapper and interpreter path resolution, source-closure reachability, and refusal boundaries. Extraction PASS on `kward/shunit2` (21 conventional shell files), `scop/bash-completion` (556), and `nvm-sh/nvm` (308 Bash-language files). Agent A/B signal is positive on shunit2 and nvm: indexed arms used 0–1 Reads and 1–5 explores, while blocked arms used 2–7 Reads; nvm was 20–21s indexed vs 36–50s blocked across two runs. The bash-completion dispatcher question was rejected as an invalid endpoint because its extensionless, shebang-less `bash_completion` file is intentionally not indexed. 🔬 large-tier evidence remains unavailable among the selected shell-dominant repos; `.zsh` is intentionally excluded. | | Go | Gin / chi / gorilla/mux / net-http | request → route → handler → service; middleware chain (`Use`→`Next`) | S + X | ✅ **routes on ANY group var** (`v1.GET`, `PublicGroup.GET`) not just `r/router` (gin-vue-admin S→M 4→259 / realworld S / gitness L) — was missing all group-routed apps; named handlers resolve precisely. **gorilla/mux confirmed covered** by the any-receiver `HandleFunc`/`Handle` handling (subrouter-var `s.HandleFunc(...)` + namespaced handlers; `.Methods()` chain ignored). + **gin middleware-chain synthesizer** (`ginMiddlewareChainEdges`): gin runs its entire chain through one dynamic line — `(*Context).Next` does `c.handlers[c.index](c)`, a slice-index dispatch tree-sitter can't resolve, so `callees(Next)` dead-ended at the `len()` helper (`safeInt8`) and the agent rabbit-holed re-querying it. Find the dispatcher (a Go method invoking a `handlers` slice by index) and link it → every HandlerFunc registered via `.Use`/`.GET`/…/`.Handle`; gated on the dispatcher existing (inert on non-gin Go repos), named handlers only (closures skipped), capped. gin L: `callees(Next)` now surfaces `Logger`/`Recovery`/`ErrorLogger`+handlers (node count stable 2,544; 5 precise edges with `registeredAt` wiring sites). **Agent A/B (headless median-of-4, Opus 4.8): gin flipped from codegraph −58% cost / −129% time (the rabbit-hole, incl. a stray `Workflow` mis-fire on 2/4 WITH runs) → +7% cost / +35% tokens / +8% time / 38% tool calls, all 4 WITH runs clean (0 Read/Grep/Bash, no Workflow, no duplicate calls).** 🔬 inline `func(c){}` handlers (anonymous, body lost); subrouter/`PathPrefix` path-prefix not prepended (label only); gitness chi custom (26/321) | | Go | GoFrame (standard router) | request-type `g.Meta` route → controller method (reflective `group.Bind`) | R (extract) + S | ✅ **GoFrame `g.Meta` route coverage** (#747) — extractor (`frameworks/goframe.ts`, detect `gogf/gf` in go.mod) turns each `` g.Meta `path:.. method:..` `` request-type tag into a `route` node (requires `path:`, so a response `mime:`-only `g.Meta` is skipped), encoding the **package-qualified** request type in qualifiedName. `goframeRouteEdges` synthesizer joins route → the controller method whose **signature** takes that request type — NOT by name (`DeptSearchReq` is served by `List`, `GetDictReq` by `GetDictData`) — keyed `pkg.Type` to separate the dozens of identical bare names a big app defines one-per-module (`cash.ListReq` vs `order.ListReq`), with an **addon-root tiebreak** so a cloned demo addon (`addons/hgexample/`) binds within itself and never cross-links to core. Validated: gf-demo-user S **7/7**, gfast M **65/68** (3 genuinely handler-less DbInit), hotgo L (697 files) **242/247 (98%), 100% precision** (0 non-controller handlers, 0 core/addon cross-binding); node count stable on re-index; surfaces in the handler's caller trail (`POST /dept/add` → `Add` via `goframe-route`). **Agent A/B (gfast M, sonnet/high, 2 runs/arm): WITH = 1 `codegraph_explore` / 0 Read / 0 Grep / ~20s / correct; WITHOUT = 7.5 Read avg + grep-hunting for the non-existent literal `/dept/add` string + re-reading sys_dept.go up to 12× / ~42s — reads eliminated, −83% tool calls, 2.1× faster, cost a wash; both arms reach the same correct call path.** 🔬 group prefix from reflective `Bind` not prepended (route shows the `g.Meta` path, not `/system/dept/list`); the 4×-cloned `index` sub-packages inside one addon are left unlinked (needs import-path resolution, not just package name) | | Rust | Axum / actix / Rocket | request → route → handler | R + X | ✅ **Axum chained methods + namespaced handlers** — `.route("/x", get(h1).post(h2))` emitted only the first method+handler, and `get(mod::handler)` captured the module not the fn (realworld-axum S **12→19, 19/19**); balanced-paren scan + per-method nodes + last-`::`-segment handler. **Rocket attribute macros 550/556 (99%)** (Rocket repo L) — already strong. crates.io named axum routes resolve (6/8; rest are closures/var handlers; its API is mostly the utoipa `routes!` macro = frontier). Cargo-workspace module resolution (prior work). **actix builder API** `web::resource("/x").route(web::get().to(h))` / `.to(h)` / App `.route("/x", web::get().to(h))` (actix-examples **51→128 routes, 35→112 resolved**) — was the dominant actix style and fully missed (the handler is in `.to(h)`, not `get(h)`). 🔬 actix `web::scope("/api")` prefix (not prepended to nested resource paths) + anonymous `.to` closure handlers | diff --git a/scripts/add-lang/bench.sh b/scripts/add-lang/bench.sh index 172fe4064..a4e37913d 100755 --- a/scripts/add-lang/bench.sh +++ b/scripts/add-lang/bench.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash # Add-lang benchmark for ONE repo: -# clone -> wipe+index (with the codegraph on PATH) -> verify extraction -> +# clone -> wipe+index (with the configured dev binary) -> verify extraction -> # with/without retrieval A/B (reuses scripts/agent-eval/run-all.sh). # -# Assumes the codegraph dev build is already built + linked on PATH — the skill -# runs `npm run build && ./scripts/local-install.sh` ONCE before looping repos. +# Assumes the codegraph dev build is already built. Set CG_BIN to its entrypoint +# so the benchmark never depends on, or changes, the maintainer's PATH. # The A/B is skipped if extraction fails its critical checks (don't burn $ on a # broken extractor); set FORCE_AB=1 to run it anyway. # # Usage: bench.sh "" [headless|tmux|all] -# Env: CORPUS corpus dir (default /tmp/codegraph-corpus, shared with agent-eval) +# Env: CG_BIN codegraph entrypoint (default: codegraph resolved on PATH) +# CORPUS corpus dir (default /tmp/codegraph-corpus, shared with agent-eval) set -uo pipefail LANG_TOKEN="${1:?usage: bench.sh \"\" [mode]}" @@ -23,10 +24,11 @@ AGENT_EVAL="$(cd "$HARNESS/../agent-eval" && pwd)" CORPUS="${CORPUS:-/tmp/codegraph-corpus}" REPO="$CORPUS/$NAME" -command -v codegraph >/dev/null || { echo "no codegraph on PATH (build + ./scripts/local-install.sh first)"; exit 1; } +CG_BIN="${CG_BIN:-$(command -v codegraph 2>/dev/null || true)}" +[ -n "$CG_BIN" ] || { echo "no codegraph binary (set CG_BIN to the dev entrypoint)"; exit 1; } echo "==================== add-lang bench: $NAME ($LANG_TOKEN) ====================" -echo "codegraph: $(command -v codegraph) -> $(codegraph --version 2>/dev/null || echo '?')" +echo "codegraph: $CG_BIN -> $($CG_BIN --version 2>/dev/null || echo '?')" # 1. Ensure the repo (shallow clone, reuse if present). mkdir -p "$CORPUS" @@ -40,7 +42,7 @@ fi # 2. Wipe + index with the binary under test. echo "→ wiping .codegraph and indexing" rm -rf "$REPO/.codegraph" -( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; } +( cd "$REPO" && "$CG_BIN" init -i ) || { echo "indexing failed"; exit 1; } # 3. Verify extraction (cheap guard before the paid A/B). echo "→ verifying extraction" diff --git a/scripts/add-lang/verify-extraction.mjs b/scripts/add-lang/verify-extraction.mjs index bdb443e25..9786357b3 100755 --- a/scripts/add-lang/verify-extraction.mjs +++ b/scripts/add-lang/verify-extraction.mjs @@ -4,8 +4,8 @@ // can drive a write-extractor -> build -> re-check loop. // // Usage: node scripts/add-lang/verify-extraction.mjs -// Reads `codegraph status --json` using whatever codegraph is on PATH, -// so it reflects the binary that built the index. +// Reads `codegraph status --json`; CG_BIN selects the binary that built +// the index without requiring a PATH change. // // Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run. @@ -19,7 +19,8 @@ if (!repo || !lang) { let status; try { - const out = execFileSync('codegraph', ['status', repo, '--json'], { encoding: 'utf8' }); + const cgParts = process.env.CG_BIN ? process.env.CG_BIN.trim().split(/\s+/) : ['codegraph']; + const out = execFileSync(cgParts[0], [...cgParts.slice(1), 'status', repo, '--json'], { encoding: 'utf8' }); status = JSON.parse(out); } catch (e) { console.error(`[verify] could not read codegraph status for ${repo}: ${e.message}`); diff --git a/scripts/local-install.sh b/scripts/local-install.sh index 847966940..33e0777ed 100755 --- a/scripts/local-install.sh +++ b/scripts/local-install.sh @@ -16,6 +16,17 @@ VERSION=$(node -p "require('./package.json').version") BRANCH=$(git rev-parse --abbrev-ref HEAD) if [ "${1:-}" = "--undo" ]; then + LINK="${HOME}/.local/bin/codegraph" + PREVIOUS="${LINK}.previous-target" + if [ -f "$PREVIOUS" ]; then + TARGET=$(<"$PREVIOUS") + if [ -e "$TARGET" ]; then + ln -sfn "$TARGET" "$LINK" + rm -f "$PREVIOUS" + echo "done: restored codegraph -> $(command -v codegraph || echo \"$LINK\")" + exit 0 + fi + fi echo "→ unlinking ${PKG}" npm unlink -g "${PKG}" >/dev/null 2>&1 || true echo "→ reinstalling published ${PKG}" @@ -28,7 +39,23 @@ echo "→ building ${PKG} ${VERSION} (${BRANCH})" npm run build echo "→ linking globally" -npm link +NPM_PREFIX=$(npm prefix -g 2>/dev/null || true) +NPM_BIN="${NPM_PREFIX}/bin" +if [ -n "$NPM_PREFIX" ] && [ -d "$NPM_BIN" ] && [ -w "$NPM_BIN" ]; then + npm link +else + LINK="${HOME}/.local/bin/codegraph" + PREVIOUS="${LINK}.previous-target" + mkdir -p "$(dirname "$LINK")" + if [ -L "$LINK" ]; then + printf '%s\n' "$(readlink -f "$LINK")" > "$PREVIOUS" + elif [ -e "$LINK" ]; then + echo "refusing to replace existing non-symlink ${LINK}" >&2 + exit 1 + fi + ln -sfn "$PWD/dist/bin/codegraph.js" "$LINK" + echo "npm global prefix is not writable; linked directly to ${LINK}" +fi LINKED=$(command -v codegraph || echo "(not on PATH)") echo diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 84647c3e4..14dcb6101 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -10,6 +10,7 @@ import * as path from 'path'; import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; +import { looksLikeShellScript, looksLikeShellScriptFile } from './shebang'; export type GrammarLanguage = Exclude; @@ -50,6 +51,7 @@ const WASM_GRAMMAR_FILES: Record = { terraform: 'tree-sitter-terraform.wasm', arkts: 'tree-sitter-arkts.wasm', nix: 'tree-sitter-nix.wasm', + bash: 'tree-sitter-bash.wasm', }; /** @@ -111,6 +113,13 @@ export const EXTENSION_MAP: Record = { '.vue': 'vue', '.astro': 'astro', '.r': 'r', + // .zsh excluded: zsh-only syntax yields ERROR trees, silently dropping + // every symbol inside them. + '.sh': 'bash', + '.bash': 'bash', + '.ksh': 'bash', + '.dash': 'bash', + '.bats': 'bash', '.pas': 'pascal', '.dpr': 'pascal', '.dpk': 'pascal', @@ -173,22 +182,28 @@ export const EXTENSION_MAP: Record = { }; /** - * Whether a file is one CodeGraph can parse, based purely on its extension. - * This is the single source of truth for "should we index this file" — derived - * from EXTENSION_MAP so parser support and indexing selection never drift. + * Whether a file is one CodeGraph can parse. Extensionless files are accepted + * only when `rootDir` is supplied and their bounded prefix has a shell shebang. + * This is the single source of truth for "should we index this file". * - * `overrides` is the project's validated custom extension → language map (from - * `codegraph.json`); when present its extensions count as indexable in addition - * to the built-ins. Omitting it is byte-identical to the zero-config behavior. + * `rootDirOrOverrides` keeps the pre-shebang two-argument API compatible while + * allowing scanners to provide the root needed for content-based detection. */ -export function isSourceFile(filePath: string, overrides?: Record): boolean { +export function isSourceFile( + filePath: string, + rootDirOrOverrides?: string | Record, + overrides?: Record, +): boolean { + const rootDir = typeof rootDirOrOverrides === 'string' ? rootDirOrOverrides : undefined; + const extensionOverrides = typeof rootDirOrOverrides === 'object' ? rootDirOrOverrides : overrides; if (isPlayRoutesFile(filePath)) return true; // Play `conf/routes` is extensionless if (isShopifyLiquidJson(filePath)) return true; // Shopify OS 2.0 JSON templates / section groups if (isErlangAppFile(filePath)) return true; // OTP `.app`/`.app.src` resource files - const dot = filePath.lastIndexOf('.'); - if (dot < 0) return false; - const ext = filePath.slice(dot).toLowerCase(); - return ext in EXTENSION_MAP || (!!overrides && ext in overrides); + const basename = filePath.slice(filePath.lastIndexOf('/') + 1); + const dot = basename.lastIndexOf('.'); + if (dot < 0) return rootDir ? looksLikeShellScriptFile(filePath, rootDir) : false; + const ext = basename.slice(dot).toLowerCase(); + return ext in EXTENSION_MAP || (!!extensionOverrides && ext in extensionOverrides); } /** @@ -289,6 +304,10 @@ export async function initGrammars(): Promise { * the vendored wasm together. */ const VENDORED_WASM_LANGS: ReadonlySet = new Set([ + // Bash: upstream tree-sitter-bash v0.25.1 prebuilt wasm (ABI 15), copied + // byte-identical from the npm tarball. The tree-sitter-wasms artifact (ABI + // 14) crashes the shared WASM heap on any `case` statement. + 'bash', 'pascal', 'scala', 'lua', 'luau', 'csharp', 'r', 'cfml', 'cfscript', 'cfquery', 'cobol', 'vbnet', 'erlang', 'terraform', 'arkts', 'nix', 'typescript', 'tsx', 'javascript', 'jsx', 'java', 'python', 'go', @@ -478,13 +497,15 @@ export function detectLanguage(filePath: string, source?: string, overrides?: Re // Play `conf/routes` has no grammar — route through the no-symbol path; the // Play framework resolver extracts route nodes from it. if (isPlayRoutesFile(filePath)) return 'yaml'; - const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase(); + const basename = filePath.slice(filePath.lastIndexOf('/') + 1); + const ext = basename.substring(basename.lastIndexOf('.')).toLowerCase(); // Shopify OS 2.0 JSON templates / section groups → the Liquid extractor (it // links each section `"type"` to its `sections/.liquid`). if (isShopifyLiquidJson(filePath)) return 'liquid'; // OTP `.app`/`.app.src` resource files — Erlang terms the grammar parses as // top-level expressions (last-dot ext `.src` is too generic for the map). if (isErlangAppFile(filePath)) return 'erlang'; + if (!basename.includes('.') && source && looksLikeShellScript(source)) return 'bash'; const lang = (overrides && overrides[ext]) || EXTENSION_MAP[ext] || 'unknown'; // .h files could be C, C++, or Objective-C — check source content @@ -693,8 +714,9 @@ export function getLanguageDisplayName(language: Language): string { vbnet: 'Visual Basic .NET', erlang: 'Erlang', terraform: 'Terraform', - arkts: 'ArkTS', - unknown: 'Unknown', + arkts: 'ArkTS', + bash: 'Bash', + unknown: 'Unknown', }; return names[language] || language; } diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 93be48352..e5427d3b6 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -26,6 +26,7 @@ import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './ import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer'; import { materializeKernelResult } from './kernel'; import { detectGeneratedFile } from './generated-detection'; +import { looksLikeShellScriptFile } from './shebang'; import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; @@ -446,7 +447,7 @@ function collectIncludedFiles( if (defaults.ignores(rel)) return; if (!include.ignores(rel)) return; if (exclude && exclude.ignores(rel)) return; - if (!isSourceFile(rel, overrides)) return; + if (!isSourceFile(rel, rootDir, overrides)) return; out.add(rel); } }; @@ -1156,10 +1157,8 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, over continue; } - const filePath = normalizePath(prefix + rel); - if (!isSourceFile(filePath, overrides)) continue; - if (statusCode.includes('D')) { + const filePath = normalizePath(prefix + rel); // Deletions stay unfiltered: getChangedFiles acts on one only when the // path is already tracked in the DB, where removal is always correct — and // that lets a newly-excluded dir's stale rows clean themselves up. (#766) @@ -1167,6 +1166,9 @@ function collectGitStatus(repoDir: string, prefix: string, out: GitChanges, over continue; } + const filePath = normalizePath(prefix + rel); + if (!isSourceFile(rel, repoDir, overrides)) continue; + // Added (`??`) / modified files inside an excluded dir must not enter the // index — match against the repo-relative path, same as the full scan. (#766) if (ig.ignores(rel)) continue; @@ -1218,7 +1220,7 @@ export function scanDirectory( const files: string[] = []; let count = 0; for (const filePath of gitFiles) { - if (isSourceFile(filePath, overrides)) { + if (isSourceFile(filePath, rootDir, overrides)) { files.push(filePath); count++; onProgress?.(count, filePath); @@ -1247,7 +1249,7 @@ export async function scanDirectoryAsync( const files: string[] = []; let count = 0; for (const filePath of gitFiles) { - if (isSourceFile(filePath, overrides)) { + if (isSourceFile(filePath, rootDir, overrides)) { files.push(filePath); count++; onProgress?.(count, filePath); @@ -1351,7 +1353,7 @@ function scanDirectoryWalk( walk(fullPath, active); } } else if (stat.isFile()) { - if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) { + if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, rootDir, overrides)) { files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -1368,7 +1370,7 @@ function scanDirectoryWalk( walk(fullPath, active); } } else if (entry.isFile()) { - if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, overrides)) { + if (!isIgnored(fullPath, false, active) && isSourceFile(relativePath, rootDir, overrides)) { files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -1662,7 +1664,11 @@ export class ExtractionOrchestrator { await new Promise(resolve => setImmediate(resolve)); // Detect needed languages and load grammars in the parse worker - const neededLanguages = [...new Set(files.map((f) => detectLanguage(f, undefined, overrides)))]; + const neededLanguages = [...new Set(files.map((f) => ( + !f.slice(f.lastIndexOf('/') + 1).includes('.') && looksLikeShellScriptFile(f, this.rootDir) + ? 'bash' + : detectLanguage(f, undefined, overrides) + )))]; // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded when c is needed if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) { neededLanguages.push('cpp'); @@ -2874,7 +2880,11 @@ export class ExtractionOrchestrator { // Load only grammars needed for changed files if (filesToIndex.length > 0) { const overrides = loadExtensionOverrides(this.rootDir); - const neededLanguages = [...new Set(filesToIndex.map((f) => detectLanguage(f, undefined, overrides)))]; + const neededLanguages = [...new Set(filesToIndex.map((f) => ( + !f.slice(f.lastIndexOf('/') + 1).includes('.') && looksLikeShellScriptFile(f, this.rootDir) + ? 'bash' + : detectLanguage(f, undefined, overrides) + )))]; // .h files default to 'c' but may be C++ — ensure cpp grammar is loaded if (neededLanguages.includes('c') && !neededLanguages.includes('cpp')) { neededLanguages.push('cpp'); diff --git a/src/extraction/languages/bash.ts b/src/extraction/languages/bash.ts new file mode 100644 index 000000000..d4513b1d7 --- /dev/null +++ b/src/extraction/languages/bash.ts @@ -0,0 +1,1154 @@ +import type { Node as SyntaxNode } from 'web-tree-sitter'; +import * as posix from 'node:path/posix'; +import { getNodeText } from '../tree-sitter-helpers'; +import type { ExtractorContext, LanguageExtractor } from '../tree-sitter-types'; + +interface CwdState { + cwdChanged: boolean; + pathPrepends: string[]; +} + +const cwdStates = new WeakMap(); + +function stateFor(ctx: ExtractorContext): CwdState { + // ctx itself is rebuilt per visited node; the live nodes array is the + // identity that persists across one file's whole extraction. + const fileKey = ctx.nodes as unknown as object; + let st = cwdStates.get(fileKey); + if (!st) { + st = { cwdChanged: false, pathPrepends: [] }; + cwdStates.set(fileKey, st); + } + return st; +} + +function commandWord(node: SyntaxNode, source: string): string | null { + const nameNode = node.childForFieldName('name'); + const word = nameNode?.child(0); + return word ? getNodeText(word, source).trim() : null; +} + +function declarationBuiltin(node: SyntaxNode, source: string): string | null { + const keyword = node.child(0); + if (!keyword) return null; + const text = getNodeText(keyword, source); + return text === 'export' || text === 'readonly' || text === 'declare' || text === 'typeset' || text === 'local' + ? text + : null; +} + +/** `declare -g` / `typeset -g` assigns in the global scope even inside a function. */ +function carriesGlobalFlag(node: SyntaxNode, source: string): boolean { + for (let i = 1; i < node.childCount; i++) { + const child = node.child(i); + if (!child) break; + if (child.type === 'word' && /^-[a-zA-Z]*g/.test(getNodeText(child, source))) return true; + if (child.type === 'variable_assignment') break; + } + return false; +} + +function carriesReadonlyFlag(node: SyntaxNode, source: string): boolean { + for (let i = 1; i < node.childCount; i++) { + const child = node.child(i); + if (!child) break; + if (child.type === 'word' && /^-[a-zA-Z]*r/.test(getNodeText(child, source))) return true; + if (child.type === 'variable_assignment') break; + } + return false; +} + +function hasFunctionAncestor(node: SyntaxNode): boolean { + let current = node.parent; + while (current) { + if (current.type === 'function_definition') return true; + current = current.parent; + } + return false; +} + +export interface BashVariable { + name: string; + kind: 'variable' | 'constant'; + valueNode: SyntaxNode | null; + positionNode: SyntaxNode; +} + +/** Variables and constants from an assignment or declaration command; empty when suppressed. */ +export function extractBashVariables(node: SyntaxNode, source: string): BashVariable[] { + if (node.type === 'variable_assignment') { + if (hasFunctionAncestor(node)) return []; + const parent = node.parent?.type === 'command' ? node.parent : null; + const nameNode = node.childForFieldName('name'); + if (!nameNode || parent) return []; + return [{ + name: getNodeText(nameNode, source), + kind: 'variable', + valueNode: node.childForFieldName('value'), + positionNode: nameNode, + }]; + } + + const builtin = declarationBuiltin(node, source); + if (!builtin || builtin === 'local') return []; + + // Inside a function, `declare` without `-g` creates a FUNCTION-LOCAL variable + // — verified by execution: `f() { declare X=1; }; f` leaves X unset, exactly + // like `local`. So it is suppressed for the same reason `local` is, and for + // the reason no other language in the index emits function-local variables at + // all (TypeScript, Python and Go each emit only top-level ones). `readonly` + // and `export` are NOT suppressed: both assign in the global scope even when + // written inside a function, as does `declare -g`. + if (hasFunctionAncestor(node) && (builtin === 'declare' || builtin === 'typeset') && !carriesGlobalFlag(node, source)) { + return []; + } + + const kind: 'variable' | 'constant' = + builtin === 'readonly' || carriesReadonlyFlag(node, source) ? 'constant' : 'variable'; + + return node.namedChildren + .filter((c) => c.type === 'variable_assignment') + .map((assignment) => { + const nameNode = assignment.childForFieldName('name'); + return nameNode + ? { + name: getNodeText(nameNode, source), + kind, + valueNode: assignment.childForFieldName('value'), + positionNode: nameNode, + } + : null; + }) + .filter((v): v is BashVariable => v !== null); +} + +// --- script path resolution ------------------------------------------------- + +const OWN_PROCESS_TYPES = new Set(['command_substitution', 'subshell', 'pipeline']); +const SUPPRESSED_ASSIGNMENT_ANCESTORS = new Set([ + ...OWN_PROCESS_TYPES, + 'if_statement', + 'while_statement', + 'until_statement', + 'for_statement', + 'case_statement', + 'function_definition', +]); +const MAX_TRACE_DEPTH = 8; +const ROOT_ANCHOR_NAMES = new Set(['REPO_ROOT', 'PROJECT_ROOT', 'WORKSPACE_ROOT', 'CODEGRAPH_ROOT']); + +function programRoot(node: SyntaxNode): SyntaxNode { + let current = node; + while (current.parent) current = current.parent; + return current; +} + +function runsInOwnProcess(node: SyntaxNode): boolean { + let current = node.parent; + while (current) { + if (OWN_PROCESS_TYPES.has(current.type)) return true; + current = current.parent; + } + return false; +} + +function isArgumentZeroExpansion(node: SyntaxNode, source: string): boolean { + if (node.type === 'simple_expansion' || node.type === 'expansion') { + const inner = expansionNameNode(node); + if (!inner || inner.type !== 'variable_name') return false; + const name = getNodeText(inner, source); + return name === '0' || name === 'BASH_SOURCE'; + } + return false; +} + +/** The variable a simple_expansion/expansion reads: a variable_name, possibly under a subscript. */ +function expansionNameNode(node: SyntaxNode): SyntaxNode | null { + const first = node.namedChild(0); + if (!first) return null; + if (first.type === 'variable_name') return first; + if (first.type === 'subscript') { + return first.childForFieldName('name') ?? first.namedChild(0); + } + return null; +} + +/** `${VAR%/*}`-style dirname-by-suffix-removal over an own-directory anchor. */ +function isOwnDirectoryRemoval(node: SyntaxNode, source: string): boolean { + if (node.type !== 'expansion') return false; + if (!isArgumentZeroExpansion(node, source)) return false; + const regex = node.namedChildren.find((c) => c.type === 'regex'); + if (!regex) return false; + const text = getNodeText(regex, source); + return text.endsWith('/*') && !text.includes('#'); +} + +function matchDirnameSubstitution(node: SyntaxNode, source: string): boolean { + if (node.type !== 'command_substitution') return false; + const body = node.namedChild(0); + const cmd = + body?.type === 'command' + ? body + : body?.type === 'list' + ? (body.namedChildren.length === 1 && body.namedChild(0)?.type === 'command' + ? body.namedChild(0) + : null) + : null; + if (!cmd || commandWord(cmd, source) !== 'dirname') return false; + + let sawAnchor = false; + for (let i = 0; i < cmd.namedChildCount; i++) { + const child = cmd.namedChild(i)!; + if (child.type === 'command_name') continue; + if (child.type === 'word' && getNodeText(child, source) === '--') continue; + if (isArgumentZeroExpansion(child, source) && !child.namedChildren.some((c) => c.type === 'regex')) { + sawAnchor = true; + continue; + } + if (child.type === 'string') { + const exprs = child.namedChildren.filter((c) => c.type !== 'string_content'); + const content = child.namedChildren.filter((c) => c.type === 'string_content').map((c) => getNodeText(c, source)).join(''); + if ( + exprs.length === 1 && + isArgumentZeroExpansion(exprs[0]!, source) && + !exprs[0]!.namedChildren.some((c) => c.type === 'regex') && + content.trim() === '' + ) { + sawAnchor = true; + continue; + } + } + return false; + } + return sawAnchor; +} + +/** + * `$(cd && pwd)` resolves to whatever its inner expression resolves + * to — usually an own-directory anchor, but anchored parent segments + * (`$(cd "$(dirname "$0")/.." && pwd)`) climb deliberately. + */ +function resolveCdPrintSubstitution( + node: SyntaxNode, + source: string, + rootDir: string, + allowRelative: boolean, + visited: Set, + depth: number +): string | null { + if (node.type !== 'command_substitution') return null; + const body = node.namedChild(0); + const commands = + body?.type === 'list' + ? body.namedChildren.filter((c) => c.type === 'command') + : body?.type === 'command' + ? [body] + : []; + if (commands.length !== 2) return null; + const [cd, pwd] = commands as [SyntaxNode, SyntaxNode]; + if (commandWord(cd, source) !== 'cd' || commandWord(pwd, source) !== 'pwd') return null; + const args = cd.namedChildren.filter((c) => c.type !== 'command_name'); + if (args.length !== 1) return null; + return composeDirectory([args[0]!], source, rootDir, allowRelative, new Set(visited), depth + 1, Number.MAX_SAFE_INTEGER); +} + +function traceVariable( + name: string, + reference: SyntaxNode, + source: string, + containingDir: string, + cutoff: number, + visited: Set, + depth: number +): string | null { + if (visited.has(name) || depth > MAX_TRACE_DEPTH) return null; + visited.add(name); + + const root = programRoot(reference); + const matches: SyntaxNode[] = []; + + const walk = (node: SyntaxNode): void => { + if (node.type === 'variable_assignment') { + const nameNode = node.childForFieldName('name'); + if (nameNode && getNodeText(nameNode, source) === name && node.startIndex < cutoff) { + let suppressed = false; + let parent = node.parent; + while (parent && parent !== root) { + if (SUPPRESSED_ASSIGNMENT_ANCESTORS.has(parent.type)) { + suppressed = true; + break; + } + parent = parent.parent; + } + if (!suppressed) matches.push(node); + } + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child) walk(child); + } + }; + walk(root); + + if (matches.length === 0) return null; + + const resolveValueOf = (match: SyntaxNode): string | null => { + const value = match.childForFieldName('value'); + // Resolving THIS assignment's value, a reference to the same name reads + // the PREVIOUS value: cut the search off at this assignment's start. + const innerVisited = new Set(visited); + innerVisited.delete(name); + return value + ? composeDirectory([value], source, containingDir, true, innerVisited, depth + 1, match.startIndex) + : null; + }; + + const valueRefsSelf = (match: SyntaxNode): boolean => { + const value = match.childForFieldName('value'); + if (!value) return false; + let found = false; + const scan = (node: SyntaxNode): void => { + if (found) return; + if (node.type === 'simple_expansion' || node.type === 'expansion') { + const n = expansionNameNode(node); + if (n && getNodeText(n, source) === name) { + found = true; + return; + } + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child) scan(child); + } + }; + scan(value); + return found; + }; + + const last = matches[matches.length - 1]!; + if (matches.length === 1 || valueRefsSelf(last)) { + // A chained re-assignment threads the previous value deterministically. + return resolveValueOf(last); + } + + // Independent overwrites: every derivation must agree, or the forward + // scan cannot tell which one ran. + const results = new Set(); + for (const match of matches) { + const resolved = resolveValueOf(match); + if (resolved === null) return null; + results.add(resolved); + } + return results.size === 1 ? results.values().next().value! : null; +} + +/** + * Extend a partially-composed directory with literal path text ('/../lib', + * './x', ...). Null current means no anchor yet; relative literals then + * bottom out at rootDir when allowed, and stay unknowable otherwise. + */ +function appendLiteralText( + current: string | null, + text: string, + rootDir: string, + allowRelative: boolean +): string | null { + if (text === '') return current; + if (current === null && !allowRelative && !text.startsWith('/')) return null; + let dir = current; + if (dir === null) { + dir = text.startsWith('/') ? '/' : rootDir; + } + for (const part of text.split('/')) { + if (part === '' || part === '.') continue; + if (part === '..') { + dir = posix.dirname(dir); + } else { + dir = posix.join(dir, part); + } + } + return dir; +} + +function isDirMaterialNode(node: SyntaxNode): boolean { + const type = node.type; + return ( + type === 'string_content' || + type === 'word' || + type === 'raw_string' || + type === 'string' || + type === 'concatenation' || + type === 'command_substitution' || + type === 'simple_expansion' || + type === 'expansion' + ); +} + +/** + * Compose a chain of expression and literal segments into an ABSOLUTE + * directory path, or null the moment a step is not statically knowable. + */ +function composeDirectory( + segments: SyntaxNode[], + source: string, + rootDir: string, + allowRelative: boolean, + visited: Set, + depth: number, + cutoff: number +): string | null { + let current: string | null = null; + + for (const segment of segments) { + const type = segment.type; + if (!isDirMaterialNode(segment)) continue; + if (type === 'string_content' || type === 'word' || type === 'raw_string') { + let text = getNodeText(segment, source); + if (type === 'raw_string' && text.length >= 2) text = text.slice(1, -1); + current = appendLiteralText(current, text, rootDir, allowRelative); + if (current === null) return null; + continue; + } + if (type === 'string' || type === 'concatenation') { + const inner: SyntaxNode[] = []; + for (let i = 0; i < segment.namedChildCount; i++) { + const child = segment.namedChild(i); + if (child) inner.push(child); + } + const composed = composeDirectory(inner, source, rootDir, current !== null || allowRelative, visited, depth, cutoff); + if (composed === null) return null; + current = composed; + continue; + } + if (type === 'command_substitution') { + const viaCdPrint = resolveCdPrintSubstitution(segment, source, rootDir, current !== null || allowRelative, visited, depth); + if (viaCdPrint !== null) { + current = viaCdPrint; + continue; + } + if (matchDirnameSubstitution(segment, source) || isOwnDirectoryRemovalWrapped(segment, source)) { + current = rootDir; + continue; + } + return null; + } + // simple_expansion | expansion + if (isOwnDirectoryRemoval(segment, source)) { + current = rootDir; + continue; + } + const nameNode = expansionNameNode(segment); + if (!nameNode) return null; + const name = getNodeText(nameNode, source); + // These conventional repository-root variables are process/environment + // anchors, not ordinary values we can trace from shell assignments. + if (ROOT_ANCHOR_NAMES.has(name)) { + // The extractor receives repository-relative paths. An empty POSIX path + // is therefore the repository root; normalizeScriptPath later converts + // the resulting target back relative to the referencing file. + current = ''; + continue; + } + const resolved = traceVariable(name, segment, source, rootDir, cutoff, visited, depth + 1); + if (resolved === null) return null; + // traceVariable composes against its own rootDir already — never re-join. + current = resolved; + } + + return current; +} + +function isOwnDirectoryRemovalWrapped(node: SyntaxNode, source: string): boolean { + if (node.type !== 'command_substitution') return false; + const body = node.namedChild(0); + if (!body) return false; + return isOwnDirectoryRemoval(body, source) || + (body.type === 'string' && body.namedChildren.some((c) => isOwnDirectoryRemoval(c, source))); +} + +/** + * Resolve a script path argument to a normalized relative path against the + * referencing file, or null. `base` is the containing file's directory, or + * null once a working-directory change makes relative literals unknowable. + */ +export function normalizeScriptPath( + argNode: SyntaxNode, + source: string, + containingFileDir: string, + base: string | null +): string | null { + const segments: SyntaxNode[] = + argNode.type === 'string' || argNode.type === 'concatenation' + ? (argNode.namedChildren.filter((c) => c !== null) as SyntaxNode[]) + : [argNode]; + + // Split the argument into directory material and the trailing literal + // filename: the last '/'-bearing literal child carries it. + let fileIdx = -1; + let filename = ''; + let prefixText = ''; + for (let i = segments.length - 1; i >= 0; i--) { + const child = segments[i]!; + if (!isDirMaterialNode(child)) return null; + if ( + fileIdx === -1 && + (child.type === 'string_content' || child.type === 'word' || child.type === 'raw_string') + ) { + let text = getNodeText(child, source); + if (child.type === 'raw_string' && text.length >= 2) text = text.slice(1, -1); + const slash = text.lastIndexOf('/'); + if (slash >= 0) { + fileIdx = i; + filename = text.slice(slash + 1); + prefixText = text.slice(0, slash + 1); + continue; + } + if (i === 0 || !segments.slice(0, i).some((c) => isDirMaterialNode(c))) { + // No slash anywhere before this literal either: a bare name, which + // bash resolves through PATH rather than the script's directory. + return null; + } + // Literal tail after the last expression: it IS the filename. + fileIdx = i; + filename = text; + prefixText = ''; + continue; + } + } + if (fileIdx === -1 || !filename) return null; + + // A failed anchor composition is fatal when expressions precede the + // filename; only a purely literal argument may root itself at BASE. + const exprBefore = segments + .slice(0, fileIdx) + .some((c) => c.type === 'command_substitution' || c.type === 'simple_expansion' || c.type === 'expansion'); + + let dir = composeDirectory( + segments.slice(0, fileIdx), + source, + containingFileDir, + base !== null, + new Set(), + 0, + argNode.startIndex + ); + if (dir === null && exprBefore) return null; + if (prefixText !== '') { + dir = appendLiteralText(dir, prefixText, containingFileDir, base !== null); + } + if (dir === null) return null; + + const target = posix.join(dir, filename); + const rel = posix.relative(containingFileDir, target); + if (rel === '') return null; + return rel.startsWith('.') ? rel : `./${rel}`; +} + +// --- interpreter wrappers --------------------------------------------------- + +interface InterpreterInvocation { + word: string; + pathNode: SyntaxNode | null; + startupPathNode: SyntaxNode | null; + normalizedPath: string | null; +} + +const WRAPPERS_NO_OPTS = new Set(['builtin', 'nohup']); +const ROOT_REMAP_WRAPPERS = new Set(['chroot', 'unshare', 'nsenter', 'bwrap']); + +/** + * Refusal register: ambient PATH words, remote/container arguments, dynamic + * roots and interactive-only startup hooks are intentionally not guessed. + */ +const WRAPPERS_FLAGS_NO_ARG: Record> = { + env: new Set(['-i']), + command: new Set(['-p']), + sudo: new Set(['-i', '-s', '-E']), + timeout: new Set(['--preserve-status']), + exec: new Set(['-cl']), +}; +const WRAPPERS_FLAG_WITH_ARG: Record> = { + env: { '-u': 'unset', '--unset': 'unset' }, + nice: { '-n': 'adjustment' }, + stdbuf: { '-i': 'mode', '-o': 'mode', '-e': 'mode' }, + sudo: { '-u': 'user', '-g': 'group' }, + timeout: { '-s': 'signal', '--signal': 'signal' }, + exec: { '-a': 'name' }, +}; +const INTERPRETER_WORDS = new Set(['sh', 'bash', 'ksh', 'zsh', 'dash']); +const SCRIPT_INTERPRETER_WORDS = new Set([ + ...INTERPRETER_WORDS, + 'python', 'python2', 'python3', 'node', 'nodejs', 'php', 'ruby', 'perl', + 'deno', 'bun', +]); +const INTERPRETER_CODE_FLAGS: Record> = { + python: new Set(['c', '--command']), python2: new Set(['c', '--command']), + python3: new Set(['c', '--command']), + node: new Set(['e', 'p', '--eval', '--print']), nodejs: new Set(['e', 'p', '--eval', '--print']), + php: new Set(['r']), ruby: new Set(['e']), perl: new Set(['e']), + deno: new Set(['e', '--eval']), bun: new Set(['e', '--eval']), +}; + +function wordsOfCommand(node: SyntaxNode, source: string): { text: string; node: SyntaxNode }[] { + const out: { text: string; node: SyntaxNode }[] = []; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i)!; + if (child.type === 'command_name') { + const word = child.child(0); + if (word) out.push({ text: getNodeText(word, source), node: word }); + } else if (child.type === 'word') { + out.push({ text: getNodeText(child, source), node: child }); + } else if (child.type === 'variable_assignment') { + out.push({ text: '=', node: child }); + } else if (child.type === 'number') { + out.push({ text: getNodeText(child, source), node: child }); + } else { + out.push({ text: '\0', node: child }); + } + } + return out; +} + +function looksLikeAssignmentWord(text: string): boolean { + return /^[A-Za-z_][A-Za-z0-9_]*=/.test(text); +} + +export function resolveInterpreterInvocation( + node: SyntaxNode, + source: string, + rootDir: string +): InterpreterInvocation | null { + const words = wordsOfCommand(node, source); + let i = 0; + let startupPathNode: SyntaxNode | null = null; + let rootRemap: string | null = null; + let bindRemap: { source: string; destination: string } | null = null; + + while (i < words.length && (words[i]!.text === '=' || looksLikeAssignmentWord(words[i]!.text))) { + const assignment = words[i]!.node; + const name = assignment.type === 'variable_assignment' + ? assignment.childForFieldName('name') + : null; + if (name && getNodeText(name, source) === 'BASH_ENV') { + startupPathNode = assignment.childForFieldName('value'); + } + i++; + } + + let sawWrapper = false; + while (i < words.length) { + const { text } = words[i]!; + if (!WRAPPERS_NO_OPTS.has(text) && !ROOT_REMAP_WRAPPERS.has(text) && !(text in WRAPPERS_FLAGS_NO_ARG) && !(text in WRAPPERS_FLAG_WITH_ARG)) break; + + // The query forms of `command` test a name: credit the tested word. + const next = words[i + 1]; + if (text === 'command' && next && (next.text === '-v' || next.text === '-V')) { + const queried = words[i + 2]; + if (!queried || queried.text === '\0') return null; + return { word: queried.text, pathNode: null, startupPathNode: null, normalizedPath: null }; + } + + sawWrapper = true; + i++; + if (text === 'bwrap') { + while (i < words.length && words[i]!.text !== '--') { + const option = words[i]!.text; + if (option === '--bind' || option === '--ro-bind') { + const source = words[i + 1]; + const destination = words[i + 2]; + if (!source || !destination || source.text === '\0' || destination.text === '\0') return null; + bindRemap = { source: source.text, destination: destination.text }; + i += 3; + continue; + } + if (option === '--chdir') { + if (!words[i + 1] || words[i + 1]!.text === '\0') return null; + i += 2; + continue; + } + return null; + } + if (words[i]?.text === '--') i++; + continue; + } + if (text !== 'chroot' && ROOT_REMAP_WRAPPERS.has(text)) return null; + if (text === 'chroot') { + const newRoot = words[i]; + if (!newRoot || newRoot.text === '\0' || newRoot.text.startsWith('-') || !newRoot.text.includes('/')) return null; + rootRemap = newRoot.text; + i++; + continue; + } + const flagsNoArg = WRAPPERS_FLAGS_NO_ARG[text]; + const flagsWithArg = WRAPPERS_FLAG_WITH_ARG[text]; + const noOpts = WRAPPERS_NO_OPTS.has(text); + while (i < words.length) { + const w = words[i]!; + if (w.text === '\0') return null; + if (looksLikeAssignmentWord(w.text)) { i++; continue; } + if (!w.text.startsWith('-')) break; + if (flagsWithArg && (w.text in flagsWithArg)) { + const next = words[i + 1]; + if (!next || next.text === '\0' || next.text.startsWith('-')) return null; + i += 2; + continue; + } + if (text === 'timeout' && flagsNoArg && w.text === '--preserve-status') { i++; continue; } + if (flagsNoArg && flagsNoArg.has(w.text)) { i++; continue; } + if (w.text === '--') { i++; break; } + return null; + } + if (noOpts) continue; + if (text === 'timeout') { + const duration = words[i]; + if (!duration || duration.text.startsWith('-') || duration.text === '\0') return null; + i++; + } + } + + if (i >= words.length) return null; + const candidate = words[i]!; + if (candidate.text === '\0') return null; + + // An interpreter named through a shell-naming variable: resolve its traced + // value and test the basename against the interpreter set. + let effectiveWord = candidate.text.includes('/') ? posix.basename(candidate.text) : candidate.text; + if (candidate.node.type === 'simple_expansion' || candidate.node.type === 'expansion') { + const nameNode = expansionNameNode(candidate.node); + if (!nameNode) return null; + const resolved = traceVariable( + getNodeText(nameNode, source), + candidate.node, + source, + rootDir, + node.startIndex, + new Set(), + 0 + ); + if (resolved === null) return null; + effectiveWord = posix.basename(resolved); + } + + // A direct script path behind the wrappers is itself the invocation target. + if ( + sawWrapper && + candidate.text.includes('/') && + !candidate.text.startsWith('-') && + !SCRIPT_INTERPRETER_WORDS.has(effectiveWord) + ) { + const remappedTarget = rootRemap && candidate.text.startsWith('/') + ? posix.join(rootDir, rootRemap, candidate.text.slice(1)) + : bindRemap && candidate.text.startsWith(bindRemap.destination) + ? posix.join(rootDir, bindRemap.source, candidate.text.slice(bindRemap.destination.length).replace(/^\//, '')) + : null; + const normalizedPath = remappedTarget ? posix.relative(rootDir, remappedTarget) : null; + return { + word: candidate.text, + pathNode: candidate.node, + startupPathNode, + normalizedPath: normalizedPath ? (normalizedPath.startsWith('.') ? normalizedPath : `./${normalizedPath}`) : null, + }; + } + + if (SCRIPT_INTERPRETER_WORDS.has(effectiveWord)) { + i++; + // Skip the interpreter's own options. `-c` (the next word is a command + // string, not a script) and `-s` (read the script from stdin) mean there is + // no script path to resolve at all; short options bundle, so `bash -ec ...` + // has to count too. + while (i < words.length) { + const w = words[i]!; + if (w.text === '--') { i++; break; } + if (!w.text.startsWith('-') || w.text === '-') break; + const codeFlags = INTERPRETER_CODE_FLAGS[effectiveWord] ?? new Set(['c', 's']); + if (codeFlags.has(w.text)) return { word: effectiveWord, pathNode: null, startupPathNode, normalizedPath: null }; + if (effectiveWord === 'bash' && (w.text === '--rcfile' || w.text === '--init-file')) { + if (!words[i + 1] || words[i + 1]!.text === '\0') return { word: effectiveWord, pathNode: null, startupPathNode, normalizedPath: null }; + i += 2; + continue; + } + if (!w.text.startsWith('--') && [...codeFlags].some((flag) => w.text.slice(1).includes(flag))) { + return { word: effectiveWord, pathNode: null, startupPathNode, normalizedPath: null }; + } + i++; + } + const script = words[i]; + if (!script) return { word: effectiveWord, pathNode: null, startupPathNode, normalizedPath: null }; + // A quoted or expansion-bearing argument reaches us as wordsOfCommand's + // '\0' sentinel but still carries its real node. normalizeScriptPath traces + // such a node exactly as it does on the `source` path — the AST is identical + // — so hand it over rather than discarding it, which is what made + // `bash "$HERE/x.sh"` emit no relation at all while `source "$HERE/x.sh"` + // resolved. A bare name still declines there, since bash resolves that + // against the runtime cwd/PATH rather than the script's directory. + const remappedTarget = rootRemap && script.text.startsWith('/') + ? posix.join(rootDir, rootRemap, script.text.slice(1)) + : bindRemap && script.text.startsWith(bindRemap.destination) + ? posix.join(rootDir, bindRemap.source, script.text.slice(bindRemap.destination.length).replace(/^\//, '')) + : null; + const normalizedPath = remappedTarget + ? posix.relative(rootDir, remappedTarget) + : null; + return { + word: effectiveWord, + pathNode: script.node, + startupPathNode, + normalizedPath: normalizedPath ? (normalizedPath.startsWith('.') ? normalizedPath : `./${normalizedPath}`) : null, + }; + } + + return { word: candidate.text, pathNode: null, startupPathNode, normalizedPath: null }; +} + +// --- function-named-as-argument forms ---------------------------------------- + +const SIGNAL_NAMES = new Set([ + 'HUP', 'INT', 'QUIT', 'ILL', 'TRAP', 'ABRT', 'BUS', 'FPE', 'KILL', 'USR1', + 'SEGV', 'USR2', 'PIPE', 'ALRM', 'TERM', 'STKFLT', 'CHLD', 'CONT', 'STOP', + 'TSTP', 'TTIN', 'TTOU', 'URG', 'XCPU', 'XFS', 'VTALRM', 'PROF', 'WINCH', + 'IO', 'PWR', 'SYS', 'EXIT', 'DEBUG', 'RETURN', 'ERR', +]); + +function isSignalSpecification(text: string): boolean { + if (/^\d+$/.test(text)) return true; + const bare = text.replace(/^SIG/i, ''); + return SIGNAL_NAMES.has(bare.toUpperCase()); +} + +function stripQuotes(text: string): string { + if (text.length >= 2 && ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'")))) { + return text.slice(1, -1); + } + return text; +} + +/** + * A function named as an argument to trap/complete/export — the guarded set + * that genuinely CALLS the function. Returns the function name, or null. + */ +export function functionRefArguments( + node: SyntaxNode, + source: string, + filePath: string +): string | null { + const word = commandWord(node, source); + if (!word) return null; + + const operandTexts = (): { text: string; node: SyntaxNode }[] => { + const out: { text: string; node: SyntaxNode }[] = []; + let pastName = false; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i)!; + if (!pastName) { + // Wrapper objects are re-created per access — never compare nodes + // with ===; the single command_name child IS the name. + if (child.type === 'command_name') pastName = true; + continue; + } + if (child.type === 'variable_assignment') continue; + const type = child.type; + if (type === 'word' || type === 'raw_string' || type === 'string') { + out.push({ text: getNodeText(child, source), node: child }); + } else { + out.push({ text: '\0', node: child }); + } + } + return out; + }; + + if (word === 'trap') { + const operands = operandTexts(); + let i = 0; + while (i < operands.length) { + const t = operands[i]!.text; + if (t === '--') { i++; break; } + if (t.startsWith('-')) { i++; continue; } + break; + } + const rest = operands.slice(i); + if (rest.length < 2) return null; + const rawAction = rest[0]!.text; + const action = stripQuotes(rawAction); + if (action === '' || action.includes(' ') || action.startsWith('-')) return null; + if (isSignalSpecification(action)) return null; + return action; + } + + if (word === 'complete') { + const operands = operandTexts(); + for (let i = 0; i < operands.length - 1; i++) { + if (operands[i]!.text === '-F' || operands[i]!.text === '--command') { + const fn = stripQuotes(operands[i + 1]!.text); + if (fn && !fn.startsWith('-')) return fn; + } + } + return null; + } + + if (word === 'export') { + const operands = operandTexts(); + let sawFuncOpt = false; + for (let i = 0; i < operands.length; i++) { + const t = operands[i]!.text; + if (t === '--') continue; + if (/^-[a-zA-Z]*f/.test(t)) { sawFuncOpt = true; continue; } + if (t.startsWith('-')) continue; + if (sawFuncOpt) { + const fn = stripQuotes(t).split('=').pop()!; + return fn || null; + } + } + return null; + } + + // The bats runner: an ordinary word everywhere else, meaningful only in a + // .bats file (an extension the grammar health check kept). + if (word === 'run' && filePath.endsWith('.bats')) { + const operands = operandTexts().filter((o) => o.text !== '\0'); + const first = operands[0]; + if (!first) return null; + const fn = stripQuotes(first.text); + if (!fn || fn.startsWith('-') || fn.startsWith('$')) return null; + return fn; + } + + return null; +} + +// --- classification --------------------------------------------------------- + +const SOURCING_WORDS = new Set(['source', '.']); +const DIRECTORY_CHANGE_WORDS = new Set(['cd', 'pushd', 'popd']); +const SHELL_SCRIPT_EXTENSIONS = new Set(['.sh', '.bash', '.ksh', '.zsh', '.dash', '.bats']); + +function isShellScriptReference(filePath: string): boolean { + const basename = posix.basename(filePath).toLowerCase(); + const dot = basename.lastIndexOf('.'); + return dot < 0 || SHELL_SCRIPT_EXTENSIONS.has(basename.slice(dot)); +} + +function localRedirectPathNodes(node: SyntaxNode): SyntaxNode[] { + const paths: SyntaxNode[] = []; + const walk = (current: SyntaxNode): void => { + if ((current.type === 'file_redirect' || current.type.endsWith('_redirect')) && current.type !== 'heredoc_redirect') { + const candidate = current.namedChild(current.namedChildCount - 1); + if (candidate && ['word', 'string', 'raw_string', 'concatenation'].includes(candidate.type)) { + paths.push(candidate); + } + return; + } + for (let i = 0; i < current.namedChildCount; i++) { + const child = current.namedChild(i); + if (child) walk(child); + } + }; + walk(node); + return paths; +} + +function emitLocalRedirectRelations(node: SyntaxNode, ctx: ExtractorContext, state: CwdState): void { + const containingFileDir = posix.dirname(ctx.filePath); + const base = state.cwdChanged ? null : containingFileDir; + for (const redirectPath of localRedirectPathNodes(node)) { + const normalized = normalizeScriptPath(redirectPath, ctx.source, containingFileDir, base); + if (normalized && isShellScriptReference(normalized)) { + emitScriptRelation(ctx, normalized, 'references', redirectPath); + } + } +} + +function isInsideCommandSubstitution(node: SyntaxNode): boolean { + let current = node.parent; + while (current) { + if (current.type === 'command_substitution') return true; + current = current.parent; + } + return false; +} + +function recordPathPrepend(node: SyntaxNode, ctx: ExtractorContext, state: CwdState): void { + const commandText = getNodeText(node, ctx.source); + const pathMatch = commandText.match(/(?:^|\s)(?:export\s+)?PATH\s*=\s*([^\n;]+)/); + if (!pathMatch) return; + const value = pathMatch[1]!.replace(/^['"]|['"]$/g, ''); + const anchored = value.match(/^\$\(dirname\s+["']?\$0["']?\)([^:]+)(?::\$PATH)?$/) + ?? commandText.match(/\$\(dirname\s+["']?\$0["']?\)([^:$\s"']+)/); + const literal = value.match(/^(\.\.?\/[^:]+)(?::\$PATH)?$/); + if (anchored) state.pathPrepends.push(posix.join(posix.dirname(ctx.filePath), anchored[1]!)); + else if (literal) state.pathPrepends.push(posix.normalize(posix.join(posix.dirname(ctx.filePath), literal[1]!))); +} + +function emitScriptRelation( + ctx: ExtractorContext, + normalizedPath: string, + kind: 'imports' | 'references', + anchorNode: SyntaxNode +): void { + if (kind === 'imports') { + ctx.createNode('import', normalizedPath, anchorNode, { + signature: getNodeText(anchorNode, ctx.source).trim().slice(0, 100), + }); + } + if (ctx.nodeStack.length === 0) return; + const fromNodeId = ctx.nodeStack[ctx.nodeStack.length - 1]; + if (!fromNodeId) return; + ctx.addUnresolvedReference({ + fromNodeId, + referenceName: normalizedPath, + referenceKind: kind, + line: anchorNode.startPosition.row + 1, + column: anchorNode.startPosition.column, + }); +} + +function classifyShellCommand(node: SyntaxNode, ctx: ExtractorContext, state: CwdState): void { + const source = ctx.source; + const word = commandWord(node, source); + if (!word) return; + + const containingFileDir = posix.dirname(ctx.filePath); + const base = state.cwdChanged ? null : containingFileDir; + + // Redirections are performed by the local shell before a remote/container + // command receives control. This keeps `ssh host 'bash -s' < ./x.sh` and + // `docker run -i image bash -s < ./x.sh` visible without admitting the + // remote command's own path arguments. + emitLocalRedirectRelations(node, ctx, state); + + // A command substitution is evaluated by this shell even when its output + // becomes an argument to a remote command (`ssh host "$(cat ./x.sh)"`). + // Credit the local file operand, but only for the explicit file-reading + // form and only when it names a shell script. + if (word === 'cat' && isInsideCommandSubstitution(node)) { + const operand = node.namedChildren.find((child) => + child.type !== 'command_name' && ['word', 'string', 'raw_string', 'concatenation'].includes(child.type)); + if (operand) { + const normalized = normalizeScriptPath(operand, source, containingFileDir, base); + if (normalized && isShellScriptReference(normalized)) { + emitScriptRelation(ctx, normalized, 'references', operand); + } + } + } + + if (SOURCING_WORDS.has(word)) { + const args = node.namedChildren.filter((c) => c.type !== 'command_name'); + const first = args[0]; + if (!first) return; + const text = getNodeText(first, source).replace(/^['"]|['"]$/g, ''); + if (!text.includes('/')) return; + const normalized = normalizeScriptPath(first, source, containingFileDir, base); + if (normalized && isShellScriptReference(normalized)) emitScriptRelation(ctx, normalized, 'imports', first); + return; + } + + if (word.includes('/') && !SCRIPT_INTERPRETER_WORDS.has(posix.basename(word))) { + const nameNode = node.childForFieldName('name'); + const nameWord = nameNode?.child(0); + if (!nameWord) return; + const normalized = normalizeScriptPath(nameWord, source, containingFileDir, base); + if (normalized && isShellScriptReference(normalized)) emitScriptRelation(ctx, normalized, 'references', nameNode!); + return; + } + + const invocation = resolveInterpreterInvocation(node, source, containingFileDir); + if (!invocation) return; + + if (invocation.startupPathNode) { + const startupPath = normalizeScriptPath(invocation.startupPathNode, source, containingFileDir, base); + if (startupPath && isShellScriptReference(startupPath)) { + emitScriptRelation(ctx, startupPath, 'imports', invocation.startupPathNode); + } + } + + if (invocation.pathNode) { + const normalized = invocation.normalizedPath ?? normalizeScriptPath(invocation.pathNode, source, containingFileDir, base); + if (normalized) { + emitScriptRelation(ctx, normalized, 'references', invocation.pathNode); + return; + } + return; + } + + // Resolve only directories explicitly prepended by this script. The + // inherited PATH is intentionally outside the static-analysis boundary. + if (state.pathPrepends.length > 0 && invocation.word && !invocation.word.includes('/')) { + const targetDir = state.pathPrepends[state.pathPrepends.length - 1]!; + const rel = posix.relative(containingFileDir, posix.join(targetDir, invocation.word)); + emitScriptRelation(ctx, rel.startsWith('.') ? rel : `./${rel}`, 'references', node.childForFieldName('name')!); + return; + } + + // A function named as an argument (trap action, export -f, complete -F, + // the bats runner) credits the FUNCTION, not the builtin word. + const fnArg = functionRefArguments(node, source, ctx.filePath); + if (fnArg) { + if (ctx.nodeStack.length > 0) { + const fromNodeId = ctx.nodeStack[ctx.nodeStack.length - 1]; + if (fromNodeId) { + ctx.addUnresolvedReference({ + fromNodeId, + referenceName: fnArg, + referenceKind: 'calls', + line: node.startPosition.row + 1, + column: node.startPosition.column, + }); + } + } + return; + } + + if (ctx.nodeStack.length > 0) { + const fromNodeId = ctx.nodeStack[ctx.nodeStack.length - 1]; + if (fromNodeId) { + ctx.addUnresolvedReference({ + fromNodeId, + referenceName: invocation.word, + referenceKind: 'calls', + line: node.startPosition.row + 1, + column: node.startPosition.column, + }); + } + } +} + +export const bashExtractor: LanguageExtractor = { + functionTypes: ['function_definition'], + classTypes: [], + methodTypes: [], + interfaceTypes: [], + structTypes: [], + enumTypes: [], + typeAliasTypes: [], + importTypes: [], + callTypes: [], + variableTypes: ['variable_assignment', 'declaration_command'], + nameField: 'name', + bodyField: 'body', + paramsField: '', + + visitNode: (node, ctx) => { + const state = stateFor(ctx); + if (node.type === 'redirected_statement') { + emitLocalRedirectRelations(node, ctx, state); + return false; + } + if (node.type === 'declaration_command') { + recordPathPrepend(node, ctx, state); + return false; + } + if (node.type !== 'command') return false; + + classifyShellCommand(node, ctx, state); + + const word = commandWord(node, ctx.source); + if (word && DIRECTORY_CHANGE_WORDS.has(word) && !runsInOwnProcess(node)) { + state.cwdChanged = true; + } + + for (const child of node.namedChildren) { + if (child.type === 'heredoc_redirect') continue; + ctx.visitNode(child); + } + return true; + }, +}; diff --git a/src/extraction/languages/index.ts b/src/extraction/languages/index.ts index 6b760b01d..b1d7d4221 100644 --- a/src/extraction/languages/index.ts +++ b/src/extraction/languages/index.ts @@ -36,6 +36,7 @@ import { solidityExtractor } from './solidity'; import { terraformExtractor } from './terraform'; import { arktsExtractor } from './arkts'; import { nixExtractor } from './nix'; +import { bashExtractor } from './bash'; export const EXTRACTORS: Partial> = { typescript: typescriptExtractor, @@ -69,4 +70,5 @@ export const EXTRACTORS: Partial> = { terraform: terraformExtractor, arkts: arktsExtractor, nix: nixExtractor, + bash: bashExtractor, }; diff --git a/src/extraction/shebang.ts b/src/extraction/shebang.ts new file mode 100644 index 000000000..0ea121182 --- /dev/null +++ b/src/extraction/shebang.ts @@ -0,0 +1,30 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +const SHELL_SHEBANG = /^#!\s*(?:\/usr\/bin\/env(?:\s+-S)?\s+|(?:\/\S+\/)*)(?:ba|z|k|da)?sh(?:\s|$)/i; + +const SHEBANG_SCAN_BYTES = 256; + +export function looksLikeShellScript(source: string): boolean { + const firstLine = source.slice(0, SHEBANG_SCAN_BYTES).split(/\r?\n/, 1)[0] ?? ''; + return SHELL_SHEBANG.test(firstLine); +} + +export function looksLikeShellScriptFile(filePath: string, rootDir: string): boolean { + try { + return looksLikeShellScript(fsReadPrefix(path.join(rootDir, filePath))); + } catch { + return false; + } +} + +function fsReadPrefix(filePath: string): string { + const fd = fs.openSync(filePath, 'r'); + try { + const buffer = Buffer.allocUnsafe(SHEBANG_SCAN_BYTES); + const bytes = fs.readSync(fd, buffer, 0, SHEBANG_SCAN_BYTES, 0); + return buffer.subarray(0, bytes).toString('utf8'); + } finally { + fs.closeSync(fd); + } +} diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index c34dc4716..6b2bf1f6d 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -23,6 +23,7 @@ import type { LanguageExtractor, ExtractorContext } from './tree-sitter-types'; import { EXTRACTORS } from './languages'; import { stripCppTemplateArgs } from './languages/c-cpp'; import { rustImplTypeName } from './languages/rust'; +import { extractBashVariables } from './languages/bash'; import { LiquidExtractor } from './liquid-extractor'; import { RazorExtractor } from './razor-extractor'; import { SvelteExtractor } from './svelte-extractor'; @@ -2883,6 +2884,21 @@ export class TreeSitterExtractor { isExported, }); } + } else if (this.language === 'bash') { + // Bash: `variable_assignment` carries its name in a nested + // `variable_name` child, and a `declaration_command` wraps the builtin's + // assignments — neither matches the generic identifier walk. A plain + // assignment inside a function body is suppressed (deliberate + // simplification); export/readonly/declare forms surface anywhere, + // readonly / declare -r as constants. + for (const v of extractBashVariables(node, this.source)) { + const initValue = v.valueNode ? getNodeText(v.valueNode, this.source).slice(0, 100) : undefined; + const initSignature = initValue ? `= ${initValue}${initValue.length >= 100 ? '...' : ''}` : undefined; + this.createNode(v.kind, v.name, v.positionNode, { docstring, signature: initSignature }); + if (v.valueNode) { + this.visitFunctionBody(v.valueNode, ''); + } + } } else { // Generic fallback for other languages // Try to find identifier children @@ -5199,6 +5215,19 @@ export class TreeSitterExtractor { const visitForCallsAndStructure = (node: SyntaxNode): void => { const nodeType = node.type; + // Bash: the body walker is the ONLY path into function bodies (the main + // visitNode walker stops at function_definitions), and both commands and + // declaration forms must flow through the same extractor hooks the + // top-level walk uses — one seam for every command node. + if (this.language === 'bash' && this.extractor) { + if (this.extractor.visitNode && nodeType === 'command') { + const ctx = this.makeExtractorContext(); + this.extractor.visitNode(node, ctx); + } else if (this.extractor.variableTypes.includes(nodeType)) { + this.extractVariable(node); + } + } + // Function-as-value capture (#756) — function bodies are walked here, // not in visitNode, so the capture hook must fire in both walkers. this.maybeCaptureFnRefs(node, nodeType); diff --git a/src/extraction/wasm/tree-sitter-bash.wasm b/src/extraction/wasm/tree-sitter-bash.wasm new file mode 100644 index 000000000..4e4198c95 Binary files /dev/null and b/src/extraction/wasm/tree-sitter-bash.wasm differ diff --git a/src/resolution/bash-scope.ts b/src/resolution/bash-scope.ts new file mode 100644 index 000000000..b8ca88f0c --- /dev/null +++ b/src/resolution/bash-scope.ts @@ -0,0 +1,107 @@ +import * as posix from 'node:path/posix'; +import type { Node } from '../types'; +import type { ResolutionContext } from './types'; + +/** + * Bash keeps ONE function namespace per process: sourcing shares functions, + * executing does not. Reachability is co-membership in some file's forward + * closure — the model the two-tier gate below implements. Derived from the + * import NODES the extractor created for sourcing statements, so it never + * depends on whether a particular reference has become an edge yet. + */ + +export type BashSourceClosure = Map>; + +const closureMemos = new WeakMap(); + +export function clearBashScopeMemos(context: ResolutionContext): void { + closureMemos.delete(context); +} + +/** Forward closure per bash file: the file itself plus everything it transitively sources. */ +export function buildSourceClosure(context: ResolutionContext): BashSourceClosure { + const memo = closureMemos.get(context); + if (memo) return memo; + + const sourcingEdges = new Map>(); + for (const node of context.getNodesByKind('import')) { + if (node.language !== 'bash') continue; + const target = resolveScriptPath(node.name, node.filePath); + if (!target) continue; + let set = sourcingEdges.get(node.filePath); + if (!set) { + set = new Set(); + sourcingEdges.set(node.filePath, set); + } + set.add(target); + } + + const closures: BashSourceClosure = new Map(); + const compute = (file: string, seen: Set): Set => { + const cached = closures.get(file); + if (cached) return cached; + if (seen.has(file)) return seen; + seen.add(file); + const out = new Set([file]); + for (const target of sourcingEdges.get(file) ?? []) { + for (const f of compute(target, seen)) out.add(f); + } + seen.delete(file); + closures.set(file, out); + return out; + }; + for (const file of sourcingEdges.keys()) compute(file, new Set()); + + closureMemos.set(context, closures); + return closures; +} + +function resolveScriptPath(name: string, fromFile: string): string | null { + if (!name.startsWith('./') && !name.startsWith('../')) return null; + const dir = posix.dirname(fromFile); + return posix.normalize(posix.join(dir, name)); +} + +export type GateVerdict = + | { accept: 'full' | 'reduced'; confidence: number } + | { accept: false }; + +/** Pure decision: is `candidate` reachable from the referencing file's process? */ +export function gateBashNameMatch( + refFilePath: string, + candidateFilePath: string, + closures: BashSourceClosure +): GateVerdict { + if (refFilePath === candidateFilePath) return { accept: 'full', confidence: 0.92 }; + for (const closure of closures.values()) { + if (closure.has(refFilePath) && closure.has(candidateFilePath)) { + return { accept: 'reduced', confidence: 0.75 }; + } + } + return { accept: false }; +} + +/** Pick the reachable winner among same-named bash function candidates, or null. */ +export function selectReachableBashFunction( + refFilePath: string, + candidates: Node[], + closures: BashSourceClosure +): Node | null { + let best: Node | null = null; + let bestConfidence = 0; + let tied = false; + for (const candidate of candidates) { + const verdict = gateBashNameMatch(refFilePath, candidate.filePath, closures); + if (!verdict.accept) continue; + if (verdict.confidence > bestConfidence) { + best = candidate; + bestConfidence = verdict.confidence; + tied = false; + } else if (verdict.confidence === bestConfidence && candidate.filePath !== best?.filePath) { + // Several closure files define the name — stay unresolved rather than + // guess which one the process actually sees. + tied = true; + } + } + return tied ? null : best; +} diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index 60c7b3008..e3cdc5b7d 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -47,8 +47,23 @@ const EXTENSION_RESOLUTION: Record = { ruby: ['.rb'], objc: ['.h', '.m', '.mm'], nix: ['.nix', '/default.nix'], + // Shell extensions the grammar health check parsed cleanly, plus the empty + // suffix: sourcing an extensionless helper by bare name is common. + bash: ['.sh', '.bash', '.ksh', '.dash', '.bats', ''], }; +export function isBashScriptPathRef(ref: UnresolvedRef): boolean { + if (ref.language !== 'bash') return false; + if (ref.referenceKind !== 'imports' && ref.referenceKind !== 'references') return false; + const name = ref.referenceName; + if (/[\s"'`$;&|<>(){}[\]*?!]/.test(name)) return false; + // Bash emits repository-relative candidates for conventional root anchors + // (for example `$REPO_ROOT/bin/deploy`), so a path with a directory segment + // is eligible even when it has no shell extension. Bare command words stay + // excluded: they belong to PATH/name resolution, not file-path resolution. + return name.startsWith('./') || name.startsWith('../') || (name.includes('/') && !name.startsWith('/')) || /\.(sh|bash|ksh|dash|bats)$/.test(name); +} + export function isNixPathImportRef(ref: UnresolvedRef): boolean { return ( ref.language === 'nix' && @@ -1427,6 +1442,30 @@ export function resolveViaImport( return null; } + // Bash sourced/executed script paths resolve to file nodes only, mirroring + // the nix branch. The classifier normalizes interpolated paths + // (SCRIPT_DIR anchors, dirname/cd-print substitutions) to dot-slash-prefixed + // relative paths; a dynamic path never becomes a reference at all. + if (isBashScriptPathRef(ref)) { + const resolvedPath = resolveImportPath(ref.referenceName, ref.filePath, ref.language, context); + if (!resolvedPath) return null; + + const basename = resolvedPath.split('/').pop()!; + const fileNode = context + .getNodesByName(basename) + .find((n) => n.kind === 'file' && n.filePath === resolvedPath); + + if (fileNode) { + return { + original: ref, + targetNodeId: fileNode.id, + confidence: 0.9, + resolvedBy: 'import', + }; + } + return null; + } + // Use cached import mappings (avoids re-reading and re-parsing per ref) const imports = context.getImportMappings(ref.filePath, ref.language); if (imports.length === 0 && !context.readFile(ref.filePath)) { diff --git a/src/resolution/index.ts b/src/resolution/index.ts index 7b4bccc18..3e27352ba 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -17,7 +17,7 @@ import { ImportMapping, } from './types'; import { matchReference, matchFunctionRef, matchDottedCallChain, matchScopedCallChain, matchMethodCall, sameLanguageFamily, crossesKnownFamily, dumpNameMatcherProfile, clearNameMatcherMemos } from './name-matcher'; -import { resolveViaImport, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, clearImportResolverMemos } from './import-resolver'; +import { resolveViaImport, resolveJvmImport, extractImportMappings, extractReExports, loadCppIncludeDirs, isPhpIncludePathRef, isCobolCopybookRef, isNixPathImportRef, isBashScriptPathRef, clearImportResolverMemos } from './import-resolver'; import { ResolverPool, minRefsForPool } from './resolver-pool'; import { detectFrameworks } from './frameworks'; import { synthesizeCallbackEdges } from './callback-synthesizer'; @@ -884,8 +884,9 @@ export class ReferenceResolver { // ArkTS chained-attribute refs carry a leading dot (`.titleStyle`) that // routes them to the decorator-gated matcher; the symbol itself is // indexed under the bare name, so the existence check strips the dot. - // Nix static path imports (`import ./x.nix`) name a FILE, not a symbol — - // they bypass the symbol-existence check and resolve via resolveViaImport. + // Nix static path imports (`import ./x.nix`) and bash sourced/executed + // script paths name a FILE, not a symbol — they bypass the symbol- + // existence check and resolve via resolveViaImport. let existenceName = ref.language === 'arkts' && ref.referenceName.startsWith('.') ? ref.referenceName.slice(1) @@ -896,6 +897,7 @@ export class ReferenceResolver { const tPre = this.profileStages ? process.hrtime.bigint() : 0n; const preFilterPass = isNixPathImportRef(ref) || + isBashScriptPathRef(ref) || this.hasAnyPossibleMatch(existenceName) || this.matchesAnyImport(ref) || this.frameworks.some((f) => f.claimsReference?.(ref.referenceName)); @@ -992,7 +994,10 @@ export class ReferenceResolver { // qualified-name fallback would only ever add wrong cross-module edges. // Nix static path imports are file references for the same reason — // falling through would let "./x.nix" name-match an unrelated node. - if (isPhpIncludePathRef(ref) || isCobolCopybookRef(ref) || isNixPathImportRef(ref) || ref.language === 'terraform') { + // Bash sourced/executed script paths are file references too: the + // name-match decision they must never reach lives in matchReference's + // bash branch (reachability goal), not in this gate. + if (isPhpIncludePathRef(ref) || isCobolCopybookRef(ref) || isNixPathImportRef(ref) || isBashScriptPathRef(ref) || ref.language === 'terraform') { return candidates.length > 0 ? candidates.reduce((best, curr) => curr.confidence > best.confidence ? curr : best @@ -2419,6 +2424,10 @@ export class ReferenceResolver { private gateLanguage(result: ResolvedRef | null, ref: UnresolvedRef): ResolvedRef | null { if (!result) return result; + // A Bash script-path reference can deliberately launch an indexed Python, + // Node, PHP, or other executable through its interpreter. This is a + // file-to-file execution edge, not a language-family symbol binding. + if (isBashScriptPathRef(ref)) return result; const tgt = this.getLanguageFromNodeId(result.targetNodeId); if (!tgt || !ref.language) return result; if ((ref.referenceKind === 'references' || ref.referenceKind === 'function_ref') && !sameLanguageFamily(tgt, ref.language)) return null; diff --git a/src/resolution/name-matcher.ts b/src/resolution/name-matcher.ts index c74d8f272..e2a42b025 100644 --- a/src/resolution/name-matcher.ts +++ b/src/resolution/name-matcher.ts @@ -6,6 +6,7 @@ import { Language, Node } from '../types'; import { UnresolvedRef, ResolvedRef, ResolutionContext } from './types'; +import { buildSourceClosure, gateBashNameMatch, selectReachableBashFunction } from './bash-scope'; /** * Ceiling on how many same-named definitions a FUZZY name-match strategy will @@ -2491,6 +2492,33 @@ export function matchReference( return matchFunctionRef(ref, context); } + // Bash calls bind ONLY through reachability: a call names a function in the + // same file or in the transitive source closure of the calling file. The + // generic single-winner matcher would pick whichever same-named function was + // indexed first — nearly every script defines log/die/usage — so this branch + // enumerates candidates itself and gates each through gateBashNameMatch. + // Nothing reachable → null; the fuzzy and qualified-name fallbacks must + // never connect a shell call to an arbitrary same-named symbol. + if (ref.language === 'bash' && ref.referenceKind === 'calls' && !ref.referenceName.includes('/')) { + const candidates = context + .getNodesByName(ref.referenceName) + .filter((n) => n.language === 'bash' && n.kind === 'function'); + if (candidates.length > 0) { + const closures = buildSourceClosure(context); + const chosen = selectReachableBashFunction(ref.filePath, candidates, closures); + if (chosen) { + const verdict = gateBashNameMatch(ref.filePath, chosen.filePath, closures); + return { + original: ref, + targetNodeId: chosen.id, + confidence: verdict.accept ? verdict.confidence : 0.75, + resolvedBy: 'exact-match', + }; + } + } + return null; + } + // ArkTS chained UI attributes — emitted with a leading dot (`.titleStyle`, // `.width`) by the extractor — resolve ONLY to decorator-marked attribute // helpers: `@Extend`/`@Styles`/`@AnimatableExtend` functions (and global diff --git a/src/sync/watcher.ts b/src/sync/watcher.ts index 034be858a..94565484b 100644 --- a/src/sync/watcher.ts +++ b/src/sync/watcher.ts @@ -406,7 +406,7 @@ export class FileWatcher { this.ready = true; for (const cb of this.readyWaiters) cb(); this.readyWaiters.length = 0; - if (IS_TEST_RUNTIME) liveWatchersForTests.set(this.projectRoot, this); + if (IS_TEST_RUNTIME || this.inertForTests) liveWatchersForTests.set(this.projectRoot, this); logDebug('File watcher started', { projectRoot: this.projectRoot, @@ -593,7 +593,7 @@ export class FileWatcher { this.refreshScope(rel); return; } - if (!isSourceFile(rel, loadExtensionOverrides(this.projectRoot))) { + if (!isSourceFile(rel, this.projectRoot, loadExtensionOverrides(this.projectRoot))) { this.maybeScheduleForRemovedDir(rel); return; } @@ -788,7 +788,7 @@ export class FileWatcher { this.pendingFiles.clear(); this.ready = false; this.ignoreMatcher = null; - if (IS_TEST_RUNTIME) liveWatchersForTests.delete(this.projectRoot); + if (IS_TEST_RUNTIME || this.inertForTests) liveWatchersForTests.delete(this.projectRoot); logDebug('File watcher stopped'); } diff --git a/src/types.ts b/src/types.ts index 186f57adc..e94709f37 100644 --- a/src/types.ts +++ b/src/types.ts @@ -102,6 +102,7 @@ export const LANGUAGES = [ 'scala', 'lua', 'luau', + 'bash', 'objc', 'r', 'solidity',