diff --git a/.gitignore b/.gitignore index 58205348b..8568550de 100644 --- a/.gitignore +++ b/.gitignore @@ -161,6 +161,10 @@ target/ !graph/test/java/cases/62-build-output-is-not-source/src/**/target/** !graph/test/java/cases/62-build-output-is-not-source/src/**/build/ !graph/test/java/cases/62-build-output-is-not-source/src/**/build/** +# ...and the Gradle case: build/ beside a build script is the construct (the parser must skip it by that +# anchor, not because git ignores it), and src/main/java/app/build/ is the control. +!graph/test/java/cases/64-gradle-build-output-is-not-source/src/**/build/ +!graph/test/java/cases/64-gradle-build-output-is-not-source/src/**/build/** # ── Secrets & local configuration ──────────────────────────────────────────── .env diff --git a/README.md b/README.md index 8522bc8c8..93413fa5b 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,10 @@ command keeps reading the previous graph until the new one is indexed and swappe naming the files it predates. The MCP server also checks every repository it has answered for once 15 minutes have passed since its last update (`AXIOMCODE_REFRESH_INTERVAL`, seconds; 0 turns it off), which catches edits made while a session sits idle. The graph records when and why it was built in `index_meta` (`refreshed_at`, `refresh_reason`). -`AXIOMCODE_NO_REFRESH=1` turns all of this off; the log is `.axiomcode/refresh.log`. +`AXIOMCODE_NO_REFRESH=1` turns the rebuilds off, not the check: an answer from a graph older than an edit still +ends with a `graph refresh: OFF` line naming the files it predates. When a name asked about finds nothing and an +edit since the graph was built writes that name, the line says so, since the declaration may simply be too new +for the graph. The log is `.axiomcode/refresh.log`. ## Graph output diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build.gradle b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build.gradle new file mode 100644 index 000000000..3a0ee1153 --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build.gradle @@ -0,0 +1,7 @@ +plugins { + id 'java' +} + +dependencies { + implementation project(':lib') +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedStage.java b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedStage.java new file mode 100644 index 000000000..f58ff966b --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedStage.java @@ -0,0 +1,10 @@ +package app.gen; + +// Written by an annotation processor into Gradle's output directory: not the project's source. +// Its only call is to the JDK, so the bytecode oracle scores nothing here; if the parser read +// this file, the .edges golden would gain its boundary_lib row. +public class GeneratedStage { + public String label(String name) { + return name.trim(); + } +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build.gradle.kts b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build.gradle.kts new file mode 100644 index 000000000..f6727627a --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build.gradle.kts @@ -0,0 +1,3 @@ +plugins { + `java-library` +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedLibStep.java b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedLibStep.java new file mode 100644 index 000000000..280d007ef --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/build/generated/sources/annotationProcessor/java/main/app/gen/GeneratedLibStep.java @@ -0,0 +1,8 @@ +package app.gen; + +// A Gradle MODULE's output (beside lib/build.gradle.kts): skipped like the root project's. +public class GeneratedLibStep { + public String label(String name) { + return name.strip(); + } +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/src/main/java/app/lib/build/LibStep.java b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/src/main/java/app/lib/build/LibStep.java new file mode 100644 index 000000000..352c08b93 --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/lib/src/main/java/app/lib/build/LibStep.java @@ -0,0 +1,8 @@ +package app.lib.build; + +// The same inside a Gradle module: lib/src/main/java/app/lib/build is a package, not lib/build. +public class LibStep { + public String apply(String name) { + return name + "-applied"; + } +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/settings.gradle b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/settings.gradle new file mode 100644 index 000000000..e325b7ebc --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'app' +include 'lib' diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/Pipeline.java b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/Pipeline.java new file mode 100644 index 000000000..9b2364cfd --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/Pipeline.java @@ -0,0 +1,11 @@ +package app; + +import app.build.StageBuilder; +import app.lib.build.LibStep; + +public class Pipeline { + public String run(String name) { + String staged = new StageBuilder().stage(name); + return new LibStep().apply(staged); + } +} diff --git a/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/build/StageBuilder.java b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/build/StageBuilder.java new file mode 100644 index 000000000..7ac14a55e --- /dev/null +++ b/graph/test/java/cases/64-gradle-build-output-is-not-source/src/src/main/java/app/build/StageBuilder.java @@ -0,0 +1,8 @@ +package app.build; + +// A source package named `build`, in a Gradle project: it has no build script beside it, so it is source. +public class StageBuilder { + public String stage(String name) { + return name + "-staged"; + } +} diff --git a/graph/test/java/expected/62-build-output-is-not-source.edges b/graph/test/java/expected/62-build-output-is-not-source.edges index 9768acfde..1779c8a8c 100644 --- a/graph/test/java/expected/62-build-output-is-not-source.edges +++ b/graph/test/java/expected/62-build-output-is-not-source.edges @@ -1,5 +1,5 @@ -ambiguous_unknown method app.WidgetValidator#check(String) -> - -ambiguous_unknown new app.WidgetValidator#check(String) -> - boundary_lib method app.WidgetValidator#check(String) -> external:String.isEmpty +known_edge method app.WidgetValidator#check(String) -> app.build.StepBuilder#next(String) known_edge method app.WidgetValidator#check(String) -> app.site.PageRenderer#render(String) +known_edge new app.WidgetValidator#check(String) -> app.build.StepBuilder#() known_edge new app.WidgetValidator#check(String) -> app.site.PageRenderer#() diff --git a/graph/test/java/expected/62-build-output-is-not-source.oracle b/graph/test/java/expected/62-build-output-is-not-source.oracle new file mode 100644 index 000000000..518b0e5c3 --- /dev/null +++ b/graph/test/java/expected/62-build-output-is-not-source.oracle @@ -0,0 +1 @@ +oracle=4 engine=4 agree=4 missing=0 (known 0, NEW 0) extra=0 diff --git a/graph/test/java/expected/62-build-output-is-not-source.type-use b/graph/test/java/expected/62-build-output-is-not-source.type-use index 6661dc348..237028423 100644 --- a/graph/test/java/expected/62-build-output-is-not-source.type-use +++ b/graph/test/java/expected/62-build-output-is-not-source.type-use @@ -1,5 +1,7 @@ ambiguous_unknown METHOD_PARAM 0 app.WidgetValidator#check(String) [METHOD_PARAM] -> - +ambiguous_unknown METHOD_PARAM 0 app.build.StepBuilder#next(String) [METHOD_PARAM] -> - ambiguous_unknown METHOD_PARAM 0 app.site.PageRenderer#render(String) [METHOD_PARAM] -> - +ambiguous_unknown METHOD_RETURN 0 app.build.StepBuilder#next(String) [METHOD] -> - ambiguous_unknown METHOD_RETURN 0 app.site.PageRenderer#render(String) [METHOD] -> - -ambiguous_unknown OBJECT_CREATION_TYPE 0 app.WidgetValidator#check(String) [EXPRESSION] -> - +known_edge OBJECT_CREATION_TYPE 0 app.WidgetValidator#check(String) [EXPRESSION] -> app.build.StepBuilder known_edge OBJECT_CREATION_TYPE 0 app.WidgetValidator#check(String) [EXPRESSION] -> app.site.PageRenderer diff --git a/graph/test/java/expected/63-javadoc-root-is-not-a-project.oracle b/graph/test/java/expected/63-javadoc-root-is-not-a-project.oracle new file mode 100644 index 000000000..97a1dbf27 --- /dev/null +++ b/graph/test/java/expected/63-javadoc-root-is-not-a-project.oracle @@ -0,0 +1 @@ +oracle=1 engine=1 agree=1 missing=0 (known 0, NEW 0) extra=0 diff --git a/graph/test/java/expected/64-gradle-build-output-is-not-source.edges b/graph/test/java/expected/64-gradle-build-output-is-not-source.edges new file mode 100644 index 000000000..7065eed6f --- /dev/null +++ b/graph/test/java/expected/64-gradle-build-output-is-not-source.edges @@ -0,0 +1,4 @@ +known_edge method app.Pipeline#run(String) -> app.build.StageBuilder#stage(String) +known_edge method app.Pipeline#run(String) -> app.lib.build.LibStep#apply(String) +known_edge new app.Pipeline#run(String) -> app.build.StageBuilder#() +known_edge new app.Pipeline#run(String) -> app.lib.build.LibStep#() diff --git a/graph/test/java/expected/64-gradle-build-output-is-not-source.oracle b/graph/test/java/expected/64-gradle-build-output-is-not-source.oracle new file mode 100644 index 000000000..518b0e5c3 --- /dev/null +++ b/graph/test/java/expected/64-gradle-build-output-is-not-source.oracle @@ -0,0 +1 @@ +oracle=4 engine=4 agree=4 missing=0 (known 0, NEW 0) extra=0 diff --git a/graph/test/java/expected/64-gradle-build-output-is-not-source.type-use b/graph/test/java/expected/64-gradle-build-output-is-not-source.type-use new file mode 100644 index 000000000..f7c8f9094 --- /dev/null +++ b/graph/test/java/expected/64-gradle-build-output-is-not-source.type-use @@ -0,0 +1,9 @@ +ambiguous_unknown LOCAL_VARIABLE 0 app.Pipeline#run(String) [LOCAL_VARIABLE] -> - +ambiguous_unknown METHOD_PARAM 0 app.Pipeline#run(String) [METHOD_PARAM] -> - +ambiguous_unknown METHOD_PARAM 0 app.build.StageBuilder#stage(String) [METHOD_PARAM] -> - +ambiguous_unknown METHOD_PARAM 0 app.lib.build.LibStep#apply(String) [METHOD_PARAM] -> - +ambiguous_unknown METHOD_RETURN 0 app.Pipeline#run(String) [METHOD] -> - +ambiguous_unknown METHOD_RETURN 0 app.build.StageBuilder#stage(String) [METHOD] -> - +ambiguous_unknown METHOD_RETURN 0 app.lib.build.LibStep#apply(String) [METHOD] -> - +known_edge OBJECT_CREATION_TYPE 0 app.Pipeline#run(String) [EXPRESSION] -> app.build.StageBuilder +known_edge OBJECT_CREATION_TYPE 0 app.Pipeline#run(String) [EXPRESSION] -> app.lib.build.LibStep diff --git a/graph/test/java/expected/oracle-agreement.txt b/graph/test/java/expected/oracle-agreement.txt index bad554eae..4bb656bac 100644 --- a/graph/test/java/expected/oracle-agreement.txt +++ b/graph/test/java/expected/oracle-agreement.txt @@ -1,4 +1,4 @@ -cases compared 48 agreeing 48 +cases compared 49 agreeing 49 CONSTRUCTOR-RULE disagreements: 0 cases, 0 rows OTHER disagreements: 0 rows naming no method: 0 diff --git a/parser/src/constants/consts.ts b/parser/src/constants/consts.ts index 5142f59a0..000999d09 100644 --- a/parser/src/constants/consts.ts +++ b/parser/src/constants/consts.ts @@ -6,7 +6,9 @@ import * as path from 'path'; export const HASH_ALGO = 'md5'; /** - * Directories to exclude when scanning projects or source files + * Directories to exclude when scanning projects or source files. + * The Java source walk does not skip `build` by this name: only a Gradle project's output + * (utils/generated-output.ts isGradleBuildOutput), since `build` is also a Java package. */ export const EXCLUDED_DIRS = new Set([ 'node_modules', diff --git a/parser/src/utils/generated-output.ts b/parser/src/utils/generated-output.ts index 3f2176c2d..02a169304 100644 --- a/parser/src/utils/generated-output.ts +++ b/parser/src/utils/generated-output.ts @@ -51,3 +51,25 @@ export function isGeneratedOutputDirectory(parent: string, name: string): boolea || (fs.existsSync(path.join(directory, 'navigation.html')) && fs.existsSync(path.join(directory, 'scripts', 'sourceset_dependencies.js'))); } + +/** + * The files that make a directory the root of a Gradle project or module. Gradle writes that + * project's output (compiled classes, processed resources, and generated sources such as + * `build/generated/sources/annotationProcessor/java/main`) into a `build` directory beside them. + */ +export const GRADLE_BUILD_FILES = ['build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts'] as const; + +/** + * Is `parent/name` a Gradle project's output directory, whose `.java` files a build wrote? + * + * Anchored on its owner, the way `target` is anchored on a `pom.xml` above: a `build` beside a + * Gradle build or settings script. `build` is also an ordinary Java package name (builders, build + * steps: `src/main/java/app/build/StepBuilder.java`), and pruning every directory of that name + * dropped such a package from the graph with no skipped-file row to say so. A `build` inside a + * source tree or a package path has no build script beside it, so it is walked. + * Used by the Java source walk (java-project-analyzer.ts); the refresher's file table + * (ax_fresh.py `gradle_output`) applies the same rule, so freshness and the language count agree. + */ +export function isGradleBuildOutput(parent: string, name: string): boolean { + return name === 'build' && GRADLE_BUILD_FILES.some((file) => fs.existsSync(path.join(parent, file))); +} diff --git a/parser/src/workflows/java/java-project-analyzer.ts b/parser/src/workflows/java/java-project-analyzer.ts index 118e4c82e..625e8880a 100644 --- a/parser/src/workflows/java/java-project-analyzer.ts +++ b/parser/src/workflows/java/java-project-analyzer.ts @@ -18,6 +18,7 @@ import { TypeAnnotation } from '@/analysis-types/java/TypeAnnotation'; import { TypeParameter } from '@/analysis-types/java/TypeParameter'; import { TypeReference } from '@/analysis-types/java/TypeReference'; import { TypeRegistry } from '@/analysis-types/java/TypeRegistry'; +import { isGradleBuildOutput } from '@/utils/generated-output'; import { EXCLUDED_DIRS, isJavaTestDir, ANALYSIS_OUTPUT_DIR, OUTPUT_TYPE_REGISTRY_CSV_FILENAME, OUTPUT_TYPE_PARAMETER_CSV_FILENAME, OUTPUT_TYPE_REFERENCE_CSV_FILENAME, OUTPUT_TYPE_ANNOTATION_CSV_FILENAME, OUTPUT_ANNOTATION_ARGUMENT_CSV_FILENAME, OUTPUT_METHOD_REGISTRY_CSV_FILENAME, OUTPUT_METHOD_PARAMETER_CSV_FILENAME, OUTPUT_METHOD_TYPE_PARAMETER_CSV_FILENAME, OUTPUT_ENUM_CONSTANT_CSV_FILENAME, OUTPUT_FIELD_REGISTRY_CSV_FILENAME, OUTPUT_FIELD_POSITION_CSV_FILENAME, OUTPUT_IMPORT_REGISTRY_CSV_FILENAME, OUTPUT_MODULE_REGISTRY_CSV_FILENAME, OUTPUT_MODULE_DIRECTIVE_CSV_FILENAME, OUTPUT_EXPRESSION_REFERENCE_CSV_FILENAME, OUTPUT_LOCAL_VARIABLE_REGISTRY_CSV_FILENAME, OUTPUT_BLOCK_REGISTRY_CSV_FILENAME, OUTPUT_COMMENT_REGISTRY_CSV_FILENAME, OUTPUT_SKIPPED_JAVA_FILES_CSV_FILENAME, JAVA_ENTITY_TYPES, FILE_EXTENSIONS, LARGE_FILE_LINE_THRESHOLD } from '@/constants/consts'; import { ENTITY_IDENTIFIERS } from '@/constants/entity-constants'; import { SkippedFileReason } from '@/enums/SkippedFileReason'; @@ -321,7 +322,12 @@ export class JavaProjectAnalyzer { for (const entry of entries) { if (entry.isDirectory()) { const isTestDir = isJavaTestDir(entry.name); - if (!EXCLUDED_DIRS.has(entry.name) && !entry.name.startsWith('.') && !(excludeTests && isTestDir) + // `build` is skipped only as a Gradle project's output, never by name: it is also a + // Java package name (see isGradleBuildOutput). + const excluded = entry.name === 'build' + ? isGradleBuildOutput(dirPath, entry.name) + : EXCLUDED_DIRS.has(entry.name); + if (!excluded && !entry.name.startsWith('.') && !(excludeTests && isTestDir) && !isGitIgnoredDir(path.join(dirPath, entry.name))) { const subPath = path.join(dirPath, entry.name); await this.scanForJavaFiles(subPath, files, excludeTests); diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index bcb55b70e..82e9850d7 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -66,7 +66,8 @@ # reads: a directory pruned here that the parser DOES read is an edit no refresh ever sees and that `index` calls up to # date, which is what one shared list did to `out`, `build`, `target` and `coverage` in Python and C# (and `coverage` # in Java). A directory watched here that the parser skips costs only a needless rebuild. Each set is the parser's own: -# java parser/src/constants/consts.ts EXCLUDED_DIRS, and every directory whose name starts with a dot +# java parser/src/constants/consts.ts EXCLUDED_DIRS except `build`, and every directory whose name starts with +# a dot; a `build` directory is skipped only as a Gradle project's output (gradle_output below) # typescript parser/src/constants/typescript-constants.ts TS_SKIP_DIRECTORIES, and dot directories # javascript parser/src/constants/javascript-constants.ts JS_SKIP_DIRECTORIES # python parser/src/workflows/python/python-project-analyzer.ts DEFAULT_EXCLUDES, `*.egg-info`, and under a @@ -75,7 +76,7 @@ # .axiomcode (the graph's own directory) and .git are pruned for every language: the build puts .axiomcode in # .git/info/exclude, so the parser skips it too. Directories git ignores are pruned for every language (git_ignored_dirs). SKIP = { - 'java': frozenset({'node_modules', '.git', '.idea', '.vscode', 'dist', 'build', 'target', 'out', '__pycache__', + 'java': frozenset({'node_modules', '.git', '.idea', '.vscode', 'dist', 'target', 'out', '__pycache__', '.pytest_cache', 'venv', 'env'}), 'typescript': frozenset({'node_modules', '.git', 'dist', 'build', 'out', 'coverage', '.next', '.nuxt', '.turbo', '.cache', '.yarn', 'bower_components'}), @@ -112,6 +113,15 @@ def generated_output(parent, name): return os.path.isfile(os.path.join(d, 'element-list')) or os.path.isfile(os.path.join(d, 'package-list')) or \ (os.path.isfile(os.path.join(d, 'navigation.html')) and os.path.isfile(os.path.join(d, 'scripts', 'sourceset_dependencies.js'))) +# A GRADLE PROJECT'S OUTPUT. The Java parser skips a directory named `build` only when a Gradle build or settings +# script sits beside it (parser/src/utils/generated-output.ts isGradleBuildOutput): `build` is also a Java package name, +# and one inside a source tree is read, so an edit there must be watched and its files counted. +GRADLE_BUILD_FILES = ('build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts') + +def gradle_output(parent, name): + """True when `parent`/`name` is the `build` directory a Gradle project or module writes its output into""" + return name == 'build' and any(os.path.isfile(os.path.join(parent, f)) for f in GRADLE_BUILD_FILES) + def out_dir(repo): return os.path.join(repo, '.axiomcode', 'out') def table_path(repo): return os.path.join(out_dir(repo), 'files.json') def state_path(repo): return os.path.join(repo, '.axiomcode', 'refresh.json') @@ -170,6 +180,7 @@ def watched(root, lang): if ignored and os.path.normpath(os.path.join(rd, s)) in ignored: continue o = off | {l for l in langs if prunes(l, s, parent)} if web and not web <= o and generated_output(d, s): o = o | web # a build's output (#1545) + if 'java' in langs and 'java' not in o and gradle_output(d, s): o = o | {'java'} # Gradle's output if all(l in o for l in langs): continue keep.append(s); hidden[os.path.join(d, s)] = o subdirs[:] = keep @@ -377,6 +388,12 @@ def change_key(c): def enabled(repo): return not os.environ.get('AXIOMCODE_NO_REFRESH') and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) +def refresh_off(repo): + """the refresher is switched off (AXIOMCODE_NO_REFRESH) over a graph of this repository's own: nothing rebuilds it, + but an answer from it can still say which files it predates. A graph placed by AXIOMCODE_GRAPH has no file table + of this tree to compare with""" + return bool(os.environ.get('AXIOMCODE_NO_REFRESH')) and not os.environ.get('AXIOMCODE_GRAPH') and has_graph(repo) + def kick(repo, trigger='an edit'): """start the worker and return at once; a no-op without a graph (the FIRST build takes minutes and is the caller's decision, see ax_contract.ensure_graph) or when one is already running""" @@ -530,9 +547,11 @@ def pending_note(s): return (f"graph refresh: the {', '.join(s['pending'])} graph{'s are' if len(s['pending']) > 1 else ' is'} still being built; " + '; '.join(says) + "; ask again when `axiomcode index` finishes") -def note(s, marked=None): +def note(s, marked=None, named=None, off=False): """one line for an answer given from a graph that is behind the files, or '' when it is not. `marked` is how many - of the answer's rows lie in those files (and carry the mark), None when the answer was not looked at""" + of the answer's rows lie in those files (and carry the mark), None when the answer was not looked at. `named` is + [(name, file)]: a name the query asked about that is written in one of those files, so a "nothing named X" is the + graph predating the edit that wrote X, not X being absent. `off`: the refresher is switched off, nothing rebuilds""" # the languages a running build has still to publish (#1555) get a line of their own, before any line about edits first = pending_note(s) if s.get('state') not in ('stale', 'building'): return first @@ -543,6 +562,13 @@ def note(s, marked=None): rows = ('' if marked is None else f"; {marked} row(s) lie in those files and are marked {MARK.strip()}: read them for their current text" if marked else "; no row of this answer lies in those files") + if named: + rows += '; ' + ', '.join(f"'{n}' is written in {f}" for n, f in named[:3]) + \ + ", edited since the graph was built: a declaration added there is not in this graph yet, so finding nothing by that name does not mean it is absent" + if off: + return first + (f"graph refresh: OFF (AXIOMCODE_NO_REFRESH is set), no refresh is running — this answer is from a graph that " + f"predates edits to {head}" + (rows if marked is not None else '; read those files for their current text') + + "; `axiomcode index` rebuilds it") if s.get('failed'): why = s.get('failed_reason') or '' return first + (f"graph refresh: the last rebuild FAILED" + (f" — {why}" if why else '') + f" (see {s['failed']}); this answer is from the " @@ -563,6 +589,16 @@ def note(s, marked=None): _CODE_LINE = re.compile(r'^\s*(\d+ )?\| ') # a line of quoted source (context --source): never marked _TOKEN = re.compile(r'[\w.$+@/-]+') _LOC_KEYS = ('at', 'declared_at', 'call_at', 'file', 'path', 'site', 'location') +# how impact and path say a name matched nothing (axiomcode-impact, axiomcode-path), and the name they say it of +NOT_FOUND = re.compile(r"(?:nothing named|nothing of kind \w+ named|no type named|no declaration in the graph contains|no declaration named)" + r" '?([^\s',]+?)'?(?=[\s,.]|$)", re.M) + +def not_found(named, out, code): + """the (name, file) of `named` the answer found nothing by: those its refusal line names, or, for a refusal that + names none, all of them. `path A B` with only A missing blames A, not B""" + said = {re.split(r'[.#:/$()<>,\s]+', m.strip('()'))[-1] for m in NOT_FOUND.findall(out)} - {''} + if said: return [(n, f) for n, f in named if re.split(r'[.#:/$()<>,\s]+', n.strip('()'))[-1] in said] + return list(named) if code != 0 else [] class Stale: """the set of edited files, and whether a path printed in an answer is one of them. Answers print paths relative to @@ -631,22 +667,25 @@ def query_names(verb, args, repo): return names def names_in_edits(repo, names, stale): - """True when a name asked about is declared or written in an edited file: a target given as file:line in one, or a - name whose last part appears as a word in one (a declaration just added is in no graph yet)""" - words = set() + """[(name, file)] for each name asked about that is declared or written in an edited file: a target given as + file:line in one, or a name whose last part appears as a word in one (a declaration just added is in no graph yet). + Empty when none is""" + words, found = {}, [] for n in names: - if stale.hit(n): return True + if stale.hit(n): found.append((n, n.split(':', 1)[0])); continue w = re.split(r'[.#:/$()<>,\s]+', n.strip('()')) w = [x for x in w if re.match(r'^\w+$', x)] - if w: words.add(w[-1]) - if not words: return False + if w: words.setdefault(w[-1], n) + if not words: return found pat = re.compile(r'\b(' + '|'.join(re.escape(w) for w in sorted(words)) + r')\b') for f in stale.files: try: with open(os.path.join(repo, f), 'rb') as fh: text = fh.read(4 << 20).decode('utf-8', 'replace') except OSError: continue - if pat.search(text): return True - return False + for w in sorted(set(pat.findall(text))): + if words.pop(w, None) is not None: found.append((w, f)) + if not words: break + return found def build_seconds(repo): """(seconds the last build of this repository took to a usable graph, whether it compiled the engine's rules), from @@ -698,22 +737,28 @@ def run(): def passthrough(): os.execvp(argv[0], argv) def with_note(n): # the answer as it is, then one line about it on stderr r = subprocess.run(argv); print(n, file=sys.stderr); return r.returncode - if not enabled(repo): + # REFRESH SWITCHED OFF IS NOT "UP TO DATE". With AXIOMCODE_NO_REFRESH nothing rebuilds the graph, which is exactly when + # an answer from it most needs to say which edits it predates: a "nothing named X" for an X an edit just added read + # as X not existing. It is checked, marked and noted as below; it only never kicks a refresh or waits for one. + off = not enabled(repo) and refresh_off(repo) + if not enabled(repo) and not off: # the refresh switched off still leaves a build that is solving the repository's other languages (#1555); a # graph placed by AXIOMCODE_GRAPH is not this build's, and says nothing n = '' if os.environ.get('AXIOMCODE_GRAPH') or not has_graph(repo) else pending_note(pending(repo)) if not n: passthrough() return with_note(n) s = status(repo) - if s['state'] == 'unknown': kick(repo, 'a query'); passthrough() + if s['state'] == 'unknown': + if not off: kick(repo, 'a query') + passthrough() if s['state'] in ('fresh', 'no graph') or not edited(s): # a build that published this tree's main graph and is solving the others (#1555): the answer is current and is # given now, and says which languages it cannot see yet if not pending_note(s): passthrough() return with_note(pending_note(s)) - if not s.get('failed'): kick(repo, 'a query') + if not s.get('failed') and not off: kick(repo, 'a query') as_json = '--json' in argv - if fresh and not s.get('failed'): + if fresh and not s.get('failed') and not off: left = expected_left(repo) print(f"waiting for the graph to refresh (--fresh): {len(edited(s))} file(s) changed since the graph was built" + (f", the last build took {int(build_seconds(repo)[0])} s" if left is not None else '') + " …", file=sys.stderr, flush=True) @@ -722,8 +767,9 @@ def with_note(n): # the answer as it is, then stale = Stale(edited(s)); r = run() out = r.stdout.decode('utf-8', 'replace') marked, n, touched = mark_answer(out, stale, as_json) - touched = touched or names_in_edits(repo, query_names(verb, argv, repo), stale) - if touched and not fresh and not s.get('failed'): + named = names_in_edits(repo, query_names(verb, argv, repo), stale) + touched = touched or bool(named) + if touched and not fresh and not s.get('failed') and not off: budget = float(os.environ.get('AXIOMCODE_FRESH_WAIT') or 30) left = expected_left(repo) if budget > 0 and not compiling(repo) and (left is None or left <= budget): @@ -736,15 +782,20 @@ def with_note(n): # the answer as it is, then r = run(); out = r.stdout.decode('utf-8', 'replace') marked, n, _ = mark_answer(out, stale, as_json) else: passthrough() + # a name asked about that an edit wrote is worth a word only when the answer found nothing by it: a refusal (the + # verb's non-zero exit) or, in a repository in several languages whose other graph did answer, that graph's line + missed = not_found(named, out, r.returncode) if named else [] if as_json: try: obj = json.loads(marked) if isinstance(obj, dict): - obj['freshness'] = dict(state=s['state'], edited=edited(s), rows_marked=n, **({'failed': s['failed']} if s.get('failed') else {})) + obj['freshness'] = dict(state='off' if off else s['state'], edited=edited(s), rows_marked=n, + **({'failed': s['failed']} if s.get('failed') else {}), + **({'named_in_edits': [dict(name=a, file=b) for a, b in missed]} if missed else {})) marked = json.dumps(obj, indent=1, ensure_ascii=False) + '\n' except ValueError: pass sys.stdout.write(marked); sys.stdout.flush() - msg = note(s, n) + msg = note(s, n, named=missed, off=off) if msg: print(msg, file=sys.stderr) return r.returncode diff --git a/tests/freshness.py b/tests/freshness.py index 6f190cb71..09ef0b699 100644 --- a/tests/freshness.py +++ b/tests/freshness.py @@ -16,6 +16,8 @@ when the running build is compiling rules, or when the answer touches no edited file fresh --fresh waits whatever the estimate, with progress on stderr, and answers unmarked; a current graph is answered with no mark and no note + named "nothing named X" for an X an edit newer than the graph wrote says so, names the file and whether a refresh + runs, also with AXIOMCODE_NO_REFRESH; a name no edit writes, or one the answer found, is not blamed mcp the MCP tools take fresh=true and pass --fresh, and the CLI's --fresh is written fresh=True in an answer No engine: the wait checks drive `ax_fresh.py query` with a stand-in verb, and a stand-in refresh that brings the file @@ -55,7 +57,8 @@ def ts_list(rel, name): def prune_checks(): parser = { - 'java': ts_list('constants/consts.ts', 'EXCLUDED_DIRS'), + # the Java walk skips `build` only beside a Gradle build script (isGradleBuildOutput), never by name + 'java': ts_list('constants/consts.ts', 'EXCLUDED_DIRS') - {'build'}, 'typescript': ts_list('constants/typescript-constants.ts', 'TS_SKIP_DIRECTORIES'), 'javascript': ts_list('constants/javascript-constants.ts', 'JS_SKIP_DIRECTORIES'), 'python': ts_list('workflows/python/python-project-analyzer.ts', 'DEFAULT_EXCLUDES'), @@ -70,13 +73,16 @@ def prune_checks(): 'python': ['app/out/calc.py', 'app/build/calc.py', 'target/t.py', 'coverage/c.py', 'build/lib/gen.py', 'build/frontend.py', '.venv/v.py', 'pkg.egg-info/e.py', 'dist/d.py', 'app/ok.py'], 'java': ['src/main/java/app/coverage/Calc.java', 'src/main/java/app/Ok.java', 'out/O.java', 'target/T.java', - 'build/B.java', '.hidden/H.java'], + 'build/B.java', '.hidden/H.java', 'src/main/java/app/build/StepBuilder.java', + 'mod/build.gradle.kts', 'mod/build/generated/sources/annotationProcessor/java/main/app/Gen.java', + 'mod/src/main/java/app/build/Step.java'], 'csharp': ['src/Shop/build/Calc.cs', 'src/Shop/out/O.cs', 'target/T.cs', 'coverage/C.cs', 'src/Shop/Ok.cs', 'src/Shop/obj/G.cs', 'bin/B.cs', 'packages/P.cs'], } read = { # what the parser reads (True) or skips (False), by its own lists 'python': [True, True, True, True, False, True, False, False, False, True], - 'java': [True, True, False, False, False, False], + # a `build` package is read; only a Gradle module's build/ (beside mod/build.gradle.kts, itself watched) is skipped + 'java': [True, True, False, False, True, False, True, True, False, True], 'csharp': [True, True, True, True, True, False, False, False], } for lang, files in tree.items(): @@ -168,9 +174,12 @@ def refresh(): open(os.path.join(repo, 'swapped'), 'w').write('new graph\n') th = threading.Thread(target=refresh) if after is not None else None if th: th.start() - verb_cmd = [sys.executable, '-c', "import os,sys; print(open('swapped').read().strip() if os.path.exists('swapped') else sys.argv[1])", verb_out] - e = dict(os.environ, AXIOMCODE_FRESH_WAIT='30', **(env or {})) - for k in ('AXIOMCODE_NO_REFRESH', 'AXIOMCODE_GRAPH'): e.pop(k, None) + # the stand-in verb exits VERB_EXIT (a refusal such as "nothing named X" exits 2) until the refresh swaps its graph in + verb_cmd = [sys.executable, '-c', "import os,sys; s=os.path.exists('swapped'); print(open('swapped').read().strip() if s else sys.argv[1]); " + "sys.exit(0 if s else int(os.environ.get('VERB_EXIT') or 0))", verb_out] + e = dict(os.environ, AXIOMCODE_FRESH_WAIT='30') + for k in ('AXIOMCODE_NO_REFRESH', 'AXIOMCODE_GRAPH', 'VERB_EXIT'): e.pop(k, None) + e.update(env or {}) if 'AXIOMCODE_FRESH' not in (env or {}): e.pop('AXIOMCODE_FRESH', None) t0 = time.time() r = subprocess.run([sys.executable, driver, SCRIPTS, repo, verb, '--', *verb_cmd, *args, repo], cwd=repo, @@ -216,6 +225,54 @@ def wait_checks(): shutil.rmtree(work, ignore_errors=True) +# ── named ───────────────────────────────────────────────────────────────────────────────────────────────────────── +NOT_FOUND = "nothing named 'audit' in the graph, and nothing close to it." + + +def named_checks(): + """a "nothing named X" for an X an edit newer than the graph wrote says the graph is stale, names the file, and says + whether a refresh is running — including when the refresher is switched off. Each with a control that stays quiet""" + work = tempfile.mkdtemp(prefix='axiomcode-named-') + try: + # the gap as it was met: AXIOMCODE_NO_REFRESH set, the answer came back bare, as if audit did not exist + out, err, took = ask(fake_repo(work, 'off'), NOT_FOUND, env=dict(AXIOMCODE_NO_REFRESH='1', VERB_EXIT='2'), args=('audit',)) + check(f"named: refresh OFF — a name an edit added says the graph predates that edit and that no refresh runs ({took:.1f}s)", + out.strip() == NOT_FOUND and took < 1.5 and 'graph refresh: OFF' in err and 'predates edits to shop/api.py' in err + and "'audit' is written in shop/api.py" in err and '`axiomcode index` rebuilds it' in err, (out, err)) + # a refresh too long to wait for: the note already said "queued"; it now also says why nothing was found + out, err, took = ask(fake_repo(work, 'queued', build_seconds='300 0'), NOT_FOUND, env=dict(VERB_EXIT='2'), args=('audit',)) + check(f"named: refresh queued — the note names the edited file that writes the name ({took:.1f}s)", + took < 1.5 and 'graph refresh: queued' in err and "'audit' is written in shop/api.py" in err, (out, err)) + # --json carries it as data + out, err, took = ask(fake_repo(work, 'json', build_seconds='300 0'), json.dumps(dict(error=NOT_FOUND)), + env=dict(VERB_EXIT='2'), args=('audit', '--json')) + fr = (json.loads(out) if out.strip().startswith('{') else {}).get('freshness', {}) + check("named: --json says named_in_edits", + fr.get('named_in_edits') == [dict(name='audit', file='shop/api.py')] and fr.get('state') == 'stale', (out, err)) + # CONTROL: a name that no edit writes, refresh off — the graph is still said to be stale, but no name is blamed + out, err, took = ask(fake_repo(work, 'ghost'), "nothing named 'ghost' in the graph, and nothing close to it.", + env=dict(AXIOMCODE_NO_REFRESH='1', VERB_EXIT='2'), args=('ghost',)) + check("named: control — a name no edit writes gets the stale note but no 'is written in'", + 'graph refresh: OFF' in err and 'is written in' not in err, (out, err)) + # CONTROL: `path audit total` with only audit missing blames audit, not total (which the edited file also writes) + out, err, took = ask(fake_repo(work, 'path', build_seconds='300 0'), NOT_FOUND, env=dict(VERB_EXIT='2'), verb='path', + args=('audit', 'total')) + check("named: control — path with one endpoint missing names only that one", + "'audit' is written in shop/api.py" in err and "'total' is written" not in err, (out, err)) + # CONTROL: a name found in the graph that the edited file also writes: the answer found it, nothing to explain + out, err, took = ask(fake_repo(work, 'found', build_seconds='300 0'), ROWS, args=('run',)) + check("named: control — a name the answer found says nothing about where it is written", + 'graph refresh: queued' in err and 'is written in' not in err, (out, err)) + # CONTROL: refresh off over a graph that matches the files: no mark, no note, as before + repo = fake_repo(work, 'offcurrent'); driver = os.path.join(work, 'driver.py') + r = subprocess.run([sys.executable, driver, SCRIPTS, repo, 'impact', '--', sys.executable, '-c', f"print({ROWS!r})"], + capture_output=True, text=True, env=dict(os.environ, AXIOMCODE_NO_REFRESH='1')) + check("named: control — refresh off over a current graph answers with no mark and no note", + r.stdout.strip() == ROWS.strip() and not r.stderr.strip(), (r.stdout, r.stderr)) + finally: + shutil.rmtree(work, ignore_errors=True) + + # ── mcp ─────────────────────────────────────────────────────────────────────────────────────────────────────────── def mcp_checks(): spec = importlib.util.spec_from_file_location('axiomcode_mcp_server', os.path.join(ROOT, 'plugins', 'axiomcode', 'mcp', 'server.py')) @@ -239,7 +296,7 @@ def mcp_checks(): if __name__ == '__main__': - prune_checks(); marks_checks(); wait_checks(); mcp_checks() + prune_checks(); marks_checks(); wait_checks(); named_checks(); mcp_checks() bad = [n for n, ok in RESULTS if not ok] print(f"\n{len(RESULTS) - len(bad)} of {len(RESULTS)} passed" + (f"; FAILED: {len(bad)}" if bad else '')) sys.exit(1 if bad or not RESULTS else 0)