diff --git a/bin/axiomcode b/bin/axiomcode index fa44d4bbb..3e1d32c87 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -20,6 +20,8 @@ # --language [,…] Restrict the run to these languages: java, typescript, python, # javascript or csharp. # --debug Also write csv/*.csv and keep raw/ beside each graph. +# --progress Append " ok " or " failed" to as each language +# finishes, so a caller can use one graph while the next is solved. # # Engine options: # --dispatch-cap --lib-depth --jdk-depth --taint @@ -188,10 +190,10 @@ case "$cmd" in ;; all) need_parser - lang=""; src=""; out=""; version=""; libs=""; popts=(); rest=(); pos=() + lang=""; src=""; out=""; version=""; libs=""; progress=""; popts=(); rest=(); pos=() while [ $# -gt 0 ]; do case "$1" in --language) lang="$2"; shift 2;; --src) src="$2"; shift 2;; --out) out="$2"; shift 2;; - --library) libs="$2"; shift 2;; + --library) libs="$2"; shift 2;; --progress) progress="$2"; shift 2;; --version|--slug) version="$2"; shift 2;; --exclude-tests) popts+=(--exclude-tests); shift;; --intermediate|--dispatch-cap|--lib-depth|--jdk-depth|--taint) rest+=("$1" "$2"); shift 2;; --debug) rest+=(--debug); shift;; @@ -259,7 +261,10 @@ case "$cmd" in # which languages the IR holds — one folder each — and which of them have an engine found=(); for d in "$ir"/*/; do l="$(basename "$d")"; [ -d "$ROOT/graph/$l/engine" ] && found+=("$l"); done [ ${#found[@]} -gt 0 ] || { echo "❌ no language with an engine in the IR (folders: $(ls "$ir" | tr '\n' ' '))" >&2; exit 1; } - langs=("${found[@]}") + # THE LARGEST LANGUAGE FIRST (#1555): a caller that publishes each graph as it is solved (--progress) gets the one + # most of the repository is written in first, not whichever sorts first. The IR's size stands in for its share; a + # caller that names --language chooses the order itself. + langs=($(for l in "${found[@]}"; do echo "$(du -sk "$ir/$l" 2>/dev/null | cut -f1) $l"; done | sort -s -k1,1nr | cut -d' ' -f2)) if [ ${#want[@]} -eq 1 ]; then case " ${found[*]} " in *" $lang "*) langs=("$lang");; *) echo "❌ --language=$lang, but the IR holds only: ${found[*]}" >&2; exit 1;; esac elif [ ${#want[@]} -gt 1 ]; then @@ -287,9 +292,10 @@ case "$cmd" in echo "▶ solving $l → $out/$l/graph.sqlite" if "$0" engine --language "$l" --client-ir "$ir/$l" --out "$out/$l" --intermediate "$int" --library "$libroots" \ --meta "source_version=$version" --meta "source_dir=$(cd "$src" && pwd)" ${rest[@]+"${rest[@]}"}; then - solved+=("$l") + solved+=("$l"); [ -z "$progress" ] || echo "$l ok $out/$l/graph.sqlite" >> "$progress" else failed+=("$l"); echo "❌ $l failed; continuing with the remaining languages" >&2 + [ -z "$progress" ] || echo "$l failed" >> "$progress" fi done for l in ${solved[@]+"${solved[@]}"}; do echo "$out/$l/graph.sqlite"; done diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py index 5da78a88b..252f79a8a 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py @@ -391,9 +391,9 @@ def ensure_graph(repo, db): # a build is running: never a silent wait. Under the MCP server answer at once with the stage it is at; from a # shell wait for it, printing the stage as it moves; either way, use the graph the moment it exists if auto and not os.environ.get('AXIOMCODE_BUILD_NOWAIT'): - print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr) - _follow(rr, lambda: ax_fresh.building(rr)) - if os.path.exists(db): return True + print(f"a graph build is already running for {repo}; waiting for its first graph …", file=sys.stderr) + _follow(rr, lambda: ax_fresh.building(rr) and not os.path.exists(db)) + if os.path.exists(db): return _published(rr) if auto and os.environ.get('AXIOMCODE_BUILD_NOWAIT'): _BUILD_NOTE.append(building_note(rr)); return False if not auto or os.environ.get('AXIOMCODE_GRAPH'): return False @@ -411,9 +411,9 @@ def ensure_graph(repo, db): nowait = bool(os.environ.get('AXIOMCODE_BUILD_NOWAIT')) if ax_fresh.building(repo): if nowait: _BUILD_NOTE.append(building_note(repo)); return False - print(f"a graph build is already running for {repo}; waiting for it …", file=sys.stderr) - _follow(repo, lambda: ax_fresh.building(repo)) - return os.path.exists(db) + print(f"a graph build is already running for {repo}; waiting for its first graph …", file=sys.stderr) + _follow(repo, lambda: ax_fresh.building(repo) and not os.path.exists(db)) + return os.path.exists(db) and _published(repo) env = None if ax_fresh.has_graph(rr): # a graph WAS built here and its pointer is broken: a repair, which keeps the baseline `changed` and test-impact @@ -432,14 +432,43 @@ def ensure_graph(repo, db): up = time.time() + 10 while p.poll() is None and not ax_fresh.building(repo) and time.time() < up: time.sleep(0.05) end = time.time() + float(os.environ.get('AXIOMCODE_BUILD_WAIT') or 60) - while p.poll() is None and time.time() < end: time.sleep(0.5) - if p.poll() is None: _BUILD_NOTE.append(building_note(repo)); return False + while p.poll() is None and not os.path.exists(db) and time.time() < end: time.sleep(0.5) + if p.poll() is None: return _published(repo) if os.path.exists(db) else (_BUILD_NOTE.append(building_note(repo)) or False) return p.returncode == 0 and os.path.exists(db) - p = subprocess.Popen([bash, build, repo], env=env) - _follow(repo, lambda: p.poll() is None) + # THE ANSWER WAITS FOR ITS GRAPH, NOT FOR THE WHOLE BUILD (#1555). The build publishes the main language's graph as + # soon as that language is solved and goes on solving the others under its lock, so the wait ends when graph.sqlite + # appears. The build writes to a log, copied here to stderr as it comes, not to this process's pipes: a caller that + # captures them (an agent's shell tool) reads until they close, and a build still solving other languages would hold + # them open to its end. + os.makedirs(os.path.join(repo, '.axiomcode'), exist_ok=True) + logp = os.path.join(repo, '.axiomcode', 'first-build.log') + with open(logp, 'w') as log: + p = subprocess.Popen([bash, build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env) + shown = [0] + def relay(): + try: + with open(logp, 'rb') as f: f.seek(shown[0]); b = f.read() + except OSError: return + shown[0] += len(b); sys.stderr.write(b.decode('utf-8', 'replace')); sys.stderr.flush() + _follow(repo, lambda: relay() or (p.poll() is None and not os.path.exists(db))) + relay() + if p.poll() is None: return os.path.exists(db) and _published(repo) return p.returncode == 0 and os.path.exists(db) +def _published(repo): + """the main graph is out while the build goes on solving the repository's other languages: the answer is given now, + and names on stderr (a `graph refresh:` line, which the MCP server carries into the answer) what it cannot see yet""" + import ax_fresh + n = '' + for _ in range(20): # the build names what is still to come just before the pointer moves + n = ax_fresh.note(ax_fresh.status(repo)) + if n or not ax_fresh.building(repo): break + time.sleep(0.1) + if n: print(n, file=sys.stderr) + return True + + _BUILD_NOTE = [] TICK = 20.0 diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index 8b375db2f..e94735c23 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -325,6 +325,18 @@ def wait_build(repo, seconds, say=True): time.sleep(0.5) return True +def pending(repo): + """what a running build has still to publish once the main graph is out (axiomcode-build, #1555): {} when nothing, + or when no build holds the lock (a build that was killed leaves the file, and it means nothing then). Else + `pending` (the languages, in solve order), `pending_old` (those a previous graph still answers for), and `first` + (a first build: there is no earlier graph of anything to wait for).""" + try: rows = [l.split() for l in open(os.path.join(out_dir(repo), 'building')) if l.strip()] + except OSError: return {} + if not rows or not building(repo): return {} + langs = [r[0] for r in rows] + old = [l for l in langs if os.path.isfile(os.path.join(repo, '.axiomcode', 'lang', l, 'out', 'graph.sqlite'))] + return dict(pending=langs, pending_old=old, first=all(len(r) > 1 and r[1] == 'first' for r in rows)) + def status(repo): if not has_graph(repo): return dict(state='no graph') if graph_broken(repo): return dict(state='building' if building(repo) else 'stale', changed=['.axiomcode/out/graph.sqlite (the pointer to the graph is broken)'], added=[], removed=[]) @@ -333,7 +345,10 @@ def status(repo): if c is None: return dict(state='unknown', note='the graph predates the file table; the next `axiomcode index` records it') changed, added, removed = c busy = building(repo) - if not (changed or added or removed): return dict(state='building' if busy else 'fresh') + if not (changed or added or removed): + # a build that already published this tree's main graph and is still solving other languages: the graph a query + # reads is current, and waiting would be waiting for other languages' compiles + return dict(state='building', **pending(repo)) if busy else dict(state='fresh') d = dict(state='building' if busy else 'stale', changed=changed, added=added, removed=removed) if not busy and st.get('failed_table') == change_key(c): d['failed'] = st.get('failed_log', ''); d['failed_reason'] = st.get('failed_reason', '') return d @@ -438,12 +453,16 @@ def failure_reason(out): def wait(repo, seconds): """kick, then wait until the graph is fresh or `seconds` have passed. Returns the status at the end.""" - if not enabled(repo): return dict(state='off') # switched off, or a graph placed by AXIOMCODE_GRAPH: nothing to say + if not enabled(repo): # switched off, or a graph placed by AXIOMCODE_GRAPH: nothing to say, + p = {} if os.environ.get('AXIOMCODE_GRAPH') or not has_graph(repo) else pending(repo) # but the languages a build + return dict(state='building', **p) if p else dict(state='off') # has yet to publish end = time.time() + seconds while True: s = status(repo) if s['state'] == 'unknown': kick(repo, 'a query'); return s # a graph from before the file table: its first refresh records one - if s['state'] in ('fresh', 'no graph') or s.get('failed'): return s + # a FIRST build's other languages are not waited for: there is no graph of theirs to refresh, only a compile + # that can take an hour (#1555). A rebuild's are, for the bounded while any refresh is. + if s['state'] in ('fresh', 'no graph') or s.get('failed') or s.get('first'): return s if s['state'] == 'stale': kick(repo, 'a query') # idempotent: a no-op while a worker holds its lock if time.time() >= end: return s time.sleep(0.5) @@ -484,21 +503,34 @@ def edited(s): """the files a stale graph predates: changed, added and removed since it was built""" return list(s.get('changed', [])) + list(s.get('added', [])) + list(s.get('removed', [])) +def pending_note(s): + """one line naming the languages a build has published the main graph without, or '' (#1555)""" + if not s.get('pending'): return '' + new = [l for l in s['pending'] if l not in s.get('pending_old', [])]; old = s.get('pending_old', []) + says = [] + if new: says.append(f"this answer has no {', '.join(new)} code yet") + if old: says.append(f"its {', '.join(old)} code comes from the previous graph") + return (f"graph refresh: the {', '.join(s['pending'])} graph{'s are' if len(s['pending']) > 1 else ' is'} still being built; " + + '; '.join(says) + "; ask again when `axiomcode index` finishes") + def note(s, marked=None): """one line for an answer given from a graph that is behind the files, or '' when it is not. `marked` is how many of the answer's rows lie in those files (and carry the mark), None when the answer was not looked at""" - if s.get('state') not in ('stale', 'building'): return '' + # the languages a running build has still to publish (#1555) get a line of their own, before any line about edits + first = pending_note(s) + if s.get('state') not in ('stale', 'building'): return first files = edited(s) - if not files: return '' + if not files: return first + first = first + '\n' if first else '' head = ', '.join(files[:5]) + (f" … +{len(files) - 5}" if len(files) > 5 else '') rows = ('' if marked is None else f"; {marked} row(s) lie in those files and are marked {MARK.strip()}: read them for their current text" if marked else "; no row of this answer lies in those files") if s.get('failed'): why = s.get('failed_reason') or '' - return (f"graph refresh: the last rebuild FAILED" + (f" — {why}" if why else '') + f" (see {s['failed']}); this answer is from the " + return first + (f"graph refresh: the last rebuild FAILED" + (f" — {why}" if why else '') + f" (see {s['failed']}); this answer is from the " f"previous graph, which predates edits to {head}" + rows) - return (f"graph refresh: {'rebuilding' if s['state'] == 'building' else 'queued'} — this answer is from the previous graph, " + return first + (f"graph refresh: {'rebuilding' if s['state'] == 'building' else 'queued'} — this answer is from the previous graph, " f"which predates edits to {head}" + (rows if marked is not None else '; read those files for their current text') + "; --fresh waits for the rebuild") @@ -647,10 +679,21 @@ def query(repo, verb, argv, fresh=False): def run(): return subprocess.run(argv, stdout=subprocess.PIPE) def passthrough(): os.execvp(argv[0], argv) - if not enabled(repo): passthrough() + def with_note(n): # the answer as it is, then one line about it on stderr + r = subprocess.run(argv); print(n, file=sys.stderr); return r.returncode + if not enabled(repo): + # the refresh switched off still leaves a build that is solving the repository's other languages (#1555); a + # graph placed by AXIOMCODE_GRAPH is not this build's, and says nothing + n = '' if os.environ.get('AXIOMCODE_GRAPH') or not has_graph(repo) else pending_note(pending(repo)) + if not n: passthrough() + return with_note(n) s = status(repo) if s['state'] == 'unknown': kick(repo, 'a query'); passthrough() - if s['state'] in ('fresh', 'no graph') or not edited(s): passthrough() + if s['state'] in ('fresh', 'no graph') or not edited(s): + # a build that published this tree's main graph and is solving the others (#1555): the answer is current and is + # given now, and says which languages it cannot see yet + if not pending_note(s): passthrough() + return with_note(pending_note(s)) if not s.get('failed'): kick(repo, 'a query') as_json = '--json' in argv if fresh and not s.get('failed'): diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index feeeb48af..a2c2b97d2 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -11,6 +11,8 @@ # A repository in several languages gets one graph per language. The main language's is the one it always had, # .axiomcode/out/graph.sqlite; every other language's is .axiomcode/lang//out/graph.sqlite, and the query verbs # (the `axiomcode` dispatcher) ask each of them. Each graph is one language: no call is followed from one to another. +# Each is published as soon as its own language is solved, the main one first, so a query never waits for another +# language's compile; until the last one is in, .axiomcode/out/building names the languages still to come. set -euo pipefail H="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "${1:-.}" && pwd)"; SRC="$REPO/${AXIOMCODE_SRC:-}"; SRC="${SRC%/}" @@ -98,13 +100,21 @@ else fi unset AXIOMCODE_GRAPH_LANG # a build writes every language's graph, never one chosen for a query OTHER_LANGS=""; [ "$LANGS" != "$LANG_ARG" ] && OTHER_LANGS="${LANGS#*,}" +# THE ORDER THE ENGINE SOLVES IN (#1555): the main language first, then the rest by file count, most first. Each graph is +# published the moment its language is solved, so this is the order they become queryable in. LANGS keeps its own order: +# it is part of the stamp and the file table, and a new order there would rebuild every mixed repository once. +SOLVE="$LANG_ARG" +if [ -n "$OTHER_LANGS" ]; then + files_of(){ case "$1" in (java) echo "$ja";; (typescript) echo "$ts";; (python) echo "$py";; (javascript) echo "$js";; (csharp) echo "$cs";; (*) echo 0;; esac; } + for l in $(for o in ${OTHER_LANGS//,/ }; do echo "$(files_of "$o") $o"; done | sort -s -k1,1nr | cut -d' ' -f2); do SOLVE="$SOLVE,$l"; done +fi OUT="$REPO/.axiomcode/out"; mkdir -p "$OUT" # ONE BUILD AT A TIME PER REPOSITORY. The background refresher (ax_fresh.py) and an explicit `axiomcode index` both # write $OUT/$LANG_ARG; two at once clobber each other's graph. The lock is on fd 9, which this shell holds for its # whole life, so the OS drops it when the build ends however it ends: no stale lock to time out. exec 9>"$REPO/.axiomcode/build.lock" if ! python3 "$H/ax_fresh.py" lock 9 --try; then - [ -n "${AXIOMCODE_BACKGROUND:-}" ] || echo "a graph build is already running for $REPO (the background refresh); waiting for it …" + [ -n "${AXIOMCODE_BACKGROUND:-}" ] || echo "a graph build is already running for $REPO; waiting for it …" python3 "$H/ax_fresh.py" lock 9 || { echo "could not take the build lock ($REPO/.axiomcode/build.lock)"; exit 75; } fi BUILD_T0=$SECONDS # how long a build takes to a usable graph: a query decides from it whether to wait (#1595) @@ -249,6 +259,9 @@ fi # build succeeds every verb keeps answering from it. A graph built for another language is not in the # way and is left where it is — which is also what $OUT/graph.sqlite still points at. PREV="$OUT/.previous-$LANG_ARG" +# A FIRST BUILD has no graph for any query to wait for; a rebuild (a refresh, an upgrade that adds a language) has the +# previous one, and a query waits a bounded while for its successor as it always did. $PENDING says which it is. +MODE=refresh; [ -e "$OUT/graph.sqlite" ] || MODE=first rm -rf "$PREV" # graph.sqlite NEVER DANGLES (#1305). With a refresher running after every edit, queries arrive mid-build all the time. # Moving the directory first and re-pointing after left a moment with no graph.sqlite at all, and a verb that finds @@ -287,27 +300,11 @@ restore(){ rm -rf "$OUT/$LANG_ARG"; if [ -d "$PREV" ]; then mv "$PREV" "$OUT/$LA [ -f "$OUT/.live.sqlite" ] && [ "$(real "$OUT/graph.sqlite")" != "$(real "$OUT/.live.sqlite")" ] && rm -f "$OUT/.live.sqlite"; return 0; } LIB=(); [ -n "${AXIOMCODE_LIBRARY:-}" ] && LIB=(--library "$AXIOMCODE_LIBRARY") ENGINE_RC=0 -"$ENGINE/bin/axiomcode" all "$SRC" "$OUT" --language "$LANGS" ${LIB[@]+"${LIB[@]}"} > "$REPO/.axiomcode/build.log" 2>&1 || ENGINE_RC=$? -# one language failing does not cost the others their graphs (the engine solves each on its own): only the MAIN -# language's failure fails the build. Another language's is named below, never dropped in silence. -if [ "$ENGINE_RC" != 0 ] && { [ -z "$OTHER_LANGS" ] || [ ! -f "$OUT/$LANG_ARG/graph.sqlite" ]; }; then - restore; echo "build failed — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; tail -5 "$REPO/.axiomcode/build.log"; exit 1 -fi -# the new database: a FILE under $OUT/$LANG_ARG. Matched as `*/$LANG_ARG/*` it was any path with the language's name -# in it, the repository's own included: a repository at .../python/app matched the graph.sqlite SYMLINK, the link was -# pointed at itself, the previous graph deleted, and every verb after it reported no graph. -DB="$(find "$OUT/$LANG_ARG" -type f -name graph.sqlite 2>/dev/null | head -1)"; NOT_OTHER=(); for l in ${OTHER_LANGS//,/ }; do NOT_OTHER+=(-not -path "$OUT/$l/*"); done # never another language's graph -[ -n "$DB" ] || DB="$(find "$OUT" -type f -name graph.sqlite -not -path "$PREV/*" ${NOT_OTHER[@]+"${NOT_OTHER[@]}"} | head -1)" -[ -n "$DB" ] || { restore; echo "build produced no graph.sqlite — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; exit 1; } -# A QUERY NEVER READS A HALF-BUILT GRAPH. The new database gets its symbols, refs and views BEFORE graph.sqlite points -# at it; until the pointer moves, every verb (and the hooks) keeps answering from the previous graph. Only then is the -# old one removed: a reader that opened it keeps the file it opened. -AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; } +REASON="$(printf %s "${AXIOMCODE_REFRESH_REASON:-axiomcode index}" | tr -d "'")" # WHEN AND WHY THIS GRAPH WAS BUILT, in the graph itself (index_meta): refreshed_at (UTC), refresh_reason (`axiomcode # index`, or what the background refresher saw: files changed, HEAD moved, found by an edit, a query or the timer), and # the tree and commit it describes. Written before the swap: a write to the live graph later would change its mtime, # which keys the impact-facts cache, and cost the next query a re-export. -REASON="$(printf %s "${AXIOMCODE_REFRESH_REASON:-axiomcode index}" | tr -d "'")" stamp_meta(){ python3 - "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$REASON" "$INDEXED_TREE" "$HEAD_SHA" <<'PY' 2>/dev/null || true import sqlite3, sys db, at, reason, tree, commit = sys.argv[1:6] @@ -317,54 +314,131 @@ with sqlite3.connect(db) as c: ('refreshed_tree', tree), ('refreshed_commit', commit)]) PY } -stamp_meta "$DB" -# EVERY OTHER LANGUAGE'S GRAPH, indexed where the engine wrote it and only then moved into place, as the main one is. -# One that did not build is NAMED: its previous graph, if any, keeps answering, and this build says it is behind. -READY=(); FAILED=() -for l in ${OTHER_LANGS//,/ }; do - LDB="$OUT/$l/graph.sqlite" - if [ -f "$LDB" ] && AXIOMCODE_INDEX_DB="$LDB" python3 "$H/axiomcode-index" "$REPO"; then stamp_meta "$LDB"; READY+=("$l"); else FAILED+=("$l"); fi -done -echo "$STAMP" > "$OUT/stamp" -# the engine that built this graph: the next build, the background refresh above all, starts its search there (see the top) -printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine.new" && mv -f "$REPO/.axiomcode/engine.new" "$REPO/.axiomcode/engine" || true -if [ -n "$INDEXED_TREE" ]; then echo "$INDEXED_TREE" > "$OUT/indexed-tree"; else rm -f "$OUT/indexed-tree"; fi -set_base "$INDEXED_TREE" -point "$DB" || { restore; exit 1; }; rm -f "$OUT/.live.sqlite" -if [ -f "$OUT/.files.json.new" ]; then mv "$OUT/.files.json.new" "$OUT/files.json"; else rm -f "$OUT/files.json"; fi -# the graph now matches the files: record how long that took, and whether it included compiling the engine's rules (a -# one-time cost a refresh does not pay, so such a duration predicts nothing) -printf '%s %s\n' "$((SECONDS - BUILD_T0))" "$(grep -q 'compiling souffle program' "$REPO/.axiomcode/build.log" 2>/dev/null && echo 1 || echo 0)" > "$OUT/build-seconds" 2>/dev/null || true -keep_base_graph -for l in ${READY[@]+"${READY[@]}"}; do - mkdir -p "$LANGDIR/$l/out"; swap "$OUT/$l/graph.sqlite" "$LANGDIR/$l/out/graph.sqlite" || true # a rename: atomic - rmdir "$OUT/$l" 2>/dev/null || true -done -# a language the repository no longer has (or that --lang left out) must not keep answering from an old graph -for d in "$LANGDIR"/*/; do - [ -d "$d" ] || continue; l="$(basename "$d")" - case ",$OTHER_LANGS," in *",$l,"*) ;; *) rm -rf "$d" "$REPO/.axiomcode/base/lang/$l";; esac -done -commit_base -rm -rf "$PREV" -python3 - "$OUT/graph.sqlite" <<'PY' +tiers(){ python3 -c 'import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); print(", ".join(f"{t} {n}" for t, n in c.execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC")))' "$1"; } +# EACH LANGUAGE'S GRAPH IS PUBLISHED WHEN ITS OWN LANGUAGE IS SOLVED (#1555). The engine solves one language after +# another, and nearly all of each is that language's Souffle compile. Published only when the whole run ended, the main +# language's graph sat finished while three JavaScript files compiled: on a C# web application with a small ClientApp, +# written at 131 s and queryable at 5,304 s, and every query in between waited on the build lock. So the engine runs in +# the background and names each language as it finishes it ($PROGRESS, `all --progress`); the main one is indexed and +# pointed at right then, every other one moved into .axiomcode/lang as it comes. $PENDING lists the languages still +# being solved: ax_fresh.py reads it while this build holds the lock, so a query neither waits for them nor reads the +# graph as stale, and says which languages its answer cannot see yet. +# $PENDING is written when the main graph is published, not before: until then the graph a query reads is the previous +# one, and a rebuild in progress must keep saying so. +PROGRESS="$OUT/.progress"; PENDING="$OUT/building"; rm -f "$PROGRESS" "$PENDING"; : > "$PROGRESS" +unpend(){ grep -v "^$1 " "$PENDING" > "$PENDING.new" 2>/dev/null || true; mv -f "$PENDING.new" "$PENDING"; } +# the engine gets its own process group (set -m), so a build that is stopped stops all of it -- the Souffle compile +# included -- rather than leaving it running under the build lock it inherited +EPID="" +stop_engine(){ if [ -n "$EPID" ] && kill -0 "$EPID" 2>/dev/null; then kill -TERM -- "-$EPID" 2>/dev/null || kill -TERM "$EPID" 2>/dev/null || true; fi; rm -f "$PROGRESS" "$PENDING" "$PENDING.new"; } +trap stop_engine EXIT +trap 'exit 130' INT; trap 'exit 143' TERM HUP +# an engine older than this script has no --progress and refuses an option it does not know: it gets none, and its +# graphs are published together when it ends, as they always were +PROG=(); grep -q -- '--progress' "$ENGINE/bin/axiomcode" 2>/dev/null && PROG=(--progress "$PROGRESS") +set -m +"$ENGINE/bin/axiomcode" all "$SRC" "$OUT" --language "$SOLVE" ${PROG[@]+"${PROG[@]}"} ${LIB[@]+"${LIB[@]}"} < /dev/null > "$REPO/.axiomcode/build.log" 2>&1 & +EPID=$! +set +m +MAIN_DONE=""; QUEUED=(); READY=(); FAILED=(); DONE=" "; SEEN=0 +# THE MAIN LANGUAGE'S GRAPH BECOMES THE GRAPH. Everything a finished build did for it happens here, as soon as it is +# solved: index, stamp, file table, baseline, pointer. +publish_main(){ + # the new database: a FILE under $OUT/$LANG_ARG. Matched as `*/$LANG_ARG/*` it was any path with the language's name + # in it, the repository's own included: a repository at .../python/app matched the graph.sqlite SYMLINK, the link was + # pointed at itself, the previous graph deleted, and every verb after it reported no graph. + DB="$(find "$OUT/$LANG_ARG" -type f -name graph.sqlite 2>/dev/null | head -1)"; NOT_OTHER=(); for l in ${OTHER_LANGS//,/ }; do NOT_OTHER+=(-not -path "$OUT/$l/*"); done # never another language's graph + [ -n "$DB" ] || DB="$(find "$OUT" -type f -name graph.sqlite -not -path "$PREV/*" ${NOT_OTHER[@]+"${NOT_OTHER[@]}"} | head -1)" + [ -n "$DB" ] || { restore; echo "build produced no graph.sqlite — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; exit 1; } + # A QUERY NEVER READS A HALF-BUILT GRAPH. The new database gets its symbols, refs and views BEFORE graph.sqlite points + # at it; until the pointer moves, every verb (and the hooks) keeps answering from the previous graph. Only then is the + # old one removed: a reader that opened it keeps the file it opened. + AXIOMCODE_INDEX_DB="$DB" python3 "$H/axiomcode-index" "$REPO" || { restore; echo "indexing the new graph failed — the graph that was there is untouched"; exit 1; } + stamp_meta "$DB" + echo "$STAMP" > "$OUT/stamp" + # the engine that built this graph: the next build, the background refresh above all, starts its search there (see the top) + printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine.new" && mv -f "$REPO/.axiomcode/engine.new" "$REPO/.axiomcode/engine" || true + if [ -n "$INDEXED_TREE" ]; then echo "$INDEXED_TREE" > "$OUT/indexed-tree"; else rm -f "$OUT/indexed-tree"; fi + set_base "$INDEXED_TREE" + if [ -n "$OTHER_LANGS" ]; then S="${SOLVE#*,}"; for l in ${S//,/ }; do echo "$l $MODE"; done > "$PENDING"; fi # before the pointer: the first query may follow at once + point "$DB" || { restore; exit 1; }; rm -f "$OUT/.live.sqlite" + if [ -f "$OUT/.files.json.new" ]; then mv "$OUT/.files.json.new" "$OUT/files.json"; else rm -f "$OUT/files.json"; fi + # the graph now matches the files: record how long that took, and whether it included compiling the engine's rules (a + # one-time cost a refresh does not pay, so such a duration predicts nothing). With other languages still to come this + # is the time to the MAIN graph, which is what a query waits for. + printf '%s %s\n' "$((SECONDS - BUILD_T0))" "$(grep -q 'compiling souffle program' "$REPO/.axiomcode/build.log" 2>/dev/null && echo 1 || echo 0)" > "$OUT/build-seconds" 2>/dev/null || true + keep_base_graph + # a language the repository no longer has (or that --lang left out) must not keep answering from an old graph + for d in "$LANGDIR"/*/; do + [ -d "$d" ] || continue; l="$(basename "$d")" + case ",$OTHER_LANGS," in *",$l,"*) ;; *) rm -rf "$d" "$REPO/.axiomcode/base/lang/$l";; esac + done + commit_base + rm -rf "$PREV" + MAIN_DONE=1 + python3 - "$OUT/graph.sqlite" <<'PY' import sqlite3, sys rows = sqlite3.connect(sys.argv[1]).execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC").fetchall() w = max([len('tier')] + [len(t) for t, _ in rows]) print(f"{'tier':<{w}} edges") for t, n in rows: print(f"{t:<{w}} {n}") PY -python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first -# query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. -python3 "$H/dl_program.py" || true # compile the query rules once, here where ~20 s is noise against the build: -# a later query under a caller's timeout (hooks/changes.py runs impact with timeout=14) cannot outlast the compile, so -# left to first use it is killed every time, caches nothing, and repeats on the next edit forever. -[ -n "$BASE_SAVED" ] && AXIOMCODE_GRAPH="$REPO/.axiomcode/base" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true -for l in ${READY[@]+"${READY[@]}"}; do - echo "$l: $(python3 -c 'import sqlite3,sys; c=sqlite3.connect(sys.argv[1]); print(", ".join(f"{t} {n}" for t, n in c.execute("SELECT tier, count(*) FROM call_edges GROUP BY tier ORDER BY 2 DESC")))' "$LANGDIR/$l/out/graph.sqlite")" - AXIOMCODE_GRAPH="$LANGDIR/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true - [ -n "$BASE_SAVED" ] && [ -f "$REPO/.axiomcode/base/lang/$l/out/graph.sqlite" ] && { AXIOMCODE_GRAPH="$REPO/.axiomcode/base/lang/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true; } + python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first + # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. + python3 "$H/dl_program.py" || true # compile the query rules once, here where ~20 s is noise against the build: + # a later query under a caller's timeout (hooks/changes.py runs impact with timeout=14) cannot outlast the compile, so + # left to first use it is killed every time, caches nothing, and repeats on the next edit forever. + [ -n "$BASE_SAVED" ] && AXIOMCODE_GRAPH="$REPO/.axiomcode/base" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true + [ -z "$OTHER_LANGS" ] || [ ! -s "$PENDING" ] || echo "$LANG_ARG graph ready at $OUT/graph.sqlite; still building: $(cut -d' ' -f1 "$PENDING" | paste -sd, - | sed 's/,/, /g')" +} +# EVERY OTHER LANGUAGE'S GRAPH, indexed where the engine wrote it and only then moved into place, as the main one is. +# One that did not build is NAMED: its previous graph, if any, keeps answering, and this build says it is behind. +publish_other(){ + local l="$1" LDB="$OUT/$1/graph.sqlite" + if [ "$2" = ok ] && [ -f "$LDB" ] && AXIOMCODE_INDEX_DB="$LDB" python3 "$H/axiomcode-index" "$REPO"; then + stamp_meta "$LDB" + mkdir -p "$LANGDIR/$l/out"; swap "$LDB" "$LANGDIR/$l/out/graph.sqlite" || true # a rename: atomic + rmdir "$OUT/$l" 2>/dev/null || true + READY+=("$l") + echo "$l: $(tiers "$LANGDIR/$l/out/graph.sqlite")" + AXIOMCODE_GRAPH="$LANGDIR/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true + [ -n "$BASE_SAVED" ] && [ -f "$REPO/.axiomcode/base/lang/$l/out/graph.sqlite" ] && { AXIOMCODE_GRAPH="$REPO/.axiomcode/base/lang/$l" python3 "$H/axiomcode-impact" --warm "$REPO" >/dev/null 2>&1 || true; } + else FAILED+=("$l"); fi + DONE="$DONE$l "; unpend "$l"; return 0 +} +# one language failing does not cost the others their graphs (the engine solves each on its own): only the MAIN +# language's failure fails the build, and it stops the rest, whose graphs would be dropped with it. +main_failed(){ restore; echo "build failed — the graph that was there is untouched; see $REPO/.axiomcode/build.log"; tail -5 "$REPO/.axiomcode/build.log"; exit 1; } +take(){ # one line of $PROGRESS: ok|failed [] + local l="${1%% *}" st; st="${1#* }"; st="${st%% *}" + if [ "$l" = "$LANG_ARG" ]; then + [ "$st" = ok ] || main_failed + publish_main + local q; for q in ${QUEUED[@]+"${QUEUED[@]}"}; do publish_other "${q%% *}" "${q#* }"; done; QUEUED=() + elif [ -n "$MAIN_DONE" ]; then publish_other "$l" "$st" + else QUEUED+=("$l $st"); fi # solved before the main one (it never is, but the main graph is published first) +} +drain(){ local n; n="$(wc -l < "$PROGRESS" | tr -d ' ')"; while [ "$SEEN" -lt "$n" ]; do SEEN=$((SEEN + 1)); take "$(sed -n "${SEEN}p" "$PROGRESS")"; done; } +# `running` is sampled BEFORE the drain: a line written just before the engine exited is still read +while :; do + running=1; kill -0 "$EPID" 2>/dev/null || running=0 + drain + [ "$running" = 1 ] || break + sleep 1 +done +ENGINE_RC=0; wait "$EPID" || ENGINE_RC=$?; EPID="" +drain +if [ -z "$MAIN_DONE" ]; then + # an engine that named nothing (one without --progress, or a run that failed before solving): as before, a failure + # fails the build unless it was another language's and the main graph is there + [ "$ENGINE_RC" = 0 ] || { [ -n "$OTHER_LANGS" ] && [ -f "$OUT/$LANG_ARG/graph.sqlite" ]; } || main_failed + take "$LANG_ARG ok" +fi +# a language the engine never named (the parser found no project of it) did not build, and is named below +for l in ${OTHER_LANGS//,/ }; do + case "$DONE" in *" $l "*) ;; *) publish_other "$l" ok;; esac done +rm -f "$PROGRESS" "$PENDING" if [ ${#FAILED[@]} -gt 0 ]; then for l in "${FAILED[@]}"; do if [ -f "$LANGDIR/$l/out/graph.sqlite" ]; then echo "⚠️ the $l graph did not build: $l questions are answered from its PREVIOUS graph, which predates this tree; see $REPO/.axiomcode/build.log" diff --git a/tests/README.md b/tests/README.md index 1f2d8476b..5424bfcf2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -61,6 +61,10 @@ One check needs no graph and is its own script: --no-engine for the rest) python3 tests/facts_cache.py path and impact do not answer from edges an older plugin exported: the export's stamp carries a version, so an upgrade re-exports (#1402; indexes a case) + python3 tests/publish_order.py a repository in several languages is queryable when its main language is solved, not + when the last one is: the others are held back, a query answers and names what it + cannot see, a first query does not wait for them, one language alone is unchanged + (#1555; builds real graphs, needs the engine) python3 tests/tiers.py every call_edges tier the schema documents is ranked, labelled and given a certainty by the frontend, so a new tier cannot read as the weakest claim diff --git a/tests/publish_order.py b/tests/publish_order.py new file mode 100644 index 000000000..5f2dc307e --- /dev/null +++ b/tests/publish_order.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""tests/publish_order.py: a repository in several languages is queryable as soon as its main language is solved. + +The engine solves one language after another, and nearly all of each is that language's Souffle compile. The build +used to publish nothing until every language was done: on a C# web application with a small front end in another +language, the C# graph was written 131 s into the first index and the first query answered at 5,304 s (#1555). + +The other languages are held back here on purpose: the engine is wrapped by one that runs the real engine, +reports the first language it solved, and holds the rest back until the test lets them go. Nothing else is faked. + +One throwaway repository: a C# web API (the main language: most files), a small Java report job and some Python +scripts, more Python files than Java ones. + + order the engine solves the main language first, then the others by file count (python before + java), though the fixed language order puts java first + published while the other languages are held, graph.sqlite already points at the C# graph and the build is + still running + query a C# question is answered then, at once, and says which languages it cannot see yet; a Java + name is refused without waiting for its compile + rebuild a rebuild that is past its main language still has the other languages' previous graphs: a query answers + at once, as every query during a refresh does, and says which languages it answered from old graphs + finished released, the build ends with every language's graph and no trace of the pending list + first query with no graph at all, a query builds one and answers when the C# graph is ready, not when the + build ends; the build goes on and still ends with every language's graph; the answer on stdout + carries none of the build's output + main fails a main language that fails fails the build, keeps the graph that was there, and publishes nothing + control a repository in one language builds as before: no pending list, no .axiomcode/lang, no note + + python3 tests/publish_order.py [-v] +""" +import json, os, shutil, subprocess, sys, tempfile, time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AX = os.path.join(ROOT, 'bin', 'axiomcode') +FRESH = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'ax_fresh.py') + +CS = { + 'WebApp/WebApp.csproj': '\n \n net8.0\n \n\n', + 'WebApp/Program.cs': 'using WebApp.Services;\n\nvar builder = WebApplication.CreateBuilder(args);\nbuilder.Services.AddScoped();\nvar app = builder.Build();\napp.Run();\n', + 'WebApp/Controllers/OrdersController.cs': 'using WebApp.Services;\n\nnamespace WebApp.Controllers;\n\npublic class OrdersController\n{\n private readonly IOrderService _orders;\n\n public OrdersController(IOrderService orders) => _orders = orders;\n\n public decimal Get(int id) => _orders.Total(id);\n}\n', + 'WebApp/Services/OrderService.cs': 'namespace WebApp.Services;\n\npublic interface IOrderService\n{\n decimal Total(int orderId);\n}\n\npublic class OrderService : IOrderService\n{\n public decimal Total(int orderId) => Subtotal(orderId) * 1.2m;\n\n private decimal Subtotal(int orderId) => orderId * 10m;\n}\n', + 'WebApp/Models/Order.cs': 'namespace WebApp.Models;\n\npublic record Order(int Id, decimal Amount);\n', +} +TOOLS = { + 'tools/report/pom.xml': '\n 4.0.0\n app\n report\n 1.0\n\n', + 'tools/report/src/main/java/app/report/Formatter.java': 'package app.report;\n\npublic class Formatter {\n public String money(double x) {\n return String.format("%.2f", x);\n }\n}\n', + 'tools/report/src/main/java/app/report/ReportJob.java': 'package app.report;\n\npublic class ReportJob {\n private final Formatter formatter = new Formatter();\n\n public String render(double total) {\n return formatter.money(total);\n }\n}\n', + 'tools/scripts/totals.py': 'def sum_totals(rows):\n return sum(r["total"] for r in rows)\n', + 'tools/scripts/export.py': 'from totals import sum_totals\n\n\ndef export(rows):\n return {"sum": sum_totals(rows)}\n', + 'tools/scripts/labels.py': 'def label(x):\n return "total " + str(x)\n', +} + +# the engine, wrapped: the real `all` runs, and the first language it names is passed on the moment it is named; the +# rest only once the real run has ended AND the HOLD file is gone. So the main graph is published while the real engine +# is still solving (a cold compile of the others' rules takes as long as it takes), and the test decides when the other +# languages arrive. Without --progress (a build that publishes nothing early) it holds the same way. +SHIM = r'''#!/usr/bin/env bash +REAL="$AXIOMCODE_TEST_REAL/bin/axiomcode" +[ "${1:-}" = all ] || exec "$REAL" "$@" +prog=""; args=(); next="" +for a in "$@"; do + if [ -n "$next" ]; then prog="$a"; args+=("$a.real"); next=""; continue; fi + [ "$a" = --progress ] && next=1; args+=("$a") +done +[ -z "$prog" ] || rm -f "$prog.real" +"$REAL" "${args[@]}" & pid=$! +if [ -n "$prog" ]; then + while kill -0 "$pid" 2>/dev/null && [ ! -s "$prog.real" ]; do sleep 0.2; done + first="$(head -1 "$prog.real" 2>/dev/null)" + if [ -n "$first" ]; then + [ -n "${AXIOMCODE_TEST_FAIL_MAIN:-}" ] && first="${first%% *} failed" + echo "$first" >> "$prog" + fi +fi +wait "$pid"; rc=$? +while [ -e "$AXIOMCODE_TEST_HOLD" ]; do sleep 0.2; done +[ -z "$prog" ] || tail -n +2 "$prog.real" >> "$prog" +exit $rc +''' + + +def sh(cwd, *cmd, env=None): + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + + +def make(root, files): + for rel, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True) + open(os.path.join(root, rel), 'w').write(text) + for cmd in (('git', 'init', '-q'), ('git', 'add', '-A'), ('git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'base')): + sh(root, *cmd) + + +def until(cond, seconds): + end = time.time() + seconds + while time.time() < end: + if cond(): return True + time.sleep(0.2) + return cond() + + +def main(argv): + verbose = '-v' in argv + fails = [] + + def check(ok, why, detail=''): + print(('ok ' if ok else 'FAIL ') + why + ('' if ok and not verbose or not detail else '\n ' + str(detail).strip()[-1500:].replace('\n', '\n '))) + if not ok: fails.append(why) + + work = os.path.realpath(tempfile.mkdtemp(prefix='axiomcode-publish-')) + shim = os.path.join(work, 'engine') + os.makedirs(os.path.join(shim, 'bin')) + open(os.path.join(shim, 'bin', 'axiomcode'), 'w').write(SHIM); os.chmod(os.path.join(shim, 'bin', 'axiomcode'), 0o755) + for n in ('graph', 'package.json', 'parser'): os.symlink(os.path.join(ROOT, n), os.path.join(shim, n)) + hold = os.path.join(work, 'hold') + env = dict(os.environ, AXIOMCODE_ENGINE=shim, AXIOMCODE_TEST_REAL=ROOT, AXIOMCODE_TEST_HOLD=hold, AXIOMCODE_NO_REFRESH='1') + for k in ('AXIOMCODE_LANG', 'AXIOMCODE_GRAPH', 'AXIOMCODE_GRAPH_LANG', 'AXIOMCODE_LIBRARY'): env.pop(k, None) + procs = [] + try: + repo = os.path.join(work, 'webapp'); make(repo, dict(CS, **TOOLS)) + out = os.path.join(repo, '.axiomcode', 'out'); lang = os.path.join(repo, '.axiomcode', 'lang') + graph = os.path.join(out, 'graph.sqlite') + + # ── published ───────────────────────────────────────────────────────────────────────────────────────── + open(hold, 'w').close() + b = subprocess.Popen([AX, 'index', '.'], cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True); procs.append(b) + up = until(lambda: os.path.exists(graph) or b.poll() is not None, 600) + running = b.poll() is None + check(up and os.path.exists(graph) and running, + 'published: graph.sqlite points at the main (C#) graph while the other languages are still being solved', + f"graph={os.path.exists(graph)} running={running}") + log = open(os.path.join(repo, '.axiomcode', 'build.log')).read() if os.path.exists(os.path.join(repo, '.axiomcode', 'build.log')) else '' + check('▶ languages: csharp python java' in log, + 'order: the main language is solved first, then the others by file count (python 3 before java 2)', + [l for l in log.splitlines() if 'languages' in l]) + if not (up and running): return 1 + until(lambda: os.path.exists(os.path.join(out, 'building')), 30) + + # ── query ───────────────────────────────────────────────────────────────────────────────────────────── + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and b.poll() is None and dt < 60, + 'query: a C# question is answered while the other languages are held, without waiting for them', + f"{dt:.1f}s\n{q.stdout}\n{q.stderr}") + check('the python, java graphs are still being built' in q.stderr, + 'query: the answer names the languages whose graphs it cannot see yet', q.stderr) + live = dict(env); live.pop('AXIOMCODE_NO_REFRESH') + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=live); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and 'still being built' in q.stderr and dt < 60 and b.poll() is None, + 'query: with the refresher on, the same answer and note, and no wait for a refresh', f"{dt:.1f}s\n{q.stderr}") + st = json.loads(sh(repo, sys.executable, FRESH, 'status', '.', '--json', env=env).stdout or '{}') + check(st.get('state') == 'building' and st.get('pending') == ['python', 'java'] and st.get('first') is True, + 'query: status says building, with the pending languages in solve order, on a first build', json.dumps(st)) + t0 = time.time(); r = sh(repo, AX, 'impact', 'Formatter.money', '.', env=env); dt = time.time() - t0 + check(r.returncode != 0 and 'Formatter.money' in r.stdout + r.stderr and b.poll() is None and dt < 60, + 'query: a Java name is refused at once, not after the Java solve', f"{dt:.1f}s\n{r.stdout}\n{r.stderr}") + + # ── finished ────────────────────────────────────────────────────────────────────────────────────────── + os.remove(hold) + bout, _ = b.communicate(timeout=5400) + have = {l: os.path.exists(os.path.join(lang, l, 'out', 'graph.sqlite')) for l in ('python', 'java')} + check(b.returncode == 0 and all(have.values()) and not os.path.exists(os.path.join(out, 'building')) and not os.path.exists(os.path.join(out, '.progress')), + 'finished: the build ends with every language\'s graph, and the pending list is gone', bout + json.dumps(have)) + check(bout.index('csharp graph ready') < bout.index('python:') < bout.index('java:') and 'still building: python, java' in bout, + 'finished: the build says when the main graph is ready, then names each language as it is published', bout) + r = sh(repo, AX, 'impact', 'Formatter.money', '.', env=env) + check(r.returncode == 0 and 'ReportJob.render' in r.stdout and 'still being built' not in r.stderr, + 'finished: the Java name is answered from its own graph, with no note', r.stdout + r.stderr) + + # ── rebuild ─────────────────────────────────────────────────────────────────────────────────────────── + api = os.path.join(repo, 'tools/scripts/totals.py') + open(api, 'a').write('\n\ndef added_later():\n return 1\n') + open(hold, 'w').close() + b = subprocess.Popen([AX, 'index', '.'], cwd=repo, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True); procs.append(b) + until(lambda: os.path.exists(os.path.join(out, 'building')) or b.poll() is not None, 600) + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=live); dt = time.time() - t0 + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and dt < 60 and b.poll() is None + and 'its python, java code comes from the previous graph' in q.stderr, + 'rebuild: past its main language, a query answers at once and says the other languages\' code is from their previous graphs', + f"{dt:.1f}s\n{q.stderr}") + os.remove(hold); b.communicate(timeout=5400) + r = sh(repo, AX, 'impact', 'added_later', '.', env=env) + check(b.returncode == 0 and r.returncode == 0 and 'totals.py' in r.stdout, 'rebuild: released, the edit is in the Python graph', r.stdout + r.stderr) + sh(repo, 'git', 'checkout', '-q', '--', '.') + + # ── main fails ──────────────────────────────────────────────────────────────────────────────────────── + before = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env).stdout + model = os.path.join(repo, 'WebApp/Models/Order.cs') + open(model, 'a').write('\npublic record Line(int Qty);\n') # a change, so the index rebuilds + f = sh(repo, AX, 'index', '.', env=dict(env, AXIOMCODE_TEST_FAIL_MAIN='1')) + after = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env) + check(f.returncode != 0 and 'build failed' in f.stdout and after.returncode == 0 and after.stdout == before + and not os.path.exists(os.path.join(out, 'building')), + 'main fails: the build fails, and the graph that was there answers exactly as before', f.stdout + f.stderr) + sh(repo, 'git', 'checkout', '-q', '--', '.') + + # ── first query ─────────────────────────────────────────────────────────────────────────────────────── + shutil.rmtree(os.path.join(repo, '.axiomcode')) + open(hold, 'w').close() + t0 = time.time(); q = sh(repo, AX, 'impact', 'OrderService.Total', '.', env=env); dt = time.time() - t0 + held = os.path.exists(hold) and not os.path.exists(os.path.join(lang, 'java')) + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and held, + 'first query: with no graph, the answer comes when the C# graph is ready, while the other languages are held', + f"{dt:.1f}s held={held}\n{q.stdout}\n{q.stderr}") + check('tier ' not in q.stdout and 'indexed ' not in q.stdout and 'indexed ' in q.stderr, + 'first query: the build\'s output goes to stderr, and the answer on stdout is only the answer', q.stdout[:600]) + check('still being built' in q.stderr, 'first query: the answer names the languages it cannot see yet', q.stderr[-600:]) + os.remove(hold) + done = until(lambda: all(os.path.exists(os.path.join(lang, l, 'out', 'graph.sqlite')) for l in ('python', 'java')) + and not os.path.exists(os.path.join(out, 'building')), 600) + check(done, 'first query: the build goes on after the answer and ends with every language\'s graph', + open(os.path.join(repo, '.axiomcode', 'index.log')).read()[-800:] if os.path.exists(os.path.join(repo, '.axiomcode', 'index.log')) else '') + + # ── control ─────────────────────────────────────────────────────────────────────────────────────────── + one = os.path.join(work, 'api-only'); make(one, CS) + c = sh(one, AX, 'index', '.', env=env) + q = sh(one, AX, 'impact', 'OrderService.Total', '.', env=env) + check(c.returncode == 0 and 'still building' not in c.stdout and not os.path.exists(os.path.join(one, '.axiomcode', 'lang')) + and not os.path.exists(os.path.join(one, '.axiomcode', 'out', 'building')) and c.stdout.rstrip().endswith('graph at ' + os.path.join(one, '.axiomcode', 'out', 'graph.sqlite')), + 'control: a repository in one language builds as it did, with no pending list and no other graph', c.stdout + c.stderr) + check(q.returncode == 0 and 'OrdersController.Get' in q.stdout and 'graph refresh' not in q.stderr, + 'control: its answer carries no note', q.stdout + q.stderr) + finally: + try: os.remove(hold) + except OSError: pass + for p in procs: + if p.poll() is None: p.kill() + time.sleep(1) + shutil.rmtree(work, ignore_errors=True) + print(f"\n{'FAILED: ' + str(len(fails)) if fails else 'all passed'}") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:]))