diff --git a/.github/workflows/libdt.yml b/.github/workflows/libdt.yml new file mode 100644 index 00000000..2d677de7 --- /dev/null +++ b/.github/workflows/libdt.yml @@ -0,0 +1,73 @@ +# Rebuilds the libdt native JVMTI agent binaries (libdt/build/*.so). +# Manually triggered; download the artifacts and commit them to libdt/build/. +# Adapted from nREPL's libnrepl workflow (https://github.com/nrepl/nrepl). +name: Build libdt native library + +on: + workflow_dispatch: + +jobs: + build-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Compile native library + run: cd libdt/ && make all + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: libdt-linux-x64 + path: libdt/build + + build-linux-arm64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Run the build process with Docker + uses: addnab/docker-run-action@v3 + with: + image: eclipse-temurin:21 + options: | + --platform linux/arm64 + --volume ${{ github.workspace }}:/build + run: | + apt-get update && apt-get install -y make gcc + cd /build/libdt/ && make all + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: libdt-linux-arm64 + path: libdt/build + + build-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Compile native library + run: cd libdt/ && make all + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: libdt-macos-universal + path: libdt/build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5fb1074..191abb1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ on: push: branches: - master - - feature/dockerfile + - fix/break-handling jobs: Config: @@ -67,14 +67,20 @@ jobs: bb: latest - name: Update VERSION run: echo "${{ needs.Config.outputs.version }}" > resources/VERSION + - name: Compile Java + run: bb java:compile - name: Zip it run: zip -r deps-try.zip . -x ".git/*" - name: Build uberjar run: | - rm -rf .git + # remove stuff not wanted in JAR + rm -rf .git .github mv deps-try.zip .. - bb uberjar deps-try-bb.jar -m eval.deps-try + + bb bb:uberjar + ls -lah deps-try-bb.jar jar -tf deps-try-bb.jar + mv ../deps-try.zip . - name: Testrun uberjar run: bb deps-try-bb.jar -h diff --git a/.gitignore b/.gitignore index d99d61a8..e05d29d8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ /.clj-kondo .lsp /VERSION +*.class +/tmp/* +!/tmp/.keep diff --git a/bb.edn b/bb.edn index 18eca27c..4b0ef685 100644 --- a/bb.edn +++ b/bb.edn @@ -1,7 +1,11 @@ {:deps {io.github.eval/deps-try {:local/root "."}} :bbin/bin {deps-try {:main-opts ["-m" "eval.deps-try"]}} :tasks - {gen-manifest {:doc "Generate recipe-manifest and print to stdout" + {java:compile {:doc "Compile files under java/" + :task (shell {:dir "java"} "javac --release 11 dt/JvmtiAgent.java")} + bb:uberjar {:doc "Create the app's uberjar" + :task (shell "bb uberjar deps-try-bb.jar -m eval.deps-try")} + gen-manifest {:doc "Generate recipe-manifest and print to stdout" :requires ([babashka.fs :as fs]) :task (exec 'eval.deps-try.recipe/generate&print-manifest {:exec-args diff --git a/deps.edn b/deps.edn index c2e3b840..e1b2956e 100644 --- a/deps.edn +++ b/deps.edn @@ -1,11 +1,14 @@ -{:paths ["src" "resources" "."] +;; NOTE keep "." out of :paths: `bb uberjar` jars every classpath folder +;; wholesale, so "." would put the entire repo in the released jar (and the +;; libdt binaries in twice). +{:paths ["src" "resources" "java" "libdt/build"] :deps {borkdude/edamame {:mvn/version "1.4.24"} - cider/orchard {:mvn/version "0.22.0"} + cider/orchard {:mvn/version "0.30.0"} com.bhauman/rebel-readline {:local/root "./vendor/rebel-readline/rebel-readline"} deps-try/cli {:local/root "./vendor/deps-try.cli"} deps-try/http-client {:local/root "./vendor/deps-try.http-client"} io.github.eval/compliment {:git/sha "cd9706db27d456e8940dcd1118174c94effa9358"} - org.clojure/clojure {:mvn/version "1.12.0-alpha11"} + org.clojure/clojure {:mvn/version "1.12.5"} org.clojure/tools.gitlibs {:mvn/version "2.5.197"} ;; suppress logging org.slf4j/slf4j-nop {:mvn/version "2.0.5"}} diff --git a/java/dt/JvmtiAgent.java b/java/dt/JvmtiAgent.java new file mode 100644 index 00000000..4b700259 --- /dev/null +++ b/java/dt/JvmtiAgent.java @@ -0,0 +1,24 @@ +package dt; + +/** + * Java facade for the C side of the libdt JVMTI agent. Currently used to + * stop threads on JDK20+. + */ +public class JvmtiAgent { + + // This will bind to Java_dt_JvmtiAgent_stopThread function once the + // agent containing this function will be attached. + public static native void stopThread(Thread thread, Throwable throwable); + + /** + * Forcibly stop a given thread. + */ + public static void stopThread(Thread thread) { + // ThreadDeath is deprecated, but so is Thread.stop(). We can revisit + // this when JVM actually decides to remove this class. + @SuppressWarnings("deprecation") + Throwable throwable = new ThreadDeath(); + throwable.setStackTrace(new StackTraceElement[0]); + stopThread(thread, throwable); + } +} diff --git a/libdt/Makefile b/libdt/Makefile new file mode 100644 index 00000000..53ac2a1a --- /dev/null +++ b/libdt/Makefile @@ -0,0 +1,49 @@ +LIB=libdt-$(OS)-$(ARCH).so +CXXFLAGS=-O3 -fno-exceptions -fvisibility=hidden +INCLUDES=-I$(JAVA_HOME)/include +LIBS=-ldl +SOURCES := $(wildcard src/*.c) + +ifeq ($(JAVA_HOME),) + $(error JAVA_HOME is not set (e.g. on macOS: JAVA_HOME=$$(/usr/libexec/java_home) make)) +endif + +ARCH:=$(shell uname -m) +ifeq ($(ARCH),x86_64) + ARCH=x64 +else + ifeq ($(findstring arm,$(ARCH)),arm) + ifeq ($(findstring 64,$(ARCH)),64) + ARCH=arm64 + else + ARCH=arm32 + endif + else ifeq ($(findstring aarch64,$(ARCH)),aarch64) + ARCH=arm64 + else + ARCH=x86 + endif +endif + +OS:=$(shell uname -s) +ifeq ($(OS),Darwin) + CXXFLAGS += -arch x86_64 -arch arm64 -mmacos-version-min=10.12 + INCLUDES += -I$(JAVA_HOME)/include/darwin + OS=macos + ARCH=universal +else + CXXFLAGS += -Wl,-z,defs + LIBS += -lrt + INCLUDES += -I$(JAVA_HOME)/include/linux + OS=linux + CC=gcc +endif + +all: build/$(LIB) + +clean: + $(RM) -r build + +build/$(LIB): $(SOURCES) + mkdir -p build + $(CC) $(CXXFLAGS) $(INCLUDES) -fPIC -shared -o $@ $(SOURCES) $(LIBS) diff --git a/libdt/build/libdt-linux-arm64.so b/libdt/build/libdt-linux-arm64.so new file mode 100755 index 00000000..2cb34f8d Binary files /dev/null and b/libdt/build/libdt-linux-arm64.so differ diff --git a/libdt/build/libdt-linux-x64.so b/libdt/build/libdt-linux-x64.so new file mode 100755 index 00000000..08f10749 Binary files /dev/null and b/libdt/build/libdt-linux-x64.so differ diff --git a/libdt/build/libdt-macos-universal.so b/libdt/build/libdt-macos-universal.so new file mode 100755 index 00000000..00fcb6f2 Binary files /dev/null and b/libdt/build/libdt-macos-universal.so differ diff --git a/libdt/src/dt_agent.c b/libdt/src/dt_agent.c new file mode 100644 index 00000000..76f8bff6 --- /dev/null +++ b/libdt/src/dt_agent.c @@ -0,0 +1,51 @@ +// Native agent that deps-try uses for deeply buried JDK functionality. +// Currently it is used to bring back Thread.stop() that was lost in JDK20+. +// Adapted from nREPL's libnrepl agent (https://github.com/nrepl/nrepl/pull/318). + +#include +#include + +static jvmtiEnv* jvmti_env; + +// https://docs.oracle.com/en/java/javase/21/docs/specs/jvmti.html#StopThread +JNIEXPORT void JNICALL Java_dt_JvmtiAgent_stopThread +(JNIEnv* env, jclass cls, jthread thread, jobject throwable) { + jvmtiError err; + jvmtiThreadInfo threadInfo; + + err = (*jvmti_env)->GetThreadInfo(jvmti_env, thread, &threadInfo); + if (err != JVMTI_ERROR_NONE) { + fprintf(stderr, "Error getting thread info: %d\n", err); + return; + } + + //printf("Stopping thread \"%s\" using JVMTI...\n", threadInfo.name); + + err = (*jvmti_env)->StopThread(jvmti_env, thread, throwable); + if (err != JVMTI_ERROR_NONE) { + fprintf(stderr, "Error stopping thread: %d\n", err); + return; + } +} + +JNIEXPORT jint JNICALL Agent_OnAttach(JavaVM* vm, char* options, void* _) { + //printf("libdt native agent loaded\n"); + + // Initialize JVMTI environment. + jint res = (*vm)->GetEnv(vm, (void**)&jvmti_env, JVMTI_VERSION_1_2); + if (res != JNI_OK) { + fprintf(stderr, "Failed to get JVMTI environment\n"); + return JNI_ERR; + } + + // Request capabilities for StopThread + jvmtiCapabilities capabilities = {0}; + capabilities.can_signal_thread = 1; + jvmtiError err = (*jvmti_env)->AddCapabilities(jvmti_env, &capabilities); + if (err != JVMTI_ERROR_NONE) { + fprintf(stderr, "Failed to add JVMTI capabilities: %d\n", err); + return JNI_ERR; + } + + return JNI_OK; +} diff --git a/src/eval/deps_try.clj b/src/eval/deps_try.clj index 12d84582..b943ce0b 100644 --- a/src/eval/deps_try.clj +++ b/src/eval/deps_try.clj @@ -26,10 +26,29 @@ (def ^:private dev? (nil? (io/resource "VERSION"))) +(defn- repl-java-version + "Major version of the JVM that the `clojure` CLI will start (found via + JAVA_CMD, JAVA_HOME or PATH, mirroring the CLI's lookup), nil when it can't + be determined." + [] + (let [java-cmd (or (System/getenv "JAVA_CMD") + (some-> (System/getenv "JAVA_HOME") (fs/path "bin" "java") str) + "java") + ;; `java -version` prints to stderr, e.g. `openjdk version "21.0.2" ...` + version-line (try (:err (p/sh {:err :string} java-cmd "-version")) + (catch Exception _ nil)) + [_ major minor] (some->> version-line (re-find #"version \"(\d+)(?:\.(\d+))?")) + major (some-> major parse-long)] + (when major + (if (> major 1) major (some-> minor parse-long))))) + (def ^:private version (string/trim (if dev? - (let [git-dir (fs/file (io/resource ".git"))] + ;; the repo-root isn't on the classpath (see NOTE in deps.edn), so derive + ;; it from a source file that is + (let [src-file (fs/file (io/resource "eval/deps_try.clj")) + git-dir (fs/path (-> src-file fs/parent fs/parent fs/parent) ".git")] (:out (p/sh {} "git" "--git-dir" (str git-dir) "describe" "--tags"))) (slurp (io/resource "VERSION"))))) @@ -100,20 +119,44 @@ ;; TODO re-enable? ;; see https://github.com/clojure/brew-install/blob/1.11.3/CHANGELOG.md #_(warn-unless-minimum-clojure-cli-version "1.11.1.1273" tdeps-version) - (let [paths (into ["."] ;; needed for clojure.java.io/resource - (string/split init-cp #":")) + (let [cp-parts (string/split init-cp #":") + ;; When running from the packaged uberjar, its path is external to the + ;; project and tools.deps deprecates such :paths entries, so it's added as + ;; a :local/root dep instead. In dev the classpath is the project's own + ;; source dirs and resolved deps, which belong in :paths as before. + ;; "." is kept for clojure.java.io/resource. + {jars true, dirs false} (if dev? + {false cp-parts} + (group-by #(string/ends-with? % ".jar") cp-parts)) + paths (into ["."] dirs) + self-deps (into {} (map-indexed (fn [i jar] + [(symbol "deps-try.self" (str "cp" i)) + {:local/root jar}]) + jars)) deps (merge - {'org.clojure/clojure {:mvn/version "1.12.0-alpha11"}} + {'org.clojure/clojure {:mvn/version "1.12.5"}} + self-deps recipe-deps requested-deps) main-args (cond-> ["--version" version] recipe-location (into (if ns-only ["--recipe-ns" recipe-location] ["--recipe" recipe-location])) - prepare (conj "-P"))] - (apply run-repl "-Sdeps" (str {:paths paths + prepare (conj "-P")) + ;; Flags allowing the JVMTI agent to be loaded (needed for breaks on + ;; Java 20+, see eval.deps-try.util.jvmti). The -XX flag (which also + ;; suppresses the JEP 451 warning) only exists on Java 9+; Java 8 + ;; refuses to boot on unrecognized -XX options, so skip it there. + java-major (repl-java-version) + jvm-flags (cond-> ["-J-Djdk.attach.allowAttachSelf"] + (or (nil? java-major) (>= java-major 9)) + (conj "-J-XX:+EnableDynamicAgentLoading"))] + (apply run-repl + (concat jvm-flags + ["-Sdeps" (str {:paths paths :deps deps}) - "-M" "-m" "eval.deps-try.try" main-args))) + "-M" "-m" "eval.deps-try.try"] + main-args)))) (defn- recipe-manifest-contents [{:keys [refresh] :as _cli-opts}] (let [remote-manifest-file "https://raw.githubusercontent.com/eval/deps-try/master/recipes/manifest.edn" diff --git a/src/eval/deps_try/deps.cljc b/src/eval/deps_try/deps.cljc index ddd5eff6..9eaa67b9 100644 --- a/src/eval/deps_try/deps.cljc +++ b/src/eval/deps_try/deps.cljc @@ -370,6 +370,9 @@ (defn deps-available [] (->> (clojure.java.basis/current-basis) :libs + ;; synthetic deps used to put the packaged uberjar on the classpath + ;; (see eval.deps-try/start-repl!) shouldn't show as user libraries + (remove (comp #{"deps-try.self"} namespace key)) (filter (comp #(every? empty? %) :parents val))))) #?(:bb nil diff --git a/src/eval/deps_try/try.clj b/src/eval/deps_try/try.clj index daf55352..9018338c 100644 --- a/src/eval/deps_try/try.clj +++ b/src/eval/deps_try/try.clj @@ -9,6 +9,7 @@ [eval.deps-try.history :as history] [eval.deps-try.recipe :as recipe] [eval.deps-try.rr-service :as rebel-service] + [eval.deps-try.util :as util] [rebel-readline.clojure.line-reader :as clj-line-reader] [rebel-readline.clojure.main :as rebel-main] [rebel-readline.commands :as rebel-readline] @@ -108,11 +109,30 @@ (pp/pprint x))))) +(def ^:private jvmti-stop-thread + ;; Resolved eagerly (see `:init` in `-main`): `requiring-resolve` takes + ;; Clojure's require-lock, which the eval being interrupted may itself hold + ;; (e.g. Ctrl-C during a hung require), deadlocking the break handler. + (delay (requiring-resolve 'eval.deps-try.util.jvmti/stop-thread))) + +(defn break-thread! + "Forcibly stop `thread`. Called from the SIGINT-handler thread; public as + it's referenced from an eval'ed form (see `handle-sigint-form`)." + [^Thread thread] + (try + (if (<= util/java-version 19) + (.stop thread) + (@jvmti-stop-thread thread)) + (catch Throwable e + (println (str "Break failed: " (ex-message e)))))) + ;; SOURCE https://github.com/bhauman/rebel-readline/pull/199 (defn- handle-sigint-form [] `(let [thread# (Thread/currentThread)] - (clj-repl/set-break-handler! (fn [_signal#] (.stop thread#))))) + (clj-repl/set-break-handler! + (fn [_signal#] + (break-thread! thread#))))) (defn- recipe-instructions [{:keys [ns-only]}] (str "Recipe" (when ns-only " namespace") " successfully loaded in the REPL-history. Press arrow-up to start with the first step.")) @@ -172,6 +192,18 @@ [ex] (swap! api/*line-reader* assoc :repl/just-caught ex)) +(defn- interrupted? + "Returns true if the given throwable was ultimately caused by the break + handler stopping the thread (see `break-thread!`). + The ThreadDeath arrives bare when the eval was interrupted at runtime, or + wrapped in a CompilerException (where `clojure.main/root-cause` stops + unwrapping) when interrupted during compilation. + SOURCE `nrepl.middleware.interruptible-eval/interrupted?`" + [^Throwable ex] + (or (instance? ThreadDeath (clojure.main/root-cause ex)) + (and (instance? clojure.lang.Compiler$CompilerException ex) + (instance? ThreadDeath (.getCause ex))))) + (defn- reset-just-caught [] `(swap! api/*line-reader* dissoc :repl/just-caught)) @@ -191,10 +223,14 @@ repl-opts (cond-> {:deps-try/version (:version opts) :deps-try/data-path data-path :caught (fn [ex] - (persist-just-caught ex) - (clojure.main/repl-caught ex)) + (when-not (interrupted? ex) + (persist-just-caught ex) + (clojure.main/repl-caught ex))) :init (fn [] (load-slow-deps!) + (when (>= util/java-version 20) + ;; see comment at jvmti-stop-thread + (try @jvmti-stop-thread (catch Throwable _))) (apply require clojure.main/repl-requires) (set! clojure.core/*print-namespace-maps* false)) :eval (fn [form] @@ -210,6 +246,6 @@ (comment - (recipe/parse "/Users/gert/projects/deps-try/deps-try-recipes/recipes/next-jdbc/postgresql.clj") + (recipe/parse "/Users/gert/projects/deps-try/deps-try/recipes/next_jdbc/postgresql.clj") #_:end) diff --git a/src/eval/deps_try/util.clj b/src/eval/deps_try/util.clj index d0745c44..e925857a 100644 --- a/src/eval/deps_try/util.clj +++ b/src/eval/deps_try/util.clj @@ -4,6 +4,19 @@ [eval.deps-try.fs :as fs] [eval.deps-try.process :as p])) +(defn parse-java-version + "Parse Java version string according to JEP 223 and return version as a number." + [] + (try (let [s (System/getProperty "java.specification.version") + [major minor _] (string/split s #"\.") + major (Integer/parseInt major)] + (if (> major 1) + major + (Integer/parseInt minor))) + (catch Exception _ 8))) + +(def java-version "Current Java version number." (parse-java-version)) + (defn duration->millis [{:keys [seconds minutes hours days weeks] :or {seconds 0 minutes 0 hours 0 days 0 weeks 0}}] (let [days (+ days (* weeks 7)) diff --git a/src/eval/deps_try/util/jvmti.clj b/src/eval/deps_try/util/jvmti.clj new file mode 100644 index 00000000..3f2dcebb --- /dev/null +++ b/src/eval/deps_try/util/jvmti.clj @@ -0,0 +1,66 @@ +(ns eval.deps-try.util.jvmti + "This namespace contains code exclusive to JDK9+ and should not be attempted to + load with earlier JDKs." + (:require + [clojure.java.io :as io]) + (:import + (com.sun.tools.attach VirtualMachine) + (java.lang ProcessHandle) + (java.nio.file Files) + (java.nio.file.attribute FileAttribute) + (dt JvmtiAgent))) + +;;; Agent unpacking + +(defonce ^:private temp-directory + ;; deleteOnExit runs in reverse order of registration: the unpacked lib + ;; (registered later) is deleted first, leaving the directory empty. + (doto (.toFile (Files/createTempDirectory "deps_try" (into-array FileAttribute []))) + (.deleteOnExit))) + +(defn- unpack-from-jar [resource-name] + (let [path (io/file temp-directory resource-name)] + (if-let [resource (io/resource resource-name)] + (with-open [in (io/input-stream resource)] + (io/copy in path)) + (throw (ex-info (str "Could not find " resource-name " in resources.") {}))) + (.deleteOnExit path) + (.getAbsolutePath path))) + +(defn- macos? [] + (re-find #"(?i)mac" (System/getProperty "os.name"))) + +(defn- linux? [] + (re-find #"(?i)linux" (System/getProperty "os.name"))) + +(defn- aarch64? [] + (re-find #"(?i)aarch64" (System/getProperty "os.arch"))) + +(def ^:private libdt-path + (delay + (let [os (System/getProperty "os.name") + lib (cond (macos?) "libdt-macos-universal.so" + (and (linux?) (aarch64?)) "libdt-linux-arm64.so" + (linux?) "libdt-linux-x64.so" + :else (throw (ex-info (str "no native agent bundled for " os + " (only Linux and macOS)") {:os os})))] + (unpack-from-jar lib)))) + +;;; Agent loading + +(defn- attach-self ^VirtualMachine [] + (VirtualMachine/attach (str (.pid (ProcessHandle/current))))) + +(defn- load-libdt-agent [] + (doto (attach-self) + (.loadAgentPath @libdt-path) + (.detach))) + +(def ^:private agent-loaded (delay (load-libdt-agent))) + +(defn stop-thread + "Stop the given `thread` using JVMTI StopThread function. Risks state + corruption. Should not be used prior to JDK20." + [thread] + @agent-loaded + (JvmtiAgent/stopThread thread)) diff --git a/tmp/.keep b/tmp/.keep new file mode 100644 index 00000000..e69de29b