diff --git a/Cargo.lock b/Cargo.lock index 0ad1f4300..c83292541 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -688,7 +688,7 @@ version = "0.1.10" dependencies = [ "anyhow", "bumpalo", - "cc", + "diffr-grammars", "dirs", "glob", "hashbrown 0.17.1", @@ -700,7 +700,6 @@ dependencies = [ "log", "pretty_assertions", "radix-heap", - "rayon", "regex", "rustc-hash", "schemars", @@ -776,6 +775,21 @@ dependencies = [ "typed-arena", ] +[[package]] +name = "diffr-grammars" +version = "0.1.10" +dependencies = [ + "cc", + "object", + "rayon", + "serde_json", + "tree-sitter", + "tree-sitter-c", + "tree-sitter-language", + "zstd", + "zstd-sys", +] + [[package]] name = "diffr-plugin-sdk" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index be804a238..1ef866327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,8 +66,8 @@ tokio = { version = "1", features = ["macros", "net", "process"] } version_check = "0.9.4" [workspace] -members = ["xtask", "crates/diffr-core", "crates/diffr-plugin-sdk"] -default-members = [".", "crates/diffr-core", "crates/diffr-plugin-sdk"] +members = ["xtask", "crates/diffr-core", "crates/diffr-grammars", "crates/diffr-plugin-sdk"] +default-members = [".", "crates/diffr-core", "crates/diffr-grammars", "crates/diffr-plugin-sdk"] exclude = ["plugins", "tests/fixtures/concurrent-guest"] [profile.release] diff --git a/crates/diffr-core/Cargo.toml b/crates/diffr-core/Cargo.toml index 6d3f88226..64409d2ff 100644 --- a/crates/diffr-core/Cargo.toml +++ b/crates/diffr-core/Cargo.toml @@ -17,6 +17,7 @@ include = [ ] [dependencies] +diffr-grammars = { path = "../diffr-grammars", version = "0.1.10" } regex = "1.10.4" typed-arena = "2.0.2" rustc-hash = "2.0.0" @@ -119,21 +120,17 @@ tempfile = "3.27.0" [build-dependencies] toml = "0.8" -# TODO: enable parallel mode, see discussion in -# https://github.com/rust-lang/cc-rs/pull/849 -cc = "1.2.48" -rayon = "1.7.0" [features] all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog", "lang-ocaml", "lang-julia", "lang-haskell", "lang-vhdl", "lang-apex", "lang-qml"] -lang-fortran = ["dep:tree-sitter-fortran"] -lang-fsharp = ["dep:tree-sitter-fsharp"] -lang-verilog = ["dep:tree-sitter-verilog"] -lang-ocaml = ["dep:tree-sitter-ocaml"] -lang-julia = ["dep:tree-sitter-julia"] -lang-haskell = ["dep:tree-sitter-haskell"] -lang-vhdl = ["dep:tree-sitter-vhdl"] -lang-apex = ["dep:tree-sitter-sfapex"] -lang-qml = ["dep:tree-sitter-qmljs"] +lang-fortran = ["dep:tree-sitter-fortran", "diffr-grammars/lang-fortran"] +lang-fsharp = ["dep:tree-sitter-fsharp", "diffr-grammars/lang-fsharp"] +lang-verilog = ["dep:tree-sitter-verilog", "diffr-grammars/lang-verilog"] +lang-ocaml = ["dep:tree-sitter-ocaml", "diffr-grammars/lang-ocaml"] +lang-julia = ["dep:tree-sitter-julia", "diffr-grammars/lang-julia"] +lang-haskell = ["dep:tree-sitter-haskell", "diffr-grammars/lang-haskell"] +lang-vhdl = ["dep:tree-sitter-vhdl", "diffr-grammars/lang-vhdl"] +lang-apex = ["dep:tree-sitter-sfapex", "diffr-grammars/lang-apex"] +lang-qml = ["dep:tree-sitter-qmljs", "diffr-grammars/lang-qml"] # Constructors and helpers the diffr package's tests build on. test-support = [] diff --git a/crates/diffr-core/build.rs b/crates/diffr-core/build.rs index ef30f8348..a0ba3a666 100644 --- a/crates/diffr-core/build.rs +++ b/crates/diffr-core/build.rs @@ -8,72 +8,9 @@ use std::path::PathBuf; -use rayon::prelude::*; - -struct TreeSitterParser { - name: &'static str, - src_dir: &'static str, - extra_files: Vec<&'static str>, -} - -impl TreeSitterParser { - fn build(&self) { - let dir = PathBuf::from(&self.src_dir); - - let mut c_files = vec!["parser.c"]; - c_files.extend_from_slice(&self.extra_files); - - let mut build = cc::Build::new(); - if cfg!(target_env = "msvc") { - build.flag("/utf-8"); - } - build.include(&dir).warnings(false); // ignore unused parameter warnings - for file in c_files { - build.file(dir.join(file)); - } - - build.link_lib_modifier("+whole-archive"); - - build.compile(self.name); - } -} - -fn main() { - bundled_plugins(); - let parsers = vec![ - TreeSitterParser { - name: "tree-sitter-janet-simple", - src_dir: "../../vendored_parsers/tree-sitter-janet-simple-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-kotlin", - src_dir: "../../vendored_parsers/tree-sitter-kotlin-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-latex", - src_dir: "../../vendored_parsers/tree-sitter-latex-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-smali", - src_dir: "../../vendored_parsers/tree-sitter-smali-src", - extra_files: vec!["scanner.c"], - }, - ]; - - // Only rerun if relevant files in the vendored_parsers/ directory change. - for parser in &parsers { - println!("cargo:rerun-if-changed={}", parser.src_dir); - } - - parsers.par_iter().for_each(|p| p.build()); -} - /// Embed the component, manifest and queries from every bundled shape plugin /// folder under `plugins/shape/`, and the classifier in `plugins/classify/`. -fn bundled_plugins() { +fn main() { let root = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("../.."); println!("cargo:rerun-if-changed={}", root.join("plugins").display()); let mut files = String::from("const FILES: &[(&str, &str)] = &[\n"); diff --git a/crates/diffr-core/src/parse/tree_sitter_parser.rs b/crates/diffr-core/src/parse/tree_sitter_parser.rs index a3a7055d4..b78aa4811 100644 --- a/crates/diffr-core/src/parse/tree_sitter_parser.rs +++ b/crates/diffr-core/src/parse/tree_sitter_parser.rs @@ -84,13 +84,6 @@ pub struct TreeSitterConfig { pub(crate) sub_languages: Vec, } -extern "C" { - fn tree_sitter_janet_simple() -> ts::Language; - fn tree_sitter_kotlin() -> ts::Language; - fn tree_sitter_latex() -> ts::Language; - fn tree_sitter_smali() -> ts::Language; -} - // TODO: begin/end and object/end. #[cfg(feature = "lang-ocaml")] const OCAML_ATOM_NODES: [&str; 6] = [ @@ -122,7 +115,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { use guess::Language::*; match language { Ada => { - let language_fn = tree_sitter_ada::LANGUAGE; + let language_fn = diffr_grammars::ADA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -141,7 +134,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-apex")] Apex => { - let language_fn = tree_sitter_sfapex::apex::LANGUAGE; + let language_fn = diffr_grammars::APEX; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -168,7 +161,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Asm => { - let language_fn = tree_sitter_asm::LANGUAGE; + let language_fn = diffr_grammars::ASM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -182,7 +175,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Bash => { - let language_fn = tree_sitter_bash::LANGUAGE; + let language_fn = diffr_grammars::BASH; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -198,7 +191,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } C => { - let language_fn = tree_sitter_c::LANGUAGE; + let language_fn = diffr_grammars::C; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -210,7 +203,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CPlusPlus => { - let language_fn = tree_sitter_cpp::LANGUAGE; + let language_fn = diffr_grammars::CPP; let language = tree_sitter::Language::new(language_fn); let mut highlight_query = tree_sitter_c::HIGHLIGHT_QUERY.to_owned(); @@ -230,7 +223,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Clojure => { - let language_fn = tree_sitter_clojure_orchard::LANGUAGE; + let language_fn = diffr_grammars::CLOJURE_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -249,7 +242,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CMake => { - let language_fn = tree_sitter_cmake::LANGUAGE; + let language_fn = diffr_grammars::CMAKE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -265,7 +258,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CommonLisp => { - let language_fn = tree_sitter_commonlisp::LANGUAGE_COMMONLISP; + let language_fn = diffr_grammars::COMMONLISP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -278,7 +271,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CSharp => { - let language_fn = tree_sitter_c_sharp::LANGUAGE; + let language_fn = diffr_grammars::C_SHARP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -301,7 +294,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Css => { - let language_fn = tree_sitter_css::LANGUAGE; + let language_fn = diffr_grammars::CSS; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -322,7 +315,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Dart => { - let language_fn = tree_sitter_dart_orchard::LANGUAGE; + let language_fn = diffr_grammars::DART_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -338,7 +331,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } DeviceTree => { - let language_fn = tree_sitter_devicetree::LANGUAGE; + let language_fn = diffr_grammars::DEVICETREE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -356,7 +349,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Dockerfile => { - let language_fn = tree_sitter_containerfile::LANGUAGE; + let language_fn = diffr_grammars::CONTAINERFILE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -383,7 +376,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Elixir => { - let language_fn = tree_sitter_elixir::LANGUAGE; + let language_fn = diffr_grammars::ELIXIR; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -399,7 +392,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Elm => { - let language_fn = tree_sitter_elm::LANGUAGE; + let language_fn = diffr_grammars::ELM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -413,7 +406,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } EmacsLisp => { - let language_fn = tree_sitter_elisp::LANGUAGE; + let language_fn = diffr_grammars::ELISP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -429,7 +422,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Erlang => { - let language_fn = tree_sitter_erlang::LANGUAGE; + let language_fn = diffr_grammars::ERLANG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -443,7 +436,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Fish => { - let language = tree_sitter_fish::language(); + let language = tree_sitter::Language::new(diffr_grammars::FISH); let highlight_query = ts::Query::new(&language, tree_sitter_fish::HIGHLIGHTS_QUERY).unwrap(); @@ -460,7 +453,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-fsharp")] FSharp => { - let language_fn = tree_sitter_fsharp::LANGUAGE_FSHARP; + let language_fn = diffr_grammars::FSHARP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -476,7 +469,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-fortran")] Fortran => { - let language_fn = tree_sitter_fortran::LANGUAGE; + let language_fn = diffr_grammars::FORTRAN; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -489,7 +482,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Gleam => { - let language_fn = tree_sitter_gleam::LANGUAGE; + let language_fn = diffr_grammars::GLEAM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -502,7 +495,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Go => { - let language_fn = tree_sitter_go::LANGUAGE; + let language_fn = diffr_grammars::GO; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -521,7 +514,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-haskell")] Haskell => { - let language_fn = tree_sitter_haskell::LANGUAGE; + let language_fn = diffr_grammars::HASKELL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -534,7 +527,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Hcl => { - let language_fn = tree_sitter_hcl::LANGUAGE; + let language_fn = diffr_grammars::HCL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -557,7 +550,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Html => { - let language_fn = tree_sitter_html::LANGUAGE; + let language_fn = diffr_grammars::HTML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -592,7 +585,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Janet => { - let language = unsafe { tree_sitter_janet_simple() }; + let language = tree_sitter::Language::new(diffr_grammars::JANET_SIMPLE); TreeSitterConfig { language: language.clone(), atom_nodes: [].into_iter().collect(), @@ -616,7 +609,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Java => { - let language_fn = tree_sitter_java_orchard::LANGUAGE; + let language_fn = diffr_grammars::JAVA_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -649,7 +642,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } JavaScript | JavascriptJsx => { - let language_fn = tree_sitter_javascript::LANGUAGE; + let language_fn = diffr_grammars::JAVASCRIPT; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -679,7 +672,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Json => { - let language_fn = tree_sitter_json::LANGUAGE; + let language_fn = diffr_grammars::JSON; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -694,7 +687,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-julia")] Julia => { - let language_fn = tree_sitter_julia::LANGUAGE; + let language_fn = diffr_grammars::JULIA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -718,7 +711,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Kotlin => { - let language = unsafe { tree_sitter_kotlin() }; + let language = tree_sitter::Language::new(diffr_grammars::KOTLIN); TreeSitterConfig { language: language.clone(), // Flattening nullable type means we can't diff the @@ -746,7 +739,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } LaTeX => { - let language = unsafe { tree_sitter_latex() }; + let language = tree_sitter::Language::new(diffr_grammars::LATEX); TreeSitterConfig { language: language.clone(), atom_nodes: [].into_iter().collect(), @@ -761,7 +754,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Lua => { - let language_fn = tree_sitter_lua::LANGUAGE; + let language_fn = diffr_grammars::LUA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -777,7 +770,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Make => { - let language_fn = tree_sitter_make::LANGUAGE; + let language_fn = diffr_grammars::MAKE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -795,7 +788,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Newick => { - let language_fn = tree_sitter_newick::LANGUAGE; + let language_fn = diffr_grammars::NEWICK; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -812,7 +805,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Nix => { - let language_fn = tree_sitter_nix::LANGUAGE; + let language_fn = diffr_grammars::NIX; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -828,7 +821,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } ObjC => { - let language_fn = tree_sitter_objc::LANGUAGE; + let language_fn = diffr_grammars::OBJC; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -849,7 +842,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-ocaml")] OCaml => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML; + let language_fn = diffr_grammars::OCAML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -863,7 +856,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-ocaml")] OCamlInterface => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE; + let language_fn = diffr_grammars::OCAML_INTERFACE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -876,7 +869,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Pascal => { - let language_fn = tree_sitter_pascal::LANGUAGE; + let language_fn = diffr_grammars::PASCAL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -893,7 +886,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Perl => { - let language_fn = ts_parser_perl::LANGUAGE; + let language_fn = diffr_grammars::PERL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -917,7 +910,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Php => { - let language_fn = tree_sitter_php::LANGUAGE_PHP; + let language_fn = diffr_grammars::PHP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -931,7 +924,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Proto => { - let language_fn = tree_sitter_proto::LANGUAGE; + let language_fn = diffr_grammars::PROTO; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -947,7 +940,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Python => { - let language_fn = tree_sitter_python::LANGUAGE; + let language_fn = diffr_grammars::PYTHON; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -969,7 +962,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-qml")] Qml => { - let language_fn = tree_sitter_qmljs::LANGUAGE; + let language_fn = diffr_grammars::QMLJS; let language = tree_sitter::Language::new(language_fn); let mut highlight_query = tree_sitter_javascript::HIGHLIGHT_QUERY.to_owned(); @@ -986,7 +979,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } R => { - let language_fn = tree_sitter_r::LANGUAGE; + let language_fn = diffr_grammars::R; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -999,7 +992,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Racket => { - let language_fn = tree_sitter_racket::LANGUAGE; + let language_fn = diffr_grammars::RACKET; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1014,7 +1007,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Ruby => { - let language_fn = tree_sitter_ruby::LANGUAGE; + let language_fn = diffr_grammars::RUBY; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1035,7 +1028,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Rust => { - let language_fn = tree_sitter_rust_orchard::LANGUAGE; + let language_fn = diffr_grammars::RUST_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1068,7 +1061,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Scala => { - let language_fn = tree_sitter_scala::LANGUAGE; + let language_fn = diffr_grammars::SCALA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1094,7 +1087,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Scheme => { - let language_fn = tree_sitter_scheme::LANGUAGE; + let language_fn = diffr_grammars::SCHEME; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1107,7 +1100,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Smali => { - let language = unsafe { tree_sitter_smali() }; + let language = tree_sitter::Language::new(diffr_grammars::SMALI); TreeSitterConfig { language: language.clone(), atom_nodes: ["string"].into_iter().collect(), @@ -1122,7 +1115,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Solidity => { - let language_fn = tree_sitter_solidity::LANGUAGE; + let language_fn = diffr_grammars::SOLIDITY; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1137,7 +1130,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Sql => { - let language_fn = tree_sitter_sequel::LANGUAGE; + let language_fn = diffr_grammars::SQL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1150,7 +1143,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Swift => { - let language_fn = tree_sitter_swift::LANGUAGE; + let language_fn = diffr_grammars::SWIFT; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1168,7 +1161,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Toml => { - let language_fn = tree_sitter_toml_ng::LANGUAGE; + let language_fn = diffr_grammars::TOML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1189,9 +1182,9 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } TypeScript | TypeScriptTsx => { let language_fn = if language == TypeScript { - tree_sitter_typescript::LANGUAGE_TYPESCRIPT + diffr_grammars::TYPESCRIPT } else { - tree_sitter_typescript::LANGUAGE_TSX + diffr_grammars::TSX }; let language = tree_sitter::Language::new(language_fn); @@ -1218,7 +1211,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Xml => { - let language_fn = tree_sitter_xml::LANGUAGE_XML; + let language_fn = diffr_grammars::XML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1235,7 +1228,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Yaml => { - let language_fn = tree_sitter_yaml::LANGUAGE; + let language_fn = diffr_grammars::YAML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1257,7 +1250,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-verilog")] Verilog => { - let language_fn = tree_sitter_verilog::LANGUAGE; + let language_fn = diffr_grammars::VERILOG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { ignore_trailing_tokens: vec![], @@ -1274,7 +1267,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-vhdl")] Vhdl => { - let language_fn = tree_sitter_vhdl::LANGUAGE; + let language_fn = diffr_grammars::VHDL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1288,7 +1281,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Zig => { - let language_fn = tree_sitter_zig::LANGUAGE; + let language_fn = diffr_grammars::ZIG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { diff --git a/crates/diffr-grammars/Cargo.toml b/crates/diffr-grammars/Cargo.toml new file mode 100644 index 000000000..bd2b1e45d --- /dev/null +++ b/crates/diffr-grammars/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "diffr-grammars" +description = "Tree-sitter grammars for diffr, with their largest parse tables stored compressed." +repository = "https://github.com/devdotfast/diffr" +homepage = "https://github.com/devdotfast/diffr" +license = "MIT" +version = "0.1.10" +authors = ["dev.fast"] +edition = "2021" +rust-version = "1.99.0" + +include = [ + "/build.rs", + "/src/", +] + +[dependencies] +tree-sitter-language = "0.1.7" +# The grammars' C code calls ZSTD_decompress. +zstd-sys = { version = "2.1.0", default-features = false } + +[build-dependencies] +cc = "1.2.48" +object = { version = "0.40", default-features = false, features = ["read_core", "coff", "elf", "macho", "std"] } +rayon = "1.7.0" +serde_json = "1.0" +tree-sitter = "0.26.10" +tree-sitter-c = "0.24" +zstd = { version = "0.13.3", default-features = false } + +[features] +lang-apex = [] +lang-fortran = [] +lang-fsharp = [] +lang-haskell = [] +lang-julia = [] +lang-ocaml = [] +lang-qml = [] +lang-verilog = [] +lang-vhdl = [] diff --git a/crates/diffr-grammars/build.rs b/crates/diffr-grammars/build.rs new file mode 100644 index 000000000..0eaa4d884 --- /dev/null +++ b/crates/diffr-grammars/build.rs @@ -0,0 +1,534 @@ +//! Link the pinned grammars with their parse tables compressed. +//! +//! Tree-sitter generates each grammar as one `parser.c`: constant tables plus a +//! lexer. This script packs every table whose elements are plain data. It reads +//! the table's bytes from an object file compiled for the target, stores them +//! as zstd bytes, and leaves an empty array in its place. A generated C +//! function, `diffr_unpack_`, fills the arrays, and `src/lib.rs` runs it +//! once before the grammar's first use. +use object::{Object, ObjectSection, ObjectSymbol}; +use rayon::prelude::*; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::process::Command; + +// Grammar crates: (constructor symbol, crate name, source folder, required feature). +const PACKAGES: &[(&str, &str, &str, Option<&str>)] = &[ + ("tree_sitter_go", "tree-sitter-go", "src", None), + ( + "tree_sitter_javascript", + "tree-sitter-javascript", + "src", + None, + ), + ("tree_sitter_python", "tree-sitter-python", "src", None), + ( + "tree_sitter_rust_orchard", + "tree-sitter-rust-orchard", + "src", + None, + ), + ( + "tree_sitter_typescript", + "tree-sitter-typescript", + "typescript/src", + None, + ), + ("tree_sitter_tsx", "tree-sitter-typescript", "tsx/src", None), + ("tree_sitter_ada", "tree-sitter-ada", "src", None), + ( + "tree_sitter_apex", + "tree-sitter-sfapex", + "apex/src", + Some("lang-apex"), + ), + ("tree_sitter_asm", "tree-sitter-asm", "src", None), + ("tree_sitter_bash", "tree-sitter-bash", "src", None), + ("tree_sitter_c", "tree-sitter-c", "src", None), + ("tree_sitter_c_sharp", "tree-sitter-c-sharp", "src", None), + ( + "tree_sitter_clojure_orchard", + "tree-sitter-clojure-orchard", + "src", + None, + ), + ("tree_sitter_cmake", "tree-sitter-cmake", "src", None), + ( + "tree_sitter_commonlisp", + "tree-sitter-commonlisp", + "src", + None, + ), + ( + "tree_sitter_containerfile", + "tree-sitter-containerfile", + "src", + None, + ), + ("tree_sitter_cpp", "tree-sitter-cpp", "src", None), + ("tree_sitter_css", "tree-sitter-css", "src", None), + ( + "tree_sitter_dart_orchard", + "tree-sitter-dart-orchard", + "src", + None, + ), + ( + "tree_sitter_devicetree", + "tree-sitter-devicetree", + "src", + None, + ), + ("tree_sitter_elisp", "tree-sitter-elisp", "src", None), + ("tree_sitter_elixir", "tree-sitter-elixir", "src", None), + ("tree_sitter_elm", "tree-sitter-elm", "src", None), + ("tree_sitter_erlang", "tree-sitter-erlang", "src", None), + ("tree_sitter_fish", "tree-sitter-fish", "src", None), + ( + "tree_sitter_fortran", + "tree-sitter-fortran", + "src", + Some("lang-fortran"), + ), + ( + "tree_sitter_fsharp", + "tree-sitter-fsharp", + "fsharp/src", + Some("lang-fsharp"), + ), + ("tree_sitter_gleam", "tree-sitter-gleam", "src", None), + ( + "tree_sitter_haskell", + "tree-sitter-haskell", + "src", + Some("lang-haskell"), + ), + ("tree_sitter_hcl", "tree-sitter-hcl", "src", None), + ("tree_sitter_html", "tree-sitter-html", "src", None), + ( + "tree_sitter_java_orchard", + "tree-sitter-java-orchard", + "src", + None, + ), + ("tree_sitter_json", "tree-sitter-json", "src", None), + ( + "tree_sitter_julia", + "tree-sitter-julia", + "src", + Some("lang-julia"), + ), + ("tree_sitter_lua", "tree-sitter-lua", "src", None), + ("tree_sitter_make", "tree-sitter-make", "src", None), + ("tree_sitter_newick", "tree-sitter-newick", "src", None), + ("tree_sitter_nix", "tree-sitter-nix", "src", None), + ("tree_sitter_objc", "tree-sitter-objc", "src", None), + ( + "tree_sitter_ocaml", + "tree-sitter-ocaml", + "grammars/ocaml/src", + Some("lang-ocaml"), + ), + ( + "tree_sitter_ocaml_interface", + "tree-sitter-ocaml", + "grammars/interface/src", + Some("lang-ocaml"), + ), + ("tree_sitter_pascal", "tree-sitter-pascal", "src", None), + ("tree_sitter_perl", "ts-parser-perl", "src", None), + ("tree_sitter_php", "tree-sitter-php", "php/src", None), + ("tree_sitter_proto", "tree-sitter-proto", "src", None), + ( + "tree_sitter_qmljs", + "tree-sitter-qmljs", + "src", + Some("lang-qml"), + ), + ("tree_sitter_r", "tree-sitter-r", "src", None), + ("tree_sitter_racket", "tree-sitter-racket", "src", None), + ("tree_sitter_ruby", "tree-sitter-ruby", "src", None), + ("tree_sitter_scala", "tree-sitter-scala", "src", None), + ("tree_sitter_scheme", "tree-sitter-scheme", "src", None), + ("tree_sitter_solidity", "tree-sitter-solidity", "src", None), + ("tree_sitter_sql", "tree-sitter-sequel", "src", None), + ("tree_sitter_swift", "tree-sitter-swift", "src", None), + ("tree_sitter_toml", "tree-sitter-toml-ng", "src", None), + ( + "tree_sitter_verilog", + "tree-sitter-verilog", + "src", + Some("lang-verilog"), + ), + ( + "tree_sitter_vhdl", + "tree-sitter-vhdl", + "src", + Some("lang-vhdl"), + ), + ("tree_sitter_xml", "tree-sitter-xml", "xml/src", None), + ("tree_sitter_yaml", "tree-sitter-yaml", "src", None), + ("tree_sitter_zig", "tree-sitter-zig", "src", None), +]; + +// Grammars in vendored_parsers: (constructor symbol, folder). +const VENDORED: &[(&str, &str)] = &[ + ("tree_sitter_janet_simple", "tree-sitter-janet-simple-src"), + ("tree_sitter_kotlin", "tree-sitter-kotlin-src"), + ("tree_sitter_latex", "tree-sitter-latex-src"), + ("tree_sitter_smali", "tree-sitter-smali-src"), +]; + +fn main() { + let roots = package_roots(); + let mut sources: Vec<(&str, PathBuf)> = PACKAGES + .iter() + .filter(|(_, _, _, feature)| { + feature.is_none_or(|feature| { + let name = feature.replace('-', "_").to_uppercase(); + std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some() + }) + }) + .map(|&(symbol, package, folder, _)| (symbol, roots[package].join(folder))) + .collect(); + sources.extend( + VENDORED + .iter() + .map(|&(symbol, folder)| (symbol, Path::new("../../vendored_parsers").join(folder))), + ); + for (_, source) in &sources { + println!("cargo:rerun-if-changed={}", source.display()); + } + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + sources + .par_iter() + .for_each(|(symbol, source)| pack(symbol, source, &out.join(symbol))); + let mut grammars = String::new(); + for (symbol, _) in &sources { + let name = symbol.strip_prefix("tree_sitter_").unwrap().to_uppercase(); + writeln!( + grammars, + "grammar!({name}, diffr_unpack_{symbol}, diffr_{symbol});" + ) + .unwrap(); + } + std::fs::write(out.join("grammars.rs"), grammars).unwrap(); +} + +/// Map each locked package name to its source directory, including vendored registries. +fn package_roots() -> BTreeMap { + let metadata = Command::new(std::env::var_os("CARGO").unwrap()) + .args([ + "metadata", + "--locked", + "--all-features", + "--format-version", + "1", + ]) + .output() + .expect("reading Cargo dependency metadata"); + assert!( + metadata.status.success(), + "{}", + String::from_utf8_lossy(&metadata.stderr) + ); + let metadata: serde_json::Value = serde_json::from_slice(&metadata.stdout).unwrap(); + metadata["packages"] + .as_array() + .unwrap() + .iter() + .map(|package| { + let manifest = Path::new(package["manifest_path"].as_str().unwrap()); + ( + package["name"].as_str().unwrap().to_owned(), + manifest.parent().unwrap().to_owned(), + ) + }) + .collect() +} + +/// Write `/grammar.c` with packed tables, the `diffr_unpack_` function that +/// fills them, and the grammar's constructor renamed to `diffr_`, then compile it. +fn pack(symbol: &str, source: &Path, out: &Path) { + std::fs::create_dir_all(out).unwrap(); + let parser = std::fs::read_to_string(source.join("parser.c")).unwrap(); + let header = std::fs::read_to_string(source.join("tree_sitter/parser.h")).unwrap(); + let tables = tables(&parser, &header); + let bytes = table_bytes(symbol, source, out, &parser, &tables); + + let mut grammar = format!( + "#include \n\ + size_t ZSTD_decompress(void *dst, size_t dst_capacity, const void *src, size_t src_size);\n\ + #define {symbol} diffr_{symbol}\n" + ); + grammar.push_str(&empty_tables(&parser, &tables, &bytes)); + for (table, bytes) in tables.iter().zip(&bytes) { + let packed = zstd::bulk::compress(bytes, 19).unwrap(); + write!( + grammar, + "\nstatic const unsigned char packed_{}[] = {{", + table.name + ) + .unwrap(); + for byte in packed { + write!(grammar, "{byte},").unwrap(); + } + grammar.push_str("};\n"); + } + grammar.push_str( + "\nstatic void unpack(void *table, size_t size, const unsigned char *packed, size_t packed_size) {\n\ + \x20 if (ZSTD_decompress(table, size, packed, packed_size) != size) abort();\n\ + }\n", + ); + writeln!(grammar, "\nvoid diffr_unpack_{symbol}(void) {{").unwrap(); + for Table { name, .. } in &tables { + writeln!( + grammar, + " unpack({name}, sizeof {name}, packed_{name}, sizeof packed_{name});" + ) + .unwrap(); + } + grammar.push_str("}\n"); + std::fs::write(out.join("grammar.c"), grammar).unwrap(); + + let mut build = compiler(source, out); + build.file(out.join("grammar.c")); + if source.join("scanner.c").is_file() { + build.file(source.join("scanner.c")); + } + build.compile(symbol); +} + +/// Compile the tables for the target and read each one's bytes out of the object file. +fn table_bytes( + symbol: &str, + source: &Path, + out: &Path, + parser: &str, + tables: &[Table], +) -> Vec> { + // Global symbols can be found by name. Mach-O and COFF symbols carry no + // size, so each table's size is compiled in beside it. + let mut globals = parser.to_owned(); + for storage in tables + .iter() + .rev() + .filter_map(|table| table.storage.clone()) + { + globals.replace_range(storage, ""); + } + for Table { name, .. } in tables { + writeln!( + globals, + "const unsigned long long diffr_size_{name} = sizeof {name};" + ) + .unwrap(); + } + let tables_c = out.join("tables.c"); + std::fs::write(&tables_c, globals).unwrap(); + let [object_path] = <[PathBuf; 1]>::try_from( + compiler(source, out) + .file(&tables_c) + .opt_level(0) + .compile_intermediates(), + ) + .unwrap(); + + let data = std::fs::read(object_path).unwrap(); + let file = object::File::parse(&*data).unwrap(); + let prefix = if file.format() == object::BinaryFormat::MachO { + "_" + } else { + "" + }; + let read = |name: &str, size: usize| -> Vec { + let symbol = file + .symbol_by_name(&format!("{prefix}{name}")) + .unwrap_or_else(|| panic!("{name} is missing from the {symbol} tables object")); + let section = file + .section_by_index(symbol.section_index().unwrap()) + .unwrap(); + if section.kind() == object::SectionKind::UninitializedData { + return vec![0; size]; + } + let start = usize::try_from(symbol.address() - section.address()).unwrap(); + section.data().unwrap()[start..start + size].to_vec() + }; + tables + .iter() + .map(|Table { name, .. }| { + let size: [u8; 8] = read(&format!("diffr_size_{name}"), 8).try_into().unwrap(); + let size = if file.is_little_endian() { + u64::from_le_bytes(size) + } else { + u64::from_be_bytes(size) + }; + read(name, usize::try_from(size).unwrap()) + }) + .collect() +} + +fn compiler(source: &Path, out: &Path) -> cc::Build { + let mut build = cc::Build::new(); + build.include(source).warnings(false).out_dir(out); + if std::env::var("CARGO_CFG_TARGET_ENV").unwrap() == "msvc" { + build.flag("/utf-8"); + } + build +} + +/// A top-level array definition in `parser.c`, `T name[...] = {...};`, whose +/// elements are plain data. +struct Table<'a> { + name: &'a str, + element: &'a str, + /// The array declarator when it states its size; `None` for `name[]`. + sized_declarator: Option<&'a str>, + /// The `static` keyword, if any. + storage: Option>, + definition: Range, +} + +/// Find the tables to pack: top-level arrays whose element type is plain data. +fn tables<'a>(parser: &'a str, header: &str) -> Vec> { + let mut c = tree_sitter::Parser::new(); + c.set_language(&tree_sitter_c::LANGUAGE.into()).unwrap(); + let header_tree = c.parse(header, None).unwrap(); + let mut typedefs = BTreeMap::new(); + let mut nodes = vec![header_tree.root_node()]; + while let Some(node) = nodes.pop() { + let mut cursor = node.walk(); + if node.kind() != "type_definition" { + nodes.extend(node.children(&mut cursor)); + continue; + } + let ty = node.child_by_field_name("type").unwrap(); + // A typedef declared through a pointer is left out, so a table of it fails to resolve. + for name in node + .children_by_field_name("declarator", &mut cursor) + .filter(|declarator| declarator.kind() == "type_identifier") + { + typedefs.insert(&header[name.byte_range()], ty); + } + } + + let tree = c.parse(parser, None).unwrap(); + let text = |node: tree_sitter::Node| &parser[node.byte_range()]; + let mut cursor = tree.walk(); + tree.root_node() + .children(&mut cursor) + .filter_map(|definition| { + let init = definition + .child_by_field_name("declarator") + .filter(|init| { + definition.kind() == "declaration" && init.kind() == "init_declarator" + })?; + init.child_by_field_name("value") + .filter(|value| value.kind() == "initializer_list")?; + // An array of pointers, such as `ts_symbol_names`, has a pointer declarator here. + let array = init + .child_by_field_name("declarator") + .filter(|array| array.kind() == "array_declarator")?; + let mut name = array; + while name.kind() == "array_declarator" { + name = name.child_by_field_name("declarator").unwrap(); + } + let ty = definition.child_by_field_name("type").unwrap(); + let mut cursor = definition.walk(); + let storage = definition + .children(&mut cursor) + .find(|child| child.kind() == "storage_class_specifier" && text(*child) == "static") + .map(|keyword| keyword.byte_range()); + plain_data(ty, parser, header, &typedefs).then(|| Table { + name: text(name), + element: text(ty), + sized_declarator: array.child_by_field_name("size").map(|_| text(array)), + storage, + definition: definition.byte_range(), + }) + }) + .collect() +} + +/// Whether values of the C type `ty` are plain data: integers, `bool`, enums, +/// and structs or unions of them, with no pointers. Panics on a type it cannot +/// resolve, so an unknown type never gets packed by accident. +fn plain_data( + ty: tree_sitter::Node, + source: &str, + header: &str, + typedefs: &BTreeMap<&str, tree_sitter::Node>, +) -> bool { + match ty.kind() { + "primitive_type" | "sized_type_specifier" | "enum_specifier" => true, + "type_identifier" => { + let name = &source[ty.byte_range()]; + let ty = typedefs + .get(name) + .unwrap_or_else(|| panic!("cannot resolve the C type {name}")); + plain_data(*ty, header, header, typedefs) + } + "struct_specifier" | "union_specifier" => { + let body = ty.child_by_field_name("body").unwrap_or_else(|| { + panic!("cannot resolve the C type {}", &source[ty.byte_range()]) + }); + let mut cursor = body.walk(); + let plain = body + .named_children(&mut cursor) + .filter(|field| field.kind() != "comment") + .all(|field| { + assert_eq!( + field.kind(), + "field_declaration", + "unexpected struct member" + ); + let mut cursor = field.walk(); + let plain_declarators = field + .children_by_field_name("declarator", &mut cursor) + .all(|mut declarator| { + while declarator.kind() == "array_declarator" { + declarator = declarator.child_by_field_name("declarator").unwrap(); + } + declarator.kind() == "field_identifier" + }); + plain_declarators + && plain_data( + field.child_by_field_name("type").unwrap(), + source, + header, + typedefs, + ) + }); + plain + } + kind => panic!( + "cannot resolve the C type {} ({kind})", + &source[ty.byte_range()] + ), + } +} + +/// Replace each table's definition with an empty, writable array of the same size. +fn empty_tables(parser: &str, tables: &[Table], bytes: &[Vec]) -> String { + let mut emptied = parser.to_owned(); + for (table, bytes) in tables.iter().zip(bytes).rev() { + let declarator = table.sized_declarator.map_or_else( + || { + format!( + "{}[{} / sizeof({})]", + table.name, + bytes.len(), + table.element + ) + }, + str::to_owned, + ); + emptied.replace_range( + table.definition.clone(), + &format!("static {} {declarator};", table.element), + ); + } + emptied +} diff --git a/crates/diffr-grammars/src/lib.rs b/crates/diffr-grammars/src/lib.rs new file mode 100644 index 000000000..4d8748793 --- /dev/null +++ b/crates/diffr-grammars/src/lib.rs @@ -0,0 +1,30 @@ +//! Tree-sitter grammars with their largest parse tables stored compressed. +//! +//! Each constant is a grammar constructor, like a grammar crate's `LANGUAGE`. +//! The first call unpacks that grammar's tables. +use std::sync::Once; +use tree_sitter_language::LanguageFn; +// Links libzstd for the grammars' C code. +use zstd_sys as _; + +macro_rules! grammar { + ($name:ident, $unpack:ident, $constructor:ident) => { + pub const $name: LanguageFn = { + unsafe extern "C" fn language() -> *const () { + extern "C" { + fn $unpack(); + fn $constructor() -> *const (); + } + static UNPACKED: Once = Once::new(); + // Safety: the unpack function fills this grammar's tables, and + // `Once` runs it to completion before any caller reads them. + UNPACKED.call_once(|| unsafe { $unpack() }); + unsafe { $constructor() } + } + // Safety: `language` returns the grammar's `TSLanguage` pointer. + unsafe { LanguageFn::from_raw(language) } + }; + }; +} + +include!(concat!(env!("OUT_DIR"), "/grammars.rs"));