From ba9ef132856ea9e0ec72a9a15fbcb02dca223377 Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Sat, 3 Oct 2026 22:32:32 -0700 Subject: [PATCH 1/2] Add diffr-grammars: Tree-sitter grammars with compressed parse tables Each grammar is linked statically, but its tables of plain data are stored as zstd bytes. The build script parses the locked parser.c with tree-sitter-c and picks every top-level array whose element type is plain data: integers, bool, enums, and structs or unions of them, resolved through the grammar's tree_sitter/parser.h. It compiles those tables for the target, reads their bytes out of the object file, replaces their definitions with zero-filled arrays, and compiles a C function that decompresses into them with libzstd. The crate exposes one safe LanguageFn constant per enabled grammar; the first call unpacks that grammar's tables once. Nothing uses the crate yet. AI assistance: written with Claude Code; reviewed by the author. Claude-Session: https://claude.ai/code/session_01KDZ845xr2ngWfAabx9dg8z Agent-Session: a21f1781-9b0d-47f6-a269-e93bb179c266 Agent-Session: dca03de1-4350-4b24-b376-cb54c86fcc2d --- Cargo.lock | 15 + Cargo.toml | 4 +- crates/diffr-grammars/Cargo.toml | 40 +++ crates/diffr-grammars/build.rs | 534 +++++++++++++++++++++++++++++++ crates/diffr-grammars/src/lib.rs | 30 ++ 5 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 crates/diffr-grammars/Cargo.toml create mode 100644 crates/diffr-grammars/build.rs create mode 100644 crates/diffr-grammars/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0ad1f4300..b322fc8ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -776,6 +776,21 @@ dependencies = [ "typed-arena", ] +[[package]] +name = "diffr-grammars" +version = "0.1.10" +dependencies = [ + "cc", + "object", + "rayon", + "serde_json", + "tree-sitter", + "tree-sitter-c", + "tree-sitter-language", + "zstd", + "zstd-sys", +] + [[package]] name = "diffr-plugin-sdk" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index be804a238..1ef866327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,8 +66,8 @@ tokio = { version = "1", features = ["macros", "net", "process"] } version_check = "0.9.4" [workspace] -members = ["xtask", "crates/diffr-core", "crates/diffr-plugin-sdk"] -default-members = [".", "crates/diffr-core", "crates/diffr-plugin-sdk"] +members = ["xtask", "crates/diffr-core", "crates/diffr-grammars", "crates/diffr-plugin-sdk"] +default-members = [".", "crates/diffr-core", "crates/diffr-grammars", "crates/diffr-plugin-sdk"] exclude = ["plugins", "tests/fixtures/concurrent-guest"] [profile.release] diff --git a/crates/diffr-grammars/Cargo.toml b/crates/diffr-grammars/Cargo.toml new file mode 100644 index 000000000..bd2b1e45d --- /dev/null +++ b/crates/diffr-grammars/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "diffr-grammars" +description = "Tree-sitter grammars for diffr, with their largest parse tables stored compressed." +repository = "https://github.com/devdotfast/diffr" +homepage = "https://github.com/devdotfast/diffr" +license = "MIT" +version = "0.1.10" +authors = ["dev.fast"] +edition = "2021" +rust-version = "1.99.0" + +include = [ + "/build.rs", + "/src/", +] + +[dependencies] +tree-sitter-language = "0.1.7" +# The grammars' C code calls ZSTD_decompress. +zstd-sys = { version = "2.1.0", default-features = false } + +[build-dependencies] +cc = "1.2.48" +object = { version = "0.40", default-features = false, features = ["read_core", "coff", "elf", "macho", "std"] } +rayon = "1.7.0" +serde_json = "1.0" +tree-sitter = "0.26.10" +tree-sitter-c = "0.24" +zstd = { version = "0.13.3", default-features = false } + +[features] +lang-apex = [] +lang-fortran = [] +lang-fsharp = [] +lang-haskell = [] +lang-julia = [] +lang-ocaml = [] +lang-qml = [] +lang-verilog = [] +lang-vhdl = [] diff --git a/crates/diffr-grammars/build.rs b/crates/diffr-grammars/build.rs new file mode 100644 index 000000000..0eaa4d884 --- /dev/null +++ b/crates/diffr-grammars/build.rs @@ -0,0 +1,534 @@ +//! Link the pinned grammars with their parse tables compressed. +//! +//! Tree-sitter generates each grammar as one `parser.c`: constant tables plus a +//! lexer. This script packs every table whose elements are plain data. It reads +//! the table's bytes from an object file compiled for the target, stores them +//! as zstd bytes, and leaves an empty array in its place. A generated C +//! function, `diffr_unpack_`, fills the arrays, and `src/lib.rs` runs it +//! once before the grammar's first use. +use object::{Object, ObjectSection, ObjectSymbol}; +use rayon::prelude::*; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::process::Command; + +// Grammar crates: (constructor symbol, crate name, source folder, required feature). +const PACKAGES: &[(&str, &str, &str, Option<&str>)] = &[ + ("tree_sitter_go", "tree-sitter-go", "src", None), + ( + "tree_sitter_javascript", + "tree-sitter-javascript", + "src", + None, + ), + ("tree_sitter_python", "tree-sitter-python", "src", None), + ( + "tree_sitter_rust_orchard", + "tree-sitter-rust-orchard", + "src", + None, + ), + ( + "tree_sitter_typescript", + "tree-sitter-typescript", + "typescript/src", + None, + ), + ("tree_sitter_tsx", "tree-sitter-typescript", "tsx/src", None), + ("tree_sitter_ada", "tree-sitter-ada", "src", None), + ( + "tree_sitter_apex", + "tree-sitter-sfapex", + "apex/src", + Some("lang-apex"), + ), + ("tree_sitter_asm", "tree-sitter-asm", "src", None), + ("tree_sitter_bash", "tree-sitter-bash", "src", None), + ("tree_sitter_c", "tree-sitter-c", "src", None), + ("tree_sitter_c_sharp", "tree-sitter-c-sharp", "src", None), + ( + "tree_sitter_clojure_orchard", + "tree-sitter-clojure-orchard", + "src", + None, + ), + ("tree_sitter_cmake", "tree-sitter-cmake", "src", None), + ( + "tree_sitter_commonlisp", + "tree-sitter-commonlisp", + "src", + None, + ), + ( + "tree_sitter_containerfile", + "tree-sitter-containerfile", + "src", + None, + ), + ("tree_sitter_cpp", "tree-sitter-cpp", "src", None), + ("tree_sitter_css", "tree-sitter-css", "src", None), + ( + "tree_sitter_dart_orchard", + "tree-sitter-dart-orchard", + "src", + None, + ), + ( + "tree_sitter_devicetree", + "tree-sitter-devicetree", + "src", + None, + ), + ("tree_sitter_elisp", "tree-sitter-elisp", "src", None), + ("tree_sitter_elixir", "tree-sitter-elixir", "src", None), + ("tree_sitter_elm", "tree-sitter-elm", "src", None), + ("tree_sitter_erlang", "tree-sitter-erlang", "src", None), + ("tree_sitter_fish", "tree-sitter-fish", "src", None), + ( + "tree_sitter_fortran", + "tree-sitter-fortran", + "src", + Some("lang-fortran"), + ), + ( + "tree_sitter_fsharp", + "tree-sitter-fsharp", + "fsharp/src", + Some("lang-fsharp"), + ), + ("tree_sitter_gleam", "tree-sitter-gleam", "src", None), + ( + "tree_sitter_haskell", + "tree-sitter-haskell", + "src", + Some("lang-haskell"), + ), + ("tree_sitter_hcl", "tree-sitter-hcl", "src", None), + ("tree_sitter_html", "tree-sitter-html", "src", None), + ( + "tree_sitter_java_orchard", + "tree-sitter-java-orchard", + "src", + None, + ), + ("tree_sitter_json", "tree-sitter-json", "src", None), + ( + "tree_sitter_julia", + "tree-sitter-julia", + "src", + Some("lang-julia"), + ), + ("tree_sitter_lua", "tree-sitter-lua", "src", None), + ("tree_sitter_make", "tree-sitter-make", "src", None), + ("tree_sitter_newick", "tree-sitter-newick", "src", None), + ("tree_sitter_nix", "tree-sitter-nix", "src", None), + ("tree_sitter_objc", "tree-sitter-objc", "src", None), + ( + "tree_sitter_ocaml", + "tree-sitter-ocaml", + "grammars/ocaml/src", + Some("lang-ocaml"), + ), + ( + "tree_sitter_ocaml_interface", + "tree-sitter-ocaml", + "grammars/interface/src", + Some("lang-ocaml"), + ), + ("tree_sitter_pascal", "tree-sitter-pascal", "src", None), + ("tree_sitter_perl", "ts-parser-perl", "src", None), + ("tree_sitter_php", "tree-sitter-php", "php/src", None), + ("tree_sitter_proto", "tree-sitter-proto", "src", None), + ( + "tree_sitter_qmljs", + "tree-sitter-qmljs", + "src", + Some("lang-qml"), + ), + ("tree_sitter_r", "tree-sitter-r", "src", None), + ("tree_sitter_racket", "tree-sitter-racket", "src", None), + ("tree_sitter_ruby", "tree-sitter-ruby", "src", None), + ("tree_sitter_scala", "tree-sitter-scala", "src", None), + ("tree_sitter_scheme", "tree-sitter-scheme", "src", None), + ("tree_sitter_solidity", "tree-sitter-solidity", "src", None), + ("tree_sitter_sql", "tree-sitter-sequel", "src", None), + ("tree_sitter_swift", "tree-sitter-swift", "src", None), + ("tree_sitter_toml", "tree-sitter-toml-ng", "src", None), + ( + "tree_sitter_verilog", + "tree-sitter-verilog", + "src", + Some("lang-verilog"), + ), + ( + "tree_sitter_vhdl", + "tree-sitter-vhdl", + "src", + Some("lang-vhdl"), + ), + ("tree_sitter_xml", "tree-sitter-xml", "xml/src", None), + ("tree_sitter_yaml", "tree-sitter-yaml", "src", None), + ("tree_sitter_zig", "tree-sitter-zig", "src", None), +]; + +// Grammars in vendored_parsers: (constructor symbol, folder). +const VENDORED: &[(&str, &str)] = &[ + ("tree_sitter_janet_simple", "tree-sitter-janet-simple-src"), + ("tree_sitter_kotlin", "tree-sitter-kotlin-src"), + ("tree_sitter_latex", "tree-sitter-latex-src"), + ("tree_sitter_smali", "tree-sitter-smali-src"), +]; + +fn main() { + let roots = package_roots(); + let mut sources: Vec<(&str, PathBuf)> = PACKAGES + .iter() + .filter(|(_, _, _, feature)| { + feature.is_none_or(|feature| { + let name = feature.replace('-', "_").to_uppercase(); + std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some() + }) + }) + .map(|&(symbol, package, folder, _)| (symbol, roots[package].join(folder))) + .collect(); + sources.extend( + VENDORED + .iter() + .map(|&(symbol, folder)| (symbol, Path::new("../../vendored_parsers").join(folder))), + ); + for (_, source) in &sources { + println!("cargo:rerun-if-changed={}", source.display()); + } + let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + sources + .par_iter() + .for_each(|(symbol, source)| pack(symbol, source, &out.join(symbol))); + let mut grammars = String::new(); + for (symbol, _) in &sources { + let name = symbol.strip_prefix("tree_sitter_").unwrap().to_uppercase(); + writeln!( + grammars, + "grammar!({name}, diffr_unpack_{symbol}, diffr_{symbol});" + ) + .unwrap(); + } + std::fs::write(out.join("grammars.rs"), grammars).unwrap(); +} + +/// Map each locked package name to its source directory, including vendored registries. +fn package_roots() -> BTreeMap { + let metadata = Command::new(std::env::var_os("CARGO").unwrap()) + .args([ + "metadata", + "--locked", + "--all-features", + "--format-version", + "1", + ]) + .output() + .expect("reading Cargo dependency metadata"); + assert!( + metadata.status.success(), + "{}", + String::from_utf8_lossy(&metadata.stderr) + ); + let metadata: serde_json::Value = serde_json::from_slice(&metadata.stdout).unwrap(); + metadata["packages"] + .as_array() + .unwrap() + .iter() + .map(|package| { + let manifest = Path::new(package["manifest_path"].as_str().unwrap()); + ( + package["name"].as_str().unwrap().to_owned(), + manifest.parent().unwrap().to_owned(), + ) + }) + .collect() +} + +/// Write `/grammar.c` with packed tables, the `diffr_unpack_` function that +/// fills them, and the grammar's constructor renamed to `diffr_`, then compile it. +fn pack(symbol: &str, source: &Path, out: &Path) { + std::fs::create_dir_all(out).unwrap(); + let parser = std::fs::read_to_string(source.join("parser.c")).unwrap(); + let header = std::fs::read_to_string(source.join("tree_sitter/parser.h")).unwrap(); + let tables = tables(&parser, &header); + let bytes = table_bytes(symbol, source, out, &parser, &tables); + + let mut grammar = format!( + "#include \n\ + size_t ZSTD_decompress(void *dst, size_t dst_capacity, const void *src, size_t src_size);\n\ + #define {symbol} diffr_{symbol}\n" + ); + grammar.push_str(&empty_tables(&parser, &tables, &bytes)); + for (table, bytes) in tables.iter().zip(&bytes) { + let packed = zstd::bulk::compress(bytes, 19).unwrap(); + write!( + grammar, + "\nstatic const unsigned char packed_{}[] = {{", + table.name + ) + .unwrap(); + for byte in packed { + write!(grammar, "{byte},").unwrap(); + } + grammar.push_str("};\n"); + } + grammar.push_str( + "\nstatic void unpack(void *table, size_t size, const unsigned char *packed, size_t packed_size) {\n\ + \x20 if (ZSTD_decompress(table, size, packed, packed_size) != size) abort();\n\ + }\n", + ); + writeln!(grammar, "\nvoid diffr_unpack_{symbol}(void) {{").unwrap(); + for Table { name, .. } in &tables { + writeln!( + grammar, + " unpack({name}, sizeof {name}, packed_{name}, sizeof packed_{name});" + ) + .unwrap(); + } + grammar.push_str("}\n"); + std::fs::write(out.join("grammar.c"), grammar).unwrap(); + + let mut build = compiler(source, out); + build.file(out.join("grammar.c")); + if source.join("scanner.c").is_file() { + build.file(source.join("scanner.c")); + } + build.compile(symbol); +} + +/// Compile the tables for the target and read each one's bytes out of the object file. +fn table_bytes( + symbol: &str, + source: &Path, + out: &Path, + parser: &str, + tables: &[Table], +) -> Vec> { + // Global symbols can be found by name. Mach-O and COFF symbols carry no + // size, so each table's size is compiled in beside it. + let mut globals = parser.to_owned(); + for storage in tables + .iter() + .rev() + .filter_map(|table| table.storage.clone()) + { + globals.replace_range(storage, ""); + } + for Table { name, .. } in tables { + writeln!( + globals, + "const unsigned long long diffr_size_{name} = sizeof {name};" + ) + .unwrap(); + } + let tables_c = out.join("tables.c"); + std::fs::write(&tables_c, globals).unwrap(); + let [object_path] = <[PathBuf; 1]>::try_from( + compiler(source, out) + .file(&tables_c) + .opt_level(0) + .compile_intermediates(), + ) + .unwrap(); + + let data = std::fs::read(object_path).unwrap(); + let file = object::File::parse(&*data).unwrap(); + let prefix = if file.format() == object::BinaryFormat::MachO { + "_" + } else { + "" + }; + let read = |name: &str, size: usize| -> Vec { + let symbol = file + .symbol_by_name(&format!("{prefix}{name}")) + .unwrap_or_else(|| panic!("{name} is missing from the {symbol} tables object")); + let section = file + .section_by_index(symbol.section_index().unwrap()) + .unwrap(); + if section.kind() == object::SectionKind::UninitializedData { + return vec![0; size]; + } + let start = usize::try_from(symbol.address() - section.address()).unwrap(); + section.data().unwrap()[start..start + size].to_vec() + }; + tables + .iter() + .map(|Table { name, .. }| { + let size: [u8; 8] = read(&format!("diffr_size_{name}"), 8).try_into().unwrap(); + let size = if file.is_little_endian() { + u64::from_le_bytes(size) + } else { + u64::from_be_bytes(size) + }; + read(name, usize::try_from(size).unwrap()) + }) + .collect() +} + +fn compiler(source: &Path, out: &Path) -> cc::Build { + let mut build = cc::Build::new(); + build.include(source).warnings(false).out_dir(out); + if std::env::var("CARGO_CFG_TARGET_ENV").unwrap() == "msvc" { + build.flag("/utf-8"); + } + build +} + +/// A top-level array definition in `parser.c`, `T name[...] = {...};`, whose +/// elements are plain data. +struct Table<'a> { + name: &'a str, + element: &'a str, + /// The array declarator when it states its size; `None` for `name[]`. + sized_declarator: Option<&'a str>, + /// The `static` keyword, if any. + storage: Option>, + definition: Range, +} + +/// Find the tables to pack: top-level arrays whose element type is plain data. +fn tables<'a>(parser: &'a str, header: &str) -> Vec> { + let mut c = tree_sitter::Parser::new(); + c.set_language(&tree_sitter_c::LANGUAGE.into()).unwrap(); + let header_tree = c.parse(header, None).unwrap(); + let mut typedefs = BTreeMap::new(); + let mut nodes = vec![header_tree.root_node()]; + while let Some(node) = nodes.pop() { + let mut cursor = node.walk(); + if node.kind() != "type_definition" { + nodes.extend(node.children(&mut cursor)); + continue; + } + let ty = node.child_by_field_name("type").unwrap(); + // A typedef declared through a pointer is left out, so a table of it fails to resolve. + for name in node + .children_by_field_name("declarator", &mut cursor) + .filter(|declarator| declarator.kind() == "type_identifier") + { + typedefs.insert(&header[name.byte_range()], ty); + } + } + + let tree = c.parse(parser, None).unwrap(); + let text = |node: tree_sitter::Node| &parser[node.byte_range()]; + let mut cursor = tree.walk(); + tree.root_node() + .children(&mut cursor) + .filter_map(|definition| { + let init = definition + .child_by_field_name("declarator") + .filter(|init| { + definition.kind() == "declaration" && init.kind() == "init_declarator" + })?; + init.child_by_field_name("value") + .filter(|value| value.kind() == "initializer_list")?; + // An array of pointers, such as `ts_symbol_names`, has a pointer declarator here. + let array = init + .child_by_field_name("declarator") + .filter(|array| array.kind() == "array_declarator")?; + let mut name = array; + while name.kind() == "array_declarator" { + name = name.child_by_field_name("declarator").unwrap(); + } + let ty = definition.child_by_field_name("type").unwrap(); + let mut cursor = definition.walk(); + let storage = definition + .children(&mut cursor) + .find(|child| child.kind() == "storage_class_specifier" && text(*child) == "static") + .map(|keyword| keyword.byte_range()); + plain_data(ty, parser, header, &typedefs).then(|| Table { + name: text(name), + element: text(ty), + sized_declarator: array.child_by_field_name("size").map(|_| text(array)), + storage, + definition: definition.byte_range(), + }) + }) + .collect() +} + +/// Whether values of the C type `ty` are plain data: integers, `bool`, enums, +/// and structs or unions of them, with no pointers. Panics on a type it cannot +/// resolve, so an unknown type never gets packed by accident. +fn plain_data( + ty: tree_sitter::Node, + source: &str, + header: &str, + typedefs: &BTreeMap<&str, tree_sitter::Node>, +) -> bool { + match ty.kind() { + "primitive_type" | "sized_type_specifier" | "enum_specifier" => true, + "type_identifier" => { + let name = &source[ty.byte_range()]; + let ty = typedefs + .get(name) + .unwrap_or_else(|| panic!("cannot resolve the C type {name}")); + plain_data(*ty, header, header, typedefs) + } + "struct_specifier" | "union_specifier" => { + let body = ty.child_by_field_name("body").unwrap_or_else(|| { + panic!("cannot resolve the C type {}", &source[ty.byte_range()]) + }); + let mut cursor = body.walk(); + let plain = body + .named_children(&mut cursor) + .filter(|field| field.kind() != "comment") + .all(|field| { + assert_eq!( + field.kind(), + "field_declaration", + "unexpected struct member" + ); + let mut cursor = field.walk(); + let plain_declarators = field + .children_by_field_name("declarator", &mut cursor) + .all(|mut declarator| { + while declarator.kind() == "array_declarator" { + declarator = declarator.child_by_field_name("declarator").unwrap(); + } + declarator.kind() == "field_identifier" + }); + plain_declarators + && plain_data( + field.child_by_field_name("type").unwrap(), + source, + header, + typedefs, + ) + }); + plain + } + kind => panic!( + "cannot resolve the C type {} ({kind})", + &source[ty.byte_range()] + ), + } +} + +/// Replace each table's definition with an empty, writable array of the same size. +fn empty_tables(parser: &str, tables: &[Table], bytes: &[Vec]) -> String { + let mut emptied = parser.to_owned(); + for (table, bytes) in tables.iter().zip(bytes).rev() { + let declarator = table.sized_declarator.map_or_else( + || { + format!( + "{}[{} / sizeof({})]", + table.name, + bytes.len(), + table.element + ) + }, + str::to_owned, + ); + emptied.replace_range( + table.definition.clone(), + &format!("static {} {declarator};", table.element), + ); + } + emptied +} diff --git a/crates/diffr-grammars/src/lib.rs b/crates/diffr-grammars/src/lib.rs new file mode 100644 index 000000000..4d8748793 --- /dev/null +++ b/crates/diffr-grammars/src/lib.rs @@ -0,0 +1,30 @@ +//! Tree-sitter grammars with their largest parse tables stored compressed. +//! +//! Each constant is a grammar constructor, like a grammar crate's `LANGUAGE`. +//! The first call unpacks that grammar's tables. +use std::sync::Once; +use tree_sitter_language::LanguageFn; +// Links libzstd for the grammars' C code. +use zstd_sys as _; + +macro_rules! grammar { + ($name:ident, $unpack:ident, $constructor:ident) => { + pub const $name: LanguageFn = { + unsafe extern "C" fn language() -> *const () { + extern "C" { + fn $unpack(); + fn $constructor() -> *const (); + } + static UNPACKED: Once = Once::new(); + // Safety: the unpack function fills this grammar's tables, and + // `Once` runs it to completion before any caller reads them. + UNPACKED.call_once(|| unsafe { $unpack() }); + unsafe { $constructor() } + } + // Safety: `language` returns the grammar's `TSLanguage` pointer. + unsafe { LanguageFn::from_raw(language) } + }; + }; +} + +include!(concat!(env!("OUT_DIR"), "/grammars.rs")); From ceb27ea063b238b5dec3ba8855499a5c47dd510d Mon Sep 17 00:00:00 2001 From: Sidharth Menon Date: Sat, 3 Oct 2026 22:32:33 -0700 Subject: [PATCH 2/2] Build every grammar from diffr-grammars diffr-core now takes each Tree-sitter language from diffr-grammars instead of the grammar crates, which it keeps only for their query files. The vendored grammars' separate static build goes away. Release binaries on aarch64 macOS drop from 108.07 MB to 58.00 MB; startup-bound commands are within 3 ms of main, and parse trees for every file in sample_files are unchanged. AI assistance: written with Claude Code; reviewed by the author. Claude-Session: https://claude.ai/code/session_01KDZ845xr2ngWfAabx9dg8z Agent-Session: a21f1781-9b0d-47f6-a269-e93bb179c266 Agent-Session: dca03de1-4350-4b24-b376-cb54c86fcc2d --- Cargo.lock | 3 +- crates/diffr-core/Cargo.toml | 23 ++- crates/diffr-core/build.rs | 65 +-------- .../src/parse/tree_sitter_parser.rs | 135 +++++++++--------- 4 files changed, 76 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b322fc8ab..c83292541 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -688,7 +688,7 @@ version = "0.1.10" dependencies = [ "anyhow", "bumpalo", - "cc", + "diffr-grammars", "dirs", "glob", "hashbrown 0.17.1", @@ -700,7 +700,6 @@ dependencies = [ "log", "pretty_assertions", "radix-heap", - "rayon", "regex", "rustc-hash", "schemars", diff --git a/crates/diffr-core/Cargo.toml b/crates/diffr-core/Cargo.toml index 6d3f88226..64409d2ff 100644 --- a/crates/diffr-core/Cargo.toml +++ b/crates/diffr-core/Cargo.toml @@ -17,6 +17,7 @@ include = [ ] [dependencies] +diffr-grammars = { path = "../diffr-grammars", version = "0.1.10" } regex = "1.10.4" typed-arena = "2.0.2" rustc-hash = "2.0.0" @@ -119,21 +120,17 @@ tempfile = "3.27.0" [build-dependencies] toml = "0.8" -# TODO: enable parallel mode, see discussion in -# https://github.com/rust-lang/cc-rs/pull/849 -cc = "1.2.48" -rayon = "1.7.0" [features] all-languages = ["lang-fortran", "lang-fsharp", "lang-verilog", "lang-ocaml", "lang-julia", "lang-haskell", "lang-vhdl", "lang-apex", "lang-qml"] -lang-fortran = ["dep:tree-sitter-fortran"] -lang-fsharp = ["dep:tree-sitter-fsharp"] -lang-verilog = ["dep:tree-sitter-verilog"] -lang-ocaml = ["dep:tree-sitter-ocaml"] -lang-julia = ["dep:tree-sitter-julia"] -lang-haskell = ["dep:tree-sitter-haskell"] -lang-vhdl = ["dep:tree-sitter-vhdl"] -lang-apex = ["dep:tree-sitter-sfapex"] -lang-qml = ["dep:tree-sitter-qmljs"] +lang-fortran = ["dep:tree-sitter-fortran", "diffr-grammars/lang-fortran"] +lang-fsharp = ["dep:tree-sitter-fsharp", "diffr-grammars/lang-fsharp"] +lang-verilog = ["dep:tree-sitter-verilog", "diffr-grammars/lang-verilog"] +lang-ocaml = ["dep:tree-sitter-ocaml", "diffr-grammars/lang-ocaml"] +lang-julia = ["dep:tree-sitter-julia", "diffr-grammars/lang-julia"] +lang-haskell = ["dep:tree-sitter-haskell", "diffr-grammars/lang-haskell"] +lang-vhdl = ["dep:tree-sitter-vhdl", "diffr-grammars/lang-vhdl"] +lang-apex = ["dep:tree-sitter-sfapex", "diffr-grammars/lang-apex"] +lang-qml = ["dep:tree-sitter-qmljs", "diffr-grammars/lang-qml"] # Constructors and helpers the diffr package's tests build on. test-support = [] diff --git a/crates/diffr-core/build.rs b/crates/diffr-core/build.rs index ef30f8348..a0ba3a666 100644 --- a/crates/diffr-core/build.rs +++ b/crates/diffr-core/build.rs @@ -8,72 +8,9 @@ use std::path::PathBuf; -use rayon::prelude::*; - -struct TreeSitterParser { - name: &'static str, - src_dir: &'static str, - extra_files: Vec<&'static str>, -} - -impl TreeSitterParser { - fn build(&self) { - let dir = PathBuf::from(&self.src_dir); - - let mut c_files = vec!["parser.c"]; - c_files.extend_from_slice(&self.extra_files); - - let mut build = cc::Build::new(); - if cfg!(target_env = "msvc") { - build.flag("/utf-8"); - } - build.include(&dir).warnings(false); // ignore unused parameter warnings - for file in c_files { - build.file(dir.join(file)); - } - - build.link_lib_modifier("+whole-archive"); - - build.compile(self.name); - } -} - -fn main() { - bundled_plugins(); - let parsers = vec![ - TreeSitterParser { - name: "tree-sitter-janet-simple", - src_dir: "../../vendored_parsers/tree-sitter-janet-simple-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-kotlin", - src_dir: "../../vendored_parsers/tree-sitter-kotlin-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-latex", - src_dir: "../../vendored_parsers/tree-sitter-latex-src", - extra_files: vec!["scanner.c"], - }, - TreeSitterParser { - name: "tree-sitter-smali", - src_dir: "../../vendored_parsers/tree-sitter-smali-src", - extra_files: vec!["scanner.c"], - }, - ]; - - // Only rerun if relevant files in the vendored_parsers/ directory change. - for parser in &parsers { - println!("cargo:rerun-if-changed={}", parser.src_dir); - } - - parsers.par_iter().for_each(|p| p.build()); -} - /// Embed the component, manifest and queries from every bundled shape plugin /// folder under `plugins/shape/`, and the classifier in `plugins/classify/`. -fn bundled_plugins() { +fn main() { let root = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("../.."); println!("cargo:rerun-if-changed={}", root.join("plugins").display()); let mut files = String::from("const FILES: &[(&str, &str)] = &[\n"); diff --git a/crates/diffr-core/src/parse/tree_sitter_parser.rs b/crates/diffr-core/src/parse/tree_sitter_parser.rs index a3a7055d4..b78aa4811 100644 --- a/crates/diffr-core/src/parse/tree_sitter_parser.rs +++ b/crates/diffr-core/src/parse/tree_sitter_parser.rs @@ -84,13 +84,6 @@ pub struct TreeSitterConfig { pub(crate) sub_languages: Vec, } -extern "C" { - fn tree_sitter_janet_simple() -> ts::Language; - fn tree_sitter_kotlin() -> ts::Language; - fn tree_sitter_latex() -> ts::Language; - fn tree_sitter_smali() -> ts::Language; -} - // TODO: begin/end and object/end. #[cfg(feature = "lang-ocaml")] const OCAML_ATOM_NODES: [&str; 6] = [ @@ -122,7 +115,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { use guess::Language::*; match language { Ada => { - let language_fn = tree_sitter_ada::LANGUAGE; + let language_fn = diffr_grammars::ADA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -141,7 +134,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-apex")] Apex => { - let language_fn = tree_sitter_sfapex::apex::LANGUAGE; + let language_fn = diffr_grammars::APEX; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -168,7 +161,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Asm => { - let language_fn = tree_sitter_asm::LANGUAGE; + let language_fn = diffr_grammars::ASM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -182,7 +175,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Bash => { - let language_fn = tree_sitter_bash::LANGUAGE; + let language_fn = diffr_grammars::BASH; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -198,7 +191,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } C => { - let language_fn = tree_sitter_c::LANGUAGE; + let language_fn = diffr_grammars::C; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -210,7 +203,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CPlusPlus => { - let language_fn = tree_sitter_cpp::LANGUAGE; + let language_fn = diffr_grammars::CPP; let language = tree_sitter::Language::new(language_fn); let mut highlight_query = tree_sitter_c::HIGHLIGHT_QUERY.to_owned(); @@ -230,7 +223,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Clojure => { - let language_fn = tree_sitter_clojure_orchard::LANGUAGE; + let language_fn = diffr_grammars::CLOJURE_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -249,7 +242,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CMake => { - let language_fn = tree_sitter_cmake::LANGUAGE; + let language_fn = diffr_grammars::CMAKE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -265,7 +258,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CommonLisp => { - let language_fn = tree_sitter_commonlisp::LANGUAGE_COMMONLISP; + let language_fn = diffr_grammars::COMMONLISP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -278,7 +271,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } CSharp => { - let language_fn = tree_sitter_c_sharp::LANGUAGE; + let language_fn = diffr_grammars::C_SHARP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -301,7 +294,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Css => { - let language_fn = tree_sitter_css::LANGUAGE; + let language_fn = diffr_grammars::CSS; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -322,7 +315,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Dart => { - let language_fn = tree_sitter_dart_orchard::LANGUAGE; + let language_fn = diffr_grammars::DART_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -338,7 +331,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } DeviceTree => { - let language_fn = tree_sitter_devicetree::LANGUAGE; + let language_fn = diffr_grammars::DEVICETREE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -356,7 +349,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Dockerfile => { - let language_fn = tree_sitter_containerfile::LANGUAGE; + let language_fn = diffr_grammars::CONTAINERFILE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -383,7 +376,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Elixir => { - let language_fn = tree_sitter_elixir::LANGUAGE; + let language_fn = diffr_grammars::ELIXIR; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -399,7 +392,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Elm => { - let language_fn = tree_sitter_elm::LANGUAGE; + let language_fn = diffr_grammars::ELM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -413,7 +406,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } EmacsLisp => { - let language_fn = tree_sitter_elisp::LANGUAGE; + let language_fn = diffr_grammars::ELISP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -429,7 +422,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Erlang => { - let language_fn = tree_sitter_erlang::LANGUAGE; + let language_fn = diffr_grammars::ERLANG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -443,7 +436,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Fish => { - let language = tree_sitter_fish::language(); + let language = tree_sitter::Language::new(diffr_grammars::FISH); let highlight_query = ts::Query::new(&language, tree_sitter_fish::HIGHLIGHTS_QUERY).unwrap(); @@ -460,7 +453,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-fsharp")] FSharp => { - let language_fn = tree_sitter_fsharp::LANGUAGE_FSHARP; + let language_fn = diffr_grammars::FSHARP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -476,7 +469,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-fortran")] Fortran => { - let language_fn = tree_sitter_fortran::LANGUAGE; + let language_fn = diffr_grammars::FORTRAN; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -489,7 +482,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Gleam => { - let language_fn = tree_sitter_gleam::LANGUAGE; + let language_fn = diffr_grammars::GLEAM; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -502,7 +495,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Go => { - let language_fn = tree_sitter_go::LANGUAGE; + let language_fn = diffr_grammars::GO; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -521,7 +514,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-haskell")] Haskell => { - let language_fn = tree_sitter_haskell::LANGUAGE; + let language_fn = diffr_grammars::HASKELL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -534,7 +527,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Hcl => { - let language_fn = tree_sitter_hcl::LANGUAGE; + let language_fn = diffr_grammars::HCL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -557,7 +550,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Html => { - let language_fn = tree_sitter_html::LANGUAGE; + let language_fn = diffr_grammars::HTML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -592,7 +585,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Janet => { - let language = unsafe { tree_sitter_janet_simple() }; + let language = tree_sitter::Language::new(diffr_grammars::JANET_SIMPLE); TreeSitterConfig { language: language.clone(), atom_nodes: [].into_iter().collect(), @@ -616,7 +609,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Java => { - let language_fn = tree_sitter_java_orchard::LANGUAGE; + let language_fn = diffr_grammars::JAVA_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -649,7 +642,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } JavaScript | JavascriptJsx => { - let language_fn = tree_sitter_javascript::LANGUAGE; + let language_fn = diffr_grammars::JAVASCRIPT; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -679,7 +672,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Json => { - let language_fn = tree_sitter_json::LANGUAGE; + let language_fn = diffr_grammars::JSON; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -694,7 +687,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-julia")] Julia => { - let language_fn = tree_sitter_julia::LANGUAGE; + let language_fn = diffr_grammars::JULIA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -718,7 +711,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Kotlin => { - let language = unsafe { tree_sitter_kotlin() }; + let language = tree_sitter::Language::new(diffr_grammars::KOTLIN); TreeSitterConfig { language: language.clone(), // Flattening nullable type means we can't diff the @@ -746,7 +739,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } LaTeX => { - let language = unsafe { tree_sitter_latex() }; + let language = tree_sitter::Language::new(diffr_grammars::LATEX); TreeSitterConfig { language: language.clone(), atom_nodes: [].into_iter().collect(), @@ -761,7 +754,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Lua => { - let language_fn = tree_sitter_lua::LANGUAGE; + let language_fn = diffr_grammars::LUA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -777,7 +770,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Make => { - let language_fn = tree_sitter_make::LANGUAGE; + let language_fn = diffr_grammars::MAKE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -795,7 +788,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Newick => { - let language_fn = tree_sitter_newick::LANGUAGE; + let language_fn = diffr_grammars::NEWICK; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -812,7 +805,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Nix => { - let language_fn = tree_sitter_nix::LANGUAGE; + let language_fn = diffr_grammars::NIX; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -828,7 +821,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } ObjC => { - let language_fn = tree_sitter_objc::LANGUAGE; + let language_fn = diffr_grammars::OBJC; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -849,7 +842,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-ocaml")] OCaml => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML; + let language_fn = diffr_grammars::OCAML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -863,7 +856,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-ocaml")] OCamlInterface => { - let language_fn = tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE; + let language_fn = diffr_grammars::OCAML_INTERFACE; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -876,7 +869,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Pascal => { - let language_fn = tree_sitter_pascal::LANGUAGE; + let language_fn = diffr_grammars::PASCAL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -893,7 +886,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Perl => { - let language_fn = ts_parser_perl::LANGUAGE; + let language_fn = diffr_grammars::PERL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -917,7 +910,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Php => { - let language_fn = tree_sitter_php::LANGUAGE_PHP; + let language_fn = diffr_grammars::PHP; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -931,7 +924,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Proto => { - let language_fn = tree_sitter_proto::LANGUAGE; + let language_fn = diffr_grammars::PROTO; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -947,7 +940,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Python => { - let language_fn = tree_sitter_python::LANGUAGE; + let language_fn = diffr_grammars::PYTHON; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -969,7 +962,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-qml")] Qml => { - let language_fn = tree_sitter_qmljs::LANGUAGE; + let language_fn = diffr_grammars::QMLJS; let language = tree_sitter::Language::new(language_fn); let mut highlight_query = tree_sitter_javascript::HIGHLIGHT_QUERY.to_owned(); @@ -986,7 +979,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } R => { - let language_fn = tree_sitter_r::LANGUAGE; + let language_fn = diffr_grammars::R; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -999,7 +992,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Racket => { - let language_fn = tree_sitter_racket::LANGUAGE; + let language_fn = diffr_grammars::RACKET; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1014,7 +1007,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Ruby => { - let language_fn = tree_sitter_ruby::LANGUAGE; + let language_fn = diffr_grammars::RUBY; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1035,7 +1028,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Rust => { - let language_fn = tree_sitter_rust_orchard::LANGUAGE; + let language_fn = diffr_grammars::RUST_ORCHARD; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1068,7 +1061,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Scala => { - let language_fn = tree_sitter_scala::LANGUAGE; + let language_fn = diffr_grammars::SCALA; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1094,7 +1087,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Scheme => { - let language_fn = tree_sitter_scheme::LANGUAGE; + let language_fn = diffr_grammars::SCHEME; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1107,7 +1100,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Smali => { - let language = unsafe { tree_sitter_smali() }; + let language = tree_sitter::Language::new(diffr_grammars::SMALI); TreeSitterConfig { language: language.clone(), atom_nodes: ["string"].into_iter().collect(), @@ -1122,7 +1115,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Solidity => { - let language_fn = tree_sitter_solidity::LANGUAGE; + let language_fn = diffr_grammars::SOLIDITY; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1137,7 +1130,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Sql => { - let language_fn = tree_sitter_sequel::LANGUAGE; + let language_fn = diffr_grammars::SQL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { language: language.clone(), @@ -1150,7 +1143,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Swift => { - let language_fn = tree_sitter_swift::LANGUAGE; + let language_fn = diffr_grammars::SWIFT; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1168,7 +1161,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Toml => { - let language_fn = tree_sitter_toml_ng::LANGUAGE; + let language_fn = diffr_grammars::TOML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1189,9 +1182,9 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } TypeScript | TypeScriptTsx => { let language_fn = if language == TypeScript { - tree_sitter_typescript::LANGUAGE_TYPESCRIPT + diffr_grammars::TYPESCRIPT } else { - tree_sitter_typescript::LANGUAGE_TSX + diffr_grammars::TSX }; let language = tree_sitter::Language::new(language_fn); @@ -1218,7 +1211,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Xml => { - let language_fn = tree_sitter_xml::LANGUAGE_XML; + let language_fn = diffr_grammars::XML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1235,7 +1228,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Yaml => { - let language_fn = tree_sitter_yaml::LANGUAGE; + let language_fn = diffr_grammars::YAML; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1257,7 +1250,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-verilog")] Verilog => { - let language_fn = tree_sitter_verilog::LANGUAGE; + let language_fn = diffr_grammars::VERILOG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { ignore_trailing_tokens: vec![], @@ -1274,7 +1267,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } #[cfg(feature = "lang-vhdl")] Vhdl => { - let language_fn = tree_sitter_vhdl::LANGUAGE; + let language_fn = diffr_grammars::VHDL; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig { @@ -1288,7 +1281,7 @@ fn build_config(language: guess::Language) -> TreeSitterConfig { } } Zig => { - let language_fn = tree_sitter_zig::LANGUAGE; + let language_fn = diffr_grammars::ZIG; let language = tree_sitter::Language::new(language_fn); TreeSitterConfig {