diff --git a/crates/diffr-core/src/config.rs b/crates/diffr-core/src/config.rs index 103b92260..be31e3996 100644 --- a/crates/diffr-core/src/config.rs +++ b/crates/diffr-core/src/config.rs @@ -458,6 +458,14 @@ mod tests { for (query, message) in [ ("(not_a_rust_node) @fold", "NodeType error"), ("(block) @typo", "unsupported capture @typo"), + ( + r#"(block "{" @fold.open "}" @fold.close) @fold"#, + "@fold.open needs exactly one @fold.indent", + ), + ( + r#"[(block "{" @fold.open . (_) @fold.indent "}" @fold.close) (match_block)] @fold"#, + "@fold.open must be in every branch", + ), ( r#"((block) @fold (#set! tag "body"))"#, "must be written :", @@ -519,7 +527,7 @@ mod query_tests { fn arbitrary_tags_and_delimiter_captures_reach_the_domain() { let params = with_queries(&[( "rust", - r#"((block "{" @fold.open "}" @fold.close) @fold (#set! tag "removed-runs:user.validation"))"#, + r#"((block "{" @fold.open . (_) @fold.indent "}" @fold.close) @fold (#set! tag "removed-runs:user.validation"))"#, )]); let source = "fn f() { println!(\"☕\"); }"; let diff = DiffResult::from_sources_with_params("a.rs", "", source, ¶ms); @@ -611,7 +619,7 @@ mod tag_tests { #[test] fn an_opening_capture_alone_folds_to_the_end_of_the_fold_node() { - let query = r#"((function_definition ":" @fold.open body: (block) @fold) (#set! tag "removed-runs:body"))"#; + let query = r#"((function_definition ":" @fold.open body: (block . (_) @fold.indent) @fold) (#set! tag "removed-runs:body"))"#; let params = with_queries(&[("python", query)]); let rhs = "def f(a):\n x = a\n return x\n"; let result = DiffResult::from_sources_with_params("a.py", "", rhs, ¶ms); @@ -629,7 +637,7 @@ mod tag_tests { fn a_node_captured_with_two_ranges_is_a_query_conflict_naming_both_files() { let whole = "((block) @fold (#set! tag \"removed-runs:whole\"))"; let interior = - "((block \"{\" @fold.open \"}\" @fold.close) @fold (#set! tag \"summarize:inside\"))"; + "((block \"{\" @fold.open . (_) @fold.indent \"}\" @fold.close) @fold (#set! tag \"summarize:inside\"))"; let dir = tempfile::tempdir().unwrap(); std::fs::create_dir_all(dir.path().join("queries/rust")).unwrap(); let removed_runs = dir.path().join("queries/rust/removed-runs.scm"); @@ -670,7 +678,7 @@ mod tag_tests { assert!( message.contains("queries/rust/removed-runs.scm and ") && message.contains( - "queries/rust/summarize.scm capture the same block with different fold ranges" + "queries/rust/summarize.scm capture the same block with different fold ranges or indents" ), "{message}" ); @@ -799,8 +807,8 @@ mod format_tests { .to_string() .contains("unsupported config version 2")); let config = - Config::from_toml("version = 1\n[plugins]\norder = ['bundled.group']\n").unwrap(); + Config::from_toml("version = 1\n[plugins]\norder = ['bundled.context']\n").unwrap(); assert_eq!(config.plugins.entries.len(), 1); - assert!(config.plugins.entries.contains_key("bundled.group")); + assert!(config.plugins.entries.contains_key("bundled.context")); } } diff --git a/crates/diffr-core/src/config/README.md b/crates/diffr-core/src/config/README.md index e697fe646..4b11276b7 100644 --- a/crates/diffr-core/src/config/README.md +++ b/crates/diffr-core/src/config/README.md @@ -23,7 +23,7 @@ typescript = "queries/javascript.scm" ```scheme ; inherits: builtin:shared/queries/rust.scm -((function_item body: (block "{" @fold.open "}" @fold.close) @fold) +((function_item body: (block "{" @fold.open . (_) @fold.indent "}" @fold.close) @fold) (#set! tag "deleted-bodies:function")) ``` @@ -46,7 +46,7 @@ Tree-sitter coordinates; labels and comments before the opening brace stay visib With both delimiter captures, they must be ordered and contained in the fold node. A `@fold.open` without `@fold.close` hides from the opening node's end to the end of the fold node; the opening node may precede the fold node, so -`(function_definition ":" @fold.open body: (block) @fold)` folds a Python body +`(function_definition ":" @fold.open body: (block . (_) @fold.indent) @fold)` folds a Python body from its header's `:`. A `@fold.close` without `@fold.open` selects no fold. These fold-boundary captures are our convention. There are no arbitrary byte or diff --git a/crates/diffr-core/src/config/default.toml b/crates/diffr-core/src/config/default.toml index 228984574..b3f300ee7 100644 --- a/crates/diffr-core/src/config/default.toml +++ b/crates/diffr-core/src/config/default.toml @@ -3,12 +3,11 @@ version = 1 [plugins] order = [ - "bundled.context", "bundled.deleted-bodies", "bundled.summarize", "bundled.test-bodies", "bundled.removed-runs", - "bundled.group", + "bundled.context", ] [plugins.bundled.context] @@ -19,9 +18,6 @@ lines = 3 enabled = true min_lines = 12 -[plugins.bundled.group] -enabled = true - [plugins.bundled.removed-runs] enabled = true min_lines = 5 diff --git a/crates/diffr-core/src/config/query.rs b/crates/diffr-core/src/config/query.rs index 85e157e69..d07c5cb41 100644 --- a/crates/diffr-core/src/config/query.rs +++ b/crates/diffr-core/src/config/query.rs @@ -4,7 +4,7 @@ //! order. Every pattern remembers which source it came from, so errors and //! conflicts can name the file that wrote it. use super::ConfigError; -use tree_sitter::Query; +use tree_sitter::{CaptureQuantifier, Query}; pub(crate) struct AnnotationQuery { pub(crate) query: Query, @@ -60,7 +60,7 @@ impl AnnotationQuery { for name in query.capture_names() { if name.starts_with("related.") || name.starts_with('_') - || matches!(*name, "fold" | "fold.open" | "fold.close") + || matches!(*name, "fold" | "fold.open" | "fold.close" | "fold.indent") { continue; } @@ -79,6 +79,30 @@ impl AnnotationQuery { tags: Vec::new(), source: owner(query.start_byte_for_pattern(index)), }; + // Every match with `@fold.open` has exactly one `@fold.indent`. + let quantifier = |name: &str| { + query + .capture_index_for_name(name) + .map_or(CaptureQuantifier::Zero, |capture| { + query.capture_quantifiers(index)[capture as usize] + }) + }; + match (quantifier("fold.open"), quantifier("fold.indent")) { + (CaptureQuantifier::Zero, _) | (CaptureQuantifier::One, CaptureQuantifier::One) => { + } + (CaptureQuantifier::One, _) => { + return Err(named( + index, + "@fold.open needs exactly one @fold.indent".into(), + )) + } + _ => { + return Err(named( + index, + "@fold.open must be in every branch of its pattern".into(), + )) + } + } if !query.property_predicates(index).is_empty() { return Err(named(index, "#is? and #is-not? are not supported".into())); } diff --git a/crates/diffr-core/src/config/store.rs b/crates/diffr-core/src/config/store.rs index e95f1fd9f..9961f1866 100644 --- a/crates/diffr-core/src/config/store.rs +++ b/crates/diffr-core/src/config/store.rs @@ -368,7 +368,7 @@ mod tests { ) .unwrap(); let path = dir.path().join("config.toml"); - let order = "order = ['bundled.context', 'bundled.deleted-bodies', 'bundled.test-bodies', 'bundled.removed-runs', 'bundled.summarize', 'bundled.group', 'external.mine']"; + let order = "order = ['bundled.context', 'bundled.deleted-bodies', 'bundled.test-bodies', 'bundled.removed-runs', 'bundled.summarize', 'external.mine']"; std::fs::write( &path, format!("[plugins]\n{order}\n[plugins.external.mine]\npath = 'plugins/mine'\n"), @@ -673,15 +673,15 @@ mod sparse_tests { let path = dir.path().join("config.toml"); std::fs::write( &path, - "# just grouping\n[plugins]\norder = ['bundled.group']\n", + "# just context\n[plugins]\norder = ['bundled.context']\n", ) .unwrap(); set(&path, "diff.graph_limit", "42").unwrap(); let text = std::fs::read_to_string(&path).unwrap(); let config = Config::from_toml(&text).unwrap(); assert_eq!(config.plugins.entries.len(), 1); - assert!(text.contains("# just grouping\n[plugins]"), "{text}"); - assert!(!text.contains("bundled.context")); + assert!(text.contains("# just context\n[plugins]"), "{text}"); + assert!(!text.contains("bundled.deleted-bodies")); } const LEGACY: &str = "For each listed fold, rewrite that function body as short pseudocode. Keep the names. No prose, no comments, no code fences. Use as few lines as possible: about one pseudocode line per five source lines, and never more than a third of the body's lines. When a fold lists a doc, also set \"summary\" to one sentence copied verbatim from that doc; otherwise leave it empty. Answer with a JSON array of {\"id\", \"summary\", \"pseudocode\"} objects, one per fold."; diff --git a/crates/diffr-core/src/engine.rs b/crates/diffr-core/src/engine.rs index e807ac66a..f53693c46 100644 --- a/crates/diffr-core/src/engine.rs +++ b/crates/diffr-core/src/engine.rs @@ -13,7 +13,7 @@ use crate::parse::folds; use crate::parse::guess_language::{guess, language_name, LanguageOverride}; use crate::parse::syntax::{self, init_next_prev}; use crate::parse::tree_sitter_parser as tsp; -use crate::summary::{DiffResult, FallbackCause, FileContent, FileFormat}; +use crate::summary::{DiffResult, FallbackCause, FileContent, FileFormat, Highlight}; use humansize::{format_size, FormatSizeOptions, BINARY}; use std::{env, fmt, path::Path}; use typed_arena::Arena; @@ -45,7 +45,7 @@ impl fmt::Display for QueryConflict { let (first, second) = &self.conflict.sources; write!( f, - "{}:{}{}: {first} and {second} capture the same {} with different fold ranges", + "{}:{}{}: {first} and {second} capture the same {} with different fold ranges or indents", self.path, self.conflict.line + 1, match self.side { @@ -111,11 +111,31 @@ pub fn diff_file_content( let language = guess(Path::new(display_path), guess_src, overrides); let lang_config = language.map(|lang| (lang, params.language(lang))); + // Highlights come from the same parse as the folds, so a side that + // parsed has them whether or not the match runs. + let highlights = |lhs_tree: &tree_sitter::Tree, rhs_tree: &tree_sitter::Tree, parser| { + if diff_options.syntax { + ( + tsp::highlight_captures(lhs_tree, lhs_src, parser), + tsp::highlight_captures(rhs_tree, rhs_src, parser), + ) + } else { + (Vec::new(), Vec::new()) + } + }; + if lhs_src == rhs_src { let file_format = match language { Some(language) => FileFormat::SupportedLanguage(language), None => FileFormat::PlainText, }; + let (lhs_highlights, rhs_highlights) = match lang_config { + Some((_, lang_config)) if diff_options.syntax => { + let tree = tsp::to_tree(lhs_src, lang_config.parser); + highlights(&tree, &tree, lang_config.parser) + } + _ => (Vec::new(), Vec::new()), + }; // If the two files are byte-for-byte identical, return early // rather than doing any more work. @@ -127,10 +147,14 @@ pub fn diff_file_content( rhs_positions: vec![], lhs_folds: vec![], rhs_folds: vec![], + lhs_highlights, + rhs_highlights, }); } let mut lhs_folds = Vec::new(); + let mut lhs_highlights: Vec = Vec::new(); + let mut rhs_highlights: Vec = Vec::new(); let mut rhs_folds = Vec::new(); let (file_format, lhs_positions, rhs_positions) = match lang_config { _ if diff_options.by_line.is_some() => { @@ -151,6 +175,8 @@ pub fn diff_file_content( let arena = Arena::new(); match tsp::to_tree_with_limit(diff_options, lang_config.parser, lhs_src, rhs_src) { Ok((lhs_tree, rhs_tree)) => { + (lhs_highlights, rhs_highlights) = + highlights(&lhs_tree, &rhs_tree, lang_config.parser); match tsp::to_syntax_with_limit( lhs_src, rhs_src, @@ -348,5 +374,7 @@ pub fn diff_file_content( rhs_positions, lhs_folds, rhs_folds, + lhs_highlights, + rhs_highlights, }) } diff --git a/crates/diffr-core/src/options.rs b/crates/diffr-core/src/options.rs index 503298897..cf35f22c7 100644 --- a/crates/diffr-core/src/options.rs +++ b/crates/diffr-core/src/options.rs @@ -18,6 +18,8 @@ pub struct DiffOptions { /// Diff the file by line without parsing, for this reason: it is /// generated or hidden. pub by_line: Option, + /// Collect highlight captures from each parsed side. + pub syntax: bool, } impl Default for DiffOptions { @@ -28,6 +30,7 @@ impl Default for DiffOptions { parse_error_limit: DEFAULT_PARSE_ERROR_LIMIT, ignore_comments: false, by_line: None, + syntax: false, } } } diff --git a/crates/diffr-core/src/parse/fold_tests.rs b/crates/diffr-core/src/parse/fold_tests.rs index b989b665f..28b3a0529 100644 --- a/crates/diffr-core/src/parse/fold_tests.rs +++ b/crates/diffr-core/src/parse/fold_tests.rs @@ -255,6 +255,51 @@ mod folds { } } + #[test] + fn a_body_fold_indents_at_its_first_statement_and_keeps_its_exact_span() { + // Each body's (indent line, column) and the text between its opener + // and closer; Go's label is outdented, so the statement it labels + // gives the indent. + for (path, src, indent, span) in [ + ( + "a.rs", + "fn run() {\n work();\n rest();\n}\n", + (1, 4), + "\n work();\n rest();\n", + ), + ( + "a.py", + "def run():\n work()\n rest()\n", + (1, 4), + "\n work()\n rest()", + ), + ( + "a.js", + "function run() {\n work();\n rest();\n}\n", + (1, 2), + "\n work();\n rest();\n", + ), + ( + "a.go", + "package main\nfunc run() {\nretry:\n\tif try() {\n\t\tgoto retry\n\t}\n\trest()\n}\n", + (3, 1), + "\nretry:\n\tif try() {\n\t\tgoto retry\n\t}\n\trest()\n", + ), + ] { + let review = DiffResult::from_sources(path, "", src); + let body = review + .rhs_folds + .iter() + .find(|fold| fold.syntax.is_some_and(|syntax| text(src, &syntax) == span)) + .unwrap_or_else(|| panic!("no body fold in {path}")); + assert_eq!( + (body.indent.line.as_usize(), body.indent.byte_column), + indent, + "{path}" + ); + } + } + #[test] fn two_queries_folding_the_same_lines_make_one_fold() { // A function whose body is one `match`: the block the shared query diff --git a/crates/diffr-core/src/parse/folds.rs b/crates/diffr-core/src/parse/folds.rs index 416683cec..6c2aa8376 100644 --- a/crates/diffr-core/src/parse/folds.rs +++ b/crates/diffr-core/src/parse/folds.rs @@ -23,6 +23,10 @@ pub struct Fold { pub(crate) tags: Vec, /// Source on this side; may span multiple syntax nodes. pub(crate) range: SourceRange, + /// See `FoldMetadata::indent`. + pub(crate) indent: SourcePosition, + /// See `FoldMetadata::syntax`. + pub(crate) syntax: Option, /// The syntax node the fold was built on. Engine-internal identity: /// syntax ids are unique across both sides of a file and never reach the /// wire, where the projection numbers regions itself. @@ -45,8 +49,8 @@ pub(crate) enum FoldMatch { } /// Two query patterns captured the same syntax node with different fold -/// ranges. Which range wins would depend on query order, so the file is not -/// diffed. +/// ranges or indents. Which wins would depend on query order, so the file is +/// not diffed. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Conflict { /// 0-based line the node starts on. @@ -160,6 +164,11 @@ pub(crate) fn classify( if region.start == region.end { continue; } + let syntax = named("fold.open").next_back().map(|_| region); + let indent = match named("fold.indent").next_back() { + Some(indent) => node_range(indent.node).start, + None => node_range(first.node).start, + }; match kinds.entry(first.node.id()) { Entry::Vacant(entry) => { let mut tags = pattern.tags.clone(); @@ -170,13 +179,15 @@ pub(crate) fn classify( relations: Vec::new(), tags, range_override: Some(region), + indent, + syntax, }, pattern.source, )); } Entry::Occupied(mut entry) => { let (metadata, source) = entry.get_mut(); - if metadata.range_override != Some(region) { + if metadata.range_override != Some(region) || metadata.indent != indent { let mut sources = [ compiled.sources[*source].clone(), compiled.sources[pattern.source].clone(), @@ -408,6 +419,8 @@ pub(crate) fn project(node: &Syntax<'_>, partner: Option<&Syntax<'_>>) -> Option relations: metadata.relations.clone(), tags: metadata.tags.clone(), range: own_range, + indent: metadata.indent, + syntax: metadata.syntax, syntax_id: node.id(), match_kind: match opposite { Some(partner) => FoldMatch::Matched { @@ -462,6 +475,8 @@ pub(crate) fn merge_spans(folds: &mut Vec, lines: &[&str]) { let merged = &mut kept[*entry.get()]; if inside(&fold.range, &merged.range) { merged.range = fold.range; + merged.indent = fold.indent; + merged.syntax = fold.syntax; merged.syntax_id = fold.syntax_id; merged.match_kind = fold.match_kind; } diff --git a/crates/diffr-core/src/parse/syntax.rs b/crates/diffr-core/src/parse/syntax.rs index 575bd34df..c0be39b9b 100644 --- a/crates/diffr-core/src/parse/syntax.rs +++ b/crates/diffr-core/src/parse/syntax.rs @@ -17,7 +17,7 @@ use crate::diff::changes::ChangeKind::*; use crate::diff::changes::{ChangeKind, ChangeMap}; use crate::diff::lcs_diff; use crate::hash::DftHashMap; -use crate::lines::{is_all_whitespace, split_on_newlines, SourceRange}; +use crate::lines::{is_all_whitespace, split_on_newlines, SourcePosition, SourceRange}; use crate::words::split_words_and_numbers; /// A Debug implementation that does not recurse into the @@ -59,6 +59,10 @@ pub(crate) struct FoldMetadata { pub(crate) tags: Vec, /// Query-selected boundary, retained when flattening removes its owner. pub(crate) range_override: Option, + /// The start of the `@fold.indent` node, else of the `@fold` node. + pub(crate) indent: SourcePosition, + /// The range between `@fold.open` and `@fold.close`. None without an opener. + pub(crate) syntax: Option, } /// Fields that are common to both `Syntax::List` and `Syntax::Atom`. @@ -1261,6 +1265,11 @@ mod tests { relations: Vec::new(), tags: vec![tag.to_owned()], range_override: Some(folds::interior_range(&[point(0)], &[point(10)])), + indent: SourcePosition { + line: 0.into(), + byte_column: 0, + }, + syntax: None, }; let atom = |fold| { Syntax::new_atom_with_fold( diff --git a/crates/diffr-core/src/parse/tree_sitter_parser.rs b/crates/diffr-core/src/parse/tree_sitter_parser.rs index fd496547a..2eb370ad2 100644 --- a/crates/diffr-core/src/parse/tree_sitter_parser.rs +++ b/crates/diffr-core/src/parse/tree_sitter_parser.rs @@ -15,6 +15,7 @@ use crate::hash::{DftHashMap, DftHashSet}; use crate::options::DiffOptions; use crate::parse::guess_language as guess; use crate::parse::syntax::{AtomKind, Syntax}; +use crate::summary::Highlight; /// A language may contain certain nodes that are in other languages /// and should be parsed as such (e.g. HTML `