Context folding as outlines, fold guides and streamed file slots in the TUI

crates/diffr-core/src/config.rs

@@ -458,6 +458,14 @@ mod tests {
         for (query, message) in [
             ("(not_a_rust_node) @fold", "NodeType error"),
             ("(block) @typo", "unsupported capture @typo"),
+            (
+                r#"(block "{" @fold.open "}" @fold.close) @fold"#,
+                "@fold.open needs exactly one @fold.indent",
+            ),
+            (
+                r#"[(block "{" @fold.open . (_) @fold.indent "}" @fold.close) (match_block)] @fold"#,
+                "@fold.open must be in every branch",
+            ),
             (
                 r#"((block) @fold (#set! tag "body"))"#,
                 "must be written <plugin>:<name>",
@@ -519,7 +527,7 @@ mod query_tests {
     fn arbitrary_tags_and_delimiter_captures_reach_the_domain() {
         let params = with_queries(&[(
             "rust",
-            r#"((block "{" @fold.open "}" @fold.close) @fold (#set! tag "removed-runs:user.validation"))"#,
+            r#"((block "{" @fold.open . (_) @fold.indent "}" @fold.close) @fold (#set! tag "removed-runs:user.validation"))"#,
         )]);
         let source = "fn f() { println!(\"☕\"); }";
         let diff = DiffResult::from_sources_with_params("a.rs", "", source, &params);
@@ -611,7 +619,7 @@ mod tag_tests {
 
     #[test]
     fn an_opening_capture_alone_folds_to_the_end_of_the_fold_node() {
-        let query = r#"((function_definition ":" @fold.open body: (block) @fold) (#set! tag "removed-runs:body"))"#;
+        let query = r#"((function_definition ":" @fold.open body: (block . (_) @fold.indent) @fold) (#set! tag "removed-runs:body"))"#;
         let params = with_queries(&[("python", query)]);
         let rhs = "def f(a):\n    x = a\n    return x\n";
         let result = DiffResult::from_sources_with_params("a.py", "", rhs, &params);
@@ -629,7 +637,7 @@ mod tag_tests {
     fn a_node_captured_with_two_ranges_is_a_query_conflict_naming_both_files() {
         let whole = "((block) @fold (#set! tag \"removed-runs:whole\"))";
         let interior =
-            "((block \"{\" @fold.open \"}\" @fold.close) @fold (#set! tag \"summarize:inside\"))";
+            "((block \"{\" @fold.open . (_) @fold.indent \"}\" @fold.close) @fold (#set! tag \"summarize:inside\"))";
         let dir = tempfile::tempdir().unwrap();
         std::fs::create_dir_all(dir.path().join("queries/rust")).unwrap();
         let removed_runs = dir.path().join("queries/rust/removed-runs.scm");
@@ -670,7 +678,7 @@ mod tag_tests {
             assert!(
                 message.contains("queries/rust/removed-runs.scm and ")
                     && message.contains(
-                        "queries/rust/summarize.scm capture the same block with different fold ranges"
+                        "queries/rust/summarize.scm capture the same block with different fold ranges or indents"
                     ),
                 "{message}"
             );
@@ -799,8 +807,8 @@ mod format_tests {
             .to_string()
             .contains("unsupported config version 2"));
         let config =
-            Config::from_toml("version = 1\n[plugins]\norder = ['bundled.group']\n").unwrap();
+            Config::from_toml("version = 1\n[plugins]\norder = ['bundled.context']\n").unwrap();
         assert_eq!(config.plugins.entries.len(), 1);
-        assert!(config.plugins.entries.contains_key("bundled.group"));
+        assert!(config.plugins.entries.contains_key("bundled.context"));
     }
 }

crates/diffr-core/src/config/README.md

@@ -23,7 +23,7 @@ typescript = "queries/javascript.scm"
 
 ```scheme
 ; inherits: builtin:shared/queries/rust.scm
-((function_item body: (block "{" @fold.open "}" @fold.close) @fold)
+((function_item body: (block "{" @fold.open . (_) @fold.indent "}" @fold.close) @fold)
   (#set! tag "deleted-bodies:function"))
 ```
 
@@ -46,7 +46,7 @@ Tree-sitter coordinates; labels and comments before the opening brace stay visib
 With both delimiter captures, they must be ordered and contained in the fold node.
 A `@fold.open` without `@fold.close` hides from the opening node's end to the end
 of the fold node; the opening node may precede the fold node, so
-`(function_definition ":" @fold.open body: (block) @fold)` folds a Python body
+`(function_definition ":" @fold.open body: (block . (_) @fold.indent) @fold)` folds a Python body
 from its header's `:`. A `@fold.close` without `@fold.open` selects no fold.
 
 These fold-boundary captures are our convention. There are no arbitrary byte or

crates/diffr-core/src/config/default.toml

@@ -3,12 +3,11 @@ version = 1
 
 [plugins]
 order = [
-    "bundled.context",
     "bundled.deleted-bodies",
     "bundled.summarize",
     "bundled.test-bodies",
     "bundled.removed-runs",
-    "bundled.group",
+    "bundled.context",
 ]
 
 [plugins.bundled.context]
@@ -19,9 +18,6 @@ lines = 3
 enabled = true
 min_lines = 12
 
-[plugins.bundled.group]
-enabled = true
-
 [plugins.bundled.removed-runs]
 enabled = true
 min_lines = 5

crates/diffr-core/src/config/query.rs

@@ -4,7 +4,7 @@
 //! order. Every pattern remembers which source it came from, so errors and
 //! conflicts can name the file that wrote it.
 use super::ConfigError;
-use tree_sitter::Query;
+use tree_sitter::{CaptureQuantifier, Query};
 
 pub(crate) struct AnnotationQuery {
     pub(crate) query: Query,
@@ -60,7 +60,7 @@ impl AnnotationQuery {
         for name in query.capture_names() {
             if name.starts_with("related.")
                 || name.starts_with('_')
-                || matches!(*name, "fold" | "fold.open" | "fold.close")
+                || matches!(*name, "fold" | "fold.open" | "fold.close" | "fold.indent")
             {
                 continue;
             }
@@ -79,6 +79,30 @@ impl AnnotationQuery {
                 tags: Vec::new(),
                 source: owner(query.start_byte_for_pattern(index)),
             };
+            // Every match with `@fold.open` has exactly one `@fold.indent`.
+            let quantifier = |name: &str| {
+                query
+                    .capture_index_for_name(name)
+                    .map_or(CaptureQuantifier::Zero, |capture| {
+                        query.capture_quantifiers(index)[capture as usize]
+                    })
+            };
+            match (quantifier("fold.open"), quantifier("fold.indent")) {
+                (CaptureQuantifier::Zero, _) | (CaptureQuantifier::One, CaptureQuantifier::One) => {
+                }
+                (CaptureQuantifier::One, _) => {
+                    return Err(named(
+                        index,
+                        "@fold.open needs exactly one @fold.indent".into(),
+                    ))
+                }
+                _ => {
+                    return Err(named(
+                        index,
+                        "@fold.open must be in every branch of its pattern".into(),
+                    ))
+                }
+            }
             if !query.property_predicates(index).is_empty() {
                 return Err(named(index, "#is? and #is-not? are not supported".into()));
             }

crates/diffr-core/src/config/store.rs

@@ -368,7 +368,7 @@ mod tests {
         )
         .unwrap();
         let path = dir.path().join("config.toml");
-        let order = "order = ['bundled.context', 'bundled.deleted-bodies', 'bundled.test-bodies', 'bundled.removed-runs', 'bundled.summarize', 'bundled.group', 'external.mine']";
+        let order = "order = ['bundled.context', 'bundled.deleted-bodies', 'bundled.test-bodies', 'bundled.removed-runs', 'bundled.summarize', 'external.mine']";
         std::fs::write(
             &path,
             format!("[plugins]\n{order}\n[plugins.external.mine]\npath = 'plugins/mine'\n"),
@@ -673,15 +673,15 @@ mod sparse_tests {
         let path = dir.path().join("config.toml");
         std::fs::write(
             &path,
-            "# just grouping\n[plugins]\norder = ['bundled.group']\n",
+            "# just context\n[plugins]\norder = ['bundled.context']\n",
         )
         .unwrap();
         set(&path, "diff.graph_limit", "42").unwrap();
         let text = std::fs::read_to_string(&path).unwrap();
         let config = Config::from_toml(&text).unwrap();
         assert_eq!(config.plugins.entries.len(), 1);
-        assert!(text.contains("# just grouping\n[plugins]"), "{text}");
-        assert!(!text.contains("bundled.context"));
+        assert!(text.contains("# just context\n[plugins]"), "{text}");
+        assert!(!text.contains("bundled.deleted-bodies"));
     }
 
     const LEGACY: &str = "For each listed fold, rewrite that function body as short pseudocode. Keep the names. No prose, no comments, no code fences. Use as few lines as possible: about one pseudocode line per five source lines, and never more than a third of the body's lines. When a fold lists a doc, also set \"summary\" to one sentence copied verbatim from that doc; otherwise leave it empty. Answer with a JSON array of {\"id\", \"summary\", \"pseudocode\"} objects, one per fold.";

crates/diffr-core/src/engine.rs

@@ -13,7 +13,7 @@ use crate::parse::folds;
 use crate::parse::guess_language::{guess, language_name, LanguageOverride};
 use crate::parse::syntax::{self, init_next_prev};
 use crate::parse::tree_sitter_parser as tsp;
-use crate::summary::{DiffResult, FallbackCause, FileContent, FileFormat};
+use crate::summary::{DiffResult, FallbackCause, FileContent, FileFormat, Highlight};
 use humansize::{format_size, FormatSizeOptions, BINARY};
 use std::{env, fmt, path::Path};
 use typed_arena::Arena;
@@ -45,7 +45,7 @@ impl fmt::Display for QueryConflict {
         let (first, second) = &self.conflict.sources;
         write!(
             f,
-            "{}:{}{}: {first} and {second} capture the same {} with different fold ranges",
+            "{}:{}{}: {first} and {second} capture the same {} with different fold ranges or indents",
             self.path,
             self.conflict.line + 1,
             match self.side {
@@ -111,11 +111,31 @@ pub fn diff_file_content(
     let language = guess(Path::new(display_path), guess_src, overrides);
     let lang_config = language.map(|lang| (lang, params.language(lang)));
 
+    // Highlights come from the same parse as the folds, so a side that
+    // parsed has them whether or not the match runs.
+    let highlights = |lhs_tree: &tree_sitter::Tree, rhs_tree: &tree_sitter::Tree, parser| {
+        if diff_options.syntax {
+            (
+                tsp::highlight_captures(lhs_tree, lhs_src, parser),
+                tsp::highlight_captures(rhs_tree, rhs_src, parser),
+            )
+        } else {
+            (Vec::new(), Vec::new())
+        }
+    };
+
     if lhs_src == rhs_src {
         let file_format = match language {
             Some(language) => FileFormat::SupportedLanguage(language),
             None => FileFormat::PlainText,
         };
+        let (lhs_highlights, rhs_highlights) = match lang_config {
+            Some((_, lang_config)) if diff_options.syntax => {
+                let tree = tsp::to_tree(lhs_src, lang_config.parser);
+                highlights(&tree, &tree, lang_config.parser)
+            }
+            _ => (Vec::new(), Vec::new()),
+        };
 
         // If the two files are byte-for-byte identical, return early
         // rather than doing any more work.
@@ -127,10 +147,14 @@ pub fn diff_file_content(
             rhs_positions: vec![],
             lhs_folds: vec![],
             rhs_folds: vec![],
+            lhs_highlights,
+            rhs_highlights,
         });
     }
 
     let mut lhs_folds = Vec::new();
+    let mut lhs_highlights: Vec<Highlight> = Vec::new();
+    let mut rhs_highlights: Vec<Highlight> = Vec::new();
     let mut rhs_folds = Vec::new();
     let (file_format, lhs_positions, rhs_positions) = match lang_config {
         _ if diff_options.by_line.is_some() => {
@@ -151,6 +175,8 @@ pub fn diff_file_content(
             let arena = Arena::new();
             match tsp::to_tree_with_limit(diff_options, lang_config.parser, lhs_src, rhs_src) {
                 Ok((lhs_tree, rhs_tree)) => {
+                    (lhs_highlights, rhs_highlights) =
+                        highlights(&lhs_tree, &rhs_tree, lang_config.parser);
                     match tsp::to_syntax_with_limit(
                         lhs_src,
                         rhs_src,
@@ -348,5 +374,7 @@ pub fn diff_file_content(
         rhs_positions,
         lhs_folds,
         rhs_folds,
+        lhs_highlights,
+        rhs_highlights,
     })
 }

crates/diffr-core/src/options.rs

@@ -18,6 +18,8 @@ pub struct DiffOptions {
     /// Diff the file by line without parsing, for this reason: it is
     /// generated or hidden.
     pub by_line: Option<FallbackCause>,
+    /// Collect highlight captures from each parsed side.
+    pub syntax: bool,
 }
 
 impl Default for DiffOptions {
@@ -28,6 +30,7 @@ impl Default for DiffOptions {
             parse_error_limit: DEFAULT_PARSE_ERROR_LIMIT,
             ignore_comments: false,
             by_line: None,
+            syntax: false,
         }
     }
 }

crates/diffr-core/src/parse/fold_tests.rs

@@ -255,6 +255,51 @@ mod folds {
         }
     }
 
+    #[test]
+    fn a_body_fold_indents_at_its_first_statement_and_keeps_its_exact_span() {
+        // Each body's (indent line, column) and the text between its opener
+        // and closer; Go's label is outdented, so the statement it labels
+        // gives the indent.
+        for (path, src, indent, span) in [
+            (
+                "a.rs",
+                "fn run() {\n    work();\n    rest();\n}\n",
+                (1, 4),
+                "\n    work();\n    rest();\n",
+            ),
+            (
+                "a.py",
+                "def run():\n    work()\n    rest()\n",
+                (1, 4),
+                "\n    work()\n    rest()",
+            ),
+            (
+                "a.js",
+                "function run() {\n  work();\n  rest();\n}\n",
+                (1, 2),
+                "\n  work();\n  rest();\n",
+            ),
+            (
+                "a.go",
+                "package main\nfunc run() {\nretry:\n\tif try() {\n\t\tgoto retry\n\t}\n\trest()\n}\n",
+                (3, 1),
+                "\nretry:\n\tif try() {\n\t\tgoto retry\n\t}\n\trest()\n",
+            ),
+        ] {
+            let review = DiffResult::from_sources(path, "", src);
+            let body = review
+                .rhs_folds
+                .iter()
+                .find(|fold| fold.syntax.is_some_and(|syntax| text(src, &syntax) == span))
+                .unwrap_or_else(|| panic!("no body fold in {path}"));
+            assert_eq!(
+                (body.indent.line.as_usize(), body.indent.byte_column),
+                indent,
+                "{path}"
+            );
+        }
+    }
+
     #[test]
     fn two_queries_folding_the_same_lines_make_one_fold() {
         // A function whose body is one `match`: the block the shared query