Fix/sentence starts

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
         run: cargo test
 
       - name: Pandoc AST fixtures (JSON on disk, no live pandoc)
-        run: cargo test --features "cli,pandoc" --test pandoc_ast_backend -- --nocapture
+        run: cargo test --features "cli,pandoc" --test integration pandoc_ast_backend -- --nocapture
 
       - name: Colink opt-in without archive fails closed
         run: |

Cargo.toml

@@ -2,6 +2,7 @@
 name = "snapper-fmt"
 version = "0.11.7"
 edition = "2024"
+autotests = false
 rust-version = "1.85"
 description = "Semantic line break formatter for Org, LaTeX, Markdown, RST, and plaintext"
 repository = "https://github.com/TurtleTech-ehf/snapper"
@@ -160,3 +161,11 @@ pkg-fmt = "txz"
 [[bench]]
 name = "reflow_bench"
 harness = false
+
+[[test]]
+name = "integration"
+path = "tests/integration.rs"
+
+[[test]]
+name = "pandoc_parity_speed"
+path = "tests/pandoc_parity_speed.rs"

cog.toml

@@ -7,7 +7,9 @@ tag_prefix = "v"
 # Bump version everywhere: Cargo.toml drives pyproject.toml via dynamic = ["version"]
 pre_bump_hooks = [
     "sed -i '0,/^version = /s/^version = \".*\"/version = \"{{version}}\"/' Cargo.toml",
-    "cargo generate-lockfile",
+    # Only the workspace crates move; generate-lockfile would re-resolve every
+    # dependency and ship upgrades no test ran against.
+    "cargo update --workspace",
     "sed -i 's/^release = \".*\"/release = \"{{version}}\"/' docs/source/conf.py",
     "sed -i 's/rev: v[0-9][0-9.\\-]*/rev: v{{version}}/' README.md docs/source/howto/vale-integration.rst",
     "sed -i 's/\"version\": \".*\"/\"version\": \"{{version}}\"/' editors/vscode/package.json editors/obsidian/package.json editors/obsidian/manifest.json editors/word/package.json packages/snapper-wasm/package.json npm/package.json",

docs/orgmode/reference/cli.org

@@ -290,7 +290,8 @@ Uses =git show REF:path= to retrieve the old version.
 :END:
 
 Watch files and auto-reformat on save.
-Accepts file paths or glob patterns. 200ms debounce to avoid reformatting during rapid saves.
+Accepts file paths or glob patterns.
+200ms debounce to avoid reformatting during rapid saves.
 Press Ctrl+C to stop.
 
 *** =snapper lsp=

src/abbreviations.rs

@@ -4,7 +4,9 @@ pub static EN_ABBREVIATIONS: &[&str] = &[
     "Mr", "Mrs", "Ms", "Dr", "Prof", "Sr", "Jr", "St", "Rev", "Gen", "Gov", "Sgt", "Cpl", "Pvt",
     "Capt", "Lt", "Col", "Maj", "Cmdr", "Adm", // Academic / scientific
     "Fig", "Figs", "Eq", "Eqs", "Ref", "Refs", "Tab", "Sec", "Ch", "Vol", "No", "Nos", "Ed", "Eds",
-    "Trans", "Dept", "Thm", "Lem", "Prop", "Def", "Cor", "Rem", "Ex",
+    "Trans", "Dept", "Thm", "Lem", "Prop", "Def", "Cor", "Rem",
+    "Ex", // The same, lowercase in running text
+    "fig", "figs", "eq", "eqs", "ref", "refs", "tab", "sec", "ch", "vol", "nos", "p",
     // Latin abbreviations
     "al", "approx", "ca", "cf", "etc", "et", "ibid", "viz", // Common
     "vs", "misc", "est", "govt", "dept", "univ", "inc", "Inc", "corp", "Corp", "ltd", "Ltd", "Ave",

src/check.rs

@@ -345,9 +345,8 @@ mod tests {
 
     #[test]
     fn long_advisory_needs_clause_boundary() {
-        let long_with_comma = format!(
-            "The quick brown fox jumps over the lazy dog, then continues running across a very long meadow without pausing for breath at all today.\n"
-        );
+        let long_with_comma =
+            "The quick brown fox jumps over the lazy dog, then continues running across a very long meadow without pausing for breath at all today.\n".to_string();
         assert!(
             long_with_comma.trim_end().chars().count() > DEFAULT_LONG_THRESHOLD,
             "fixture must exceed the default long threshold"

src/parser/latex.rs

@@ -4740,7 +4740,7 @@ Some text.
         // Whole-line prefix: leftover scan `from` can sit after another
         // leftover on the same line. `\lstinline` must stay Prose so
         // `See \verb|x| here.` is not the only mid-line case.
-        let after_other = concat!("\\py{print(1)} \\lstinline|print(1)|\n",);
+        let after_other = "\\py{print(1)} \\lstinline|print(1)|\n";
         let after_other_regions = LatexParser::default().parse(after_other);
         assert!(
             after_other_regions.iter().any(|r| matches!(
@@ -6120,7 +6120,8 @@ Some text.
             Region::Code { body, footer, .. } => Some((body.as_str(), footer.as_str())),
             _ => None,
         });
-        let (body, footer) = code.expect(&format!("lstlisting must be Code, got: {regions:?}"));
+        let (body, footer) =
+            code.unwrap_or_else(|| panic!("lstlisting must be Code, got: {regions:?}"));
         assert!(
             body.contains("print(1)"),
             "listing body must keep source, got body={body:?} regions={regions:?}"
@@ -6159,7 +6160,8 @@ Some text.
             Region::Code { body, footer, .. } => Some((body.as_str(), footer.as_str())),
             _ => None,
         });
-        let (body, footer) = code.expect(&format!("verbatim must be Code, got: {regions:?}"));
+        let (body, footer) =
+            code.unwrap_or_else(|| panic!("verbatim must be Code, got: {regions:?}"));
         assert!(
             body.contains("still body"),
             "inner \\end must not close the outer verbatim; still body stays in the listing, got body={body:?} regions={regions:?}"
@@ -6196,7 +6198,8 @@ Some text.
             Region::Code { body, footer, .. } => Some((body.as_str(), footer.as_str())),
             _ => None,
         });
-        let (body, footer) = code.expect(&format!("lstlisting must be Code, got: {regions:?}"));
+        let (body, footer) =
+            code.unwrap_or_else(|| panic!("lstlisting must be Code, got: {regions:?}"));
         assert!(
             body.contains("print(1)"),
             "listing body must keep source before %, got body={body:?} regions={regions:?}"
@@ -7646,9 +7649,8 @@ Some text.
             Region::Code { body, footer, .. } => Some((body.as_str(), footer.as_str())),
             _ => None,
         });
-        let (raw_body, raw_footer) = raw_code.expect(&format!(
-            "lstlisting* must stay Code on %, got: {raw_regions:?}"
-        ));
+        let (raw_body, raw_footer) = raw_code
+            .unwrap_or_else(|| panic!("lstlisting* must stay Code on %, got: {raw_regions:?}"));
         assert!(
             raw_body.contains("print(1)"),
             "lstlisting* raw scan must keep source before %, got body={raw_body:?}"

src/parser/markdown.rs

@@ -5740,7 +5740,8 @@ mod tests {
             Region::Structure(s) if s.contains("<!--") => Some(s.as_str()),
             _ => None,
         });
-        let comment = comment.expect(&format!("comment must be Structure, got {regions:?}"));
+        let comment =
+            comment.unwrap_or_else(|| panic!("comment must be Structure, got {regions:?}"));
         assert!(comment.contains("<!--"), "{comment}");
         assert!(comment.contains("Hidden. With a period."), "{comment}");
         assert!(comment.contains("Still comment."), "{comment}");
@@ -6129,7 +6130,7 @@ mod tests {
             Region::Structure(s) if s.contains("<div") => Some(s.as_str()),
             _ => None,
         });
-        let div = div.expect(&format!("div block must be Structure, got {regions:?}"));
+        let div = div.unwrap_or_else(|| panic!("div block must be Structure, got {regions:?}"));
         assert!(div.contains("<div class=\"note\">"), "{div}");
         assert!(div.contains("<p>Hello. World.</p>"), "{div}");
         assert!(div.contains("</div>"), "{div}");
@@ -6161,7 +6162,7 @@ mod tests {
             Region::Structure(s) if s.contains("<div>") => Some(s.as_str()),
             _ => None,
         });
-        let div = div.expect(&format!("div block must be Structure, got {regions:?}"));
+        let div = div.unwrap_or_else(|| panic!("div block must be Structure, got {regions:?}"));
         assert!(div.contains("<div>"), "{div}");
         assert!(div.contains("First. Second."), "{div}");
         assert!(div.contains("</div>"), "{div}");
@@ -6196,7 +6197,7 @@ mod tests {
             Region::Structure(s) if s.contains("<hr>") => Some(s.as_str()),
             _ => None,
         });
-        let hr = hr.expect(&format!("hr block must be Structure, got {regions:?}"));
+        let hr = hr.unwrap_or_else(|| panic!("hr block must be Structure, got {regions:?}"));
         assert!(
             !hr.contains("After html"),
             "void type-6 must end on the hr line, got {hr}"
@@ -6478,7 +6479,7 @@ mod tests {
             Region::Structure(s) if s.contains("<span") => Some(s.as_str()),
             _ => None,
         });
-        let span = span.expect(&format!("span block must be Structure, got {regions:?}"));
+        let span = span.unwrap_or_else(|| panic!("span block must be Structure, got {regions:?}"));
         assert!(span.contains("<span class=\"note\">"), "{span}");
         assert!(span.contains("First. Second."), "{span}");
         assert!(span.contains("</span>"), "{span}");

src/parser/pandoc/ast.rs

@@ -509,7 +509,7 @@ mod tests {
             .filter(|r| matches!(r, Region::Prose(_)))
             .collect();
         assert!(
-            prose.len() >= 1,
+            !prose.is_empty(),
             "expected ≥1 prose region from Para nodes, got {regions:?}"
         );
         assert!(
@@ -578,7 +578,7 @@ mod tests {
             Region::Structure(s) if s.contains('|') && s.contains("---") => Some(s.as_str()),
             _ => None,
         });
-        let table = table.expect(&format!("pipe table structure: {regions:?}"));
+        let table = table.unwrap_or_else(|| panic!("pipe table structure: {regions:?}"));
         assert!(table.contains('a') && table.contains('b') && table.contains('1'));
         assert!(
             !regions
@@ -627,9 +627,8 @@ mod tests {
             Region::Structure(s) if s.contains("cargo binstall") => Some(s.as_str()),
             _ => None,
         });
-        let heading = heading.expect(&format!(
-            "expected Structure from Header node, got {regions:?}"
-        ));
+        let heading = heading
+            .unwrap_or_else(|| panic!("expected Structure from Header node, got {regions:?}"));
         assert!(
             heading.contains("1.") && heading.contains("cargo binstall"),
             "Header title text preserved as structure payload: {heading:?}"

src/sentence/unicode.rs

@@ -120,6 +120,9 @@ pub struct UnicodeSentenceSplitter {
     lang_multi_pattern: Regex,
     /// Extra LaTeX command names tokenized like `\verb` before split.
     extra_verbatim_commands: Vec<String>,
+    /// Whether a lowercase word after a terminator and a space opens a
+    /// sentence. UAX #29 SB8 says no; a reader counting claims says yes.
+    lowercase_starts: bool,
 }
 
 impl UnicodeSentenceSplitter {
@@ -160,9 +163,19 @@ impl UnicodeSentenceSplitter {
             lang_abbrev_pattern,
             lang_multi_pattern,
             extra_verbatim_commands: Vec::new(),
+            lowercase_starts: false,
         }
     }
 
+    /// Open a sentence at a lowercase word after a terminator and a space
+    /// (`It stopped. packsetd restarts.`), which UAX #29 SB8 keeps joined.
+    /// A known abbreviation before the terminator (`e.g. the`) still joins.
+    /// Off by default: a formatter reflowing prose keeps SB8.
+    pub fn with_lowercase_starts(mut self, on: bool) -> Self {
+        self.lowercase_starts = on;
+        self
+    }
+
     /// Extra LaTeX command names tokenized like `\verb` before split.
     pub fn with_verbatim_commands(mut self, cmds: Vec<String>) -> Self {
         self.extra_verbatim_commands = cmds;
@@ -3134,6 +3147,9 @@ impl SentenceSplitter for UnicodeSentenceSplitter {
         // lowercase. Split same-line `iCloud` starts before abbreviation
         // merge so `e.g. iCloud` can rejoin.
         let expanded = split_before_lowercase_proper_nouns(raw_segments.iter().copied());
+        // SB8 also keeps `stopped. 3 tests` joined; a number opens a
+        // sentence, and the abbreviation merge below rejoins `Fig. 3`.
+        let expanded = split_before_sentence_starts(&expanded, self.lowercase_starts);
         let refs: Vec<&str> = expanded.iter().map(String::as_str).collect();
         let merged = self.refine_segments_from_strs(&refs);
         let restored = restore_inline_tokens(merged, &placeholders);
@@ -3257,6 +3273,45 @@ fn take_lowercase_proper_noun_sentence(seg: &str) -> Option<(String, String)> {
     Some((head.to_string(), rest.to_string()))
 }
 
+/// Split each segment after a terminator and whitespace when the next word
+/// opens with a digit, or with a lowercase letter when `lowercase` is set.
+/// A head with no letter (`1.` opening a list item, `...`) is not a
+/// sentence, so the scan moves on to the next terminator.
+fn split_before_sentence_starts(segments: &[String], lowercase: bool) -> Vec<String> {
+    let mut out = Vec::with_capacity(segments.len());
+    for seg in segments {
+        let chars: Vec<(usize, char)> = seg.char_indices().collect();
+        let mut from = 0usize;
+        let mut i = 0usize;
+        while i < chars.len() {
+            let (_, c) = chars[i];
+            if !matches!(c, '.' | '!' | '?') {
+                i += 1;
+                continue;
+            }
+            let mut j = i;
+            while j < chars.len() && matches!(chars[j].1, '.' | '!' | '?') {
+                j += 1;
+            }
+            let mut k = j;
+            while k < chars.len() && chars[k].1.is_whitespace() {
+                k += 1;
+            }
+            let opens = chars
+                .get(k)
+                .is_some_and(|&(_, n)| n.is_ascii_digit() || (lowercase && n.is_lowercase()));
+            let end = chars.get(j).map_or(seg.len(), |&(at, _)| at);
+            if k > j && opens && seg[from..end].chars().any(char::is_alphabetic) {
+                out.push(seg[from..end].to_string());
+                from = chars[k].0;
+            }
+            i = j.max(i + 1);
+        }
+        out.push(seg[from..].to_string());
+    }
+    out.into_iter().filter(|s| !s.trim().is_empty()).collect()
+}
+
 fn merge_abbreviation_splits(
     segments: &[&str],
     abbrev_re: &Regex,
@@ -3847,6 +3902,41 @@ mod tests {
         );
     }
 
+    #[test]
+    fn a_number_opens_a_sentence() {
+        assert_eq!(
+            split("It stopped. 3 tests failed."),
+            vec!["It stopped.", "3 tests failed."]
+        );
+        assert_eq!(
+            split("See Fig. 3 and Eq. 2 for the rate. Done."),
+            vec!["See Fig. 3 and Eq. 2 for the rate.", "Done."]
+        );
+        assert_eq!(
+            split("The tracker has 0.9.3 now. It holds."),
+            vec!["The tracker has 0.9.3 now.", "It holds."]
+        );
+    }
+
+    #[test]
+    fn a_lowercase_start_opens_a_sentence_only_when_asked() {
+        let text = "Restart it. packsetd reloads the pack.";
+        assert_eq!(split(text), vec![text.to_string()], "SB8 by default");
+        let claims = UnicodeSentenceSplitter::new().with_lowercase_starts(true);
+        assert_eq!(
+            claims.split(text),
+            vec!["Restart it.", "packsetd reloads the pack."]
+        );
+        assert_eq!(
+            claims.split("Use a ring, e.g. the instanton. It converges."),
+            vec!["Use a ring, e.g. the instanton.", "It converges."]
+        );
+        assert_eq!(
+            claims.split("1. first item. 2. second item."),
+            vec!["1. first item.", "2. second item."]
+        );
+    }
+
     #[test]
     fn simple_sentences() {
         assert_eq!(
@@ -8035,24 +8125,20 @@ mod tests {
         let spans = atomic_inline_spans(text);
         let tokens: Vec<&str> = spans.iter().map(|&(s, e)| &text[s..e]).collect();
         assert!(
-            tokens
-                .iter()
-                .any(|t| *t == "[the example site](https://ex.com)"),
+            tokens.contains(&"[the example site](https://ex.com)"),
             "markdown link: {tokens:?}"
         );
         assert!(
-            tokens.iter().any(|t| *t == "`some long code`"),
+            tokens.contains(&"`some long code`"),
             "inline code: {tokens:?}"
         );
-        assert!(tokens.iter().any(|t| *t == "$E = m$"), "math: {tokens:?}");
+        assert!(tokens.contains(&"$E = m$"), "math: {tokens:?}");
         assert!(
-            tokens
-                .iter()
-                .any(|t| *t == "[[https://example.com][the example site]]"),
+            tokens.contains(&"[[https://example.com][the example site]]"),
             "org link: {tokens:?}"
         );
         assert!(
-            tokens.iter().any(|t| *t == "<https://ex.com/a>"),
+            tokens.contains(&"<https://ex.com/a>"),
             "autolink: {tokens:?}"
         );
     }