fix: support Hunspell flag aliases and lazy affix expansion

internal/check/spellfilter.go

@@ -1,41 +1,52 @@
 package check
 
+import (
+	"unicode"
+	"unicode/utf8"
+)
+
 // The default spelling filters, hand-written.
 //
 // A spelling rule runs these against every word of every block, and blocks
 // overlap -- a sentence is also part of a paragraph -- so the same word is
 // tested several times over. Profiling a spell-check of a 120 KB file put 72%
 // of the run inside regexp.(*machine).match, all of it here.
 //
-// The three patterns are simple enough to read directly, and the regexes are
-// ASCII-only, so a byte scan answers each of them exactly. Order matters:
+// The three patterns are simple enough to read directly. Order matters:
 // skipsNonWord is both the cheapest and the most often true, so it goes first.
 
 // skipsNonWord reports whether a word contains anything outside the pattern
-// `[^a-zA-Z_']` -- that is, whether that pattern would match.
-//
-// Bytes above ASCII count, as they do for the regex: a rune outside the class
-// is a rune the class does not contain.
+// `[^\p{L}_']` -- that is, whether that pattern would match.
 func skipsNonWord(word string) bool {
 	for i := 0; i < len(word); i++ {
-		c := word[i]
-		switch {
-		case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c == '_', c == '\'':
-		default:
+		char := word[i]
+		if isUpper(char) || isLower(char) || char == '_' || char == '\'' {
+			continue
+		}
+		if char < utf8.RuneSelf {
 			return true
 		}
+
+		// Preserve the byte-scan fast path for ASCII words, then decode only
+		// the suffix that actually contains Unicode.
+		for _, r := range word[i:] {
+			if !unicode.IsLetter(r) && r != '_' && r != '\'' {
+				return true
+			}
+		}
+		return false
 	}
 	return false
 }
 
-// skipsTrailingCaps reports whether `[A-Z]+$` would match: the word ends in at
-// least one capital.
+// skipsTrailingCaps reports whether `[\p{Lu}]+$` would match: the word ends in
+// at least one Unicode uppercase letter.
 func skipsTrailingCaps(word string) bool {
 	if word == "" {
 		return false
 	}
-	c := word[len(word)-1]
-	return c >= 'A' && c <= 'Z'
+	r, _ := utf8.DecodeLastRuneInString(word)
+	return unicode.IsUpper(r)
 }
 
 // skipsCamel reports whether `[A-Z]{1}[a-z]+[A-Z]+\w+` would match: a capital,

internal/check/spellfilter_test.go

@@ -2,29 +2,63 @@ package check
 
 import (
 	"math/rand"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/vale-cli/vale/v3/internal/core"
+	"github.com/vale-cli/vale/v3/internal/nlp"
 )
 
+// TestSkippedByDefaultUnicodeWords checks that the default filters pass
+// Unicode words to the spell checker while still skipping uppercase and
+// non-word tokens.
+func TestSkippedByDefaultUnicodeWords(t *testing.T) {
+	cases := map[string]bool{
+		"pozícióben":   false,
+		"pozícióban":   false,
+		"árvíztűrő":    false,
+		"tükörfúrógép": false,
+		"ÁRVÍZTŰRŐ":    true,
+		"TÜKÖRFÚRÓGÉP": true,
+		"célkitűzés":   false,
+		"café":         false,
+		"naïve":        false,
+		"Straße":       false,
+		"foo-bar":      true,
+		"foo.bar":      true,
+	}
+
+	for word, want := range cases {
+		if got := skippedByDefault(word); got != want {
+			t.Errorf("skippedByDefault(%q) = %v, want %v", word, got, want)
+		}
+	}
+}
+
 // TestSpellFiltersMatchRegex checks the hand-written filters against the
 // patterns they replace, over both fixed cases and random strings.
 func TestSpellFiltersMatchRegex(t *testing.T) {
 	cases := []string{
 		"", "a", "A", "hello", "Hello", "HELLO", "helloW", "CamelCase",
 		"camelCase", "XMLHttpRequest", "iOS", "IDs", "don't", "it's", "_foo",
 		"foo_bar", "foo-bar", "foo.bar", "café", "naïve", "Straße", "42",
+		"pozícióben", "pozícióban", "árvíztűrő", "tükörfúrógép",
+		"ÁRVÍZTŰRŐ", "TÜKÖRFÚRÓGÉP", "célkitűzés",
 		"a1b2", "ABCd", "aBC", "aBCd", "HTTPServer", "getHTTPResponse",
 		"McDonald", "O'Brien", "e.g", "U.S.A", "ZZ", "aZ", "AaB", "AaBc",
 	}
 	// Coverage, not secrecy: these strings are fed to two implementations of
 	// the same filter to check they agree, so a predictable generator is all
 	// that is wanted -- and a fixed seed makes a failure reproducible.
 	rng := rand.New(rand.NewSource(1)) //nolint:gosec // not security-sensitive
+	alphabet := []rune(" aAzZ_'0-.áéíóöőúüűÁÉÍÓÖŐÚÜŰß")
 	for i := 0; i < 4000; i++ {
 		n := 1 + rng.Intn(12)
 		var b strings.Builder
 		for j := 0; j < n; j++ {
-			b.WriteByte(" aAzZ_'0-.é"[rng.Intn(11)])
+			b.WriteRune(alphabet[rng.Intn(len(alphabet))])
 		}
 		cases = append(cases, b.String())
 	}
@@ -47,3 +81,51 @@ func TestSpellFiltersMatchRegex(t *testing.T) {
 		}
 	}
 }
+
+// TestSpellingRunChecksUnicodeWordsWithDefaultFilters verifies that a spelling
+// rule without `custom: true` reports a misspelled Unicode word and accepts a
+// correctly spelled one.
+func TestSpellingRunChecksUnicodeWordsWithDefaultFilters(t *testing.T) {
+	dir := t.TempDir()
+	aff := filepath.Join(dir, "hu.aff")
+	dic := filepath.Join(dir, "hu.dic")
+
+	if err := os.WriteFile(aff, []byte("SET UTF-8\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(dic, []byte("1\npozícióban\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg, err := core.NewConfig(&core.CLIFlags{IgnoreGlobal: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg.AddStylesPath(dir)
+
+	rule, err := NewSpelling(cfg, baseCheck{
+		"name":    "Test.Spelling",
+		"extends": "spelling",
+		"message": "Did you really mean '%s'?",
+		"level":   "error",
+		"aff":     filepath.Base(aff),
+		"dic":     filepath.Base(dic),
+	}, filepath.Join(dir, "Spelling.yml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rule.Custom || !rule.stdFilters {
+		t.Fatal("test rule must exercise the default filters without custom: true")
+	}
+
+	alerts, err := rule.Run(nlp.NewBlock("", "pozícióben pozícióban", "text"), nil, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(alerts) != 1 {
+		t.Fatalf("got %d alerts, want 1: %#v", len(alerts), alerts)
+	}
+	if got := alerts[0].Match; got != "pozícióben" {
+		t.Errorf("alert matched %q, want %q", got, "pozícióben")
+	}
+}

internal/check/spelling.go

@@ -19,8 +19,8 @@ import (
 
 var defaultFilters = []*regexp.Regexp{
 	regexp.MustCompile(`[A-Z]{1}[a-z]+[A-Z]+\w+`),
-	regexp.MustCompile(`[A-Z]+$`),
-	regexp.MustCompile(`[^a-zA-Z_']`),
+	regexp.MustCompile(`[\p{Lu}]+$`),
+	regexp.MustCompile(`[^\p{L}_']`),
 }
 
 // Spelling checks text against a Hunspell dictionary.