From 4c0077c9a65e8ac307ed2ad4be5d6795048e3428 Mon Sep 17 00:00:00 2001 From: Ferenc Magnucz <7318601+fmagnucz@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:37:03 +0200 Subject: [PATCH 1/2] fix: support Hunspell flag aliases and lazy affix expansion --- internal/check/spellfilter.go | 39 +- internal/check/spellfilter_test.go | 84 +++- internal/check/spelling.go | 4 +- internal/spell/aff.go | 428 +++++++++++++++++--- internal/spell/aff_test.go | 618 +++++++++++++++++++++++++++++ internal/spell/gospell.go | 323 +++++++++++++-- internal/spell/lazy.go | 143 +++++++ internal/spell/multi.go | 4 +- testdata/e2e/checks.yaml | 1 + 9 files changed, 1545 insertions(+), 99 deletions(-) create mode 100644 internal/spell/lazy.go diff --git a/internal/check/spellfilter.go b/internal/check/spellfilter.go index f29895a2..41b6a141 100644 --- a/internal/check/spellfilter.go +++ b/internal/check/spellfilter.go @@ -1,5 +1,10 @@ package check +import ( + "unicode" + "unicode/utf8" +) + // The default spelling filters, hand-written. // // A spelling rule runs these against every word of every block, and blocks @@ -7,35 +12,41 @@ package check // tested several times over. Profiling a spell-check of a 120 KB file put 72% // of the run inside regexp.(*machine).match, all of it here. // -// The three patterns are simple enough to read directly, and the regexes are -// ASCII-only, so a byte scan answers each of them exactly. Order matters: +// The three patterns are simple enough to read directly. Order matters: // skipsNonWord is both the cheapest and the most often true, so it goes first. // skipsNonWord reports whether a word contains anything outside the pattern -// `[^a-zA-Z_']` -- that is, whether that pattern would match. -// -// Bytes above ASCII count, as they do for the regex: a rune outside the class -// is a rune the class does not contain. +// `[^\p{L}_']` -- that is, whether that pattern would match. func skipsNonWord(word string) bool { for i := 0; i < len(word); i++ { - c := word[i] - switch { - case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c == '_', c == '\'': - default: + char := word[i] + if isUpper(char) || isLower(char) || char == '_' || char == '\'' { + continue + } + if char < utf8.RuneSelf { return true } + + // Preserve the byte-scan fast path for ASCII words, then decode only + // the suffix that actually contains Unicode. + for _, r := range word[i:] { + if !unicode.IsLetter(r) && r != '_' && r != '\'' { + return true + } + } + return false } return false } -// skipsTrailingCaps reports whether `[A-Z]+$` would match: the word ends in at -// least one capital. +// skipsTrailingCaps reports whether `[\p{Lu}]+$` would match: the word ends in +// at least one Unicode uppercase letter. func skipsTrailingCaps(word string) bool { if word == "" { return false } - c := word[len(word)-1] - return c >= 'A' && c <= 'Z' + r, _ := utf8.DecodeLastRuneInString(word) + return unicode.IsUpper(r) } // skipsCamel reports whether `[A-Z]{1}[a-z]+[A-Z]+\w+` would match: a capital, diff --git a/internal/check/spellfilter_test.go b/internal/check/spellfilter_test.go index f5a9eeda..bfb4de7f 100644 --- a/internal/check/spellfilter_test.go +++ b/internal/check/spellfilter_test.go @@ -2,10 +2,41 @@ package check import ( "math/rand" + "os" + "path/filepath" "strings" "testing" + + "github.com/vale-cli/vale/v3/internal/core" + "github.com/vale-cli/vale/v3/internal/nlp" ) +// TestSkippedByDefaultUnicodeWords checks that the default filters pass +// Unicode words to the spell checker while still skipping uppercase and +// non-word tokens. +func TestSkippedByDefaultUnicodeWords(t *testing.T) { + cases := map[string]bool{ + "pozícióben": false, + "pozícióban": false, + "árvíztűrő": false, + "tükörfúrógép": false, + "ÁRVÍZTŰRŐ": true, + "TÜKÖRFÚRÓGÉP": true, + "célkitűzés": false, + "café": false, + "naïve": false, + "Straße": false, + "foo-bar": true, + "foo.bar": true, + } + + for word, want := range cases { + if got := skippedByDefault(word); got != want { + t.Errorf("skippedByDefault(%q) = %v, want %v", word, got, want) + } + } +} + // TestSpellFiltersMatchRegex checks the hand-written filters against the // patterns they replace, over both fixed cases and random strings. func TestSpellFiltersMatchRegex(t *testing.T) { @@ -13,6 +44,8 @@ func TestSpellFiltersMatchRegex(t *testing.T) { "", "a", "A", "hello", "Hello", "HELLO", "helloW", "CamelCase", "camelCase", "XMLHttpRequest", "iOS", "IDs", "don't", "it's", "_foo", "foo_bar", "foo-bar", "foo.bar", "café", "naïve", "Straße", "42", + "pozícióben", "pozícióban", "árvíztűrő", "tükörfúrógép", + "ÁRVÍZTŰRŐ", "TÜKÖRFÚRÓGÉP", "célkitűzés", "a1b2", "ABCd", "aBC", "aBCd", "HTTPServer", "getHTTPResponse", "McDonald", "O'Brien", "e.g", "U.S.A", "ZZ", "aZ", "AaB", "AaBc", } @@ -20,11 +53,12 @@ func TestSpellFiltersMatchRegex(t *testing.T) { // the same filter to check they agree, so a predictable generator is all // that is wanted -- and a fixed seed makes a failure reproducible. rng := rand.New(rand.NewSource(1)) //nolint:gosec // not security-sensitive + alphabet := []rune(" aAzZ_'0-.áéíóöőúüűÁÉÍÓÖŐÚÜŰß") for i := 0; i < 4000; i++ { n := 1 + rng.Intn(12) var b strings.Builder for j := 0; j < n; j++ { - b.WriteByte(" aAzZ_'0-.é"[rng.Intn(11)]) + b.WriteRune(alphabet[rng.Intn(len(alphabet))]) } cases = append(cases, b.String()) } @@ -47,3 +81,51 @@ func TestSpellFiltersMatchRegex(t *testing.T) { } } } + +// TestSpellingRunChecksUnicodeWordsWithDefaultFilters verifies that a spelling +// rule without `custom: true` reports a misspelled Unicode word and accepts a +// correctly spelled one. +func TestSpellingRunChecksUnicodeWordsWithDefaultFilters(t *testing.T) { + dir := t.TempDir() + aff := filepath.Join(dir, "hu.aff") + dic := filepath.Join(dir, "hu.dic") + + if err := os.WriteFile(aff, []byte("SET UTF-8\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dic, []byte("1\npozícióban\n"), 0o600); err != nil { + t.Fatal(err) + } + + cfg, err := core.NewConfig(&core.CLIFlags{IgnoreGlobal: true}) + if err != nil { + t.Fatal(err) + } + cfg.AddStylesPath(dir) + + rule, err := NewSpelling(cfg, baseCheck{ + "name": "Test.Spelling", + "extends": "spelling", + "message": "Did you really mean '%s'?", + "level": "error", + "aff": filepath.Base(aff), + "dic": filepath.Base(dic), + }, filepath.Join(dir, "Spelling.yml")) + if err != nil { + t.Fatal(err) + } + if rule.Custom || !rule.stdFilters { + t.Fatal("test rule must exercise the default filters without custom: true") + } + + alerts, err := rule.Run(nlp.NewBlock("", "pozícióben pozícióban", "text"), nil, cfg) + if err != nil { + t.Fatal(err) + } + if len(alerts) != 1 { + t.Fatalf("got %d alerts, want 1: %#v", len(alerts), alerts) + } + if got := alerts[0].Match; got != "pozícióben" { + t.Errorf("alert matched %q, want %q", got, "pozícióben") + } +} diff --git a/internal/check/spelling.go b/internal/check/spelling.go index 3959d2cc..52589dce 100644 --- a/internal/check/spelling.go +++ b/internal/check/spelling.go @@ -19,8 +19,8 @@ import ( var defaultFilters = []*regexp.Regexp{ regexp.MustCompile(`[A-Z]{1}[a-z]+[A-Z]+\w+`), - regexp.MustCompile(`[A-Z]+$`), - regexp.MustCompile(`[^a-zA-Z_']`), + regexp.MustCompile(`[\p{Lu}]+$`), + regexp.MustCompile(`[^\p{L}_']`), } // Spelling checks text against a Hunspell dictionary. diff --git a/internal/spell/aff.go b/internal/spell/aff.go index 40b1963d..13e94d68 100644 --- a/internal/spell/aff.go +++ b/internal/spell/aff.go @@ -69,6 +69,35 @@ type rule struct { matcher *regexp.Regexp // matcher to see if this rule applies or not } +// hunspellConditionPattern converts the part of Hunspell's affix-condition +// syntax that differs from Go regular expressions. A '-' inside a Hunspell +// character group is always literal; it never introduces a range. Encoding it +// as a hexadecimal escape preserves that meaning when regexp compiles it. +func hunspellConditionPattern(condition string, atype affixType) string { + var pattern strings.Builder + insideGroup := false + + for _, char := range condition { + switch char { + case '[': + insideGroup = true + case ']': + insideGroup = false + case '-': + if insideGroup { + pattern.WriteString(`\x2d`) + continue + } + } + pattern.WriteRune(char) + } + + if atype == Prefix { + return "^" + pattern.String() + } + return pattern.String() + "$" +} + // dictConfig is a partial representation of a Hunspell AFF (Affix) file. const ( // defaultCompoundMin is Hunspell's own default for COMPOUNDMIN. @@ -78,12 +107,15 @@ const ( maxCompoundMin = 100 // maxCompoundRules caps what a COMPOUNDRULE count may preallocate. maxCompoundRules = 1 << 16 + // maxFlagAliasCapacity caps only the initial allocation for an AF table. + maxFlagAliasCapacity = 1 << 14 ) type dictConfig struct { IconvReplacements []string Replacements [][2]string CompoundRule []string + FlagAliases []string Flag string TryChars string WordChars string @@ -110,7 +142,7 @@ func (a *dictConfig) compoundingEnabled() bool { // parseFlags splits a flag string into individual flags based on the FLAG type. // // Hunspell supports several flag formats: -// - "ASCII" (default): each character is a flag +// - "ASCII" (default): each byte is a flag // - "num": flags are comma-separated numbers (e.g., "14308,10482,4720") // - "UTF-8": each UTF-8 character is a flag // - "long": each pair of ASCII characters is a flag @@ -124,37 +156,241 @@ func (a dictConfig) parseFlags(flagStr string) []string { flags = append(flags, flagStr[i:i+2]) } return flags - default: // "ASCII" or "UTF-8" + case "UTF-8": flags := make([]string, 0, len(flagStr)) for _, r := range flagStr { flags = append(flags, string(r)) } return flags + default: // "ASCII", Hunspell's default extended 8-bit format. + flags := make([]string, 0, len(flagStr)) + for i := range len(flagStr) { + flags = append(flags, flagStr[i:i+1]) + } + return flags + } +} + +// parseSingleFlag decodes directives that name exactly one flag. In the +// default 8-bit mode this intentionally returns the first byte even when the +// AFF file itself is UTF-8 encoded. Hunspell applies the same byte identifier +// to AFF class names and DIC flag vectors. +func (a dictConfig) parseSingleFlag(flagStr string) (string, error) { + flags := a.parseFlags(flagStr) + if len(flags) == 0 || flags[0] == "" { + return "", fmt.Errorf("empty flag") } + return flags[0], nil } // expand expands a word/affix using dictionary/affix rules // -// This also supports CompoundRule flags +// This is the dictionary-entry expansion path, so an AF table makes the text +// after the slash a one-based alias index. +// +// This also supports CompoundRule flags. func (a dictConfig) expand(wordAffix string, out []string) ([]string, error) { - return a.expandDepth(wordAffix, out, 0) + word, flags, err := a.dictionaryEntry(wordAffix) + if err != nil { + return nil, err + } + return a.expandDepth(word, flags, out, 0) } -// expandDepth is expand, tracking how many continuation classes deep it is. -func (a dictConfig) expandDepth(wordAffix string, out []string, depth int) ([]string, error) { - out = out[:0] +// dictionaryEntry separates a dictionary root from its normalized flag +// vector. AF aliases apply only here, at the original .dic entry. +func (a dictConfig) dictionaryEntry(wordAffix string) (string, string, error) { idx := strings.Index(wordAffix, "/") - - // not found if idx == -1 { - out = append(out, wordAffix) - return out, nil + return wordAffix, "", nil } if idx == 0 || idx+1 == len(wordAffix) { - return nil, fmt.Errorf("slash char found in first or last position") + return "", "", fmt.Errorf("slash char found in first or last position") + } + + word, flags := wordAffix[:idx], wordAffix[idx+1:] + return word, a.resolveFlagAlias(flags), nil +} + +// resolveFlagAlias expands the one-based AF alias used by both dictionary +// entries and affix continuation classes. Hunspell treats an invalid alias as +// an empty flag vector, leaving the generated word without further affixes. +func (a dictConfig) resolveFlagAlias(flags string) string { + if len(a.FlagAliases) == 0 { + return flags + } + + aliasIndex, err := strconv.ParseInt(flags, 10, 64) + if err != nil || aliasIndex < 1 || aliasIndex > int64(len(a.FlagAliases)) { + return "" + } + return a.FlagAliases[aliasIndex-1] +} + +// expandDepth is expand, tracking how many continuation classes deep it is. +func (a dictConfig) expandDepth(word, keyString string, out []string, depth int) ([]string, error) { + out = out[:0] + _, err := a.walkDepth(word, keyString, depth, func(generated string) bool { + out = append(out, generated) + return false + }) + return out, err +} + +// expandsTo reports whether an entry can generate target. It shares the exact +// forward traversal used by expand, but stops at the first match and does not +// retain unrelated forms. +func (a dictConfig) expandsTo(word, keyString, target string) bool { + return a.expandsToWithin(word, keyString, target, nil) +} + +// expandsToWithin reports whether an entry can generate target while limiting +// continuation traversal to forms in allowed. A nil set permits every form. +func (a dictConfig) expandsToWithin( + word, keyString, target string, + allowed map[string]struct{}, +) bool { + found, _ := a.walkDepthWithin(word, keyString, 0, allowed, func(generated string) bool { + return generated == target + }) + return found +} + +// hasFlaggedForm reports whether target occurs at an expansion state carrying +// flag. COMPOUNDRULE groups are populated from exactly these states in the +// eager implementation, including continuation-produced forms. +func (a dictConfig) hasFlaggedForm(word, keyString, target, flag string) bool { + return a.hasFlaggedFormWithin(word, keyString, target, flag, nil) +} + +// hasFlaggedFormWithin reports whether target occurs at an expansion state +// carrying flag while limiting continuation traversal to forms in allowed. +func (a dictConfig) hasFlaggedFormWithin( + word, keyString, target, flag string, + allowed map[string]struct{}, +) bool { + return a.hasFlaggedFormDepth(word, keyString, target, flag, 0, allowed) +} + +// hasFlaggedFormDepth recursively searches affix and continuation states for +// target carrying wanted, stopping after the configured continuation depth. +func (a dictConfig) hasFlaggedFormDepth( + word, keyString, target, wanted string, + depth int, + allowed map[string]struct{}, +) bool { + flags := a.parseFlags(keyString) + if word == target && containsFlag(flags, wanted) { + return true + } + + for _, flag := range flags { + if flag == a.CompoundOnly { + return false + } + } + + prefixes := make([]affix, 0, 5) + suffixes := make([]affix, 0, 5) + for _, flag := range flags { + class, found := a.AffixMap[flag] + if !found { + continue + } + if !class.CrossProduct { + if a.formsHaveFlag(class.forms(word), target, wanted, depth, allowed) { + return true + } + continue + } + if class.Type == Prefix { + prefixes = append(prefixes, class) + } else { + suffixes = append(suffixes, class) + } + } + + for _, suffix := range suffixes { + if a.formsHaveFlag(suffix.forms(word), target, wanted, depth, allowed) { + return true + } + } + for _, prefix := range prefixes { + prefixForms := prefix.forms(word) + if a.formsHaveFlag(prefixForms, target, wanted, depth, allowed) { + return true + } + for _, suffix := range suffixes { + for _, prefixForm := range prefixForms { + if !wordAllowed(prefixForm.Word, allowed) { + continue + } + if a.formsHaveFlag(suffix.forms(prefixForm.Word), target, wanted, depth, allowed) { + return true + } + } + } + } + return false +} + +// formsHaveFlag reports whether one of forms, or one of its continuation +// expansions, produces target with wanted in its continuation flags. +func (a dictConfig) formsHaveFlag( + forms []form, + target, wanted string, + depth int, + allowed map[string]struct{}, +) bool { + for _, current := range forms { + continuation := a.resolveFlagAlias(current.Cont) + if continuation == "" || depth >= maxAffixDepth { + continue + } + if current.Word == target && containsFlag(a.parseFlags(continuation), wanted) { + return true + } + if !wordAllowed(current.Word, allowed) { + continue + } + if a.hasFlaggedFormDepth(current.Word, continuation, target, wanted, depth+1, allowed) { + return true + } + } + return false +} + +// containsFlag reports whether wanted occurs in flags. +func containsFlag(flags []string, wanted string) bool { + for _, flag := range flags { + if flag == wanted { + return true + } + } + return false +} + +// walkDepth visits the forms of one dictionary or continuation entry. A true +// visitor result stops the traversal. +func (a dictConfig) walkDepth( + word, keyString string, + depth int, + visit func(string) bool, +) (bool, error) { + return a.walkDepthWithin(word, keyString, depth, nil, visit) +} + +// walkDepthWithin visits generated forms while limiting continuation and +// cross-product traversal to forms in allowed. A nil set permits every form. +func (a dictConfig) walkDepthWithin( + word, keyString string, + depth int, + allowed map[string]struct{}, + visit func(string) bool, +) (bool, error) { + if keyString == "" { + return visit(word), nil } - // safe - word, keyString := wordAffix[:idx], wordAffix[idx+1:] flags := a.parseFlags(keyString) @@ -166,19 +402,15 @@ func (a dictConfig) expandDepth(wordAffix string, out []string, depth int) ([]st compoundOnly = true continue } - if _, ok := a.compoundMap[key]; !ok { - // the isn't a compound flag - continue - } - // is a compound flag - a.compoundMap[key] = append(a.compoundMap[key], word) } if compoundOnly { - return out, nil + return false, nil } - out = append(out, word) + if visit(word) { + return true, nil + } prefixes := make([]affix, 0, 5) suffixes := make([]affix, 0, 5) for _, key := range flags { @@ -187,7 +419,9 @@ func (a dictConfig) expandDepth(wordAffix string, out []string, depth int) ([]st continue } if !af.CrossProduct { - out = a.appendForms(af.forms(word), out, depth) + if a.walkForms(af.forms(word), depth, allowed, visit) { + return true, nil + } continue } if af.Type == Prefix { @@ -199,20 +433,29 @@ func (a dictConfig) expandDepth(wordAffix string, out []string, depth int) ([]st // expand all suffixes with out any prefixes for _, suf := range suffixes { - out = a.appendForms(suf.forms(word), out, depth) + if a.walkForms(suf.forms(word), depth, allowed, visit) { + return true, nil + } } for _, pre := range prefixes { prewords := pre.forms(word) - out = a.appendForms(prewords, out, depth) + if a.walkForms(prewords, depth, allowed, visit) { + return true, nil + } // now do cross product for _, suf := range suffixes { for _, w := range prewords { - out = a.appendForms(suf.forms(w.Word), out, depth) + if !wordAllowed(w.Word, allowed) { + continue + } + if a.walkForms(suf.forms(w.Word), depth, allowed, visit) { + return true, nil + } } } } - return out, nil + return false, nil } // maxAffixDepth bounds how many times a continuation class may be followed. @@ -223,34 +466,51 @@ func (a dictConfig) expandDepth(wordAffix string, out []string, depth int) ([]st // continue to itself, and following that faithfully would not terminate. const maxAffixDepth = 2 -// appendForms adds each generated form to out, then follows any continuation -// flags it carries. +// walkForms visits each generated form, then follows any continuation flags it +// carries. // // This is the step Hunspell calls twofold affixation: `SFX 1 0 t/34,22 e` says // that after the rule builds its form, classes 34 and 22 apply to *that*. Not // following them leaves the further-inflected words unrecognized, which reads // to a user as their own dictionary not knowing an ordinary word -- most // visibly in Danish, Dutch and Hungarian, where inflection is built this way. -func (a dictConfig) appendForms(forms []form, out []string, depth int) []string { +func (a dictConfig) walkForms( + forms []form, + depth int, + allowed map[string]struct{}, + visit func(string) bool, +) bool { for _, f := range forms { - out = append(out, f.Word) - if f.Cont == "" || depth >= maxAffixDepth { + if visit(f.Word) { + return true + } + continuation := a.resolveFlagAlias(f.Cont) + if continuation == "" || depth >= maxAffixDepth { continue } - // The continuation is expressed exactly like a dictionary entry, so - // it is expanded as one. - more, err := a.expandDepth(f.Word+"/"+f.Cont, nil, depth+1) - if err != nil { + if !wordAllowed(f.Word, allowed) { continue } - // expandDepth re-emits the word it was given; it is already in out. - for _, w := range more { - if w != f.Word { - out = append(out, w) - } + // walkDepth re-emits the word it was given. Filter every equal form as + // appendForms historically did because it is already visited above. + found, _ := a.walkDepthWithin(f.Word, continuation, depth+1, allowed, func(word string) bool { + return word != f.Word && visit(word) + }) + if found { + return true } } - return out + return false +} + +// wordAllowed reports whether word is in allowed, treating a nil set as +// unrestricted. +func wordAllowed(word string, allowed map[string]struct{}) bool { + if allowed == nil { + return true + } + _, found := allowed[word] + return found } // allDigits reports whether s is non-empty and contains only ASCII digits. It @@ -286,6 +546,9 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen compoundMap: make(map[string][]string), CompoundMin: defaultCompoundMin, } + // A negative value means that no AF table header has been seen. Keep the + // expected size separately because an alias vector may itself be numeric. + var flagAliasExpected int64 = -1 scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() @@ -343,7 +606,11 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen if len(parts) < 2 { return nil, fmt.Errorf("ONLYINCOMPOUND stanza had %d fields, expected 2", len(parts)) } - aff.CompoundOnly = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("ONLYINCOMPOUND stanza had invalid flag %q", parts[1]) + } + aff.CompoundOnly = flag case "COMPOUNDRULE": if len(parts) < 2 { return nil, fmt.Errorf("COMPOUNDRULE stanza had %d fields, expected 2", len(parts)) @@ -366,22 +633,42 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen if len(parts) < 2 { return nil, fmt.Errorf("NOSUGGEST stanza had %d fields, expected 2", len(parts)) } - aff.NoSuggestFlag = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("NOSUGGEST stanza had invalid flag %q", parts[1]) + } + aff.NoSuggestFlag = flag case "COMPOUNDFLAG": if len(parts) >= 2 { - aff.CompoundFlag = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("COMPOUNDFLAG stanza had invalid flag %q", parts[1]) + } + aff.CompoundFlag = flag } case "COMPOUNDBEGIN": if len(parts) >= 2 { - aff.CompoundBegin = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("COMPOUNDBEGIN stanza had invalid flag %q", parts[1]) + } + aff.CompoundBegin = flag } case "COMPOUNDMIDDLE": if len(parts) >= 2 { - aff.CompoundMiddle = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("COMPOUNDMIDDLE stanza had invalid flag %q", parts[1]) + } + aff.CompoundMiddle = flag } case "COMPOUNDEND": if len(parts) >= 2 { - aff.CompoundEnd = parts[1] + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("COMPOUNDEND stanza had invalid flag %q", parts[1]) + } + aff.CompoundEnd = flag } case "WORDCHARS": if len(parts) < 2 { @@ -393,11 +680,41 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen return nil, fmt.Errorf("FLAG stanza had %d, expected 1", len(parts)) } aff.Flag = parts[1] + case "AF": + if len(parts) < 2 { + return nil, fmt.Errorf("AF stanza had %d fields, expected at least 2", len(parts)) + } + + if flagAliasExpected < 0 { + count, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil || count < 1 { + return nil, fmt.Errorf("AF stanza had %q, expected positive number", parts[1]) + } + flagAliasExpected = count + // The count comes from the file, so cap only the initial + // allocation. append grows the slice if a larger table is valid. + aff.FlagAliases = make([]string, 0, int(min(count, int64(maxFlagAliasCapacity)))) + continue + } + + if int64(len(aff.FlagAliases)) >= flagAliasExpected { + return nil, fmt.Errorf("AF table had more than %d entries", flagAliasExpected) + } + // Keep the vector in its original encoding. It is decoded according + // to FLAG only when a dictionary entry refers to this alias. + aff.FlagAliases = append(aff.FlagAliases, parts[1]) case "PFX", "SFX": atype := Prefix if parts[0] == "SFX" { atype = Suffix } + if len(parts) < 2 { + return nil, fmt.Errorf("%s stanza had %d fields, expected at least 2", parts[0], len(parts)) + } + flag, err := aff.parseSingleFlag(parts[1]) + if err != nil { + return nil, fmt.Errorf("%s stanza had invalid flag %q", parts[0], parts[1]) + } sections := len(parts) // A header line is `PFX/SFX flag Y|N count`; a rule line is @@ -414,12 +731,11 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen return nil, err } // this is a new Affix! - aff.AffixMap[parts[1]] = affix{ + aff.AffixMap[flag] = affix{ Type: atype, CrossProduct: cross, } case sections >= 4: - flag := parts[1] a, ok := aff.AffixMap[flag] if !ok { return nil, fmt.Errorf("got rules for flag %q but no definition", flag) @@ -440,12 +756,7 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen var matcher *regexp.Regexp var err error if cond != "." { - pat := cond - if a.Type == Prefix { - pat = "^" + pat - } else { - pat += "$" - } + pat := hunspellConditionPattern(cond, a.Type) matcher, err = regexp.Compile(pat) if err != nil { return nil, fmt.Errorf("unable to compile %s", pat) @@ -490,6 +801,11 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen if err := scanner.Err(); err != nil { return nil, err } + if flagAliasExpected >= 0 && int64(len(aff.FlagAliases)) != flagAliasExpected { + return nil, fmt.Errorf( + "AF table had %d entries, expected %d", + len(aff.FlagAliases), flagAliasExpected) + } return &aff, nil } diff --git a/internal/spell/aff_test.go b/internal/spell/aff_test.go index 6b1e4ad4..cc649c1f 100644 --- a/internal/spell/aff_test.go +++ b/internal/spell/aff_test.go @@ -1,7 +1,9 @@ package spell import ( + "fmt" "strings" + "sync" "testing" "time" ) @@ -14,6 +16,17 @@ func TestParseFlagsASCII(t *testing.T) { } } +// TestParseFlagsDefaultEightBit verifies that the default flag mode treats a +// UTF-8 sequence as separate eight-bit flags. +func TestParseFlagsDefaultEightBit(t *testing.T) { + dc := dictConfig{Flag: "ASCII"} + flagStr := "\xc3\xa9" + flags := dc.parseFlags(flagStr) + if len(flags) != 2 || flags[0] != "\xc3" || flags[1] != "\xa9" { + t.Errorf("ASCII parseFlags(%q) = %q, want [\\xc3 \\xa9]", flagStr, flags) + } +} + func TestParseFlagsNum(t *testing.T) { dc := dictConfig{Flag: "num"} flags := dc.parseFlags("14308,10482,4720") @@ -38,6 +51,194 @@ func TestParseFlagsUTF8(t *testing.T) { } } +// TestFlagAliasesParsing verifies that AF directives retain their declared +// order and support Unicode flag vectors. +func TestFlagAliasesParsing(t *testing.T) { + affContent := `SET UTF-8 +FLAG UTF-8 +AF 2 +AF A +AF AŐ # second alias +` + + aff, err := newDictConfig(strings.NewReader(affContent)) + if err != nil { + t.Fatalf("newDictConfig error: %v", err) + } + + if len(aff.FlagAliases) != 2 { + t.Fatalf("FlagAliases = %v, want [A AŐ]", aff.FlagAliases) + } + if aff.FlagAliases[0] != "A" || aff.FlagAliases[1] != "AŐ" { + t.Errorf("FlagAliases = %v, want [A AŐ]", aff.FlagAliases) + } +} + +// TestFlagAliasIndexing verifies that dictionary alias numbers select the +// corresponding one-based AF entry. +func TestFlagAliasIndexing(t *testing.T) { + affContent := `SET UTF-8 +AF 2 +AF A +AF B + +SFX A N 1 +SFX A 0 s . + +SFX B N 1 +SFX B 0 ed . +` + dicContent := `2 +first/1 +second/2 +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + tests := []struct { + word string + want bool + }{ + {"first", true}, + {"firsts", true}, + {"firsted", false}, + {"second", true}, + {"seconds", false}, + {"seconded", true}, + } + + for _, tt := range tests { + if got := gs.spell(tt.word); got != tt.want { + t.Errorf("spell(%q) = %v, want %v", tt.word, got, tt.want) + } + } +} + +// TestFlagAliasWithMultipleFlags verifies that one AF alias can enable more +// than one affix class. +func TestFlagAliasWithMultipleFlags(t *testing.T) { + affContent := `SET UTF-8 +AF 1 +AF AB + +SFX A N 1 +SFX A 0 s . + +SFX B N 1 +SFX B 0 ed . +` + dicContent := `1 +root/1 +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"root", "roots", "rooted"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } +} + +// TestDefaultEightBitFlagExpansion verifies that a non-ASCII byte can identify +// an affix class in Hunspell's default flag mode. +func TestDefaultEightBitFlagExpansion(t *testing.T) { + const flag = "\xc1" + affContent := "SET UTF-8\n" + + "SFX " + flag + " N 1\n" + + "SFX " + flag + " 0 s .\n" + dicContent := "1\nword/" + flag + "\n" + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + if !gs.spell("words") { + t.Error("spell(\"words\") = false, want true") + } +} + +// TestUTF8FlagExpansion verifies that FLAG UTF-8 permits a Unicode code point +// to identify an affix class. +func TestUTF8FlagExpansion(t *testing.T) { + affContent := `SET UTF-8 +FLAG UTF-8 + +SFX Ő N 1 +SFX Ő 0 s . +` + dicContent := `1 +word/Ő +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + if !gs.spell("words") { + t.Error("spell(\"words\") = false, want true") + } +} + +// TestFlagAliasWithContinuationClass verifies that a numeric continuation is +// resolved through the AF table instead of being treated as a literal flag. +func TestFlagAliasWithContinuationClass(t *testing.T) { + affContent := `SET UTF-8 +AF 2 +AF A +AF B + +SFX A N 1 +SFX A 0 ed/2 . + +SFX B N 1 +SFX B 0 ly . + +SFX 2 N 1 +SFX 2 0 wrong . +` + dicContent := `1 +root/1 +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"root", "rooted", "rootedly"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } + if gs.spell("rootedwrong") { + t.Error("continuation alias was treated as a literal flag") + } +} + func TestFlagNumAffixParsing(t *testing.T) { // Minimal FLAG num AFF file affContent := `SET UTF-8 @@ -100,6 +301,33 @@ SFX Aa 0 s . } } +// TestDefaultFlagAffixUsesSingleByteIdentifier verifies that directives naming +// one flag use a single byte in the default flag mode. +func TestDefaultFlagAffixUsesSingleByteIdentifier(t *testing.T) { + // The Italian dictionary is UTF-8 text but uses Hunspell's default 8-bit + // flag mode. Its `d/£$` entry and `SFX £` class must therefore resolve + // through the same first-byte flag identifier, as they do in Hunspell. + affContent := `SET UTF-8 + +SFX £ Y 1 +SFX £ 0 i [ivxlcdm] +` + dicContent := `1 +d/£$ +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + if !gs.spell("di") { + t.Error("spell(\"di\") = false, want true") + } +} + func TestFlagNumExpand(t *testing.T) { affContent := `SET UTF-8 FLAG num @@ -300,6 +528,37 @@ func TestCompoundSegmentation(t *testing.T) { } } +// TestCompoundRuleIncludesContinuationForms verifies that forms produced by a +// continuation class participate in COMPOUNDRULE matching. +func TestCompoundRuleIncludesContinuationForms(t *testing.T) { + affContent := `SET UTF-8 + +COMPOUNDRULE 1 +COMPOUNDRULE CD + +SFX A N 1 +SFX A 0 s/C . +` + dicContent := `2 +root/A +word/D +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + if !gs.spell("rootsword") { + t.Error("spell(\"rootsword\") = false, want true") + } + if gs.spell("rootword") { + t.Error("spell(\"rootword\") = true, want false") + } +} + func TestConditionlessAffixRule(t *testing.T) { // OpenTaal's Dutch dictionary writes affix rules without the (optional) // condition field, e.g. `SFX CA 0 /CaCp`. A 4-field rule must not be @@ -320,6 +579,365 @@ func TestConditionlessAffixRule(t *testing.T) { } } +// TestHungarianAffixConditionsWithLiteralHyphens verifies that representative +// Hungarian character classes containing hyphens match Hunspell semantics. +func TestHungarianAffixConditionsWithLiteralHyphens(t *testing.T) { + affContent := `SET UTF-8 +FLAG UTF-8 + +SFX A N 1 +SFX A 0 x [áéiíoóőuúůüű-ø] + +SFX B N 1 +SFX B 0 y [áéiíóőuúůüű-àùø] +` + dicContent := `7 +tű/AB +tø/AB +tő/AB +tà/AB +tù/AB +ta/AB +tz/AB +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + tests := []struct { + word string + want bool + }{ + {"tűx", true}, + {"tøx", true}, + {"tőx", true}, + {"tàx", false}, + {"tùx", false}, + {"tax", false}, + {"tzx", false}, + {"tűy", true}, + {"tøy", true}, + {"tőy", true}, + {"tày", true}, + {"tùy", true}, + {"tay", false}, + {"tzy", false}, + } + + for _, tt := range tests { + if got := gs.spell(tt.word); got != tt.want { + t.Errorf("spell(%q) = %v, want %v", tt.word, got, tt.want) + } + } +} + +// TestAffixConditionDoesNotTreatHyphenAsRange verifies that a hyphen inside a +// Hunspell character class is matched literally rather than as a range. +func TestAffixConditionDoesNotTreatHyphenAsRange(t *testing.T) { + affContent := `SET UTF-8 + +SFX A N 1 +SFX A 0 s [a-z] +` + dicContent := `4 +a/A +-/A +z/A +m/A +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"as", "-s", "zs"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } + if gs.spell("ms") { + t.Error("spell(\"ms\") = true, want false") + } +} + +// TestDictionaryEntriesAreExpandedLazily verifies that derived forms are +// recognized without being materialized in the stored dictionary. +func TestDictionaryEntriesAreExpandedLazily(t *testing.T) { + const ruleCount = 40 + var affContent strings.Builder + affContent.WriteString("SET UTF-8\n\nSFX A Y 40\n") + for i := range ruleCount { + fmt.Fprintf(&affContent, "SFX A 0 a%d/B .\n", i) + } + affContent.WriteString("\nSFX B Y 40\n") + for i := range ruleCount { + fmt.Fprintf(&affContent, "SFX B 0 b%d .\n", i) + } + + gs, err := newGoSpellReader( + strings.NewReader(affContent.String()), + strings.NewReader("1\nroot/A\n"), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + if got := len(gs.dict); got != 1 { + t.Errorf("materialized dictionary size = %d, want only 1 root", got) + } + for _, word := range []string{"root", "roota0", "roota39b39"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } + if gs.spell("roota40b40") { + t.Error("spell(\"roota40b40\") = true, want false") + } +} + +// TestLazyCrossProduct verifies that lazy expansion preserves valid prefix and +// suffix cross-products. +func TestLazyCrossProduct(t *testing.T) { + affContent := `SET UTF-8 + +PFX P Y 1 +PFX P 0 re . + +SFX S Y 1 +SFX S 0 s . +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader("1\nroot/PS\n"), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"root", "reroot", "roots", "reroots"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } +} + +// TestLazyLookupAtMaximumAffixDepth verifies that continuation expansion +// reaches the configured depth limit without exceeding it. +func TestLazyLookupAtMaximumAffixDepth(t *testing.T) { + affContent := `SET UTF-8 +FLAG num + +PFX 1 Y 1 +PFX 1 0 p0 . +SFX 2 Y 1 +SFX 2 0 s0/3,4 . + +PFX 3 Y 1 +PFX 3 0 p1 . +SFX 4 Y 1 +SFX 4 0 s1/5,6 . + +PFX 5 Y 1 +PFX 5 0 p2 . +SFX 6 Y 1 +SFX 6 0 s2 . +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader("1\nroot/1,2\n"), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + if !gs.spell("p2p1p0roots0s1s2") { + t.Error("spell at maximum affix depth = false, want true") + } +} + +// TestLazyLookupPreservesSuffixStripBehavior verifies that reverse lookup +// mirrors the existing forward behavior for suffix strip rules. +func TestLazyLookupPreservesSuffixStripBehavior(t *testing.T) { + affContent := `SET UTF-8 + +SFX A N 2 +SFX A e 0 e +SFX A ing s . +` + dicContent := `2 +make/A +root/A +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"mak", "roots"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } +} + +// TestLazySuggestionIncludesDerivedWord verifies that edit-distance candidates +// generated through affixes can be returned as suggestions. +func TestLazySuggestionIncludesDerivedWord(t *testing.T) { + affContent := `SET UTF-8 + +SFX A N 1 +SFX A 0 s . +` + dicContent := `6 +root/A +alpha +bravo +charlie +delta +echo +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + found := false + for _, match := range gs.suggest("rootss") { + if match.word == "roots" { + found = true + break + } + } + if !found { + t.Error("suggest(\"rootss\") does not include derived word \"roots\"") + } +} + +// TestLazyHomographsDoNotCombineFlags verifies that separate entries for the +// same root do not combine their flags into an invalid expansion. +func TestLazyHomographsDoNotCombineFlags(t *testing.T) { + affContent := `SET UTF-8 + +PFX P Y 1 +PFX P 0 re . + +SFX S Y 1 +SFX S 0 s . +` + dicContent := `2 +root/P +root/S +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + for _, word := range []string{"root", "reroot", "roots"} { + if !gs.spell(word) { + t.Errorf("spell(%q) = false, want true", word) + } + } + if gs.spell("reroots") { + t.Error("spell(\"reroots\") = true, want false") + } +} + +// TestLazySpellConcurrent verifies that lazy lookups and their caches are safe +// when spell checks run concurrently. +func TestLazySpellConcurrent(t *testing.T) { + affContent := `SET UTF-8 + +SFX A Y 1 +SFX A 0 ed/B . + +SFX B Y 1 +SFX B 0 ly . +` + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader("1\nroot/A\n"), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + words := map[string]bool{ + "root": true, + "rooted": true, + "rootedly": true, + "rootedness": false, + } + var wait sync.WaitGroup + for range 8 { + wait.Add(1) + go func() { + defer wait.Done() + for range 100 { + for word, want := range words { + if got := gs.spell(word); got != want { + t.Errorf("spell(%q) = %v, want %v", word, got, want) + } + } + } + }() + } + wait.Wait() +} + +// TestLazySpellCacheIsBounded verifies that lazy spelling results cannot grow +// the cache beyond its configured capacity. +func TestLazySpellCacheIsBounded(t *testing.T) { + cache := newSpellCache() + for index := 0; index <= lazySpellCacheSize; index++ { + cache.set(fmt.Sprintf("word-%d", index), index%2 == 0) + } + + if got := len(cache.values); got != lazySpellCacheSize { + t.Errorf("cache size = %d, want %d", got, lazySpellCacheSize) + } + if _, found := cache.get("word-0"); found { + t.Error("oldest cache entry was not evicted") + } + if value, found := cache.get(fmt.Sprintf("word-%d", lazySpellCacheSize)); !found || !value { + t.Error("newest cache entry is missing") + } +} + +// TestSuggestionWithSmallDictionary verifies suggestion generation when the +// dictionary contains fewer entries than the requested result limit. +func TestSuggestionWithSmallDictionary(t *testing.T) { + gs, err := newGoSpellReader( + strings.NewReader("SET UTF-8\n"), + strings.NewReader("1\nword\n"), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + matches := gs.suggest("wrod") + if len(matches) != 1 || matches[0].word != "word" { + t.Errorf("suggest(\"wrod\") = %v, want word", matches) + } +} + // TestContinuationCycleTerminates covers an .aff file whose affix class // continues to itself. Nothing in the format forbids it, and following it // faithfully would not terminate, so expansion is bounded -- the point of the diff --git a/internal/spell/gospell.go b/internal/spell/gospell.go index c475b42b..8e2f443b 100644 --- a/internal/spell/gospell.go +++ b/internal/spell/gospell.go @@ -20,7 +20,14 @@ type wordMatch struct { } type goSpell struct { - dict map[string]struct{} + dict map[string]struct{} + roots map[string][]dictionaryFlags + affix *dictConfig + reverse reverseAffixIndex + cache *spellCache + compoundFlagCache *spellCache + compoundMatchCache *spellCache + lazyCompoundRules bool ireplacer *strings.Replacer compounds []*regexp.Regexp @@ -107,18 +114,92 @@ func (s *goSpell) keys() []string { func (s *goSpell) suggest(word string) []wordMatch { metric := metrics.NewLevenshtein() + matches := make([]wordMatch, 0, maxSuggestionMatches) + seen := make(map[string]struct{}) + checked := make(map[string]struct{}) + checks := 0 + consider := func(candidate string) bool { + if candidate == word { + return true + } + if _, found := checked[candidate]; found { + return true + } + if checks >= maxSuggestionChecks || len(matches) >= maxSuggestionMatches { + return false + } + checked[candidate] = struct{}{} + checks++ + if s.inLexicon(candidate) || s.inLexicon(strings.ToLower(candidate)) { + seen[candidate] = struct{}{} + matches = append(matches, wordMatch{ + word: candidate, + score: strutil.Similarity(candidate, word, metric), + }) + } + return true + } - matches := []wordMatch{} - for _, option := range s.keys() { - sim := strutil.Similarity(option, word, metric) - matches = append(matches, wordMatch{option, sim}) + runes := []rune(word) +mutationLoop: + for i := range runes { + candidate := string(append(append([]rune{}, runes[:i]...), runes[i+1:]...)) + if !consider(candidate) { + break mutationLoop + } + if i+1 < len(runes) { + swapped := append([]rune{}, runes...) + swapped[i], swapped[i+1] = swapped[i+1], swapped[i] + if !consider(string(swapped)) { + break mutationLoop + } + } + } + + tryChars := uniqueRunes(s.affix.TryChars) + for i := 0; i < len(runes) && checks < maxSuggestionChecks && len(matches) < maxSuggestionMatches; i++ { + for _, char := range tryChars { + replaced := append([]rune{}, runes...) + replaced[i] = char + if !consider(string(replaced)) { + break + } + } + } + for i := 0; i <= len(runes) && checks < maxSuggestionChecks && len(matches) < maxSuggestionMatches; i++ { + for _, char := range tryChars { + inserted := make([]rune, 0, len(runes)+1) + inserted = append(inserted, runes[:i]...) + inserted = append(inserted, char) + inserted = append(inserted, runes[i:]...) + if !consider(string(inserted)) { + break + } + } + } + + // Mutations preserve derived suggestions without materializing every + // surface. Roots remain a bounded-memory fallback for more distant typos. + if len(matches) < 5 { + for _, option := range s.keys() { + if _, found := seen[option]; found { + continue + } + matches = append(matches, wordMatch{ + word: option, + score: strutil.Similarity(option, word, metric), + }) + } } sort.Slice(matches, func(i, j int) bool { + if matches[i].score == matches[j].score { + return matches[i].word < matches[j].word + } return matches[i].score > matches[j].score }) - hits := matches[:5] + hits := matches[:min(5, len(matches))] if word == strings.Title(word) { //nolint:staticcheck // Capitalized word, so capitalize the suggestions for i := range hits { @@ -129,14 +210,31 @@ func (s *goSpell) suggest(word string) []wordMatch { return hits } +const ( + maxSuggestionChecks = 4096 + maxSuggestionMatches = 256 +) + +// uniqueRunes returns the runes in text once each, preserving first-seen order. +func uniqueRunes(text string) []rune { + seen := make(map[rune]struct{}) + result := make([]rune, 0, len(text)) + for _, char := range text { + if _, found := seen[char]; found { + continue + } + seen[char] = struct{}{} + result = append(result, char) + } + return result +} + // spell checks to see if a given word is in the internal dictionaries func (s *goSpell) spell(word string) bool { - _, ok := s.dict[word] - if ok { + if s.inLexicon(word) { return true } - _, ok = s.dict[strings.ToLower(word)] - if ok { + if s.inLexicon(strings.ToLower(word)) { return true } @@ -161,6 +259,9 @@ func (s *goSpell) spell(word string) bool { return true } } + if s.matchesLazyCompoundRule(word) { + return true + } // Affix-flag compounding (German, Dutch, ...): accept a word that splits // into dictionary segments. See #848. @@ -172,7 +273,7 @@ func (s *goSpell) spell(word string) bool { units := isNumberUnits(word) if units != "" { // dictionary appears to have list of units - if _, ok = s.dict[units]; ok { + if s.inLexicon(units) { return true } } @@ -180,18 +281,177 @@ func (s *goSpell) spell(word string) bool { return false } +// inLexicon reports whether word is an explicit dictionary entry or can be +// derived through affixes, caching derived lookups. +func (s *goSpell) inLexicon(word string) bool { + if _, ok := s.dict[word]; ok { + return true + } + if value, found := s.cache.get(word); found { + return value + } + value := s.isDerived(word) + s.cache.set(word, value) + return value +} + +// isDerived reports whether an indexed dictionary root can generate target. +func (s *goSpell) isDerived(target string) bool { + return s.matchesRoot(target, func( + root string, + entry dictionaryFlags, + allowed map[string]struct{}, + ) bool { + return s.affix.expandsToWithin(root, entry.text, target, allowed) + }) +} + +// matchesRoot walks reverse-affix candidates for target and invokes match for +// every dictionary root found along the way. +func (s *goSpell) matchesRoot( + target string, + match func(string, dictionaryFlags, map[string]struct{}) bool, +) bool { + current := map[string]struct{}{target: {}} + tested := make(map[string]struct{}) + for depth := 0; depth <= maxReverseAffixes; depth++ { + next := make(map[string]struct{}) + for candidate := range current { + if _, seen := tested[candidate]; !seen { + tested[candidate] = struct{}{} + for _, entry := range s.roots[candidate] { + if match(candidate, entry, tested) { + return true + } + } + } + if depth < maxReverseAffixes { + s.reverse.predecessors(candidate, next) + } + } + current = next + } + return false +} + +// hasCompoundFlag reports whether word can be generated in a state carrying +// flag, caching results separately for each word-and-flag pair. +func (s *goSpell) hasCompoundFlag(word, flag string) bool { + cacheKey := flag + "\x00" + word + if value, found := s.compoundFlagCache.get(cacheKey); found { + return value + } + value := s.matchesRoot(word, func( + root string, + entry dictionaryFlags, + allowed map[string]struct{}, + ) bool { + return s.affix.hasFlaggedFormWithin(root, entry.text, word, flag, allowed) + }) + s.compoundFlagCache.set(cacheKey, value) + return value +} + +// matchesLazyCompoundRule reports whether word satisfies any COMPOUNDRULE that +// depends on lazily generated affix forms. +func (s *goSpell) matchesLazyCompoundRule(word string) bool { + if !s.lazyCompoundRules { + return false + } + if value, found := s.compoundMatchCache.get(word); found { + return value + } + + boundaries := []int{0} + for index := range word { + if index > 0 { + boundaries = append(boundaries, index) + } + } + boundaries = append(boundaries, len(word)) + for _, compoundRule := range s.affix.CompoundRule { + var pattern strings.Builder + pattern.WriteByte('^') + for _, flag := range s.affix.parseFlags(compoundRule) { + if len(flag) == 1 && strings.ContainsRune("()+?*", rune(flag[0])) { + // Preserve the existing COMPOUNDRULE regexp construction: these + // characters are treated as literal rule tokens by Vale. + pattern.WriteString(regexp.QuoteMeta(flag)) + continue + } + + alternatives := make(map[string]struct{}) + for start := 0; start+1 < len(boundaries); start++ { + for end := start + 1; end < len(boundaries); end++ { + part := word[boundaries[start]:boundaries[end]] + if s.hasCompoundFlag(part, flag) { + alternatives[part] = struct{}{} + } + } + } + pattern.WriteString("(?:") + if len(alternatives) == 0 { + // A NUL cannot occur in a token produced by Vale's splitters. + pattern.WriteString(`\x00`) + } else { + parts := make([]string, 0, len(alternatives)) + for part := range alternatives { + parts = append(parts, part) + } + sort.Slice(parts, func(i, j int) bool { + return len(parts[i]) > len(parts[j]) + }) + for index, part := range parts { + if index > 0 { + pattern.WriteByte('|') + } + pattern.WriteString(regexp.QuoteMeta(part)) + } + } + pattern.WriteByte(')') + } + pattern.WriteByte('$') + compiled, err := regexp.Compile(pattern.String()) + if err == nil && compiled.MatchString(word) { + s.compoundMatchCache.set(word, true) + return true + } + } + s.compoundMatchCache.set(word, false) + return false +} + +// hasLazyCompoundForms reports whether an affix continuation can attach a flag +// referenced by a COMPOUNDRULE. +func hasLazyCompoundForms(affix *dictConfig) bool { + if len(affix.CompoundRule) == 0 { + return false + } + for _, class := range affix.AffixMap { + for _, current := range class.Rules { + continuation := affix.resolveFlagAlias(current.Cont) + for _, flag := range affix.parseFlags(continuation) { + if _, found := affix.compoundMap[flag]; found { + return true + } + } + } + } + return false +} + // inDict reports whether word is a dictionary entry, trying its exact, // lower-cased, and title-cased forms. The latter two matter for compound // segments: e.g. a German compound writes interior nouns lower-case, while the // dictionary stores them capitalized. func (s *goSpell) inDict(word string) bool { - if _, ok := s.dict[word]; ok { + if s.inLexicon(word) { return true } - if _, ok := s.dict[strings.ToLower(word)]; ok { + if s.inLexicon(strings.ToLower(word)) { return true } - if _, ok := s.dict[capitalize(word)]; ok { + if s.inLexicon(capitalize(word)) { return true } return false @@ -263,14 +523,20 @@ func newGoSpellReader(aff, dic io.Reader) (*goSpell, error) { gs := goSpell{ // TODO: Use fixed size from first list? - dict: make(map[string]struct{}), - compounds: make([]*regexp.Regexp, 0, len(affix.CompoundRule)), - splitter: newSplitter(affix.WordChars), - canCompound: affix.compoundingEnabled(), - compoundMin: affix.CompoundMin, + dict: make(map[string]struct{}), + roots: make(map[string][]dictionaryFlags), + affix: affix, + reverse: newReverseAffixIndex(affix.AffixMap), + cache: newSpellCache(), + compoundFlagCache: newSpellCache(), + compoundMatchCache: newSpellCache(), + lazyCompoundRules: hasLazyCompoundForms(affix), + compounds: make([]*regexp.Regexp, 0, len(affix.CompoundRule)), + splitter: newSplitter(affix.WordChars), + canCompound: affix.compoundingEnabled(), + compoundMin: affix.CompoundMin, } - words := []string{} for scanner.Scan() { line := scanner.Text() // A .dic entry is `word/flags` optionally followed by whitespace- @@ -290,7 +556,8 @@ func newGoSpellReader(aff, dic io.Reader) (*goSpell, error) { } line = fields[0] - words, err = affix.expand(line, words) + word, flags, entryErr := affix.dictionaryEntry(line) + err = entryErr if err != nil { // Skip malformed entries (e.g., a line with flags but no word) // rather than abandoning the entire dictionary, which would leave @@ -298,11 +565,17 @@ func newGoSpellReader(aff, dic io.Reader) (*goSpell, error) { continue } - if len(words) == 0 { - continue + gs.roots[word] = append(gs.roots[word], dictionaryFlags{text: flags}) + compoundOnly := false + for _, flag := range affix.parseFlags(flags) { + if flag == affix.CompoundOnly { + compoundOnly = true + } + if _, found := affix.compoundMap[flag]; found { + affix.compoundMap[flag] = append(affix.compoundMap[flag], word) + } } - - for _, word := range words { + if !compoundOnly { gs.dict[word] = struct{}{} } } diff --git a/internal/spell/lazy.go b/internal/spell/lazy.go new file mode 100644 index 00000000..0f222930 --- /dev/null +++ b/internal/spell/lazy.go @@ -0,0 +1,143 @@ +package spell + +import ( + "sort" + "strings" + "sync" +) + +// dictionaryFlags is one normalized flag vector attached to a root. Keeping +// homographic entries separate prevents flags from unrelated entries from +// forming an invalid cross-product. +type dictionaryFlags struct { + text string +} + +type reverseAffixIndex struct { + prefixes map[string][]rule + suffixes map[string][]rule + prefixLengths []int + suffixLengths []int +} + +// newReverseAffixIndex indexes affix rules by their added prefix or suffix so +// candidate roots can be recovered from a target word. +func newReverseAffixIndex(affixes map[string]affix) reverseAffixIndex { + index := reverseAffixIndex{ + prefixes: make(map[string][]rule), + suffixes: make(map[string][]rule), + } + prefixLengths := make(map[int]struct{}) + suffixLengths := make(map[int]struct{}) + for _, class := range affixes { + for _, current := range class.Rules { + if class.Type == Prefix { + index.prefixes[current.AffixText] = append(index.prefixes[current.AffixText], current) + prefixLengths[len(current.AffixText)] = struct{}{} + } else { + index.suffixes[current.AffixText] = append(index.suffixes[current.AffixText], current) + suffixLengths[len(current.AffixText)] = struct{}{} + } + } + } + for length := range prefixLengths { + index.prefixLengths = append(index.prefixLengths, length) + } + for length := range suffixLengths { + index.suffixLengths = append(index.suffixLengths, length) + } + sort.Ints(index.prefixLengths) + sort.Ints(index.suffixLengths) + return index +} + +// An expandDepth invocation can apply one prefix and one suffix. The initial +// invocation and each permitted continuation depth may therefore contribute +// two affixes to a generated form. +const maxReverseAffixes = 2 * (maxAffixDepth + 1) + +// predecessors adds every plausible one-affix predecessor of word to out. +func (i reverseAffixIndex) predecessors(word string, out map[string]struct{}) { + for _, length := range i.prefixLengths { + if length > len(word) { + break + } + text := word[:length] + for _, current := range i.prefixes[text] { + candidate := word[length:] + if current.matcher == nil || current.matcher.MatchString(candidate) { + out[candidate] = struct{}{} + } + } + } + + for _, length := range i.suffixLengths { + if length > len(word) { + break + } + text := word[len(word)-length:] + for _, current := range i.suffixes[text] { + stem := word[:len(word)-length] + if current.Strip == "" { + if current.matcher == nil || current.matcher.MatchString(stem) { + out[stem] = struct{}{} + } + continue + } + + candidate := stem + current.Strip + if current.matcher == nil || current.matcher.MatchString(candidate) { + out[candidate] = struct{}{} + } + // The existing forward path appends the affix without stripping when + // the input does not end in Strip. Preserve that behavior here. + if !strings.HasSuffix(stem, current.Strip) && + (current.matcher == nil || current.matcher.MatchString(stem)) { + out[stem] = struct{}{} + } + } + } +} + +const lazySpellCacheSize = 4096 + +type spellCache struct { + mu sync.Mutex + values map[string]bool + keys []string + nextKey int +} + +// newSpellCache creates a bounded cache for lazy spelling results. +func newSpellCache() *spellCache { + return &spellCache{ + values: make(map[string]bool, lazySpellCacheSize), + keys: make([]string, 0, lazySpellCacheSize), + } +} + +// get returns the cached result for word and whether it was present. +func (c *spellCache) get(word string) (bool, bool) { + c.mu.Lock() + defer c.mu.Unlock() + value, found := c.values[word] + return value, found +} + +// set stores value for word, evicting the oldest ring-buffer slot when full. +func (c *spellCache) set(word string, value bool) { + c.mu.Lock() + defer c.mu.Unlock() + if _, found := c.values[word]; found { + c.values[word] = value + return + } + if len(c.keys) < cap(c.keys) { + c.keys = append(c.keys, word) + } else { + delete(c.values, c.keys[c.nextKey]) + c.keys[c.nextKey] = word + c.nextKey = (c.nextKey + 1) % len(c.keys) + } + c.values[word] = value +} diff --git a/internal/spell/multi.go b/internal/spell/multi.go index 613f58d4..b09879f0 100644 --- a/internal/spell/multi.go +++ b/internal/spell/multi.go @@ -167,7 +167,9 @@ func (m *Checker) Suggest(word string) []string { return suggestions } -// Dict returns the underlying dictionary for the provided index. +// Dict returns the stored roots and explicit word-list entries for the +// provided index. Derived affix forms are resolved lazily and are not +// materialized in this map. func (m *Checker) Dict(i int) map[string]struct{} { return m.checkers[i].dict } diff --git a/testdata/e2e/checks.yaml b/testdata/e2e/checks.yaml index 7f225900..906ee09b 100644 --- a/testdata/e2e/checks.yaml +++ b/testdata/e2e/checks.yaml @@ -91,6 +91,7 @@ cases: exit: 1 want: | test.md:16:1:Vale.Spelling:Did you really mean 'gitlab'? + test.md:18:1:Vale.Spelling:Did you really mean 'オプション'? test.md:28:11:Vale.Spelling:Did you really mean 'typpo'? test.md:32:17:Vale.Spelling:Did you really mean 'gfm'? test.md:34:17:Vale.Spelling:Did you really mean 'remmark'? From 4c516daa32e7300751c8afc605a7b9bea99caca5 Mon Sep 17 00:00:00 2001 From: Ferenc Magnucz <7318601+fmagnucz@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:50:42 +0200 Subject: [PATCH 2/2] fix: handle Hunspell zero affixes and prefix stripping --- internal/spell/aff.go | 16 ++++++--- internal/spell/aff_test.go | 68 ++++++++++++++++++++++++++++++++++++++ internal/spell/lazy.go | 4 ++- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/internal/spell/aff.go b/internal/spell/aff.go index 13e94d68..c4b33665 100644 --- a/internal/spell/aff.go +++ b/internal/spell/aff.go @@ -41,8 +41,11 @@ func (a affix) forms(word string) []form { continue } if a.Type == Prefix { - out = append(out, form{Word: r.AffixText + word, Cont: r.Cont}) - // TODO is does Strip apply to prefixes too? + stripWord := word + if r.Strip != "" && strings.HasPrefix(word, r.Strip) { + stripWord = word[len(r.Strip):] + } + out = append(out, form{Word: r.AffixText + stripWord, Cont: r.Cont}) } else { stripWord := word if r.Strip != "" && strings.HasSuffix(word, r.Strip) { @@ -767,9 +770,7 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen // // TODO: Is this safe to do in all cases? affixText, cont := parts[3], "" - if affixText == "0" { - affixText = "" - } else if text, flags, found := strings.Cut(affixText, "/"); found { + if text, flags, found := strings.Cut(affixText, "/"); found { // Split off the affix's own continuation flags, e.g. the // "/34,22" in `SFX 1 0 t/34,22 e`. Left in place they would // be appended to the generated word ("stavet/34,22"), so @@ -781,6 +782,11 @@ func newDictConfig(file io.Reader) (*dictConfig, error) { //nolint:funlen // `stave` in two steps. See expand. affixText, cont = text, flags } + // Hunspell uses 0 for an empty affix, including when it carries + // continuation flags such as `0/L`. + if affixText == "0" { + affixText = "" + } a.Rules = append(a.Rules, rule{ Strip: strip, diff --git a/internal/spell/aff_test.go b/internal/spell/aff_test.go index cc649c1f..da65ac2a 100644 --- a/internal/spell/aff_test.go +++ b/internal/spell/aff_test.go @@ -239,6 +239,74 @@ root/1 } } +// TestZeroAffixWithContinuationFlags verifies that a zero affix is normalized +// to empty text before its continuation flags are applied. +func TestZeroAffixWithContinuationFlags(t *testing.T) { + affContent := `SET UTF-8 +PFX L Y 1 +PFX L 0 l' . + +SFX F Y 1 +SFX F 0 0/L . +` + dicContent := `1 +ordinateur/F +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + tests := []struct { + word string + want bool + }{ + {"l'ordinateur", true}, + {"ordinateur0", false}, + {"l'ordinateur0", false}, + } + for _, tt := range tests { + if got := gs.spell(tt.word); got != tt.want { + t.Errorf("spell(%q) = %v, want %v", tt.word, got, tt.want) + } + } +} + +func TestPrefixStrip(t *testing.T) { + affContent := `SET UTF-8 +PFX A N 1 +PFX A a l'A a +` + dicContent := `1 +ami/A +` + + gs, err := newGoSpellReader( + strings.NewReader(affContent), + strings.NewReader(dicContent), + ) + if err != nil { + t.Fatalf("newGoSpellReader error: %v", err) + } + + tests := []struct { + word string + want bool + }{ + {"l'Ami", true}, + {"l'Aami", false}, + } + for _, tt := range tests { + if got := gs.spell(tt.word); got != tt.want { + t.Errorf("spell(%q) = %v, want %v", tt.word, got, tt.want) + } + } +} + func TestFlagNumAffixParsing(t *testing.T) { // Minimal FLAG num AFF file affContent := `SET UTF-8 diff --git a/internal/spell/lazy.go b/internal/spell/lazy.go index 0f222930..7f1f73c1 100644 --- a/internal/spell/lazy.go +++ b/internal/spell/lazy.go @@ -64,7 +64,9 @@ func (i reverseAffixIndex) predecessors(word string, out map[string]struct{}) { } text := word[:length] for _, current := range i.prefixes[text] { - candidate := word[length:] + // Reverse the forward prefix transformation by removing the added + // prefix and restoring the text stripped from the dictionary root. + candidate := current.Strip + word[length:] if current.matcher == nil || current.matcher.MatchString(candidate) { out[candidate] = struct{}{} }