feat: generic per-language text corpora (data/corpora)

.gitignore

@@ -32,3 +32,4 @@ node_modules/
 
 # Web app generated assets
 web/public/data/
+.cache/

MANIFEST.in

@@ -1,3 +1,4 @@
 include src/worldalphabets/py.typed
 recursive-include src/worldalphabets/data *
 recursive-exclude src/worldalphabets/data/audio *
+recursive-exclude src/worldalphabets/data/corpora *.txt

README.md

@@ -60,6 +60,52 @@ print("English tokens (first 5):", freq_en.tokens[:5])
 print("Token mode:", freq_en.mode)
 ```
 
+### Text corpora (optional data — access from every interface)
+
+Per-language corpora of **real sentences** (Tatoeba exports; one sentence per
+line, ~400KB per language — currently 167 languages, 27MB) live in
+`data/corpora/`. **The text is deliberately optional**: only a small manifest
+(`SOURCES.json`) ships in the Python wheel/sdist, the npm tarball, and the
+browser bundle — embedding 27MB of text would bloat every package. The C
+library does not embed it either.
+
+Licensing: Tatoeba sentences are **CC-BY 2.0 FR** — attribution to
+https://tatoeba.org, retained in each corpus's SOURCES.json entry; the CC0
+subset is used where a language has substantive coverage. (A brief synthetic
+mode — Zipf-sampled from the frequency lists — was removed: plausible
+character statistics over word salad is the wrong trade for a text corpus.)
+
+```python
+from worldalphabets import list_corpora, get_corpus
+
+info = list_corpora()               # works everywhere: [{lang, mode, verify}, ...]
+text = get_corpus("hu")             # needs the files (see resolution order below)
+text = get_corpus("hu", path=".../WorldAlphabets/data/corpora")  # explicit
+# or: export WA_CORPORA_DIR=.../data/corpora
+```
+
+JavaScript/TypeScript (same shape):
+
+```js
+const { listCorpora, loadCorpus } = require("worldalphabets"); // or ESM import
+await listCorpora();        // manifest — always available
+await loadCorpus("hu");     // reads data/corpora/ from a repo checkout (Node);
+                           // throws descriptive guidance in the browser
+```
+
+C (manifest only — the text is never embedded):
+
+```c
+#include <worldalphabets.h>
+size_t n = wa_corpus_count();                 // 167
+const wa_corpus_info *c = wa_get_corpus_info("hu");
+// c->mode == "tatoeba-sentences"; load data/corpora/<lang>.txt yourself
+```
+
+Corpora are generated by `scripts/build_text_corpora.py` from the Tatoeba
+per-language exports (cached under `.cache/tatoeba/`); CommonVoice transcripts
+(CC-0) are a future additional source.
+
 ### Node.js
 
 #### From npm

__tests__/worldalphabets.test.js

@@ -6,6 +6,8 @@ const {
   getAvailableCodes,
   loadAlphabet,
   loadFrequencyList,
+  listCorpora,
+  loadCorpus,
   getIndexData,
   getLanguage,
   getScripts,
@@ -98,6 +100,30 @@ describe('worldalphabets', () => {
     await expect(loadFrequencyList('invalid-code')).rejects.toThrow();
   });
 
+  describe('text corpora', () => {
+    it('should list corpora with licensing metadata', async () => {
+      const corpora = await listCorpora();
+      expect(Array.isArray(corpora)).toBe(true);
+      expect(corpora.length).toBeGreaterThan(100);
+      const hu = corpora.find((c) => c.lang === 'hu');
+      expect(hu).toBeDefined();
+      expect(typeof hu.verify).toBe('boolean');
+      expect(typeof hu.mode).toBe('string');
+    });
+
+    it('should load a corpus from the repository checkout', async () => {
+      const corpus = await loadCorpus('hu');
+      expect(corpus.language).toBe('hu');
+      expect(corpus.text.length).toBeGreaterThan(100000);
+      // One sentence per line, many lines
+      expect(corpus.text.split('\n').length).toBeGreaterThan(1000);
+    });
+
+    it('should throw a descriptive error for a missing corpus', async () => {
+      await expect(loadCorpus('invalid-code')).rejects.toThrow(/not found|not available/i);
+    });
+  });
+
   it('should get scripts for a language', async () => {
     const scripts = await getScripts('zh');
     expect(Array.isArray(scripts)).toBe(true);

c/generated/worldalphabets_data.h

@@ -6,10 +6,12 @@
 #define WA_ALPHABETS_COUNT 342u
 #define WA_FREQUENCY_LISTS_COUNT 193u
 #define WA_KEYBOARD_LAYOUTS_COUNT 194u
+#define WA_CORPORA_COUNT 167u
 
 extern const char *WA_LANGUAGE_CODES[];
 extern const wa_script_entry WA_SCRIPT_ENTRIES[];
 extern const wa_alphabet WA_ALPHABETS[];
 extern const wa_frequency_list WA_FREQUENCY_LISTS[];
 extern const wa_keyboard_layout WA_KEYBOARD_LAYOUTS[];
 extern const char *WA_LAYOUT_IDS[];
+extern const wa_corpus_info WA_CORPORA[];

c/include/worldalphabets.h

@@ -50,6 +50,12 @@ typedef struct {
     size_t token_count;
 } wa_frequency_list;
 
+typedef struct {
+    const char *language;
+    const char *mode;  // "tatoeba-sentences" or "tatoeba-sentences-cc0"
+    int verify;        // 1 until the per-language source chain is confirmed clean
+} wa_corpus_info;
+
 typedef struct {
     const char *language;
     const char **scripts;
@@ -110,6 +116,15 @@ wa_string_array wa_get_scripts(const char *code);
 // Frequency lists
 const wa_frequency_list *wa_load_frequency_list(const char *code);
 
+// Text corpora — MANIFEST ONLY. The corpus text (real Tatoeba sentences,
+// CC-BY 2.0 FR, attribution https://tatoeba.org) is deliberately not
+// embedded: 27MB would dwarf the library. Use the manifest to discover
+// availability, then load data/corpora/<lang>.txt from a WorldAlphabets
+// checkout or vendored copy. Retain the attribution when redistributing.
+size_t wa_corpus_count(void);
+const wa_corpus_info *wa_list_corpora(void);
+const wa_corpus_info *wa_get_corpus_info(const char *code);
+
 // Language detection
 wa_detect_result_array wa_detect_languages(const char *text,
                                            const char **candidate_langs,

c/src/worldalphabets.c

@@ -85,6 +85,26 @@ const wa_frequency_list *wa_load_frequency_list(const char *code) {
     return find_freq_list(code);
 }
 
+// --- text corpora (manifest only; text is not embedded) ---
+
+size_t wa_corpus_count(void) {
+    return (size_t)WA_CORPORA_COUNT;
+}
+
+const wa_corpus_info *wa_list_corpora(void) {
+    return WA_CORPORA;
+}
+
+const wa_corpus_info *wa_get_corpus_info(const char *code) {
+    if (code == NULL) return NULL;
+    for (size_t i = 0; i < WA_CORPORA_COUNT; i++) {
+        if (strcmp(WA_CORPORA[i].language, code) == 0) {
+            return &WA_CORPORA[i];
+        }
+    }
+    return NULL;
+}
+
 // --- detection ---
 
 typedef struct {