ingestlib/config.example.yaml at main · LangModule/ingestlib · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# ingestlib configuration — only what you choose lives here; every
# omitted key is a library default.

# ── Your choices ────────────────────────────────────────────────────────

# sqlite (default: one local file, no server, no keys) | pinecone | qdrant
# | pgvector | mongodb | milvus | opensearch | weaviate — secrets in .env;
# server-backed stores need their pip extra: pip install "ingestlib[qdrant]"
vector_store: sqlite

# jina (needs JINA_API_KEY) | aws (same AWS credentials) | none
reranker: jina

# Who serves LLM and embedding calls:
#   bedrock (default) | openai (needs OPENAI_API_KEY) | ollama (local server, no key)
# Switching embedding_provider changes the vector space — re-ingest after.
llm_provider: bedrock
embedding_provider: bedrock

# Where pipeline artifacts (parses, page images, chunks) live:
# s3 (durable, shareable) | local (a plain folder — zero cloud)
artifact_store: s3

# Your AWS profile from ~/.aws/credentials. Only needed while a choice
# above uses AWS (bedrock provider, s3 artifacts, aws reranker, Amazon
# OpenSearch) — delete this whole section otherwise.
aws:
  profile: your-aws-profile
  region: us-east-1
  account_id: "000000000000"

# OCR server for parse/ingest: mlx-vlm-server (Apple Silicon) | vllm-server (NVIDIA)
paddle_vl:
  backend: mlx-vlm-server
  server_url: http://localhost:8111/

# ── Optional overrides — the values shown ARE the defaults ──────────────
# Uncomment only what you want to change.

# s3:
#   bucket: ingestlib-{aws.account_id}   # bucket names are global across AWS
# artifacts:
#   path: artifacts                      # local mode's folder; relative → beside this file

# bedrock:
#   llm_model_id: us.amazon.nova-2-lite-v1:0
#   embedding_model_id: amazon.nova-2-multimodal-embeddings-v1:0
#   rerank_model_id: amazon.rerank-v1:0
#   rerank_region: us-west-2
# openai:
#   llm_model_id: gpt-5-mini
#   embedding_model_id: text-embedding-3-small
# ollama:                                # base_url accepts any OpenAI-compatible
#   base_url: http://localhost:11434/v1  # server (vLLM, LM Studio); use GGUF
#   llm_model_id: qwen3.5:9b             # builds, not -mlx
#   embedding_model_id: qwen3-embedding:0.6b
# jina:
#   base_url: https://api.jina.ai/v1
#   rerank_model_id: jina-reranker-v3
# paddle_vl:
#   api_model_name: PaddlePaddle/PaddleOCR-VL-1.6

# Indexes, collections, and tables are created on first use.
# pinecone:
#   index_name: ingestlib
#   sparse_index_name: ingestlib-sparse
#   sparse_model_id: pinecone-sparse-english-v0
#   cloud: aws
#   region: us-east-1
# qdrant:
#   collection_name: ingestlib
# sqlite:
#   path: ingestlib.db                   # relative paths anchor beside this file
# pgvector:
#   table_name: ingestlib
# mongodb:
#   database: ingestlib
#   collection_name: ingestlib
# milvus:
#   collection_name: ingestlib
# opensearch:
#   index_name: ingestlib
# weaviate:
#   collection_name: Ingestlib           # Weaviate capitalizes collection names

# MCP server (`ingestlib mcp`) — expose the pipeline as tools to agents.
# stdio needs nothing; the streamable-http transport binds host:port and
# requires MCP_TOKEN (.env). read_only hides ingest/remove/sync/backfill.
# mcp:
#   read_only: false
#   host: 127.0.0.1                      # local-only by default; change to expose
#   port: 8000