diff --git a/.wordlist.txt b/.wordlist.txt index 82778142..de9830ac 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -1,48 +1,393 @@ + +ACL +ACLs +AG2 +AGPLv +AGPLv3 +ANN +AOF +APIs +APOC +APPDATA +ASC +AST +Addon +AgentOS +Aldis +Analytics +Anthropic +ApeCloud +AutoGen +Autowired +BFS +BGSAVE +BLAS +BYOC +BackupPolicy +BackupRepo +BackupSchedule +Benchmarking +Betweenness +Bitnami +Bitnami's +Bitwise +Booleans +Brin +Bundler +CDLP +CLI +CMD +CMake +CRUD +CRoaring +CSC +CSV +CSVs +Cailliau +CentOS +Centos +Checkpointing +Claude +ClickHouse +ClusterDefinition +Cmd +CodeGraph +Cognee +Cognee's +Cohere +Colab +ColumnType +ComponentDefinition +ConsoleSpanExporter +ConversableAgent +ConversationBufferMemory +Costanza +Ctrl +Cursor +Cypher +DBaaS +DDL +DDTHH +DELUSER +DESC +DT +Dani +Databricks +Datadog +DateTime +Deduplicate +Deduplication +DefaultFalkorDBClient +Dockerfiles +Dovizioso +Dropzone +Ducati +Dunder +EMPLOYS +Embedder +EnableFalkorDBRepositories +EnableTransactionManagement +EpisodeType +ErBEqN +Esc +Explainer +FAQ +FDB +FIELDTERMINATOR +FOF +FOREACH +Falkor +FalkorClientBuilder +FalkorConnectionInfo +FalkorDB +FalkorDB's +FalkorDBClient +FalkorDBConfig +FalkorDBCustomConversions +FalkorDBEntityConverter +FalkorDBExample +FalkorDBGraph +FalkorDBGraphStoreFactory +FalkorDBLite +FalkorDBMappingContext +FalkorDBQAChain +FalkorDBRepository +FalkorDBTemplate +FalkorDBTransactionManager +FalkorGraphRagCapability +FastAPI +Forst +GCP +GCP's +GETUSER +GNNs +GPL +Gadepally +Gemini +GenAI +Geometric's +Geospatial +GitHub +GoDoc +GrAPL +Grafana +GraphBLAS +GraphBlas +GraphCypherQAChain +GraphName +GraphRAG +GraphSAGE +GraphStoreFactory +Graphiti +Groq +GroupChat +GroupChatManager +HAS +HLL +HNSW +HarmonicCentrality +HashMap +Homebrew +IDE +IDF +IDentifier +INCOMING +IPDPS +IPs +ISC +Implementers +IndexError +Instrumentor +Integrations +Interned +JFalkorDB +JPA +JS +JSDoc +JSON +JST +JWT +Jaccard +Jaeger +Jaro +Jena +Jena's +Jngle +Jupyter +KJDev +KNOWS +Kepner +Keren +Kruskal's +KubeBlocks +Kubernetes +Kuzu +LLM +LLMs +LRU +LangChain +LangGraph +LangGraph's +Levenshtein +Lex +Lightning +Lipman +LiteLLM +LlamaIndex +LoadBalancer +LocalDateTime +LocalDateTimeToStringConverter +LocalTime +Lovitz +MCP +MCPServer +MEM +MSF +MST +MSTs +MYAPP +MariaDB +Mem +Metaphone +Miesha +Mifflin +MotoGP +Munger +MyApp +MySQL +NEXTAUTH +NFalkorDB +NLB +NRedisGraph +NaN +NeighborLoader +NextAuth +NodeJS +NodePort +Nostem +NumItems +OGM +OMP +ORM +OTLP +OTel +OUTGOING +ObjectPool +Ollama +Ontologies +OpenAI +OpenAIClient +OpenAPI +OpenCypher +OpenMP +OpenTelemetry +Opire +OpsRequest +Ouaknine +PDFs +PRs +PaaS +PageRequest +Pageable +Parametrized +PascalCase +Pathfinding +Pedrosa +PersonRepository +PersonService +Pieter +Pinterest +PodMonitor +PostContent +Postgres +Prim's +ProcedureCall +PyG +PyG's +PyPI +Pydoc +QueryWeaver +QueryWeaver's +QuickJS +Quickstart +RDB +RDF +RESULTSET +RO +RSALv +RW +Railway +Redhat +RediSearch +Redis +RedisGraph +RedisInsight +Reranking +ResultSet +Roi +Rossi +Rueian +RunningQueries +SDK +SDKs +SETUSER +SHA +SLAs +SLOWLOG +SPARQL +SPpath +SPpaths +SQL's +SQLServer +SSE +SSL +SSPLv +SSpath +SSpaths +SaaS +Saldana +Sceat +SearchType +Sergey +Spring +Stopwords +Streamable +StringToLocalDateTimeConverter +StringUtils +Subcommands +Submap +Supabase +Swagger +TCP +TF +TLS +TOC +TTL +TTY +TXT +TechCorp +TenantLabel +Text2SQL +Transactional +Trendshift +TwitterUser +TypeScript +UDF +UDFs +UI +UNIQIE +URIs +UUID +UUIDStringGenerator +UX +UpperCamelCase +UpperCaseOdd +Upserts +Uvicorn +VGF +VKEY +VPC +VPCs +VPNs +ValueType +Vijay +Vin +Vite +WCC +WORKS +WSL +WaitingQueries +Winkler +XCode +YYYY +Zep's abc accessor accordion -ACL -ACLs acos acyclic addon -Addon addons ag2 -AG2 agentic -AgentOS -AGPLv -AGPLv3 ai -Aldis algo alice alister -allowInsecureImages allShortestPaths +allowInsecureImages analytics -Analytics -ANN -Anthropic -AOF apecloud -ApeCloud api -APIs -APOC -APPDATA arccosine arcsine arctangent args artifactId -ASC +asList ascii asin -asList -AST async atan atomicity @@ -50,75 +395,40 @@ auth autocomplete autocompletion autogen -AutoGen -Autowired awslabs backend -BackupPolicy -BackupRepo -BackupSchedule benchmarking -Benchmarking betweenness -Betweenness -BFS -BGSAVE bigmac birthyear bitmasks -Bitnami -Bitnami's bitwise -Bitwise -BLAS body bool boolean booleans -Booleans bottlenecked -Brin -Bundler -BYOC -Cailliau callbackUrl calmcode -camel_case camelCase +camel_case cardinality -CDLP ceil -Centos -CentOS cgraph chatbots checkpointed checkpointing -Checkpointing checksums -Claude +chunkers cli -CLI -ClickHouse cloudserve -ClusterDefinition -CMake -Cmd -CMD codebase codebases codegraph -CodeGraph cognee -Cognee -Cognee's cognify -Cohere -Colab -ColumnType communityId compat -ComponentDefinition componentId compton conf @@ -126,13 +436,9 @@ config configGet configSet configurability -ConsoleSpanExporter const -ConversableAgent -ConversationBufferMemory copyGraph cosineDistance -Costanza costProp countByAge countEmployees @@ -141,44 +447,25 @@ createConstraint createNodeIndex createQuery crflynn -CRoaring cron -CRUD -CSC csrf csrfToken csv -CSV -CSVs -Ctrl -Cursor +cutover cypher -Cypher -Dani -Databricks -Datadog datafile dataset datasets datetime -DateTime datetimes -DBaaS dbms ddl -DDL -DDTHH decapitalize dedup -Deduplicate deduplicated deduplicates deduplication -Deduplication -DefaultFalkorDBClient deleteByAge -DELUSER -DESC destinationGraphName dev dimensionality @@ -188,15 +475,9 @@ displayName distro dm dockerfile -Dockerfiles dotenv -Dovizioso drivine dropConstraint -Dropzone -DT -Ducati -Dunder duplications dups durations @@ -204,14 +485,11 @@ dylib efConstruction efRuntime emailExists -Embedder +embedder embeddings employeeCount -EMPLOYS -EnableFalkorDBRepositories -EnableTransactionManagement -endcapture endNode +endcapture endtab endtabs entrypoint @@ -219,48 +497,21 @@ enum enums env epel -EpisodeType -ErBEqN erroring -Esc euclideanDistance eval everysec existsByEmail -Explainer expr exprList extraFlags failover falkor -Falkor -FalkorClientBuilder -FalkorConnectionInfo -falkordb falkorDB -FalkorDB -FalkorDBClient -FalkorDBConfig -FalkorDBCustomConversions -FalkorDBEntityConverter -FalkorDBExample -FalkorDBGraph -FalkorDBGraphStoreFactory +falkordb falkordblite -FalkorDBLite -FalkorDBMappingContext -FalkorDBQAChain -FalkorDBRepository -FalkorDB's -FalkorDBTemplate -FalkorDBTransactionManager -FalkorGraphRagCapability fallbacks faq -FAQ -FastAPI -FDB -FIELDTERMINATOR filesystem findAllByOrderByNameAsc findByActiveFalse @@ -291,12 +542,9 @@ findTop10ByOrderByAgeDesc findTop5ByOrderByAgeDesc firstName fn -FOF followerCount -foreach forEach -FOREACH -Forst +foreach fp fromJsonList fromJsonMap @@ -305,174 +553,91 @@ frontend fsync fullMatch fulltext -Gadepally -GCP -GCP's -Gemini genai -GenAI -Geometric's geospatial -Geospatial getConfig getMessage -GETUSER getzep gg github -GitHub -GNNs godoc -GoDoc -GPL gpt gradle -Grafana graphA -GraphBlas -GraphBLAS -GraphCypherQAChain -graphiti -Graphiti graphName -GraphName -graphrag -GraphRAG -GraphSAGE -GraphStoreFactory graphZ -GrAPL -Groq -GroupChat -GroupChatManager +graphiti +graphrag groupId h1 -HarmonicCentrality -HAS -HashMap hasLabels haversin healthcheck hh highmem -HLL -HNSW -Homebrew hostname hostnames howtos html - htmlcontent http https hydre -IDE -IDentifier -IDF idx imdb -Implementers -INCOMING indegree -IndexError -indexesOf index.md indexOf +indexesOf insertListElements -Instrumentor integrations -Integrations -Interned io ioredis ioredisgraph -IPDPS -IPs -ISC isEmpty iteratively jaccard -Jaccard -Jaeger -Jaro jaroWinkler javadocs javascript jedis jena -Jena -Jena's jfalkordb -JFalkorDB jinja jl -Jngle -JPA jpbourbon js -JS -JSDoc json -JSON -JST -Jupyter -JWT kafka kbcli kebab -Kepner -Keren keyspace -KJDev -KNOWS -Kruskal's kubeblocks -KubeBlocks -Kubernetes -Kuzu +kubectl +lTrim labelPropagation labelsList langchain -LangChain -LangGraph -LangGraph's lastName learnhub len levenshtein -Levenshtein -Lex libcypher libomp -Lightning -Lipman -list_graphs listGraphs -LiteLLM -LlamaIndex +list_graphs llm -LLM -LLMs -LoadBalancer loadmodule +localStorage localdatetime -LocalDateTime -LocalDateTimeToStringConverter localhost -localStorage -LocalTime lon lookups -Lovitz lowercased lowercasing lpad -LRU -lTrim macOS malte -MariaDB matchRegEx maxAge maxCost @@ -482,136 +647,76 @@ maxLen maxmemory mb mcp -MCP -mcpserver -MCPServer mcpServers +mcpserver md mem -Mem -MEM memoryInfo memoryUsage -Metaphone -Miesha -Mifflin minAge misconfigured modelled -MotoGP -MSF -MST -MSTs multi multithreaded -Munger mut -MyApp -MYAPP myGraph myPassword -myproject -MySQL myUsername +myproject namespace namespaces nan -NaN natively nav nd -NeighborLoader neo -NextAuth -NEXTAUTH -NFalkorDB nlb -NLB nodeId -NodeJS -NodePort -Nostem npm npx -NRedisGraph nredisstack nuget numerics -NumItems -ObjectPool observability ogm -OGM -Ollama -OMP onboarding oniguruma ontologies -Ontologies ontology -OpenAI -OpenAIClient -OpenAPI -opencypher openCypher -OpenCypher -OpenMP +opencypher opentelemetry -OpenTelemetry -Opire -OpsRequest orderability orm -ORM orphaned -OTel -OTLP -Ouaknine outdegree -OUTGOING p -PaaS -Pageable -pagerank pageRank -PageRequest +pagerank parallelizing -Parametrized params pascal_case -PascalCase pathCost pathCount -pathfinding -Pathfinding pathNodes pathWeight +pathfinding pc pdf -PDFs -Pedrosa percentileCont percentileDisc performant permalink -PersonRepository -PersonService php -Pieter -Pinterest pipx -PodMonitor pom -PostContent pre prem preprocessing prev -Prim's printf println proc -ProcedureCall programmatically propCount propertyKeys @@ -620,12 +725,8 @@ propvalue prune psutil py -Pydoc pyg -PyG -PyG's pypi -PyPI pzac qdrant queryData @@ -633,86 +734,60 @@ queryEmbedding queryNodes queryRelationships queryVector -QueryWeaver -QueryWeaver's -QuickJS +queryable quickstart -Quickstart quorum -Railway +rTrim randomUUID rax rb -RDB rdf -RDF reachability -readme readOnlyQuery +readme rebalancing recomputation redeployments redgraph -Redhat redis -Redis -RediSearch redisgraph -RedisGraph -RedisInsight redislite regexGroups +relDirection +relTypes relationshipId relationshipTypes -relDirection reltype reltypeList -relTypes removeKey removeKeys replaceRegEx replicas resultSet -ResultSet -RESULTSET rgraph ro -RO -Roi roQuery -Rossi +rollforward rpad -RSALv -rTrim -Rueian rueidis -RunningQueries +runnable runtime rustis -RW -SaaS -Saldana samplingSeed samplingSize sanitization sbin scalability scalable -Sceat schemaName schemas scranton sdist sdk -SDK -SDKs sds -SearchType segregation selectGraph -Sergey setConfig -SETUSER sharded sharding shiftLeft @@ -725,44 +800,26 @@ signout signum signup similarityFunction -SLAs -slowlog slowLog -SLOWLOG -snake_case +slowlog snakeCase +snake_case snapshotting sourceName sourceNode -SPARQL -SPpath -SPpaths -Spring sprintf -SQL's -SQLServer sqrt ss sse -SSE -SSL -SSpath -SSpaths -SSPLv -startNode -stateful stDev stDevP +startNode +stateful stdio stopwords -Stopwords str -Streamable -StringToLocalDateTimeConverter -StringUtils strList subcommands -Subcommands subdirectories subdirectory subfolder @@ -770,121 +827,78 @@ subgraph subgraphs sublist submap -Submap subqueries subquery substring sudo -Swagger swapCase sz table targetNode -TCP -TechCorp tenant tenantId -TenantLabel -Text2SQL -TF thpool -TLS toBoolean toBooleanList toBooleanOrNull -toc -TOC toFloat toFloatList toFloatOrNull toInteger toIntegerList toIntegerOrNull -toJson toJSON +toJson +toLower +toString +toStringList +toStringOrNull +toTimeZone +toUpper +toc tokenId tokenization tokenized tokenizer tokio tolist -toLower tombatron tompro toolchain topologies topoteretes -toString -toStringList -toStringOrNull -toTimeZone -toUpper tracebacks trackpad tradeoff tradeoffs transactional -Transactional -Trendshift ttl -TTL -TTY -TwitterUser txt -TXT typeOf -TypeScript uber -UDF -UDFs -UI unaliased undirected -UNIQIE unnormalized unparseable unserializable updateLastLogin upperCamelCase -UpperCamelCase -UpperCaseOdd upsert -Upserts uri -URIs url urlencoded userName utf util -UUID -UUIDStringGenerator -Uvicorn -UX -ValueType vec vecf versine -VGF viewport -Vijay -Vin -Vite -VKEY -VPC -VPCs -VPNs -WaitingQueries walkthrough -WCC weightProp whitespace -Winkler -WORKS -WSL www xcode -XCode xlarge xml xxHash @@ -893,15 +907,4 @@ yaml yml yourQuery yourSourceName -YYYY zepai -Zep's -chunkers -cutover -embedder -PRs -queryable -Reranking -rollforward -runnable -SHA diff --git a/operations/migration/sql-to-falkordb.md b/operations/migration/sql-to-falkordb.md index 305c58bd..74177f18 100644 --- a/operations/migration/sql-to-falkordb.md +++ b/operations/migration/sql-to-falkordb.md @@ -106,7 +106,7 @@ cargo run --release -- --config clickhouse.incremental.yaml --daemon --interval- ```bash cd Databricks-to-FalkorDB/databricks-to-falkordb cargo build --release -cargo run --release -- --config path/to/config.yaml +cargo run --release -- --config ../databricks_sample_to_falkordb.yaml ``` Note: Databricks currently supports one-shot runs only (no daemon/purge flags). @@ -150,13 +150,14 @@ cd PostgreSQL-to-FalkorDB/postgres-to-falkordb cargo build --release # One-shot run -cargo run --release -- --config path/to/config.yaml +cargo run --release -- --config example.config.yaml # Continuous sync -cargo run --release -- --config path/to/config.yaml --daemon --interval-secs 60 +cargo run --release -- --config example.config.yaml --daemon --interval-secs 60 ``` Note: PostgreSQL currently supports daemon mode but not purge flags. +For Supabase-compatible Postgres endpoints, include `sslmode=require` (minimum) or `sslmode=verify-full` in `POSTGRES_URL`. ### Snowflake → FalkorDB @@ -167,10 +168,10 @@ cd Snowflake-to-FalkorDB cargo build --release # One-shot run -cargo run --release -- --config path/to/config.yaml +cargo run --release -- --config snowflake_stream_example.yaml # Continuous sync -cargo run --release -- --config path/to/config.yaml --daemon --interval-secs 300 +cargo run --release -- --config snowflake_stream_example.yaml --daemon --interval-secs 300 ``` ### Spark SQL (Livy) → FalkorDB @@ -180,7 +181,7 @@ cargo run --release -- --config path/to/config.yaml --daemon --interval-secs 300 ```bash cd Spark-to-FalkorDB/spark-to-falkordb cargo build --release -cargo run --release -- --config path/to/config.yaml +cargo run --release -- --config ../spark_to_falkordb_e2e.sample.yaml ``` Note: Spark currently supports one-shot runs only (no daemon/purge flags). @@ -206,10 +207,10 @@ Purge flags are supported by: BigQuery, ClickHouse, MariaDB, MySQL, Snowflake, S ```bash # Purge full graph before loading -cargo run --release -- --config path/to/config.yaml --purge-graph +cargo run --release -- --config ClickHouse-to-FalkorDB/clickhouse.incremental.yaml --purge-graph # Purge selected mappings -cargo run --release -- --config path/to/config.yaml --purge-mapping customers +cargo run --release -- --config ClickHouse-to-FalkorDB/clickhouse.incremental.yaml --purge-mapping customers ``` ## Schema introspection + template scaffolding @@ -218,10 +219,10 @@ Supported by: BigQuery, ClickHouse, Databricks, MariaDB, MySQL, PostgreSQL, Snow ```bash # Print normalized source schema summary -cargo run --release -- --config path/to/config.yaml --introspect-schema +cargo run --release -- --config PostgreSQL-to-FalkorDB/postgres-to-falkordb/example.config.yaml --introspect-schema # Generate starter mapping template -cargo run --release -- --config path/to/config.yaml --generate-template --output scaffold.yaml +cargo run --release -- --config PostgreSQL-to-FalkorDB/postgres-to-falkordb/example.config.yaml --generate-template --output PostgreSQL-to-FalkorDB/postgres.generated.template.yaml ``` Notes: @@ -237,7 +238,7 @@ The control plane discovers tools by scanning for `tool.manifest.json` and provi - Preview extracted source schema - Generate scaffold templates from source metadata - Visualize graph topology from mappings -- Start runs (one-shot and daemon where supported) +- Start runs (one-shot and daemon where supported) on local or Kubernetes execution backend - Stop active runs - Stream live logs (SSE) and view persisted logs for historical runs - Inspect and clear per-config incremental state/watermarks @@ -263,6 +264,16 @@ Configuration (environment variables): - `CONTROL_PLANE_DATA_DIR` (default `control-plane/data/`) - `CONTROL_PLANE_UI_DIST` (default `control-plane/ui/dist/`) - `CONTROL_PLANE_API_KEY` (optional bearer token requirement) +- `CONTROL_PLANE_ENABLED_TOOLS` (optional comma-separated allow-list, for example `postgres,snowflake`) +- `CONTROL_PLANE_EXECUTION_BACKEND` (`local` or `kubernetes`, default `local`) +- `CONTROL_PLANE_K8S_NAMESPACE` (namespace where Kubernetes run workloads are created) +- `CONTROL_PLANE_K8S_RUNNER_IMAGE` (multi-tool runner image reference) +- `CONTROL_PLANE_K8S_IMAGE_PULL_POLICY` (runner image pull policy) +- `CONTROL_PLANE_K8S_SERVICE_ACCOUNT` (service account used for run workloads) +- `CONTROL_PLANE_K8S_SHARED_PVC` (optional shared PVC for file-backed state) +- `CONTROL_PLANE_K8S_ENV_SECRET` / `CONTROL_PLANE_K8S_ENV_CONFIGMAP` (optional env sources projected to run pods) +- `CONTROL_PLANE_K8S_KUBECTL_BIN` (kubectl binary path; default `kubectl`) +- `CONTROL_PLANE_K8S_BINARY_DIR` (tool binary directory in runner image; default `/opt/falkordb/bin`) Selected API endpoints: @@ -282,7 +293,7 @@ Selected API endpoints: Notes: -- Runs execute locally on the host machine running the control plane server. +- Runs execute either locally on the control-plane host (`local`) or as Kubernetes workloads (`kubernetes`). - Runtime artifacts are persisted under `CONTROL_PLANE_DATA_DIR`, including a SQLite DB and per-run files. - SSE auth with API key may use query-string token fallback because browser `EventSource` does not support custom auth headers. @@ -294,6 +305,38 @@ npm install npm run dev ``` +## Container + Kubernetes single-deployment model + +The DM-SQL repository includes a single-release deployment path that pairs one control-plane instance with one multi-tool runner image: + +- `control-plane/Dockerfile` builds the control-plane API + UI image. +- `docker/runner.Dockerfile` builds a runner image containing all SQL-to-FalkorDB binaries. +- `docker/build-images.sh [registry]` builds both images with one version tag. +- `deploy/helm/dm-sql-to-falkordb/` provides a Helm chart for a unified deployment. + +Example image build: + +```bash +./docker/build-images.sh v0.1.0 ghcr.io/falkordb +``` + +Example Helm install (single control plane with selected tools enabled): + +```bash +helm upgrade --install dm-sql deploy/helm/dm-sql-to-falkordb \ + --namespace dm-sql --create-namespace \ + --set global.version=v0.1.0 \ + --set tools.enabled.postgres=true \ + --set tools.enabled.snowflake=true \ + --set tools.enabled.mysql=false \ + --set tools.enabled.mariadb=false \ + --set tools.enabled.clickhouse=false \ + --set tools.enabled.bigquery=false \ + --set tools.enabled.databricks=false \ + --set tools.enabled.spark=false \ + --set tools.enabled.sqlserver=false +``` + ## Metrics feature All current SQL loaders expose Prometheus-style metrics with: