From 61b2dda93003b81d646c3d7b8bf39381be88a19d Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 18:14:41 +0300 Subject: [PATCH 1/7] codegen: add built-in C++, C# and Java targets --- .github/workflows/ci.yml | 68 ++- .gitignore | 4 + README.md | 18 +- docs/architecture.md | 7 +- docs/compatibility.md | 7 + docs/cpp.md | 94 ++++ docs/csharp.md | 63 +++ docs/development.md | 34 +- docs/java-research.md | 111 ++++ docs/java.md | 85 +++ docs/targets.md | 24 +- examples/authors/cpp/CMakeLists.txt | 14 + examples/authors/cpp/Dockerfile | 18 + examples/authors/cpp/native/CMakeLists.txt | 8 + examples/authors/cpp/native/main.cpp | 122 +++++ examples/authors/cpp/native/models.hpp | 26 + examples/authors/cpp/native/probe.sh | 6 + examples/authors/cpp/native/queries.cpp | 157 ++++++ examples/authors/cpp/native/queries.hpp | 29 ++ examples/authors/cpp/run-smoke.sh | 36 ++ examples/authors/cpp/userver/CMakeLists.txt | 2 + examples/authors/cpp/userver/main.cpp | 17 + examples/authors/cpp/userver/models.hpp | 28 + examples/authors/cpp/userver/probe.sh | 5 + examples/authors/cpp/userver/queries.cpp | 103 ++++ examples/authors/cpp/userver/queries.hpp | 29 ++ examples/authors/cpp/userver/run.sh | 17 + .../authors/cpp/userver/smoke_handler.cpp | 85 +++ .../authors/cpp/userver/smoke_handler.hpp | 28 + .../authors/cpp/userver/static_config.yaml | 38 ++ .../csharp/adonet/Authors.AdoNet.csproj | 12 + examples/authors/csharp/adonet/Models.cs | 32 ++ examples/authors/csharp/adonet/Program.cs | 33 ++ examples/authors/csharp/adonet/Queries.cs | 121 +++++ examples/authors/csharp/adonet/README.md | 23 + examples/authors/csharp/adonet/Smoke.cs | 45 ++ examples/authors/java/hibernate/pom.xml | 30 ++ .../main/java/authors/hibernate/Authors.java | 4 + .../authors/hibernate/GetAuthorNameRow.java | 4 + .../java/authors/hibernate/GetAuthorRow.java | 4 + .../authors/hibernate/ListAuthorsRow.java | 4 + .../main/java/authors/hibernate/Queries.java | 120 +++++ .../test/java/authors/hibernate/Smoke.java | 83 +++ examples/authors/java/jdbc/pom.xml | 20 + .../src/main/java/authors/jdbc/Authors.java | 4 + .../java/authors/jdbc/GetAuthorNameRow.java | 4 + .../main/java/authors/jdbc/GetAuthorRow.java | 4 + .../java/authors/jdbc/ListAuthorsRow.java | 4 + .../src/main/java/authors/jdbc/Queries.java | 108 ++++ .../src/test/java/authors/jdbc/Smoke.java | 61 +++ examples/authors/java/native/pom.xml | 19 + .../main/java/authors/nativeapi/Authors.java | 4 + .../authors/nativeapi/GetAuthorNameRow.java | 4 + .../java/authors/nativeapi/GetAuthorRow.java | 4 + .../authors/nativeapi/ListAuthorsRow.java | 4 + .../main/java/authors/nativeapi/Queries.java | 109 ++++ .../test/java/authors/nativeapi/Smoke.java | 77 +++ examples/authors/java/pom.xml | 42 ++ examples/authors/java/run-smoke.sh | 28 + examples/authors/java/spring/pom.xml | 25 + .../src/main/java/authors/spring/Authors.java | 4 + .../java/authors/spring/GetAuthorNameRow.java | 4 + .../java/authors/spring/GetAuthorRow.java | 4 + .../java/authors/spring/ListAuthorsRow.java | 4 + .../src/main/java/authors/spring/Queries.java | 120 +++++ .../src/test/java/authors/spring/Smoke.java | 64 +++ examples/authors/sqlc.yaml | 37 ++ internal/cli/cli.go | 36 +- internal/codegen/cpp/generator.go | 488 ++++++++++++++++++ internal/codegen/cpp/generator_test.go | 331 ++++++++++++ internal/codegen/csharp/generator.go | 436 ++++++++++++++++ internal/codegen/csharp/generator_test.go | 225 ++++++++ internal/codegen/java/generator.go | 410 +++++++++++++++ internal/codegen/java/generator_test.go | 356 +++++++++++++ internal/config/config.go | 60 ++- internal/config/config_test.go | 34 ++ internal/endtoend/golden_test.go | 6 +- .../authors/expected/cpp/native/models.hpp | 16 + .../authors/expected/cpp/native/queries.cpp | 48 ++ .../authors/expected/cpp/native/queries.hpp | 25 + .../authors/expected/cpp/userver/models.hpp | 18 + .../authors/expected/cpp/userver/queries.cpp | 34 ++ .../authors/expected/cpp/userver/queries.hpp | 25 + .../testdata/authors/expected/cs/Models.cs | 16 + .../testdata/authors/expected/cs/Queries.cs | 49 ++ .../expected/java/hibernate/Authors.java | 4 + .../expected/java/hibernate/GetAuthorRow.java | 4 + .../expected/java/hibernate/Queries.java | 38 ++ .../authors/expected/java/jdbc/Authors.java | 4 + .../expected/java/jdbc/GetAuthorRow.java | 4 + .../authors/expected/java/jdbc/Queries.java | 36 ++ .../authors/expected/java/native/Authors.java | 4 + .../expected/java/native/GetAuthorRow.java | 4 + .../authors/expected/java/native/Queries.java | 39 ++ .../authors/expected/java/spring/Authors.java | 4 + .../expected/java/spring/GetAuthorRow.java | 4 + .../authors/expected/java/spring/Queries.java | 38 ++ internal/endtoend/testdata/authors/sqlc.yaml | 39 ++ 98 files changed, 5371 insertions(+), 20 deletions(-) create mode 100644 docs/cpp.md create mode 100644 docs/csharp.md create mode 100644 docs/java-research.md create mode 100644 docs/java.md create mode 100644 examples/authors/cpp/CMakeLists.txt create mode 100644 examples/authors/cpp/Dockerfile create mode 100644 examples/authors/cpp/native/CMakeLists.txt create mode 100644 examples/authors/cpp/native/main.cpp create mode 100644 examples/authors/cpp/native/models.hpp create mode 100755 examples/authors/cpp/native/probe.sh create mode 100644 examples/authors/cpp/native/queries.cpp create mode 100644 examples/authors/cpp/native/queries.hpp create mode 100755 examples/authors/cpp/run-smoke.sh create mode 100644 examples/authors/cpp/userver/CMakeLists.txt create mode 100644 examples/authors/cpp/userver/main.cpp create mode 100644 examples/authors/cpp/userver/models.hpp create mode 100755 examples/authors/cpp/userver/probe.sh create mode 100644 examples/authors/cpp/userver/queries.cpp create mode 100644 examples/authors/cpp/userver/queries.hpp create mode 100755 examples/authors/cpp/userver/run.sh create mode 100644 examples/authors/cpp/userver/smoke_handler.cpp create mode 100644 examples/authors/cpp/userver/smoke_handler.hpp create mode 100644 examples/authors/cpp/userver/static_config.yaml create mode 100644 examples/authors/csharp/adonet/Authors.AdoNet.csproj create mode 100644 examples/authors/csharp/adonet/Models.cs create mode 100644 examples/authors/csharp/adonet/Program.cs create mode 100644 examples/authors/csharp/adonet/Queries.cs create mode 100644 examples/authors/csharp/adonet/README.md create mode 100644 examples/authors/csharp/adonet/Smoke.cs create mode 100644 examples/authors/java/hibernate/pom.xml create mode 100644 examples/authors/java/hibernate/src/main/java/authors/hibernate/Authors.java create mode 100644 examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorNameRow.java create mode 100644 examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorRow.java create mode 100644 examples/authors/java/hibernate/src/main/java/authors/hibernate/ListAuthorsRow.java create mode 100644 examples/authors/java/hibernate/src/main/java/authors/hibernate/Queries.java create mode 100644 examples/authors/java/hibernate/src/test/java/authors/hibernate/Smoke.java create mode 100644 examples/authors/java/jdbc/pom.xml create mode 100644 examples/authors/java/jdbc/src/main/java/authors/jdbc/Authors.java create mode 100644 examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorNameRow.java create mode 100644 examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorRow.java create mode 100644 examples/authors/java/jdbc/src/main/java/authors/jdbc/ListAuthorsRow.java create mode 100644 examples/authors/java/jdbc/src/main/java/authors/jdbc/Queries.java create mode 100644 examples/authors/java/jdbc/src/test/java/authors/jdbc/Smoke.java create mode 100644 examples/authors/java/native/pom.xml create mode 100644 examples/authors/java/native/src/main/java/authors/nativeapi/Authors.java create mode 100644 examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorNameRow.java create mode 100644 examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorRow.java create mode 100644 examples/authors/java/native/src/main/java/authors/nativeapi/ListAuthorsRow.java create mode 100644 examples/authors/java/native/src/main/java/authors/nativeapi/Queries.java create mode 100644 examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java create mode 100644 examples/authors/java/pom.xml create mode 100755 examples/authors/java/run-smoke.sh create mode 100644 examples/authors/java/spring/pom.xml create mode 100644 examples/authors/java/spring/src/main/java/authors/spring/Authors.java create mode 100644 examples/authors/java/spring/src/main/java/authors/spring/GetAuthorNameRow.java create mode 100644 examples/authors/java/spring/src/main/java/authors/spring/GetAuthorRow.java create mode 100644 examples/authors/java/spring/src/main/java/authors/spring/ListAuthorsRow.java create mode 100644 examples/authors/java/spring/src/main/java/authors/spring/Queries.java create mode 100644 examples/authors/java/spring/src/test/java/authors/spring/Smoke.java create mode 100644 internal/codegen/cpp/generator.go create mode 100644 internal/codegen/cpp/generator_test.go create mode 100644 internal/codegen/csharp/generator.go create mode 100644 internal/codegen/csharp/generator_test.go create mode 100644 internal/codegen/java/generator.go create mode 100644 internal/codegen/java/generator_test.go create mode 100644 internal/endtoend/testdata/authors/expected/cpp/native/models.hpp create mode 100644 internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp create mode 100644 internal/endtoend/testdata/authors/expected/cpp/native/queries.hpp create mode 100644 internal/endtoend/testdata/authors/expected/cpp/userver/models.hpp create mode 100644 internal/endtoend/testdata/authors/expected/cpp/userver/queries.cpp create mode 100644 internal/endtoend/testdata/authors/expected/cpp/userver/queries.hpp create mode 100644 internal/endtoend/testdata/authors/expected/cs/Models.cs create mode 100644 internal/endtoend/testdata/authors/expected/cs/Queries.cs create mode 100644 internal/endtoend/testdata/authors/expected/java/hibernate/Authors.java create mode 100644 internal/endtoend/testdata/authors/expected/java/hibernate/GetAuthorRow.java create mode 100644 internal/endtoend/testdata/authors/expected/java/hibernate/Queries.java create mode 100644 internal/endtoend/testdata/authors/expected/java/jdbc/Authors.java create mode 100644 internal/endtoend/testdata/authors/expected/java/jdbc/GetAuthorRow.java create mode 100644 internal/endtoend/testdata/authors/expected/java/jdbc/Queries.java create mode 100644 internal/endtoend/testdata/authors/expected/java/native/Authors.java create mode 100644 internal/endtoend/testdata/authors/expected/java/native/GetAuthorRow.java create mode 100644 internal/endtoend/testdata/authors/expected/java/native/Queries.java create mode 100644 internal/endtoend/testdata/authors/expected/java/spring/Authors.java create mode 100644 internal/endtoend/testdata/authors/expected/java/spring/GetAuthorRow.java create mode 100644 internal/endtoend/testdata/authors/expected/java/spring/Queries.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 150184a..8c255f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,10 +14,24 @@ jobs: with: go-version-file: go.mod cache: true + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' - name: Build and test standalone generator run: | - go test ./... + go test -p 1 ./... make build + - name: Compile all generated scalar bindings against published SDKs + env: + SQLC_YDB_CSHARP_DOTNET: dotnet + SQLC_YDB_TEST_MAVEN: mvn + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + run: go test -p 1 -count=1 ./internal/codegen/csharp ./internal/codegen/java - name: Verify generated examples run: | ./bin/sqlc-ydb compile -f examples/authors/sqlc.yaml @@ -29,7 +43,7 @@ jobs: ydb-acceptance: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 services: ydb: image: ydbplatform/local-ydb:26.3.1.8 @@ -54,6 +68,14 @@ jobs: with: go-version-file: go.mod cache: true + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' - name: Install tested Python runtimes run: python3 -m pip install -r examples/authors/python/requirements.txt # Keep local-ydb acceptance sequential: concurrent tests/compilers can @@ -70,3 +92,45 @@ jobs: go test -p 1 -count=1 -timeout=90s ./... -v cd .. python3 -m python.smoke + - name: Run C# ADO.NET example + working-directory: examples/authors + env: + SQLC_YDB_TEST_DSN: Host=localhost;Port=2136;Database=/local + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + run: dotnet run --project csharp/adonet/Authors.AdoNet.csproj + - name: Run Java native, JDBC, Spring and Hibernate examples sequentially + run: sh examples/authors/java/run-smoke.sh + + cpp-acceptance: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + # Finish SDK compilation before starting YDB on this host. + - name: Build C++ native and userver examples with the pinned SDK image + run: | + docker build -t sqlc-ydb-cpp-tests -f examples/authors/cpp/Dockerfile examples/authors/cpp + docker run --rm -v "$PWD:/workspace" -w /workspace sqlc-ydb-cpp-tests \ + bash -lc 'cmake -S examples/authors/cpp -B examples/authors/cpp/build -GNinja -DCMAKE_PREFIX_PATH=/usr/share/yandex && cmake --build examples/authors/cpp/build --target authors_native authors_userver -j1' + - name: Start one disposable YDB service + run: | + docker run -d --name sqlc-ydb-cpp-server --hostname localhost \ + -p 2136:2136 -e GRPC_PORT=2136 \ + -e YDB_USE_IN_MEMORY_PDISKS=true -e YDB_DEFAULT_LOG_LEVEL=ERROR \ + ydbplatform/local-ydb:26.3.1.8 + for attempt in {1..60}; do + if timeout 10 docker exec sqlc-ydb-cpp-server /health_check; then + exit 0 + fi + sleep 2 + done + docker logs --tail 100 sqlc-ydb-cpp-server + exit 1 + - name: Run C++ native and userver examples sequentially + run: | + docker run --rm --network host -v "$PWD:/workspace" -w /workspace \ + -e SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ + sqlc-ydb-cpp-tests bash examples/authors/cpp/run-smoke.sh + - name: Stop the disposable YDB service + if: always() + run: docker rm -f sqlc-ydb-cpp-server || true diff --git a/.gitignore b/.gitignore index 23ae400..b4087d5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ __pycache__/ *.pyc .venv/ +**/target/ +**/obj/ +examples/**/bin/ +examples/**/build/ diff --git a/README.md b/README.md index eb61e7a..256fa30 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # sqlc-ydb -Generate typed Go and Python query code from YQL for YDB. One executable contains +Generate typed Go, Python, C++, C#, and Java query code from YQL for YDB. One executable contains the parser, semantic analyzer, and generators. Generation works offline and does not require a running YDB, Python, or any separately installed codegen plugin. @@ -18,13 +18,16 @@ go build -o bin/sqlc-ydb ./cmd/sqlc-ydb ./bin/sqlc-ydb diff -f examples/authors/sqlc.yaml ``` -The example generates Go using the native YDB SDK and `database/sql`, and Python -using the native SDK, DB-API, and SQLAlchemy. The generated application needs the +The example generates Go using the native YDB SDK and `database/sql`; Python +using the native SDK, DB-API, and SQLAlchemy; C++ using the native SDK and userver; +C# using `Ydb.Sdk.Ado`; and Java using the native SDK, JDBC, Spring JDBC, and +Hibernate. The generated application needs the corresponding runtime library; the generator itself does not. The example groups generated code and dependencies by language: `go/database/sql`, `go/native`, `python/dbapi`, `python/sqlalchemy`, and -`python/native`. Shared `schema.sql`, `queries.sql`, and `sqlc.yaml` stay in +`python/native`, `cpp/native`, `cpp/userver`, `csharp/adonet`, and +`java/{native,jdbc,spring,hibernate}`. Shared `schema.sql`, `queries.sql`, and `sqlc.yaml` stay in `examples/authors`. ```yaml @@ -59,7 +62,7 @@ exits with status 1 if generated contents differ. The familiar sqlc workflow is the compatibility target. This project has its own implementation and release cycle. It supports only YDB, with built-in generators; external engine/codegen plugins and their protocols are intentionally excluded. -Plugin configurations require migration to `gen.go` / `gen.python`. +Plugin configurations require migration to built-in `gen` entries. The analyzer reads the ANTLR YQL parse tree directly. It resolves names and types before generators see a query. The shared semantic result describes parameters @@ -68,5 +71,6 @@ unresolved types must produce an error rather than an untyped fallback. See [compatibility](docs/compatibility.md), [targets](docs/targets.md), [architecture](docs/architecture.md), and [development](docs/development.md) for -the implemented scope and remaining work. More languages follow after the Go and -Python pipeline is established. +the implemented scope and remaining work. Target-specific configuration and +examples are described in [C++](docs/cpp.md), [C#](docs/csharp.md), and +[Java](docs/java.md). diff --git a/docs/architecture.md b/docs/architecture.md index 143cab7..e244fb7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,6 +7,9 @@ flowchart LR C --> D[Resolved catalog and queries] D --> E[Built-in Go generator] D --> F[Built-in Python generator] + D --> G[Built-in C++ generator] + D --> H[Built-in C# generator] + D --> I[Built-in Java generator] ``` `internal/source` loads files and migration inputs. `internal/analyzer` owns @@ -19,7 +22,7 @@ identity, parameters, result sets and source locations. Nullability is an `Optional` type, and compound type metadata is retained. A table catalog and a query projection are distinct: `SELECT name` does not generate the whole table. -`internal/codegen/golang` and `internal/codegen/python` produce files from that +The language packages in `internal/codegen` produce files from that resolved model. They handle naming, runtime-specific parameter binding, result decoding and resource lifetimes. They do not analyze SQL or load external code. Lexical adaptation of parameter placeholders for a driver is separate from @@ -38,7 +41,7 @@ filesystem transaction covering every output. it and cannot serve as the semantic correctness baseline. - Do not add an intermediate AST. The typed analysis result is necessary for code generation and is not a syntax representation. -- Start with Go and Python. C++, Java, C#, then JS/PHP/Rust follow later. SDK +- Built-in generators cover Go, Python, C++, Java, and C#. JS/PHP/Rust follow later. SDK maintainers are already in the product team and can review generated APIs. - Track upstream product behavior and selectively adapt relevant tests. Record source provenance and retain license notices whenever code is copied. diff --git a/docs/compatibility.md b/docs/compatibility.md index 83dbf06..42ec03d 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -12,6 +12,10 @@ internal data structures and source history are not a dependency. message. A new generator must be implemented in this repository. - `gen.python` selects a built-in generator. `runtime` chooses `ydb`, `dbapi`, or `sqlalchemy`. Go adds `sql_package: ydb` alongside `database/sql`. +- `gen.cpp` selects `runtime: ydb|userver`; `gen.java` selects + `runtime: ydb|jdbc|spring|hibernate`. `native` is an alias for `ydb` in these + two targets. `gen.csharp` uses the modern YDB ADO.NET SDK without a runtime + selector: the SDK's native API is already ADO.NET. - No intermediate AST. ANTLR parse contexts feed semantic analysis directly. ## Implemented workflow @@ -32,6 +36,9 @@ internal data structures and source history are not a dependency. - Python options `package`, `out`, `runtime`, `emit_sync_querier`, `emit_async_querier`. Synchronous generation defaults to enabled; requesting asynchronous generation currently fails explicitly. +- C++ options `namespace`, `out`, `runtime`; C# options `namespace`, `out`; + Java options `package`, `out`, `runtime`. These are built-in extensions to + the sqlc configuration shape, not external plugin options. - Unknown configuration options produce errors. Generation never silently discards an option that has not been implemented. diff --git a/docs/cpp.md b/docs/cpp.md new file mode 100644 index 0000000..7f290c6 --- /dev/null +++ b/docs/cpp.md @@ -0,0 +1,94 @@ +# C++ generation + +The built-in C++ generator emits C++20 for two runtimes: + +- `runtime: ydb` uses the native YDB C++ Query SDK. +- `runtime: userver` uses userver's asynchronous YDB driver. + +Each output contains `models.hpp`, `queries.hpp`, and `queries.cpp`. The configured namespace may contain nested components such as `authors::native`. Generated query methods implement `:one`, `:many`, and `:exec`. A `:one` method returns `std::optional` and reads the first row, so an empty result becomes `std::nullopt`. A `:many` method returns `std::vector`. An `:exec` method checks execution success and intentionally discards any result sets. `:execrows` and unsupported types are rejected during generation. + +## Configuration + +```yaml +gen: + cpp: + namespace: authors::native + out: cpp/native + runtime: ydb +``` + +Use `runtime: userver` for the userver adapter. The configuration layer also accepts `runtime: native` as an alias for `ydb`; the generator's canonical runtime names are `ydb` and `userver`. + +## Ownership and transactions + +`Queries` stores a non-owning reference to the runtime client. The caller must keep `NYdb::NQuery::TQueryClient` or `userver::ydb::TableClient` alive longer than the generated `Queries` object. The native driver and userver component remain caller-owned as well. + +Every native generated method calls `TQueryClient::RetryQuerySync`, obtains a retry-managed `TSession`, and executes one query with `BeginTx(SerializableRW()).CommitTx()`. Every retry attempt rebuilds the parameter object. This is a self-contained transaction per generated method; generated methods do not join a caller-owned transaction. + +Every userver generated method calls `TableClient::ExecuteQuery`. userver performs retries internally and its default operation settings select a committed serializable read-write transaction for that call. Multi-statement caller transactions belong in handwritten code using `TableClient::RetryTx`; generated methods do not join them. + +## Types + +| YQL | Native SDK C++ | userver C++ | +| --- | --- | --- | +| `Bool` | `bool` | `bool` | +| `Int8` / `Uint8` | `std::int8_t` / `std::uint8_t` | same | +| `Int16` / `Uint16` | `std::int16_t` / `std::uint16_t` | same | +| `Int32` / `Uint32` | `std::int32_t` / `std::uint32_t` | same | +| `Int64` / `Uint64` | `std::int64_t` / `std::uint64_t` | same | +| `Float` | `float` | unsupported by userver | +| `Double` | `double` | `double` | +| `String` | `std::string` through SDK `String` accessors | `std::string` | +| `Utf8` | `std::string` through SDK `Utf8` accessors | `userver::ydb::Utf8` | +| `Optional` | `std::optional` | `std::optional` | + +The native mapping retains the `String` versus `Utf8` distinction in its parameter builders and result parsers even though both values use `std::string`. userver uses its strong `Utf8` typedef, so the distinction is also visible in the public C++ type. Nested optionals and non-scalar containers are rejected explicitly. + +Identifiers must be ASCII C++ identifiers, must not be C++20 keywords, and must not start with `_` or the generator-reserved `sqlc_` prefix. Duplicate query, parameter, or result-column names are rejected, as are names that collide with generated row types, the `Queries` class, its client member, or per-query SQL constants. Generated SQL normally remains readable as a multiline raw string. The generator selects a raw-string delimiter absent from the SQL and switches to length-preserving escaped fragments for control bytes, carriage returns, byte-order marks, and invalid UTF-8. + +## Upstream API evidence + +The API was checked on 2026-09-07 against these exact revisions: + +- YDB C++ SDK `main`: [`6ea7a0f93bd97bcb92dca3a4ac948bb743077e50`](https://github.com/ydb-platform/ydb-cpp-sdk/tree/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50). `TQueryClient` declares the `ExecuteQuery` and `RetryQuerySync` overloads in [`client.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/query/client.h#L74-L120). The SDK value API provides width-specific, `String`, `Utf8`, and optional builders/parsers in [`value.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/value/value.h#L328-L510). The maintained basic example demonstrates `RetryQuerySync`, transaction control, parameter construction, and `TResultSetParser` in [`basic_example.cpp`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/examples/basic_example/basic_example.cpp#L163-L247). +- userver `develop`: [`86759637d175baa64f0b3b01f1a027bfedbf795a`](https://github.com/userver-framework/userver/tree/86759637d175baa64f0b3b01f1a027bfedbf795a). `TableClient::ExecuteQuery` and its retry contract are declared in [`table.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/table.hpp#L90-L235). Cursor and typed row extraction are defined in [`response.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/response.hpp#L35-L180). The public primitive mapping, including the absence of `Float` and the distinct `Utf8` strong type, is documented in [`types.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/types.hpp#L15-L75). The implementation shows retry-managed Query SDK execution and per-call transaction selection in [`table.cpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/src/ydb/table.cpp#L420-L445). The official Ubuntu image enables YDB in [`ubuntu-24.04-userver.dockerfile`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/scripts/docker/ubuntu-24.04-userver.dockerfile), and [`SetupYdbCppSDK.cmake`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/cmake/SetupYdbCppSDK.cmake#L4-L55) pins SDK 3.21.1 and requests its `Iam` component. +- YDB documentation `main`: [`2a1fce8e188f51004950d1e8684b395304a229aa`](https://github.com/ydb-platform/ydb/tree/2a1fce8e188f51004950d1e8684b395304a229aa). The [retry guide](https://github.com/ydb-platform/ydb/blob/2a1fce8e188f51004950d1e8684b395304a229aa/ydb/docs/en/core/recipes/ydb-sdk/retry.md) recommends native `RetryQuerySync` with a session and states that all userver `TableClient` methods include retry handling. + +## Build and smoke + +Generate all authors adapters from the repository root: + +```bash +go run ./cmd/sqlc-ydb generate -f examples/authors/sqlc.yaml +``` + +The current YDB C++ SDK release is `v3.22.0`, which publishes Ubuntu 24.04 `libydb-cpp-dev` and `yandex-googleapis-api-common-protos` packages. Its CMake package installs below `/usr/share/yandex`. Native links the real `YDB-CPP-SDK::Driver`, `YDB-CPP-SDK::Params`, and `YDB-CPP-SDK::Query` targets; userver links `userver::ydb`. The pinned userver target also links `YDB-CPP-SDK::ydb-cpp-iam`, so the top-level CMake file requests the SDK `Iam` component before loading userver's installed targets. + +The official `ghcr.io/userver-framework/ubuntu-24.04-userver` image is built with `USERVER_FEATURE_YDB=1` and includes the YDB SDK packages. The compile environment is pinned in `examples/authors/cpp/Dockerfile` to `ghcr.io/userver-framework/ubuntu-24.04-userver@sha256:8b71ba0bdc5f79038d2e639cc7d8f669405db7377b851f7581b09c67183151e4`, which contains userver 3.2-rc and YDB C++ SDK 3.21.1. + +That image's installed `userver-ydb-config.cmake` asks for the obsolete CMake package name `googleapis`, while its real installed SDK package exports `yandex-googleapis-api-common-protos::api-common-protos` from `yandex-googleapis-api-common-protosConfig.cmake`. The Dockerfile makes the exact dependency-name correction and verifies both the old line and the real package file before changing it. It does not add replacement headers, targets, or libraries. Build the small derived image and compile serially to stay within the 2 GB Docker VM: + +```bash +docker build \ + -t sqlc-ydb-authors-cpp \ + -f examples/authors/cpp/Dockerfile . + +docker run --rm \ + -v "$PWD:/workspace" -w /workspace \ + sqlc-ydb-authors-cpp \ + bash -lc 'cmake -S examples/authors/cpp -B /tmp/authors-cpp -GNinja -DCMAKE_PREFIX_PATH="/usr/share/yandex;/usr/local" && cmake --build /tmp/authors-cpp --target authors_native authors_userver -j1' +``` + +Both live smokes expect to run with `examples/authors` as the working directory and use `SQLC_YDB_TEST_DSN`, defaulting in the wrappers to `grpc://localhost:2136/local`. The smoke launchers split that value into the SDK endpoint `grpc://localhost:2136` and database `/local`; this matches `TDriverConfig::SetEndpoint` plus `SetDatabase` and userver's YDB component schema. They create the `authors` table from `schema.sql` without `IF NOT EXISTS`, test maximum `Uint64`, present and null optionals, missing `:one`, the named single-column query, `:many`, and `:exec`, then drop the table. Cleanup is armed only after table creation succeeds. + +```bash +cd examples/authors +SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ + cpp/native/probe.sh /tmp/authors-cpp/native/authors_native + +SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ + cpp/userver/run.sh /tmp/authors-cpp/userver/authors_userver & +userver_pid=$! +trap 'kill "$userver_pid" 2>/dev/null || true' EXIT +cpp/userver/probe.sh +``` diff --git a/docs/csharp.md b/docs/csharp.md new file mode 100644 index 0000000..6c8ce23 --- /dev/null +++ b/docs/csharp.md @@ -0,0 +1,63 @@ +# C# generation + +The built-in C# generator targets the current `Ydb.Sdk` ADO.NET provider. It +generates `Models.cs` and `Queries.cs`; it does not generate a data source, +open a connection, begin a transaction, or dispose caller-owned resources. +Construct `Queries` with an already-open `YdbConnection`. Pass a +caller-owned `YdbTransaction` to the constructor, or use `WithTransaction`. +Every generated operation is async and accepts a `CancellationToken`. + +```yaml +version: "2" +sql: + - engine: ydb + schema: schema.sql + queries: queries.sql + gen: + csharp: + namespace: Authors.AdoNet + out: csharp/adonet +``` + +The generated methods cover `:one`, `:many`, and `:exec`. `:execrows` is +rejected because YDB does not return affected-row counts. SQL is emitted as +portable, escaped C# string fragments so quotes, backslashes, CRLF, and C0 +control characters preserve their exact values without depending on a raw +literal delimiter. + +`:one` returns the first row and throws `InvalidOperationException` if no row +exists. `:many` returns `IReadOnlyList` and `:exec` returns a `Task`. +There is one C# profile: the modern SDK is already an ADO.NET provider. + +## Types and parameters + +The supported scalar types are `Bool`, signed and unsigned integer types, +`Float`, `Double`, `Utf8`, `String`, and `Uuid`, plus one level of +`Optional`. They map to `bool`, the corresponding C# numeric type, +`float`, `double`, `string`, `byte[]`, and `Guid`. Unsupported YQL types fail +generation; there is no `object` or inferred-type fallback. + +Parameters are constructed as `YdbParameter` with an explicit standard +`DbType`, which the YDB provider maps to its concrete YDB type. In particular, +`Uint64` always binds as `DbType.UInt64`, `Utf8` as `DbType.String`, and +`String` as `DbType.Binary`. An optional parameter uses the same explicit type +for a value and for `DBNull.Value`, which lets the provider create a correctly +typed YDB null. + +## SDK evidence and build target + +The API choice was checked against `ydb-platform/ydb` main at +`204baf30e62446f850fc0271979aa309e2932d63` in +`ydb/docs/en/core/reference/languages-and-apis/ado-net/basic-usage.md` and +`type-mapping.md`, and `ydb-platform/ydb-dotnet-sdk` main at +`e35785a671b88f0f05ab6f4f9e15a260c44600f8`. The SDK README calls +`Ydb.Sdk` the ADO.NET provider and demonstrates `YdbDataSource`, +`YdbConnection`, and `YdbCommand`; the provider source exposes +`YdbParameter(string, DbType, object?)` and +`YdbCommand.ExecuteReaderAsync(CancellationToken)`. `YdbParameter` emits a +typed null when its `DbType` is explicit, so `IsNullable` is not the +mechanism used for YQL null typing. + +The authors smoke project targets `net8.0` and pins `Ydb.Sdk` `0.33.3`. +`Ydb.Sdk` belongs to generated-project dependencies, never to sqlc-ydb's Go +module. Use .NET SDK 8.x to build the smoke project. diff --git a/docs/development.md b/docs/development.md index 69444fe..ce73474 100644 --- a/docs/development.md +++ b/docs/development.md @@ -22,8 +22,20 @@ and baseline update command. Generator tests compile generated Go against the selected SDK in a temporary module and execute generated code using mock adapters. Python 3.9 or newer must -be available as `python3` for the Python generator's execution tests. A normal -generator build has no dependency on Python. +be available as `python3` for the Python generator's execution tests. Java 17+ +(`javac` and `java`) and a C++20 compiler are used for exact SQL literal +round-trip checks. A normal generator build needs only Go. + +Published SDK checks are opt-in locally and enabled in CI: + +```sh +SQLC_YDB_CSHARP_DOTNET=dotnet go test ./internal/codegen/csharp +SQLC_YDB_TEST_MAVEN=mvn go test ./internal/codegen/java +``` + +These compile every supported scalar and nullable scalar binding using the +real .NET and Java dependencies. C# also compiles and runs its SQL byte checks. +No generated runtime imports are added to the generator's Go module. The authors example keeps its Go module and tests in `go/`, and Python packages, requirements and tests in `python/`. Schema, queries and generator configuration @@ -50,6 +62,12 @@ The `make test` and `make check` targets also serialize Go packages, so setting the live-test environment variables does not accidentally run language suites in parallel. +C# and the four Java profiles run in successive acceptance steps too. C++ uses +the pinned userver/SDK development image to compile both executables before +starting local-ydb; native and userver runtime probes then execute sequentially. +The test image and its CMake packaging workaround are in +`examples/authors/cpp/Dockerfile`; see [C++](cpp.md) for commands. + ```sh SQLC_YDB_TEST_DSN=grpc://localhost:2136/local go test -p 1 -count=1 -timeout=180s ./internal/codegen/golang -run TestLiveYDB -v ``` @@ -78,6 +96,18 @@ the installed SQLAlchemy library. Integration checks include the maximum Uint64 value, UTF-8 text, optional values, single-column projections, list queries, writes and missing rows. +From `examples/authors`, the additional live checks are: + +```sh +SQLC_YDB_TEST_DSN='Host=localhost;Port=2136;Database=/local' \ + dotnet run --project csharp/adonet/Authors.AdoNet.csproj +SQLC_YDB_TEST_DSN=grpc://localhost:2136/local sh java/run-smoke.sh +``` + +Each creates and drops its own `authors` table only after a successful create. +Use an otherwise empty disposable database. Java/.NET runtime builds and tests +are separate from offline SQL generation. + Build a container with `docker build -t sqlc-ydb:dev .`. The ANTLR-generated Go parser is large; the Docker build limits compile concurrency and uses more frequent garbage collection to reduce peak memory use. diff --git a/docs/java-research.md b/docs/java-research.md new file mode 100644 index 0000000..134d339 --- /dev/null +++ b/docs/java-research.md @@ -0,0 +1,111 @@ +# Java target research + +Checked 2026-09-07 against these current default-branch snapshots: + +| repository | commit | +| --- | --- | +| `ydb-platform/ydb-java-sdk` | `98aab7828816c9b92cd7583c3383865b834da0af` | +| `ydb-platform/ydb-jdbc-driver` | `a2a43af922ae90b01341a116a6cac81364656b24` | +| `ydb-platform/ydb-java-dialects` | `ddd81338501c074f93671914fe914aa1addca3a5` | + +The example pins published artifacts rather than these source snapshots: +SDK BOM `2.4.11`, JDBC `2.4.1`, Hibernate dialect `1.7.0`. Maven Central metadata +on 2026-09-07 reports these as latest/release values. +The Spring Data JDBC dialect is intentionally absent: the generated Spring +profile uses `JdbcTemplate`, not Spring Data repository support. + +## Final generated architecture + +All four profiles emit a final `Queries` class and top-level Java records. +Methods are lower camel case and follow SQL command cardinality: +`Optional` for `:one`, `List` for `:many`, and `void` for `:exec`. +`Uint64` is a Java `long` carrying the original 64-bit pattern. Nullable +`Utf8` is `String`; nullable output is guarded with the driver's `wasNull()` +for JDBC paths and optional-item checks for native paths. + +The constructors borrow application resources: + +* native: `Queries(SessionRetryContext)`; +* JDBC: `Queries(Connection)`; +* Spring: `Queries(JdbcTemplate)`, using `JdbcTemplate.execute` with a + `ConnectionCallback`; +* Hibernate: `Queries(Session)`, using `Session.doReturningWork` and the same + typed JDBC operations. Query projections are records, not generated JPA + entities. + +Generated JDBC, Spring, and Hibernate methods prepare the original declared +YQL, unwrap `tech.ydb.jdbc.YdbPreparedStatement`, and bind `author_id`, +`author_name`, and `biography` by name (the setter adds `$`). This follows the current driver's +`YdbPreparedStatement` API and avoids relying on positional order. The driver +source in `jdbc/src/main/java/tech/ydb/jdbc/query/params/PreparedQuery.java` +(lines 42-73) sorts indexed `$pN` parameters first and then other names; this is +why generated code uses name setters. `MappingSetters.castToUint64` in +`jdbc/src/main/java/tech/ydb/jdbc/common/MappingSetters.java` (lines 370-418) +passes a `Long` to `PrimitiveValue.newUint64`, preserving `-1L` as `2^64-1`. +Generated setters pass a concrete SDK `Value`, which is explicitly handled +by both `SimpleJdbcPrm.setValue` and `ValueFactory.readValue`. This preserves +unsigned and optional types for both declared and inferred parameters. The +custom `setObject(name, object, Type)` overload is not used: its implementation +does not use the supplied `Type` argument. + +SDK `Uint8`, `Uint16`, and `Uint32` constructors mask their signed Java carrier. +Generated methods reject negative or oversized values before calling any SDK +or JDBC method; `Uint64` intentionally preserves all bits of `long`. + +## Verified native SDK path + +The published SDK API used by the native profile is: + +```java +try (GrpcTransport transport = GrpcTransport.forConnectionString(dsn).build(); + QueryClient client = QueryClient.newClient(transport).build()) { + SessionRetryContext retry = SessionRetryContext.create(client).build(); + QueryReader reader = retry.supplyResult(session -> + QueryReader.readFrom(session.createQuery(sql, TxMode.SERIALIZABLE_RW, params))) + .join().getValue(); +} +``` + +The exact classes are in `query/src/main/java/tech/ydb/query/QueryClient.java`, +`QuerySession.java`, `QueryStream.java`, and +`tools/{QueryReader,SessionRetryContext}.java`; `TxMode` is in +`common/src/main/java/tech/ydb/common/transaction/TxMode.java`. DDL in the +smoke uses `TxMode.NONE`; reads and writes use the appropriate query transaction +mode. `QueryReader.getResultSetCount/getResultSet` return +`ResultSetReader`; its `next`, `getColumn`, and `ValueReader` getters decode +rows. `PrimitiveValue.newUint64(long)`, `newText(String)`, and +`OptionalType.emptyValue/newValue` are the verified parameter factories. + +The caller closes transport and query client. `SessionRetryContext` creates and +closes per-operation query sessions internally, so generated methods borrow the +retry context and never close it. + +## Framework notes + +The SQL-first JVM reference is sqlc's own +[Kotlin JDBC output](https://github.com/sqlc-dev/sqlc-gen-kotlin/blob/2c6a78075b1b9a075427b403a07b187bc36e7451/examples/src/main/kotlin/com/example/authors/postgresql/QueriesImpl.kt): +query constants, typed methods/results, a borrowed `Connection`, and owned +prepared statements. The new Java implementation follows that shape without +copying the Kotlin implementation or its plugin protocol. Its `:one` behavior +matches this project's Go/Python adapters (first row), rather than Kotlin's +additional multiple-row check. + +Spring's documented +[JdbcTemplate callbacks](https://docs.spring.io/spring-framework/reference/data-access/jdbc/core.html) +provide connection management and exception translation for handwritten SQL. +Hibernate's documented +[doReturningWork](https://docs.hibernate.org/orm/6.6/javadocs/org/hibernate/SharedSessionContract.html#doReturningWork(org.hibernate.jdbc.ReturningWork)) +provides JDBC access using the session's connection. These are the framework +integration points selected here; inferring JPA entities from SQL projections +is not part of this generator. + +Spring's generated API stays SQL first and works with `JdbcTemplate`; the smoke +uses `SingleConnectionDataSource` only to make connection ownership explicit. +Hibernate uses its native JDBC connection callback, so no entity mapping is +needed for a manual SQL projection. The YDB Hibernate 6 dialect class checked +in the current dialect source is `tech.ydb.hibernate.dialect.YdbDialect`. + +The four smoke programs read and execute `examples/authors/schema.sql`, then +drop `authors` only after their own successful create. They require +`SQLC_YDB_TEST_DSN` and are intended to run sequentially against a disposable +database. They do not run automatically during Maven compilation. diff --git a/docs/java.md b/docs/java.md new file mode 100644 index 0000000..9dd3386 --- /dev/null +++ b/docs/java.md @@ -0,0 +1,85 @@ +# Java generation + +The authors example has four independent Java 17 profiles. The generated API is +SQL first: each profile keeps the query text and produces a final `Queries` +class with top-level row records. `:one` methods return `Optional`, +`:many` methods return `List`, and `:exec` methods return `void`. + +Configure the Java package and its output directory relative to `sqlc.yaml`: + +```yaml +gen: + java: + package: authors.nativeapi + out: java/native/src/main/java/authors/nativeapi + runtime: ydb +``` + +`runtime` accepts `ydb` (also `native`), `jdbc`, `spring`, or `hibernate`. +Files are emitted directly into `out`; match it to your Java package directory. +Each schema table and query projection gets a record, without ORM annotations. + +Build with Java 17 or newer and Maven, then run the smoke programs: + +```sh +cd examples/authors +mvn -f java/pom.xml test-compile +export SQLC_YDB_TEST_DSN=grpc://localhost:2136/local +# Each smoke creates and drops `authors`; use a disposable database. +sh java/run-smoke.sh +``` + +The smoke sources are compile-time examples and require a running target only +when their `main` methods are invoked. Each checks both nullable biography +states, a `Uint64` whose bit pattern is `2^64-1` (`-1L` in Java), result +mapping, and deletion. The runner invokes profiles sequentially. Each creates +and drops its own `authors` table and fails if a table already exists. + +Published dependencies are pinned to YDB SDK BOM `2.4.11`, JDBC `2.4.1`, and +Hibernate YDB dialect `1.7.0`. Spring uses `spring-jdbc` directly. + +The native constructor receives a borrowed +`tech.ydb.query.tools.SessionRetryContext`. The application owns and closes +`GrpcTransport` and `QueryClient`; generated query methods do not close either. +The JDBC constructor receives a borrowed `java.sql.Connection`; statements and +result sets are method-owned and the connection remains application-owned. +The Spring constructor receives a `JdbcTemplate`; the example wraps one +borrowed connection in `SingleConnectionDataSource`. The Hibernate constructor +receives an open `org.hibernate.Session` and runs the same typed JDBC operations +inside `Session.doReturningWork`, so a projection does not require a generated +JPA entity. + +For JDBC, variables are bound through the driver's `YdbPreparedStatement` +name-based `setObject` with an explicitly typed SDK `Value`. Names omit the +leading `$`, which the driver adds itself. This is required because the driver orders +indexed `$p1` parameters first and then other names alphabetically. `Uint64` +uses `long` as a bit-preserving representation, so `-1L` must remain `-1L`; do +not convert it through `int` or floating point. Nullable `Utf8` is `String` +with a null binding and nullable result; binary YQL `String` values are +`byte[]` in the generated Java API. +Explicit values also retain inferred YQL types when the SQL has no `DECLARE`. +`Uint8`, `Uint16`, and `Uint32` inputs are checked before execution, so a wider +Java integer cannot be silently truncated by an SDK constructor. + +Supported scalar types are `Bool`, signed and unsigned integers, `Float`, +`Double`, `Utf8`, and `String`, plus one level of `Optional`. Optional +primitives use boxed Java types; unsupported types fail generation. Native +query methods execute one transaction per method using `SERIALIZABLE_RW`. +JDBC, Spring, and Hibernate methods use the caller's transaction; they never +commit, roll back, or close caller-owned connections or sessions. +Before using the Hibernate adapter, flush any pending ORM changes that the +query must see: `doReturningWork` does not infer Hibernate entity flush rules +from YQL. Transaction boundaries and entity lifecycle remain application-owned. + +SQL uses Java 17 text blocks with escaped delimiters, control characters, and +trailing whitespace. Literal tests compile and execute the emitted Java and +compare exact UTF-8 bytes with the original SQL. + +The native implementation is based on the current SDK source snapshot +`98aab7828816c9b92cd7583c3383865b834da0af`: +`GrpcTransport.forConnectionString(url).build()`, +`QueryClient.newClient(transport).build()`, and +`SessionRetryContext.create(client).build()`. The JDBC snapshot is +`a2a43af922ae90b01341a116a6cac81364656b24`; the dialect snapshot is +`ddd81338501c074f93671914fe914aa1addca3a5`. Exact source paths and published +version notes are in [java-research.md](java-research.md). diff --git a/docs/targets.md b/docs/targets.md index 9fcb048..ee9c47d 100644 --- a/docs/targets.md +++ b/docs/targets.md @@ -7,6 +7,13 @@ | Python native SDK | `gen.python.runtime: ydb` | dataclasses and `Querier(QuerySessionPool)` | | Python DB-API | `gen.python.runtime: dbapi` | dataclasses and a connection-based `Querier` | | Python SQLAlchemy | `gen.python.runtime: sqlalchemy` | dataclasses and synchronous `Querier` | +| C++ native SDK | `gen.cpp.runtime: ydb` | `Queries(TQueryClient&)`, structs, optional/vector results | +| C++ userver | `gen.cpp.runtime: userver` | `Queries(TableClient&)`, userver YDB bindings | +| C# ADO.NET | `gen.csharp` | async `Queries(YdbConnection)`, records, cancellation and transactions | +| Java native SDK | `gen.java.runtime: ydb` | `Queries(SessionRetryContext)`, Java 17 records | +| Java JDBC | `gen.java.runtime: jdbc` | `Queries(Connection)`, named YDB prepared statements | +| Java Spring JDBC | `gen.java.runtime: spring` | `Queries(JdbcTemplate)`, framework-owned connections | +| Java Hibernate | `gen.java.runtime: hibernate` | `Queries(Session)`, JDBC work inside the session | Output references: the legacy YDB generators preserved at archive commit `da046efe95d7ec65c13cd1f88a9f55804c322f73`, sqlc's Go generator, and @@ -25,8 +32,23 @@ row exists. `:many` returns a Go slice or Python iterable. `:exec` returns only execution status. The selected YDB SDK/driver APIs do not expose a portable affected-row count, so all generators reject `:execrows`. +C++ and Java `:one` results are optional; C# throws `InvalidOperationException` +when no row exists. All +return the first row when present. `:many` returns a typed collection. Java +represents `Uint64` as the full 64-bit `long` bit pattern; use +`Long.toUnsignedString` for unsigned decimal formatting. C++ uses `uint64_t` +and C# uses `ulong`. Binary YQL `String` stays binary in every target. + +The C++/C#/Java generators initially cover scalar primitives and their optional +forms. Unsupported temporal, decimal, or container types fail explicitly; see +the individual [C++](cpp.md), [C#](csharp.md), and [Java](java.md) target docs. +Spring and Hibernate integrations generate query projections and methods; +they do not infer ORM entities from query results. + Generated code uses caller-provided clients/connections. The caller controls -connection lifetime, credentials and transactions. Generated DB-API code closes +connection lifetime and credentials. Transaction behavior is target-specific: +native C++ and Java execute a transaction per method; connection and framework +profiles use the caller's transaction. Generated DB-API code closes its own cursors and does not commit caller-owned transactions. The verified `ydb-sqlalchemy` 0.1.22 has no asynchronous dialect. Requests for diff --git a/examples/authors/cpp/CMakeLists.txt b/examples/authors/cpp/CMakeLists.txt new file mode 100644 index 0000000..8688b32 --- /dev/null +++ b/examples/authors/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.22) +project(sqlc_ydb_authors_cpp LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# userver::ydb links the SDK IAM library in the pinned distribution, so load +# that component before userver imports its installed targets. +find_package(ydb-cpp-sdk REQUIRED COMPONENTS Driver Params Query Iam) +find_package(userver REQUIRED COMPONENTS core ydb) + +add_subdirectory(native) +add_subdirectory(userver) diff --git a/examples/authors/cpp/Dockerfile b/examples/authors/cpp/Dockerfile new file mode 100644 index 0000000..415f04b --- /dev/null +++ b/examples/authors/cpp/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/userver-framework/ubuntu-24.04-userver@sha256:8b71ba0bdc5f79038d2e639cc7d8f669405db7377b851f7581b09c67183151e4 + +# This image's userver 3.2-rc YDB package asks CMake for the obsolete +# "googleapis" package name. The installed YDB SDK 3.21.1 package exports the +# real protobuf target through "yandex-googleapis-api-common-protos" instead. +# Fail if either package layout changes so this metadata-only correction cannot +# silently turn into a partial or stubbed SDK setup. +RUN set -eux; \ + config=/usr/lib/cmake/userver/userver-ydb-config.cmake; \ + package=/usr/share/yandex/lib/cmake/yandex-googleapis-api-common-protos/yandex-googleapis-api-common-protosConfig.cmake; \ + test -f "${config}"; \ + test -f "${package}"; \ + grep -Fq 'find_dependency(googleapis CONFIG)' "${config}"; \ + sed -i 's/find_dependency(googleapis CONFIG)/find_dependency(yandex-googleapis-api-common-protos CONFIG)/' "${config}"; \ + ! grep -Fq 'find_dependency(googleapis CONFIG)' "${config}"; \ + grep -Fq 'find_dependency(yandex-googleapis-api-common-protos CONFIG)' "${config}" + +WORKDIR /workspace diff --git a/examples/authors/cpp/native/CMakeLists.txt b/examples/authors/cpp/native/CMakeLists.txt new file mode 100644 index 0000000..806dcfe --- /dev/null +++ b/examples/authors/cpp/native/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(authors_native main.cpp models.hpp queries.hpp queries.cpp) +target_link_libraries( + authors_native + PRIVATE + YDB-CPP-SDK::Driver + YDB-CPP-SDK::Params + YDB-CPP-SDK::Query +) diff --git a/examples/authors/cpp/native/main.cpp b/examples/authors/cpp/native/main.cpp new file mode 100644 index 0000000..b3759b1 --- /dev/null +++ b/examples/authors/cpp/native/main.cpp @@ -0,0 +1,122 @@ +#include "queries.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::string ReadSchema() { + std::ifstream input{"schema.sql"}; + if (!input) { + throw std::runtime_error("open schema.sql from the examples/authors working directory"); + } + std::ostringstream contents; + contents << input.rdbuf(); + return contents.str(); +} + +struct TestDatabase final { + std::string endpoint; + std::string database; +}; + +TestDatabase ParseTestDsn(const char* dsn) { + const std::string value{dsn}; + constexpr std::string_view kPrefix{"grpc://"}; + const auto database_pos = value.find('/', kPrefix.size()); + if (!value.starts_with(kPrefix) || database_pos == std::string::npos || database_pos == kPrefix.size()) { + throw std::runtime_error("SQLC_YDB_TEST_DSN must look like grpc://host:port/database"); + } + return {value.substr(0, database_pos), value.substr(database_pos)}; +} + +void ExecuteStatement(NYdb::NQuery::TQueryClient& client, const std::string& statement) { + const auto status = client.RetryQuerySync([&](NYdb::NQuery::TSession session) -> NYdb::TStatus { + return session.ExecuteQuery( + statement, + NYdb::NQuery::TTxControl::NoTx() + ).GetValueSync(); + }); + NYdb::ThrowOnError(status); +} + +class CreatedAuthorsTable final { +public: + explicit CreatedAuthorsTable(NYdb::NQuery::TQueryClient& client) : client_(client) {} + ~CreatedAuthorsTable() { + if (active_) { + try { + ExecuteStatement(client_, "DROP TABLE authors;"); + } catch (...) { + } + } + } + + void Drop() { + ExecuteStatement(client_, "DROP TABLE authors;"); + active_ = false; + } + +private: + NYdb::NQuery::TQueryClient& client_; + bool active_{true}; +}; + +} // namespace + +int main() { + const char* dsn = std::getenv("SQLC_YDB_TEST_DSN"); + if (!dsn || !*dsn) { + std::cerr << "SQLC_YDB_TEST_DSN is required\n"; + return 2; + } + + try { + const auto test_database = ParseTestDsn(dsn); + NYdb::TDriverConfig driver_config; + driver_config.SetEndpoint(test_database.endpoint).SetDatabase(test_database.database); + NYdb::TDriver driver{driver_config}; + NYdb::NQuery::TQueryClient client{driver}; + ExecuteStatement(client, ReadSchema()); + CreatedAuthorsTable created_table{client}; + authors::native::Queries queries{client}; + + constexpr std::uint64_t kMaxId = std::numeric_limits::max(); + queries.UpsertAuthor(kMaxId, "C++ SDK", std::optional{"present"}); + queries.UpsertAuthor(kMaxId - 1, "optional null", std::nullopt); + + const auto max_author = queries.GetAuthor(kMaxId); + const auto null_author = queries.GetAuthor(kMaxId - 1); + const auto missing_author = queries.GetAuthor(kMaxId - 2); + const auto name = queries.GetAuthorName(kMaxId); + if (!max_author || max_author->id != kMaxId || max_author->bio != std::optional{"present"} || + !null_author || null_author->bio || missing_author || !name || name->name != "C++ SDK") { + throw std::runtime_error("native C++ generated adapter returned unexpected boundary values"); + } + const auto authors = queries.ListAuthors(); + if (authors.size() != 2) { + throw std::runtime_error("native C++ generated adapter returned an unexpected row count"); + } + queries.DeleteAuthor(kMaxId); + queries.DeleteAuthor(kMaxId - 1); + created_table.Drop(); + driver.Stop(true); + std::cout << "native C++ generated adapter ok; rows=" << authors.size() << '\n'; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/examples/authors/cpp/native/models.hpp b/examples/authors/cpp/native/models.hpp new file mode 100644 index 0000000..21a23e4 --- /dev/null +++ b/examples/authors/cpp/native/models.hpp @@ -0,0 +1,26 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include +#include +#include + +namespace authors::native { + +struct GetAuthorRow final { + std::uint64_t id; + std::string name; + std::optional bio; +}; + +struct ListAuthorsRow final { + std::uint64_t id; + std::string name; + std::optional bio; +}; + +struct GetAuthorNameRow final { + std::string name; +}; + +} // namespace authors::native diff --git a/examples/authors/cpp/native/probe.sh b/examples/authors/cpp/native/probe.sh new file mode 100755 index 0000000..f2d7e37 --- /dev/null +++ b/examples/authors/cpp/native/probe.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${SQLC_YDB_TEST_DSN:=grpc://localhost:2136/local}" +export SQLC_YDB_TEST_DSN +exec "${1:-./authors_native}" diff --git a/examples/authors/cpp/native/queries.cpp b/examples/authors/cpp/native/queries.cpp new file mode 100644 index 0000000..41713cb --- /dev/null +++ b/examples/authors/cpp/native/queries.cpp @@ -0,0 +1,157 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#include "queries.hpp" + +#include +#include + +namespace authors::native { +namespace { + +const std::string kGetAuthorSql = R"sqlc(-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;)sqlc"; + +const std::string kListAuthorsSql = R"sqlc(-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;)sqlc"; + +const std::string kGetAuthorNameSql = R"sqlc(-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;)sqlc"; + +const std::string kUpsertAuthorSql = R"sqlc(-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);)sqlc"; + +const std::string kDeleteAuthorSql = R"sqlc(-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;)sqlc"; + +} // namespace + +std::optional Queries::GetAuthor(std::uint64_t author_id) const { + std::optional sqlc_result_set; + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_params = NYdb::TParamsBuilder() + .AddParam("$author_id").Uint64(author_id).Build() + .Build(); + auto sqlc_result = sqlc_session.ExecuteQuery( + kGetAuthorSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), + sqlc_params + ).GetValueSync(); + if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) { + sqlc_result_set = sqlc_result.GetResultSet(0); + } + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); + if (!sqlc_result_set) { + throw std::runtime_error("GetAuthor: successful query returned no result set"); + } + NYdb::TResultSetParser sqlc_parser(*sqlc_result_set); + if (!sqlc_parser.TryNextRow()) { + return std::nullopt; + } + GetAuthorRow sqlc_row{ + sqlc_parser.ColumnParser("id").GetUint64(), + sqlc_parser.ColumnParser("name").GetUtf8(), + sqlc_parser.ColumnParser("bio").GetOptionalUtf8(), + }; + return sqlc_row; +} + +std::vector Queries::ListAuthors() const { + std::optional sqlc_result_set; + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_result = sqlc_session.ExecuteQuery( + kListAuthorsSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx() + ).GetValueSync(); + if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) { + sqlc_result_set = sqlc_result.GetResultSet(0); + } + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); + if (!sqlc_result_set) { + throw std::runtime_error("ListAuthors: successful query returned no result set"); + } + NYdb::TResultSetParser sqlc_parser(*sqlc_result_set); + std::vector sqlc_rows; + sqlc_rows.reserve(sqlc_result_set->RowsCount()); + while (sqlc_parser.TryNextRow()) { + sqlc_rows.push_back(ListAuthorsRow{ + sqlc_parser.ColumnParser("id").GetUint64(), + sqlc_parser.ColumnParser("name").GetUtf8(), + sqlc_parser.ColumnParser("bio").GetOptionalUtf8(), + }); + } + return sqlc_rows; +} + +std::optional Queries::GetAuthorName(std::uint64_t author_id) const { + std::optional sqlc_result_set; + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_params = NYdb::TParamsBuilder() + .AddParam("$author_id").Uint64(author_id).Build() + .Build(); + auto sqlc_result = sqlc_session.ExecuteQuery( + kGetAuthorNameSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), + sqlc_params + ).GetValueSync(); + if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) { + sqlc_result_set = sqlc_result.GetResultSet(0); + } + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); + if (!sqlc_result_set) { + throw std::runtime_error("GetAuthorName: successful query returned no result set"); + } + NYdb::TResultSetParser sqlc_parser(*sqlc_result_set); + if (!sqlc_parser.TryNextRow()) { + return std::nullopt; + } + GetAuthorNameRow sqlc_row{ + sqlc_parser.ColumnParser("name").GetUtf8(), + }; + return sqlc_row; +} + +void Queries::UpsertAuthor(std::uint64_t author_id, const std::string& author_name, const std::optional& biography) const { + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_params = NYdb::TParamsBuilder() + .AddParam("$author_id").Uint64(author_id).Build() + .AddParam("$author_name").Utf8(author_name).Build() + .AddParam("$biography").OptionalUtf8(biography).Build() + .Build(); + auto sqlc_result = sqlc_session.ExecuteQuery( + kUpsertAuthorSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), + sqlc_params + ).GetValueSync(); + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); +} + +void Queries::DeleteAuthor(std::uint64_t author_id) const { + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_params = NYdb::TParamsBuilder() + .AddParam("$author_id").Uint64(author_id).Build() + .Build(); + auto sqlc_result = sqlc_session.ExecuteQuery( + kDeleteAuthorSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), + sqlc_params + ).GetValueSync(); + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); +} + +} // namespace authors::native diff --git a/examples/authors/cpp/native/queries.hpp b/examples/authors/cpp/native/queries.hpp new file mode 100644 index 0000000..b41631f --- /dev/null +++ b/examples/authors/cpp/native/queries.hpp @@ -0,0 +1,29 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include "models.hpp" + +#include +#include +#include +#include + +#include + +namespace authors::native { + +class Queries final { +public: + explicit Queries(NYdb::NQuery::TQueryClient& client) noexcept : client_(client) {} + + std::optional GetAuthor(std::uint64_t author_id) const; + std::vector ListAuthors() const; + std::optional GetAuthorName(std::uint64_t author_id) const; + void UpsertAuthor(std::uint64_t author_id, const std::string& author_name, const std::optional& biography) const; + void DeleteAuthor(std::uint64_t author_id) const; + +private: + NYdb::NQuery::TQueryClient& client_; +}; + +} // namespace authors::native diff --git a/examples/authors/cpp/run-smoke.sh b/examples/authors/cpp/run-smoke.sh new file mode 100755 index 0000000..1b726c3 --- /dev/null +++ b/examples/authors/cpp/run-smoke.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${SQLC_YDB_TEST_DSN:?set SQLC_YDB_TEST_DSN to a disposable YDB database}" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +build_dir="${1:-${script_dir}/build}" +cd "${script_dir}/.." + +"${build_dir}/native/authors_native" + +cpp/userver/run.sh "${build_dir}/userver/authors_userver" & +userver_pid=$! +cleanup() { + kill "${userver_pid}" 2>/dev/null || true + wait "${userver_pid}" 2>/dev/null || true +} +trap cleanup EXIT + +# Wait for the listener without retrying a failed database exercise. +ready=false +for attempt in {1..60}; do + if ! kill -0 "${userver_pid}" 2>/dev/null; then + echo 'userver exited before opening its listener' >&2 + exit 1 + fi + if (echo > /dev/tcp/127.0.0.1/8080) 2>/dev/null; then + ready=true + break + fi + sleep 1 +done +if [[ "${ready}" != true ]]; then + echo 'userver did not open its listener within 60 seconds' >&2 + exit 1 +fi +cpp/userver/probe.sh diff --git a/examples/authors/cpp/userver/CMakeLists.txt b/examples/authors/cpp/userver/CMakeLists.txt new file mode 100644 index 0000000..e38e0d4 --- /dev/null +++ b/examples/authors/cpp/userver/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(authors_userver main.cpp smoke_handler.cpp models.hpp queries.hpp queries.cpp) +target_link_libraries(authors_userver PRIVATE userver::ydb) diff --git a/examples/authors/cpp/userver/main.cpp b/examples/authors/cpp/userver/main.cpp new file mode 100644 index 0000000..3cce506 --- /dev/null +++ b/examples/authors/cpp/userver/main.cpp @@ -0,0 +1,17 @@ +#include "smoke_handler.hpp" + +#include +#include +#include +#include +#include + +int main(int argc, char* argv[]) { + auto components = + components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, components); +} diff --git a/examples/authors/cpp/userver/models.hpp b/examples/authors/cpp/userver/models.hpp new file mode 100644 index 0000000..6c9b2b7 --- /dev/null +++ b/examples/authors/cpp/userver/models.hpp @@ -0,0 +1,28 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include +#include +#include + +#include + +namespace authors::userver { + +struct GetAuthorRow final { + std::uint64_t id; + ::userver::ydb::Utf8 name; + std::optional<::userver::ydb::Utf8> bio; +}; + +struct ListAuthorsRow final { + std::uint64_t id; + ::userver::ydb::Utf8 name; + std::optional<::userver::ydb::Utf8> bio; +}; + +struct GetAuthorNameRow final { + ::userver::ydb::Utf8 name; +}; + +} // namespace authors::userver diff --git a/examples/authors/cpp/userver/probe.sh b/examples/authors/cpp/userver/probe.sh new file mode 100755 index 0000000..f700ed8 --- /dev/null +++ b/examples/authors/cpp/userver/probe.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${AUTHORS_USERVER_URL:=http://127.0.0.1:8080/smoke}" +curl --fail --silent --show-error "${AUTHORS_USERVER_URL}" diff --git a/examples/authors/cpp/userver/queries.cpp b/examples/authors/cpp/userver/queries.cpp new file mode 100644 index 0000000..b5bd527 --- /dev/null +++ b/examples/authors/cpp/userver/queries.cpp @@ -0,0 +1,103 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#include "queries.hpp" + +#include +#include + +namespace authors::userver { +namespace { + +const ::userver::ydb::Query kGetAuthorQuery{ + R"sqlc(-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;)sqlc", + ::userver::ydb::Query::NameLiteral{"GetAuthor"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +const ::userver::ydb::Query kListAuthorsQuery{ + R"sqlc(-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;)sqlc", + ::userver::ydb::Query::NameLiteral{"ListAuthors"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +const ::userver::ydb::Query kGetAuthorNameQuery{ + R"sqlc(-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;)sqlc", + ::userver::ydb::Query::NameLiteral{"GetAuthorName"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +const ::userver::ydb::Query kUpsertAuthorQuery{ + R"sqlc(-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);)sqlc", + ::userver::ydb::Query::NameLiteral{"UpsertAuthor"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +const ::userver::ydb::Query kDeleteAuthorQuery{ + R"sqlc(-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;)sqlc", + ::userver::ydb::Query::NameLiteral{"DeleteAuthor"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +} // namespace + +std::optional Queries::GetAuthor(std::uint64_t author_id) const { + auto sqlc_response = this->client_.ExecuteQuery(kGetAuthorQuery, "$author_id", author_id); + auto sqlc_cursor = sqlc_response.GetSingleCursor(); + if (sqlc_cursor.empty()) { + return std::nullopt; + } + auto sqlc_row = sqlc_cursor.GetFirstRow(); + return GetAuthorRow{ + sqlc_row.Get("id"), + sqlc_row.Get<::userver::ydb::Utf8>("name"), + sqlc_row.Get>("bio"), + }; +} + +std::vector Queries::ListAuthors() const { + auto sqlc_response = this->client_.ExecuteQuery(kListAuthorsQuery); + auto sqlc_cursor = sqlc_response.GetSingleCursor(); + std::vector sqlc_rows; + sqlc_rows.reserve(sqlc_cursor.size()); + for (auto sqlc_row : sqlc_cursor) { + sqlc_rows.push_back(ListAuthorsRow{ + sqlc_row.Get("id"), + sqlc_row.Get<::userver::ydb::Utf8>("name"), + sqlc_row.Get>("bio"), + }); + } + return sqlc_rows; +} + +std::optional Queries::GetAuthorName(std::uint64_t author_id) const { + auto sqlc_response = this->client_.ExecuteQuery(kGetAuthorNameQuery, "$author_id", author_id); + auto sqlc_cursor = sqlc_response.GetSingleCursor(); + if (sqlc_cursor.empty()) { + return std::nullopt; + } + auto sqlc_row = sqlc_cursor.GetFirstRow(); + return GetAuthorNameRow{ + sqlc_row.Get<::userver::ydb::Utf8>("name"), + }; +} + +void Queries::UpsertAuthor(std::uint64_t author_id, const ::userver::ydb::Utf8& author_name, const std::optional<::userver::ydb::Utf8>& biography) const { + static_cast(this->client_.ExecuteQuery(kUpsertAuthorQuery, "$author_id", author_id, "$author_name", author_name, "$biography", biography)); +} + +void Queries::DeleteAuthor(std::uint64_t author_id) const { + static_cast(this->client_.ExecuteQuery(kDeleteAuthorQuery, "$author_id", author_id)); +} + +} // namespace authors::userver diff --git a/examples/authors/cpp/userver/queries.hpp b/examples/authors/cpp/userver/queries.hpp new file mode 100644 index 0000000..62669a5 --- /dev/null +++ b/examples/authors/cpp/userver/queries.hpp @@ -0,0 +1,29 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include "models.hpp" + +#include +#include +#include +#include + +#include + +namespace authors::userver { + +class Queries final { +public: + explicit Queries(::userver::ydb::TableClient& client) noexcept : client_(client) {} + + std::optional GetAuthor(std::uint64_t author_id) const; + std::vector ListAuthors() const; + std::optional GetAuthorName(std::uint64_t author_id) const; + void UpsertAuthor(std::uint64_t author_id, const ::userver::ydb::Utf8& author_name, const std::optional<::userver::ydb::Utf8>& biography) const; + void DeleteAuthor(std::uint64_t author_id) const; + +private: + ::userver::ydb::TableClient& client_; +}; + +} // namespace authors::userver diff --git a/examples/authors/cpp/userver/run.sh b/examples/authors/cpp/userver/run.sh new file mode 100755 index 0000000..6e6cfe2 --- /dev/null +++ b/examples/authors/cpp/userver/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${SQLC_YDB_TEST_DSN:=grpc://localhost:2136/local}" + +if [[ ! "${SQLC_YDB_TEST_DSN}" =~ ^(grpc://[^/]+)(/.*)$ ]]; then + echo "SQLC_YDB_TEST_DSN must look like grpc://host:port/database" >&2 + exit 2 +fi + +endpoint="${BASH_REMATCH[1]}" +database="${BASH_REMATCH[2]}" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +exec "${1:-./authors_userver}" \ + --config "${2:-${script_dir}/static_config.yaml}" \ + --config_vars <(printf '{"ydb-endpoint":"%s","ydb-database":"%s"}\n' "${endpoint}" "${database}") diff --git a/examples/authors/cpp/userver/smoke_handler.cpp b/examples/authors/cpp/userver/smoke_handler.cpp new file mode 100644 index 0000000..ebb0ac4 --- /dev/null +++ b/examples/authors/cpp/userver/smoke_handler.cpp @@ -0,0 +1,85 @@ +#include "smoke_handler.hpp" + +#include "queries.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace authors::userver_example { + +namespace { + +std::string ReadSchema() { + std::ifstream input{"schema.sql"}; + if (!input) { + throw std::runtime_error("open schema.sql from the examples/authors working directory"); + } + std::ostringstream contents; + contents << input.rdbuf(); + return contents.str(); +} + +class CreatedAuthorsTable final { +public: + explicit CreatedAuthorsTable(ydb::TableClient& client) : client_(client) {} + ~CreatedAuthorsTable() { + if (active_) { + try { + client_.ExecuteSchemeQuery("DROP TABLE authors;"); + } catch (...) { + } + } + } + + void Drop() { + client_.ExecuteSchemeQuery("DROP TABLE authors;"); + active_ = false; + } + +private: + ydb::TableClient& client_; + bool active_{true}; +}; + +} // namespace + +SmokeHandler::SmokeHandler(const components::ComponentConfig& config, const components::ComponentContext& context) + : HttpHandlerBase(config, context), + client_(context.FindComponent().GetTableClient("authors")) {} + +std::string SmokeHandler::HandleRequest(server::http::HttpRequest&, server::request::RequestContext&) const { + client_->ExecuteSchemeQuery(ReadSchema()); + CreatedAuthorsTable created_table{*client_}; + ::authors::userver::Queries queries{*client_}; + constexpr std::uint64_t kMaxId = std::numeric_limits::max(); + queries.UpsertAuthor( + kMaxId, + ydb::Utf8{"userver"}, + std::optional{ydb::Utf8{"present"}} + ); + queries.UpsertAuthor(kMaxId - 1, ydb::Utf8{"optional null"}, std::nullopt); + const auto max_author = queries.GetAuthor(kMaxId); + const auto null_author = queries.GetAuthor(kMaxId - 1); + const auto missing_author = queries.GetAuthor(kMaxId - 2); + const auto name = queries.GetAuthorName(kMaxId); + if (!max_author || max_author->id != kMaxId || !max_author->bio || + max_author->bio->GetUnderlying() != "present" || !null_author || null_author->bio || missing_author || !name || + name->name.GetUnderlying() != "userver") { + throw std::runtime_error("userver generated adapter returned unexpected boundary values"); + } + const auto row_count = queries.ListAuthors().size(); + if (row_count != 2) { + throw std::runtime_error("userver generated adapter returned an unexpected row count"); + } + queries.DeleteAuthor(kMaxId); + queries.DeleteAuthor(kMaxId - 1); + created_table.Drop(); + return "userver generated adapter ok; rows=" + std::to_string(row_count) + "\n"; +} + +} // namespace authors::userver_example diff --git a/examples/authors/cpp/userver/smoke_handler.hpp b/examples/authors/cpp/userver/smoke_handler.hpp new file mode 100644 index 0000000..2c7c30c --- /dev/null +++ b/examples/authors/cpp/userver/smoke_handler.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace authors::userver_example { + +class SmokeHandler final : public server::handlers::HttpHandlerBase { +public: + static constexpr std::string_view kName = "handler-authors-smoke"; + + SmokeHandler(const components::ComponentConfig& config, const components::ComponentContext& context); + + std::string HandleRequest( + server::http::HttpRequest& request, + server::request::RequestContext& context + ) const override; + +private: + std::shared_ptr client_; +}; + +} // namespace authors::userver_example diff --git a/examples/authors/cpp/userver/static_config.yaml b/examples/authors/cpp/userver/static_config.yaml new file mode 100644 index 0000000..1741380 --- /dev/null +++ b/examples/authors/cpp/userver/static_config.yaml @@ -0,0 +1,38 @@ +components_manager: + task_processors: + main-task-processor: + worker_threads: 2 + fs-task-processor: + worker_threads: 1 + default_task_processor: main-task-processor + components: + server: + listener: + port: 8080 + task_processor: main-task-processor + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: info + overflow_behavior: discard + default-secdist-provider: + inline: {} + dynamic-config: + updates-enabled: false + fs-task-processor: fs-task-processor + ydb: + operation-settings: + client-timeout: 2s + retries: 3 + databases: + authors: + endpoint: $ydb-endpoint + database: $ydb-database + min_pool_size: 1 + max_pool_size: 2 + handler-authors-smoke: + method: GET + path: /smoke + task_processor: main-task-processor diff --git a/examples/authors/csharp/adonet/Authors.AdoNet.csproj b/examples/authors/csharp/adonet/Authors.AdoNet.csproj new file mode 100644 index 0000000..534cd6f --- /dev/null +++ b/examples/authors/csharp/adonet/Authors.AdoNet.csproj @@ -0,0 +1,12 @@ + + + Exe + net8.0 + enable + enable + true + + + + + diff --git a/examples/authors/csharp/adonet/Models.cs b/examples/authors/csharp/adonet/Models.cs new file mode 100644 index 0000000..c564b9c --- /dev/null +++ b/examples/authors/csharp/adonet/Models.cs @@ -0,0 +1,32 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#nullable enable +using System; + +namespace Authors.AdoNet; +public sealed record Authors( + ulong ID, + string Name, + string? Bio +); + +public sealed record GetAuthorRow( + ulong ID, + string Name, + string? Bio +); + +public sealed record ListAuthorsRow( + ulong ID, + string Name, + string? Bio +); + +public sealed record GetAuthorNameRow( + string Name +); + +public sealed record UpsertAuthorParams( + ulong AuthorID, + string AuthorName, + string? Biography +); diff --git a/examples/authors/csharp/adonet/Program.cs b/examples/authors/csharp/adonet/Program.cs new file mode 100644 index 0000000..64a3874 --- /dev/null +++ b/examples/authors/csharp/adonet/Program.cs @@ -0,0 +1,33 @@ +using System; +using System.Threading; +using Ydb.Sdk.Ado; + +namespace Authors.AdoNet; + +public static class Program +{ + public static async Task Main() + { + var dsn = Environment.GetEnvironmentVariable("SQLC_YDB_TEST_DSN"); + if (string.IsNullOrWhiteSpace(dsn)) + { + Console.Error.WriteLine("SQLC_YDB_TEST_DSN is required (for example Host=localhost;Port=2136;Database=/local)"); + return 2; + } + using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(45)); + await using var dataSource = new YdbDataSource(dsn); + await using var connection = await dataSource.OpenConnectionAsync(cancellationSource.Token); + await using (var create = new YdbCommand(await File.ReadAllTextAsync("schema.sql", cancellationSource.Token), connection)) + await create.ExecuteNonQueryAsync(cancellationSource.Token); + try + { + await Smoke.ExerciseAsync(connection, cancellationSource.Token); + } + finally + { + await using var drop = new YdbCommand("DROP TABLE authors;", connection); + await drop.ExecuteNonQueryAsync(CancellationToken.None); + } + return 0; + } +} diff --git a/examples/authors/csharp/adonet/Queries.cs b/examples/authors/csharp/adonet/Queries.cs new file mode 100644 index 0000000..6a940cd --- /dev/null +++ b/examples/authors/csharp/adonet/Queries.cs @@ -0,0 +1,121 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#nullable enable +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; +using Ydb.Sdk.Ado; + +namespace Authors.AdoNet; + + +public sealed class Queries +{ + private readonly YdbConnection _connection; + private readonly YdbTransaction? _transaction; + + public Queries(YdbConnection connection, YdbTransaction? transaction = null) + { + _connection = connection ?? throw new ArgumentNullException(nameof(connection)); + _transaction = transaction; + } + + public Queries WithTransaction(YdbTransaction transaction) => new(_connection, transaction ?? throw new ArgumentNullException(nameof(transaction))); + + private const string SqlGetAuthor = + "-- name: GetAuthor :one\n" + + "DECLARE $author_id AS Uint64;\n" + + "SELECT id, name, bio FROM authors WHERE id = $author_id;"; + + public async Task GetAuthorAsync(ulong AuthorID, CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlGetAuthor, _connection) { Transaction = _transaction }; + command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, AuthorID)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("query returned no rows"); + } + return GetAuthorRowFrom(reader); + } + + private static GetAuthorRow GetAuthorRowFrom(DbDataReader reader) => new( + reader.GetFieldValue(0), + reader.GetFieldValue(1), + reader.IsDBNull(2) ? null : reader.GetFieldValue(2) + ); + + private const string SqlListAuthors = + "-- name: ListAuthors :many\n" + + "SELECT id, name, bio FROM authors ORDER BY id;"; + + public async Task> ListAuthorsAsync(CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlListAuthors, _connection) { Transaction = _transaction }; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + var rows = new List(); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rows.Add(ListAuthorsRowFrom(reader)); + } + return rows; + } + + private static ListAuthorsRow ListAuthorsRowFrom(DbDataReader reader) => new( + reader.GetFieldValue(0), + reader.GetFieldValue(1), + reader.IsDBNull(2) ? null : reader.GetFieldValue(2) + ); + + private const string SqlGetAuthorName = + "-- name: GetAuthorName :one\n" + + "DECLARE $author_id AS Uint64;\n" + + "SELECT name FROM authors WHERE id = $author_id;"; + + public async Task GetAuthorNameAsync(ulong AuthorID, CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlGetAuthorName, _connection) { Transaction = _transaction }; + command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, AuthorID)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("query returned no rows"); + } + return GetAuthorNameRowFrom(reader); + } + + private static GetAuthorNameRow GetAuthorNameRowFrom(DbDataReader reader) => new( + reader.GetFieldValue(0) + ); + + private const string SqlUpsertAuthor = + "-- name: UpsertAuthor :exec\n" + + "DECLARE $author_id AS Uint64;\n" + + "DECLARE $author_name AS Utf8;\n" + + "DECLARE $biography AS Optional;\n" + + "UPSERT INTO authors (id, name, bio)\n" + + "VALUES ($author_id, $author_name, $biography);"; + + public async Task UpsertAuthorAsync(UpsertAuthorParams args, CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlUpsertAuthor, _connection) { Transaction = _transaction }; + command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, args.AuthorID)); + command.Parameters.Add(new YdbParameter("$author_name", DbType.String, args.AuthorName)); + command.Parameters.Add(new YdbParameter("$biography", DbType.String, (object?)args.Biography ?? DBNull.Value)); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private const string SqlDeleteAuthor = + "-- name: DeleteAuthor :exec\n" + + "DECLARE $author_id AS Uint64;\n" + + "DELETE FROM authors WHERE id = $author_id;"; + + public async Task DeleteAuthorAsync(ulong AuthorID, CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlDeleteAuthor, _connection) { Transaction = _transaction }; + command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, AuthorID)); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/examples/authors/csharp/adonet/README.md b/examples/authors/csharp/adonet/README.md new file mode 100644 index 0000000..0e94963 --- /dev/null +++ b/examples/authors/csharp/adonet/README.md @@ -0,0 +1,23 @@ +# Authors ADO.NET smoke + +`Models.cs` and `Queries.cs` in this directory are generated by `sqlc-ydb` +from the parent authors schema and queries. Generated `Queries` accepts a +caller-owned `YdbConnection` and optional `YdbTransaction`. `Program.cs` is an +opt-in live smoke: it owns a `YdbDataSource` only at the application boundary. + +Run `sqlc-ydb generate -f sqlc.yaml` from `examples/authors`, then build: + +```sh +dotnet build csharp/adonet/Authors.AdoNet.csproj +``` + +Run the live smoke against a disposable database without an `authors` table: + +```sh +SQLC_YDB_TEST_DSN='Host=localhost;Port=2136;Database=/local' \ + dotnet run --project csharp/adonet/Authors.AdoNet.csproj +``` + +It creates the table using `schema.sql`, uses `ulong.MaxValue`, checks both +null and non-null `Optional` and absent rows, then drops its own table. +A failed create never causes an existing table to be dropped. diff --git a/examples/authors/csharp/adonet/Smoke.cs b/examples/authors/csharp/adonet/Smoke.cs new file mode 100644 index 0000000..56e343f --- /dev/null +++ b/examples/authors/csharp/adonet/Smoke.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Ydb.Sdk.Ado; + +namespace Authors.AdoNet; + +// Exercises every generated command. The caller owns YdbDataSource and the +// connection lifetime; this method neither creates nor disposes either. +public static class Smoke +{ + public static async Task ExerciseAsync(YdbConnection connection, CancellationToken cancellationToken = default) + { + var queries = new Queries(connection); + const ulong id = ulong.MaxValue; + await queries.UpsertAuthorAsync(new UpsertAuthorParams(id, "sqlc-ydb C# smoke", null), cancellationToken); + GetAuthorRow author = await queries.GetAuthorAsync(id, cancellationToken); + if (author.ID != id || author.Name != "sqlc-ydb C# smoke" || author.Bio is not null) + throw new InvalidOperationException("optional Utf8 null or Uint64 binding changed"); + + await queries.UpsertAuthorAsync(new UpsertAuthorParams(id, "sqlc-ydb C# smoke", "present"), cancellationToken); + author = await queries.GetAuthorAsync(id, cancellationToken); + if (author.Bio != "present") + throw new InvalidOperationException("optional Utf8 value binding changed"); + + GetAuthorNameRow name = await queries.GetAuthorNameAsync(id, cancellationToken); + if (name.Name != author.Name) + throw new InvalidOperationException("single-column :one mapping changed"); + IReadOnlyList authors = await queries.ListAuthorsAsync(cancellationToken); + if (!authors.Any(row => row.ID == id && row.Bio == "present")) + throw new InvalidOperationException(":many mapping changed"); + await queries.DeleteAuthorAsync(id, cancellationToken); + try + { + await queries.GetAuthorAsync(id, cancellationToken); + throw new Exception("missing-row query unexpectedly succeeded"); + } + catch (InvalidOperationException error) when (error.Message == "query returned no rows") + { + // :one reports an absent row without inventing a default record. + } + } +} diff --git a/examples/authors/java/hibernate/pom.xml b/examples/authors/java/hibernate/pom.xml new file mode 100644 index 0000000..efc79e1 --- /dev/null +++ b/examples/authors/java/hibernate/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + example.com.sqlc-ydb-authors + authors-java + 0.1.0-SNAPSHOT + .. + + authors-java-hibernate + + + org.hibernate.orm + hibernate-core + ${hibernate.version} + + + tech.ydb.jdbc + ydb-jdbc-driver + ${ydb.jdbc.version} + + + tech.ydb.dialects + hibernate-ydb-dialect + ${hibernate.ydb.version} + + + diff --git a/examples/authors/java/hibernate/src/main/java/authors/hibernate/Authors.java b/examples/authors/java/hibernate/src/main/java/authors/hibernate/Authors.java new file mode 100644 index 0000000..75d3b66 --- /dev/null +++ b/examples/authors/java/hibernate/src/main/java/authors/hibernate/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record Authors(long id, String name, String bio) {} diff --git a/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorNameRow.java b/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorNameRow.java new file mode 100644 index 0000000..d368ed0 --- /dev/null +++ b/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorNameRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record GetAuthorNameRow(String name) {} diff --git a/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorRow.java b/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorRow.java new file mode 100644 index 0000000..e9ce2d3 --- /dev/null +++ b/examples/authors/java/hibernate/src/main/java/authors/hibernate/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/examples/authors/java/hibernate/src/main/java/authors/hibernate/ListAuthorsRow.java b/examples/authors/java/hibernate/src/main/java/authors/hibernate/ListAuthorsRow.java new file mode 100644 index 0000000..abb9a61 --- /dev/null +++ b/examples/authors/java/hibernate/src/main/java/authors/hibernate/ListAuthorsRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record ListAuthorsRow(long id, String name, String bio) {} diff --git a/examples/authors/java/hibernate/src/main/java/authors/hibernate/Queries.java b/examples/authors/java/hibernate/src/main/java/authors/hibernate/Queries.java new file mode 100644 index 0000000..9dc1040 --- /dev/null +++ b/examples/authors/java/hibernate/src/main/java/authors/hibernate/Queries.java @@ -0,0 +1,120 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final org.hibernate.Session client; + + public Queries(org.hibernate.Session client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + return client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + }); + } + + private static final String listAuthorsSql = """ +-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;\ +"""; + + public java.util.List listAuthors() { + return client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(listAuthorsSql)) { + try (var _rows = _prepared.executeQuery()) { + var _items = new java.util.ArrayList(); + while (_rows.next()) { + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + _items.add(new ListAuthorsRow(_value0, _value1, _value2)); + } + return _items; + } + } + }); + } + + private static final String getAuthorNameSql = """ +-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthorName(long authorId) { + return client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorNameSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + String _value0 = _rows.getString(1); + return java.util.Optional.of(new GetAuthorNameRow(_value0)); + } + } + }); + } + + private static final String upsertAuthorSql = """ +-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);\ +"""; + + public void upsertAuthor(long authorId, String authorName, String biography) { + client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(upsertAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _statement.setObject("author_name", PrimitiveValue.newText(authorName)); + _statement.setObject("biography", biography == null ? OptionalType.of(PrimitiveType.Text).emptyValue() : OptionalType.of(PrimitiveType.Text).newValue(PrimitiveValue.newText(biography))); + _prepared.execute(); + return null; + } + }); + } + + private static final String deleteAuthorSql = """ +-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;\ +"""; + + public void deleteAuthor(long authorId) { + client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(deleteAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _prepared.execute(); + return null; + } + }); + } +} diff --git a/examples/authors/java/hibernate/src/test/java/authors/hibernate/Smoke.java b/examples/authors/java/hibernate/src/test/java/authors/hibernate/Smoke.java new file mode 100644 index 0000000..093fdc7 --- /dev/null +++ b/examples/authors/java/hibernate/src/test/java/authors/hibernate/Smoke.java @@ -0,0 +1,83 @@ +package authors.hibernate; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.cfg.AvailableSettings; +import org.hibernate.cfg.Configuration; +import tech.ydb.hibernate.dialect.YdbDialect; +import tech.ydb.jdbc.YdbDriver; + +/** Run from examples/authors; the smoke creates and drops its authors table. */ +public final class Smoke { + private static final long MAX_UINT64 = -1L; + private static final long SECOND_ID = 7L; + + private Smoke() { } + + public static void main(String[] args) throws Exception { + String endpoint = System.getenv("SQLC_YDB_TEST_DSN"); + if (endpoint == null || endpoint.isBlank()) throw new IllegalStateException("SQLC_YDB_TEST_DSN is required"); + String schema = readSchema(); + String jdbcUrl = "jdbc:ydb:" + endpoint; + try (SessionFactory factory = new Configuration() + .setProperty(AvailableSettings.DRIVER, YdbDriver.class.getName()) + .setProperty(AvailableSettings.DIALECT, YdbDialect.class.getName()) + .setProperty(AvailableSettings.URL, jdbcUrl) + .buildSessionFactory(); + Session session = factory.openSession()) { + session.doWork(connection -> { try (var statement = connection.createStatement()) { statement.execute(schema); } }); + try { + session.beginTransaction(); + try { + exercise(new Queries(session)); + session.getTransaction().commit(); + } catch (RuntimeException | Error e) { + if (session.getTransaction().isActive()) { + try { + session.getTransaction().rollback(); + } catch (RuntimeException rollbackError) { + e.addSuppressed(rollbackError); + } + } + throw e; + } + } finally { + session.doWork(connection -> { try (var statement = connection.createStatement()) { statement.execute("DROP TABLE authors;"); } }); + } + } + } + + private static void exercise(Queries queries) { + queries.upsertAuthor(MAX_UINT64, "Unsigned", null); + GetAuthorRow emptyBio = queries.getAuthor(MAX_UINT64).orElseThrow(); + check(emptyBio.id() == MAX_UINT64 && "Unsigned".equals(emptyBio.name()) && emptyBio.bio() == null, + "nullable Hibernate row"); + check("Unsigned".equals(queries.getAuthorName(MAX_UINT64).orElseThrow().name()), "Hibernate name"); + + queries.upsertAuthor(MAX_UINT64, "Unsigned", "Biography"); + check("Biography".equals(queries.getAuthor(MAX_UINT64).orElseThrow().bio()), "non-null Hibernate bio"); + queries.upsertAuthor(SECOND_ID, "Second", null); + List rows = queries.listAuthors(); + check(rows.stream().anyMatch(row -> row.id() == MAX_UINT64), "Hibernate list result"); + + queries.deleteAuthor(SECOND_ID); + check(queries.getAuthor(SECOND_ID).isEmpty(), "Hibernate delete result"); + queries.deleteAuthor(MAX_UINT64); + } + + private static String readSchema() throws Exception { + Path schema = Path.of("schema.sql"); + if (!Files.isRegularFile(schema) || Files.readString(schema).isBlank()) { + throw new IllegalStateException("run this smoke from examples/authors with schema.sql present"); + } + return Files.readString(schema); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/examples/authors/java/jdbc/pom.xml b/examples/authors/java/jdbc/pom.xml new file mode 100644 index 0000000..ac3bb81 --- /dev/null +++ b/examples/authors/java/jdbc/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + example.com.sqlc-ydb-authors + authors-java + 0.1.0-SNAPSHOT + .. + + authors-java-jdbc + + + tech.ydb.jdbc + ydb-jdbc-driver + ${ydb.jdbc.version} + + + diff --git a/examples/authors/java/jdbc/src/main/java/authors/jdbc/Authors.java b/examples/authors/java/jdbc/src/main/java/authors/jdbc/Authors.java new file mode 100644 index 0000000..8b7b7b8 --- /dev/null +++ b/examples/authors/java/jdbc/src/main/java/authors/jdbc/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record Authors(long id, String name, String bio) {} diff --git a/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorNameRow.java b/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorNameRow.java new file mode 100644 index 0000000..acc5a05 --- /dev/null +++ b/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorNameRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record GetAuthorNameRow(String name) {} diff --git a/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorRow.java b/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorRow.java new file mode 100644 index 0000000..0d87a81 --- /dev/null +++ b/examples/authors/java/jdbc/src/main/java/authors/jdbc/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/examples/authors/java/jdbc/src/main/java/authors/jdbc/ListAuthorsRow.java b/examples/authors/java/jdbc/src/main/java/authors/jdbc/ListAuthorsRow.java new file mode 100644 index 0000000..5de1c10 --- /dev/null +++ b/examples/authors/java/jdbc/src/main/java/authors/jdbc/ListAuthorsRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record ListAuthorsRow(long id, String name, String bio) {} diff --git a/examples/authors/java/jdbc/src/main/java/authors/jdbc/Queries.java b/examples/authors/java/jdbc/src/main/java/authors/jdbc/Queries.java new file mode 100644 index 0000000..cf67f3f --- /dev/null +++ b/examples/authors/java/jdbc/src/main/java/authors/jdbc/Queries.java @@ -0,0 +1,108 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final java.sql.Connection client; + + public Queries(java.sql.Connection client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + } + + private static final String listAuthorsSql = """ +-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;\ +"""; + + public java.util.List listAuthors() throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(listAuthorsSql)) { + try (var _rows = _prepared.executeQuery()) { + var _items = new java.util.ArrayList(); + while (_rows.next()) { + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + _items.add(new ListAuthorsRow(_value0, _value1, _value2)); + } + return _items; + } + } + } + + private static final String getAuthorNameSql = """ +-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthorName(long authorId) throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(getAuthorNameSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + String _value0 = _rows.getString(1); + return java.util.Optional.of(new GetAuthorNameRow(_value0)); + } + } + } + + private static final String upsertAuthorSql = """ +-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);\ +"""; + + public void upsertAuthor(long authorId, String authorName, String biography) throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(upsertAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _statement.setObject("author_name", PrimitiveValue.newText(authorName)); + _statement.setObject("biography", biography == null ? OptionalType.of(PrimitiveType.Text).emptyValue() : OptionalType.of(PrimitiveType.Text).newValue(PrimitiveValue.newText(biography))); + _prepared.execute(); + } + } + + private static final String deleteAuthorSql = """ +-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;\ +"""; + + public void deleteAuthor(long authorId) throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(deleteAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _prepared.execute(); + } + } +} diff --git a/examples/authors/java/jdbc/src/test/java/authors/jdbc/Smoke.java b/examples/authors/java/jdbc/src/test/java/authors/jdbc/Smoke.java new file mode 100644 index 0000000..74f2156 --- /dev/null +++ b/examples/authors/java/jdbc/src/test/java/authors/jdbc/Smoke.java @@ -0,0 +1,61 @@ +package authors.jdbc; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.List; + +/** Run from examples/authors; the smoke creates and drops its authors table. */ +public final class Smoke { + private static final long MAX_UINT64 = -1L; + private static final long SECOND_ID = 7L; + + private Smoke() { } + + public static void main(String[] args) throws Exception { + String endpoint = System.getenv("SQLC_YDB_TEST_DSN"); + if (endpoint == null || endpoint.isBlank()) throw new IllegalStateException("SQLC_YDB_TEST_DSN is required"); + String schema = readSchema(); + try (Connection connection = DriverManager.getConnection("jdbc:ydb:" + endpoint)) { + try (var statement = connection.createStatement()) { + statement.execute(schema); + } + try { + Queries queries = new Queries(connection); + exercise(queries); + } finally { + try (var statement = connection.createStatement()) { statement.execute("DROP TABLE authors;"); } + } + } + } + + private static void exercise(Queries queries) throws java.sql.SQLException { + queries.upsertAuthor(MAX_UINT64, "Unsigned", null); + GetAuthorRow emptyBio = queries.getAuthor(MAX_UINT64).orElseThrow(); + check(emptyBio.id() == MAX_UINT64 && "Unsigned".equals(emptyBio.name()) && emptyBio.bio() == null, + "nullable JDBC row"); + check("Unsigned".equals(queries.getAuthorName(MAX_UINT64).orElseThrow().name()), "JDBC name"); + + queries.upsertAuthor(MAX_UINT64, "Unsigned", "Biography"); + check("Biography".equals(queries.getAuthor(MAX_UINT64).orElseThrow().bio()), "non-null JDBC bio"); + queries.upsertAuthor(SECOND_ID, "Second", null); + List rows = queries.listAuthors(); + check(rows.stream().anyMatch(row -> row.id() == MAX_UINT64), "JDBC list result"); + + queries.deleteAuthor(SECOND_ID); + check(queries.getAuthor(SECOND_ID).isEmpty(), "JDBC delete result"); + } + + private static String readSchema() throws Exception { + Path schema = Path.of("schema.sql"); + if (!Files.isRegularFile(schema) || Files.readString(schema).isBlank()) { + throw new IllegalStateException("run this smoke from examples/authors with schema.sql present"); + } + return Files.readString(schema); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/examples/authors/java/native/pom.xml b/examples/authors/java/native/pom.xml new file mode 100644 index 0000000..77c5ed7 --- /dev/null +++ b/examples/authors/java/native/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + example.com.sqlc-ydb-authors + authors-java + 0.1.0-SNAPSHOT + .. + + authors-java-native + + + tech.ydb + ydb-sdk-query + + + diff --git a/examples/authors/java/native/src/main/java/authors/nativeapi/Authors.java b/examples/authors/java/native/src/main/java/authors/nativeapi/Authors.java new file mode 100644 index 0000000..4d2a874 --- /dev/null +++ b/examples/authors/java/native/src/main/java/authors/nativeapi/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record Authors(long id, String name, String bio) {} diff --git a/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorNameRow.java b/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorNameRow.java new file mode 100644 index 0000000..f6acc75 --- /dev/null +++ b/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorNameRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record GetAuthorNameRow(String name) {} diff --git a/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorRow.java b/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorRow.java new file mode 100644 index 0000000..2e50eae --- /dev/null +++ b/examples/authors/java/native/src/main/java/authors/nativeapi/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/examples/authors/java/native/src/main/java/authors/nativeapi/ListAuthorsRow.java b/examples/authors/java/native/src/main/java/authors/nativeapi/ListAuthorsRow.java new file mode 100644 index 0000000..d739636 --- /dev/null +++ b/examples/authors/java/native/src/main/java/authors/nativeapi/ListAuthorsRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record ListAuthorsRow(long id, String name, String bio) {} diff --git a/examples/authors/java/native/src/main/java/authors/nativeapi/Queries.java b/examples/authors/java/native/src/main/java/authors/nativeapi/Queries.java new file mode 100644 index 0000000..696e053 --- /dev/null +++ b/examples/authors/java/native/src/main/java/authors/nativeapi/Queries.java @@ -0,0 +1,109 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +import tech.ydb.query.tools.SessionRetryContext; +import tech.ydb.query.tools.QueryReader; +import tech.ydb.common.transaction.TxMode; +import tech.ydb.table.query.Params; +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final SessionRetryContext client; + + public Queries(SessionRetryContext client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + var _params = Params.create(); + _params.put("$author_id", PrimitiveValue.newUint64(authorId)); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(getAuthorSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + if (_query.getResultSetCount() != 1) throw new IllegalStateException("Expected one result set"); + var _rows = _query.getResultSet(0); + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getColumn(0).getUint64(); + String _value1 = _rows.getColumn(1).getText(); + String _value2 = _rows.getColumn(2).isOptionalItemPresent() ? _rows.getColumn(2).getOptionalItem().getText() : null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + + private static final String listAuthorsSql = """ +-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;\ +"""; + + public java.util.List listAuthors() { + var _params = Params.create(); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(listAuthorsSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + if (_query.getResultSetCount() != 1) throw new IllegalStateException("Expected one result set"); + var _rows = _query.getResultSet(0); + var _items = new java.util.ArrayList(); + while (_rows.next()) { + long _value0 = _rows.getColumn(0).getUint64(); + String _value1 = _rows.getColumn(1).getText(); + String _value2 = _rows.getColumn(2).isOptionalItemPresent() ? _rows.getColumn(2).getOptionalItem().getText() : null; + _items.add(new ListAuthorsRow(_value0, _value1, _value2)); + } + return _items; + } + + private static final String getAuthorNameSql = """ +-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthorName(long authorId) { + var _params = Params.create(); + _params.put("$author_id", PrimitiveValue.newUint64(authorId)); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(getAuthorNameSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + if (_query.getResultSetCount() != 1) throw new IllegalStateException("Expected one result set"); + var _rows = _query.getResultSet(0); + if (!_rows.next()) return java.util.Optional.empty(); + String _value0 = _rows.getColumn(0).getText(); + return java.util.Optional.of(new GetAuthorNameRow(_value0)); + } + + private static final String upsertAuthorSql = """ +-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);\ +"""; + + public void upsertAuthor(long authorId, String authorName, String biography) { + var _params = Params.create(); + _params.put("$author_id", PrimitiveValue.newUint64(authorId)); + _params.put("$author_name", PrimitiveValue.newText(authorName)); + _params.put("$biography", biography == null ? OptionalType.of(PrimitiveType.Text).emptyValue() : OptionalType.of(PrimitiveType.Text).newValue(PrimitiveValue.newText(biography))); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(upsertAuthorSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + } + + private static final String deleteAuthorSql = """ +-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;\ +"""; + + public void deleteAuthor(long authorId) { + var _params = Params.create(); + _params.put("$author_id", PrimitiveValue.newUint64(authorId)); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(deleteAuthorSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + } +} diff --git a/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java b/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java new file mode 100644 index 0000000..ab4b2d4 --- /dev/null +++ b/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java @@ -0,0 +1,77 @@ +package authors.nativeapi; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import tech.ydb.core.grpc.GrpcTransport; +import tech.ydb.query.QueryClient; +import tech.ydb.query.tools.SessionRetryContext; + +/** Run from examples/authors; the smoke creates and drops its authors table. */ +public final class Smoke { + private static final long MAX_UINT64 = -1L; + private static final long SECOND_ID = 7L; + + private Smoke() { } + + public static void main(String[] args) throws Exception { + String url = System.getenv("SQLC_YDB_TEST_DSN"); + if (url == null || url.isBlank()) { + throw new IllegalStateException("SQLC_YDB_TEST_DSN is required"); + } + String schema = readSchema(); + try (GrpcTransport transport = GrpcTransport.forConnectionString(url).build(); + QueryClient client = QueryClient.newClient(transport).build()) { + SessionRetryContext retry = SessionRetryContext.create(client).build(); + createSchema(retry, schema); + Queries queries = new Queries(retry); + try { + exercise(queries); + } finally { + queries.deleteAuthor(MAX_UINT64); + queries.deleteAuthor(SECOND_ID); + dropSchema(retry); + } + } + } + + private static void exercise(Queries queries) { + queries.upsertAuthor(MAX_UINT64, "Unsigned", null); + GetAuthorRow emptyBio = queries.getAuthor(MAX_UINT64).orElseThrow(); + check(emptyBio.id() == MAX_UINT64 && "Unsigned".equals(emptyBio.name()) && emptyBio.bio() == null, + "nullable native row"); + check("Unsigned".equals(queries.getAuthorName(MAX_UINT64).orElseThrow().name()), "native name"); + + queries.upsertAuthor(MAX_UINT64, "Unsigned", "Biography"); + check("Biography".equals(queries.getAuthor(MAX_UINT64).orElseThrow().bio()), "non-null native bio"); + queries.upsertAuthor(SECOND_ID, "Second", null); + List rows = queries.listAuthors(); + check(rows.stream().anyMatch(row -> row.id() == MAX_UINT64), "native list result"); + + queries.deleteAuthor(SECOND_ID); + check(queries.getAuthor(SECOND_ID).isEmpty(), "native delete result"); + } + + private static String readSchema() throws Exception { + Path schema = Path.of("schema.sql"); + if (!Files.isRegularFile(schema) || Files.readString(schema).isBlank()) { + throw new IllegalStateException("run this smoke from examples/authors with schema.sql present"); + } + return Files.readString(schema); + } + + private static void createSchema(SessionRetryContext retry, String schema) { + var result = retry.supplyResult(session -> tech.ydb.query.tools.QueryReader.readFrom(session.createQuery(schema, tech.ydb.common.transaction.TxMode.NONE))).join(); + if (!result.isSuccess()) throw new IllegalStateException("CREATE TABLE authors failed: " + result); + } + + private static void dropSchema(SessionRetryContext retry) { + var result = retry.supplyResult(session -> tech.ydb.query.tools.QueryReader.readFrom(session.createQuery("DROP TABLE authors;", tech.ydb.common.transaction.TxMode.NONE))).join(); + if (!result.isSuccess()) throw new IllegalStateException("DROP TABLE authors failed: " + result); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/examples/authors/java/pom.xml b/examples/authors/java/pom.xml new file mode 100644 index 0000000..b2e4021 --- /dev/null +++ b/examples/authors/java/pom.xml @@ -0,0 +1,42 @@ + + + 4.0.0 + + example.com.sqlc-ydb-authors + authors-java + 0.1.0-SNAPSHOT + pom + + sqlc-ydb authors Java examples + + + native + jdbc + spring + hibernate + + + + UTF-8 + 17 + 2.4.11 + 2.4.1 + 6.2.0 + 6.2.7.Final + 1.7.0 + + + + + + tech.ydb + ydb-sdk-bom + ${ydb.sdk.version} + pom + import + + + + diff --git a/examples/authors/java/run-smoke.sh b/examples/authors/java/run-smoke.sh new file mode 100755 index 0000000..cf40608 --- /dev/null +++ b/examples/authors/java/run-smoke.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +if [ -z "${SQLC_YDB_TEST_DSN:-}" ]; then + echo "SQLC_YDB_TEST_DSN is required" >&2 + exit 2 +fi + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +AUTHORS_DIR=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +cd "$AUTHORS_DIR" + +mvn -f java/pom.xml -DskipTests test-compile + +for module in native jdbc spring hibernate; do + mvn -q -f "java/$module/pom.xml" dependency:build-classpath \ + -Dmdep.includeScope=test \ + -Dmdep.outputFile="target/smoke-classpath.txt" + classpath="java/$module/target/test-classes:java/$module/target/classes:$(cat "java/$module/target/smoke-classpath.txt")" + case "$module" in + native) class=authors.nativeapi.Smoke ;; + jdbc) class=authors.jdbc.Smoke ;; + spring) class=authors.spring.Smoke ;; + hibernate) class=authors.hibernate.Smoke ;; + esac + echo "Running $class" + java -cp "$classpath" "$class" +done diff --git a/examples/authors/java/spring/pom.xml b/examples/authors/java/spring/pom.xml new file mode 100644 index 0000000..f9bce62 --- /dev/null +++ b/examples/authors/java/spring/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + example.com.sqlc-ydb-authors + authors-java + 0.1.0-SNAPSHOT + .. + + authors-java-spring + + + org.springframework + spring-jdbc + ${spring.jdbc.version} + + + tech.ydb.jdbc + ydb-jdbc-driver + ${ydb.jdbc.version} + + + diff --git a/examples/authors/java/spring/src/main/java/authors/spring/Authors.java b/examples/authors/java/spring/src/main/java/authors/spring/Authors.java new file mode 100644 index 0000000..f74f054 --- /dev/null +++ b/examples/authors/java/spring/src/main/java/authors/spring/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record Authors(long id, String name, String bio) {} diff --git a/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorNameRow.java b/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorNameRow.java new file mode 100644 index 0000000..b11293a --- /dev/null +++ b/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorNameRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record GetAuthorNameRow(String name) {} diff --git a/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorRow.java b/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorRow.java new file mode 100644 index 0000000..89eca40 --- /dev/null +++ b/examples/authors/java/spring/src/main/java/authors/spring/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/examples/authors/java/spring/src/main/java/authors/spring/ListAuthorsRow.java b/examples/authors/java/spring/src/main/java/authors/spring/ListAuthorsRow.java new file mode 100644 index 0000000..0c7fd75 --- /dev/null +++ b/examples/authors/java/spring/src/main/java/authors/spring/ListAuthorsRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record ListAuthorsRow(long id, String name, String bio) {} diff --git a/examples/authors/java/spring/src/main/java/authors/spring/Queries.java b/examples/authors/java/spring/src/main/java/authors/spring/Queries.java new file mode 100644 index 0000000..d9f3317 --- /dev/null +++ b/examples/authors/java/spring/src/main/java/authors/spring/Queries.java @@ -0,0 +1,120 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final org.springframework.jdbc.core.JdbcTemplate client; + + public Queries(org.springframework.jdbc.core.JdbcTemplate client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT id, name, bio FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + return client.execute((org.springframework.jdbc.core.ConnectionCallback>) _connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + }); + } + + private static final String listAuthorsSql = """ +-- name: ListAuthors :many +SELECT id, name, bio FROM authors ORDER BY id;\ +"""; + + public java.util.List listAuthors() { + return client.execute((org.springframework.jdbc.core.ConnectionCallback>) _connection -> { + try (var _prepared = _connection.prepareStatement(listAuthorsSql)) { + try (var _rows = _prepared.executeQuery()) { + var _items = new java.util.ArrayList(); + while (_rows.next()) { + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + _items.add(new ListAuthorsRow(_value0, _value1, _value2)); + } + return _items; + } + } + }); + } + + private static final String getAuthorNameSql = """ +-- name: GetAuthorName :one +DECLARE $author_id AS Uint64; +SELECT name FROM authors WHERE id = $author_id;\ +"""; + + public java.util.Optional getAuthorName(long authorId) { + return client.execute((org.springframework.jdbc.core.ConnectionCallback>) _connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorNameSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + String _value0 = _rows.getString(1); + return java.util.Optional.of(new GetAuthorNameRow(_value0)); + } + } + }); + } + + private static final String upsertAuthorSql = """ +-- name: UpsertAuthor :exec +DECLARE $author_id AS Uint64; +DECLARE $author_name AS Utf8; +DECLARE $biography AS Optional; +UPSERT INTO authors (id, name, bio) +VALUES ($author_id, $author_name, $biography);\ +"""; + + public void upsertAuthor(long authorId, String authorName, String biography) { + client.execute((org.springframework.jdbc.core.ConnectionCallback) _connection -> { + try (var _prepared = _connection.prepareStatement(upsertAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _statement.setObject("author_name", PrimitiveValue.newText(authorName)); + _statement.setObject("biography", biography == null ? OptionalType.of(PrimitiveType.Text).emptyValue() : OptionalType.of(PrimitiveType.Text).newValue(PrimitiveValue.newText(biography))); + _prepared.execute(); + return null; + } + }); + } + + private static final String deleteAuthorSql = """ +-- name: DeleteAuthor :exec +DECLARE $author_id AS Uint64; +DELETE FROM authors WHERE id = $author_id;\ +"""; + + public void deleteAuthor(long authorId) { + client.execute((org.springframework.jdbc.core.ConnectionCallback) _connection -> { + try (var _prepared = _connection.prepareStatement(deleteAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + _prepared.execute(); + return null; + } + }); + } +} diff --git a/examples/authors/java/spring/src/test/java/authors/spring/Smoke.java b/examples/authors/java/spring/src/test/java/authors/spring/Smoke.java new file mode 100644 index 0000000..93dfd6f --- /dev/null +++ b/examples/authors/java/spring/src/test/java/authors/spring/Smoke.java @@ -0,0 +1,64 @@ +package authors.spring; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.List; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.SingleConnectionDataSource; + +/** Run from examples/authors; the smoke creates and drops its authors table. */ +public final class Smoke { + private static final long MAX_UINT64 = -1L; + private static final long SECOND_ID = 7L; + + private Smoke() { } + + public static void main(String[] args) throws Exception { + String endpoint = System.getenv("SQLC_YDB_TEST_DSN"); + if (endpoint == null || endpoint.isBlank()) throw new IllegalStateException("SQLC_YDB_TEST_DSN is required"); + String schema = readSchema(); + try (Connection connection = DriverManager.getConnection("jdbc:ydb:" + endpoint)) { + try (var statement = connection.createStatement()) { statement.execute(schema); } + try { + JdbcTemplate template = new JdbcTemplate(new SingleConnectionDataSource(connection, true)); + Queries queries = new Queries(template); + exercise(queries); + check(!connection.isClosed(), "Spring borrowed connection remains open"); + } finally { + try (var statement = connection.createStatement()) { statement.execute("DROP TABLE authors;"); } + } + } + } + + private static void exercise(Queries queries) { + queries.upsertAuthor(MAX_UINT64, "Unsigned", null); + GetAuthorRow emptyBio = queries.getAuthor(MAX_UINT64).orElseThrow(); + check(emptyBio.id() == MAX_UINT64 && "Unsigned".equals(emptyBio.name()) && emptyBio.bio() == null, + "nullable Spring row"); + check("Unsigned".equals(queries.getAuthorName(MAX_UINT64).orElseThrow().name()), "Spring name"); + + queries.upsertAuthor(MAX_UINT64, "Unsigned", "Biography"); + check("Biography".equals(queries.getAuthor(MAX_UINT64).orElseThrow().bio()), "non-null Spring bio"); + queries.upsertAuthor(SECOND_ID, "Second", null); + List rows = queries.listAuthors(); + check(rows.stream().anyMatch(row -> row.id() == MAX_UINT64), "Spring list result"); + + queries.deleteAuthor(SECOND_ID); + check(queries.getAuthor(SECOND_ID).isEmpty(), "Spring delete result"); + } + + private static String readSchema() throws Exception { + Path schema = Path.of("schema.sql"); + if (!Files.isRegularFile(schema) || Files.readString(schema).isBlank()) { + throw new IllegalStateException("run this smoke from examples/authors with schema.sql present"); + } + return Files.readString(schema); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/examples/authors/sqlc.yaml b/examples/authors/sqlc.yaml index 5b6e1cc..d350e68 100644 --- a/examples/authors/sqlc.yaml +++ b/examples/authors/sqlc.yaml @@ -16,6 +16,17 @@ sql: package: authors out: python/native runtime: ydb + csharp: + namespace: Authors.AdoNet + out: csharp/adonet + cpp: + namespace: authors::native + out: cpp/native + runtime: ydb + java: + package: authors.nativeapi + out: java/native/src/main/java/authors/nativeapi + runtime: ydb - name: portable engine: ydb schema: schema.sql @@ -29,6 +40,10 @@ sql: package: authors out: python/dbapi runtime: dbapi + java: + package: authors.jdbc + out: java/jdbc/src/main/java/authors/jdbc + runtime: jdbc - name: sqlalchemy engine: ydb schema: schema.sql @@ -39,3 +54,25 @@ sql: out: python/sqlalchemy runtime: sqlalchemy emit_sync_querier: true + - name: spring + engine: ydb + schema: schema.sql + queries: queries.sql + gen: + java: + package: authors.spring + out: java/spring/src/main/java/authors/spring + runtime: spring + cpp: + namespace: authors::userver + out: cpp/userver + runtime: userver + - name: hibernate + engine: ydb + schema: schema.sql + queries: queries.sql + gen: + java: + package: authors.hibernate + out: java/hibernate/src/main/java/authors/hibernate + runtime: hibernate diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 82eb487..d54cf80 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -12,7 +12,10 @@ import ( "strings" "github.com/ydb-platform/sqlc-engine-ydb/internal/analyzer" + "github.com/ydb-platform/sqlc-engine-ydb/internal/codegen/cpp" + "github.com/ydb-platform/sqlc-engine-ydb/internal/codegen/csharp" "github.com/ydb-platform/sqlc-engine-ydb/internal/codegen/golang" + "github.com/ydb-platform/sqlc-engine-ydb/internal/codegen/java" "github.com/ydb-platform/sqlc-engine-ydb/internal/codegen/python" "github.com/ydb-platform/sqlc-engine-ydb/internal/config" "github.com/ydb-platform/sqlc-engine-ydb/internal/model" @@ -21,7 +24,7 @@ import ( var Version = "0.1.0-dev" -const help = `sqlc-ydb generates typed Go and Python code from YQL. +const help = `sqlc-ydb generates typed code from YQL. Usage: sqlc-ydb [-f sqlc.yaml] @@ -194,8 +197,8 @@ func prepare(c *config.Config, generate bool) ([]output, error) { if !generate { continue } - if s.Gen.Go == nil && s.Gen.Python == nil { - return nil, errors.New("generation requires gen.go or gen.python") + if s.Gen.Go == nil && s.Gen.Python == nil && s.Gen.CSharp == nil && s.Gen.Java == nil && s.Gen.CPP == nil { + return nil, errors.New("generation requires a built-in generator in gen") } add := func(dir string, files []model.File) error { if !filepath.IsAbs(dir) { @@ -236,6 +239,33 @@ func prepare(c *config.Config, generate bool) ([]output, error) { return nil, err } } + if g := s.Gen.CSharp; g != nil { + files, err := csharp.Generate(result, csharp.Options{Namespace: g.Namespace}) + if err != nil { + return nil, fmt.Errorf("C# generation: %w", err) + } + if err := add(g.Out, files); err != nil { + return nil, err + } + } + if g := s.Gen.Java; g != nil { + files, err := java.Generate(result, java.Options{Package: g.Package, Runtime: g.Runtime}) + if err != nil { + return nil, fmt.Errorf("Java generation: %w", err) + } + if err := add(g.Out, files); err != nil { + return nil, err + } + } + if g := s.Gen.CPP; g != nil { + files, err := cpp.Generate(result, cpp.Options{Namespace: g.Namespace, Runtime: g.Runtime}) + if err != nil { + return nil, fmt.Errorf("C++ generation: %w", err) + } + if err := add(g.Out, files); err != nil { + return nil, err + } + } } for _, f := range outputs { key, err := canonicalPath(f.path) diff --git a/internal/codegen/cpp/generator.go b/internal/codegen/cpp/generator.go new file mode 100644 index 0000000..0e836af --- /dev/null +++ b/internal/codegen/cpp/generator.go @@ -0,0 +1,488 @@ +// Package cpp renders the resolved YQL model as C++20 source for the native +// YDB C++ SDK or userver's YDB driver. +package cpp + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +type Options struct { + Namespace string + Runtime string // ydb or userver +} + +type scalarType struct { + cpp string + builder string + parser string + reference bool +} + +func Generate(in *model.AnalysisResult, options Options) ([]model.File, error) { + if in == nil { + return nil, fmt.Errorf("analysis result is nil") + } + if len(in.Diagnostics) != 0 { + return nil, fmt.Errorf("cannot generate with diagnostics: %s", in.Diagnostics[0]) + } + if options.Namespace == "" { + options.Namespace = "db" + } + if err := validateNamespace(options.Namespace); err != nil { + return nil, err + } + if options.Runtime == "" { + options.Runtime = "ydb" + } + if options.Runtime != "ydb" && options.Runtime != "userver" { + return nil, fmt.Errorf("unsupported C++ runtime %q", options.Runtime) + } + if err := validate(in, options); err != nil { + return nil, err + } + + models, err := renderModels(in, options) + if err != nil { + return nil, err + } + header, err := renderHeader(in, options) + if err != nil { + return nil, err + } + source, err := renderSource(in, options) + if err != nil { + return nil, err + } + return []model.File{ + {Name: "models.hpp", Content: []byte(models)}, + {Name: "queries.hpp", Content: []byte(header)}, + {Name: "queries.cpp", Content: []byte(source)}, + }, nil +} + +func validate(in *model.AnalysisResult, options Options) error { + rowTypes := make(map[string]string) + for _, query := range in.Queries { + if query.Command == model.One || query.Command == model.Many { + rowTypes[query.Name+"Row"] = query.Name + } + } + + seenQueries := map[string]bool{} + for _, query := range in.Queries { + if err := validateIdent(query.Name); err != nil || query.Name == "Queries" || query.Name == "client_" { + return fmt.Errorf("invalid C++ query name %q", query.Name) + } + if owner, exists := rowTypes[query.Name]; exists { + return fmt.Errorf("query name %q collides with generated row type for %s", query.Name, owner) + } + if seenQueries[query.Name] { + return fmt.Errorf("duplicate C++ query name %q", query.Name) + } + seenQueries[query.Name] = true + switch query.Command { + case model.One, model.Many, model.Exec: + case model.ExecRows: + return fmt.Errorf("%s: :execrows is unavailable for C++ runtime %s", query.Name, options.Runtime) + default: + return fmt.Errorf("%s: unsupported command %q", query.Name, query.Command) + } + if (query.Command == model.One || query.Command == model.Many) && + (len(query.ResultSets) != 1 || len(query.ResultSets[0].Columns) == 0) { + return fmt.Errorf("%s: %s requires one non-empty result set", query.Name, query.Command) + } + + seenParams := map[string]bool{} + for _, parameter := range query.Parameters { + if err := validateIdent(parameter.Name); err != nil { + return fmt.Errorf("%s: invalid C++ parameter %q: %w", query.Name, parameter.Name, err) + } + if seenParams[parameter.Name] { + return fmt.Errorf("%s: duplicate C++ parameter %q", query.Name, parameter.Name) + } + if (query.Command == model.One || query.Command == model.Many) && parameter.Name == query.Name+"Row" { + return fmt.Errorf("%s: parameter %q collides with generated row type", query.Name, parameter.Name) + } + constantName := "k" + query.Name + "Sql" + if options.Runtime == "userver" { + constantName = "k" + query.Name + "Query" + } + if parameter.Name == constantName { + return fmt.Errorf("%s: parameter %q collides with generated SQL constant", query.Name, parameter.Name) + } + seenParams[parameter.Name] = true + if _, err := typeInfo(parameter.Type, options.Runtime); err != nil { + return fmt.Errorf("%s parameter %s: %w", query.Name, parameter.Name, err) + } + } + for _, resultSet := range query.ResultSets { + seenColumns := map[string]bool{} + for _, column := range resultSet.Columns { + if err := validateIdent(column.Name); err != nil { + return fmt.Errorf("%s: invalid C++ result column %q: %w", query.Name, column.Name, err) + } + if seenColumns[column.Name] { + return fmt.Errorf("%s: duplicate C++ result column %q", query.Name, column.Name) + } + if (query.Command == model.One || query.Command == model.Many) && column.Name == query.Name+"Row" { + return fmt.Errorf("%s: result column %q collides with generated row type", query.Name, column.Name) + } + seenColumns[column.Name] = true + if _, err := typeInfo(column.Type, options.Runtime); err != nil { + return fmt.Errorf("%s column %s: %w", query.Name, column.Name, err) + } + } + } + } + return nil +} + +func validateNamespace(namespace string) error { + parts := strings.Split(namespace, "::") + if len(parts) == 0 { + return fmt.Errorf("invalid C++ namespace %q", namespace) + } + for _, part := range parts { + if err := validateIdent(part); err != nil { + return fmt.Errorf("invalid C++ namespace %q: %w", namespace, err) + } + } + return nil +} + +func validateIdent(name string) error { + if name == "" { + return fmt.Errorf("identifier is empty") + } + if strings.HasPrefix(name, "sqlc_") { + return fmt.Errorf("reserved sqlc_ prefix") + } + if strings.HasPrefix(name, "_") { + return fmt.Errorf("leading underscore is not supported") + } + if cppKeywords[name] { + return fmt.Errorf("C++ keyword") + } + for index, r := range name { + if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (index > 0 && r >= '0' && r <= '9') || (index > 0 && r == '_')) { + return fmt.Errorf("only ASCII C++ identifiers are supported") + } + } + return nil +} + +var cppKeywords = map[string]bool{ + "alignas": true, "alignof": true, "and": true, "and_eq": true, "asm": true, + "auto": true, "bitand": true, "bitor": true, "bool": true, "break": true, + "case": true, "catch": true, "char": true, "char8_t": true, "char16_t": true, + "char32_t": true, "class": true, "compl": true, "concept": true, "const": true, + "consteval": true, "constexpr": true, "constinit": true, "const_cast": true, "continue": true, + "co_await": true, "co_return": true, "co_yield": true, "decltype": true, "default": true, + "delete": true, "do": true, "double": true, "dynamic_cast": true, "else": true, + "enum": true, "explicit": true, "export": true, "extern": true, "false": true, + "float": true, "for": true, "friend": true, "goto": true, "if": true, + "inline": true, "int": true, "long": true, "mutable": true, "namespace": true, + "new": true, "noexcept": true, "not": true, "not_eq": true, "nullptr": true, + "operator": true, "or": true, "or_eq": true, "private": true, "protected": true, + "public": true, "register": true, "reinterpret_cast": true, "requires": true, "return": true, + "short": true, "signed": true, "sizeof": true, "static": true, "static_assert": true, + "static_cast": true, "struct": true, "switch": true, "template": true, "this": true, + "thread_local": true, "throw": true, "true": true, "try": true, "typedef": true, + "typeid": true, "typename": true, "union": true, "unsigned": true, "using": true, + "virtual": true, "void": true, "volatile": true, "wchar_t": true, "while": true, + "xor": true, "xor_eq": true, +} + +func typeInfo(yqlType model.Type, runtime string) (scalarType, error) { + if yqlType.IsOptional() { + if yqlType.Elem == nil { + return scalarType{}, fmt.Errorf("Optional lacks element") + } + if yqlType.Elem.IsOptional() { + return scalarType{}, fmt.Errorf("nested Optional is unsupported") + } + base, err := typeInfo(*yqlType.Elem, runtime) + if err != nil { + return scalarType{}, err + } + base.cpp = "std::optional<" + base.cpp + ">" + base.builder = "Optional" + base.builder + base.parser = "GetOptional" + strings.TrimPrefix(base.parser, "Get") + base.reference = true + return base, nil + } + + kind := strings.ToLower(yqlType.Kind) + if runtime == "userver" && kind == "float" { + return scalarType{}, fmt.Errorf("unsupported YQL type %q for userver", yqlType.Kind) + } + var info scalarType + switch kind { + case "bool": + info = scalarType{cpp: "bool", builder: "Bool", parser: "GetBool"} + case "int8": + info = scalarType{cpp: "std::int8_t", builder: "Int8", parser: "GetInt8"} + case "uint8": + info = scalarType{cpp: "std::uint8_t", builder: "Uint8", parser: "GetUint8"} + case "int16": + info = scalarType{cpp: "std::int16_t", builder: "Int16", parser: "GetInt16"} + case "uint16": + info = scalarType{cpp: "std::uint16_t", builder: "Uint16", parser: "GetUint16"} + case "int32": + info = scalarType{cpp: "std::int32_t", builder: "Int32", parser: "GetInt32"} + case "uint32": + info = scalarType{cpp: "std::uint32_t", builder: "Uint32", parser: "GetUint32"} + case "int64": + info = scalarType{cpp: "std::int64_t", builder: "Int64", parser: "GetInt64"} + case "uint64": + info = scalarType{cpp: "std::uint64_t", builder: "Uint64", parser: "GetUint64"} + case "float": + info = scalarType{cpp: "float", builder: "Float", parser: "GetFloat"} + case "double": + info = scalarType{cpp: "double", builder: "Double", parser: "GetDouble"} + case "string": + info = scalarType{cpp: "std::string", builder: "String", parser: "GetString", reference: true} + case "utf8": + cppType := "std::string" + if runtime == "userver" { + cppType = "::userver::ydb::Utf8" + } + info = scalarType{cpp: cppType, builder: "Utf8", parser: "GetUtf8", reference: true} + default: + return scalarType{}, fmt.Errorf("unsupported YQL type %q for %s", yqlType.Kind, runtime) + } + return info, nil +} + +func renderModels(in *model.AnalysisResult, options Options) (string, error) { + var out strings.Builder + out.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n#pragma once\n\n#include \n#include \n#include \n") + if options.Runtime == "userver" { + out.WriteString("\n#include \n") + } + out.WriteString("\nnamespace " + options.Namespace + " {\n\n") + for _, query := range in.Queries { + if query.Command != model.One && query.Command != model.Many { + continue + } + out.WriteString("struct " + query.Name + "Row final {\n") + for _, column := range query.ResultSets[0].Columns { + info, err := typeInfo(column.Type, options.Runtime) + if err != nil { + return "", err + } + out.WriteString(" " + info.cpp + " " + column.Name + ";\n") + } + out.WriteString("};\n\n") + } + out.WriteString("} // namespace " + options.Namespace + "\n") + return out.String(), nil +} + +func renderHeader(in *model.AnalysisResult, options Options) (string, error) { + var out strings.Builder + out.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n#pragma once\n\n#include \"models.hpp\"\n\n#include \n#include \n#include \n#include \n") + clientType := "NYdb::NQuery::TQueryClient" + if options.Runtime == "ydb" { + out.WriteString("\n#include \n") + } else { + clientType = "::userver::ydb::TableClient" + out.WriteString("\n#include \n") + } + out.WriteString("\nnamespace " + options.Namespace + " {\n\nclass Queries final {\npublic:\n") + out.WriteString(" explicit Queries(" + clientType + "& client) noexcept : client_(client) {}\n\n") + for _, query := range in.Queries { + returnType := "void" + if query.Command == model.One { + returnType = "std::optional<" + query.Name + "Row>" + } else if query.Command == model.Many { + returnType = "std::vector<" + query.Name + "Row>" + } + out.WriteString(" " + returnType + " " + query.Name + "(" + methodParameters(query, options.Runtime) + ") const;\n") + } + out.WriteString("\nprivate:\n " + clientType + "& client_;\n};\n\n} // namespace " + options.Namespace + "\n") + return out.String(), nil +} + +func methodParameters(query model.AnalyzedQuery, runtime string) string { + parts := make([]string, 0, len(query.Parameters)) + for _, parameter := range query.Parameters { + info, _ := typeInfo(parameter.Type, runtime) + declaration := info.cpp + " " + parameter.Name + if info.reference { + declaration = "const " + info.cpp + "& " + parameter.Name + } + parts = append(parts, declaration) + } + return strings.Join(parts, ", ") +} + +func renderSource(in *model.AnalysisResult, options Options) (string, error) { + var out strings.Builder + out.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n#include \"queries.hpp\"\n\n#include \n#include \n\nnamespace " + options.Namespace + " {\nnamespace {\n\n") + for _, query := range in.Queries { + if options.Runtime == "ydb" { + out.WriteString("const std::string k" + query.Name + "Sql = " + sqlLiteral(query.SQL) + ";\n\n") + } else { + out.WriteString("const ::userver::ydb::Query k" + query.Name + "Query{\n " + sqlLiteral(query.SQL) + ",\n") + out.WriteString(" ::userver::ydb::Query::NameLiteral{" + strconv.Quote(query.Name) + "},\n") + out.WriteString(" ::userver::ydb::Query::LogMode::kNameOnly,\n};\n\n") + } + } + out.WriteString("} // namespace\n\n") + for _, query := range in.Queries { + var err error + if options.Runtime == "ydb" { + err = renderNativeMethod(&out, query, options) + } else { + err = renderUserverMethod(&out, query, options) + } + if err != nil { + return "", err + } + } + out.WriteString("} // namespace " + options.Namespace + "\n") + return out.String(), nil +} + +func renderNativeMethod(out *strings.Builder, query model.AnalyzedQuery, options Options) error { + returnType := "void" + if query.Command == model.One { + returnType = "std::optional<" + query.Name + "Row>" + } else if query.Command == model.Many { + returnType = "std::vector<" + query.Name + "Row>" + } + out.WriteString(returnType + " Queries::" + query.Name + "(" + methodParameters(query, options.Runtime) + ") const {\n") + if query.Command == model.One || query.Command == model.Many { + out.WriteString(" std::optional sqlc_result_set;\n") + } + out.WriteString(" const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus {\n") + if len(query.Parameters) != 0 { + out.WriteString(" auto sqlc_params = NYdb::TParamsBuilder()") + for _, parameter := range query.Parameters { + info, _ := typeInfo(parameter.Type, options.Runtime) + out.WriteString("\n .AddParam(" + strconv.Quote("$"+parameter.Name) + ")." + info.builder + "(" + parameter.Name + ").Build()") + } + out.WriteString("\n .Build();\n") + } + out.WriteString(" auto sqlc_result = sqlc_session.ExecuteQuery(\n k" + query.Name + "Sql,\n NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx()") + if len(query.Parameters) != 0 { + out.WriteString(",\n sqlc_params") + } + out.WriteString("\n ).GetValueSync();\n") + if query.Command == model.One || query.Command == model.Many { + out.WriteString(" if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) {\n sqlc_result_set = sqlc_result.GetResultSet(0);\n }\n") + } + out.WriteString(" return sqlc_result;\n });\n NYdb::ThrowOnError(sqlc_status);\n") + if query.Command == model.Exec { + out.WriteString("}\n\n") + return nil + } + out.WriteString(" if (!sqlc_result_set) {\n throw std::runtime_error(" + strconv.Quote(query.Name+": successful query returned no result set") + ");\n }\n NYdb::TResultSetParser sqlc_parser(*sqlc_result_set);\n") + if query.Command == model.One { + out.WriteString(" if (!sqlc_parser.TryNextRow()) {\n return std::nullopt;\n }\n " + query.Name + "Row sqlc_row{\n") + writeNativeRow(out, query.ResultSets[0], options.Runtime, " ") + out.WriteString(" };\n return sqlc_row;\n}\n\n") + return nil + } + out.WriteString(" std::vector<" + query.Name + "Row> sqlc_rows;\n sqlc_rows.reserve(sqlc_result_set->RowsCount());\n while (sqlc_parser.TryNextRow()) {\n sqlc_rows.push_back(" + query.Name + "Row{\n") + writeNativeRow(out, query.ResultSets[0], options.Runtime, " ") + out.WriteString(" });\n }\n return sqlc_rows;\n}\n\n") + return nil +} + +func writeNativeRow(out *strings.Builder, resultSet model.ResultSet, runtime, indent string) { + for _, column := range resultSet.Columns { + info, _ := typeInfo(column.Type, runtime) + out.WriteString(indent + "sqlc_parser.ColumnParser(" + strconv.Quote(column.Name) + ")." + info.parser + "(),\n") + } +} + +func renderUserverMethod(out *strings.Builder, query model.AnalyzedQuery, options Options) error { + returnType := "void" + if query.Command == model.One { + returnType = "std::optional<" + query.Name + "Row>" + } else if query.Command == model.Many { + returnType = "std::vector<" + query.Name + "Row>" + } + out.WriteString(returnType + " Queries::" + query.Name + "(" + methodParameters(query, options.Runtime) + ") const {\n") + call := "this->client_.ExecuteQuery(k" + query.Name + "Query" + for _, parameter := range query.Parameters { + call += ", " + strconv.Quote("$"+parameter.Name) + ", " + parameter.Name + } + call += ")" + if query.Command == model.Exec { + out.WriteString(" static_cast(" + call + ");\n}\n\n") + return nil + } + out.WriteString(" auto sqlc_response = " + call + ";\n auto sqlc_cursor = sqlc_response.GetSingleCursor();\n") + if query.Command == model.One { + out.WriteString(" if (sqlc_cursor.empty()) {\n return std::nullopt;\n }\n auto sqlc_row = sqlc_cursor.GetFirstRow();\n return " + query.Name + "Row{\n") + writeUserverRow(out, query.ResultSets[0], options.Runtime, " ") + out.WriteString(" };\n}\n\n") + return nil + } + out.WriteString(" std::vector<" + query.Name + "Row> sqlc_rows;\n sqlc_rows.reserve(sqlc_cursor.size());\n for (auto sqlc_row : sqlc_cursor) {\n sqlc_rows.push_back(" + query.Name + "Row{\n") + writeUserverRow(out, query.ResultSets[0], options.Runtime, " ") + out.WriteString(" });\n }\n return sqlc_rows;\n}\n\n") + return nil +} + +func writeUserverRow(out *strings.Builder, resultSet model.ResultSet, runtime, indent string) { + for _, column := range resultSet.Columns { + info, _ := typeInfo(column.Type, runtime) + out.WriteString(indent + "sqlc_row.Get<" + info.cpp + ">(" + strconv.Quote(column.Name) + "),\n") + } +} + +// sqlLiteral prefers readable raw strings. Bytes that C++ translation phases +// may rewrite or reject are emitted as isolated hexadecimal string fragments. +func sqlLiteral(sql string) string { + delimiter := "sqlc" + for suffix := 0; strings.Contains(sql, ")"+delimiter+"\""); suffix++ { + delimiter = "sqlc" + strconv.Itoa(suffix+1) + } + + var parts []string + var raw bytes.Buffer + flushRaw := func() { + if raw.Len() == 0 { + return + } + parts = append(parts, "R\""+delimiter+"("+raw.String()+")"+delimiter+"\"") + raw.Reset() + } + for index := 0; index < len(sql); { + r, size := utf8.DecodeRuneInString(sql[index:]) + if r == utf8.RuneError && size == 1 { + flushRaw() + parts = append(parts, fmt.Sprintf("\"\\x%02X\"", sql[index])) + index++ + continue + } + unsafe := r == '\r' || r == 0x7f || r == 0xfeff || (r < 0x20 && r != '\n' && r != '\t') + if unsafe { + flushRaw() + for _, value := range []byte(sql[index : index+size]) { + parts = append(parts, fmt.Sprintf("\"\\x%02X\"", value)) + } + } else { + raw.WriteString(sql[index : index+size]) + } + index += size + } + flushRaw() + if len(parts) == 0 { + return "std::string{}" + } + if len(parts) == 1 && strings.HasPrefix(parts[0], "R\"") { + return parts[0] + } + return "std::string{\n " + strings.Join(parts, "\n ") + ",\n " + strconv.Itoa(len(sql)) + "\n }" +} diff --git a/internal/codegen/cpp/generator_test.go b/internal/codegen/cpp/generator_test.go new file mode 100644 index 0000000..8c5a65d --- /dev/null +++ b/internal/codegen/cpp/generator_test.go @@ -0,0 +1,331 @@ +package cpp + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +func authorsAnalysis() *model.AnalysisResult { + uint64Type := model.Type{Kind: "Uint64"} + utf8Type := model.Type{Kind: "Utf8"} + optionalUtf8 := model.Optional(utf8Type) + row := model.ResultSet{Columns: []model.Column{ + {Name: "id", Type: uint64Type}, + {Name: "name", Type: utf8Type}, + {Name: "bio", Type: optionalUtf8}, + }} + return &model.AnalysisResult{Queries: []model.AnalyzedQuery{ + { + Name: "GetAuthor", Command: model.One, + SQL: "DECLARE $author_id AS Uint64;\nSELECT id, name, bio FROM authors WHERE id = $author_id;", + Parameters: []model.Parameter{{Name: "author_id", Type: uint64Type}}, + ResultSets: []model.ResultSet{row}, + }, + { + Name: "ListAuthors", Command: model.Many, + SQL: "SELECT id, name, bio FROM authors ORDER BY id;", + ResultSets: []model.ResultSet{row}, + }, + { + Name: "UpsertAuthor", Command: model.Exec, + SQL: "DECLARE $author_id AS Uint64;\nDECLARE $author_name AS Utf8;\nDECLARE $biography AS Optional;\nUPSERT INTO authors (id, name, bio)\nVALUES ($author_id, $author_name, $biography);", + Parameters: []model.Parameter{ + {Name: "author_id", Type: uint64Type}, + {Name: "author_name", Type: utf8Type}, + {Name: "biography", Type: optionalUtf8}, + }, + }, + }} +} + +func generatedContent(t *testing.T, files []model.File, name string) string { + t.Helper() + for _, file := range files { + if file.Name == name { + return string(file.Content) + } + } + t.Fatalf("missing generated file %q", name) + return "" +} + +func TestGenerateNativeYDBAuthorsAPI(t *testing.T) { + files, err := Generate(authorsAnalysis(), Options{Namespace: "example::authors", Runtime: "ydb"}) + if err != nil { + t.Fatal(err) + } + if len(files) != 3 { + t.Fatalf("got %d files, want 3", len(files)) + } + models := generatedContent(t, files, "models.hpp") + header := generatedContent(t, files, "queries.hpp") + source := generatedContent(t, files, "queries.cpp") + for _, want := range []string{ + "namespace example::authors {", + "struct GetAuthorRow final {", + "std::uint64_t id;", + "std::string name;", + "std::optional bio;", + } { + if !strings.Contains(models, want) { + t.Errorf("models.hpp missing %q:\n%s", want, models) + } + } + for _, want := range []string{ + "explicit Queries(NYdb::NQuery::TQueryClient& client) noexcept", + "std::optional GetAuthor(std::uint64_t author_id) const;", + "std::vector ListAuthors() const;", + "void UpsertAuthor(std::uint64_t author_id, const std::string& author_name, const std::optional& biography) const;", + "NYdb::NQuery::TQueryClient& client_;", + } { + if !strings.Contains(header, want) { + t.Errorf("queries.hpp missing %q:\n%s", want, header) + } + } + for _, want := range []string{ + "client_.RetryQuerySync", + "NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx()", + ".AddParam(\"$author_id\").Uint64(author_id).Build()", + ".AddParam(\"$author_name\").Utf8(author_name).Build()", + ".AddParam(\"$biography\").OptionalUtf8(biography).Build()", + "GetUint64()", + "GetUtf8()", + "GetOptionalUtf8()", + } { + if !strings.Contains(source, want) { + t.Errorf("queries.cpp missing %q:\n%s", want, source) + } + } +} + +func TestGenerateUserverAuthorsAPI(t *testing.T) { + files, err := Generate(authorsAnalysis(), Options{Namespace: "example::authors", Runtime: "userver"}) + if err != nil { + t.Fatal(err) + } + models := generatedContent(t, files, "models.hpp") + header := generatedContent(t, files, "queries.hpp") + source := generatedContent(t, files, "queries.cpp") + for _, want := range []string{ + "::userver::ydb::Utf8 name;", + "std::optional<::userver::ydb::Utf8> bio;", + } { + if !strings.Contains(models, want) { + t.Errorf("models.hpp missing %q:\n%s", want, models) + } + } + for _, want := range []string{ + "explicit Queries(::userver::ydb::TableClient& client) noexcept", + "void UpsertAuthor(std::uint64_t author_id, const ::userver::ydb::Utf8& author_name, const std::optional<::userver::ydb::Utf8>& biography) const;", + "::userver::ydb::TableClient& client_;", + } { + if !strings.Contains(header, want) { + t.Errorf("queries.hpp missing %q:\n%s", want, header) + } + } + for _, want := range []string{ + "const ::userver::ydb::Query kGetAuthorQuery", + "::userver::ydb::Query::NameLiteral{\"GetAuthor\"}", + "::userver::ydb::Query::LogMode::kNameOnly", + "client_.ExecuteQuery(kGetAuthorQuery, \"$author_id\", author_id)", + "sqlc_row.Get(\"id\")", + "sqlc_row.Get<::userver::ydb::Utf8>(\"name\")", + "sqlc_row.Get>(\"bio\")", + } { + if !strings.Contains(source, want) { + t.Errorf("queries.cpp missing %q:\n%s", want, source) + } + } +} + +func TestGenerateRejectsUnsupportedAndUnsafeInput(t *testing.T) { + tests := []struct { + name string + mutate func(*model.AnalysisResult) + opts Options + message string + }{ + {"unknown runtime", func(*model.AnalysisResult) {}, Options{Runtime: "grpc"}, "unsupported C++ runtime"}, + {"execrows", func(a *model.AnalysisResult) { a.Queries[0].Command = model.ExecRows }, Options{Runtime: "ydb"}, ":execrows"}, + {"missing result", func(a *model.AnalysisResult) { a.Queries[0].ResultSets = nil }, Options{Runtime: "ydb"}, "requires one non-empty result set"}, + {"keyword parameter", func(a *model.AnalysisResult) { a.Queries[0].Parameters[0].Name = "class" }, Options{Runtime: "ydb"}, "invalid C++ parameter"}, + {"reserved prefix", func(a *model.AnalysisResult) { a.Queries[0].ResultSets[0].Columns[0].Name = "sqlc_row" }, Options{Runtime: "ydb"}, "reserved sqlc_ prefix"}, + {"userver float", func(a *model.AnalysisResult) { a.Queries[0].Parameters[0].Type = model.Type{Kind: "Float"} }, Options{Runtime: "userver"}, "unsupported YQL type \"Float\" for userver"}, + {"nested optional", func(a *model.AnalysisResult) { + a.Queries[0].Parameters[0].Type = model.Optional(model.Optional(model.Type{Kind: "Utf8"})) + }, Options{Runtime: "ydb"}, "nested Optional"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := authorsAnalysis() + tc.mutate(a) + _, err := Generate(a, tc.opts) + if err == nil || !strings.Contains(err.Error(), tc.message) { + t.Fatalf("got error %v, want substring %q", err, tc.message) + } + }) + } +} + +func TestOneReturnsFirstRowWithoutRejectingAdditionalRows(t *testing.T) { + for _, runtime := range []string{"ydb", "userver"} { + t.Run(runtime, func(t *testing.T) { + files, err := Generate(authorsAnalysis(), Options{Runtime: runtime}) + if err != nil { + t.Fatal(err) + } + if source := generatedContent(t, files, "queries.cpp"); strings.Contains(source, ":one query returned more than one row") { + t.Fatalf(":one must return the first row, not reject extra rows:\n%s", source) + } + }) + } +} + +func TestRejectsGeneratedNameCollisions(t *testing.T) { + t.Run("query method hides row type", func(t *testing.T) { + a := authorsAnalysis() + a.Queries = append(a.Queries, model.AnalyzedQuery{Name: "GetAuthorRow", Command: model.Exec, SQL: "SELECT 1;"}) + _, err := Generate(a, Options{Runtime: "ydb"}) + if err == nil || !strings.Contains(err.Error(), "GetAuthorRow") || !strings.Contains(err.Error(), "row type") { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("field has its row type name", func(t *testing.T) { + a := authorsAnalysis() + a.Queries[0].ResultSets[0].Columns[0].Name = "GetAuthorRow" + _, err := Generate(a, Options{Runtime: "ydb"}) + if err == nil || !strings.Contains(err.Error(), "GetAuthorRow") || !strings.Contains(err.Error(), "row type") { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("parameter hides row type", func(t *testing.T) { + a := authorsAnalysis() + a.Queries[0].Parameters[0].Name = "GetAuthorRow" + _, err := Generate(a, Options{Runtime: "ydb"}) + if err == nil || !strings.Contains(err.Error(), "GetAuthorRow") || !strings.Contains(err.Error(), "row type") { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("parameter hides SQL constant", func(t *testing.T) { + a := authorsAnalysis() + a.Queries[0].Parameters[0].Name = "kGetAuthorSql" + _, err := Generate(a, Options{Runtime: "ydb"}) + if err == nil || !strings.Contains(err.Error(), "kGetAuthorSql") || !strings.Contains(err.Error(), "SQL constant") { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("query method conflicts with client member", func(t *testing.T) { + a := authorsAnalysis() + a.Queries[0].Name = "client_" + _, err := Generate(a, Options{Runtime: "ydb"}) + if err == nil || !strings.Contains(err.Error(), "invalid C++ query name") { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +func TestParameterCannotShadowClientMember(t *testing.T) { + a := authorsAnalysis() + a.Queries[0].Parameters[0].Name = "client_" + files, err := Generate(a, Options{Runtime: "ydb"}) + if err != nil { + t.Fatal(err) + } + source := generatedContent(t, files, "queries.cpp") + if !strings.Contains(source, "this->client_.RetryQuerySync") { + t.Fatalf("client member is not explicitly qualified:\n%s", source) + } +} + +func TestScalarWidthsAndStringKindsRemainDistinct(t *testing.T) { + tests := []struct { + kind string + cpp string + nativeBuilder string + nativeParser string + }{ + {"Bool", "bool", "Bool", "GetBool"}, + {"Int8", "std::int8_t", "Int8", "GetInt8"}, + {"Uint8", "std::uint8_t", "Uint8", "GetUint8"}, + {"Int16", "std::int16_t", "Int16", "GetInt16"}, + {"Uint16", "std::uint16_t", "Uint16", "GetUint16"}, + {"Int32", "std::int32_t", "Int32", "GetInt32"}, + {"Uint32", "std::uint32_t", "Uint32", "GetUint32"}, + {"Int64", "std::int64_t", "Int64", "GetInt64"}, + {"Uint64", "std::uint64_t", "Uint64", "GetUint64"}, + {"Float", "float", "Float", "GetFloat"}, + {"Double", "double", "Double", "GetDouble"}, + {"String", "std::string", "String", "GetString"}, + {"Utf8", "std::string", "Utf8", "GetUtf8"}, + } + for _, tc := range tests { + t.Run(tc.kind, func(t *testing.T) { + info, err := typeInfo(model.Type{Kind: tc.kind}, "ydb") + if err != nil { + t.Fatal(err) + } + if info.cpp != tc.cpp || info.builder != tc.nativeBuilder || info.parser != tc.nativeParser { + t.Fatalf("got %+v, want C++ %q builder %q parser %q", info, tc.cpp, tc.nativeBuilder, tc.nativeParser) + } + }) + } + stringInfo, err := typeInfo(model.Type{Kind: "String"}, "userver") + if err != nil { + t.Fatal(err) + } + utf8Info, err := typeInfo(model.Type{Kind: "Utf8"}, "userver") + if err != nil { + t.Fatal(err) + } + if stringInfo.cpp != "std::string" || utf8Info.cpp != "::userver::ydb::Utf8" { + t.Fatalf("userver string types collapsed: String=%q Utf8=%q", stringInfo.cpp, utf8Info.cpp) + } +} + +func TestSQLLiteralReadableAndRoundTripsAllBytes(t *testing.T) { + compiler, err := exec.LookPath("clang++") + if err != nil { + t.Skip("clang++ is unavailable") + } + tests := []struct { + name string + sql string + }{ + {"multiline", "-- Привет\nSELECT '\\\"', `name`\nFROM authors;\n"}, + {"delimiter collision", "SELECT ')sqlc\"', ')sqlc1\"';"}, + {"controls", "SELECT '\x00\a\b\f\r\v\x1b\x7f';\n"}, + {"invalid utf8", string([]byte{'S', 'E', 'L', 'E', 'C', 'T', ' ', 0xff, ';'})}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + literal := sqlLiteral(tc.sql) + if tc.name == "multiline" && (!strings.HasPrefix(literal, `R"sqlc(`) || strings.Contains(literal, `\nSELECT`)) { + t.Fatalf("multiline SQL is not a readable raw literal: %s", literal) + } + dir := t.TempDir() + source := "#include \n#include \nint main() { const std::string value = " + literal + "; std::cout.write(value.data(), value.size()); }\n" + input := filepath.Join(dir, "literal.cpp") + binary := filepath.Join(dir, "literal") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(compiler, "-std=c++20", input, "-o", binary) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("compile SQL literal: %v\n%s\n%s", err, out, source) + } + got, err := exec.Command(binary).Output() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(tc.sql)) { + t.Fatalf("round trip mismatch:\n got: %q\nwant: %q\nliteral: %s", got, []byte(tc.sql), literal) + } + }) + } +} diff --git a/internal/codegen/csharp/generator.go b/internal/codegen/csharp/generator.go new file mode 100644 index 0000000..f78e1f7 --- /dev/null +++ b/internal/codegen/csharp/generator.go @@ -0,0 +1,436 @@ +// Package csharp renders the resolved YQL model as modern C# ADO.NET code. +package csharp + +import ( + "bytes" + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +// Options controls the generated namespace. +// Generated code always targets the concrete modern Ydb.Sdk.Ado provider. +type Options struct{ Namespace string } + +func Generate(in *model.AnalysisResult, o Options) ([]model.File, error) { + if in == nil { + return nil, fmt.Errorf("csharp generator: analysis result is nil") + } + if len(in.Diagnostics) != 0 { + return nil, fmt.Errorf("csharp generator: cannot generate with diagnostics: %s", in.Diagnostics[0]) + } + if o.Namespace == "" { + o.Namespace = "Db" + } + if !namespace(o.Namespace) { + return nil, fmt.Errorf("csharp generator: invalid namespace %q", o.Namespace) + } + if err := validate(in); err != nil { + return nil, err + } + return []model.File{ + {Name: "Models.cs", Content: renderModels(in, o)}, + {Name: "Queries.cs", Content: renderQueries(in, o)}, + }, nil +} + +func validate(in *model.AnalysisResult) error { + queryNames, methodNames := map[string]string{}, map[string]string{} + // Queries is emitted by this generator, so no record may reuse its name. + modelNames := map[string]string{"Queries": "generated query class"} + for _, n := range []string{"Guid", "Task", "CancellationToken", "List", "IReadOnlyList", "DbType", "DBNull", "YdbConnection", "YdbTransaction", "YdbCommand", "YdbParameter"} { + modelNames[n] = "framework type" + } + add := func(dst map[string]string, name, original, what string) error { + if old, ok := dst[name]; ok && old != original { + return fmt.Errorf("csharp generator: %s collision %q (%q and %q)", what, name, old, original) + } + dst[name] = original + return nil + } + for _, table := range in.Catalog.Tables { + if err := add(modelNames, csName(table.Name), "table:"+table.Name, "model name"); err != nil { + return err + } + if err := fields("table "+table.Name, table.Columns); err != nil { + return err + } + } + for _, q := range in.Queries { + if !csIdent(q.Name) { + return fmt.Errorf("csharp generator: invalid query name %q", q.Name) + } + if err := add(queryNames, csName(q.Name), q.Name, "SQL constant"); err != nil { + return err + } + if err := add(methodNames, csName(q.Name)+"Async", q.Name, "method name"); err != nil { + return err + } + switch q.Command { + case model.One, model.Many, model.Exec: + default: + return fmt.Errorf("csharp generator: query %q: unsupported command %q", q.Name, q.Command) + } + if (q.Command == model.One || q.Command == model.Many) && (len(q.ResultSets) != 1 || len(q.ResultSets[0].Columns) == 0) { + return fmt.Errorf("csharp generator: query %q: %s requires one non-empty result set", q.Name, q.Command) + } + if !utf8.ValidString(q.SQL) { + return fmt.Errorf("csharp generator: query %q: SQL is not valid UTF-8", q.Name) + } + seen := map[string]bool{} + for _, p := range q.Parameters { + n := csName(p.Name) + if !csIdent(n) || seen[n] { + return fmt.Errorf("csharp generator: query %q: parameter name collision at %q", q.Name, p.Name) + } + seen[n] = true + if _, err := csType(p.Type); err != nil { + return fmt.Errorf("csharp generator: query %q parameter %q: %w", q.Name, p.Name, err) + } + } + if len(q.Parameters) > 1 { + if err := add(modelNames, csName(q.Name)+"Params", "params:"+q.Name, "model name"); err != nil { + return err + } + } + if q.Command == model.One || q.Command == model.Many { + if err := add(modelNames, csName(q.Name)+"Row", "row:"+q.Name, "model name"); err != nil { + return err + } + if err := fields("query "+q.Name, q.ResultSets[0].Columns); err != nil { + return err + } + } + } + return nil +} + +func fields(where string, columns []model.Column) error { + seen := map[string]bool{} + for _, c := range columns { + n := csName(c.Name) + if !csIdent(n) || seen[n] { + return fmt.Errorf("csharp generator: %s: column name collision at %q", where, c.Name) + } + seen[n] = true + if _, err := csType(c.Type); err != nil { + return fmt.Errorf("csharp generator: %s column %q: %w", where, c.Name, err) + } + } + return nil +} + +func csType(t model.Type) (string, error) { + if t.IsOptional() { + if t.Elem == nil { + return "", fmt.Errorf("Optional lacks element") + } + if t.Elem.IsOptional() { + return "", fmt.Errorf("nested Optional is unsupported") + } + e, err := csType(*t.Elem) + if err != nil { + return "", err + } + if e == "string" || e == "byte[]" { + return e + "?", nil + } + return e + "?", nil + } + switch strings.ToLower(t.Kind) { + case "bool": + return "bool", nil + case "int8": + return "sbyte", nil + case "int16": + return "short", nil + case "int32": + return "int", nil + case "int64": + return "long", nil + case "uint8": + return "byte", nil + case "uint16": + return "ushort", nil + case "uint32": + return "uint", nil + case "uint64": + return "ulong", nil + case "float": + return "float", nil + case "double": + return "double", nil + case "utf8": + return "string", nil + case "string": + return "byte[]", nil + case "uuid": + return "Guid", nil + default: + return "", fmt.Errorf("unsupported YQL type %q", t.Kind) + } +} + +func renderModels(in *model.AnalysisResult, o Options) []byte { + var b bytes.Buffer + b.WriteString(modelsHeader(o) + "\n") + emitted := map[string]bool{} + write := func(name string, cols []model.Column) { + if emitted[name] { + return + } + emitted[name] = true + b.WriteString("public sealed record " + name + "(\n") + for i, c := range cols { + typ, _ := csType(c.Type) + comma := "," + if i == len(cols)-1 { + comma = "" + } + fmt.Fprintf(&b, " %s %s%s\n", typ, csName(c.Name), comma) + } + b.WriteString(");\n\n") + } + for _, t := range in.Catalog.Tables { + write(csName(t.Name), t.Columns) + } + for _, q := range in.Queries { + if len(q.Parameters) > 1 { + ps := make([]model.Column, len(q.Parameters)) + for i, p := range q.Parameters { + ps[i] = model.Column{Name: p.Name, Type: p.Type} + } + write(csName(q.Name)+"Params", ps) + } + if q.Command == model.One || q.Command == model.Many { + write(csName(q.Name)+"Row", q.ResultSets[0].Columns) + } + } + return []byte(strings.TrimRight(b.String(), "\n") + "\n") +} + +func renderQueries(in *model.AnalysisResult, o Options) []byte { + var b bytes.Buffer + b.WriteString(queriesHeader(o)) + b.WriteString("\npublic sealed class Queries\n{\n") + b.WriteString(" private readonly YdbConnection _connection;\n private readonly YdbTransaction? _transaction;\n\n public Queries(YdbConnection connection, YdbTransaction? transaction = null)\n {\n _connection = connection ?? throw new ArgumentNullException(nameof(connection));\n _transaction = transaction;\n }\n\n public Queries WithTransaction(YdbTransaction transaction) => new(_connection, transaction ?? throw new ArgumentNullException(nameof(transaction)));\n") + for _, q := range in.Queries { + writeSQLConstant(&b, q) + writeMethod(&b, q) + } + b.WriteString("}\n") + return b.Bytes() +} + +func modelsHeader(o Options) string { + return "// Code generated by sqlc-ydb. DO NOT EDIT.\n#nullable enable\nusing System;\n\nnamespace " + o.Namespace + ";" +} +func queriesHeader(o Options) string { + return "// Code generated by sqlc-ydb. DO NOT EDIT.\n#nullable enable\nusing System;\nusing System.Collections.Generic;\nusing System.Data;\nusing System.Data.Common;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing Ydb.Sdk.Ado;\n\nnamespace " + o.Namespace + ";\n\n" +} +func writeSQLConstant(b *bytes.Buffer, q model.AnalyzedQuery) { + fmt.Fprintf(b, "\n private const string Sql%s =\n", csName(q.Name)) + parts := strings.SplitAfter(q.SQL, "\n") + if len(parts) > 1 && parts[len(parts)-1] == "" { + parts = parts[:len(parts)-1] + } + if len(parts) == 0 { + parts = []string{""} + } + for i, p := range parts { + end := ";" + if i != len(parts)-1 { + end = " +" + } + fmt.Fprintf(b, " %s%s\n", csString(p), end) + } +} + +func writeMethod(b *bytes.Buffer, q model.AnalyzedQuery) { + name := csName(q.Name) + ret := "Task" + if q.Command == model.One { + ret = "Task<" + name + "Row>" + } + if q.Command == model.Many { + ret = "Task>" + } + fmt.Fprintf(b, "\n public async %s %sAsync(%sCancellationToken cancellationToken = default)\n {\n", ret, name, methodParameters(q)) + fmt.Fprintf(b, " await using var command = new YdbCommand(Sql%s, _connection) { Transaction = _transaction };\n", name) + for _, p := range q.Parameters { + writeParameter(b, q, p) + } + switch q.Command { + case model.Exec: + b.WriteString(" await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);\n") + case model.One: + b.WriteString(" await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);\n if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))\n {\n throw new InvalidOperationException(\"query returned no rows\");\n }\n return " + name + "RowFrom(reader);\n") + case model.Many: + b.WriteString(" await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);\n var rows = new List<" + name + "Row>();\n while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))\n {\n rows.Add(" + name + "RowFrom(reader));\n }\n return rows;\n") + } + b.WriteString(" }\n") + if q.Command == model.One || q.Command == model.Many { + fmt.Fprintf(b, "\n private static %sRow %sRowFrom(DbDataReader reader) => new(\n", name, name) + for i, c := range q.ResultSets[0].Columns { + comma := "," + if i == len(q.ResultSets[0].Columns)-1 { + comma = "" + } + fmt.Fprintf(b, " %s%s\n", readValue(c, i), comma) + } + b.WriteString(" );\n") + } +} + +func methodParameters(q model.AnalyzedQuery) string { + if len(q.Parameters) == 0 { + return "" + } + if len(q.Parameters) > 1 { + return csName(q.Name) + "Params args, " + } + typ, _ := csType(q.Parameters[0].Type) + return typ + " " + csName(q.Parameters[0].Name) + ", " +} +func parameterRef(q model.AnalyzedQuery, p model.Parameter) string { + if len(q.Parameters) > 1 { + return "args." + csName(p.Name) + } + return csName(p.Name) +} +func writeParameter(b *bytes.Buffer, q model.AnalyzedQuery, p model.Parameter) { + v := parameterRef(q, p) + fmt.Fprintf(b, " command.Parameters.Add(new YdbParameter(%q, DbType.%s, %s));\n", "$"+p.Name, dbType(p.Type), nullableValue(p.Type, v)) +} +func nullableValue(t model.Type, v string) string { + if t.IsOptional() { + return "(object?)" + v + " ?? DBNull.Value" + } + return v +} +func dbType(t model.Type) string { + if t.IsOptional() { + t = *t.Elem + } + switch strings.ToLower(t.Kind) { + case "utf8": + return "String" + case "string": + return "Binary" + case "bool": + return "Boolean" + case "float": + return "Single" + case "double": + return "Double" + case "uuid": + return "Guid" + case "int8": + return "SByte" + case "int16": + return "Int16" + case "int32": + return "Int32" + case "int64": + return "Int64" + case "uint8": + return "Byte" + case "uint16": + return "UInt16" + case "uint32": + return "UInt32" + case "uint64": + return "UInt64" + default: + return csName(t.Kind) + } +} +func readValue(c model.Column, i int) string { + typ, _ := csType(c.Type) + bare := strings.TrimSuffix(typ, "?") + get := fmt.Sprintf("reader.GetFieldValue<%s>(%d)", bare, i) + if c.Type.IsOptional() { + return fmt.Sprintf("reader.IsDBNull(%d) ? null : %s", i, get) + } + return get +} + +func csString(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch r { + case '\\': + b.WriteString("\\\\") + case '"': + b.WriteString("\\\"") + case '\n': + b.WriteString("\\n") + case '\r': + b.WriteString("\\r") + case '\t': + b.WriteString("\\t") + case '\b': + b.WriteString("\\b") + case '\f': + b.WriteString("\\f") + default: + if r < 0x20 || r == 0x7f { + fmt.Fprintf(&b, "\\u%04X", r) + } else { + b.WriteRune(r) + } + } + } + b.WriteByte('"') + return b.String() +} + +func csName(s string) string { + var b strings.Builder + for _, p := range strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) { + if strings.EqualFold(p, "id") { + b.WriteString("ID") + continue + } + for i, r := range p { + if i == 0 { + b.WriteRune(unicode.ToUpper(r)) + } else { + b.WriteRune(r) + } + } + } + n := b.String() + if n == "" { + return "Value" + } + if unicode.IsDigit([]rune(n)[0]) { + return "Value" + n + } + return n +} +func csIdent(s string) bool { + if s == "" || csKeywords[s] { + return false + } + for i, r := range s { + if !(r == '_' || unicode.IsLetter(r) || (i > 0 && unicode.IsDigit(r))) { + return false + } + } + return true +} +func namespace(s string) bool { + for _, p := range strings.Split(s, ".") { + if !csIdent(p) { + return false + } + } + return true +} + +var csKeywords = map[string]bool{"abstract": true, "as": true, "base": true, "bool": true, "break": true, "byte": true, "case": true, "catch": true, "char": true, "checked": true, "class": true, "const": true, "continue": true, "decimal": true, "default": true, "delegate": true, "do": true, "double": true, "else": true, "enum": true, "event": true, "explicit": true, "extern": true, "false": true, "finally": true, "fixed": true, "float": true, "for": true, "foreach": true, "goto": true, "if": true, "implicit": true, "in": true, "int": true, "interface": true, "internal": true, "is": true, "lock": true, "long": true, "namespace": true, "new": true, "null": true, "object": true, "operator": true, "out": true, "override": true, "params": true, "private": true, "protected": true, "public": true, "readonly": true, "ref": true, "return": true, "sbyte": true, "sealed": true, "short": true, "sizeof": true, "stackalloc": true, "static": true, "string": true, "struct": true, "switch": true, "this": true, "throw": true, "true": true, "try": true, "typeof": true, "uint": true, "ulong": true, "unchecked": true, "unsafe": true, "ushort": true, "using": true, "virtual": true, "void": true, "volatile": true, "while": true} diff --git a/internal/codegen/csharp/generator_test.go b/internal/codegen/csharp/generator_test.go new file mode 100644 index 0000000..2f6bb99 --- /dev/null +++ b/internal/codegen/csharp/generator_test.go @@ -0,0 +1,225 @@ +package csharp + +import ( + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +func authorsAnalysis() *model.AnalysisResult { + utf8 := model.Type{Kind: "Utf8"} + return &model.AnalysisResult{ + Catalog: model.Catalog{Tables: []model.Table{{Name: "authors", Columns: []model.Column{ + {Name: "id", Type: model.Type{Kind: "Uint64"}}, {Name: "name", Type: utf8}, {Name: "bio", Type: model.Optional(utf8)}, + }}}}, + Queries: []model.AnalyzedQuery{ + {Name: "GetAuthor", Command: model.One, SQL: "DECLARE $author_id AS Uint64;\nSELECT id, name, bio FROM authors WHERE id = $author_id;\n", Parameters: []model.Parameter{{Name: "author_id", Type: model.Type{Kind: "Uint64"}}}, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "id", Type: model.Type{Kind: "Uint64"}}, {Name: "name", Type: utf8}, {Name: "bio", Type: model.Optional(utf8)}}}}}, + {Name: "ListAuthors", Command: model.Many, SQL: "SELECT id, name, bio FROM authors;", ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "id", Type: model.Type{Kind: "Uint64"}}, {Name: "name", Type: utf8}, {Name: "bio", Type: model.Optional(utf8)}}}}}, + {Name: "UpsertAuthor", Command: model.Exec, SQL: "DECLARE $author_id AS Uint64; DECLARE $author_name AS Utf8; DECLARE $biography AS Optional; UPSERT INTO authors VALUES ($author_id, $author_name, $biography);", Parameters: []model.Parameter{{Name: "author_id", Type: model.Type{Kind: "Uint64"}}, {Name: "author_name", Type: utf8}, {Name: "biography", Type: model.Optional(utf8)}}}, + }, + } +} + +func generated(t *testing.T, a *model.AnalysisResult) (string, string) { + t.Helper() + files, err := Generate(a, Options{Namespace: "Authors.AdoNet"}) + if err != nil { + t.Fatal(err) + } + return string(files[0].Content), string(files[1].Content) +} + +func TestGenerateUsesConcreteModernYdbAdoSurface(t *testing.T) { + models, queries := generated(t, authorsAnalysis()) + for _, want := range []string{ + "namespace Authors.AdoNet;", "public sealed record Authors(", "public sealed record GetAuthorRow(", "string? Bio", "public sealed record UpsertAuthorParams(", + } { + if !strings.Contains(models, want) { + t.Errorf("Models.cs missing %q:\n%s", want, models) + } + } + for _, want := range []string{ + "using Ydb.Sdk.Ado;", "private readonly YdbConnection _connection;", "private readonly YdbTransaction? _transaction;", "WithTransaction(YdbTransaction transaction)", + "new YdbCommand(SqlGetAuthor, _connection) { Transaction = _transaction }", "new YdbParameter(\"$author_id\", DbType.UInt64, AuthorID)", + "new YdbParameter(\"$biography\", DbType.String, (object?)args.Biography ?? DBNull.Value)", "ExecuteReaderAsync(cancellationToken)", "ReadAsync(cancellationToken)", "reader.IsDBNull(2) ? null : reader.GetFieldValue(2)", + } { + if !strings.Contains(queries, want) { + t.Errorf("Queries.cs missing %q:\n%s", want, queries) + } + } + if strings.Contains(queries, "YdbDataSource") || strings.Contains(queries, "TableClient") || strings.Contains(queries, "DbConnection") { + t.Fatalf("generator must not own a data source or use legacy/generic surface:\n%s", queries) + } +} + +func TestSQLLiteralPreservesControlsQuotesAndBackslashes(t *testing.T) { + a := authorsAnalysis() + sql := "SELECT '\"', '\\\\', '" + string(rune(0)) + "', '" + string(rune(0x1f)) + "';\r\n-- \"\"\" delimiter-looking text\n" + a.Queries = a.Queries[:1] + a.Queries[0].SQL = sql + _, queries := generated(t, a) + for _, want := range []string{`"SELECT '\"', '\\\\', '\u0000', '\u001F';\r\n" +`, `"-- \"\"\" delimiter-looking text\n";`} { + if !strings.Contains(queries, want) { + t.Errorf("SQL literal did not use portable exact escaping %q:\n%s", want, queries) + } + } +} + +func TestRejectsUnsupportedOrCollidingInput(t *testing.T) { + for _, tc := range []struct { + name string + in *model.AnalysisResult + opts Options + want string + }{ + {"list", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, Parameters: []model.Parameter{{Name: "x", Type: model.Type{Kind: "List"}}}}}}, Options{}, "unsupported YQL type"}, + {"execrows", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.ExecRows}}}, Options{}, "unsupported command"}, + {"field collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "t", Columns: []model.Column{{Name: "a_b", Type: model.Type{Kind: "Utf8"}}, {Name: "a b", Type: model.Type{Kind: "Utf8"}}}}}}}, Options{}, "column name collision"}, + {"generated type collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "queries"}}}}, Options{}, "model name collision"}, + {"namespace", authorsAnalysis(), Options{Namespace: "Bad.class"}, "invalid namespace"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := Generate(tc.in, tc.opts) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Generate() error = %v, want %q", err, tc.want) + } + }) + } +} + +// This opt-in check validates the generated source against the published SDK, +// rather than a mock provider. It is opt-in because contributors may not have +// the .NET SDK installed. Example: +// SQLC_YDB_CSHARP_DOTNET=/path/to/dotnet go test ./internal/codegen/csharp -run Published +func TestGeneratedCodeBuildsAgainstPublishedSDK(t *testing.T) { + dotnet := os.Getenv("SQLC_YDB_CSHARP_DOTNET") + if dotnet == "" { + t.Skip("set SQLC_YDB_CSHARP_DOTNET to run the published-SDK build") + } + files, err := Generate(authorsAnalysis(), Options{Namespace: "Authors.AdoNet"}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + for _, file := range files { + if err := os.WriteFile(filepath.Join(dir, file.Name), file.Content, 0600); err != nil { + t.Fatal(err) + } + } + project := `net8.0enabletrue` + if err := os.WriteFile(filepath.Join(dir, "generated.csproj"), []byte(project), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(dotnet, "build", "--nologo") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("generated C# does not build against Ydb.Sdk 0.33.3: %v\n%s", err, out) + } +} + +func TestAllSupportedScalarsBuildAgainstPublishedSDK(t *testing.T) { + dotnet := os.Getenv("SQLC_YDB_CSHARP_DOTNET") + if dotnet == "" { + t.Skip("set SQLC_YDB_CSHARP_DOTNET to run the published-SDK build") + } + types := []model.Type{ + {Kind: "Bool"}, {Kind: "Int8"}, {Kind: "Int16"}, {Kind: "Int32"}, {Kind: "Int64"}, + {Kind: "Uint8"}, {Kind: "Uint16"}, {Kind: "Uint32"}, {Kind: "Uint64"}, + {Kind: "Float"}, {Kind: "Double"}, {Kind: "Utf8"}, {Kind: "String"}, {Kind: "Uuid"}, + } + var parameters []model.Parameter + var columns []model.Column + for _, typ := range types { + parameters = append(parameters, model.Parameter{Name: strings.ToLower(typ.Kind), Type: typ}) + parameters = append(parameters, model.Parameter{Name: "optional_" + strings.ToLower(typ.Kind), Type: model.Optional(typ)}) + columns = append(columns, model.Column{Name: strings.ToLower(typ.Kind), Type: typ}) + columns = append(columns, model.Column{Name: "optional_" + strings.ToLower(typ.Kind), Type: model.Optional(typ)}) + } + in := &model.AnalysisResult{Queries: []model.AnalyzedQuery{{ + Name: "AllScalars", + Command: model.One, + SQL: "SELECT 1;", + Parameters: parameters, + ResultSets: []model.ResultSet{{Columns: columns}}, + }}} + files, err := Generate(in, Options{Namespace: "Scalars"}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + for _, file := range files { + if err := os.WriteFile(filepath.Join(dir, file.Name), file.Content, 0600); err != nil { + t.Fatal(err) + } + } + project := `net8.0enabletrue` + if err := os.WriteFile(filepath.Join(dir, "scalars.csproj"), []byte(project), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(dotnet, "build", "--nologo") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("all generated scalar bindings must compile against Ydb.Sdk 0.33.3: %v\n%s", err, out) + } +} + +func TestRejectsModelNamesThatShadowFrameworkTypes(t *testing.T) { + for _, table := range []string{"guid", "task", "cancellation_token"} { + t.Run(table, func(t *testing.T) { + _, err := Generate(&model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: table}}}}, Options{}) + if err == nil || !strings.Contains(err.Error(), "model name collision") { + t.Fatalf("Generate() error = %v, want framework model-name collision", err) + } + }) + } +} + +// This is an execution check, not merely a source inspection: C# evaluates the +// emitted literal and compares its UTF-8 bytes with the original SQL. +func TestSQLLiteralRoundTripsThroughCSharpRuntime(t *testing.T) { + dotnet := os.Getenv("SQLC_YDB_CSHARP_DOTNET") + if dotnet == "" { + t.Skip("set SQLC_YDB_CSHARP_DOTNET to run the C# SQL-literal check") + } + sql := "SELECT '\"', '\\\\', '" + string(rune(0)) + "', '" + string(rune(0x1f)) + "';\r\n-- \"\"\" delimiter-looking text\n" + in := authorsAnalysis() + in.Queries = in.Queries[:1] + in.Queries[0].SQL = sql + files, err := Generate(in, Options{Namespace: "Authors.AdoNet"}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + for _, file := range files { + if err := os.WriteFile(filepath.Join(dir, file.Name), file.Content, 0600); err != nil { + t.Fatal(err) + } + } + project := `Exenet8.0enabletrue` + if err := os.WriteFile(filepath.Join(dir, "literal.csproj"), []byte(project), 0600); err != nil { + t.Fatal(err) + } + program := fmt.Sprintf(`using System; using System.Reflection; using System.Text; using Authors.AdoNet; internal static class Program { static int Main() { var actual = (string)typeof(Queries).GetField("SqlGetAuthor", BindingFlags.Static | BindingFlags.NonPublic)!.GetValue(null)!; if (Convert.ToBase64String(Encoding.UTF8.GetBytes(actual)) != %q) throw new Exception("SQL changed"); return 0; } }`, base64.StdEncoding.EncodeToString([]byte(sql))) + if err := os.WriteFile(filepath.Join(dir, "Program.cs"), []byte(program), 0600); err != nil { + t.Fatal(err) + } + env := append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + build := exec.Command(dotnet, "build", "--nologo") + build.Dir, build.Env = dir, env + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("literal build: %v\n%s", err, out) + } + run := exec.Command(dotnet, "run", "--no-build", "--nologo") + run.Dir, run.Env = dir, env + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("literal runtime: %v\n%s", err, out) + } +} diff --git a/internal/codegen/java/generator.go b/internal/codegen/java/generator.go new file mode 100644 index 0000000..2466e4e --- /dev/null +++ b/internal/codegen/java/generator.go @@ -0,0 +1,410 @@ +// Package java generates SQL-first Java APIs for the YDB SDK and JDBC integrations. +package java + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +type Options struct{ Package, Runtime string } + +type scalar struct{ typ, boxed, sdk, jdbc, sqlType string } + +var scalars = map[string]scalar{ + "Bool": {"boolean", "Boolean", "Bool", "Boolean", "BOOLEAN"}, + "Int8": {"byte", "Byte", "Int8", "Byte", "TINYINT"}, + "Uint8": {"int", "Integer", "Uint8", "Int", "INTEGER"}, + "Int16": {"short", "Short", "Int16", "Short", "SMALLINT"}, + "Uint16": {"int", "Integer", "Uint16", "Int", "INTEGER"}, + "Int32": {"int", "Integer", "Int32", "Int", "INTEGER"}, + "Uint32": {"long", "Long", "Uint32", "Long", "BIGINT"}, + "Int64": {"long", "Long", "Int64", "Long", "BIGINT"}, + "Uint64": {"long", "Long", "Uint64", "Long", "BIGINT"}, + "Float": {"float", "Float", "Float", "Float", "FLOAT"}, + "Double": {"double", "Double", "Double", "Double", "DOUBLE"}, + "Utf8": {"String", "String", "Text", "String", "VARCHAR"}, + "String": {"byte[]", "byte[]", "Bytes", "Bytes", "BINARY"}, +} + +func typeInfo(t model.Type) (scalar, string, error) { + s, ok := scalars[t.UnwrapOptional().Kind] + if !ok { + return s, "", fmt.Errorf("unsupported Java type %s", t.Kind) + } + if t.IsOptional() { + return s, s.boxed, nil + } + return s, s.typ, nil +} + +var reserved = func() map[string]bool { + m := map[string]bool{} + for _, s := range strings.Fields("abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for goto if implements import instanceof int interface long native new package private protected public return short static strictfp super switch synchronized this throw throws transient try void volatile while true false null _ record sealed permits var yield when clone finalize getClass hashCode notify notifyAll toString wait") { + m[s] = true + } + return m +}() + +func identifier(s string) bool { + if s == "" || reserved[s] { + return false + } + for i, r := range s { + if !((r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || (i > 0 && r >= '0' && r <= '9')) { + return false + } + } + return true +} + +func name(s string, upper bool) (string, error) { + var b strings.Builder + for _, r := range s { + if r == '_' || r == '-' || r == ' ' || r == '.' { + upper = true + continue + } + if upper && r >= 'a' && r <= 'z' { + r -= 'a' - 'A' + } else if b.Len() == 0 && !upper && r >= 'A' && r <= 'Z' { + r += 'a' - 'A' + } + b.WriteRune(r) + upper = false + } + n := b.String() + if reserved[n] { + n += "_" + } + if !identifier(n) { + return "", fmt.Errorf("cannot represent %q as a Java identifier", s) + } + return n, nil +} + +// quoted also escapes backslashes preceding u: Java Unicode escapes are processed +// before tokenization. Doubling every input backslash keeps them literal. +func quoted(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch r { + case '\\', '"': + b.WriteByte('\\') + b.WriteRune(r) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + if r < 32 || r == 127 { + fmt.Fprintf(&b, "\\%03o", r) + } else { + b.WriteRune(r) + } + } + } + b.WriteByte('"') + return b.String() +} + +// sqlLiteral emits a Java 17 text block. Escaped quotes cannot close the block; +// escaped trailing spaces survive incidental whitespace stripping. The final +// continuation suppresses only the newline introduced by the closing delimiter. +func sqlLiteral(s string) string { + var b strings.Builder + b.WriteString("\"\"\"\n") + for i, line := range strings.Split(s, "\n") { + if i > 0 { + b.WriteByte('\n') + } + encoded := quoted(line) + encoded = encoded[1 : len(encoded)-1] + if strings.HasSuffix(encoded, " ") { + encoded = strings.TrimSuffix(encoded, " ") + `\s` + } + b.WriteString(encoded) + } + b.WriteString("\\\n\"\"\"") + return b.String() +} + +func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { + if a == nil { + return nil, fmt.Errorf("nil analysis result") + } + if len(a.Diagnostics) != 0 { + return nil, fmt.Errorf("cannot generate Java with analysis diagnostics") + } + if o.Package == "" { + o.Package = "db" + } + if o.Package == "java" || strings.HasPrefix(o.Package, "java.") { + return nil, fmt.Errorf("invalid Java package %q: java packages are reserved by the JVM", o.Package) + } + for _, p := range strings.Split(o.Package, ".") { + if !identifier(p) { + return nil, fmt.Errorf("invalid Java package %q", o.Package) + } + } + if o.Runtime == "" || o.Runtime == "native" { + o.Runtime = "ydb" + } + if o.Runtime != "ydb" && o.Runtime != "jdbc" && o.Runtime != "spring" && o.Runtime != "hibernate" { + return nil, fmt.Errorf("unsupported Java runtime %q", o.Runtime) + } + header := "// Code generated by sqlc-ydb. DO NOT EDIT.\npackage " + o.Package + ";\n\n" + files := []model.File{} + types := map[string]bool{"Queries": true, "String": true, "Long": true, "Integer": true, "Short": true, "Byte": true, "Boolean": true, "Float": true, "Double": true} + for _, n := range []string{"SessionRetryContext", "QueryReader", "TxMode", "Params", "PrimitiveValue", "PrimitiveType", "OptionalType", "IllegalStateException"} { + types[n] = true + } + addRecord := func(n string, cols []model.Column) error { + if types[n] { + return fmt.Errorf("Java type name collision: %s", n) + } + types[n] = true + fields := []string{} + seen := map[string]bool{} + for _, c := range cols { + field, err := name(c.Name, false) + if err != nil { + return err + } + if seen[field] { + return fmt.Errorf("Java field name collision in %s: %s", n, field) + } + seen[field] = true + _, t, err := typeInfo(c.Type) + if err != nil { + return fmt.Errorf("%s.%s: %w", n, c.Name, err) + } + fields = append(fields, t+" "+field) + } + files = append(files, model.File{Name: n + ".java", Content: []byte(header + "public record " + n + "(" + strings.Join(fields, ", ") + ") {}\n")}) + return nil + } + for _, table := range a.Catalog.Tables { + n, err := name(table.Name, true) + if err != nil { + return nil, err + } + if err := addRecord(n, table.Columns); err != nil { + return nil, err + } + } + var b strings.Builder + b.WriteString(header) + if o.Runtime == "ydb" { + b.WriteString("import tech.ydb.query.tools.SessionRetryContext;\nimport tech.ydb.query.tools.QueryReader;\nimport tech.ydb.common.transaction.TxMode;\nimport tech.ydb.table.query.Params;\n") + } + needsValues := o.Runtime == "ydb" + for _, q := range a.Queries { + needsValues = needsValues || len(q.Parameters) > 0 + } + if needsValues { + b.WriteString("import tech.ydb.table.values.PrimitiveValue;\nimport tech.ydb.table.values.PrimitiveType;\nimport tech.ydb.table.values.OptionalType;\n\n") + } + owner := map[string]string{"ydb": "SessionRetryContext", "jdbc": "java.sql.Connection", "spring": "org.springframework.jdbc.core.JdbcTemplate", "hibernate": "org.hibernate.Session"}[o.Runtime] + fmt.Fprintf(&b, "// The caller owns the injected client and its lifecycle.\npublic final class Queries {\n private final %s client;\n\n public Queries(%s client) {\n this.client = java.util.Objects.requireNonNull(client);\n }\n", owner, owner) + methods := map[string]bool{} + for _, q := range a.Queries { + if !utf8.ValidString(q.SQL) { + return nil, fmt.Errorf("%s: Java SQL must be valid UTF-8", q.Name) + } + method, err := name(q.Name, false) + if err != nil { + return nil, err + } + if methods[method] { + return nil, fmt.Errorf("Java method name collision: %s", method) + } + methods[method] = true + row, _ := name(q.Name, true) + row += "Row" + ret := "void" + switch q.Command { + case model.One, model.Many: + if len(q.ResultSets) != 1 || len(q.ResultSets[0].Columns) == 0 { + return nil, fmt.Errorf("%s: %s requires one nonempty result set", q.Name, q.Command) + } + if err := addRecord(row, q.ResultSets[0].Columns); err != nil { + return nil, err + } + if q.Command == model.One { + ret = "java.util.Optional<" + row + ">" + } else { + ret = "java.util.List<" + row + ">" + } + case model.Exec: + default: + return nil, fmt.Errorf("%s: Java does not support %s", q.Name, q.Command) + } + params := []string{} + paramNames := []string{} + seen := map[string]bool{"client": true, "_params": true, "_query": true, "_connection": true, "_statement": true, "_prepared": true, "_rows": true, "_items": true} + for _, p := range q.Parameters { + n, err := name(p.Name, false) + if err != nil { + return nil, err + } + if seen[n] { + return nil, fmt.Errorf("%s: Java parameter name collision: %s", q.Name, n) + } + seen[n] = true + _, typ, err := typeInfo(p.Type) + if err != nil { + return nil, fmt.Errorf("%s parameter %s: %w", q.Name, p.Name, err) + } + params = append(params, typ+" "+n) + paramNames = append(paramNames, n) + } + constant := method + "Sql" + fmt.Fprintf(&b, "\n private static final String %s = %s;\n", constant, sqlLiteral(q.SQL)) + throws := "" + if o.Runtime == "jdbc" { + throws = " throws java.sql.SQLException" + } + fmt.Fprintf(&b, "\n public %s %s(%s)%s {\n", ret, method, strings.Join(params, ", "), throws) + // Java's wider signed carriers must not be silently narrowed by the SDK. + // Uint64 deliberately uses all 64 bits of long and needs no range check. + for i, p := range q.Parameters { + max := map[string]string{"Uint8": "255", "Uint16": "65535", "Uint32": "4294967295L"}[p.Type.UnwrapOptional().Kind] + if max == "" { + continue + } + n := paramNames[i] + condition := n + " < 0 || " + n + " > " + max + if p.Type.IsOptional() { + condition = n + " != null && (" + condition + ")" + } + fmt.Fprintf(&b, " if (%s) throw new IllegalArgumentException(%s);\n", condition, quoted("parameter $"+p.Name+" is outside "+p.Type.UnwrapOptional().Kind+" range")) + } + if o.Runtime == "ydb" { + emitNative(&b, q, paramNames, constant, row) + } else { + emitJDBC(&b, q, paramNames, constant, row, ret, o.Runtime) + } + b.WriteString(" }\n") + } + b.WriteString("}\n") + files = append(files, model.File{Name: "Queries.java", Content: []byte(b.String())}) + return files, nil +} + +func emitNative(b *strings.Builder, q model.AnalyzedQuery, names []string, constant, row string) { + b.WriteString(" var _params = Params.create();\n") + for i, p := range q.Parameters { + fmt.Fprintf(b, " _params.put(%s, %s);\n", quoted("$"+p.Name), parameterValue(p, names[i])) + } + b.WriteString(" var _query = client.supplyResult(_session -> QueryReader.readFrom(\n") + fmt.Fprintf(b, " _session.createQuery(%s, TxMode.SERIALIZABLE_RW, _params))).join().getValue();\n", constant) + if q.Command == model.Exec { + return + } + b.WriteString(" if (_query.getResultSetCount() != 1) throw new IllegalStateException(\"Expected one result set\");\n var _rows = _query.getResultSet(0);\n") + emitRows(b, q, row, " ", true) +} + +func parameterValue(p model.Parameter, name string) string { + s, _, _ := typeInfo(p.Type) + value := "PrimitiveValue.new" + s.sdk + "(" + name + ")" + if p.Type.IsOptional() { + o := "OptionalType.of(PrimitiveType." + s.sdk + ")" + value = name + " == null ? " + o + ".emptyValue() : " + o + ".newValue(" + value + ")" + } + return value +} + +func emitJDBC(b *strings.Builder, q model.AnalyzedQuery, names []string, constant, row, ret, runtime string) { + indent := " " + connection := "client" + if runtime == "spring" { + callbackRet := ret + if ret == "void" { + callbackRet = "Void" + } else { + b.WriteString(indent + "return ") + } + if ret == "void" { + b.WriteString(indent) + } + fmt.Fprintf(b, "client.execute((org.springframework.jdbc.core.ConnectionCallback<%s>) _connection -> {\n", callbackRet) + connection = "_connection" + indent += " " + } else if runtime == "hibernate" { + b.WriteString(indent) + if ret != "void" { + b.WriteString("return ") + } + b.WriteString("client.doReturningWork(_connection -> {\n") + connection = "_connection" + indent += " " + } + fmt.Fprintf(b, "%stry (var _prepared = %s.prepareStatement(%s)) {\n", indent, connection, constant) + indent += " " + if len(q.Parameters) > 0 { + b.WriteString(indent + "var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class);\n") + } + for i, p := range q.Parameters { + // YdbPreparedStatement adds '$' to a parameter name itself. Passing an + // SDK Value preserves inferred YQL types even when SQL omits DECLARE. + fmt.Fprintf(b, "%s_statement.setObject(%s, %s);\n", indent, quoted(p.Name), parameterValue(p, names[i])) + } + if q.Command == model.Exec { + b.WriteString(indent + "_prepared.execute();\n") + if runtime != "jdbc" { + b.WriteString(indent + "return null;\n") + } + } else { + b.WriteString(indent + "try (var _rows = _prepared.executeQuery()) {\n") + emitRows(b, q, row, indent+" ", false) + b.WriteString(indent + "}\n") + } + indent = strings.TrimSuffix(indent, " ") + b.WriteString(indent + "}\n") + if runtime != "jdbc" { + b.WriteString(" });\n") + } +} + +func emitRows(b *strings.Builder, q model.AnalyzedQuery, row, indent string, native bool) { + if q.Command == model.One { + b.WriteString(indent + "if (!_rows.next()) return java.util.Optional.empty();\n") + } else { + fmt.Fprintf(b, "%svar _items = new java.util.ArrayList<%s>();\n%swhile (_rows.next()) {\n", indent, row, indent) + indent += " " + } + values := []string{} + for i, c := range q.ResultSets[0].Columns { + s, typ, _ := typeInfo(c.Type) + n := fmt.Sprintf("_value%d", i) + if native { + reader := fmt.Sprintf("_rows.getColumn(%d)", i) + if c.Type.IsOptional() { + fmt.Fprintf(b, "%s%s %s = %s.isOptionalItemPresent() ? %s.getOptionalItem().get%s() : null;\n", indent, typ, n, reader, reader, s.sdk) + } else { + fmt.Fprintf(b, "%s%s %s = %s.get%s();\n", indent, typ, n, reader, s.sdk) + } + } else { + fmt.Fprintf(b, "%s%s %s = _rows.get%s(%d);\n", indent, typ, n, s.jdbc, i+1) + if c.Type.IsOptional() { + fmt.Fprintf(b, "%sif (_rows.wasNull()) %s = null;\n", indent, n) + } + } + values = append(values, n) + } + newRow := "new " + row + "(" + strings.Join(values, ", ") + ")" + if q.Command == model.One { + fmt.Fprintf(b, "%sreturn java.util.Optional.of(%s);\n", indent, newRow) + } else { + fmt.Fprintf(b, "%s_items.add(%s);\n", indent, newRow) + indent = strings.TrimSuffix(indent, " ") + b.WriteString(indent + "}\n" + indent + "return _items;\n") + } +} diff --git a/internal/codegen/java/generator_test.go b/internal/codegen/java/generator_test.go new file mode 100644 index 0000000..892075f --- /dev/null +++ b/internal/codegen/java/generator_test.go @@ -0,0 +1,356 @@ +package java + +import ( + "encoding/base64" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +func TestSQLLiteralRoundTripsThroughJava17(t *testing.T) { + // The query has no parameters so the JDBC surface is entirely JDK types. This + // makes the check a real javac/java 17 literal test without an SDK dependency. + var c0 strings.Builder + for r := rune(0); r < 32; r++ { + c0.WriteRune(r) + } + c0.WriteRune(0x7f) + cases := []struct { + name string + sql string + }{ + {"empty", ""}, + {"ordinary", "SELECT 1;"}, + {"leading_lf", "\nSELECT 1;"}, + {"trailing_lf", "SELECT 1;\n"}, + {"blank_lines", "\n\nSELECT 1;\n\n"}, + {"spaces_tabs", " SELECT\t1; \n\t \n"}, + {"crlf", "SELECT 1;\r\n\r\nSELECT 2;\r\n"}, + {"quotes", "SELECT '\"', '\"\"\"', '''';"}, + {"literal_unicode_escape", `SELECT '\u000A', '\\u000A';`}, + {"trailing_backslash", "SELECT 'x';\\"}, + {"unicode_bom", "\ufeffSELECT 'Автор 中文 🚀 e\u0301 \u200d \u2028 \u2029';"}, + {"nul_and_all_c0", "SELECT '" + c0.String() + "';"}, + } + queries := make([]model.AnalyzedQuery, len(cases)) + for i, tc := range cases { + queries[i] = model.AnalyzedQuery{Name: fmt.Sprintf("Case%02d", i), Command: model.Exec, SQL: tc.sql} + } + files, err := Generate(&model.AnalysisResult{Queries: queries}, Options{Package: "literal", Runtime: "jdbc"}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + for _, f := range files { + if err := os.WriteFile(filepath.Join(dir, f.Name), f.Content, 0600); err != nil { + t.Fatal(err) + } + } + var program strings.Builder + program.WriteString("package literal;\nimport java.lang.reflect.*; import java.nio.charset.StandardCharsets; import java.util.Base64;\npublic final class Main {\n") + program.WriteString(" private static void check(String field, String expected) throws Exception { Field f=Queries.class.getDeclaredField(field); f.setAccessible(true); String actual=(String)f.get(null); if (!Base64.getEncoder().encodeToString(actual.getBytes(StandardCharsets.UTF_8)).equals(expected)) throw new AssertionError(field+\" changed: \"+actual); }\n") + program.WriteString(" public static void main(String[] args) throws Exception {\n") + for i, tc := range cases { + fmt.Fprintf(&program, " check(\"case%02dSql\", \"%s\");\n", i, base64.StdEncoding.EncodeToString([]byte(tc.sql))) + } + program.WriteString(" }\n}\n") + if err := os.WriteFile(filepath.Join(dir, "Main.java"), []byte(program.String()), 0600); err != nil { + t.Fatal(err) + } + classes := filepath.Join(dir, "classes") + compile := exec.Command("javac", "--release", "17", "-d", classes, "Queries.java", "Main.java") + compile.Dir = dir + if out, err := compile.CombinedOutput(); err != nil { + t.Fatalf("generated Java 17 source does not compile: %v\n%s\n%s", err, out, files[len(files)-1].Content) + } + run := exec.Command("java", "-cp", classes, "literal.Main") + run.Dir = dir + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("generated Java SQL literal changed at runtime: %v\n%s", err, out) + } +} + +func TestGenerateRejectsInvalidContracts(t *testing.T) { + utf8 := model.Type{Kind: "Utf8"} + for _, tc := range []struct { + name string + in *model.AnalysisResult + opts Options + want string + }{ + {"nil", nil, Options{}, "nil analysis result"}, + {"diagnostics", &model.AnalysisResult{Diagnostics: []model.Diagnostic{{Message: "bad"}}}, Options{}, "analysis diagnostics"}, + {"package_keyword", &model.AnalysisResult{}, Options{Package: "bad.class"}, "invalid Java package"}, + {"package_empty_segment", &model.AnalysisResult{}, Options{Package: "bad..pkg"}, "invalid Java package"}, + {"package_java_namespace", &model.AnalysisResult{}, Options{Package: "java.sqlc"}, "java packages are reserved"}, + {"runtime", &model.AnalysisResult{}, Options{Runtime: "unknown"}, "unsupported Java runtime"}, + {"unsupported_parameter", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, Parameters: []model.Parameter{{Name: "p", Type: model.Type{Kind: "Json"}}}}}}, Options{}, "unsupported Java type"}, + {"unsupported_result", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.One, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "value", Type: model.Type{Kind: "List"}}}}}}}}, Options{}, "unsupported Java type"}, + {"execrows", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.ExecRows}}}, Options{}, "does not support"}, + {"one_no_results", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.One}}}, Options{}, "requires one nonempty result set"}, + {"many_two_results", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Many, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "value", Type: utf8}}}, {Columns: []model.Column{{Name: "other", Type: utf8}}}}}}}, Options{}, "requires one nonempty result set"}, + {"method_collision", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Get_User", Command: model.Exec}, {Name: "getUser", Command: model.Exec}}}, Options{}, "method name collision"}, + {"parameter_collision", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, Parameters: []model.Parameter{{Name: "a-b", Type: utf8}, {Name: "a_b", Type: utf8}}}}}, Options{}, "parameter name collision"}, + {"field_collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "items", Columns: []model.Column{{Name: "a-b", Type: utf8}, {Name: "a_b", Type: utf8}}}}}}, Options{}, "field name collision"}, + {"record_collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "get_author_row", Columns: []model.Column{{Name: "id", Type: utf8}}}}}, Queries: []model.AnalyzedQuery{{Name: "get_author", Command: model.One, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "id", Type: utf8}}}}}}}, Options{}, "type name collision"}, + {"invalid_utf8", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, SQL: string([]byte{0xff})}}}, Options{}, "must be valid UTF-8"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := Generate(tc.in, tc.opts) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Generate() error = %v, want %q", err, tc.want) + } + }) + } +} + +// This opt-in integration test compiles one all-scalar generated query against +// the exact dependency pins in each authors Maven profile. It intentionally +// uses the real provider APIs, not local stubs. +func TestAllSupportedScalarsCompileAgainstAuthorsMavenProfiles(t *testing.T) { + maven := os.Getenv("SQLC_YDB_TEST_MAVEN") + if maven == "" { + t.Skip("set SQLC_YDB_TEST_MAVEN to compile against the authors Maven profiles") + } + types := []model.Type{ + {Kind: "Bool"}, {Kind: "Int8"}, {Kind: "Uint8"}, {Kind: "Int16"}, {Kind: "Uint16"}, + {Kind: "Int32"}, {Kind: "Uint32"}, {Kind: "Int64"}, {Kind: "Uint64"}, {Kind: "Float"}, + {Kind: "Double"}, {Kind: "Utf8"}, {Kind: "String"}, + } + var parameters []model.Parameter + var columns []model.Column + for _, typ := range types { + name := strings.ToLower(typ.Kind) + parameters = append(parameters, model.Parameter{Name: name, Type: typ}, model.Parameter{Name: "optional_" + name, Type: model.Optional(typ)}) + columns = append(columns, model.Column{Name: name, Type: typ}, model.Column{Name: "optional_" + name, Type: model.Optional(typ)}) + } + repoRoot, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + parentPom, err := os.ReadFile(filepath.Join(repoRoot, "examples", "authors", "java", "pom.xml")) + if err != nil { + t.Fatal(err) + } + profiles := []struct{ runtime, module, pkg string }{ + {"ydb", "native", "synthetic.nativeapi"}, + {"jdbc", "jdbc", "synthetic.jdbc"}, + {"spring", "spring", "synthetic.spring"}, + {"hibernate", "hibernate", "synthetic.hibernate"}, + } + for _, profile := range profiles { + t.Run(profile.runtime, func(t *testing.T) { + files, err := Generate(&model.AnalysisResult{Queries: []model.AnalyzedQuery{{ + Name: "AllScalars", Command: model.One, SQL: "SELECT 1;", Parameters: parameters, ResultSets: []model.ResultSet{{Columns: columns}}, + }}}, Options{Package: profile.pkg, Runtime: profile.runtime}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "pom.xml"), parentPom, 0600); err != nil { + t.Fatal(err) + } + modulePom, err := os.ReadFile(filepath.Join(repoRoot, "examples", "authors", "java", profile.module, "pom.xml")) + if err != nil { + t.Fatal(err) + } + moduleDir := filepath.Join(dir, profile.module) + if err := os.MkdirAll(filepath.Join(moduleDir, "src", "main", "java", filepath.FromSlash(strings.ReplaceAll(profile.pkg, ".", "/"))), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(moduleDir, "pom.xml"), modulePom, 0600); err != nil { + t.Fatal(err) + } + for _, file := range files { + if err := os.WriteFile(filepath.Join(moduleDir, "src", "main", "java", filepath.FromSlash(strings.ReplaceAll(profile.pkg, ".", "/")), file.Name), file.Content, 0600); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command(maven, "-q", "-DskipTests", "compile") + cmd.Dir = moduleDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("generated %s all-scalar API does not compile against authors Maven pins: %v\n%s", profile.runtime, err, out) + } + }) + } +} + +// This test executes generated JDBC code without a YDB server. Its proxy only +// supplies the JDBC wrapper surface; parameter binding is delegated to the +// published driver's PreparedQuery, which verifies the driver's real name +// normalization and Value type checks. +func TestGeneratedJDBCUsesTypedDriverValuesAndGuardsUnsignedRanges(t *testing.T) { + maven := os.Getenv("SQLC_YDB_TEST_MAVEN") + if maven == "" { + t.Skip("set SQLC_YDB_TEST_MAVEN to execute the published JDBC binding regression") + } + queries := []model.AnalyzedQuery{ + { + Name: "Bind", Command: model.Exec, SQL: "SELECT 1;", + Parameters: []model.Parameter{ + {Name: "author_id", Type: model.Type{Kind: "Uint64"}}, + {Name: "maybe_id", Type: model.Optional(model.Type{Kind: "Uint16"})}, + {Name: "title", Type: model.Type{Kind: "Utf8"}}, + {Name: "payload", Type: model.Type{Kind: "String"}}, + }, + }, + {Name: "Bad8", Command: model.Exec, SQL: "SELECT 1;", Parameters: []model.Parameter{{Name: "value", Type: model.Type{Kind: "Uint8"}}}}, + {Name: "Bad16", Command: model.Exec, SQL: "SELECT 1;", Parameters: []model.Parameter{{Name: "value", Type: model.Optional(model.Type{Kind: "Uint16"})}}}, + {Name: "Bad32", Command: model.Exec, SQL: "SELECT 1;", Parameters: []model.Parameter{{Name: "value", Type: model.Type{Kind: "Uint32"}}}}, + } + files, err := Generate(&model.AnalysisResult{Queries: queries}, Options{Package: "synthetic.jdbc", Runtime: "jdbc"}) + if err != nil { + t.Fatal(err) + } + repoRoot, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + parentPom, err := os.ReadFile(filepath.Join(repoRoot, "examples", "authors", "java", "pom.xml")) + if err != nil { + t.Fatal(err) + } + jdbcPom, err := os.ReadFile(filepath.Join(repoRoot, "examples", "authors", "java", "jdbc", "pom.xml")) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "pom.xml"), parentPom, 0600); err != nil { + t.Fatal(err) + } + moduleDir := filepath.Join(dir, "jdbc") + packageDir := filepath.Join(moduleDir, "src", "main", "java", "synthetic", "jdbc") + if err := os.MkdirAll(packageDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(moduleDir, "pom.xml"), jdbcPom, 0600); err != nil { + t.Fatal(err) + } + for _, file := range files { + if err := os.WriteFile(filepath.Join(packageDir, file.Name), file.Content, 0600); err != nil { + t.Fatal(err) + } + } + const program = `package synthetic.jdbc; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import tech.ydb.jdbc.YdbPreparedStatement; +import tech.ydb.jdbc.common.YdbTypes; +import tech.ydb.jdbc.query.QueryKey; +import tech.ydb.jdbc.query.YdbQuery; +import tech.ydb.jdbc.query.params.PreparedQuery; +import tech.ydb.jdbc.settings.YdbQueryProperties; +import tech.ydb.table.query.Params; +import tech.ydb.table.values.DecimalType; +import tech.ydb.table.values.OptionalType; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.Type; + +public final class Main { + private Main() { } + + public static void main(String[] args) throws Exception { + Queries guarded = new Queries(refusingConnection()); + expectRange(() -> guarded.bad8(-1)); + expectRange(() -> guarded.bad8(256)); + expectRange(() -> guarded.bad16(-1)); + expectRange(() -> guarded.bad16(65536)); + expectRange(() -> guarded.bad32(-1L)); + expectRange(() -> guarded.bad32(4294967296L)); + + YdbTypes types = new YdbTypes(false, DecimalType.getDefault()); + YdbQuery query = YdbQuery.parseQuery(new QueryKey("SELECT 1;"), new YdbQueryProperties(new Properties()), types); + Map declared = new HashMap<>(); + declared.put("$author_id", PrimitiveType.Uint64); + declared.put("$maybe_id", OptionalType.of(PrimitiveType.Uint16)); + declared.put("$title", PrimitiveType.Text); + declared.put("$payload", PrimitiveType.Bytes); + PreparedQuery bound = new PreparedQuery(types, query, declared); + new Queries(bindingConnection(bound)).bind(-1L, null, "typed text", new byte[] { 0, 1, (byte) 255 }); + + Params values = bound.getCurrentParams(); + check(values.values().size() == 4, "wrong parameter count"); + check(PrimitiveValue.newUint64(-1L).equals(values.values().get("$author_id")), "Uint64 lost its type or name"); + check(OptionalType.of(PrimitiveType.Uint16).emptyValue().equals(values.values().get("$maybe_id")), "optional null lost its declared type"); + check(PrimitiveValue.newText("typed text").equals(values.values().get("$title")), "Utf8 lost its type"); + check(PrimitiveValue.newBytes(new byte[] { 0, 1, (byte) 255 }).equals(values.values().get("$payload")), "String lost its binary type"); + } + + private static Connection refusingConnection() { + return (Connection) Proxy.newProxyInstance(Main.class.getClassLoader(), new Class[] { Connection.class }, + (proxy, method, args) -> { throw new AssertionError("range guard reached Connection." + method.getName()); }); + } + + private static Connection bindingConnection(PreparedQuery query) { + YdbPreparedStatement named = (YdbPreparedStatement) Proxy.newProxyInstance( + Main.class.getClassLoader(), new Class[] { YdbPreparedStatement.class }, new InvocationHandler() { + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (method.getName().equals("setObject") && args != null && args.length == 2 && args[0] instanceof String) { + query.setParam((String) args[0], args[1], Types.JAVA_OBJECT); + return null; + } + if (method.getName().equals("close")) return null; + throw new AssertionError("unexpected YdbPreparedStatement." + method.getName()); + } + }); + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + Main.class.getClassLoader(), new Class[] { PreparedStatement.class }, (proxy, method, args) -> { + if (method.getName().equals("unwrap") && args != null && args.length == 1 && args[0] == YdbPreparedStatement.class) return named; + if (method.getName().equals("isWrapperFor")) return args != null && args.length == 1 && args[0] == YdbPreparedStatement.class; + if (method.getName().equals("execute")) return false; + if (method.getName().equals("close")) return null; + throw new AssertionError("unexpected PreparedStatement." + method.getName()); + }); + return (Connection) Proxy.newProxyInstance(Main.class.getClassLoader(), new Class[] { Connection.class }, (proxy, method, args) -> { + if (method.getName().equals("prepareStatement")) return statement; + if (method.getName().equals("close")) return null; + throw new AssertionError("unexpected Connection." + method.getName()); + }); + } + + private static void expectRange(ThrowingRun run) throws Exception { + try { run.run(); } catch (IllegalArgumentException expected) { return; } + throw new AssertionError("expected IllegalArgumentException"); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } + + @FunctionalInterface private interface ThrowingRun { void run() throws Exception; } +} +` + if err := os.WriteFile(filepath.Join(packageDir, "Main.java"), []byte(program), 0600); err != nil { + t.Fatal(err) + } + classpathFile := filepath.Join(moduleDir, "classpath") + cmd := exec.Command(maven, "-q", "-DskipTests", "compile", "dependency:build-classpath", "-Dmdep.outputFile="+classpathFile) + cmd.Dir = moduleDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("generated JDBC binding fixture did not compile: %v\n%s", err, out) + } + classpath, err := os.ReadFile(classpathFile) + if err != nil { + t.Fatal(err) + } + run := exec.Command("java", "-cp", filepath.Join(moduleDir, "target", "classes")+string(os.PathListSeparator)+strings.TrimSpace(string(classpath)), "synthetic.jdbc.Main") + run.Dir = moduleDir + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("generated JDBC binding fixture failed against the published driver: %v\n%s", err, out) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 58d71b4..91b2d62 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,9 +51,29 @@ type Python struct { EmitAsyncQuerier bool `yaml:"emit_async_querier"` } +type CPP struct { + Namespace string `yaml:"namespace"` + Out string `yaml:"out"` + Runtime string `yaml:"runtime"` +} + +type CSharp struct { + Namespace string `yaml:"namespace"` + Out string `yaml:"out"` +} + +type Java struct { + Package string `yaml:"package"` + Out string `yaml:"out"` + Runtime string `yaml:"runtime"` +} + type Gen struct { Go *Go `yaml:"go"` Python *Python `yaml:"python"` + CPP *CPP `yaml:"cpp"` + CSharp *CSharp `yaml:"csharp"` + Java *Java `yaml:"java"` } type SQL struct { @@ -207,10 +227,48 @@ func Parse(data []byte) (*Config, error) { return nil, errors.New("Python requires emit_sync_querier or emit_async_querier") } } + if g := s.Gen.CPP; g != nil { + if g.Out == "" { + return nil, fmt.Errorf("sql[%d].gen.cpp.out is required", i) + } + if g.Namespace == "" { + g.Namespace = "db" + } + if g.Runtime == "" || g.Runtime == "native" { + g.Runtime = "ydb" + } + if g.Runtime != "ydb" && g.Runtime != "userver" { + return nil, fmt.Errorf("sql[%d]: unsupported C++ runtime %q", i, g.Runtime) + } + } + if g := s.Gen.CSharp; g != nil { + if g.Out == "" { + return nil, fmt.Errorf("sql[%d].gen.csharp.out is required", i) + } + if g.Namespace == "" { + g.Namespace = "Db" + } + } + if g := s.Gen.Java; g != nil { + if g.Out == "" { + return nil, fmt.Errorf("sql[%d].gen.java.out is required", i) + } + if g.Package == "" { + g.Package = "db" + } + if g.Runtime == "" || g.Runtime == "native" { + g.Runtime = "ydb" + } + switch g.Runtime { + case "ydb", "jdbc", "spring", "hibernate": + default: + return nil, fmt.Errorf("sql[%d]: unsupported Java runtime %q", i, g.Runtime) + } + } } return &c, nil } func pluginError() error { - return errors.New("external plugins and codegen are not supported: migrate to sql[].gen.go or sql[].gen.python; generators are built into sqlc-ydb") + return errors.New("external plugins and codegen are not supported: migrate to built-in sql[].gen.go, python, cpp, csharp or java generators") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d627ca0..41f0734 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -49,3 +49,37 @@ func TestRejectUnsupportedConfiguration(t *testing.T) { }) } } + +func TestAdditionalBuiltinTargets(t *testing.T) { + base := "version: '2'\nsql:\n- engine: ydb\n schema: s.sql\n queries: q.sql\n gen:\n" + c, err := Parse([]byte(base + " cpp:\n out: cpp\n csharp:\n out: cs\n java:\n out: java\n")) + if err != nil { + t.Fatal(err) + } + g := c.SQL[0].Gen + if g.CPP.Namespace != "db" || g.CPP.Runtime != "ydb" || g.CSharp.Namespace != "Db" || g.Java.Package != "db" || g.Java.Runtime != "ydb" { + t.Fatalf("unexpected defaults: %+v %+v %+v", g.CPP, g.CSharp, g.Java) + } + for _, runtime := range []string{"native", "ydb", "jdbc", "spring", "hibernate"} { + if _, err := Parse([]byte(base + " java:\n out: java\n runtime: " + runtime + "\n")); err != nil { + t.Fatalf("Java %s: %v", runtime, err) + } + } + for _, runtime := range []string{"native", "ydb", "userver"} { + if _, err := Parse([]byte(base + " cpp:\n out: cpp\n runtime: " + runtime + "\n")); err != nil { + t.Fatalf("C++ %s: %v", runtime, err) + } + } + for _, options := range []string{ + " cpp:\n namespace: db\n", + " csharp:\n namespace: Db\n", + " java:\n package: db\n", + " cpp:\n out: cpp\n runtime: imaginary\n", + " java:\n out: java\n runtime: imaginary\n", + " csharp:\n out: cs\n runtime: native\n", + } { + if _, err := Parse([]byte(base + options)); err == nil { + t.Errorf("expected invalid options to fail: %s", options) + } + } +} diff --git a/internal/endtoend/golden_test.go b/internal/endtoend/golden_test.go index 75308c7..c621079 100644 --- a/internal/endtoend/golden_test.go +++ b/internal/endtoend/golden_test.go @@ -15,6 +15,8 @@ import ( var update = flag.Bool("update", false, "update end-to-end expected output") +var outputRoots = []string{"db", "py", "cpp", "cs", "java"} + func TestGolden(t *testing.T) { fixtures, err := os.ReadDir("testdata") if err != nil { @@ -102,7 +104,7 @@ func updateExpected(t *testing.T, fixture, dir string) { if err := os.RemoveAll(expected); err != nil { t.Fatal(err) } - for _, root := range []string{"db", "py"} { + for _, root := range outputRoots { from := filepath.Join(dir, root) if _, err := os.Stat(from); os.IsNotExist(err) { continue @@ -169,7 +171,7 @@ func generated(t *testing.T, root string, want map[string][]byte) map[string][]b } got[rel] = data } - for _, d := range []string{"db", "py"} { + for _, d := range outputRoots { base := filepath.Join(root, d) if _, err := os.Stat(base); os.IsNotExist(err) { continue diff --git a/internal/endtoend/testdata/authors/expected/cpp/native/models.hpp b/internal/endtoend/testdata/authors/expected/cpp/native/models.hpp new file mode 100644 index 0000000..b59000f --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/native/models.hpp @@ -0,0 +1,16 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include +#include +#include + +namespace authors::native { + +struct GetAuthorRow final { + std::uint64_t id; + std::string name; + std::optional bio; +}; + +} // namespace authors::native diff --git a/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp b/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp new file mode 100644 index 0000000..9c385ed --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp @@ -0,0 +1,48 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#include "queries.hpp" + +#include +#include + +namespace authors::native { +namespace { + +const std::string kGetAuthorSql = R"sqlc(-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;)sqlc"; + +} // namespace + +std::optional Queries::GetAuthor(std::uint64_t author_id) const { + std::optional sqlc_result_set; + const auto sqlc_status = this->client_.RetryQuerySync([&](NYdb::NQuery::TSession sqlc_session) -> NYdb::TStatus { + auto sqlc_params = NYdb::TParamsBuilder() + .AddParam("$author_id").Uint64(author_id).Build() + .Build(); + auto sqlc_result = sqlc_session.ExecuteQuery( + kGetAuthorSql, + NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), + sqlc_params + ).GetValueSync(); + if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) { + sqlc_result_set = sqlc_result.GetResultSet(0); + } + return sqlc_result; + }); + NYdb::ThrowOnError(sqlc_status); + if (!sqlc_result_set) { + throw std::runtime_error("GetAuthor: successful query returned no result set"); + } + NYdb::TResultSetParser sqlc_parser(*sqlc_result_set); + if (!sqlc_parser.TryNextRow()) { + return std::nullopt; + } + GetAuthorRow sqlc_row{ + sqlc_parser.ColumnParser("id").GetUint64(), + sqlc_parser.ColumnParser("name").GetUtf8(), + sqlc_parser.ColumnParser("bio").GetOptionalUtf8(), + }; + return sqlc_row; +} + +} // namespace authors::native diff --git a/internal/endtoend/testdata/authors/expected/cpp/native/queries.hpp b/internal/endtoend/testdata/authors/expected/cpp/native/queries.hpp new file mode 100644 index 0000000..adfc6bc --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/native/queries.hpp @@ -0,0 +1,25 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include "models.hpp" + +#include +#include +#include +#include + +#include + +namespace authors::native { + +class Queries final { +public: + explicit Queries(NYdb::NQuery::TQueryClient& client) noexcept : client_(client) {} + + std::optional GetAuthor(std::uint64_t author_id) const; + +private: + NYdb::NQuery::TQueryClient& client_; +}; + +} // namespace authors::native diff --git a/internal/endtoend/testdata/authors/expected/cpp/userver/models.hpp b/internal/endtoend/testdata/authors/expected/cpp/userver/models.hpp new file mode 100644 index 0000000..c69d64f --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/userver/models.hpp @@ -0,0 +1,18 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include +#include +#include + +#include + +namespace authors::userver { + +struct GetAuthorRow final { + std::uint64_t id; + ::userver::ydb::Utf8 name; + std::optional<::userver::ydb::Utf8> bio; +}; + +} // namespace authors::userver diff --git a/internal/endtoend/testdata/authors/expected/cpp/userver/queries.cpp b/internal/endtoend/testdata/authors/expected/cpp/userver/queries.cpp new file mode 100644 index 0000000..dad9727 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/userver/queries.cpp @@ -0,0 +1,34 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#include "queries.hpp" + +#include +#include + +namespace authors::userver { +namespace { + +const ::userver::ydb::Query kGetAuthorQuery{ + R"sqlc(-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;)sqlc", + ::userver::ydb::Query::NameLiteral{"GetAuthor"}, + ::userver::ydb::Query::LogMode::kNameOnly, +}; + +} // namespace + +std::optional Queries::GetAuthor(std::uint64_t author_id) const { + auto sqlc_response = this->client_.ExecuteQuery(kGetAuthorQuery, "$author_id", author_id); + auto sqlc_cursor = sqlc_response.GetSingleCursor(); + if (sqlc_cursor.empty()) { + return std::nullopt; + } + auto sqlc_row = sqlc_cursor.GetFirstRow(); + return GetAuthorRow{ + sqlc_row.Get("id"), + sqlc_row.Get<::userver::ydb::Utf8>("name"), + sqlc_row.Get>("bio"), + }; +} + +} // namespace authors::userver diff --git a/internal/endtoend/testdata/authors/expected/cpp/userver/queries.hpp b/internal/endtoend/testdata/authors/expected/cpp/userver/queries.hpp new file mode 100644 index 0000000..2a7bc6a --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cpp/userver/queries.hpp @@ -0,0 +1,25 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#pragma once + +#include "models.hpp" + +#include +#include +#include +#include + +#include + +namespace authors::userver { + +class Queries final { +public: + explicit Queries(::userver::ydb::TableClient& client) noexcept : client_(client) {} + + std::optional GetAuthor(std::uint64_t author_id) const; + +private: + ::userver::ydb::TableClient& client_; +}; + +} // namespace authors::userver diff --git a/internal/endtoend/testdata/authors/expected/cs/Models.cs b/internal/endtoend/testdata/authors/expected/cs/Models.cs new file mode 100644 index 0000000..4cca783 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cs/Models.cs @@ -0,0 +1,16 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#nullable enable +using System; + +namespace Authors.AdoNet; +public sealed record Authors( + ulong ID, + string Name, + string? Bio +); + +public sealed record GetAuthorRow( + ulong ID, + string Name, + string? Bio +); diff --git a/internal/endtoend/testdata/authors/expected/cs/Queries.cs b/internal/endtoend/testdata/authors/expected/cs/Queries.cs new file mode 100644 index 0000000..2fd1f9b --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/cs/Queries.cs @@ -0,0 +1,49 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +#nullable enable +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; +using Ydb.Sdk.Ado; + +namespace Authors.AdoNet; + + +public sealed class Queries +{ + private readonly YdbConnection _connection; + private readonly YdbTransaction? _transaction; + + public Queries(YdbConnection connection, YdbTransaction? transaction = null) + { + _connection = connection ?? throw new ArgumentNullException(nameof(connection)); + _transaction = transaction; + } + + public Queries WithTransaction(YdbTransaction transaction) => new(_connection, transaction ?? throw new ArgumentNullException(nameof(transaction))); + + private const string SqlGetAuthor = + "-- name: GetAuthor :one\n" + + "DECLARE $author_id AS Uint64;\n" + + "SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;"; + + public async Task GetAuthorAsync(ulong AuthorID, CancellationToken cancellationToken = default) + { + await using var command = new YdbCommand(SqlGetAuthor, _connection) { Transaction = _transaction }; + command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, AuthorID)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("query returned no rows"); + } + return GetAuthorRowFrom(reader); + } + + private static GetAuthorRow GetAuthorRowFrom(DbDataReader reader) => new( + reader.GetFieldValue(0), + reader.GetFieldValue(1), + reader.IsDBNull(2) ? null : reader.GetFieldValue(2) + ); +} diff --git a/internal/endtoend/testdata/authors/expected/java/hibernate/Authors.java b/internal/endtoend/testdata/authors/expected/java/hibernate/Authors.java new file mode 100644 index 0000000..75d3b66 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/hibernate/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record Authors(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/hibernate/GetAuthorRow.java b/internal/endtoend/testdata/authors/expected/java/hibernate/GetAuthorRow.java new file mode 100644 index 0000000..e9ce2d3 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/hibernate/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/hibernate/Queries.java b/internal/endtoend/testdata/authors/expected/java/hibernate/Queries.java new file mode 100644 index 0000000..c99ad98 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/hibernate/Queries.java @@ -0,0 +1,38 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.hibernate; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final org.hibernate.Session client; + + public Queries(org.hibernate.Session client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + return client.doReturningWork(_connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + }); + } +} diff --git a/internal/endtoend/testdata/authors/expected/java/jdbc/Authors.java b/internal/endtoend/testdata/authors/expected/java/jdbc/Authors.java new file mode 100644 index 0000000..8b7b7b8 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/jdbc/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record Authors(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/jdbc/GetAuthorRow.java b/internal/endtoend/testdata/authors/expected/java/jdbc/GetAuthorRow.java new file mode 100644 index 0000000..0d87a81 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/jdbc/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/jdbc/Queries.java b/internal/endtoend/testdata/authors/expected/java/jdbc/Queries.java new file mode 100644 index 0000000..0af30f3 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/jdbc/Queries.java @@ -0,0 +1,36 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.jdbc; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final java.sql.Connection client; + + public Queries(java.sql.Connection client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) throws java.sql.SQLException { + try (var _prepared = client.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + } +} diff --git a/internal/endtoend/testdata/authors/expected/java/native/Authors.java b/internal/endtoend/testdata/authors/expected/java/native/Authors.java new file mode 100644 index 0000000..4d2a874 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/native/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record Authors(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/native/GetAuthorRow.java b/internal/endtoend/testdata/authors/expected/java/native/GetAuthorRow.java new file mode 100644 index 0000000..2e50eae --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/native/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/native/Queries.java b/internal/endtoend/testdata/authors/expected/java/native/Queries.java new file mode 100644 index 0000000..0d1a854 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/native/Queries.java @@ -0,0 +1,39 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.nativeapi; + +import tech.ydb.query.tools.SessionRetryContext; +import tech.ydb.query.tools.QueryReader; +import tech.ydb.common.transaction.TxMode; +import tech.ydb.table.query.Params; +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final SessionRetryContext client; + + public Queries(SessionRetryContext client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + var _params = Params.create(); + _params.put("$author_id", PrimitiveValue.newUint64(authorId)); + var _query = client.supplyResult(_session -> QueryReader.readFrom( + _session.createQuery(getAuthorSql, TxMode.SERIALIZABLE_RW, _params))).join().getValue(); + if (_query.getResultSetCount() != 1) throw new IllegalStateException("Expected one result set"); + var _rows = _query.getResultSet(0); + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getColumn(0).getUint64(); + String _value1 = _rows.getColumn(1).getText(); + String _value2 = _rows.getColumn(2).isOptionalItemPresent() ? _rows.getColumn(2).getOptionalItem().getText() : null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } +} diff --git a/internal/endtoend/testdata/authors/expected/java/spring/Authors.java b/internal/endtoend/testdata/authors/expected/java/spring/Authors.java new file mode 100644 index 0000000..f74f054 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/spring/Authors.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record Authors(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/spring/GetAuthorRow.java b/internal/endtoend/testdata/authors/expected/java/spring/GetAuthorRow.java new file mode 100644 index 0000000..89eca40 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/spring/GetAuthorRow.java @@ -0,0 +1,4 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +public record GetAuthorRow(long id, String name, String bio) {} diff --git a/internal/endtoend/testdata/authors/expected/java/spring/Queries.java b/internal/endtoend/testdata/authors/expected/java/spring/Queries.java new file mode 100644 index 0000000..1922806 --- /dev/null +++ b/internal/endtoend/testdata/authors/expected/java/spring/Queries.java @@ -0,0 +1,38 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package authors.spring; + +import tech.ydb.table.values.PrimitiveValue; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.OptionalType; + +// The caller owns the injected client and its lifecycle. +public final class Queries { + private final org.springframework.jdbc.core.JdbcTemplate client; + + public Queries(org.springframework.jdbc.core.JdbcTemplate client) { + this.client = java.util.Objects.requireNonNull(client); + } + + private static final String getAuthorSql = """ +-- name: GetAuthor :one +DECLARE $author_id AS Uint64; +SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;\ +"""; + + public java.util.Optional getAuthor(long authorId) { + return client.execute((org.springframework.jdbc.core.ConnectionCallback>) _connection -> { + try (var _prepared = _connection.prepareStatement(getAuthorSql)) { + var _statement = _prepared.unwrap(tech.ydb.jdbc.YdbPreparedStatement.class); + _statement.setObject("author_id", PrimitiveValue.newUint64(authorId)); + try (var _rows = _prepared.executeQuery()) { + if (!_rows.next()) return java.util.Optional.empty(); + long _value0 = _rows.getLong(1); + String _value1 = _rows.getString(2); + String _value2 = _rows.getString(3); + if (_rows.wasNull()) _value2 = null; + return java.util.Optional.of(new GetAuthorRow(_value0, _value1, _value2)); + } + } + }); + } +} diff --git a/internal/endtoend/testdata/authors/sqlc.yaml b/internal/endtoend/testdata/authors/sqlc.yaml index 5681473..39d00d6 100644 --- a/internal/endtoend/testdata/authors/sqlc.yaml +++ b/internal/endtoend/testdata/authors/sqlc.yaml @@ -10,3 +10,42 @@ sql: python: out: py runtime: ydb + cpp: + namespace: authors::native + out: cpp/native + runtime: ydb + csharp: + namespace: Authors.AdoNet + out: cs + java: + package: authors.nativeapi + out: java/native + runtime: ydb + - engine: ydb + schema: schema.sql + queries: queries.sql + gen: + cpp: + namespace: authors::userver + out: cpp/userver + runtime: userver + java: + package: authors.jdbc + out: java/jdbc + runtime: jdbc + - engine: ydb + schema: schema.sql + queries: queries.sql + gen: + java: + package: authors.spring + out: java/spring + runtime: spring + - engine: ydb + schema: schema.sql + queries: queries.sql + gen: + java: + package: authors.hibernate + out: java/hibernate + runtime: hibernate From 71baad7712ed84b8a9e12a1435deb0c1dfbc48ed Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 18:26:05 +0300 Subject: [PATCH 2/7] fix: preserve C# optional parameter types and load C++ SDK once --- docs/cpp.md | 6 +- docs/csharp.md | 19 ++--- examples/authors/cpp/CMakeLists.txt | 5 +- examples/authors/cpp/Dockerfile | 14 ++-- examples/authors/csharp/adonet/Queries.cs | 3 +- internal/codegen/csharp/generator.go | 17 +++-- internal/codegen/csharp/generator_test.go | 75 +++++++++++++++++-- internal/codegen/java/generator.go | 2 +- internal/codegen/java/generator_test.go | 1 + .../testdata/authors/expected/cs/Queries.cs | 1 + 10 files changed, 109 insertions(+), 34 deletions(-) diff --git a/docs/cpp.md b/docs/cpp.md index 7f290c6..b685fe5 100644 --- a/docs/cpp.md +++ b/docs/cpp.md @@ -62,11 +62,13 @@ Generate all authors adapters from the repository root: go run ./cmd/sqlc-ydb generate -f examples/authors/sqlc.yaml ``` -The current YDB C++ SDK release is `v3.22.0`, which publishes Ubuntu 24.04 `libydb-cpp-dev` and `yandex-googleapis-api-common-protos` packages. Its CMake package installs below `/usr/share/yandex`. Native links the real `YDB-CPP-SDK::Driver`, `YDB-CPP-SDK::Params`, and `YDB-CPP-SDK::Query` targets; userver links `userver::ydb`. The pinned userver target also links `YDB-CPP-SDK::ydb-cpp-iam`, so the top-level CMake file requests the SDK `Iam` component before loading userver's installed targets. +The current YDB C++ SDK release is `v3.22.0`, which publishes Ubuntu 24.04 `libydb-cpp-dev` and `yandex-googleapis-api-common-protos` packages. Its CMake package installs below `/usr/share/yandex`. Native links the real `YDB-CPP-SDK::Driver`, `YDB-CPP-SDK::Params`, and `YDB-CPP-SDK::Query` targets; userver links `userver::ydb`. The pinned userver target also links `YDB-CPP-SDK::ydb-cpp-iam`. The official `ghcr.io/userver-framework/ubuntu-24.04-userver` image is built with `USERVER_FEATURE_YDB=1` and includes the YDB SDK packages. The compile environment is pinned in `examples/authors/cpp/Dockerfile` to `ghcr.io/userver-framework/ubuntu-24.04-userver@sha256:8b71ba0bdc5f79038d2e639cc7d8f669405db7377b851f7581b09c67183151e4`, which contains userver 3.2-rc and YDB C++ SDK 3.21.1. -That image's installed `userver-ydb-config.cmake` asks for the obsolete CMake package name `googleapis`, while its real installed SDK package exports `yandex-googleapis-api-common-protos::api-common-protos` from `yandex-googleapis-api-common-protosConfig.cmake`. The Dockerfile makes the exact dependency-name correction and verifies both the old line and the real package file before changing it. It does not add replacement headers, targets, or libraries. Build the small derived image and compile serially to stay within the 2 GB Docker VM: +That image's installed `userver-ydb-config.cmake` has two packaging defects. It asks for the obsolete CMake package name `googleapis`, while its real installed SDK package exports `yandex-googleapis-api-common-protos::api-common-protos` from `yandex-googleapis-api-common-protosConfig.cmake`. It also loads the SDK without components even though `userver::ydb` requires the IAM library. The SDK 3.21.1 package is not safe to load twice with different component lists because it recreates component aliases. + +The Dockerfile verifies and corrects both dependency lines. Its single SDK load requests `Driver`, `Params`, and `Query` for the native example plus `Iam` for `userver::ydb`. The top-level project therefore loads userver once and uses the real SDK targets that dependency exports. The workaround does not add replacement headers, targets, or libraries. Build the small derived image and compile serially to stay within the 2 GB Docker VM: ```bash docker build \ diff --git a/docs/csharp.md b/docs/csharp.md index 6c8ce23..cff5727 100644 --- a/docs/csharp.md +++ b/docs/csharp.md @@ -37,12 +37,13 @@ The supported scalar types are `Bool`, signed and unsigned integer types, `float`, `double`, `string`, `byte[]`, and `Guid`. Unsupported YQL types fail generation; there is no `object` or inferred-type fallback. -Parameters are constructed as `YdbParameter` with an explicit standard -`DbType`, which the YDB provider maps to its concrete YDB type. In particular, -`Uint64` always binds as `DbType.UInt64`, `Utf8` as `DbType.String`, and -`String` as `DbType.Binary`. An optional parameter uses the same explicit type -for a value and for `DBNull.Value`, which lets the provider create a correctly -typed YDB null. +Required parameters are constructed as `YdbParameter` with an explicit +standard `DbType`, which the provider maps to its concrete YDB type. In +particular, `Uint64` binds as `DbType.UInt64`, `Utf8` as `DbType.String`, and +`String` as `DbType.Binary`. Optional parameters use the SDK's typed +`YdbValue.MakeOptional*` factories. This preserves `Optional` for both a +present value and null: supplying a bare present CLR value would otherwise bind +as `T`, not `Optional`. ## SDK evidence and build target @@ -54,9 +55,9 @@ The API choice was checked against `ydb-platform/ydb` main at `Ydb.Sdk` the ADO.NET provider and demonstrates `YdbDataSource`, `YdbConnection`, and `YdbCommand`; the provider source exposes `YdbParameter(string, DbType, object?)` and -`YdbCommand.ExecuteReaderAsync(CancellationToken)`. `YdbParameter` emits a -typed null when its `DbType` is explicit, so `IsNullable` is not the -mechanism used for YQL null typing. +`YdbCommand.ExecuteReaderAsync(CancellationToken)`. `YdbValue.MakeOptional*` +serializes the optional wrapper directly; `IsNullable` and `DBNull.Value` do +not select the YQL optional type. The authors smoke project targets `net8.0` and pins `Ydb.Sdk` `0.33.3`. `Ydb.Sdk` belongs to generated-project dependencies, never to sqlc-ydb's Go diff --git a/examples/authors/cpp/CMakeLists.txt b/examples/authors/cpp/CMakeLists.txt index 8688b32..eeb3162 100644 --- a/examples/authors/cpp/CMakeLists.txt +++ b/examples/authors/cpp/CMakeLists.txt @@ -5,9 +5,8 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -# userver::ydb links the SDK IAM library in the pinned distribution, so load -# that component before userver imports its installed targets. -find_package(ydb-cpp-sdk REQUIRED COMPONENTS Driver Params Query Iam) +# The pinned image's corrected userver YDB package loads these SDK components +# once and exposes the SDK aliases used by the native example as well. find_package(userver REQUIRED COMPONENTS core ydb) add_subdirectory(native) diff --git a/examples/authors/cpp/Dockerfile b/examples/authors/cpp/Dockerfile index 415f04b..b9a4e30 100644 --- a/examples/authors/cpp/Dockerfile +++ b/examples/authors/cpp/Dockerfile @@ -1,18 +1,22 @@ FROM ghcr.io/userver-framework/ubuntu-24.04-userver@sha256:8b71ba0bdc5f79038d2e639cc7d8f669405db7377b851f7581b09c67183151e4 # This image's userver 3.2-rc YDB package asks CMake for the obsolete -# "googleapis" package name. The installed YDB SDK 3.21.1 package exports the -# real protobuf target through "yandex-googleapis-api-common-protos" instead. -# Fail if either package layout changes so this metadata-only correction cannot -# silently turn into a partial or stubbed SDK setup. +# "googleapis" package name and loads the non-idempotent YDB SDK package without +# the components required by its exported target. Correct both dependency lines +# and fail if the pinned package layout changes. This only fixes installed CMake +# metadata; all headers, targets, and libraries remain the packaged artifacts. RUN set -eux; \ config=/usr/lib/cmake/userver/userver-ydb-config.cmake; \ package=/usr/share/yandex/lib/cmake/yandex-googleapis-api-common-protos/yandex-googleapis-api-common-protosConfig.cmake; \ test -f "${config}"; \ test -f "${package}"; \ grep -Fq 'find_dependency(googleapis CONFIG)' "${config}"; \ + grep -Fq 'find_dependency(ydb-cpp-sdk CONFIG)' "${config}"; \ sed -i 's/find_dependency(googleapis CONFIG)/find_dependency(yandex-googleapis-api-common-protos CONFIG)/' "${config}"; \ + sed -i 's/find_dependency(ydb-cpp-sdk CONFIG)/find_dependency(ydb-cpp-sdk CONFIG COMPONENTS Driver Params Query Iam)/' "${config}"; \ ! grep -Fq 'find_dependency(googleapis CONFIG)' "${config}"; \ - grep -Fq 'find_dependency(yandex-googleapis-api-common-protos CONFIG)' "${config}" + ! grep -Fxq ' find_dependency(ydb-cpp-sdk CONFIG)' "${config}"; \ + grep -Fq 'find_dependency(yandex-googleapis-api-common-protos CONFIG)' "${config}"; \ + grep -Fq 'find_dependency(ydb-cpp-sdk CONFIG COMPONENTS Driver Params Query Iam)' "${config}" WORKDIR /workspace diff --git a/examples/authors/csharp/adonet/Queries.cs b/examples/authors/csharp/adonet/Queries.cs index 6a940cd..da05514 100644 --- a/examples/authors/csharp/adonet/Queries.cs +++ b/examples/authors/csharp/adonet/Queries.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Ydb.Sdk.Ado; +using Ydb.Sdk.Value; namespace Authors.AdoNet; @@ -103,7 +104,7 @@ public async Task UpsertAuthorAsync(UpsertAuthorParams args, CancellationToken c await using var command = new YdbCommand(SqlUpsertAuthor, _connection) { Transaction = _transaction }; command.Parameters.Add(new YdbParameter("$author_id", DbType.UInt64, args.AuthorID)); command.Parameters.Add(new YdbParameter("$author_name", DbType.String, args.AuthorName)); - command.Parameters.Add(new YdbParameter("$biography", DbType.String, (object?)args.Biography ?? DBNull.Value)); + command.Parameters.Add(new YdbParameter("$biography", YdbValue.MakeOptionalUtf8(args.Biography))); await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } diff --git a/internal/codegen/csharp/generator.go b/internal/codegen/csharp/generator.go index f78e1f7..bfc4bce 100644 --- a/internal/codegen/csharp/generator.go +++ b/internal/codegen/csharp/generator.go @@ -41,7 +41,7 @@ func validate(in *model.AnalysisResult) error { queryNames, methodNames := map[string]string{}, map[string]string{} // Queries is emitted by this generator, so no record may reuse its name. modelNames := map[string]string{"Queries": "generated query class"} - for _, n := range []string{"Guid", "Task", "CancellationToken", "List", "IReadOnlyList", "DbType", "DBNull", "YdbConnection", "YdbTransaction", "YdbCommand", "YdbParameter"} { + for _, n := range []string{"Guid", "Task", "CancellationToken", "List", "IReadOnlyList", "DbType", "DBNull", "DbDataReader", "ArgumentNullException", "InvalidOperationException", "YdbConnection", "YdbTransaction", "YdbCommand", "YdbParameter", "YdbValue"} { modelNames[n] = "framework type" } add := func(dst map[string]string, name, original, what string) error { @@ -229,7 +229,7 @@ func modelsHeader(o Options) string { return "// Code generated by sqlc-ydb. DO NOT EDIT.\n#nullable enable\nusing System;\n\nnamespace " + o.Namespace + ";" } func queriesHeader(o Options) string { - return "// Code generated by sqlc-ydb. DO NOT EDIT.\n#nullable enable\nusing System;\nusing System.Collections.Generic;\nusing System.Data;\nusing System.Data.Common;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing Ydb.Sdk.Ado;\n\nnamespace " + o.Namespace + ";\n\n" + return "// Code generated by sqlc-ydb. DO NOT EDIT.\n#nullable enable\nusing System;\nusing System.Collections.Generic;\nusing System.Data;\nusing System.Data.Common;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing Ydb.Sdk.Ado;\nusing Ydb.Sdk.Value;\n\nnamespace " + o.Namespace + ";\n\n" } func writeSQLConstant(b *bytes.Buffer, q model.AnalyzedQuery) { fmt.Fprintf(b, "\n private const string Sql%s =\n", csName(q.Name)) @@ -303,13 +303,14 @@ func parameterRef(q model.AnalyzedQuery, p model.Parameter) string { } func writeParameter(b *bytes.Buffer, q model.AnalyzedQuery, p model.Parameter) { v := parameterRef(q, p) - fmt.Fprintf(b, " command.Parameters.Add(new YdbParameter(%q, DbType.%s, %s));\n", "$"+p.Name, dbType(p.Type), nullableValue(p.Type, v)) -} -func nullableValue(t model.Type, v string) string { - if t.IsOptional() { - return "(object?)" + v + " ?? DBNull.Value" + if p.Type.IsOptional() { + fmt.Fprintf(b, " command.Parameters.Add(new YdbParameter(%q, YdbValue.MakeOptional%s(%s)));\n", "$"+p.Name, optionalFactory(p.Type), v) + return } - return v + fmt.Fprintf(b, " command.Parameters.Add(new YdbParameter(%q, DbType.%s, %s));\n", "$"+p.Name, dbType(p.Type), v) +} +func optionalFactory(t model.Type) string { + return csName(t.UnwrapOptional().Kind) } func dbType(t model.Type) string { if t.IsOptional() { diff --git a/internal/codegen/csharp/generator_test.go b/internal/codegen/csharp/generator_test.go index 2f6bb99..0fb62bb 100644 --- a/internal/codegen/csharp/generator_test.go +++ b/internal/codegen/csharp/generator_test.go @@ -47,7 +47,7 @@ func TestGenerateUsesConcreteModernYdbAdoSurface(t *testing.T) { for _, want := range []string{ "using Ydb.Sdk.Ado;", "private readonly YdbConnection _connection;", "private readonly YdbTransaction? _transaction;", "WithTransaction(YdbTransaction transaction)", "new YdbCommand(SqlGetAuthor, _connection) { Transaction = _transaction }", "new YdbParameter(\"$author_id\", DbType.UInt64, AuthorID)", - "new YdbParameter(\"$biography\", DbType.String, (object?)args.Biography ?? DBNull.Value)", "ExecuteReaderAsync(cancellationToken)", "ReadAsync(cancellationToken)", "reader.IsDBNull(2) ? null : reader.GetFieldValue(2)", + "using Ydb.Sdk.Value;", "new YdbParameter(\"$biography\", YdbValue.MakeOptionalUtf8(args.Biography))", "ExecuteReaderAsync(cancellationToken)", "ReadAsync(cancellationToken)", "reader.IsDBNull(2) ? null : reader.GetFieldValue(2)", } { if !strings.Contains(queries, want) { t.Errorf("Queries.cs missing %q:\n%s", want, queries) @@ -118,7 +118,7 @@ func TestGeneratedCodeBuildsAgainstPublishedSDK(t *testing.T) { } cmd := exec.Command(dotnet, "build", "--nologo") cmd.Dir = dir - cmd.Env = append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + cmd.Env = dotnetEnv(dir) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("generated C# does not build against Ydb.Sdk 0.33.3: %v\n%s", err, out) } @@ -165,14 +165,14 @@ func TestAllSupportedScalarsBuildAgainstPublishedSDK(t *testing.T) { } cmd := exec.Command(dotnet, "build", "--nologo") cmd.Dir = dir - cmd.Env = append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + cmd.Env = dotnetEnv(dir) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("all generated scalar bindings must compile against Ydb.Sdk 0.33.3: %v\n%s", err, out) } } func TestRejectsModelNamesThatShadowFrameworkTypes(t *testing.T) { - for _, table := range []string{"guid", "task", "cancellation_token"} { + for _, table := range []string{"guid", "task", "cancellation_token", "db_data_reader", "argument_null_exception", "invalid_operation_exception", "ydb_value"} { t.Run(table, func(t *testing.T) { _, err := Generate(&model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: table}}}}, Options{}) if err == nil || !strings.Contains(err.Error(), "model name collision") { @@ -211,7 +211,7 @@ func TestSQLLiteralRoundTripsThroughCSharpRuntime(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "Program.cs"), []byte(program), 0600); err != nil { t.Fatal(err) } - env := append(os.Environ(), "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet"), "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + env := dotnetEnv(dir) build := exec.Command(dotnet, "build", "--nologo") build.Dir, build.Env = dir, env if out, err := build.CombinedOutput(); err != nil { @@ -223,3 +223,68 @@ func TestSQLLiteralRoundTripsThroughCSharpRuntime(t *testing.T) { t.Fatalf("literal runtime: %v\n%s", err, out) } } + +func dotnetEnv(dir string) []string { + env := append([]string{}, os.Environ()...) + env = append(env, "DOTNET_CLI_HOME="+filepath.Join(dir, ".dotnet")) + if os.Getenv("NUGET_PACKAGES") == "" { + env = append(env, "NUGET_PACKAGES="+filepath.Join(dir, ".nuget")) + } + return env +} + +// This exercises the public 0.33.3 SDK value serializer, rather than relying +// on DbType/DBNull behavior. Generated Optional expressions use these +// values so both a present value and null retain Optional on wire. +func TestOptionalParametersSerializeAsTypedYdbValues(t *testing.T) { + dotnet := os.Getenv("SQLC_YDB_CSHARP_DOTNET") + if dotnet == "" { + t.Skip("set SQLC_YDB_CSHARP_DOTNET to run the published-SDK optional wire-type check") + } + _, queries := generated(t, authorsAnalysis()) + if !strings.Contains(queries, "YdbValue.MakeOptionalUtf8(args.Biography)") { + t.Fatalf("generated optional parameter did not use YdbValue: %s", queries) + } + dir := t.TempDir() + project := `Exenet8.0enabletrue` + if err := os.WriteFile(filepath.Join(dir, "wire.csproj"), []byte(project), 0600); err != nil { + t.Fatal(err) + } + program := `using System; +using Ydb.Sdk.Ado; +using Ydb.Sdk.Value; + +internal static class Program +{ + private static void Check(YdbValue value, Ydb.Type.Types.PrimitiveTypeId primitive, bool isNull) + { + var proto = value.GetProto(); + if (proto.Type.OptionalType?.Item.TypeId != primitive) throw new Exception("optional item type changed"); + if ((proto.Value.ValueCase == Ydb.Value.ValueOneofCase.NullFlagValue) != isNull) throw new Exception("optional presence changed"); + } + + private static int Main() + { + var present = new YdbParameter("$present", YdbValue.MakeOptionalUtf8("present")); + var absent = new YdbParameter("$absent", YdbValue.MakeOptionalUtf8(null)); + var bytes = new YdbParameter("$bytes", YdbValue.MakeOptionalString(new byte[] { 0, 255 })); + Check((YdbValue)present.Value!, Ydb.Type.Types.PrimitiveTypeId.Utf8, false); + Check((YdbValue)absent.Value!, Ydb.Type.Types.PrimitiveTypeId.Utf8, true); + Check((YdbValue)bytes.Value!, Ydb.Type.Types.PrimitiveTypeId.String, false); + return 0; + } +}` + if err := os.WriteFile(filepath.Join(dir, "Program.cs"), []byte(program), 0600); err != nil { + t.Fatal(err) + } + build := exec.Command(dotnet, "build", "--nologo") + build.Dir, build.Env = dir, dotnetEnv(dir) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("optional wire-type build: %v\n%s", err, out) + } + run := exec.Command(dotnet, "run", "--no-build", "--nologo") + run.Dir, run.Env = dir, dotnetEnv(dir) + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("optional wire-type runtime: %v\n%s", err, out) + } +} diff --git a/internal/codegen/java/generator.go b/internal/codegen/java/generator.go index 2466e4e..9312155 100644 --- a/internal/codegen/java/generator.go +++ b/internal/codegen/java/generator.go @@ -161,7 +161,7 @@ func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { header := "// Code generated by sqlc-ydb. DO NOT EDIT.\npackage " + o.Package + ";\n\n" files := []model.File{} types := map[string]bool{"Queries": true, "String": true, "Long": true, "Integer": true, "Short": true, "Byte": true, "Boolean": true, "Float": true, "Double": true} - for _, n := range []string{"SessionRetryContext", "QueryReader", "TxMode", "Params", "PrimitiveValue", "PrimitiveType", "OptionalType", "IllegalStateException"} { + for _, n := range []string{"SessionRetryContext", "QueryReader", "TxMode", "Params", "PrimitiveValue", "PrimitiveType", "OptionalType", "IllegalStateException", "IllegalArgumentException"} { types[n] = true } addRecord := func(n string, cols []model.Column) error { diff --git a/internal/codegen/java/generator_test.go b/internal/codegen/java/generator_test.go index 892075f..f646cb7 100644 --- a/internal/codegen/java/generator_test.go +++ b/internal/codegen/java/generator_test.go @@ -87,6 +87,7 @@ func TestGenerateRejectsInvalidContracts(t *testing.T) { {"diagnostics", &model.AnalysisResult{Diagnostics: []model.Diagnostic{{Message: "bad"}}}, Options{}, "analysis diagnostics"}, {"package_keyword", &model.AnalysisResult{}, Options{Package: "bad.class"}, "invalid Java package"}, {"package_empty_segment", &model.AnalysisResult{}, Options{Package: "bad..pkg"}, "invalid Java package"}, + {"framework_type_collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "illegal_argument_exception"}}}}, Options{}, "type name collision"}, {"package_java_namespace", &model.AnalysisResult{}, Options{Package: "java.sqlc"}, "java packages are reserved"}, {"runtime", &model.AnalysisResult{}, Options{Runtime: "unknown"}, "unsupported Java runtime"}, {"unsupported_parameter", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, Parameters: []model.Parameter{{Name: "p", Type: model.Type{Kind: "Json"}}}}}}, Options{}, "unsupported Java type"}, diff --git a/internal/endtoend/testdata/authors/expected/cs/Queries.cs b/internal/endtoend/testdata/authors/expected/cs/Queries.cs index 2fd1f9b..a3e60bf 100644 --- a/internal/endtoend/testdata/authors/expected/cs/Queries.cs +++ b/internal/endtoend/testdata/authors/expected/cs/Queries.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Ydb.Sdk.Ado; +using Ydb.Sdk.Value; namespace Authors.AdoNet; From 228f70f830a22414fb06d5fbf9e4b31aec7481e4 Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 18:33:08 +0300 Subject: [PATCH 3/7] fix: use C++ SDK status helpers and reject invalid C# record members --- .github/workflows/ci.yml | 2 +- docs/cpp.md | 27 ++++++----- docs/development.md | 4 +- examples/authors/cpp/native/main.cpp | 3 +- examples/authors/cpp/native/queries.cpp | 14 ++++-- .../test/java/authors/nativeapi/Smoke.java | 2 - internal/codegen/cpp/generator.go | 8 +++- internal/codegen/cpp/generator_test.go | 2 + internal/codegen/csharp/generator.go | 25 ++++++++-- internal/codegen/csharp/generator_test.go | 47 +++++++++++++++++++ .../authors/expected/cpp/native/queries.cpp | 6 ++- 11 files changed, 110 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c255f8..2ee232c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: run: | docker build -t sqlc-ydb-cpp-tests -f examples/authors/cpp/Dockerfile examples/authors/cpp docker run --rm -v "$PWD:/workspace" -w /workspace sqlc-ydb-cpp-tests \ - bash -lc 'cmake -S examples/authors/cpp -B examples/authors/cpp/build -GNinja -DCMAKE_PREFIX_PATH=/usr/share/yandex && cmake --build examples/authors/cpp/build --target authors_native authors_userver -j1' + bash -lc 'cmake -S examples/authors/cpp -B examples/authors/cpp/build -GNinja -DCMAKE_PREFIX_PATH=/usr/share/yandex && cmake --build examples/authors/cpp/build --target authors_native authors_userver -j1 -- -k 0' - name: Start one disposable YDB service run: | docker run -d --name sqlc-ydb-cpp-server --hostname localhost \ diff --git a/docs/cpp.md b/docs/cpp.md index b685fe5..90a5daa 100644 --- a/docs/cpp.md +++ b/docs/cpp.md @@ -50,7 +50,7 @@ Identifiers must be ASCII C++ identifiers, must not be C++20 keywords, and must The API was checked on 2026-09-07 against these exact revisions: -- YDB C++ SDK `main`: [`6ea7a0f93bd97bcb92dca3a4ac948bb743077e50`](https://github.com/ydb-platform/ydb-cpp-sdk/tree/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50). `TQueryClient` declares the `ExecuteQuery` and `RetryQuerySync` overloads in [`client.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/query/client.h#L74-L120). The SDK value API provides width-specific, `String`, `Utf8`, and optional builders/parsers in [`value.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/value/value.h#L328-L510). The maintained basic example demonstrates `RetryQuerySync`, transaction control, parameter construction, and `TResultSetParser` in [`basic_example.cpp`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/examples/basic_example/basic_example.cpp#L163-L247). +- YDB C++ SDK `main`: [`6ea7a0f93bd97bcb92dca3a4ac948bb743077e50`](https://github.com/ydb-platform/ydb-cpp-sdk/tree/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50). `TQueryClient` declares the `ExecuteQuery` and `RetryQuerySync` overloads in [`client.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/query/client.h#L74-L120). The SDK value API provides width-specific, `String`, `Utf8`, and optional builders/parsers in [`value.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/value/value.h#L328-L510). Status failures are raised by `NYdb::NStatusHelpers::ThrowOnError`, declared in [`status.h`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/include/ydb-cpp-sdk/client/types/status/status.h#L51-L100). The maintained basic example demonstrates `RetryQuerySync`, transaction control, parameter construction, and `TResultSetParser` in [`basic_example.cpp`](https://github.com/ydb-platform/ydb-cpp-sdk/blob/6ea7a0f93bd97bcb92dca3a4ac948bb743077e50/examples/basic_example/basic_example.cpp#L163-L247). - userver `develop`: [`86759637d175baa64f0b3b01f1a027bfedbf795a`](https://github.com/userver-framework/userver/tree/86759637d175baa64f0b3b01f1a027bfedbf795a). `TableClient::ExecuteQuery` and its retry contract are declared in [`table.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/table.hpp#L90-L235). Cursor and typed row extraction are defined in [`response.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/response.hpp#L35-L180). The public primitive mapping, including the absence of `Float` and the distinct `Utf8` strong type, is documented in [`types.hpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/include/userver/ydb/types.hpp#L15-L75). The implementation shows retry-managed Query SDK execution and per-call transaction selection in [`table.cpp`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/ydb/src/ydb/table.cpp#L420-L445). The official Ubuntu image enables YDB in [`ubuntu-24.04-userver.dockerfile`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/scripts/docker/ubuntu-24.04-userver.dockerfile), and [`SetupYdbCppSDK.cmake`](https://github.com/userver-framework/userver/blob/86759637d175baa64f0b3b01f1a027bfedbf795a/cmake/SetupYdbCppSDK.cmake#L4-L55) pins SDK 3.21.1 and requests its `Iam` component. - YDB documentation `main`: [`2a1fce8e188f51004950d1e8684b395304a229aa`](https://github.com/ydb-platform/ydb/tree/2a1fce8e188f51004950d1e8684b395304a229aa). The [retry guide](https://github.com/ydb-platform/ydb/blob/2a1fce8e188f51004950d1e8684b395304a229aa/ydb/docs/en/core/recipes/ydb-sdk/retry.md) recommends native `RetryQuerySync` with a session and states that all userver `TableClient` methods include retry handling. @@ -68,29 +68,30 @@ The official `ghcr.io/userver-framework/ubuntu-24.04-userver` image is built wit That image's installed `userver-ydb-config.cmake` has two packaging defects. It asks for the obsolete CMake package name `googleapis`, while its real installed SDK package exports `yandex-googleapis-api-common-protos::api-common-protos` from `yandex-googleapis-api-common-protosConfig.cmake`. It also loads the SDK without components even though `userver::ydb` requires the IAM library. The SDK 3.21.1 package is not safe to load twice with different component lists because it recreates component aliases. -The Dockerfile verifies and corrects both dependency lines. Its single SDK load requests `Driver`, `Params`, and `Query` for the native example plus `Iam` for `userver::ydb`. The top-level project therefore loads userver once and uses the real SDK targets that dependency exports. The workaround does not add replacement headers, targets, or libraries. Build the small derived image and compile serially to stay within the 2 GB Docker VM: +The Dockerfile verifies and corrects both dependency lines. Its single SDK load requests `Driver`, `Params`, and `Query` for the native example plus `Iam` for `userver::ydb`. The top-level project therefore loads userver once and uses the real SDK targets that dependency exports. The workaround does not add replacement headers, targets, or libraries. These commands use Linux with an amd64 Docker engine, as in CI. Build and compile before starting local-ydb, and keep compilation sequential: ```bash docker build \ -t sqlc-ydb-authors-cpp \ - -f examples/authors/cpp/Dockerfile . + -f examples/authors/cpp/Dockerfile examples/authors/cpp docker run --rm \ -v "$PWD:/workspace" -w /workspace \ sqlc-ydb-authors-cpp \ - bash -lc 'cmake -S examples/authors/cpp -B /tmp/authors-cpp -GNinja -DCMAKE_PREFIX_PATH="/usr/share/yandex;/usr/local" && cmake --build /tmp/authors-cpp --target authors_native authors_userver -j1' + bash -lc 'cmake -S examples/authors/cpp -B examples/authors/cpp/build -GNinja -DCMAKE_PREFIX_PATH=/usr/share/yandex && cmake --build examples/authors/cpp/build --target authors_native authors_userver -j1' ``` Both live smokes expect to run with `examples/authors` as the working directory and use `SQLC_YDB_TEST_DSN`, defaulting in the wrappers to `grpc://localhost:2136/local`. The smoke launchers split that value into the SDK endpoint `grpc://localhost:2136` and database `/local`; this matches `TDriverConfig::SetEndpoint` plus `SetDatabase` and userver's YDB component schema. They create the `authors` table from `schema.sql` without `IF NOT EXISTS`, test maximum `Uint64`, present and null optionals, missing `:one`, the named single-column query, `:many`, and `:exec`, then drop the table. Cleanup is armed only after table creation succeeds. ```bash -cd examples/authors -SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ - cpp/native/probe.sh /tmp/authors-cpp/native/authors_native - -SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ - cpp/userver/run.sh /tmp/authors-cpp/userver/authors_userver & -userver_pid=$! -trap 'kill "$userver_pid" 2>/dev/null || true' EXIT -cpp/userver/probe.sh +# Run from the repository root after a disposable YDB is ready on port 2136. +docker run --rm --network host \ + -v "$PWD:/workspace" -w /workspace \ + -e SQLC_YDB_TEST_DSN=grpc://localhost:2136/local \ + sqlc-ydb-authors-cpp bash examples/authors/cpp/run-smoke.sh ``` + +The runner sets the working directory, executes native first, starts userver, +waits for its listener, invokes `/smoke` once, and stops the userver process. +The compiled binaries stay in the mounted `cpp/build` directory and execute +inside the same SDK image used to build them. diff --git a/docs/development.md b/docs/development.md index ce73474..f50266f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,8 +13,8 @@ contents and expected diagnostics. Fixture updates are explicit, never an automatic part of tests. Semantic unit tests independently assert resolved parameters, result columns and errors. -CI separates this fast offline suite from one Linux acceptance job backed by a -single pinned YDB service. GitHub Actions waits for the image's health check; +CI separates this fast offline suite from Linux acceptance jobs. Each acceptance +host runs a single pinned YDB service and waits for the image's health check; there is no multi-database startup framework, engine matrix, plugin subprocess runner or optional Postgres/MySQL fallback. See [the fixture runner](../internal/endtoend/README.md) for the upstream references diff --git a/examples/authors/cpp/native/main.cpp b/examples/authors/cpp/native/main.cpp index b3759b1..9a893a1 100644 --- a/examples/authors/cpp/native/main.cpp +++ b/examples/authors/cpp/native/main.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -49,7 +50,7 @@ void ExecuteStatement(NYdb::NQuery::TQueryClient& client, const std::string& sta NYdb::NQuery::TTxControl::NoTx() ).GetValueSync(); }); - NYdb::ThrowOnError(status); + NYdb::NStatusHelpers::ThrowOnError(status); } class CreatedAuthorsTable final { diff --git a/examples/authors/cpp/native/queries.cpp b/examples/authors/cpp/native/queries.cpp index 41713cb..91e0380 100644 --- a/examples/authors/cpp/native/queries.cpp +++ b/examples/authors/cpp/native/queries.cpp @@ -1,6 +1,10 @@ // Code generated by sqlc-ydb. DO NOT EDIT. #include "queries.hpp" +#include +#include +#include + #include #include @@ -47,7 +51,7 @@ std::optional Queries::GetAuthor(std::uint64_t author_id) const { } return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); if (!sqlc_result_set) { throw std::runtime_error("GetAuthor: successful query returned no result set"); } @@ -75,7 +79,7 @@ std::vector Queries::ListAuthors() const { } return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); if (!sqlc_result_set) { throw std::runtime_error("ListAuthors: successful query returned no result set"); } @@ -108,7 +112,7 @@ std::optional Queries::GetAuthorName(std::uint64_t author_id) } return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); if (!sqlc_result_set) { throw std::runtime_error("GetAuthorName: successful query returned no result set"); } @@ -136,7 +140,7 @@ void Queries::UpsertAuthor(std::uint64_t author_id, const std::string& author_na ).GetValueSync(); return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); } void Queries::DeleteAuthor(std::uint64_t author_id) const { @@ -151,7 +155,7 @@ void Queries::DeleteAuthor(std::uint64_t author_id) const { ).GetValueSync(); return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); } } // namespace authors::native diff --git a/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java b/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java index ab4b2d4..cd263b3 100644 --- a/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java +++ b/examples/authors/java/native/src/test/java/authors/nativeapi/Smoke.java @@ -29,8 +29,6 @@ public static void main(String[] args) throws Exception { try { exercise(queries); } finally { - queries.deleteAuthor(MAX_UINT64); - queries.deleteAuthor(SECOND_ID); dropSchema(retry); } } diff --git a/internal/codegen/cpp/generator.go b/internal/codegen/cpp/generator.go index 0e836af..e345226 100644 --- a/internal/codegen/cpp/generator.go +++ b/internal/codegen/cpp/generator.go @@ -325,7 +325,11 @@ func methodParameters(query model.AnalyzedQuery, runtime string) string { func renderSource(in *model.AnalysisResult, options Options) (string, error) { var out strings.Builder - out.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n#include \"queries.hpp\"\n\n#include \n#include \n\nnamespace " + options.Namespace + " {\nnamespace {\n\n") + out.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n#include \"queries.hpp\"\n") + if options.Runtime == "ydb" { + out.WriteString("\n#include \n#include \n#include \n") + } + out.WriteString("\n#include \n#include \n\nnamespace " + options.Namespace + " {\nnamespace {\n\n") for _, query := range in.Queries { if options.Runtime == "ydb" { out.WriteString("const std::string k" + query.Name + "Sql = " + sqlLiteral(query.SQL) + ";\n\n") @@ -379,7 +383,7 @@ func renderNativeMethod(out *strings.Builder, query model.AnalyzedQuery, options if query.Command == model.One || query.Command == model.Many { out.WriteString(" if (sqlc_result.IsSuccess() && !sqlc_result.GetResultSets().empty()) {\n sqlc_result_set = sqlc_result.GetResultSet(0);\n }\n") } - out.WriteString(" return sqlc_result;\n });\n NYdb::ThrowOnError(sqlc_status);\n") + out.WriteString(" return sqlc_result;\n });\n NYdb::NStatusHelpers::ThrowOnError(sqlc_status);\n") if query.Command == model.Exec { out.WriteString("}\n\n") return nil diff --git a/internal/codegen/cpp/generator_test.go b/internal/codegen/cpp/generator_test.go index 8c5a65d..e42c44b 100644 --- a/internal/codegen/cpp/generator_test.go +++ b/internal/codegen/cpp/generator_test.go @@ -89,7 +89,9 @@ func TestGenerateNativeYDBAuthorsAPI(t *testing.T) { } } for _, want := range []string{ + "#include ", "client_.RetryQuerySync", + "NYdb::NStatusHelpers::ThrowOnError(sqlc_status);", "NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx()", ".AddParam(\"$author_id\").Uint64(author_id).Build()", ".AddParam(\"$author_name\").Utf8(author_name).Build()", diff --git a/internal/codegen/csharp/generator.go b/internal/codegen/csharp/generator.go index bfc4bce..72f3af0 100644 --- a/internal/codegen/csharp/generator.go +++ b/internal/codegen/csharp/generator.go @@ -55,7 +55,7 @@ func validate(in *model.AnalysisResult) error { if err := add(modelNames, csName(table.Name), "table:"+table.Name, "model name"); err != nil { return err } - if err := fields("table "+table.Name, table.Columns); err != nil { + if err := fields("table "+table.Name, csName(table.Name), table.Columns); err != nil { return err } } @@ -95,12 +95,19 @@ func validate(in *model.AnalysisResult) error { if err := add(modelNames, csName(q.Name)+"Params", "params:"+q.Name, "model name"); err != nil { return err } + params := make([]model.Column, len(q.Parameters)) + for i, p := range q.Parameters { + params[i] = model.Column{Name: p.Name, Type: p.Type} + } + if err := fields("query "+q.Name+" parameters", csName(q.Name)+"Params", params); err != nil { + return err + } } if q.Command == model.One || q.Command == model.Many { if err := add(modelNames, csName(q.Name)+"Row", "row:"+q.Name, "model name"); err != nil { return err } - if err := fields("query "+q.Name, q.ResultSets[0].Columns); err != nil { + if err := fields("query "+q.Name, csName(q.Name)+"Row", q.ResultSets[0].Columns); err != nil { return err } } @@ -108,13 +115,25 @@ func validate(in *model.AnalysisResult) error { return nil } -func fields(where string, columns []model.Column) error { +var recordReservedMembers = map[string]bool{ + "Clone": true, "Deconstruct": true, "EqualityContract": true, "PrintMembers": true, + "Equals": true, "GetHashCode": true, "ToString": true, + "GetType": true, "MemberwiseClone": true, "Finalize": true, "ReferenceEquals": true, +} + +func fields(where, record string, columns []model.Column) error { seen := map[string]bool{} for _, c := range columns { n := csName(c.Name) if !csIdent(n) || seen[n] { return fmt.Errorf("csharp generator: %s: column name collision at %q", where, c.Name) } + if n == record { + return fmt.Errorf("csharp generator: %s: record member %q collides with record name", where, c.Name) + } + if recordReservedMembers[n] { + return fmt.Errorf("csharp generator: %s: record member %q collides with generated record member", where, c.Name) + } seen[n] = true if _, err := csType(c.Type); err != nil { return fmt.Errorf("csharp generator: %s column %q: %w", where, c.Name, err) diff --git a/internal/codegen/csharp/generator_test.go b/internal/codegen/csharp/generator_test.go index 0fb62bb..458d090 100644 --- a/internal/codegen/csharp/generator_test.go +++ b/internal/codegen/csharp/generator_test.go @@ -182,6 +182,53 @@ func TestRejectsModelNamesThatShadowFrameworkTypes(t *testing.T) { } } +func TestRejectsRecordMemberCollisions(t *testing.T) { + utf8 := model.Type{Kind: "Utf8"} + for _, tc := range []struct { + name string + in *model.AnalysisResult + want string + }{ + { + name: "table member equals record", want: "collides with record name", + in: &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{ + Name: "authors", Columns: []model.Column{{Name: "authors", Type: utf8}}, + }}}}, + }, + { + name: "row member equals record", want: "collides with record name", + in: &model.AnalysisResult{Queries: []model.AnalyzedQuery{{ + Name: "get_author", Command: model.One, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "get_author_row", Type: utf8}}}}, + }}}, + }, + { + name: "params member equals record", want: "collides with record name", + in: &model.AnalysisResult{Queries: []model.AnalyzedQuery{{ + Name: "upsert", Command: model.Exec, Parameters: []model.Parameter{{Name: "upsert_params", Type: utf8}, {Name: "other", Type: utf8}}, + }}}, + }, + { + name: "synthesized clone", want: "generated record member", + in: &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{ + Name: "authors", Columns: []model.Column{{Name: "clone", Type: utf8}}, + }}}}, + }, + { + name: "synthesized equality contract", want: "generated record member", + in: &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{ + Name: "authors", Columns: []model.Column{{Name: "equality_contract", Type: utf8}}, + }}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := Generate(tc.in, Options{}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Generate() error = %v, want %q", err, tc.want) + } + }) + } +} + // This is an execution check, not merely a source inspection: C# evaluates the // emitted literal and compares its UTF-8 bytes with the original SQL. func TestSQLLiteralRoundTripsThroughCSharpRuntime(t *testing.T) { diff --git a/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp b/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp index 9c385ed..33faac1 100644 --- a/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp +++ b/internal/endtoend/testdata/authors/expected/cpp/native/queries.cpp @@ -1,6 +1,10 @@ // Code generated by sqlc-ydb. DO NOT EDIT. #include "queries.hpp" +#include +#include +#include + #include #include @@ -29,7 +33,7 @@ std::optional Queries::GetAuthor(std::uint64_t author_id) const { } return sqlc_result; }); - NYdb::ThrowOnError(sqlc_status); + NYdb::NStatusHelpers::ThrowOnError(sqlc_status); if (!sqlc_result_set) { throw std::runtime_error("GetAuthor: successful query returned no result set"); } From 635ad0bd52f0f4fc2fe876483e51e2735c746651 Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 18:39:42 +0300 Subject: [PATCH 4/7] fix: pass bare endpoints to C++ SDK smoke clients --- docs/cpp.md | 2 +- docs/targets.md | 4 ++-- examples/authors/cpp/native/main.cpp | 14 ++++++++------ examples/authors/cpp/userver/run.sh | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/cpp.md b/docs/cpp.md index 90a5daa..d0da672 100644 --- a/docs/cpp.md +++ b/docs/cpp.md @@ -81,7 +81,7 @@ docker run --rm \ bash -lc 'cmake -S examples/authors/cpp -B examples/authors/cpp/build -GNinja -DCMAKE_PREFIX_PATH=/usr/share/yandex && cmake --build examples/authors/cpp/build --target authors_native authors_userver -j1' ``` -Both live smokes expect to run with `examples/authors` as the working directory and use `SQLC_YDB_TEST_DSN`, defaulting in the wrappers to `grpc://localhost:2136/local`. The smoke launchers split that value into the SDK endpoint `grpc://localhost:2136` and database `/local`; this matches `TDriverConfig::SetEndpoint` plus `SetDatabase` and userver's YDB component schema. They create the `authors` table from `schema.sql` without `IF NOT EXISTS`, test maximum `Uint64`, present and null optionals, missing `:one`, the named single-column query, `:many`, and `:exec`, then drop the table. Cleanup is armed only after table creation succeeds. +Both live smokes expect to run with `examples/authors` as the working directory and use `SQLC_YDB_TEST_DSN`, defaulting in the wrappers to `grpc://localhost:2136/local`. The smoke launchers split that value into the SDK endpoint `localhost:2136` and database `/local`. SDK 3.21.1 stores `TDriverConfig::SetEndpoint` input verbatim, and userver passes its configured endpoint directly to that method, so the protocol prefix belongs only to the external smoke DSN. They create the `authors` table from `schema.sql` without `IF NOT EXISTS`, test maximum `Uint64`, present and null optionals, missing `:one`, the named single-column query, `:many`, and `:exec`, then drop the table. Cleanup is armed only after table creation succeeds. ```bash # Run from the repository root after a disposable YDB is ready on port 2136. diff --git a/docs/targets.md b/docs/targets.md index ee9c47d..ac06d3a 100644 --- a/docs/targets.md +++ b/docs/targets.md @@ -47,8 +47,8 @@ they do not infer ORM entities from query results. Generated code uses caller-provided clients/connections. The caller controls connection lifetime and credentials. Transaction behavior is target-specific: -native C++ and Java execute a transaction per method; connection and framework -profiles use the caller's transaction. Generated DB-API code closes +both C++ profiles and native Java execute a transaction per method; C#, JDBC, +Spring, and Hibernate use the caller's connection or transaction. Generated DB-API code closes its own cursors and does not commit caller-owned transactions. The verified `ydb-sqlalchemy` 0.1.22 has no asynchronous dialect. Requests for diff --git a/examples/authors/cpp/native/main.cpp b/examples/authors/cpp/native/main.cpp index 9a893a1..a750dc3 100644 --- a/examples/authors/cpp/native/main.cpp +++ b/examples/authors/cpp/native/main.cpp @@ -29,20 +29,22 @@ std::string ReadSchema() { } struct TestDatabase final { - std::string endpoint; - std::string database; + std::string_view endpoint; + std::string_view database; }; -TestDatabase ParseTestDsn(const char* dsn) { - const std::string value{dsn}; +constexpr TestDatabase ParseTestDsn(std::string_view value) { constexpr std::string_view kPrefix{"grpc://"}; const auto database_pos = value.find('/', kPrefix.size()); if (!value.starts_with(kPrefix) || database_pos == std::string::npos || database_pos == kPrefix.size()) { throw std::runtime_error("SQLC_YDB_TEST_DSN must look like grpc://host:port/database"); } - return {value.substr(0, database_pos), value.substr(database_pos)}; + return {value.substr(kPrefix.size(), database_pos - kPrefix.size()), value.substr(database_pos)}; } +static_assert(ParseTestDsn("grpc://localhost:2136/local").endpoint == "localhost:2136"); +static_assert(ParseTestDsn("grpc://localhost:2136/local").database == "/local"); + void ExecuteStatement(NYdb::NQuery::TQueryClient& client, const std::string& statement) { const auto status = client.RetryQuerySync([&](NYdb::NQuery::TSession session) -> NYdb::TStatus { return session.ExecuteQuery( @@ -87,7 +89,7 @@ int main() { try { const auto test_database = ParseTestDsn(dsn); NYdb::TDriverConfig driver_config; - driver_config.SetEndpoint(test_database.endpoint).SetDatabase(test_database.database); + driver_config.SetEndpoint(std::string{test_database.endpoint}).SetDatabase(std::string{test_database.database}); NYdb::TDriver driver{driver_config}; NYdb::NQuery::TQueryClient client{driver}; ExecuteStatement(client, ReadSchema()); diff --git a/examples/authors/cpp/userver/run.sh b/examples/authors/cpp/userver/run.sh index 6e6cfe2..86cf33a 100755 --- a/examples/authors/cpp/userver/run.sh +++ b/examples/authors/cpp/userver/run.sh @@ -3,7 +3,7 @@ set -euo pipefail : "${SQLC_YDB_TEST_DSN:=grpc://localhost:2136/local}" -if [[ ! "${SQLC_YDB_TEST_DSN}" =~ ^(grpc://[^/]+)(/.*)$ ]]; then +if [[ ! "${SQLC_YDB_TEST_DSN}" =~ ^grpc://([^/]+)(/.*)$ ]]; then echo "SQLC_YDB_TEST_DSN must look like grpc://host:port/database" >&2 exit 2 fi From 5ac2e3bb6a5aa07bb3fadfc3ed64b45d329e03db Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 18:46:54 +0300 Subject: [PATCH 5/7] test: disable userver stack monitor in Docker smoke config --- docs/cpp.md | 3 +++ examples/authors/cpp/userver/static_config.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/cpp.md b/docs/cpp.md index d0da672..bcf4662 100644 --- a/docs/cpp.md +++ b/docs/cpp.md @@ -95,3 +95,6 @@ The runner sets the working directory, executes native first, starts userver, waits for its listener, invokes `/smoke` once, and stops the userver process. The compiled binaries stay in the mounted `cpp/build` directory and execute inside the same SDK image used to build them. +The smoke config disables userver's optional coroutine stack usage monitor, +whose `userfaultfd` call is blocked by Docker's default seccomp profile. +The example runs with ordinary container permissions. diff --git a/examples/authors/cpp/userver/static_config.yaml b/examples/authors/cpp/userver/static_config.yaml index 1741380..58bbaed 100644 --- a/examples/authors/cpp/userver/static_config.yaml +++ b/examples/authors/cpp/userver/static_config.yaml @@ -1,4 +1,7 @@ components_manager: + coro_pool: + # Docker's default seccomp profile blocks userfaultfd used by this diagnostic. + stack_usage_monitor_enabled: false task_processors: main-task-processor: worker_threads: 2 From 6b12f8d7924e2d8421b2723d582c9f8522b3eb66 Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 19:53:54 +0300 Subject: [PATCH 6/7] analyzer: apply YDB schema migrations to the catalog --- README.md | 3 +- docs/architecture.md | 31 +++ docs/compatibility.md | 41 +++- docs/roadmap.md | 112 ++++++++++ internal/analyzer/analyzer_test.go | 4 +- internal/analyzer/catalog.go | 207 +++++++++++++++--- internal/analyzer/catalog_migrations_test.go | 203 +++++++++++++++++ .../testdata/migrations/expected/db/db.go | 16 ++ .../testdata/migrations/expected/db/models.go | 7 + .../migrations/expected/db/queries.sql.go | 25 +++ .../migrations/expected/py/__init__.py | 1 + .../testdata/migrations/expected/py/models.py | 15 ++ .../migrations/expected/py/queries.py | 44 ++++ .../migrations/migrations/001_create.sql | 11 + .../migrations/migrations/002_alter.sql | 6 + .../migrations/003_cleanup.down.sql | 1 + .../endtoend/testdata/migrations/queries.sql | 3 + .../endtoend/testdata/migrations/sqlc.yaml | 12 + 18 files changed, 709 insertions(+), 33 deletions(-) create mode 100644 docs/roadmap.md create mode 100644 internal/analyzer/catalog_migrations_test.go create mode 100644 internal/endtoend/testdata/migrations/expected/db/db.go create mode 100644 internal/endtoend/testdata/migrations/expected/db/models.go create mode 100644 internal/endtoend/testdata/migrations/expected/db/queries.sql.go create mode 100644 internal/endtoend/testdata/migrations/expected/py/__init__.py create mode 100644 internal/endtoend/testdata/migrations/expected/py/models.py create mode 100644 internal/endtoend/testdata/migrations/expected/py/queries.py create mode 100644 internal/endtoend/testdata/migrations/migrations/001_create.sql create mode 100644 internal/endtoend/testdata/migrations/migrations/002_alter.sql create mode 100644 internal/endtoend/testdata/migrations/migrations/003_cleanup.down.sql create mode 100644 internal/endtoend/testdata/migrations/queries.sql create mode 100644 internal/endtoend/testdata/migrations/sqlc.yaml diff --git a/README.md b/README.md index 256fa30..e779fae 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,8 @@ and result columns; it is not an intermediate AST. Unsupported constructs and unresolved types must produce an error rather than an untyped fallback. See [compatibility](docs/compatibility.md), [targets](docs/targets.md), -[architecture](docs/architecture.md), and [development](docs/development.md) for +[architecture](docs/architecture.md), [compiler roadmap](docs/roadmap.md), and +[development](docs/development.md) for the implemented scope and remaining work. Target-specific configuration and examples are described in [C++](docs/cpp.md), [C#](docs/csharp.md), and [Java](docs/java.md). diff --git a/docs/architecture.md b/docs/architecture.md index e244fb7..156eeb3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,6 +33,37 @@ prepared before writes start. Each file is replaced through a temporary sibling; this protects individual files from interrupted writes, but does not promise a filesystem transaction covering every output. +## Where is the compiler? + +Compilation is present as a stage, but there is currently no `internal/compiler` +package or `Compiler` object. Its responsibilities are distributed as follows: + +| Responsibility | Current owner | +| --- | --- | +| Resolve source paths, order migrations, keep Up sections | `internal/source` | +| Parse YQL, apply schema statements to the catalog, analyze named queries | `internal/analyzer` | +| Return resolved catalog, parameters, result columns and diagnostics | `model.AnalysisResult` | +| Invoke analysis once per `sql` configuration entry, then its generators | `internal/cli.prepare` | +| Render source files for each selected language/runtime | `internal/codegen/*` | + +In upstream sqlc, `internal/compiler.Compiler` owns catalog/query compilation, +parser selection, analysis and SQL rewrites; code generation is dispatched by +`internal/cmd`. A compiler therefore is not another syntax representation and +does not imply an intermediate AST or a native-code backend. The comparable +boundary here is `analyzer.Analyze(...) -> model.AnalysisResult`. + +Macro handling and semantic analysis must share one compilation boundary before +generation. This can remain in `internal/analyzer`; a separate `internal/compiler` +package is an implementation option, not a requirement. Extract orchestration +only if the added responsibilities justify it. The CLI retains configuration, +source/output paths and file IO. No engine registry, plugin interface or second +AST is needed. See [the compiler roadmap](roadmap.md) for ordering and acceptance +criteria. + +Reference: upstream sqlc +[`Compiler`](https://github.com/sqlc-dev/sqlc/blob/23e357a414310aa8846e64624da8b8a626b3a610/internal/compiler/engine.go) +and [generation dispatch](https://github.com/sqlc-dev/sqlc/blob/23e357a414310aa8846e64624da8b8a626b3a610/internal/cmd/generate.go). + ## Development decisions - Develop the independent implementation in the existing repository. Do not diff --git a/docs/compatibility.md b/docs/compatibility.md index 42ec03d..1b64d31 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -28,6 +28,9 @@ internal data structures and source history are not a dependency. lexical order. Hidden files and `*.down.sql` are excluded. - Schema rollback sections for goose, sql-migrate, tern and dbmate are excluded. Migration markers inside string literals are preserved. +- Schema migrations update an in-memory catalog in input order. Supported DDL: + `CREATE TABLE [IF NOT EXISTS]`, `DROP TABLE [IF EXISTS]`, and `ALTER TABLE` + with `ADD [COLUMN]`, `DROP [COLUMN]`, or table `RENAME TO`. - Query annotations `-- name: QueryName :one|:many|:exec`. `:execrows` is parsed but generation rejects it: the selected YDB APIs cannot provide its required affected-row count. @@ -57,15 +60,16 @@ fixture and, for runtime-sensitive behavior, an execution test. ## Current analyzer coverage -The initial analyzer supports explicit `CREATE TABLE` catalogs, table column +The analyzer supports explicit `CREATE TABLE` catalogs and the schema migration +operations listed below, table column projections and `*`, table/column aliases, supported joins and their optional sides, `COUNT`, `DECLARE`, direct comparison parameter inference, selected scalar local bindings, `INSERT`/`UPSERT ... VALUES`, `UPDATE ... SET`, `DELETE`, and `RETURNING`. It validates names outside the projection and conflicting parameter constraints. Diagnostics include source file, line and column. -This is a deliberately limited first semantic implementation. Schema evolution -through ALTER/DROP, general computed projections and casts, CTEs/subqueries, +This is a deliberately limited first semantic implementation. General computed +projections and casts, CTEs/subqueries, multiple result sets, FLATTEN, full function/type inference and the full YQL grammar semantics are subsequent work. Accepted syntax is not a claim of full equivalence to the YDB server's type checker. @@ -74,3 +78,34 @@ Current INSERT/UPSERT VALUES and UPDATE SET values must be direct parameters; literal and computed assignments are explicitly rejected. Shared query-file declarations must currently be moved into each named query. These are temporary coverage limits, separate from the permanent decision to exclude plugins. + +## Schema migration coverage + +Use a migration directory, glob or ordered file list as `schema`. The analyzer +applies its supported Up statements to the catalog before analyzing any query. +It never executes migrations or data statements against YDB. Dropping and +recreating a table replaces its schema; renaming updates column ownership. +Adding a column preserves its declared YQL type and nullability, and dropping a +primary-key column fails. Table/column order stays deterministic. + +An ALTER with several supported column actions is applied atomically to the +in-memory catalog. This does not describe server transaction behavior. Missing +objects, duplicate names, rename collisions and invalid primary keys produce +source-located errors. Existing tables remain unchanged by a guarded CREATE. +`RENAME TO` must currently be the only action in its ALTER statement. DDL inside +action definitions and EXPLAIN statements is rejected, not applied to the catalog. + +`ALTER COLUMN` changes to types/nullability/defaults and other ALTER actions +such as indexes, changefeeds or table settings are currently rejected. External +tables, views, table stores and CREATE TABLE AS are also outside this catalog's +scope. Physical CREATE options that do not affect modeled columns are not +represented in the catalog; this is not a full server DDL validator. + +References: YDB [columns](https://ydb.tech/docs/en/yql/reference/syntax/alter_table/columns), +[table rename](https://ydb.tech/docs/en/yql/reference/syntax/alter_table/rename), +and [DROP TABLE](https://ydb.tech/docs/en/yql/reference/syntax/drop_table). +YQL main at `d62403dadf7588c33d2d0a61296a157b61163d52` explicitly handles +[`DROP TABLE IF EXISTS` through `missingOk`](https://github.com/ydb-platform/ydb/blob/d62403dadf7588c33d2d0a61296a157b61163d52/yql/essentials/sql/v1/translation/sql_query.cpp#L575) +and [rejects combining RENAME TO with other ALTER actions](https://github.com/ydb-platform/ydb/blob/d62403dadf7588c33d2d0a61296a157b61163d52/yql/essentials/sql/v1/translation/sql_query.cpp#L2399). +See [the roadmap](roadmap.md) for shared macros and deferred database-assisted +analysis. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..18b144f --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,112 @@ +# Compiler roadmap + +Keep the familiar sqlc workflow, direct ANTLR YQL parse contexts, a resolved +semantic model and built-in generators. There is no cross-dialect AST, engine +registry, external codegen protocol or requirement to merge upstream sources. + +## 1. Schema evolution: implemented subset + +Apply supported YDB DDL to an in-memory catalog in source order, then analyze +queries against the final catalog. This reads migration files; it does not run +migrations against a database or manage a migration version table. + +- Preserve the source loader's explicit list order and lexical directory/glob + order, its supported Up/Down markers and exclusion of `*.down.sql`. +- Apply YDB `ALTER TABLE` and `DROP TABLE` catalog operations alongside `CREATE + TABLE`, with existence guards only where supported by the YQL grammar. +- Validate missing/duplicate names and primary-key constraints. Apply each + multi-action ALTER atomically to the catalog; a failed action must not leave + earlier actions partially applied. +- Keep table/column ordering deterministic and diagnostics tied to the original + file, line and column. Unsupported schema semantics must fail explicitly. +- Verify semantic results with focused tests and the real CLI with a migration + directory fixture generating both Go and Python. Reuse the existing golden + runner and CI; no database matrix or new container orchestration is needed. + +The exact supported DDL subset is recorded in [compatibility](compatibility.md). +It covers ADD/DROP COLUMN and standalone table RENAME TO. General ALTER COLUMN, +indexes, views and other schema objects remain future work. +Further schema operations should be added with corresponding YDB semantics and +fixtures, not accepted merely because the parser recognizes them. + +## 2. Shared compiler and macros: planned + +Keep one compilation entry point per `sql` configuration entry. It consumes +loaded schema/query sources and returns the semantic compilation result. Every +selected generator receives that same completed result. `compile`, `generate` +and `diff` use the same entry point; `compile` stops before generation. + +The existing `analyzer.Analyze` entry point can own this coordination. A separate +`internal/compiler` package is optional: extract it only if macro processing or +other responsibilities make that boundary useful. Keep the existing +`model.AnalysisResult`; avoid an empty wrapper or a package added for naming +consistency alone. A stateful `Compiler` object is only needed if later caching +or server resources justify its lifecycle. + +Macro processing belongs inside this compilation boundary. It happens once per +compilation unit, independent of the number or language of its generators. This +does not mean all macros can be resolved in one pass before analysis: + +1. Recognize supported sqlc macro syntax using tokens, preserving strings, + comments, quoted identifiers and source positions. Lower syntax that YQL + cannot parse into YQL parameters while recording macro metadata. +2. Build the catalog and analyze the YQL parse contexts. Resolve macro-dependent + names, types, optional parameters, result grouping and list element types. +3. Finalize executable YQL and semantic metadata once. If a rewrite changes + syntax, validate the rewritten query before passing it to generators. + +Implement `sqlc.arg` and `sqlc.narg` first: map names to YDB parameters, preserve +`narg` nullability, reconcile explicit `DECLARE` statements, infer required +types and diagnose conflicts. Cover repeated uses and collisions with existing +parameters/local bindings. Do not silently rename existing public parameters. + +Then address `sqlc.embed`: projection expansion depends on the catalog and must +retain grouping metadata for generated models. It is not only string +replacement. Plan `sqlc.slice` after defining its YDB `List` parameter +semantics and each runtime's list binding. Unsupported combinations must fail; +there is no need to copy another database's placeholder expansion strategy. + +Upstream macro behavior is the reference: +[sqlc macros](https://docs.sqlc.dev/en/latest/reference/macros.html). + +Runtime placeholder rendering is a separate last step: SQLAlchemy's `:name` is +different from executable YQL's `$name`. During shared macro implementation, record +resolved external parameter occurrences and their roles/ranges alongside the +SQL, so adapters can render their syntax without re-lexing or rediscovering +parameters. Update ranges after shared rewrites. Declarations and local bindings +must not be mistaken for external parameter occurrences. These ranges and +macro/result metadata are not a recursive AST. + +Acceptance criteria: + +- One compilation result feeds several generators; adding another generator + never repeats macro resolution or semantic analysis. +- `compile` reports the same macro errors as `generate` and `diff`. +- Diagnostics refer to original SQL even after expansion; tests cover Unicode, + CRLF, escaped identifiers, comments and macro-like text inside strings. +- Compiler tests assert rewritten SQL and semantic metadata independently of + any generator. End-to-end tests verify at least two language outputs from the + same macro fixture, and existing exact SQL-literal round-trip tests remain. +- Runtime-specific placeholder tests assert the SQL ultimately seen by YDB, + including parameter-like text in strings/comments and YQL local bindings. + +## 3. Database-assisted analysis: deferred + +Implement when a concrete feature request or a query the local analyzer cannot +resolve justifies it. Local generation remains the default, with no implicit +connection to a configured or developer database. + +First investigate the current YDB APIs for schema description and query type +metadata without executing user queries. Record which parameter/result types +they actually expose, what schema state they require and their limitations. +Do not assume server EXPLAIN/prepare can replace the local analyzer. + +Then define explicit opt-in configuration, schema-drift behavior, type metadata +reconciliation, diagnostics, timeouts and cache invalidation. If an isolated +database needs schema preparation, treat that as an explicit separate mode; +compilation must not apply migrations to an arbitrary application database. +All generators consume the same enriched semantic result. + +Acceptance includes reproducible offline behavior, clear errors when requested +server metadata is unavailable, and sequential live-YDB integration tests. +Different local-ydb images and runtime suites stay sequential on each host. diff --git a/internal/analyzer/analyzer_test.go b/internal/analyzer/analyzer_test.go index 73bb510..10bac92 100644 --- a/internal/analyzer/analyzer_test.go +++ b/internal/analyzer/analyzer_test.go @@ -186,8 +186,8 @@ SELECT id FROM authors WHERE id = $id;`}} func TestAnalyzeRejectsUnsupportedSchemaStatements(t *testing.T) { result, err := Analyze([]model.Source{{Name: "schema.sql", Text: ` CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); -DROP TABLE authors;`}}, nil) - if err == nil || result == nil || !strings.Contains(err.Error(), "only CREATE TABLE is currently supported") { +UPSERT INTO authors (id) VALUES (1);`}}, nil) + if err == nil || result == nil || !strings.Contains(err.Error(), "unsupported schema statement") { t.Fatalf("error = %v; result = %#v", err, result) } } diff --git a/internal/analyzer/catalog.go b/internal/analyzer/catalog.go index 501cabc..3472218 100644 --- a/internal/analyzer/catalog.go +++ b/internal/analyzer/catalog.go @@ -13,52 +13,188 @@ import ( func buildCatalog(sources []model.Source) (model.Catalog, []model.Diagnostic) { catalog := model.Catalog{} var diagnostics []model.Diagnostic - seen := map[string]bool{} for _, source := range sources { parsed, syntaxDiagnostics := parseYQL(source.Name, source.Text, 0) diagnostics = append(diagnostics, syntaxDiagnostics...) if len(syntaxDiagnostics) != 0 { continue } - var statements []*parser.Sql_stmtContext - descendants(parsed.tree, func(node antlr.Tree) { - if ctx, ok := node.(*parser.Sql_stmtContext); ok { - statements = append(statements, ctx) + statementList := parsed.tree.Sql_stmt_list() + if statementList == nil { + diagnostics = append(diagnostics, model.Diagnostic{Position: model.Position{File: source.Name, Line: 1, Column: 1}, Message: "unsupported schema query form"}) + continue + } + for _, statement := range statementList.AllSql_stmt() { + core := statement.Sql_stmt_core() + if statement.EXPLAIN() != nil || core == nil { + diagnostics = append(diagnostics, diagnosticAt(source.Name, 0, statement, fmt.Sprintf("unsupported schema statement %q; supported statements are CREATE TABLE, ALTER TABLE, and DROP TABLE", statement.GetText()))) + continue } - }) - for _, statement := range statements { - var creates []*parser.Create_table_stmtContext - descendants(statement, func(node antlr.Tree) { - if ctx, ok := node.(*parser.Create_table_stmtContext); ok { - creates = append(creates, ctx) - } - }) - if len(creates) != 1 { - diagnostics = append(diagnostics, diagnosticAt(source.Name, 0, statement, fmt.Sprintf("unsupported schema statement %q; only CREATE TABLE is currently supported", statement.GetText()))) + if create := core.Create_table_stmt(); create != nil { + diagnostics = append(diagnostics, applyCreateTable(&catalog, source.Name, create)...) continue } - create := creates[0] - table, tableDiagnostics := catalogTable(source.Name, create) - diagnostics = append(diagnostics, tableDiagnostics...) - if len(tableDiagnostics) != 0 { + if alter := core.Alter_table_stmt(); alter != nil { + diagnostics = append(diagnostics, applyAlterTable(&catalog, source.Name, alter)...) continue } - key := strings.ToLower(table.Name) - if seen[key] { - diagnostics = append(diagnostics, diagnosticAt(source.Name, 0, create, fmt.Sprintf("table %q is declared more than once", table.Name))) + if drop := core.Drop_table_stmt(); drop != nil { + diagnostics = append(diagnostics, applyDropTable(&catalog, source.Name, drop)...) continue } - seen[key] = true - catalog.Tables = append(catalog.Tables, table) + diagnostics = append(diagnostics, diagnosticAt(source.Name, 0, statement, fmt.Sprintf("unsupported schema statement %q; supported statements are CREATE TABLE, ALTER TABLE, and DROP TABLE", statement.GetText()))) } } return catalog, diagnostics } -func catalogTable(file string, create *parser.Create_table_stmtContext) (model.Table, []model.Diagnostic) { +func applyCreateTable(catalog *model.Catalog, file string, create parser.ICreate_table_stmtContext) []model.Diagnostic { + if diagnostic := validateCreateTableShape(file, create); diagnostic != nil { + return []model.Diagnostic{*diagnostic} + } + name := simpleTableName(create.Simple_table_ref()) + if name != "" { + if _, exists := catalogTableIndex(*catalog, name); exists { + if create.IF() != nil && create.NOT() != nil && create.EXISTS() != nil { + // YDB skips the entire CREATE TABLE IF NOT EXISTS statement when the + // object exists, including validation of the proposed replacement schema. + return nil + } + return []model.Diagnostic{diagnosticAt(file, 0, create, fmt.Sprintf("table %q already exists", name))} + } + } + table, diagnostics := catalogTable(file, create) + if len(diagnostics) != 0 { + return diagnostics + } + catalog.Tables = append(catalog.Tables, table) + return nil +} + +func applyDropTable(catalog *model.Catalog, file string, drop parser.IDrop_table_stmtContext) []model.Diagnostic { + if drop.TABLE() == nil || drop.EXTERNAL() != nil || drop.TABLESTORE() != nil { + return []model.Diagnostic{diagnosticAt(file, 0, drop, "only ordinary DROP TABLE is supported")} + } + name := simpleTableName(drop.Simple_table_ref()) + if name == "" { + return []model.Diagnostic{diagnosticAt(file, 0, drop, "DROP TABLE has no resolvable table name")} + } + index, exists := catalogTableIndex(*catalog, name) + if !exists { + if drop.IF() != nil && drop.EXISTS() != nil { + return nil + } + return []model.Diagnostic{diagnosticAt(file, 0, drop, fmt.Sprintf("table %q does not exist", name))} + } + catalog.Tables = append(catalog.Tables[:index], catalog.Tables[index+1:]...) + return nil +} + +func applyAlterTable(catalog *model.Catalog, file string, alter parser.IAlter_table_stmtContext) []model.Diagnostic { + name := simpleTableName(alter.Simple_table_ref()) + if name == "" { + return []model.Diagnostic{diagnosticAt(file, 0, alter, "ALTER TABLE has no resolvable table name")} + } + index, exists := catalogTableIndex(*catalog, name) + if !exists { + return []model.Diagnostic{diagnosticAt(file, 0, alter, fmt.Sprintf("table %q does not exist", name))} + } + working := cloneTable(catalog.Tables[index]) var diagnostics []model.Diagnostic + actions := alter.AllAlter_table_action() + if len(actions) > 1 { + for _, action := range actions { + if action.Alter_table_rename_to() != nil { + return []model.Diagnostic{diagnosticAt(file, 0, action, "RENAME TO must be the only action in an ALTER TABLE statement")} + } + } + } + for _, action := range actions { + diagnostics = append(diagnostics, applyAlterTableAction(*catalog, index, &working, file, action)...) + if len(diagnostics) != 0 { + return diagnostics + } + } + catalog.Tables[index] = working + return nil +} + +func applyAlterTableAction(catalog model.Catalog, tableIndex int, table *model.Table, file string, action parser.IAlter_table_actionContext) []model.Diagnostic { + if add := action.Alter_table_add_column(); add != nil { + column, err := catalogColumn(table.Name, add.Column_schema()) + if err != nil { + return []model.Diagnostic{diagnosticAt(file, 0, add, err.Error())} + } + if _, exists := catalogColumnIndex(*table, column.Name); exists { + return []model.Diagnostic{diagnosticAt(file, 0, add.Column_schema(), fmt.Sprintf("column %q already exists in table %q", column.Name, table.Name))} + } + table.Columns = append(table.Columns, column) + return nil + } + if drop := action.Alter_table_drop_column(); drop != nil { + name := identifier(drop.An_id().GetText()) + columnIndex, exists := catalogColumnIndex(*table, name) + if !exists { + return []model.Diagnostic{diagnosticAt(file, 0, drop, fmt.Sprintf("column %q does not exist in table %q", name, table.Name))} + } + for _, key := range table.PrimaryKey { + if strings.EqualFold(key, name) { + return []model.Diagnostic{diagnosticAt(file, 0, drop, fmt.Sprintf("cannot drop primary key column %q from table %q", name, table.Name))} + } + } + table.Columns = append(table.Columns[:columnIndex], table.Columns[columnIndex+1:]...) + return nil + } + if rename := action.Alter_table_rename_to(); rename != nil { + newName := identifier(rename.An_id_table().GetText()) + if otherIndex, exists := catalogTableIndex(catalog, newName); exists && otherIndex != tableIndex { + return []model.Diagnostic{diagnosticAt(file, 0, rename, fmt.Sprintf("table %q already exists", newName))} + } + table.Name = newName + for i := range table.Columns { + table.Columns[i].Table = newName + } + return nil + } + return []model.Diagnostic{diagnosticAt(file, 0, action, fmt.Sprintf("unsupported ALTER TABLE action %q; supported actions are ADD COLUMN, DROP COLUMN, and RENAME TO", action.GetText()))} +} + +func catalogTableIndex(catalog model.Catalog, name string) (int, bool) { + for i := range catalog.Tables { + if strings.EqualFold(catalog.Tables[i].Name, name) { + return i, true + } + } + return -1, false +} + +func catalogColumnIndex(table model.Table, name string) (int, bool) { + for i := range table.Columns { + if strings.EqualFold(table.Columns[i].Name, name) { + return i, true + } + } + return -1, false +} + +func cloneTable(table model.Table) model.Table { + table.Columns = append([]model.Column(nil), table.Columns...) + table.PrimaryKey = append([]string(nil), table.PrimaryKey...) + return table +} + +func validateCreateTableShape(file string, create parser.ICreate_table_stmtContext) *model.Diagnostic { if create.TABLE() == nil || create.EXTERNAL() != nil || create.TABLESTORE() != nil || create.Table_as_source() != nil { - return model.Table{}, []model.Diagnostic{diagnosticAt(file, 0, create, "only CREATE TABLE with an explicit column list is supported")} + diagnostic := diagnosticAt(file, 0, create, "only CREATE TABLE with an explicit column list is supported") + return &diagnostic + } + return nil +} + +func catalogTable(file string, create parser.ICreate_table_stmtContext) (model.Table, []model.Diagnostic) { + var diagnostics []model.Diagnostic + if diagnostic := validateCreateTableShape(file, create); diagnostic != nil { + return model.Table{}, []model.Diagnostic{*diagnostic} } ref := create.Simple_table_ref() if ref == nil || ref.Simple_table_ref_core() == nil { @@ -66,6 +202,8 @@ func catalogTable(file string, create *parser.Create_table_stmtContext) (model.T } table := model.Table{Name: identifier(ref.Simple_table_ref_core().GetText())} columnNames := map[string]bool{} + primaryKeyNames := map[string]bool{} + primaryKeyDeclarations := 0 for _, entry := range create.AllCreate_table_entry() { if columnContext := entry.Column_schema(); columnContext != nil { column, err := catalogColumn(table.Name, columnContext) @@ -87,8 +225,16 @@ func catalogTable(file string, create *parser.Create_table_stmtContext) (model.T diagnostics = append(diagnostics, diagnosticAt(file, 0, constraint, "only PRIMARY KEY table constraints are supported")) continue } + primaryKeyDeclarations++ for _, id := range constraint.AllAn_id() { - table.PrimaryKey = append(table.PrimaryKey, identifier(id.GetText())) + name := identifier(id.GetText()) + key := strings.ToLower(name) + if primaryKeyNames[key] { + diagnostics = append(diagnostics, diagnosticAt(file, 0, id, fmt.Sprintf("primary key column %q is declared more than once", name))) + continue + } + primaryKeyNames[key] = true + table.PrimaryKey = append(table.PrimaryKey, name) } continue } @@ -97,11 +243,18 @@ func catalogTable(file string, create *parser.Create_table_stmtContext) (model.T if len(table.Columns) == 0 { diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("table %q has no columns", table.Name))) } + if primaryKeyDeclarations == 0 { + diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("table %q must declare a PRIMARY KEY", table.Name))) + } else if primaryKeyDeclarations > 1 { + diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("table %q declares PRIMARY KEY more than once", table.Name))) + } for _, key := range table.PrimaryKey { if !columnNames[strings.ToLower(key)] { diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("primary key column %q does not exist", key))) } } + // PARTITION BY and WITH describe physical storage and do not change the + // tables, columns, types, or primary keys represented by model.Catalog. return table, diagnostics } diff --git a/internal/analyzer/catalog_migrations_test.go b/internal/analyzer/catalog_migrations_test.go new file mode 100644 index 0000000..6811352 --- /dev/null +++ b/internal/analyzer/catalog_migrations_test.go @@ -0,0 +1,203 @@ +package analyzer + +import ( + "reflect" + "strings" + "testing" + + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" +) + +func TestAnalyzeAppliesSchemaMigrationsAcrossSources(t *testing.T) { + schema := []model.Source{ + {Name: "001_create.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, name Utf8, PRIMARY KEY (id));`}, + {Name: "002_alter.sql", Text: `ALTER TABLE authors ADD COLUMN biography Utf8 NOT NULL, DROP COLUMN name;`}, + } + queries := []model.Source{{Name: "query.sql", Text: `-- name: ListAuthors :many +SELECT id, biography FROM authors;`}} + + got, err := Analyze(schema, queries) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + wantColumns := []model.Column{ + {Name: "id", Type: model.Type{Kind: "Uint64"}, Table: "authors"}, + {Name: "biography", Type: model.Type{Kind: "Utf8"}, Table: "authors"}, + } + if !reflect.DeepEqual(got.Catalog.Tables[0].Columns, wantColumns) { + t.Fatalf("catalog columns = %#v, want %#v", got.Catalog.Tables[0].Columns, wantColumns) + } + if !reflect.DeepEqual(got.Queries[0].ResultSets[0].Columns, wantColumns) { + t.Fatalf("result columns = %#v, want %#v", got.Queries[0].ResultSets[0].Columns, wantColumns) + } +} + +func TestCatalogDropRecreateAndRenamePreserveOrder(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{ + {Name: "001.sql", Text: `CREATE TABLE first (id Uint64 NOT NULL, PRIMARY KEY (id)); CREATE TABLE second (id Uint64 NOT NULL, PRIMARY KEY (id));`}, + {Name: "002.sql", Text: `DROP TABLE first; CREATE TABLE first (key Utf8 NOT NULL, PRIMARY KEY (key)); ALTER TABLE first RENAME TO final;`}, + }) + if len(diagnostics) != 0 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if got, want := []string{catalog.Tables[0].Name, catalog.Tables[1].Name}, []string{"second", "final"}; !reflect.DeepEqual(got, want) { + t.Fatalf("table order = %#v, want %#v", got, want) + } + if got := catalog.Tables[1].Columns[0]; got.Name != "key" || got.Table != "final" { + t.Fatalf("renamed column = %#v", got) + } +} + +func TestCatalogExistenceGuards(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +DROP TABLE IF EXISTS missing; +CREATE TABLE IF NOT EXISTS authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +CREATE TABLE IF NOT EXISTS authors (invalid_replacement Utf8);`}}) + if len(diagnostics) != 0 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if len(catalog.Tables) != 1 || catalog.Tables[0].Columns[0].Name != "id" { + t.Fatalf("catalog = %#v", catalog) + } +} + +func TestCatalogRenamesQuotedTablePathAndColumnOwnership(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE ` + "`dir/authors`" + ` (id Uint64 NOT NULL, PRIMARY KEY (id)); +ALTER TABLE ` + "`dir/authors`" + ` RENAME TO ` + "`archive/writers`" + `;`}}) + if len(diagnostics) != 0 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if catalog.Tables[0].Name != "archive/writers" || catalog.Tables[0].Columns[0].Table != "archive/writers" { + t.Fatalf("catalog = %#v", catalog) + } +} + +func TestCatalogReportsMigrationFailuresAtActionSource(t *testing.T) { + _, diagnostics := buildCatalog([]model.Source{ + {Name: "001.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}, + {Name: "002.sql", Text: "\nALTER TABLE authors ADD COLUMN id Utf8;"}, + }) + if len(diagnostics) != 1 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + got := diagnostics[0] + if got.Position != (model.Position{File: "002.sql", Line: 2, Column: 32}) || !strings.Contains(got.Message, `column "id" already exists`) { + t.Fatalf("diagnostic = %#v", got) + } +} + +func TestCatalogRejectsRenameCollisionWithoutMutation(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +CREATE TABLE writers (id Uint64 NOT NULL, PRIMARY KEY (id)); +ALTER TABLE authors RENAME TO writers;`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, `table "writers" already exists`) { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if got := []string{catalog.Tables[0].Name, catalog.Tables[1].Name}; !reflect.DeepEqual(got, []string{"authors", "writers"}) { + t.Fatalf("table names after failed rename = %#v", got) + } +} + +func TestCatalogRejectsRenameCombinedWithOtherActions(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +ALTER TABLE authors ADD COLUMN name Utf8, RENAME TO writers;`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "RENAME TO must be the only action") { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if catalog.Tables[0].Name != "authors" || len(catalog.Tables[0].Columns) != 1 { + t.Fatalf("catalog after rejected mixed rename = %#v", catalog) + } +} + +func TestCatalogRejectsExplainedSchemaStatementWithoutMutation(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: `EXPLAIN CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "unsupported schema statement") { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if len(catalog.Tables) != 0 { + t.Fatalf("catalog = %#v", catalog) + } +} + +func TestCatalogGuardDoesNotHideUnsupportedCreateForm(t *testing.T) { + _, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +CREATE TABLE IF NOT EXISTS authors (PRIMARY KEY (id)) AS SELECT 1 AS id;`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "only CREATE TABLE with an explicit column list is supported") { + t.Fatalf("diagnostics = %#v", diagnostics) + } +} + +func TestCatalogDoesNotApplyDDLInsideActionDefinition(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +DEFINE ACTION $change_schema() AS + ALTER TABLE authors ADD COLUMN biography Utf8; +END DEFINE;`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, "unsupported schema statement") { + t.Fatalf("diagnostics = %#v", diagnostics) + } + want := []model.Column{{Name: "id", Type: model.Type{Kind: "Uint64"}, Table: "authors"}} + if len(catalog.Tables) != 1 || !reflect.DeepEqual(catalog.Tables[0].Columns, want) { + t.Fatalf("action definition changed catalog: %#v", catalog) + } +} + +func TestCatalogRejectsMissingObjectsAndPrimaryKeyChanges(t *testing.T) { + tests := []struct { + name string + sql string + want string + }{ + {name: "drop missing table", sql: `DROP TABLE missing;`, want: `table "missing" does not exist`}, + {name: "alter missing table", sql: `ALTER TABLE missing ADD COLUMN value Utf8;`, want: `table "missing" does not exist`}, + {name: "drop missing column", sql: `CREATE TABLE t (id Uint64 NOT NULL, PRIMARY KEY (id)); ALTER TABLE t DROP COLUMN missing;`, want: `column "missing" does not exist`}, + {name: "drop key column", sql: `CREATE TABLE t (id Uint64 NOT NULL, PRIMARY KEY (id)); ALTER TABLE t DROP COLUMN id;`, want: `cannot drop primary key column "id"`}, + {name: "duplicate key column", sql: `CREATE TABLE t (id Uint64 NOT NULL, PRIMARY KEY (id, id));`, want: `primary key column "id" is declared more than once`}, + {name: "missing primary key", sql: `CREATE TABLE t (id Uint64 NOT NULL);`, want: `must declare a PRIMARY KEY`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: tt.sql}}) + if len(diagnostics) == 0 || !strings.Contains(diagnostics[0].Message, tt.want) { + t.Fatalf("diagnostics = %#v, want message containing %q", diagnostics, tt.want) + } + }) + } +} + +func TestCatalogRejectsUnsupportedSchemaOperations(t *testing.T) { + tests := []struct { + name string + sql string + want string + }{ + {name: "alter nullability", sql: `CREATE TABLE t (id Uint64 NOT NULL, value Utf8, PRIMARY KEY (id)); ALTER TABLE t ALTER COLUMN value SET NOT NULL;`, want: "unsupported ALTER TABLE action"}, + {name: "index", sql: `CREATE TABLE t (id Uint64 NOT NULL, PRIMARY KEY (id)); ALTER TABLE t ADD INDEX by_id GLOBAL ON (id);`, want: "unsupported ALTER TABLE action"}, + {name: "data statement", sql: `CREATE TABLE t (id Uint64 NOT NULL, PRIMARY KEY (id)); UPSERT INTO t (id) VALUES (1);`, want: "unsupported schema statement"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: tt.sql}}) + if len(diagnostics) == 0 || !strings.Contains(diagnostics[0].Message, tt.want) { + t.Fatalf("diagnostics = %#v, want message containing %q", diagnostics, tt.want) + } + }) + } +} + +func TestCatalogDoesNotPartiallyApplyFailedMultiActionAlter(t *testing.T) { + catalog, diagnostics := buildCatalog([]model.Source{{Name: "schema.sql", Text: ` +CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); +ALTER TABLE authors ADD COLUMN biography Utf8, DROP COLUMN missing;`}}) + if len(diagnostics) != 1 || !strings.Contains(diagnostics[0].Message, `column "missing" does not exist`) { + t.Fatalf("diagnostics = %#v", diagnostics) + } + want := []model.Column{{Name: "id", Type: model.Type{Kind: "Uint64"}, Table: "authors"}} + if !reflect.DeepEqual(catalog.Tables[0].Columns, want) { + t.Fatalf("columns after failed ALTER = %#v, want %#v", catalog.Tables[0].Columns, want) + } +} diff --git a/internal/endtoend/testdata/migrations/expected/db/db.go b/internal/endtoend/testdata/migrations/expected/db/db.go new file mode 100644 index 0000000..1dde34c --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/db/db.go @@ -0,0 +1,16 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package db + +import ( + "context" + "github.com/ydb-platform/ydb-go-sdk/v3/query" +) + +type DBTX interface { + Exec(context.Context, string, ...query.ExecuteOption) error + QueryResultSet(context.Context, string, ...query.ExecuteOption) (query.ClosableResultSet, error) + QueryRow(context.Context, string, ...query.ExecuteOption) (query.Row, error) +} +type Queries struct{ db DBTX } + +func New(db DBTX) *Queries { return &Queries{db: db} } diff --git a/internal/endtoend/testdata/migrations/expected/db/models.go b/internal/endtoend/testdata/migrations/expected/db/models.go new file mode 100644 index 0000000..9bdc385 --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/db/models.go @@ -0,0 +1,7 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +package db + +type GetAuthorRow struct { + ID uint64 + Name *string +} diff --git a/internal/endtoend/testdata/migrations/expected/db/queries.sql.go b/internal/endtoend/testdata/migrations/expected/db/queries.sql.go new file mode 100644 index 0000000..9a35408 --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/db/queries.sql.go @@ -0,0 +1,25 @@ +// Code generated by sqlc-ydb. DO NOT EDIT. +// source: queries.sql +package db + +import ( + "context" + ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/query" +) + +const getAuthor = `-- name: GetAuthor :one +DECLARE $id AS Uint64; +SELECT * FROM authors WHERE id = $id;` + +func (q *Queries) GetAuthor(ctx context.Context, id uint64) (GetAuthorRow, error) { + result, err := q.db.QueryRow(ctx, getAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$id").Uint64(id).Build())) + if err != nil { + return GetAuthorRow{}, err + } + var row GetAuthorRow + if err := result.Scan(&row.ID, &row.Name); err != nil { + return GetAuthorRow{}, err + } + return row, nil +} diff --git a/internal/endtoend/testdata/migrations/expected/py/__init__.py b/internal/endtoend/testdata/migrations/expected/py/__init__.py new file mode 100644 index 0000000..f7188d5 --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/py/__init__.py @@ -0,0 +1 @@ +# Code generated by sqlc-ydb. DO NOT EDIT. diff --git a/internal/endtoend/testdata/migrations/expected/py/models.py b/internal/endtoend/testdata/migrations/expected/py/models.py new file mode 100644 index 0000000..00e1a86 --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/py/models.py @@ -0,0 +1,15 @@ +# Code generated by sqlc-ydb. DO NOT EDIT. +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from uuid import UUID +from typing import Optional + +@dataclass +class Authors: + id: int + name: Optional[str] + +@dataclass +class Author: + id: int + name: Optional[str] diff --git a/internal/endtoend/testdata/migrations/expected/py/queries.py b/internal/endtoend/testdata/migrations/expected/py/queries.py new file mode 100644 index 0000000..5fc1fbb --- /dev/null +++ b/internal/endtoend/testdata/migrations/expected/py/queries.py @@ -0,0 +1,44 @@ +# Code generated by sqlc-ydb. DO NOT EDIT. +from __future__ import annotations +from typing import Iterable, Optional +from . import models +import ydb +from sqlalchemy import text +from sqlalchemy.engine import Connection +def _typed(value, typ): + return (value, typ) + + +SQL_GET_AUTHOR = """-- name\\: GetAuthor \\:one +DECLARE $id AS Uint64; +SELECT * FROM authors WHERE id = :id;""" + + +def _row_value(row, name, index): + try: + return row[name] + except (KeyError, IndexError, TypeError): + try: + return row[index] + except (KeyError, IndexError, TypeError): + return getattr(row, name) + + +class Querier: + def __init__(self, connection: Connection): + self._connection = connection + + def get_author(self, id: int) -> Optional[models.Author]: + parameters = {"id": _typed(id, ydb.PrimitiveType.Uint64)} + result = self._connection.execute(text(SQL_GET_AUTHOR), parameters) + try: + rows = result.fetchall() + finally: + result.close() + if not rows: + return None + row = rows[0] + return models.Author( + id=_row_value(row, "id", 0), + name=_row_value(row, "name", 1), + ) diff --git a/internal/endtoend/testdata/migrations/migrations/001_create.sql b/internal/endtoend/testdata/migrations/migrations/001_create.sql new file mode 100644 index 0000000..a34ac89 --- /dev/null +++ b/internal/endtoend/testdata/migrations/migrations/001_create.sql @@ -0,0 +1,11 @@ +-- +goose Up +CREATE TABLE authors ( + id Uint64 NOT NULL, + old_bio Utf8, + PRIMARY KEY (id) +); +CREATE TABLE obsolete (id Uint64 NOT NULL, PRIMARY KEY (id)); + +-- +goose Down +DROP TABLE authors; +DROP TABLE obsolete; diff --git a/internal/endtoend/testdata/migrations/migrations/002_alter.sql b/internal/endtoend/testdata/migrations/migrations/002_alter.sql new file mode 100644 index 0000000..e8dd76a --- /dev/null +++ b/internal/endtoend/testdata/migrations/migrations/002_alter.sql @@ -0,0 +1,6 @@ +-- +goose Up +ALTER TABLE authors ADD COLUMN name Utf8, DROP COLUMN old_bio; +DROP TABLE obsolete; + +-- +goose Down +ALTER TABLE authors DROP COLUMN name; diff --git a/internal/endtoend/testdata/migrations/migrations/003_cleanup.down.sql b/internal/endtoend/testdata/migrations/migrations/003_cleanup.down.sql new file mode 100644 index 0000000..37916eb --- /dev/null +++ b/internal/endtoend/testdata/migrations/migrations/003_cleanup.down.sql @@ -0,0 +1 @@ +DROP TABLE authors; diff --git a/internal/endtoend/testdata/migrations/queries.sql b/internal/endtoend/testdata/migrations/queries.sql new file mode 100644 index 0000000..f525f55 --- /dev/null +++ b/internal/endtoend/testdata/migrations/queries.sql @@ -0,0 +1,3 @@ +-- name: GetAuthor :one +DECLARE $id AS Uint64; +SELECT * FROM authors WHERE id = $id; diff --git a/internal/endtoend/testdata/migrations/sqlc.yaml b/internal/endtoend/testdata/migrations/sqlc.yaml new file mode 100644 index 0000000..a85a70d --- /dev/null +++ b/internal/endtoend/testdata/migrations/sqlc.yaml @@ -0,0 +1,12 @@ +version: "2" +sql: + - engine: ydb + schema: migrations + queries: queries.sql + gen: + go: + out: db + sql_package: ydb + python: + out: py + runtime: sqlalchemy From b5e3cf605e972a5a4740bd733e0f05a727a5417b Mon Sep 17 00:00:00 2001 From: Aleksey Myasnikov Date: Mon, 7 Sep 2026 22:48:10 +0300 Subject: [PATCH 7/7] feat: prepare standalone generator for manual releases --- .agents/README.md | 10 + .agents/context.md | 41 +++ .agents/decisions.md | 23 ++ .github/workflows/ci.yml | 1 + .github/workflows/publish.yml | 239 +++++++++++++ .gitignore | 1 + AGENTS.md | 71 ++++ CHANGELOG.md | 29 ++ Makefile | 7 +- README.md | 16 +- docs/architecture.md | 6 + docs/compatibility.md | 42 ++- docs/development.md | 4 + docs/implementation-contract.md | 26 -- docs/java-research.md | 111 ------ docs/java.md | 11 +- docs/provenance.md | 36 ++ docs/release-plan.md | 124 +++++++ docs/releasing.md | 120 +++++++ docs/roadmap.md | 31 +- docs/targets.md | 9 +- .../authors/go/database/sql/queries.sql.go | 26 +- examples/authors/go/go.mod | 18 +- examples/authors/go/go.sum | 68 ++-- examples/authors/go/native/models.go | 6 +- examples/authors/go/native/queries.sql.go | 26 +- examples/authors/python/dbapi/queries.py | 23 +- examples/authors/python/native/queries.py | 29 +- examples/authors/python/sqlalchemy/queries.py | 27 +- examples/authors/sqlc.yaml | 3 - internal/analyzer/analyzer.go | 9 + internal/analyzer/analyzer_test.go | 198 +++++++++++ internal/analyzer/catalog.go | 15 +- internal/analyzer/literal.go | 138 ++++++++ internal/analyzer/semantic.go | 106 +++--- internal/cli/cli.go | 59 +++- internal/cli/cli_test.go | 79 +++++ internal/cli/outputs.go | 79 +++++ internal/codegen/csharp/generator.go | 22 +- internal/codegen/csharp/generator_test.go | 2 + internal/codegen/golang/generate.go | 72 ++-- internal/codegen/golang/generate_test.go | 61 +++- internal/codegen/java/generator.go | 32 +- internal/codegen/java/generator_test.go | 1 + internal/codegen/python/generator.go | 321 +++++++----------- internal/codegen/python/generator_test.go | 61 +++- internal/config/config.go | 16 +- internal/config/config_test.go | 1 + internal/endtoend/golden_test.go | 26 +- .../authors/expected/db/queries.sql.go | 6 +- .../testdata/authors/expected/py/queries.py | 17 +- .../join_alias/expected/db/queries.sql.go | 6 +- .../join_alias/expected/py/queries.py | 19 +- .../migrations/expected/db/queries.sql.go | 6 +- .../migrations/expected/py/queries.py | 17 +- internal/model/model.go | 7 +- scripts/release | 241 +++++++++++++ scripts/release-targets | 6 + scripts/release-version.py | 282 +++++++++++++++ scripts/test_release_version.py | 248 ++++++++++++++ 60 files changed, 2590 insertions(+), 747 deletions(-) create mode 100644 .agents/README.md create mode 100644 .agents/context.md create mode 100644 .agents/decisions.md create mode 100644 .github/workflows/publish.yml create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md delete mode 100644 docs/implementation-contract.md delete mode 100644 docs/java-research.md create mode 100644 docs/release-plan.md create mode 100644 docs/releasing.md create mode 100644 internal/analyzer/literal.go create mode 100644 internal/cli/outputs.go create mode 100755 scripts/release create mode 100644 scripts/release-targets create mode 100644 scripts/release-version.py create mode 100644 scripts/test_release_version.py diff --git a/.agents/README.md b/.agents/README.md new file mode 100644 index 0000000..d8c48b3 --- /dev/null +++ b/.agents/README.md @@ -0,0 +1,10 @@ +# Project memory + +- [Context](context.md): repository map and sources of truth. +- [Decisions](decisions.md): durable choices and their rationale. +- [AGENTS.md](../AGENTS.md): editing and verification rules. + +Keep these notes small. Update existing entries when a decision changes; do not +append a transcript of each task. Implementation details, commands and user-facing +behavior belong in the linked code and documentation. Recheck live repository, +dependency and CI state rather than recording it here as a lasting fact. diff --git a/.agents/context.md b/.agents/context.md new file mode 100644 index 0000000..cfc4b43 --- /dev/null +++ b/.agents/context.md @@ -0,0 +1,41 @@ +# Context + +sqlc-ydb generates typed application code from YQL for YDB in a standalone Go +binary. It follows the familiar sqlc workflow while maintaining its own source +and release cycle. It is a development implementation with explicit coverage +limits; successful parsing alone does not establish semantic support. + +## Code map + +| Area | Responsibility | +| --- | --- | +| `cmd/sqlc-ydb`, `internal/cli` | Commands, pipeline orchestration, output validation and file IO | +| `internal/config` | Strict config parsing, supported options and defaults | +| `internal/source` | Input ordering and migration Up sections | +| `internal/analyzer` | Direct YQL parse contexts, catalog evolution, name/type resolution and diagnostics | +| `internal/model` | Resolved query/catalog data shared by generators; not an AST | +| `internal/codegen/{golang,python,cpp,csharp,java}` | Language naming and SDK-specific bindings, decoding and source rendering | +| `internal/endtoend` | CLI fixtures, expected diagnostics and generated golden files | +| `examples/authors` | Shared schema/config with language-specific dependencies and executable examples | +| `.github/workflows` | Offline verification and sequential acceptance steps per host | + +## Sources of truth + +- [README](../README.md): build and first generation. +- [Compatibility](../docs/compatibility.md): supported config, queries, schema + migrations, intentional exclusions and output ownership. +- [Architecture](../docs/architecture.md): current stages and responsibilities. +- [Targets](../docs/targets.md), [C++](../docs/cpp.md), [C#](../docs/csharp.md), + [Java](../docs/java.md): generated API and runtime contracts. +- [Development](../docs/development.md): commands and validation requirements. +- [Roadmap](../docs/roadmap.md): shared macros and deferred database-assisted analysis. +- [Release plan](../docs/release-plan.md): release gates, ydb.tech documentation, + external query corpus, and user-owned SDK reviews/consumer pilots. +- [Releasing](../docs/releasing.md): packaging, dry runs and publication workflow. +- [Changelog](../CHANGELOG.md): pending Unreleased entries and published stable versions. +- [Provenance](../docs/provenance.md): upstream references and inspected SDK sources. + +The Go module name is authoritative in `go.mod`; the executable is `sqlc-ydb`. +Historical repository/module names can differ. Do not infer a rename or restore +the old engine-plugin dependencies from an archive branch. Check the current +Git branch, remote and worktree before any publication. diff --git a/.agents/decisions.md b/.agents/decisions.md new file mode 100644 index 0000000..83be610 --- /dev/null +++ b/.agents/decisions.md @@ -0,0 +1,23 @@ +# Decisions + +These choices constrain maintenance; implementation details remain in the linked +documents. Revisit a decision explicitly rather than letting a local workaround +change the architecture. + +| Decision | Reason and reference | +| --- | --- | +| Independent YDB-only implementation | Compatibility concerns user workflow, not upstream internal code or Git history. See [compatibility](../docs/compatibility.md). | +| Keep semantic analysis; no intermediate AST | Direct ANTLR contexts avoid a second syntax representation while resolved types remain necessary for code generation. See [architecture](../docs/architecture.md). | +| Built-in generators only | New language support belongs in this repository; external engine/codegen/WASM/process plugins are deliberately excluded. See [compatibility](../docs/compatibility.md). | +| One modern C# ADO.NET target | The official SDK exposes ADO.NET already; a second nominally native profile would duplicate it. See [C#](../docs/csharp.md). | +| SQL-first Java framework adapters | Typed query methods and projection records fit JdbcTemplate and Hibernate JDBC callbacks. Do not infer ORM entities from arbitrary SQL. See [Java](../docs/java.md). | +| Shared macro processing before generators | Language count must not multiply SQL semantic work. A separate compiler package is optional; macros are still planned. See [roadmap](../docs/roadmap.md). | +| Offline generation by default | Database-assisted analysis is deferred until a concrete need defines the API and semantics. See [roadmap](../docs/roadmap.md). | +| Sequential local-ydb validation per host | Concurrent images, runtime suites and container builds have exceeded available memory. See [development](../docs/development.md). | +| User guide on ydb.tech near release | Keep technical references and examples here; publish the consumer journey with verified upstream recipes and explicit limits on the YDB site. See [release plan](../docs/release-plan.md). | +| Consumer acceptance in 0.x before 1.0.0 | The user arranges SDK reviews, production-query corpus access and real-project pilots; implementation work addresses the resulting findings. See [release plan](../docs/release-plan.md). | +| Manual publication from accumulated changelog entries | The maintainer chooses the version part in the Actions form. The workflow assigns the version and checks all artifacts before pushing the release commit/tag; RCs preserve pending notes. See [releasing](../docs/releasing.md). | + +SDK-specific behavior should be reviewed with the SDK maintainers when its public +contract is unclear. They are available within the product team; invented fallback +behavior is not a substitute for establishing that contract. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ee232c..df5658c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: dotnet-version: '8.0.x' - name: Build and test standalone generator run: | + make test-release go test -p 1 ./... make build - name: Compile all generated scalar bindings against published SDKs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..f9342c4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,239 @@ +name: publish + +on: + workflow_dispatch: + inputs: + version-change: + description: Version part (use PATCH for the first 0.0.1 release) + required: true + type: choice + options: [PATCH, MINOR, MAJOR] + release-candidate: + description: Release candidate (create a draft prerelease) + required: true + type: boolean + default: true + dry-run: + description: Build and verify without pushing commits, tags or releases + required: true + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: sqlc-ydb-publish + cancel-in-progress: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.prepare.outputs.tag }} + commit: ${{ steps.prepare.outputs.commit }} + steps: + - name: Check release branch + env: + DRY_RUN: ${{ inputs.dry-run }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + if [[ "$DRY_RUN" != true && "$GITHUB_REF" != "refs/heads/$DEFAULT_BRANCH" ]]; then + echo "Publish from the default branch; use dry-run to check another branch." >&2 + exit 1 + fi + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + cache: maven + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + - name: Prepare version and changelog locally + id: prepare + env: + VERSION_PART: ${{ inputs.version-change }} + RELEASE_CANDIDATE: ${{ inputs.release-candidate }} + run: | + python3 scripts/release-version.py prepare \ + --part "$VERSION_PART" --rc "$RELEASE_CANDIDATE" \ + --notes "$RUNNER_TEMP/release-notes.md" >"$RUNNER_TEMP/release.json" + release_tag=$(jq -r .tag "$RUNNER_TEMP/release.json") + if [[ "$RELEASE_CANDIDATE" != true ]]; then + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add CHANGELOG.md internal/cli/cli.go + git commit -m "Release $release_tag" + fi + release_commit=$(git rev-parse HEAD) + echo "tag=$release_tag" >>"$GITHUB_OUTPUT" + echo "commit=$release_commit" >>"$GITHUB_OUTPUT" + echo "RELEASE_TAG=$release_tag" >>"$GITHUB_ENV" + echo "RELEASE_COMMIT=$release_commit" >>"$GITHUB_ENV" + git bundle create "$RUNNER_TEMP/release-source.bundle" HEAD + printf '### %s\n\nSource: `%s`\n\nRelease commit: `%s`\n' \ + "$release_tag" "$GITHUB_SHA" "$release_commit" >>"$GITHUB_STEP_SUMMARY" + - name: Run standalone checks + run: make check + - name: Build and verify all six archives sequentially + run: | + scripts/release check "$RELEASE_TAG" + mkdir -p dist + printf '%s\n' "$RELEASE_COMMIT" >dist/COMMIT + while read -r release_os release_arch; do + scripts/release build "$RELEASE_TAG" "$RELEASE_COMMIT" "$release_os" "$release_arch" dist + done dist/COMMIT + scripts/release verify "$RELEASE_TAG" dist scripts/release-targets + + publish: + if: ${{ !inputs.dry-run }} + needs: [prepare, smoke] + runs-on: ubuntu-latest + permissions: + contents: write + env: + RELEASE_TAG: ${{ needs.prepare.outputs.tag }} + RELEASE_COMMIT: ${{ needs.prepare.outputs.commit }} + RELEASE_BRANCH: ${{ github.event.repository.default_branch }} + RELEASE_CANDIDATE: ${{ inputs.release-candidate }} + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/download-artifact@v4 + with: + name: release-source + path: ${{ runner.temp }}/release-source + - uses: actions/download-artifact@v4 + with: + name: release-bundle + path: dist + - name: Push the verified commit and tag + run: | + git fetch "$RUNNER_TEMP/release-source/release-source.bundle" HEAD + git checkout --detach "$RELEASE_COMMIT" + test "$(git rev-parse HEAD)" = "$RELEASE_COMMIT" + branch_head=$(git ls-remote --exit-code origin "refs/heads/$RELEASE_BRANCH" | cut -f1) + if [[ "$branch_head" != "$GITHUB_SHA" ]]; then + echo "The release branch moved while artifacts were being checked; start a new run." >&2 + exit 1 + fi + if gh release view "$RELEASE_TAG" >/dev/null 2>&1; then + echo "release already exists: $RELEASE_TAG" >&2 + exit 1 + fi + git tag "$RELEASE_TAG" "$RELEASE_COMMIT" + git push --atomic origin \ + "$RELEASE_COMMIT:refs/heads/$RELEASE_BRANCH" \ + "refs/tags/$RELEASE_TAG:refs/tags/$RELEASE_TAG" + - name: Upload verified assets and finish the release + run: | + release_flags=(--draft) + if [[ "$RELEASE_CANDIDATE" == true ]]; then + release_flags+=(--prerelease) + fi + gh release create "$RELEASE_TAG" \ + dist/*.tar.gz dist/*.zip dist/SHA256SUMS \ + "${release_flags[@]}" --verify-tag --title "$RELEASE_TAG" \ + --notes-file "$RUNNER_TEMP/release-source/release-notes.md" + if [[ "$RELEASE_CANDIDATE" != true ]]; then + gh release edit "$RELEASE_TAG" --draft=false --latest + fi diff --git a/.gitignore b/.gitignore index b4087d5..7b8a2b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /bin/ +/dist/ __pycache__/ *.pyc .venv/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ce957c0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,71 @@ +# Working on sqlc-ydb + +Read [.agents/context.md](.agents/context.md) for the code map and links to the +current contracts. Inspect the working tree before editing; preserve unrelated +changes. Repository documentation is in English. + +## Implementation rules + +- Keep the pipeline in [architecture](docs/architecture.md): direct ANTLR YQL + contexts, semantic analysis, one resolved result, built-in generators. Do not + introduce an intermediate AST, engine registry or external plugin protocol. +- Unsupported syntax, types and options must produce actionable errors. Do not + guess a type, substitute a default value or silently ignore an option to make + generation succeed. Defaults and SDK compatibility branches need a documented + contract and a test that distinguishes them from an error. +- Prefer small cohesive functions and explicit control flow. Extract duplicated + policy only when it has the same meaning and reason to change. Runtime binding, + decoding and transaction ownership differ across SDKs; similar-looking code is + not sufficient reason to build a common adapter framework. +- Keep semantic SQL work before generation. Future macros must resolve once per + compilation unit, independently of the number of generators. Driver placeholder + rendering must preserve SQL strings, comments, identifiers and local bindings. +- Validate generated identifiers and collisions. Preserve query text exactly in + readable multiline literals, including delimiters, Unicode and control bytes. + Generated code must use the actual SDK row/binding contract, not guessed object + shapes or fallback values. Make resource and transaction ownership explicit. +- Keep runtime SDK dependencies in tests/examples, out of the generator's module + imports. Do not add local sibling `replace` directives. Verify SDK APIs against + pinned dependencies and record source evidence in [provenance](docs/provenance.md). + +## Verification + +- For a bug, first add a regression that demonstrates the wrong behavior. Assert + meaningful results or diagnostics, not the implementation's internal steps. +- Run the affected package tests while editing; run `make check` for the integrated + change. Commands and prerequisites are in [development](docs/development.md). +- When output changes intentionally, regenerate examples with `make generate` and + update [golden fixtures](internal/endtoend/README.md). Review the generated diff; + do not hand-edit generated files or accept a baseline just to make tests pass. +- For SDK binding, row decoding or transaction changes, compile generated code + against the pinned SDK and run the relevant execution test. Rendering tests and + mocks alone do not establish SDK compatibility. SQL literal tests must evaluate + generated literals with the language's compiler/runtime and compare the original bytes. +- Run local-ydb images, live runtime suites and Docker builds **sequentially per + host**. Keep `go test -p 1` for live suites and no `t.Parallel` in them. Use an + isolated disposable database; never stop unrelated containers. Prefer Linux CI + for heavy acceptance checks on memory-constrained development machines. + +## Documentation and project memory + +- User documentation is planned for ydb.tech near release. Keep technical + compatibility contracts and SDK references here; once the site guide exists, + link it rather than maintaining two full user guides. Contributor commands + belong in [development](docs/development.md). Update the canonical page when + behavior changes, then link to it elsewhere. +- Release preparation is tracked in [the release plan](docs/release-plan.md). + Keep SDK maintainer reviews and consumer pilots assigned to the user. Preparing + artifacts or changelog entries does not authorize creating a tag or publishing. + Accumulate consumer-facing changes under `Unreleased`; the manual publish + workflow assigns version headings. The first release describes capabilities, + not fixes to development iterations that were never released. +- Remove dead branches, obsolete plans and comments that merely narrate code. + Keep comments that explain a constraint or non-obvious SDK behavior. Do not add + speculative abstractions or LLM task assignments to production documentation. +- `.agents/` is a small maintained project memory: code map, lasting decisions and + pointers. Update it when boundaries or decisions change. Do not copy reference + docs, generated APIs, dependency versions, machine paths or transient CI state + into it. Mark planned work as planned; tests and source establish what exists. +- When adapting upstream source or tests, preserve license notices and record the + exact source commit. This is an independent implementation, not a source fork + requiring upstream merges. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1212746 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +Add changes under Unreleased. The [publish workflow](docs/releasing.md) assigns +the version and moves these entries into a numbered section at release time. + +## Unreleased + +### Added + +- Standalone YDB-only CLI with `generate`, `compile`, `diff`, `init`, and + `version`; `version --verbose` also reports the commit embedded by release builds. +- Direct ANTLR YQL parsing, semantic analysis, and supported schema migration + operations applied to an in-memory catalog. +- Built-in Go (native SDK, database/sql), Python (native SDK, DB-API, SQLAlchemy), + C++ (native SDK, userver), C# (ADO.NET), and Java (native SDK, JDBC, Spring JDBC, + Hibernate) generators. +- Shared authors examples, exact generated-output fixtures, SQL literal + round-trip tests, SDK compilation checks, and sequential live-YDB acceptance. +- Release packaging for Linux, macOS, and Windows on amd64 and arm64, with + SHA256 checksums and version/commit metadata. +- Diagnostics for unsupported expressions, types and options, generated name + collisions, and obsolete generated files in current output directories. + +### Compatibility + +- This is an independent implementation of the sqlc workflow. Only YDB and + built-in generators are supported; external engine/codegen plugins are excluded. +- The initial version implements a documented subset of YQL and sqlc options, + not every upstream recipe. See [compatibility](docs/compatibility.md). diff --git a/Makefile b/Makefile index 536f9c2..40e8129 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test generate check clean +.PHONY: build test test-release generate check clean build: go build -trimpath -o bin/sqlc-ydb ./cmd/sqlc-ydb @@ -6,10 +6,13 @@ build: test: go test -p 1 ./... +test-release: + python3 -m unittest discover -s scripts -p 'test_*.py' + generate: go run ./cmd/sqlc-ydb generate -f examples/authors/sqlc.yaml -check: +check: test-release go test -p 1 ./... go run ./cmd/sqlc-ydb diff -f examples/authors/sqlc.yaml diff --git a/README.md b/README.md index e779fae..6606769 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ Generate typed Go, Python, C++, C#, and Java query code from YQL for YDB. One ex the parser, semantic analyzer, and generators. Generation works offline and does not require a running YDB, Python, or any separately installed codegen plugin. -This is the first standalone development version. The previous engine-plugin +This is the first standalone development version, preparing for release 0.0.1. +No release is implied by the version number; see the [changelog](CHANGELOG.md) +and [release plan](docs/release-plan.md). The previous engine-plugin implementation is preserved in `archive/engine-plugins-2026-09-07`. ## Quick start @@ -56,6 +58,10 @@ Use `sqlc-ydb init` for a starting configuration. Input and output paths are relative to the configuration file. `generate` completes analysis and rendering before writing any files; `compile` writes nothing; `diff` writes nothing and exits with status 1 if generated contents differ. +Renamed queries or models can leave obsolete generated files: `generate` and +`diff` report these for manual removal in their current output directories. +See [output ownership](docs/compatibility.md#output-ownership) when moving outputs +or sharing directories between configurations. ## Design and compatibility @@ -75,3 +81,11 @@ See [compatibility](docs/compatibility.md), [targets](docs/targets.md), the implemented scope and remaining work. Target-specific configuration and examples are described in [C++](docs/cpp.md), [C#](docs/csharp.md), and [Java](docs/java.md). + +The user guide is planned for the SQLC section of ydb.tech near release. The +repository currently contains the quick start, technical references and executable +examples; [the release plan](docs/release-plan.md) tracks site documentation and +consumer acceptance. + +For repository work, start with [AGENTS.md](AGENTS.md) and the +[project context](.agents/context.md). diff --git a/docs/architecture.md b/docs/architecture.md index 156eeb3..7388dfd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,6 +21,9 @@ another recursive syntax tree. identity, parameters, result sets and source locations. Nullability is an `Optional` type, and compound type metadata is retained. A table catalog and a query projection are distinct: `SELECT name` does not generate the whole table. +`AnalyzedQuery.SQL` retains executable YQL and its declarations. Parameter names +omit the leading `$`; their types and result column types must be resolved. +`analyzer.Analyze` returns an error whenever its result contains diagnostics. The language packages in `internal/codegen` produce files from that resolved model. They handle naming, runtime-specific parameter binding, result @@ -32,6 +35,9 @@ semantic query analysis and must preserve strings, comments and identifiers. prepared before writes start. Each file is replaced through a temporary sibling; this protects individual files from interrupted writes, but does not promise a filesystem transaction covering every output. +Before writing or comparing, the CLI checks current output directories for +obsolete files with the sqlc-ydb generated header. It reports those files for +manual removal; see [output ownership](compatibility.md#output-ownership). ## Where is the compiler? diff --git a/docs/compatibility.md b/docs/compatibility.md index 1b64d31..5d485b0 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -21,6 +21,9 @@ internal data structures and source history are not a dependency. ## Implemented workflow - `generate`, `compile`, `diff`, `init`, `version`, `--help`, `-f` / `--file`. +- `version` prints one version string; `version --verbose` also prints the + commit embedded by release builds. Ordinary source builds report `unknown` + unless the commit is supplied through linker flags. - `sqlc.yaml`, `sqlc.yml`, and `sqlc.json`; config version 2 and the basic version 1 Go `packages` format. Paths resolve relative to the configuration. - File paths, lists, nonrecursive directories, and ordinary glob patterns. @@ -36,9 +39,11 @@ internal data structures and source history are not a dependency. affected-row count. - Go options `package`, `out`, `sql_package`, `emit_json_tags`, `emit_interface`, `emit_empty_slices`. The default `sql_package` is `database/sql`, matching sqlc. -- Python options `package`, `out`, `runtime`, `emit_sync_querier`, +- Python options `out`, `runtime`, `emit_sync_querier`, `emit_async_querier`. Synchronous generation defaults to enabled; requesting asynchronous generation currently fails explicitly. + The Python package directory is selected by `out`; remove `gen.python.package` + from older configurations. That option was ignored and now produces an error. - C++ options `namespace`, `out`, `runtime`; C# options `namespace`, `out`; Java options `package`, `out`, `runtime`. These are built-in extensions to the sqlc configuration shape, not external plugin options. @@ -50,14 +55,31 @@ internal data structures and source history are not a dependency. This development version does not claim complete sqlc compatibility. Type/name overrides, the full generator option inventory, sqlc macros, batch commands, `vet`, `verify`, cloud/remote workflows and live database-assisted analysis still -need implementation. They are not successful no-op commands. Generated files no -longer produced by a query set are not automatically deleted. +need implementation. They are not successful no-op commands. Query and type coverage evolves independently of configuration compatibility. Unsupported YQL must be diagnosed by the analyzer; a resolved type unsupported by a language adapter is a generation error. Each supported behavior needs a fixture and, for runtime-sensitive behavior, an execution test. +## Output ownership + +Use separate output directories for independently invoked configurations. Several +generators in one configuration may share a directory if their filenames do not +collide. Files written by sqlc-ydb carry a generated header and are overwritten +by `generate`; handwritten files with other names are retained. + +If a query file or model is renamed or removed, `generate` and `diff` report +obsolete files with the sqlc-ydb header in the current output directories and +exit with status 1. Remove the listed files and rerun. No output is written +before this check passes. `diff` also reports missing or changed expected files. + +This check covers regular files directly in directories the current generation +writes. It does not follow unrelated symlinks, scan nested packages, or remember +previously configured output directories. When changing `out` or removing a +generator entirely, clean up its old directory yourself. `compile` does not +inspect outputs. Files are never automatically deleted. + ## Current analyzer coverage The analyzer supports explicit `CREATE TABLE` catalogs and the schema migration @@ -66,12 +88,22 @@ projections and `*`, table/column aliases, supported joins and their optional sides, `COUNT`, `DECLARE`, direct comparison parameter inference, selected scalar local bindings, `INSERT`/`UPSERT ... VALUES`, `UPDATE ... SET`, `DELETE`, and `RETURNING`. It validates names outside the projection and conflicting parameter -constraints. Diagnostics include source file, line and column. +constraints. Diagnostics include source file, line and column. Table, column, +alias and parameter names are case-sensitive, as in YQL. + +Direct scalar literal projections retain their YQL types, including integer +width/signedness, `Float` versus `Double`, and `String` versus `Utf8`. Integer +suffixes and ranges follow the [YQL lexical rules](https://ydb.tech/docs/en/yql/reference/syntax/lexer). +Non-column projections need an explicit `AS` name. A compound expression such +as `$value = 1ul` is rejected instead of inheriting the parameter's type. This is a deliberately limited first semantic implementation. General computed projections and casts, CTEs/subqueries, multiple result sets, FLATTEN, full function/type inference and the full YQL -grammar semantics are subsequent work. Accepted syntax is not a claim of full +grammar semantics are subsequent work. Unary numeric expressions in projections +or local assignments and backslash escapes in quoted identifiers are also +explicitly rejected until their YQL semantics are implemented. `EXPLAIN` cannot +be used as a named data query. Accepted syntax is not a claim of full equivalence to the YDB server's type checker. Current INSERT/UPSERT VALUES and UPDATE SET values must be direct parameters; diff --git a/docs/development.md b/docs/development.md index f50266f..cf8cb6a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,9 @@ # Development +Release artifact builds and the manual workflow's dry-run mode are described in +[releasing](releasing.md); product readiness and responsibilities are in +[the release plan](release-plan.md). + ```sh make test make build diff --git a/docs/implementation-contract.md b/docs/implementation-contract.md deleted file mode 100644 index 00ec27d..0000000 --- a/docs/implementation-contract.md +++ /dev/null @@ -1,26 +0,0 @@ -# Initial standalone implementation - -The module path follows the actual repository: `github.com/ydb-platform/sqlc-engine-ydb`. -The executable is `sqlc-ydb`. The only SQL dialect is YQL for YDB. - -Pipeline: config and source loading → ANTLR YQL parse tree → semantic analyzer → -`model.AnalysisResult` → built-in Go/Python generators → files. -There is no intermediate AST, plugin protocol, external generator, or sqlc dependency. - -Shared interfaces for parallel implementation: - -- `analyzer.Analyze(schema, queries []model.Source) (*model.AnalysisResult, error)`. -- `golang.Generate(*model.AnalysisResult, golang.Options) ([]model.File, error)`. - Options: `Package, Runtime string; EmitJSONTags, EmitInterface, EmitEmptySlices bool`. - Runtime values: `ydb` and `database/sql`. -- `python.Generate(*model.AnalysisResult, python.Options) ([]model.File, error)`. - Options: `Package, Runtime string; EmitSyncQuerier, EmitAsyncQuerier bool`. - Runtime values: `ydb`, `dbapi`, `sqlalchemy`. - -`AnalyzedQuery.SQL` contains executable YQL, including required declarations. -Parameter names do not include `$`. Column and parameter types must be resolved. -Unsupported syntax/types/commands must fail explicitly; never substitute `Any`. -Generators consume only semantic results and do not walk the ANTLR tree. -Generated files must use safe language string literals and deterministic ordering. - -Each implementation owns its directory; root owns this model, config, CLI and integration. diff --git a/docs/java-research.md b/docs/java-research.md deleted file mode 100644 index 134d339..0000000 --- a/docs/java-research.md +++ /dev/null @@ -1,111 +0,0 @@ -# Java target research - -Checked 2026-09-07 against these current default-branch snapshots: - -| repository | commit | -| --- | --- | -| `ydb-platform/ydb-java-sdk` | `98aab7828816c9b92cd7583c3383865b834da0af` | -| `ydb-platform/ydb-jdbc-driver` | `a2a43af922ae90b01341a116a6cac81364656b24` | -| `ydb-platform/ydb-java-dialects` | `ddd81338501c074f93671914fe914aa1addca3a5` | - -The example pins published artifacts rather than these source snapshots: -SDK BOM `2.4.11`, JDBC `2.4.1`, Hibernate dialect `1.7.0`. Maven Central metadata -on 2026-09-07 reports these as latest/release values. -The Spring Data JDBC dialect is intentionally absent: the generated Spring -profile uses `JdbcTemplate`, not Spring Data repository support. - -## Final generated architecture - -All four profiles emit a final `Queries` class and top-level Java records. -Methods are lower camel case and follow SQL command cardinality: -`Optional` for `:one`, `List` for `:many`, and `void` for `:exec`. -`Uint64` is a Java `long` carrying the original 64-bit pattern. Nullable -`Utf8` is `String`; nullable output is guarded with the driver's `wasNull()` -for JDBC paths and optional-item checks for native paths. - -The constructors borrow application resources: - -* native: `Queries(SessionRetryContext)`; -* JDBC: `Queries(Connection)`; -* Spring: `Queries(JdbcTemplate)`, using `JdbcTemplate.execute` with a - `ConnectionCallback`; -* Hibernate: `Queries(Session)`, using `Session.doReturningWork` and the same - typed JDBC operations. Query projections are records, not generated JPA - entities. - -Generated JDBC, Spring, and Hibernate methods prepare the original declared -YQL, unwrap `tech.ydb.jdbc.YdbPreparedStatement`, and bind `author_id`, -`author_name`, and `biography` by name (the setter adds `$`). This follows the current driver's -`YdbPreparedStatement` API and avoids relying on positional order. The driver -source in `jdbc/src/main/java/tech/ydb/jdbc/query/params/PreparedQuery.java` -(lines 42-73) sorts indexed `$pN` parameters first and then other names; this is -why generated code uses name setters. `MappingSetters.castToUint64` in -`jdbc/src/main/java/tech/ydb/jdbc/common/MappingSetters.java` (lines 370-418) -passes a `Long` to `PrimitiveValue.newUint64`, preserving `-1L` as `2^64-1`. -Generated setters pass a concrete SDK `Value`, which is explicitly handled -by both `SimpleJdbcPrm.setValue` and `ValueFactory.readValue`. This preserves -unsigned and optional types for both declared and inferred parameters. The -custom `setObject(name, object, Type)` overload is not used: its implementation -does not use the supplied `Type` argument. - -SDK `Uint8`, `Uint16`, and `Uint32` constructors mask their signed Java carrier. -Generated methods reject negative or oversized values before calling any SDK -or JDBC method; `Uint64` intentionally preserves all bits of `long`. - -## Verified native SDK path - -The published SDK API used by the native profile is: - -```java -try (GrpcTransport transport = GrpcTransport.forConnectionString(dsn).build(); - QueryClient client = QueryClient.newClient(transport).build()) { - SessionRetryContext retry = SessionRetryContext.create(client).build(); - QueryReader reader = retry.supplyResult(session -> - QueryReader.readFrom(session.createQuery(sql, TxMode.SERIALIZABLE_RW, params))) - .join().getValue(); -} -``` - -The exact classes are in `query/src/main/java/tech/ydb/query/QueryClient.java`, -`QuerySession.java`, `QueryStream.java`, and -`tools/{QueryReader,SessionRetryContext}.java`; `TxMode` is in -`common/src/main/java/tech/ydb/common/transaction/TxMode.java`. DDL in the -smoke uses `TxMode.NONE`; reads and writes use the appropriate query transaction -mode. `QueryReader.getResultSetCount/getResultSet` return -`ResultSetReader`; its `next`, `getColumn`, and `ValueReader` getters decode -rows. `PrimitiveValue.newUint64(long)`, `newText(String)`, and -`OptionalType.emptyValue/newValue` are the verified parameter factories. - -The caller closes transport and query client. `SessionRetryContext` creates and -closes per-operation query sessions internally, so generated methods borrow the -retry context and never close it. - -## Framework notes - -The SQL-first JVM reference is sqlc's own -[Kotlin JDBC output](https://github.com/sqlc-dev/sqlc-gen-kotlin/blob/2c6a78075b1b9a075427b403a07b187bc36e7451/examples/src/main/kotlin/com/example/authors/postgresql/QueriesImpl.kt): -query constants, typed methods/results, a borrowed `Connection`, and owned -prepared statements. The new Java implementation follows that shape without -copying the Kotlin implementation or its plugin protocol. Its `:one` behavior -matches this project's Go/Python adapters (first row), rather than Kotlin's -additional multiple-row check. - -Spring's documented -[JdbcTemplate callbacks](https://docs.spring.io/spring-framework/reference/data-access/jdbc/core.html) -provide connection management and exception translation for handwritten SQL. -Hibernate's documented -[doReturningWork](https://docs.hibernate.org/orm/6.6/javadocs/org/hibernate/SharedSessionContract.html#doReturningWork(org.hibernate.jdbc.ReturningWork)) -provides JDBC access using the session's connection. These are the framework -integration points selected here; inferring JPA entities from SQL projections -is not part of this generator. - -Spring's generated API stays SQL first and works with `JdbcTemplate`; the smoke -uses `SingleConnectionDataSource` only to make connection ownership explicit. -Hibernate uses its native JDBC connection callback, so no entity mapping is -needed for a manual SQL projection. The YDB Hibernate 6 dialect class checked -in the current dialect source is `tech.ydb.hibernate.dialect.YdbDialect`. - -The four smoke programs read and execute `examples/authors/schema.sql`, then -drop `authors` only after their own successful create. They require -`SQLC_YDB_TEST_DSN` and are intended to run sequentially against a disposable -database. They do not run automatically during Maven compilation. diff --git a/docs/java.md b/docs/java.md index 9dd3386..122acad 100644 --- a/docs/java.md +++ b/docs/java.md @@ -75,11 +75,6 @@ SQL uses Java 17 text blocks with escaped delimiters, control characters, and trailing whitespace. Literal tests compile and execute the emitted Java and compare exact UTF-8 bytes with the original SQL. -The native implementation is based on the current SDK source snapshot -`98aab7828816c9b92cd7583c3383865b834da0af`: -`GrpcTransport.forConnectionString(url).build()`, -`QueryClient.newClient(transport).build()`, and -`SessionRetryContext.create(client).build()`. The JDBC snapshot is -`a2a43af922ae90b01341a116a6cac81364656b24`; the dialect snapshot is -`ddd81338501c074f93671914fe914aa1addca3a5`. Exact source paths and published -version notes are in [java-research.md](java-research.md). +The inspected SDK sources and framework API references are recorded in +[source provenance](provenance.md#java-sdk-and-framework-references). +Dependency versions used by the example are pinned in [its Maven build](../examples/authors/java/pom.xml). diff --git a/docs/provenance.md b/docs/provenance.md index e4bf5d3..43544b3 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -18,3 +18,39 @@ The parser is an external dependency from `ydb-platform/yql-parsers`, pinned to `d9544073fd13d17b30f609fa4fe7b034cd28ba02`). Dependency license notices remain in their modules. If upstream source code or tests are copied in future changes, record the source commit and preserve the applicable notices alongside them. + +## Java SDK and framework references + +Source snapshots inspected on 2026-09-07: + +| Repository | Commit | +| --- | --- | +| [YDB Java SDK](https://github.com/ydb-platform/ydb-java-sdk) | `98aab7828816c9b92cd7583c3383865b834da0af` | +| [YDB JDBC driver](https://github.com/ydb-platform/ydb-jdbc-driver) | `a2a43af922ae90b01341a116a6cac81364656b24` | +| [YDB Java dialects](https://github.com/ydb-platform/ydb-java-dialects) | `ddd81338501c074f93671914fe914aa1addca3a5` | + +These snapshots explain API choices; published dependencies used by tests are +pinned in the [example Maven build](../examples/authors/java/pom.xml). +The [Java guide](java.md) defines the generated API and resource ownership. + +- The SQL-first reference is sqlc's [Kotlin JDBC output](https://github.com/sqlc-dev/sqlc-gen-kotlin/blob/2c6a78075b1b9a075427b403a07b187bc36e7451/examples/src/main/kotlin/com/example/authors/postgresql/QueriesImpl.kt): + query constants, typed results and a borrowed connection. Its implementation + and plugin protocol were not copied. Our `:one` returns the first row, as the + other sqlc-ydb adapters do; it does not add Kotlin's multiple-row check. +- Native query execution uses `QueryClient`, `QuerySession`, + `tools/SessionRetryContext` and `tools/QueryReader` in the SDK's `query` module. + `SessionRetryContext` owns operation sessions; the caller owns the transport + and client. Parameters use `PrimitiveValue` and `OptionalType` factories. +- JDBC's `query/params/PreparedQuery.java` sorts indexed `$pN` parameters before + other names. The generated code binds by name through `YdbPreparedStatement` + to avoid depending on positional order. It supplies SDK `Value` objects, + handled by `SimpleJdbcPrm.setValue` and `ValueFactory.readValue`, to preserve + unsigned and optional types. The inspected `setObject(name, object, Type)` + overload ignores its `Type` argument and is deliberately not used. +- SDK constructors for `Uint8/16/32` mask the signed Java carrier. Generated + range checks prevent truncation; `Uint64` intentionally retains every bit + of a Java `long`. +- Spring [JdbcTemplate callbacks](https://docs.spring.io/spring-framework/reference/data-access/jdbc/core.html) + and Hibernate [doReturningWork](https://docs.hibernate.org/orm/6.6/javadocs/org/hibernate/SharedSessionContract.html#doReturningWork(org.hibernate.jdbc.ReturningWork)) + provide the selected SQL execution APIs. The generator does not infer JPA + entities from query projections or require Spring Data repository support. diff --git a/docs/release-plan.md b/docs/release-plan.md new file mode 100644 index 0000000..2aac395 --- /dev/null +++ b/docs/release-plan.md @@ -0,0 +1,124 @@ +# Release plan + +The first version is **0.0.1**. Preparing its code, changelog and release workflow +does not create a tag or publish a release. Consumer pilots and feedback belong +in versions below 1.0.0; 1.0.0 follows evidence from real projects. + +This page owns release work and responsibilities. Current behavior is in +[compatibility](compatibility.md), and compiler feature design remains in +[the compiler roadmap](roadmap.md). Packaging commands are in +[releasing](releasing.md). Planned checks below are not claims of +completed acceptance. + +## Before the first public release + +| Work | Owner | Completion evidence | +| --- | --- | --- | +| Reliable analysis and generation | Implementation work | Regression tests for expressions, literals, nullability, names, column order and output ownership; unsupported cases fail explicitly. Known limitations remain documented. | +| Current Go SDK compatibility | Implementation work | Resolve the latest published stable ydb-go-sdk/v3, pin it in tests/examples, compile both native and database/sql outputs, and run their binding tests. Do not add runtime SDK imports to the generator module. | +| SDK maintainer review | User | Arrange reviews with the SDK maintainers in the team, collect their findings and agree which runtime profiles are ready for users. Implementation work addresses the findings. | +| Reproducible artifacts | Implementation work | Build and verify the six release archives, checksums, version and commit; run the release workflow without publishing first. Exercise the packaged executable, not only `go run`. | +| Module/repository naming | Implementation work | Resolve the historical `sqlc-engine-ydb` Go module path before the first tag; update internal imports, build scripts and references together if renamed. | +| User documentation on ydb.tech | Implementation work, near release | A reviewed SQLC section with a reproducible installation-to-query path and explicit compatibility boundaries, tested with release-candidate artifacts. | +| Publish the first release | User decision; prepared workflow | Start the manual publish workflow after a successful dry run. It assigns the version, prepares the changelog and creates the tag after artifact checks. No tag is created as part of preparing this plan. | + +All selected SDK profiles need their applicable compilation and acceptance +checks. Fixes already made during the maintainability review cover several +blockers, but do not establish complete YQL semantics. Local-ydb images, runtime +suites and Docker builds remain sequential on each host. + +## User documentation on ydb.tech + +Publish the user guide in the SQLC section of ydb.tech when the release is ready +or close to ready. Introduce sqlc-ydb as the recommended YDB-specific variant +with the familiar sqlc workflow. Make its independent implementation and +supported subset clear. Link upstream documentation for verified applicable +recipes; describe YDB-specific or incompatible cases locally. Do not promise +that every upstream recipe works before compatibility tests establish that. + +The first user journey should cover: + +1. Select and download a platform archive; verify its checksum and version. +2. Write schema, named YQL queries and `sqlc.yaml`; generate a minimal project. +3. Install the chosen SDK dependencies and execute the generated methods. +4. Use parameters, optional values, result cardinality and transactions correctly. +5. Apply schema migration inputs, regenerate outputs and check them in CI. +6. Diagnose unsupported queries/options, naming conflicts and obsolete files. +7. Move from upstream sqlc configuration or the old YDB plugin configuration. + +Use checked examples to establish which upstream recipes work, need YDB changes, +or are unsupported. A successful parse is not sufficient evidence. Include the +tested sqlc-ydb/SDK versions and link to their compatibility scope. + +Keep architecture, contributor commands, SDK API evidence, technical contracts +and executable examples in this repository. Until the site guide exists, retain +the README quick start and target references. Once published, link the site from +the README rather than maintaining two complete user guides. + +## External production-query corpus + +The user may provide more than one million YDB queries. Corpus availability and +context are prerequisites, not assumed access. Initially keep this corpus and +its detailed reports outside the repository. Decide later whether selected, +minimized cases can become repository regressions. + +**User responsibilities:** provide an approved input location and stable query +IDs, identify the available schema snapshots and parameter declarations, and +define the scope of the run. Query text alone is enough for a parser pass. +Semantic checks additionally need schema context, syntax settings and parameter +types; generation needs a query name, command/cardinality and selected runtime +options. Supply these as metadata when the original SQL has no sqlc annotations. +Parameter values, credentials and result rows are not needed. + +**Future implementation work, once inputs are available:** + +- Run separate stages: parse, local analysis, generation, SDK compilation. An + optional server type check requires separate API research and explicit opt-in; + the runner must not execute production queries. +- Start with a representative small sample, then a larger sample, then the full + corpus. Stream input, use bounded workers and batches, cache immutable catalogs + by schema/context hash, and checkpoint progress. Do not start a process or + rebuild the schema catalog for every query. A long-lived worker process can + provide hard timeout/RSS limits and crash isolation where in-process cancellation + is insufficient. +- Reuse the current analyzer and generators. Extract schema and query phases from + `analyzer.Analyze` only when needed for catalog reuse, preserving its existing + entry point. No second AST, plugin interface or universal DBMS harness is needed. +- Record a JSONL outcome for every input ID and stage: success, missing context, + unsupported feature, semantic error, generation error, SDK compilation error, + timeout, resource limit, crash or infrastructure failure. Introduce stable + diagnostic codes rather than grouping by human-readable error strings. +- Record code/parser/SDK/compiler versions, query/context/config hashes, limits + and run ID. Preserve the input denominator, including skipped queries and + failures. Deduplicate execution only when all stage inputs match exactly; + expand the outcome back to each original ID. Similar-query grouping is for + investigation, not a substitute for checking those inputs. +- Compile generated code in bounded shards against each selected SDK and narrow + failing shards to individual cases. Report results per stage and runtime; + parsing a million queries does not prove correct inferred types or runtime behavior. +- Turn confirmed implementation defects into small regression tests when their + inputs can be shared. Do not publish the original corpus by default. + +Today the relevant public commands are `compile`, `generate` and `diff`. +`compile` performs local semantic analysis; `vet`, `check` and `parse` are not +implemented CLI commands. The first corpus runner can call internal stages +without inventing aliases or changing that public contract. + +Complete generation of the corpus is a long-term coverage target. Report the +current unsupported subset honestly; do not weaken diagnostics to increase the +success rate. Remaining work includes computed expressions, casts, function +typing, CTEs/subqueries, multiple result sets, broader DDL and type coverage, +nullability evidence and schema drift. The current inventory is in +[compatibility](compatibility.md). + +## Consumer acceptance before 1.0.0 + +**User responsibilities:** select two or three real consumer projects, arrange +SDK maintainer reviews and coordinate installation/use from the user documentation +during 0.x releases. Collect concrete API, compatibility and documentation findings +and decide whether the product is ready for 1.0.0. + +**Implementation work:** reproduce reported failures, fix confirmed defects, +add shareable minimized regressions, update documentation and publish fixes +through the prepared release process when authorized. Do not mark this gate +complete based only on the repository's authors example or a large parser pass. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..e70df69 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,120 @@ +# Releasing + +Required reviews, documentation and consumer pilots are tracked in +[the release plan](release-plan.md). Releases are started manually from +**Actions → publish → Run workflow**. A tag push does not start a release. + +## Changelog and version selection + +Accumulate consumer-visible changes under `## Unreleased` in +[CHANGELOG.md](../CHANGELOG.md). Describe the first release's capabilities; +reserve `Fixed` for corrections to behavior in published versions. Do not add +the next version heading or change the CLI version by hand. + +The form has three inputs: + +| Input | Effect | +| --- | --- | +| Version part | `PATCH` increments the patch number; `MINOR` increments the minor number and resets patch; `MAJOR` increments major and resets both. Choose based on the accumulated changes and compatibility impact. | +| Release candidate | Enabled by default. Creates the next `-rcN` tag and a **draft prerelease** with all binaries. Leaves the source version and Unreleased entries unchanged. | +| Dry run | Builds and checks everything without pushing commits, tags or GitHub Releases. Disabled by default. Enable it to rehearse publication. | + +The first release requires `PATCH` and produces **0.0.1**, or **0.0.1-rc0** with +Release candidate enabled. Subsequent RCs use the highest existing suffix plus +one. After the first stable release, the selected part is incremented from the +last stable version; RC tags do not advance that base. + +The maintainer chooses the version part. The workflow computes the number, but +does not infer compatibility impact from prose. Empty Unreleased sections, +inconsistent source/history versions and existing target tags fail preparation. + +## Publication sequence + +1. Complete the release gates, including the applicable SDK and sequential YDB + acceptance checks. Resolve the historical module path before the first tag. +2. Select the intended branch in the form and enable **Dry run**. A rehearsal + can use a development branch; publication requires the default branch. +3. The workflow extracts release notes. For a stable release it updates the CLI + version, moves pending notes under `## vVERSION`, leaves an empty Unreleased + section, and creates a local release commit. RCs use the selected source commit. +4. It runs `make check`, builds the six archives sequentially, verifies their + contents and checksums, and runs each packaged executable on its native + OS/architecture runner. A Git bundle carries the exact prepared commit to + those jobs, including the stable version and changelog changes. +5. Inspect the rehearsal, then start **publish** from the default branch with + **Dry run** disabled. Releasing a stable version also requires clearing + **Release candidate**. This run repeats the checks before publication. +6. After all checks pass, the workflow verifies that the branch has not moved, + then pushes the prepared commit and tag together. It uploads the archives and + checksums to a draft. Stable releases become public after upload; RC releases + remain drafts, as in the SDK workflow. + +Publication uses the repository's `GITHUB_TOKEN` with `contents: write`. Branch +rules must permit its release commit; the workflow does not bypass protection. +A branch update during verification requires a new run. Tag pushes made by this +workflow do not need to trigger another workflow: packaging and verification +already ran in the same invocation. + +Existing tags and releases are not overwritten. If an upload fails after the +push, the tag and possibly a draft remain. Inspect the failed run and recover +using its verified artifacts; do not rerun version selection expecting it to +repair the same release automatically. + +## Artifacts + +The release matrix is exactly: + +| Platform | Architectures | Archive | +| --- | --- | --- | +| Linux | amd64, arm64 | `.tar.gz` | +| macOS (`darwin` in artifact names) | amd64, arm64 | `.tar.gz` | +| Windows | amd64, arm64 | `.zip` | + +Each archive contains `sqlc-ydb` (or `sqlc-ydb.exe`) and `LICENSE` in a directory +named `sqlc-ydb_VERSION_OS_ARCH`. `SHA256SUMS` covers all six archives. Builds use +`CGO_ENABLED=0`; applications using generated code still need their SDKs. + +`sqlc-ydb version` prints the version, including any RC suffix. +`sqlc-ydb version --verbose` also prints the embedded commit. Release builds +carry the exact prepared source commit. Ordinary source builds report `unknown` +for the commit unless linker flags supply it. + +## Local checks + +Version/changelog tests use Python's standard library and are part of +`make check`; they can also run separately with `make test-release`. +Packaging requires Bash, Python 3.9+, Go, `tar`, `zip`/`unzip`, and `sha256sum` +or `shasum`. From a clean repository checkout, preview the next RC locally: + +```sh +release_work=$(mktemp -d) +python3 scripts/release-version.py prepare --part PATCH --rc true \ + --notes "$release_work/notes.md" >"$release_work/plan.json" +release_tag=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["tag"])' "$release_work/plan.json") +release_commit=$(git rev-parse HEAD) +mkdir -p "$release_work/dist" +printf '%s\n' "$release_commit" >"$release_work/dist/COMMIT" +while read -r release_os release_arch; do + scripts/release build "$release_tag" "$release_commit" "$release_os" "$release_arch" "$release_work/dist" +done result; database/sql cannot scan this driver value and rejects List diff --git a/examples/authors/go/database/sql/queries.sql.go b/examples/authors/go/database/sql/queries.sql.go index 940fdf0..5e5baac 100644 --- a/examples/authors/go/database/sql/queries.sql.go +++ b/examples/authors/go/database/sql/queries.sql.go @@ -7,21 +7,21 @@ import ( "database/sql" ) -const getAuthor = `-- name: GetAuthor :one +const queryGetAuthor = `-- name: GetAuthor :one DECLARE $author_id AS Uint64; SELECT id, name, bio FROM authors WHERE id = $author_id;` -func (q *Queries) GetAuthor(ctx context.Context, author_id uint64) (GetAuthorRow, error) { +func (q *Queries) GetAuthor(ctx context.Context, arg uint64) (GetAuthorRow, error) { var row GetAuthorRow - err := q.db.QueryRowContext(ctx, getAuthor, sql.Named("author_id", author_id)).Scan(&row.ID, &row.Name, &row.Bio) + err := q.db.QueryRowContext(ctx, queryGetAuthor, sql.Named("author_id", arg)).Scan(&row.ID, &row.Name, &row.Bio) return row, err } -const listAuthors = `-- name: ListAuthors :many +const queryListAuthors = `-- name: ListAuthors :many SELECT id, name, bio FROM authors ORDER BY id;` func (q *Queries) ListAuthors(ctx context.Context) ([]ListAuthorsRow, error) { - rows, err := q.db.QueryContext(ctx, listAuthors) + rows, err := q.db.QueryContext(ctx, queryListAuthors) if err != nil { return []ListAuthorsRow(nil), err } @@ -37,17 +37,17 @@ func (q *Queries) ListAuthors(ctx context.Context) ([]ListAuthorsRow, error) { return items, rows.Err() } -const getAuthorName = `-- name: GetAuthorName :one +const queryGetAuthorName = `-- name: GetAuthorName :one DECLARE $author_id AS Uint64; SELECT name FROM authors WHERE id = $author_id;` -func (q *Queries) GetAuthorName(ctx context.Context, author_id uint64) (GetAuthorNameRow, error) { +func (q *Queries) GetAuthorName(ctx context.Context, arg uint64) (GetAuthorNameRow, error) { var row GetAuthorNameRow - err := q.db.QueryRowContext(ctx, getAuthorName, sql.Named("author_id", author_id)).Scan(&row.Name) + err := q.db.QueryRowContext(ctx, queryGetAuthorName, sql.Named("author_id", arg)).Scan(&row.Name) return row, err } -const upsertAuthor = `-- name: UpsertAuthor :exec +const queryUpsertAuthor = `-- name: UpsertAuthor :exec DECLARE $author_id AS Uint64; DECLARE $author_name AS Utf8; DECLARE $biography AS Optional; @@ -55,15 +55,15 @@ UPSERT INTO authors (id, name, bio) VALUES ($author_id, $author_name, $biography);` func (q *Queries) UpsertAuthor(ctx context.Context, arg UpsertAuthorParams) error { - _, err := q.db.ExecContext(ctx, upsertAuthor, sql.Named("author_id", arg.AuthorID), sql.Named("author_name", arg.AuthorName), sql.Named("biography", arg.Biography)) + _, err := q.db.ExecContext(ctx, queryUpsertAuthor, sql.Named("author_id", arg.AuthorID), sql.Named("author_name", arg.AuthorName), sql.Named("biography", arg.Biography)) return err } -const deleteAuthor = `-- name: DeleteAuthor :exec +const queryDeleteAuthor = `-- name: DeleteAuthor :exec DECLARE $author_id AS Uint64; DELETE FROM authors WHERE id = $author_id;` -func (q *Queries) DeleteAuthor(ctx context.Context, author_id uint64) error { - _, err := q.db.ExecContext(ctx, deleteAuthor, sql.Named("author_id", author_id)) +func (q *Queries) DeleteAuthor(ctx context.Context, arg uint64) error { + _, err := q.db.ExecContext(ctx, queryDeleteAuthor, sql.Named("author_id", arg)) return err } diff --git a/examples/authors/go/go.mod b/examples/authors/go/go.mod index 8967a79..a3ba1a8 100644 --- a/examples/authors/go/go.mod +++ b/examples/authors/go/go.mod @@ -2,18 +2,18 @@ module example.com/sqlc-ydb-authors go 1.26.0 -require github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1 +require github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1 require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/grpc v1.69.4 // indirect - google.golang.org/protobuf v1.35.1 // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect ) diff --git a/examples/authors/go/go.sum b/examples/authors/go/go.sum index 0446668..eb3300a 100644 --- a/examples/authors/go/go.sum +++ b/examples/authors/go/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= @@ -51,8 +51,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -70,20 +70,22 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1 h1:YaqzRVbcncabB34YNjOl5ADomYUFva+6l74svIIIJUo= -github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 h1:7VKlOrBIQ8L8acJ9wFCt6lWzLDbOhc05VLpL31743tU= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1 h1:T+fB2ZDHpYIGC7DWjK+rzZHLoexBF0zG/n3Q9DGNskY= +github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1/go.mod h1:dJXJ1u00IqO8Vsph8fWmYj8b1E7jyphnvJPqA69emBY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= @@ -101,28 +103,28 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -130,14 +132,16 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -145,8 +149,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -160,8 +164,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/examples/authors/go/native/models.go b/examples/authors/go/native/models.go index 1b58035..9e798ed 100644 --- a/examples/authors/go/native/models.go +++ b/examples/authors/go/native/models.go @@ -26,9 +26,9 @@ type UpsertAuthorParams struct { } type Querier interface { - GetAuthor(ctx context.Context, author_id uint64) (GetAuthorRow, error) + GetAuthor(ctx context.Context, arg uint64) (GetAuthorRow, error) ListAuthors(ctx context.Context) ([]ListAuthorsRow, error) - GetAuthorName(ctx context.Context, author_id uint64) (GetAuthorNameRow, error) + GetAuthorName(ctx context.Context, arg uint64) (GetAuthorNameRow, error) UpsertAuthor(ctx context.Context, arg UpsertAuthorParams) error - DeleteAuthor(ctx context.Context, author_id uint64) error + DeleteAuthor(ctx context.Context, arg uint64) error } diff --git a/examples/authors/go/native/queries.sql.go b/examples/authors/go/native/queries.sql.go index 00ea27c..a0bdc3d 100644 --- a/examples/authors/go/native/queries.sql.go +++ b/examples/authors/go/native/queries.sql.go @@ -8,12 +8,12 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3/query" ) -const getAuthor = `-- name: GetAuthor :one +const queryGetAuthor = `-- name: GetAuthor :one DECLARE $author_id AS Uint64; SELECT id, name, bio FROM authors WHERE id = $author_id;` -func (q *Queries) GetAuthor(ctx context.Context, author_id uint64) (GetAuthorRow, error) { - result, err := q.db.QueryRow(ctx, getAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(author_id).Build())) +func (q *Queries) GetAuthor(ctx context.Context, arg uint64) (GetAuthorRow, error) { + result, err := q.db.QueryRow(ctx, queryGetAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(arg).Build())) if err != nil { return GetAuthorRow{}, err } @@ -24,11 +24,11 @@ func (q *Queries) GetAuthor(ctx context.Context, author_id uint64) (GetAuthorRow return row, nil } -const listAuthors = `-- name: ListAuthors :many +const queryListAuthors = `-- name: ListAuthors :many SELECT id, name, bio FROM authors ORDER BY id;` func (q *Queries) ListAuthors(ctx context.Context) ([]ListAuthorsRow, error) { - result, err := q.db.QueryResultSet(ctx, listAuthors) + result, err := q.db.QueryResultSet(ctx, queryListAuthors) if err != nil { return make([]ListAuthorsRow, 0), err } @@ -47,12 +47,12 @@ func (q *Queries) ListAuthors(ctx context.Context) ([]ListAuthorsRow, error) { return items, nil } -const getAuthorName = `-- name: GetAuthorName :one +const queryGetAuthorName = `-- name: GetAuthorName :one DECLARE $author_id AS Uint64; SELECT name FROM authors WHERE id = $author_id;` -func (q *Queries) GetAuthorName(ctx context.Context, author_id uint64) (GetAuthorNameRow, error) { - result, err := q.db.QueryRow(ctx, getAuthorName, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(author_id).Build())) +func (q *Queries) GetAuthorName(ctx context.Context, arg uint64) (GetAuthorNameRow, error) { + result, err := q.db.QueryRow(ctx, queryGetAuthorName, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(arg).Build())) if err != nil { return GetAuthorNameRow{}, err } @@ -63,7 +63,7 @@ func (q *Queries) GetAuthorName(ctx context.Context, author_id uint64) (GetAutho return row, nil } -const upsertAuthor = `-- name: UpsertAuthor :exec +const queryUpsertAuthor = `-- name: UpsertAuthor :exec DECLARE $author_id AS Uint64; DECLARE $author_name AS Utf8; DECLARE $biography AS Optional; @@ -71,13 +71,13 @@ UPSERT INTO authors (id, name, bio) VALUES ($author_id, $author_name, $biography);` func (q *Queries) UpsertAuthor(ctx context.Context, arg UpsertAuthorParams) error { - return q.db.Exec(ctx, upsertAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(arg.AuthorID).Param("$author_name").Text(arg.AuthorName).Param("$biography").BeginOptional().Text(arg.Biography).EndOptional().Build())) + return q.db.Exec(ctx, queryUpsertAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(arg.AuthorID).Param("$author_name").Text(arg.AuthorName).Param("$biography").BeginOptional().Text(arg.Biography).EndOptional().Build())) } -const deleteAuthor = `-- name: DeleteAuthor :exec +const queryDeleteAuthor = `-- name: DeleteAuthor :exec DECLARE $author_id AS Uint64; DELETE FROM authors WHERE id = $author_id;` -func (q *Queries) DeleteAuthor(ctx context.Context, author_id uint64) error { - return q.db.Exec(ctx, deleteAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(author_id).Build())) +func (q *Queries) DeleteAuthor(ctx context.Context, arg uint64) error { + return q.db.Exec(ctx, queryDeleteAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$author_id").Uint64(arg).Build())) } diff --git a/examples/authors/python/dbapi/queries.py b/examples/authors/python/dbapi/queries.py index 7fc3689..99a8c9a 100644 --- a/examples/authors/python/dbapi/queries.py +++ b/examples/authors/python/dbapi/queries.py @@ -30,15 +30,6 @@ def _typed(value, typ): return (value, typ) -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - class Querier: def __init__(self, connection): @@ -54,9 +45,9 @@ def get_author(self, author_id: int) -> Optional[models.Author]: return None row = rows[0] return models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row[0], + name=row[1], + bio=row[2], ) finally: cursor.close() @@ -68,9 +59,9 @@ def list_authors(self) -> Iterable[models.Author]: cursor.execute(SQL_LIST_AUTHORS, parameters) rows = cursor.fetchall() return (models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row[0], + name=row[1], + bio=row[2], ) for row in rows) finally: cursor.close() @@ -85,7 +76,7 @@ def get_author_name(self, author_id: int) -> Optional[models.GetAuthorNameRow]: return None row = rows[0] return models.GetAuthorNameRow( - name=_row_value(row, "name", 0), + name=row[0], ) finally: cursor.close() diff --git a/examples/authors/python/native/queries.py b/examples/authors/python/native/queries.py index a927156..0e44f88 100644 --- a/examples/authors/python/native/queries.py +++ b/examples/authors/python/native/queries.py @@ -27,15 +27,6 @@ DELETE FROM authors WHERE id = $author_id;""" -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - def _typed(value, typ): return ydb.TypedValue(value, typ) @@ -47,35 +38,35 @@ def __init__(self, pool: ydb.QuerySessionPool): def get_author(self, author_id: int) -> Optional[models.Author]: parameters = {"$author_id": _typed(author_id, ydb.PrimitiveType.Uint64)} result_sets = self._pool.execute_with_retries(SQL_GET_AUTHOR, parameters) - rows = result_sets[0].rows if result_sets else [] + rows = result_sets[0].rows if not rows: return None row = rows[0] return models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row["id"], + name=row["name"], + bio=row["bio"], ) def list_authors(self) -> Iterable[models.Author]: parameters = {} result_sets = self._pool.execute_with_retries(SQL_LIST_AUTHORS, parameters) - rows = result_sets[0].rows if result_sets else [] + rows = result_sets[0].rows return (models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row["id"], + name=row["name"], + bio=row["bio"], ) for row in rows) def get_author_name(self, author_id: int) -> Optional[models.GetAuthorNameRow]: parameters = {"$author_id": _typed(author_id, ydb.PrimitiveType.Uint64)} result_sets = self._pool.execute_with_retries(SQL_GET_AUTHOR_NAME, parameters) - rows = result_sets[0].rows if result_sets else [] + rows = result_sets[0].rows if not rows: return None row = rows[0] return models.GetAuthorNameRow( - name=_row_value(row, "name", 0), + name=row["name"], ) def upsert_author(self, author_id: int, author_name: str, biography: Optional[str]) -> None: diff --git a/examples/authors/python/sqlalchemy/queries.py b/examples/authors/python/sqlalchemy/queries.py index ae18605..ba6d969 100644 --- a/examples/authors/python/sqlalchemy/queries.py +++ b/examples/authors/python/sqlalchemy/queries.py @@ -5,9 +5,6 @@ import ydb from sqlalchemy import text from sqlalchemy.engine import Connection -def _typed(value, typ): - return (value, typ) - SQL_GET_AUTHOR = """-- name\\: GetAuthor \\:one DECLARE $author_id AS Uint64; @@ -32,14 +29,8 @@ def _typed(value, typ): DELETE FROM authors WHERE id = :author_id;""" -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) +def _typed(value, typ): + return (value, typ) class Querier: @@ -57,9 +48,9 @@ def get_author(self, author_id: int) -> Optional[models.Author]: return None row = rows[0] return models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row._mapping["id"], + name=row._mapping["name"], + bio=row._mapping["bio"], ) def list_authors(self) -> Iterable[models.Author]: @@ -70,9 +61,9 @@ def list_authors(self) -> Iterable[models.Author]: finally: result.close() return (models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row._mapping["id"], + name=row._mapping["name"], + bio=row._mapping["bio"], ) for row in rows) def get_author_name(self, author_id: int) -> Optional[models.GetAuthorNameRow]: @@ -86,7 +77,7 @@ def get_author_name(self, author_id: int) -> Optional[models.GetAuthorNameRow]: return None row = rows[0] return models.GetAuthorNameRow( - name=_row_value(row, "name", 0), + name=row._mapping["name"], ) def upsert_author(self, author_id: int, author_name: str, biography: Optional[str]) -> None: diff --git a/examples/authors/sqlc.yaml b/examples/authors/sqlc.yaml index d350e68..a85dbcb 100644 --- a/examples/authors/sqlc.yaml +++ b/examples/authors/sqlc.yaml @@ -13,7 +13,6 @@ sql: emit_interface: true emit_empty_slices: true python: - package: authors out: python/native runtime: ydb csharp: @@ -37,7 +36,6 @@ sql: out: go/database/sql sql_package: database/sql python: - package: authors out: python/dbapi runtime: dbapi java: @@ -50,7 +48,6 @@ sql: queries: queries.sql gen: python: - package: authors out: python/sqlalchemy runtime: sqlalchemy emit_sync_querier: true diff --git a/internal/analyzer/analyzer.go b/internal/analyzer/analyzer.go index e8aca2c..be24fe1 100644 --- a/internal/analyzer/analyzer.go +++ b/internal/analyzer/analyzer.go @@ -96,6 +96,15 @@ func parseYQL(file, text string, lineOffset int) (parsedYQL, []model.Diagnostic) p.RemoveErrorListeners() p.AddErrorListener(listener) tree := p.Sql_query() + tokens.Fill() + for _, token := range tokens.GetAllTokens() { + if token.GetTokenType() == parser.YQLLexerID_QUOTED && strings.Contains(token.GetText(), `\`) { + listener.diagnostics = append(listener.diagnostics, model.Diagnostic{ + Position: model.Position{File: file, Line: lineOffset + token.GetLine(), Column: token.GetColumn() + 1}, + Message: "backslash escapes in quoted identifiers are unsupported", + }) + } + } return parsedYQL{tree: tree}, listener.diagnostics } diff --git a/internal/analyzer/analyzer_test.go b/internal/analyzer/analyzer_test.go index 10bac92..575e2d3 100644 --- a/internal/analyzer/analyzer_test.go +++ b/internal/analyzer/analyzer_test.go @@ -183,6 +183,35 @@ SELECT id FROM authors WHERE id = $id;`}} } } +func TestParseYQLRejectsBackslashEscapesInQuotedIdentifiers(t *testing.T) { + tests := []struct { + name string + text string + line int + column int + }{ + {name: "schema table", text: "CREATE TABLE `bad\\nname` (id Uint64, PRIMARY KEY (id));", line: 1, column: 14}, + {name: "select column", text: "SELECT `bad\\nname` FROM authors;", line: 1, column: 8}, + {name: "result alias", text: "SELECT id AS `bad\\nname` FROM authors;", line: 1, column: 14}, + {name: "bind name", text: "DECLARE $`bad\\nname` AS Uint64;", line: 1, column: 10}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, diagnostics := parseYQL("input.sql", tt.text, 2) + if len(diagnostics) != 1 { + t.Fatalf("diagnostics = %#v, want one", diagnostics) + } + got := diagnostics[0] + if got.Position != (model.Position{File: "input.sql", Line: tt.line + 2, Column: tt.column}) { + t.Fatalf("position = %#v", got.Position) + } + if !strings.Contains(got.Message, "backslash escapes in quoted identifiers are unsupported") { + t.Fatalf("message = %q", got.Message) + } + }) + } +} + func TestAnalyzeRejectsUnsupportedSchemaStatements(t *testing.T) { result, err := Analyze([]model.Source{{Name: "schema.sql", Text: ` CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id)); @@ -203,6 +232,175 @@ SELECT id = 1 AS matches FROM authors;`}}, } } +func TestAnalyzeRejectsCompositeParameterProjectionInsteadOfUsingBindType(t *testing.T) { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: `-- name: Matches :many +DECLARE $value AS Uint64; +SELECT $value = 1ul AS matches FROM authors;`}}, + ) + if err == nil || !strings.Contains(err.Error(), `unsupported result expression "$value=1ul"`) { + t.Fatalf("error = %v", err) + } +} + +func TestAnalyzeKeepsDirectParameterProjection(t *testing.T) { + got, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: `-- name: Echo :many +DECLARE $value AS Uint64; +SELECT $value AS value FROM authors;`}}, + ) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + want := model.Type{Kind: "Uint64"} + if resultType := got.Queries[0].ResultSets[0].Columns[0].Type; !reflect.DeepEqual(resultType, want) { + t.Fatalf("result type = %#v, want %#v", resultType, want) + } +} + +func TestAnalyzeUsesYQLLiteralTypes(t *testing.T) { + tests := []struct { + name string + expr string + kind string + }{ + {name: "default int32", expr: "1", kind: "Int32"}, + {name: "expanded int64", expr: "2147483648", kind: "Int64"}, + {name: "explicit int64", expr: "1l", kind: "Int64"}, + {name: "explicit int16", expr: "1s", kind: "Int16"}, + {name: "explicit int8", expr: "1t", kind: "Int8"}, + {name: "explicit uint64", expr: "18446744073709551615ul", kind: "Uint64"}, + {name: "uppercase explicit uint64", expr: "1UL", kind: "Uint64"}, + {name: "explicit uint32", expr: "1u", kind: "Uint32"}, + {name: "explicit uint16", expr: "1us", kind: "Uint16"}, + {name: "explicit uint8", expr: "1ut", kind: "Uint8"}, + {name: "hex uint8", expr: "0xffut", kind: "Uint8"}, + {name: "uppercase hex prefix", expr: "0Xffut", kind: "Uint8"}, + {name: "default double", expr: "1.5", kind: "Double"}, + {name: "explicit float", expr: "1.5f", kind: "Float"}, + {name: "default string", expr: `"hello"`, kind: "String"}, + {name: "explicit string", expr: `"hello"s`, kind: "String"}, + {name: "utf8", expr: `"hello"u`, kind: "Utf8"}, + {name: "uppercase utf8 suffix", expr: `"hello"U`, kind: "Utf8"}, + {name: "yson", expr: `"[]"y`, kind: "Yson"}, + {name: "json", expr: `"{}"j`, kind: "Json"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Literal :many\nSELECT " + tt.expr + " AS value FROM authors;"}}, + ) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + if kind := got.Queries[0].ResultSets[0].Columns[0].Type.Kind; kind != tt.kind { + t.Fatalf("literal %s type = %s, want %s", tt.expr, kind, tt.kind) + } + }) + } +} + +func TestAnalyzeRejectsUnaryNumericLiteralUntilItsResultTypeIsSupported(t *testing.T) { + for _, expr := range []string{"-1", "+1ul", "-128t"} { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Literal :many\nSELECT " + expr + " AS value FROM authors;"}}, + ) + if err == nil || !strings.Contains(err.Error(), "unsupported result expression") { + t.Errorf("literal %s error = %v", expr, err) + } + } +} + +func TestAnalyzeRejectsIntegerLiteralOutsideItsYQLTypeRange(t *testing.T) { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Literal :many\nSELECT 256ut AS value FROM authors;"}}, + ) + if err == nil || !strings.Contains(err.Error(), `integer literal "256ut" is out of range for Uint8`) { + t.Fatalf("error = %v", err) + } +} + +func TestAnalyzeRejectsLiteralSuffixesWithoutDocumentedModelTypes(t *testing.T) { + for _, expr := range []string{"1p", `"value"p`} { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Literal :many\nSELECT " + expr + " AS value FROM authors;"}}, + ) + if err == nil || !strings.Contains(err.Error(), "unsupported") { + t.Errorf("literal %s error = %v", expr, err) + } + } +} + +func TestAnalyzeUsesLiteralTypeForLocalBinding(t *testing.T) { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: `-- name: Lookup :many +$value = 1ul; +SELECT id FROM authors WHERE id = $value;`}}, + ) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } +} + +func TestAnalyzeRejectsExplainQuery(t *testing.T) { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Explained :many\nEXPLAIN SELECT id FROM authors;"}}, + ) + if err == nil || !strings.Contains(err.Error(), "EXPLAIN is unsupported in named queries") { + t.Fatalf("error = %v", err) + } +} + +func TestAnalyzeTreatsYQLIdentifiersAsCaseSensitive(t *testing.T) { + tests := []struct { + name string + query string + want string + }{ + {name: "table", query: "SELECT ID FROM authors;", want: `unknown table "authors"`}, + {name: "column", query: "SELECT id FROM Authors;", want: `unknown column "id"`}, + {name: "alias", query: "SELECT a.ID FROM Authors AS A;", want: `unknown column "a.ID"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE Authors (ID Uint64 NOT NULL, PRIMARY KEY (ID));`}}, + []model.Source{{Name: "query.sql", Text: "-- name: Lookup :many\n" + tt.query}}, + ) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestAnalyzeTreatsYQLBindNamesAsCaseSensitive(t *testing.T) { + got, err := Analyze( + []model.Source{{Name: "schema.sql", Text: `CREATE TABLE Authors (ID Uint64 NOT NULL, PRIMARY KEY (ID));`}}, + []model.Source{{Name: "query.sql", Text: `-- name: Lookup :many +DECLARE $Value AS Utf8; +SELECT ID FROM Authors WHERE ID = $value;`}}, + ) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + want := []model.Parameter{ + {Name: "Value", Type: model.Type{Kind: "Utf8"}}, + {Name: "value", Type: model.Type{Kind: "Uint64"}}, + } + if !reflect.DeepEqual(got.Queries[0].Parameters, want) { + t.Fatalf("parameters = %#v, want %#v", got.Queries[0].Parameters, want) + } +} + func TestAnalyzeRejectsCastUntilItsNullabilityCanBeProven(t *testing.T) { _, err := Analyze( []model.Source{{Name: "schema.sql", Text: `CREATE TABLE authors (id Uint64 NOT NULL, PRIMARY KEY (id));`}}, diff --git a/internal/analyzer/catalog.go b/internal/analyzer/catalog.go index 3472218..84df980 100644 --- a/internal/analyzer/catalog.go +++ b/internal/analyzer/catalog.go @@ -138,7 +138,7 @@ func applyAlterTableAction(catalog model.Catalog, tableIndex int, table *model.T return []model.Diagnostic{diagnosticAt(file, 0, drop, fmt.Sprintf("column %q does not exist in table %q", name, table.Name))} } for _, key := range table.PrimaryKey { - if strings.EqualFold(key, name) { + if key == name { return []model.Diagnostic{diagnosticAt(file, 0, drop, fmt.Sprintf("cannot drop primary key column %q from table %q", name, table.Name))} } } @@ -161,7 +161,7 @@ func applyAlterTableAction(catalog model.Catalog, tableIndex int, table *model.T func catalogTableIndex(catalog model.Catalog, name string) (int, bool) { for i := range catalog.Tables { - if strings.EqualFold(catalog.Tables[i].Name, name) { + if catalog.Tables[i].Name == name { return i, true } } @@ -170,7 +170,7 @@ func catalogTableIndex(catalog model.Catalog, name string) (int, bool) { func catalogColumnIndex(table model.Table, name string) (int, bool) { for i := range table.Columns { - if strings.EqualFold(table.Columns[i].Name, name) { + if table.Columns[i].Name == name { return i, true } } @@ -211,7 +211,7 @@ func catalogTable(file string, create parser.ICreate_table_stmtContext) (model.T diagnostics = append(diagnostics, diagnosticAt(file, 0, columnContext, err.Error())) continue } - key := strings.ToLower(column.Name) + key := column.Name if columnNames[key] { diagnostics = append(diagnostics, diagnosticAt(file, 0, columnContext, fmt.Sprintf("column %q is declared more than once", column.Name))) continue @@ -228,12 +228,11 @@ func catalogTable(file string, create parser.ICreate_table_stmtContext) (model.T primaryKeyDeclarations++ for _, id := range constraint.AllAn_id() { name := identifier(id.GetText()) - key := strings.ToLower(name) - if primaryKeyNames[key] { + if primaryKeyNames[name] { diagnostics = append(diagnostics, diagnosticAt(file, 0, id, fmt.Sprintf("primary key column %q is declared more than once", name))) continue } - primaryKeyNames[key] = true + primaryKeyNames[name] = true table.PrimaryKey = append(table.PrimaryKey, name) } continue @@ -249,7 +248,7 @@ func catalogTable(file string, create parser.ICreate_table_stmtContext) (model.T diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("table %q declares PRIMARY KEY more than once", table.Name))) } for _, key := range table.PrimaryKey { - if !columnNames[strings.ToLower(key)] { + if !columnNames[key] { diagnostics = append(diagnostics, diagnosticAt(file, 0, create, fmt.Sprintf("primary key column %q does not exist", key))) } } diff --git a/internal/analyzer/literal.go b/internal/analyzer/literal.go new file mode 100644 index 0000000..0aaa333 --- /dev/null +++ b/internal/analyzer/literal.go @@ -0,0 +1,138 @@ +package analyzer + +import ( + "fmt" + "strconv" + "strings" + + "github.com/antlr4-go/antlr/v4" + "github.com/ydb-platform/sqlc-engine-ydb/internal/model" + parser "github.com/ydb-platform/yql-parsers/go" +) + +func literalType(expr parser.IExprContext) (model.Type, bool, error) { + var literals []parser.ILiteral_valueContext + descendants(expr, func(node antlr.Tree) { + if literal, ok := node.(parser.ILiteral_valueContext); ok { + literals = append(literals, literal) + } + }) + if len(literals) != 1 { + return model.Type{}, false, nil + } + literal := literals[0] + expressionText := expr.GetText() + literalText := literal.GetText() + if expressionText != literalText { + return model.Type{}, false, nil + } + switch { + case literal.Bool_value() != nil: + return model.Type{Kind: "Bool"}, true, nil + case literal.STRING_VALUE() != nil: + typeValue, err := stringLiteralType(literal.GetText()) + return typeValue, true, err + case literal.Integer() != nil: + typeValue, err := integerLiteralType(expressionText, literal.Integer().INTEGER_VALUE() != nil) + return typeValue, true, err + case literal.Real_() != nil: + typeValue, err := realLiteralType(expressionText) + return typeValue, true, err + default: + return model.Type{}, false, nil + } +} + +func stringLiteralType(text string) (model.Type, error) { + if len(text) < 2 { + return model.Type{Kind: "String"}, nil + } + prefix, suffix := text[:len(text)-1], text[len(text)-1] + if !(strings.HasSuffix(prefix, "'") || strings.HasSuffix(prefix, "\"") || strings.HasSuffix(prefix, "@@")) { + return model.Type{Kind: "String"}, nil + } + switch strings.ToLower(string(suffix)) { + case "s": + return model.Type{Kind: "String"}, nil + case "u": + return model.Type{Kind: "Utf8"}, nil + case "y": + return model.Type{Kind: "Yson"}, nil + case "j": + return model.Type{Kind: "Json"}, nil + default: + return model.Type{}, fmt.Errorf("unsupported string literal suffix %q", string(text[len(text)-1])) + } +} + +func integerLiteralType(text string, hasSuffix bool) (model.Type, error) { + lower := strings.ToLower(text) + type suffixType struct { + suffix string + kind string + bits int + unsigned bool + } + types := []suffixType{ + {suffix: "ul", kind: "Uint64", bits: 64, unsigned: true}, + {suffix: "us", kind: "Uint16", bits: 16, unsigned: true}, + {suffix: "ut", kind: "Uint8", bits: 8, unsigned: true}, + {suffix: "l", kind: "Int64", bits: 64}, + {suffix: "s", kind: "Int16", bits: 16}, + {suffix: "t", kind: "Int8", bits: 8}, + {suffix: "u", kind: "Uint32", bits: 32, unsigned: true}, + } + for _, candidate := range types { + if !strings.HasSuffix(lower, candidate.suffix) { + continue + } + number, base := integerDigits(text[:len(text)-len(candidate.suffix)]) + var err error + if candidate.unsigned { + _, err = strconv.ParseUint(number, base, candidate.bits) + } else { + _, err = strconv.ParseInt(number, base, candidate.bits) + } + if err != nil { + return model.Type{}, fmt.Errorf("integer literal %q is out of range for %s", text, candidate.kind) + } + return model.Type{Kind: candidate.kind}, nil + } + if hasSuffix { + return model.Type{}, fmt.Errorf("unsupported integer literal suffix in %q", text) + } + number, base := integerDigits(text) + value, err := strconv.ParseInt(number, base, 64) + if err != nil { + return model.Type{}, fmt.Errorf("integer literal %q is out of range for Int64", text) + } + if value >= -1<<31 && value <= 1<<31-1 { + return model.Type{Kind: "Int32"}, nil + } + return model.Type{Kind: "Int64"}, nil +} + +func integerDigits(text string) (string, int) { + lower := strings.ToLower(text) + switch { + case strings.HasPrefix(lower, "0x"): + return text[2:], 16 + case strings.HasPrefix(lower, "0o"): + return text[2:], 8 + case strings.HasPrefix(lower, "0b"): + return text[2:], 2 + default: + return text, 10 + } +} + +func realLiteralType(text string) (model.Type, error) { + kind, bits, number := "Double", 64, text + if strings.HasSuffix(strings.ToLower(text), "f") { + kind, bits, number = "Float", 32, text[:len(text)-1] + } + if _, err := strconv.ParseFloat(number, bits); err != nil { + return model.Type{}, fmt.Errorf("floating-point literal %q is out of range for %s", text, kind) + } + return model.Type{Kind: kind}, nil +} diff --git a/internal/analyzer/semantic.go b/internal/analyzer/semantic.go index dfa9b0a..c84e2e2 100644 --- a/internal/analyzer/semantic.go +++ b/internal/analyzer/semantic.go @@ -3,7 +3,6 @@ package analyzer import ( "fmt" "sort" - "strconv" "strings" "github.com/antlr4-go/antlr/v4" @@ -174,21 +173,19 @@ func validateQueryStatements(block queryBlock, tree queryTree) []model.Diagnosti var diagnostics []model.Diagnostic mainStatements := 0 for _, statement := range tree.statements { - declares, named, data := 0, 0, 0 - descendants(statement, func(node antlr.Tree) { - switch node.(type) { - case *parser.Declare_stmtContext: - declares++ - case *parser.Named_nodes_stmtContext: - named++ - case *parser.Select_coreContext, *parser.Into_table_stmtContext, *parser.Update_stmtContext, *parser.Delete_stmtContext: - data++ - } - }) + core := statement.Sql_stmt_core() + if statement.EXPLAIN() != nil { + diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement, "EXPLAIN is unsupported in named queries")) + continue + } + if core == nil { + diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement, fmt.Sprintf("unsupported statement in named query: %q", statement.GetText()))) + continue + } switch { - case declares == 1 && named == 0 && data == 0: - case named == 1 && declares == 0 && data == 0: - case named == 0 && declares == 0 && data == 1: + case core.Declare_stmt() != nil: + case core.Named_nodes_stmt() != nil: + case core.Select_stmt() != nil, core.Into_table_stmt() != nil, core.Update_stmt() != nil, core.Delete_stmt() != nil: mainStatements++ default: diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement, fmt.Sprintf("unsupported statement in named query: %q", statement.GetText()))) @@ -215,12 +212,11 @@ func declarations(block queryBlock, tree queryTree) (map[string]model.Type, map[ diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, declaration, err.Error())) continue } - key := strings.ToLower(name) - if previous, ok := declared[key]; ok && !sameType(previous, typeValue) { + if previous, ok := declared[name]; ok && !sameType(previous, typeValue) { diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, declaration, fmt.Sprintf("parameter $%s has conflicting DECLARE types %s and %s", name, typeString(previous), typeString(typeValue)))) continue } - declared[key] = typeValue + declared[name] = typeValue } return declared, positions, diagnostics } @@ -239,14 +235,14 @@ func localBindings(block queryBlock, tree queryTree, declared map[string]model.T if bind, ok := node.(*parser.Bind_parameterContext); ok && bind.GetStart() != nil { lhs = append(lhs, bind) positions[bind.GetStart().GetStart()] = true - names[strings.ToLower(bindName(bind))] = true + names[bindName(bind)] = true } }) if len(lhs) != 1 || statement.Expr() == nil { diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement, "only single scalar local assignments are supported")) continue } - name := strings.ToLower(bindName(lhs[0])) + name := bindName(lhs[0]) if _, exists := types[name]; exists { diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement, fmt.Sprintf("local $%s is assigned more than once", name))) continue @@ -258,7 +254,7 @@ func localBindings(block queryBlock, tree queryTree, declared map[string]model.T } }) if len(rhsBinds) == 1 && statement.Expr().GetText() == rhsBinds[0].GetText() { - rhsName := strings.ToLower(bindName(rhsBinds[0])) + rhsName := bindName(rhsBinds[0]) typeValue, ok := types[rhsName] if !ok { typeValue, ok = declared[rhsName] @@ -271,7 +267,10 @@ func localBindings(block queryBlock, tree queryTree, declared map[string]model.T continue } if len(rhsBinds) == 0 { - if typeValue, ok := literalType(statement.Expr().GetText()); ok { + if typeValue, ok, err := literalType(statement.Expr()); err != nil { + diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, statement.Expr(), err.Error())) + continue + } else if ok { types[name] = typeValue continue } @@ -364,7 +363,7 @@ func selectProjection(block queryBlock, selectCore *parser.Select_coreContext, r prefix := strings.TrimSuffix(result.Opt_id_prefix().GetText(), ".") matched := false for _, rel := range relations { - if prefix != "" && !strings.EqualFold(prefix, rel.alias) && !strings.EqualFold(prefix, rel.table.Name) { + if prefix != "" && prefix != rel.alias && prefix != rel.table.Name { continue } matched = true @@ -434,8 +433,8 @@ func expressionColumn(expr parser.IExprContext, relations []relation, declared m binds = append(binds, bind) } }) - if len(refs) == 0 && len(binds) == 1 { - typeValue, ok := declared[strings.ToLower(bindName(binds[0]))] + if len(refs) == 0 && len(binds) == 1 && expr.GetText() == binds[0].GetText() { + typeValue, ok := declared[bindName(binds[0])] if !ok { return model.Column{}, false, fmt.Errorf("cannot resolve type of parameter $%s in result", bindName(binds[0])) } @@ -444,7 +443,9 @@ func expressionColumn(expr parser.IExprContext, relations []relation, declared m if len(refs) != 0 { return model.Column{}, false, fmt.Errorf("computed result expression %q is not supported", expr.GetText()) } - if literal, ok := literalType(expr.GetText()); ok { + if literal, ok, err := literalType(expr); err != nil { + return model.Column{}, false, err + } else if ok { return model.Column{Type: literal}, false, nil } return model.Column{}, false, fmt.Errorf("unsupported result expression %q", expr.GetText()) @@ -487,7 +488,7 @@ func isPureColumnExpression(expr parser.IExprContext) bool { if len(refs) != 1 { return false } - return strings.EqualFold(strings.ReplaceAll(expr.GetText(), "`", ""), qualifiedName(refs[0])) + return expr.GetStart() == refs[0].ctx.GetStart() && expr.GetStop() == refs[0].ctx.GetStop() } func qualifiedName(ref columnRef) string { @@ -536,11 +537,11 @@ func validateColumnReferences(block queryBlock, root antlr.Tree, relations []rel func resolveColumn(relations []relation, ref columnRef) (model.Column, error) { var matches []model.Column for _, rel := range relations { - if ref.qualifier != "" && !strings.EqualFold(ref.qualifier, rel.alias) && !strings.EqualFold(ref.qualifier, rel.table.Name) { + if ref.qualifier != "" && ref.qualifier != rel.alias && ref.qualifier != rel.table.Name { continue } for _, column := range rel.table.Columns { - if strings.EqualFold(column.Name, ref.name) { + if column.Name == ref.name { matches = append(matches, joinedColumn(column, rel.optional)) } } @@ -690,10 +691,9 @@ func inferDirectDMLBind(root antlr.Tree, typeValue model.Type, inferred map[stri } func inferParameter(inferred map[string]model.Type, name string, typeValue model.Type) { - key := strings.ToLower(name) - previous, ok := inferred[key] + previous, ok := inferred[name] if !ok || sameType(previous, typeValue) { - inferred[key] = typeValue + inferred[name] = typeValue return } if previous.Kind == "" { @@ -701,11 +701,11 @@ func inferParameter(inferred map[string]model.Type, name string, typeValue model } if sameType(previous.UnwrapOptional(), typeValue.UnwrapOptional()) { if previous.IsOptional() && !typeValue.IsOptional() { - inferred[key] = typeValue + inferred[name] = typeValue } return } - inferred[key] = model.Type{} + inferred[name] = model.Type{} } func externalParameters(block queryBlock, binds []parser.IBind_parameterContext, declared, inferred map[string]model.Type, declarationPositions, localPositions map[int]bool, localNames map[string]bool) ([]model.Parameter, []model.Diagnostic) { @@ -718,17 +718,16 @@ func externalParameters(block queryBlock, binds []parser.IBind_parameterContext, continue } name := bindName(bind) - key := strings.ToLower(name) - if localNames[key] && !declarationPositions[bind.GetStart().GetStart()] { + if localNames[name] && !declarationPositions[bind.GetStart().GetStart()] { continue } - if seen[key] { + if seen[name] { continue } - seen[key] = true - typeValue, ok := declared[key] + seen[name] = true + typeValue, ok := declared[name] if !ok { - typeValue, ok = inferred[key] + typeValue, ok = inferred[name] } if ok && typeValue.Kind == "" { diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, bind, fmt.Sprintf("external parameter $%s is constrained by incompatible column types", name))) @@ -738,7 +737,7 @@ func externalParameters(block queryBlock, binds []parser.IBind_parameterContext, diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, bind, fmt.Sprintf("cannot resolve type of external parameter $%s; add DECLARE", name))) continue } - if inferredType, inferredOK := inferred[key]; inferredOK && !compatibleTypes(typeValue, inferredType) { + if inferredType, inferredOK := inferred[name]; inferredOK && !compatibleTypes(typeValue, inferredType) { diagnostics = append(diagnostics, diagnosticAt(block.file, block.line-1, bind, fmt.Sprintf("parameter $%s declared as %s but used with %s", name, typeString(typeValue), typeString(inferredType)))) continue } @@ -788,7 +787,7 @@ func simpleTableName(ctx parser.ISimple_table_refContext) string { func findTable(catalog model.Catalog, name string) *model.Table { for i := range catalog.Tables { - if strings.EqualFold(catalog.Tables[i].Name, name) { + if catalog.Tables[i].Name == name { return &catalog.Tables[i] } } @@ -797,7 +796,7 @@ func findTable(catalog model.Catalog, name string) *model.Table { func tableColumn(table *model.Table, name string) *model.Column { for i := range table.Columns { - if strings.EqualFold(table.Columns[i].Name, name) { + if table.Columns[i].Name == name { return &table.Columns[i] } } @@ -830,24 +829,3 @@ func typeString(value model.Type) string { } return value.Kind } - -func literalType(text string) (model.Type, bool) { - lower := strings.ToLower(text) - if lower == "true" || lower == "false" { - return model.Type{Kind: "Bool"}, true - } - if strings.HasPrefix(text, "'") || strings.HasPrefix(text, "\"") { - return model.Type{Kind: "String"}, true - } - if strings.HasPrefix(text, "@@") { - return model.Type{Kind: "String"}, true - } - trimmed := strings.TrimRight(lower, "ulps") - if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil { - return model.Type{Kind: "Int64"}, true - } - if _, err := strconv.ParseFloat(trimmed, 64); err == nil { - return model.Type{Kind: "Double"}, true - } - return model.Type{}, false -} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index d54cf80..780702b 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -22,7 +22,9 @@ import ( "github.com/ydb-platform/sqlc-engine-ydb/internal/source" ) -var Version = "0.1.0-dev" +// Release builds set Version and Commit through linker flags. +var Version = "0.0.1" +var Commit = "unknown" const help = `sqlc-ydb generates typed code from YQL. @@ -34,7 +36,7 @@ Commands: compile Analyze schema and queries without generating files diff Compare generated code with existing files (exit 1 on differences) init Create a sqlc.yaml configuration (--v1 or --v2) - version Print the version + version Print the version (--verbose includes the build commit) Options: -f, --file Use an alternate configuration file @@ -45,6 +47,7 @@ Options: type arguments struct { command, file string v1, help bool + verbose bool } func parseArgs(args []string) (arguments, error) { @@ -73,6 +76,8 @@ func parseArgs(args []string) (arguments, error) { a.v1 = true case arg == "--v2": v2 = true + case arg == "--verbose": + a.verbose = true case strings.HasPrefix(arg, "-"): return a, fmt.Errorf("unknown option %q", arg) default: @@ -88,6 +93,9 @@ func parseArgs(args []string) (arguments, error) { if (a.v1 || v2) && a.command != "init" { return a, errors.New("--v1 and --v2 are only valid for init") } + if a.verbose && a.command != "version" { + return a, errors.New("--verbose is only valid for version") + } return a, nil } @@ -103,6 +111,9 @@ func Run(args []string, stdout, stderr io.Writer) int { } if a.command == "version" { fmt.Fprintln(stdout, Version) + if a.verbose { + fmt.Fprintf(stdout, "commit: %s\n", Commit) + } return 0 } if a.command == "init" { @@ -131,6 +142,10 @@ func Run(args []string, stdout, stderr io.Writer) int { if a.command == "compile" { return 0 } + if err := checkStaleOutputs(files); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } if a.command == "diff" { changed, err := compare(files, stdout) if err != nil { @@ -187,13 +202,6 @@ func prepare(c *config.Config, generate bool) ([]output, error) { if err != nil { return nil, err } - if len(result.Diagnostics) > 0 { - messages := make([]string, 0, len(result.Diagnostics)) - for _, d := range result.Diagnostics { - messages = append(messages, d.Error()) - } - return nil, errors.New(strings.Join(messages, "\n")) - } if !generate { continue } @@ -231,7 +239,7 @@ func prepare(c *config.Config, generate bool) ([]output, error) { } } if p := s.Gen.Python; p != nil { - files, err := python.Generate(result, python.Options{Package: p.Package, Runtime: p.Runtime, EmitSyncQuerier: *p.EmitSyncQuerier, EmitAsyncQuerier: p.EmitAsyncQuerier}) + files, err := python.Generate(result, python.Options{Runtime: p.Runtime, EmitSyncQuerier: *p.EmitSyncQuerier, EmitAsyncQuerier: p.EmitAsyncQuerier}) if err != nil { return nil, fmt.Errorf("Python generation: %w", err) } @@ -342,7 +350,11 @@ func start(s []string) int { } func writeFile(f output) error { - if old, err := os.ReadFile(f.path); err == nil && bytes.Equal(old, f.content) { + old, err := os.ReadFile(f.path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err == nil && bytes.Equal(old, f.content) { return nil } if err := os.MkdirAll(filepath.Dir(f.path), 0755); err != nil { @@ -372,9 +384,30 @@ func initialize(a arguments, w io.Writer) error { if path == "" { path = "sqlc.yaml" } - text := "version: \"2\"\nsql:\n - engine: ydb\n schema: schema.sql\n queries: query.sql\n gen:\n go:\n package: db\n out: db\n sql_package: ydb\n python:\n package: queries\n out: queries\n runtime: ydb\n" + text := `version: "2" +sql: + - engine: ydb + schema: schema.sql + queries: query.sql + gen: + go: + package: db + out: db + sql_package: ydb + python: + out: queries + runtime: ydb +` if a.v1 { - text = "version: \"1\"\npackages:\n - name: db\n engine: ydb\n path: db\n schema: schema.sql\n queries: query.sql\n sql_package: ydb\n" + text = `version: "1" +packages: + - name: db + engine: ydb + path: db + schema: schema.sql + queries: query.sql + sql_package: ydb +` } file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) if errors.Is(err, os.ErrExist) { diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 28f9953..bfd67dc 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -103,6 +103,18 @@ func TestCLIAndInit(t *testing.T) { } } +func TestVersionVerboseIsOnlyAvailableForVersion(t *testing.T) { + if code, out, err := invoke("version"); code != 0 || out != Version+"\n" || err != "" { + t.Fatalf("version: %d %q %q", code, out, err) + } + if code, out, err := invoke("version", "--verbose"); code != 0 || out != Version+"\ncommit: "+Commit+"\n" || err != "" { + t.Fatalf("verbose version: %d %q %q", code, out, err) + } + if code, _, err := invoke("generate", "--verbose"); code != 1 || !strings.Contains(err, "only valid for version") { + t.Fatalf("generate --verbose: %d %q", code, err) + } +} + func TestSymlinkOutputCannotOverwriteConfig(t *testing.T) { dir := t.TempDir() cfg := filepath.Join(dir, "models.go") @@ -121,3 +133,70 @@ func TestSymlinkOutputCannotOverwriteConfig(t *testing.T) { t.Fatal("config overwritten") } } + +func TestRenamedQueryLeavesStaleOutput(t *testing.T) { + dir := t.TempDir() + cfg := filepath.Join(dir, "sqlc.yaml") + put(t, filepath.Join(dir, "schema.sql"), "CREATE TABLE a (id Uint64 NOT NULL, PRIMARY KEY(id));") + put(t, filepath.Join(dir, "queries", "old.sql"), "-- name: GetA :one\nSELECT id FROM a;") + put(t, cfg, "version: '2'\nsql:\n- engine: ydb\n schema: schema.sql\n queries: queries\n gen:\n go:\n out: db\n") + if code, _, err := invoke("generate", "-f", cfg); code != 0 { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(dir, "queries", "old.sql"), filepath.Join(dir, "queries", "new.sql")); err != nil { + t.Fatal(err) + } + stalePath := filepath.Join(dir, "db", "old.sql.go") + staleBefore, err := os.ReadFile(stalePath) + if err != nil { + t.Fatal(err) + } + for _, command := range []string{"diff", "generate"} { + if code, _, err := invoke(command, "-f", cfg); code != 1 || !strings.Contains(err, "old.sql.go") || !strings.Contains(err, "stale") { + t.Fatalf("%s accepted stale output: %d %s", command, code, err) + } + } + if _, err := os.Stat(filepath.Join(dir, "db", "new.sql.go")); !os.IsNotExist(err) { + t.Fatal("generate wrote files before reporting stale output") + } + if data, err := os.ReadFile(stalePath); err != nil || !bytes.Equal(data, staleBefore) { + t.Fatal("stale output was removed or changed") + } + if err := os.Remove(stalePath); err != nil { + t.Fatal(err) + } + // Handwritten files and packages nested in out are not generator-owned. + put(t, filepath.Join(dir, "db", "custom.go"), "package db\n// Code generated by sqlc-ydb. DO NOT EDIT.\n") + put(t, filepath.Join(dir, "db", "nested", "models.go"), "// Code generated by sqlc-ydb. DO NOT EDIT.\npackage nested\n") + for _, command := range []string{"generate", "diff"} { + if code, _, err := invoke(command, "-f", cfg); code != 0 { + t.Fatalf("%s after cleanup: %s", command, err) + } + } +} + +func TestStaleOutputsAcrossLanguages(t *testing.T) { + for _, file := range []struct{ name, header string }{ + {"Unused.java", "// Code generated by sqlc-ydb. DO NOT EDIT.\n"}, + {"unused.py", "# Code generated by sqlc-ydb. DO NOT EDIT.\n"}, + {"Unused.cs", "// Code generated by sqlc-ydb. DO NOT EDIT.\r\n"}, + } { + t.Run(file.name, func(t *testing.T) { + dir := t.TempDir() + cfg := filepath.Join(dir, "sqlc.yaml") + put(t, filepath.Join(dir, "schema.sql"), "CREATE TABLE a (id Uint64 NOT NULL, PRIMARY KEY(id));") + put(t, filepath.Join(dir, "queries.sql"), "-- name: GetA :one\nSELECT id FROM a;") + put(t, cfg, "version: '2'\nsql:\n- engine: ydb\n schema: schema.sql\n queries: queries.sql\n gen:\n go:\n out: db\n python:\n out: db\n") + if code, _, err := invoke("generate", "-f", cfg); code != 0 { + t.Fatal(err) + } + put(t, filepath.Join(dir, "db", file.name), file.header) + if code, _, err := invoke("diff", "-f", cfg); code != 1 || !strings.Contains(err, file.name) { + t.Fatalf("diff accepted stale %s: %d %s", file.name, code, err) + } + if code, _, err := invoke("compile", "-f", cfg); code != 0 { + t.Fatalf("compile inspected output: %s", err) + } + }) + } +} diff --git a/internal/cli/outputs.go b/internal/cli/outputs.go new file mode 100644 index 0000000..00d9a01 --- /dev/null +++ b/internal/cli/outputs.go @@ -0,0 +1,79 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +// Check only directories that this invocation writes. Generated files left by +// renamed queries or models can break a build even when every expected file +// matches. Report them before writing; never delete files on the user's behalf. +func checkStaleOutputs(files []output) error { + expected := map[string]map[string]bool{} + for _, f := range files { + dir, err := canonicalPath(filepath.Dir(f.path)) + if err != nil { + return err + } + if expected[dir] == nil { + expected[dir] = map[string]bool{} + } + expected[dir][filepath.Base(f.path)] = true + } + dirs := make([]string, 0, len(expected)) + for dir := range expected { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + var stale []string + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + for _, entry := range entries { + // Do not walk nested packages or follow unrelated symbolic links. + if expected[dir][entry.Name()] || !entry.Type().IsRegular() { + continue + } + path := filepath.Join(dir, entry.Name()) + generated, err := hasGeneratedHeader(path) + if err != nil { + return err + } + if generated { + stale = append(stale, path) + } + } + } + if len(stale) != 0 { + return fmt.Errorf("stale generated files; remove them before generating or use a separate output directory for each configuration:\n%s", strings.Join(stale, "\n")) + } + return nil +} + +func hasGeneratedHeader(path string) (bool, error) { + file, err := os.Open(path) + if err != nil { + return false, err + } + defer file.Close() + var header [64]byte + n, err := io.ReadFull(file, header[:]) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return false, fmt.Errorf("read %s: %w", path, err) + } + line, _, found := bytes.Cut(header[:n], []byte("\n")) + line = bytes.TrimSuffix(line, []byte("\r")) + return found && (bytes.Equal(line, []byte("// Code generated by sqlc-ydb. DO NOT EDIT.")) || + bytes.Equal(line, []byte("# Code generated by sqlc-ydb. DO NOT EDIT."))), nil +} diff --git a/internal/codegen/csharp/generator.go b/internal/codegen/csharp/generator.go index 72f3af0..74f298a 100644 --- a/internal/codegen/csharp/generator.go +++ b/internal/codegen/csharp/generator.go @@ -45,28 +45,33 @@ func validate(in *model.AnalysisResult) error { modelNames[n] = "framework type" } add := func(dst map[string]string, name, original, what string) error { - if old, ok := dst[name]; ok && old != original { + if old, ok := dst[name]; ok { return fmt.Errorf("csharp generator: %s collision %q (%q and %q)", what, name, old, original) } dst[name] = original return nil } for _, table := range in.Catalog.Tables { - if err := add(modelNames, csName(table.Name), "table:"+table.Name, "model name"); err != nil { + modelName := csName(table.Name) + if !csIdent(modelName) { + return fmt.Errorf("csharp generator: invalid model name for table %q", table.Name) + } + if err := add(modelNames, modelName, "table:"+table.Name, "model name"); err != nil { return err } - if err := fields("table "+table.Name, csName(table.Name), table.Columns); err != nil { + if err := fields("table "+table.Name, modelName, table.Columns); err != nil { return err } } for _, q := range in.Queries { - if !csIdent(q.Name) { + generatedName := csName(q.Name) + if !csIdent(q.Name) || !csIdent(generatedName) { return fmt.Errorf("csharp generator: invalid query name %q", q.Name) } - if err := add(queryNames, csName(q.Name), q.Name, "SQL constant"); err != nil { + if err := add(queryNames, generatedName, q.Name, "SQL constant"); err != nil { return err } - if err := add(methodNames, csName(q.Name)+"Async", q.Name, "method name"); err != nil { + if err := add(methodNames, generatedName+"Async", q.Name, "method name"); err != nil { return err } switch q.Command { @@ -154,9 +159,6 @@ func csType(t model.Type) (string, error) { if err != nil { return "", err } - if e == "string" || e == "byte[]" { - return e + "?", nil - } return e + "?", nil } switch strings.ToLower(t.Kind) { @@ -426,7 +428,7 @@ func csName(s string) string { } n := b.String() if n == "" { - return "Value" + return "" } if unicode.IsDigit([]rune(n)[0]) { return "Value" + n diff --git a/internal/codegen/csharp/generator_test.go b/internal/codegen/csharp/generator_test.go index 458d090..fd341f3 100644 --- a/internal/codegen/csharp/generator_test.go +++ b/internal/codegen/csharp/generator_test.go @@ -82,6 +82,8 @@ func TestRejectsUnsupportedOrCollidingInput(t *testing.T) { {"execrows", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.ExecRows}}}, Options{}, "unsupported command"}, {"field collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "t", Columns: []model.Column{{Name: "a_b", Type: model.Type{Kind: "Utf8"}}, {Name: "a b", Type: model.Type{Kind: "Utf8"}}}}}}}, Options{}, "column name collision"}, {"generated type collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "queries"}}}}, Options{}, "model name collision"}, + {"duplicate query", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Same", Command: model.Exec}, {Name: "Same", Command: model.Exec}}}, Options{}, "SQL constant collision"}, + {"unrepresentable table name", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "---"}}}}, Options{}, "invalid model name"}, {"namespace", authorsAnalysis(), Options{Namespace: "Bad.class"}, "invalid namespace"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/codegen/golang/generate.go b/internal/codegen/golang/generate.go index 3395759..ef56c1e 100644 --- a/internal/codegen/golang/generate.go +++ b/internal/codegen/golang/generate.go @@ -33,7 +33,7 @@ func Generate(in *model.AnalysisResult, o Options) ([]model.File, error) { if o.Package == "" { o.Package = "db" } - if !ident(o.Package) { + if !ident(o.Package) || o.Package == "_" { return nil, fmt.Errorf("invalid Go package %q", o.Package) } if o.Runtime == "" { @@ -79,6 +79,9 @@ func validate(in *model.AnalysisResult, o Options) error { if !ident(q.Name) || seen[q.Name] { return fmt.Errorf("invalid or duplicate query name %q", q.Name) } + if o.Runtime == "database/sql" && q.Name == "WithTx" { + return fmt.Errorf("query name %q conflicts with generated Queries.WithTx", q.Name) + } seen[q.Name] = true source := sourceName(q) out := outputName(source) @@ -241,7 +244,7 @@ func models(in *model.AnalysisResult, o Options) []byte { if o.EmitInterface { b.WriteString("type Querier interface {\n") for _, q := range in.Queries { - _, sig := methodArgs(q) + sig := methodArgs(q) ret := "error" if q.Command == model.One { ret = "(" + q.Name + "Row, error)" @@ -249,9 +252,6 @@ func models(in *model.AnalysisResult, o Options) []byte { if q.Command == model.Many { ret = "([]" + q.Name + "Row, error)" } - if q.Command == model.ExecRows { - ret = "(int64, error)" - } b.WriteString(q.Name + "(ctx context.Context" + sig + ") " + ret + "\n") } b.WriteString("}\n") @@ -296,11 +296,23 @@ func New(db DBTX) *Queries { return &Queries{db:db} } func queryFile(source string, qs []model.AnalyzedQuery, o Options) []byte { var b bytes.Buffer b.WriteString("// Code generated by sqlc-ydb. DO NOT EDIT.\n// source: " + filepath.Base(source) + "\npackage " + o.Package + "\n\n") + hasParameters := false + for _, q := range qs { + hasParameters = hasParameters || len(q.Parameters) > 0 + } if o.Runtime == "database/sql" { - b.WriteString("import (\"context\"; \"database/sql\")\n\n") + b.WriteString("import (\n\"context\"\n") + if hasParameters { + b.WriteString("\"database/sql\"\n") + } } else { - b.WriteString("import (\"context\"; ydb \"github.com/ydb-platform/ydb-go-sdk/v3\"; \"github.com/ydb-platform/ydb-go-sdk/v3/query\")\n\n") + b.WriteString("import (\n\"context\"\n") + if hasParameters { + b.WriteString("ydb \"github.com/ydb-platform/ydb-go-sdk/v3\"\n") + b.WriteString("\"github.com/ydb-platform/ydb-go-sdk/v3/query\"\n") + } } + b.WriteString(")\n\n") for _, q := range qs { writeQuery(&b, q, o) } @@ -308,7 +320,7 @@ func queryFile(source string, qs []model.AnalyzedQuery, o Options) []byte { } func writeQuery(b *bytes.Buffer, q model.AnalyzedQuery, o Options) { - c := "const " + lower(q.Name) + " = " + sqlLiteral(q.SQL) + "\n\n" + c := "const " + queryConstName(q.Name) + " = " + sqlLiteral(q.SQL) + "\n\n" b.WriteString(c) ret := "error" if q.Command == model.One { @@ -317,15 +329,12 @@ func writeQuery(b *bytes.Buffer, q model.AnalyzedQuery, o Options) { if q.Command == model.Many { ret = "([]" + q.Name + "Row, error)" } - if q.Command == model.ExecRows { - ret = "(int64, error)" - } - args, sig := methodArgs(q) + sig := methodArgs(q) b.WriteString("func (q *Queries) " + q.Name + "(ctx context.Context" + sig + ") " + ret + " {\n") if o.Runtime == "database/sql" { - writeSQL(b, q, args, o) + writeSQL(b, q, o) } else { - writeYDB(b, q, args, o) + writeYDB(b, q, o) } b.WriteString("}\n\n") } @@ -346,21 +355,21 @@ func sqlLiteral(sql string) string { return strings.Join(lines, " +\n") } -func methodArgs(q model.AnalyzedQuery) (string, string) { +func methodArgs(q model.AnalyzedQuery) string { if len(q.Parameters) == 0 { - return "", "" + return "" } if len(q.Parameters) > 1 { - return "arg", ", arg " + q.Name + "Params" + return ", arg " + q.Name + "Params" } t, _ := goType(q.Parameters[0].Type) - return lower(q.Parameters[0].Name), ", " + lower(q.Parameters[0].Name) + " " + t + return ", arg " + t } func varRef(q model.AnalyzedQuery, p model.Parameter) string { if len(q.Parameters) > 1 { return "arg." + goName(p.Name) } - return lower(p.Name) + return "arg" } func sqlArgs(q model.AnalyzedQuery) string { x := make([]string, len(q.Parameters)) @@ -372,38 +381,36 @@ func sqlArgs(q model.AnalyzedQuery) string { } return ", " + strings.Join(x, ", ") } -func writeSQL(b *bytes.Buffer, q model.AnalyzedQuery, args string, o Options) { +func writeSQL(b *bytes.Buffer, q model.AnalyzedQuery, o Options) { a := sqlArgs(q) switch q.Command { case model.Exec: - b.WriteString("_, err := q.db.ExecContext(ctx, " + lower(q.Name) + a + ")\nreturn err\n") - case model.ExecRows: - b.WriteString("result, err := q.db.ExecContext(ctx, " + lower(q.Name) + a + ")\nif err != nil { return 0, err }; return result.RowsAffected()\n") + b.WriteString("_, err := q.db.ExecContext(ctx, " + queryConstName(q.Name) + a + ")\nreturn err\n") case model.One: - b.WriteString("var row " + q.Name + "Row\nerr := q.db.QueryRowContext(ctx, " + lower(q.Name) + a + ").Scan(" + scan(q.ResultSets[0]) + ")\nreturn row, err\n") + b.WriteString("var row " + q.Name + "Row\nerr := q.db.QueryRowContext(ctx, " + queryConstName(q.Name) + a + ").Scan(" + scan(q.ResultSets[0]) + ")\nreturn row, err\n") case model.Many: init := "[]" + q.Name + "Row(nil)" if o.EmitEmptySlices { init = "make([]" + q.Name + "Row, 0)" } - b.WriteString("rows, err := q.db.QueryContext(ctx, " + lower(q.Name) + a + ")\nif err != nil { return " + init + ", err }; defer rows.Close()\nitems := " + init + "\nfor rows.Next() { var row " + q.Name + "Row\nif err := rows.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return nil, err }; items = append(items, row) }\nreturn items, rows.Err()\n") + b.WriteString("rows, err := q.db.QueryContext(ctx, " + queryConstName(q.Name) + a + ")\nif err != nil { return " + init + ", err }; defer rows.Close()\nitems := " + init + "\nfor rows.Next() { var row " + q.Name + "Row\nif err := rows.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return nil, err }; items = append(items, row) }\nreturn items, rows.Err()\n") } } -func writeYDB(b *bytes.Buffer, q model.AnalyzedQuery, args string, o Options) { +func writeYDB(b *bytes.Buffer, q model.AnalyzedQuery, o Options) { opt := params(q) if q.Command == model.Exec { - b.WriteString("return q.db.Exec(ctx, " + lower(q.Name) + opt + ")\n") + b.WriteString("return q.db.Exec(ctx, " + queryConstName(q.Name) + opt + ")\n") return } if q.Command == model.One { - b.WriteString("result, err := q.db.QueryRow(ctx, " + lower(q.Name) + opt + ")\nif err != nil { return " + q.Name + "Row{}, err }; var row " + q.Name + "Row\nif err := result.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return " + q.Name + "Row{}, err }; return row, nil\n") + b.WriteString("result, err := q.db.QueryRow(ctx, " + queryConstName(q.Name) + opt + ")\nif err != nil { return " + q.Name + "Row{}, err }; var row " + q.Name + "Row\nif err := result.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return " + q.Name + "Row{}, err }; return row, nil\n") return } init := "[]" + q.Name + "Row(nil)" if o.EmitEmptySlices { init = "make([]" + q.Name + "Row, 0)" } - b.WriteString("result, err := q.db.QueryResultSet(ctx, " + lower(q.Name) + opt + ")\nif err != nil { return " + init + ", err }; defer result.Close(ctx)\nitems := " + init + "\nfor r, err := range result.Rows(ctx) { if err != nil { return nil, err }; var row " + q.Name + "Row; if err := r.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return nil, err }; items = append(items, row) }\nreturn items, nil\n") + b.WriteString("result, err := q.db.QueryResultSet(ctx, " + queryConstName(q.Name) + opt + ")\nif err != nil { return " + init + ", err }; defer result.Close(ctx)\nitems := " + init + "\nfor r, err := range result.Rows(ctx) { if err != nil { return nil, err }; var row " + q.Name + "Row; if err := r.Scan(" + scan(q.ResultSets[0]) + "); err != nil { return nil, err }; items = append(items, row) }\nreturn items, nil\n") } func scan(rs model.ResultSet) string { x := make([]string, len(rs.Columns)) @@ -462,12 +469,7 @@ func outputName(s string) string { s = strings.TrimSuffix(s, ".sql") return s + ".sql.go" } -func lower(s string) string { - if s == "" { - return "q" - } - return strings.ToLower(s[:1]) + s[1:] -} +func queryConstName(name string) string { return "query" + name } func goName(s string) string { var b strings.Builder for _, part := range strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) { diff --git a/internal/codegen/golang/generate_test.go b/internal/codegen/golang/generate_test.go index 02c1cc4..8a5f800 100644 --- a/internal/codegen/golang/generate_test.go +++ b/internal/codegen/golang/generate_test.go @@ -39,7 +39,7 @@ func TestGeneratedSQLIsMultilineAndPreservesText(t *testing.T) { } { for _, runtime := range []string{"database/sql", "ydb"} { source := generatedSQLSource(t, runtime, tc.sql) - if !strings.Contains(string(source), "const getUser = "+tc.wantLiteral) { + if !strings.Contains(string(source), "const queryGetUser = "+tc.wantLiteral) { t.Fatalf("%s SQL is not a readable multiline literal:\n%s", runtime, source) } if strings.Contains(tc.sql, "`id`") && !strings.Contains(string(source), "\"SELECT `id`, `bio` FROM `users`\\n\"") { @@ -109,7 +109,7 @@ func generatedSQLValue(t *testing.T, source []byte) string { found := false ast.Inspect(file, func(node ast.Node) bool { decl, ok := node.(*ast.ValueSpec) - if !ok || len(decl.Names) != 1 || decl.Names[0].Name != "getUser" { + if !ok || len(decl.Names) != 1 || decl.Names[0].Name != "queryGetUser" { return true } found = true @@ -169,7 +169,7 @@ func runLiveGenerated(t *testing.T, dsn, table, runtime string) { t.Fatal(err) } } - if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module generated\n\ngo 1.26.0\n\nrequire github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1\n"), 0600); err != nil { + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module generated\n\ngo 1.26.0\n\nrequire github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1\n"), 0600); err != nil { t.Fatal(err) } test := liveTestSource(dsn, table, runtime) @@ -277,7 +277,7 @@ var lastSQL string var fail bool type drv struct{}; func (drv) Open(string)(driver.Conn,error){return conn{},nil} type conn struct{}; func (conn) Prepare(string)(driver.Stmt,error){return nil,driver.ErrSkip}; func (conn) Close()error{return nil}; func (conn) Begin()(driver.Tx,error){return nil,driver.ErrSkip} -func (conn) QueryContext(_ context.Context, q string, a []driver.NamedValue)(driver.Rows,error){ calls=a;lastSQL=q;if fail{return nil,errors.New("query failed")}; if q==getUser {return &rows{data:[][]driver.Value{{uint64(7),nil}}},nil}; return &rows{data:[][]driver.Value{{uint64(8),"a"}}},nil } +func (conn) QueryContext(_ context.Context, q string, a []driver.NamedValue)(driver.Rows,error){ calls=a;lastSQL=q;if fail{return nil,errors.New("query failed")}; if q==queryGetUser {return &rows{data:[][]driver.Value{{uint64(7),nil}}},nil}; return &rows{data:[][]driver.Value{{uint64(8),"a"}}},nil } func (conn) ExecContext(_ context.Context, _ string, a []driver.NamedValue)(driver.Result,error){calls=a; return result(3),nil} type rows struct{data [][]driver.Value; i int}; func (r *rows) Columns()[]string{return []string{"id","bio"}}; func (r *rows) Close()error{closed=true;return nil}; func (r *rows) Next(dst []driver.Value)error{if r.i==len(r.data){return io.EOF};copy(dst,r.data[r.i]);r.i++;return nil} type result int64; func (r result) LastInsertId()(int64,error){return 0,nil};func(r result) RowsAffected()(int64,error){return int64(r),nil} @@ -302,7 +302,7 @@ func compileInput(t *testing.T, input *model.AnalysisResult, opts Options) { } mod := "module generated\n\ngo 1.26.0\n" if opts.Runtime == "ydb" { - mod += "\nrequire github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1\n" + mod += "\nrequire github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1\n" } if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte(mod), 0600); err != nil { t.Fatal(err) @@ -313,7 +313,7 @@ func compileInput(t *testing.T, input *model.AnalysisResult, opts Options) { if err != nil { t.Fatalf("generated %s code does not compile:\n%s", opts.Runtime, out) } - if opts.Runtime == "database/sql" && !strings.Contains(string(files[0].Content), "json:\"id\"") { + if opts.Runtime == "database/sql" && opts.EmitJSONTags && !strings.Contains(string(files[0].Content), "json:\"id\"") { t.Fatal("JSON tags were not emitted") } } @@ -339,3 +339,52 @@ func TestGoNameInitialismID(t *testing.T) { t.Fatalf("author_id => %q", got) } } + +func TestGeneratedIdentifiersDoNotCollideWithMethodScope(t *testing.T) { + u64 := model.Type{Kind: "Uint64"} + row := []model.ResultSet{{Columns: []model.Column{{Name: "value", Type: u64}}}} + databaseSQL := &model.AnalysisResult{Queries: []model.AnalyzedQuery{ + {Name: "ByContext", Command: model.Exec, Parameters: []model.Parameter{{Name: "ctx", Type: u64}}}, + {Name: "ByReceiver", Command: model.Exec, Parameters: []model.Parameter{{Name: "q", Type: u64}}}, + {Name: "BySQLImport", Command: model.Exec, Parameters: []model.Parameter{{Name: "sql", Type: u64}}}, + {Name: "ByRowLocal", Command: model.One, Parameters: []model.Parameter{{Name: "row", Type: u64}}, ResultSets: row}, + }} + compileInput(t, databaseSQL, Options{Package: "db", Runtime: "database/sql"}) + + ydb := &model.AnalysisResult{Queries: []model.AnalyzedQuery{ + {Name: "ByYDBImport", Command: model.Exec, Parameters: []model.Parameter{{Name: "ydb", Type: u64}}}, + {Name: "ByQueryImport", Command: model.Exec, Parameters: []model.Parameter{{Name: "query", Type: u64}}}, + }} + compileInput(t, ydb, Options{Package: "db", Runtime: "ydb"}) +} + +func TestNoParameterQueryFilesCompileWithoutUnusedRuntimeImports(t *testing.T) { + in := &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Ping", Command: model.Exec, SQL: "SELECT 1;"}}} + for _, runtime := range []string{"database/sql", "ydb"} { + t.Run(runtime, func(t *testing.T) { + compileInput(t, in, Options{Package: "db", Runtime: runtime}) + }) + } +} + +func TestDatabaseSQLRejectsQueryNamedWithTx(t *testing.T) { + _, err := Generate(&model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "WithTx", Command: model.Exec}}}, Options{Package: "db", Runtime: "database/sql"}) + if err == nil || !strings.Contains(err.Error(), `query name "WithTx" conflicts with generated Queries.WithTx`) { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestQueryConstantsPreserveCaseDistinctNames(t *testing.T) { + in := &model.AnalysisResult{Queries: []model.AnalyzedQuery{ + {Name: "Foo", Command: model.Exec, SQL: "SELECT 1;"}, + {Name: "foo", Command: model.Exec, SQL: "SELECT 2;"}, + }} + compileInput(t, in, Options{Package: "db", Runtime: "database/sql"}) +} + +func TestRejectsBlankIdentifierPackage(t *testing.T) { + _, err := Generate(&model.AnalysisResult{}, Options{Package: "_", Runtime: "database/sql"}) + if err == nil || !strings.Contains(err.Error(), `invalid Go package "_"`) { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/codegen/java/generator.go b/internal/codegen/java/generator.go index 9312155..45b5fa7 100644 --- a/internal/codegen/java/generator.go +++ b/internal/codegen/java/generator.go @@ -11,22 +11,22 @@ import ( type Options struct{ Package, Runtime string } -type scalar struct{ typ, boxed, sdk, jdbc, sqlType string } +type scalar struct{ typ, boxed, sdk, jdbc string } var scalars = map[string]scalar{ - "Bool": {"boolean", "Boolean", "Bool", "Boolean", "BOOLEAN"}, - "Int8": {"byte", "Byte", "Int8", "Byte", "TINYINT"}, - "Uint8": {"int", "Integer", "Uint8", "Int", "INTEGER"}, - "Int16": {"short", "Short", "Int16", "Short", "SMALLINT"}, - "Uint16": {"int", "Integer", "Uint16", "Int", "INTEGER"}, - "Int32": {"int", "Integer", "Int32", "Int", "INTEGER"}, - "Uint32": {"long", "Long", "Uint32", "Long", "BIGINT"}, - "Int64": {"long", "Long", "Int64", "Long", "BIGINT"}, - "Uint64": {"long", "Long", "Uint64", "Long", "BIGINT"}, - "Float": {"float", "Float", "Float", "Float", "FLOAT"}, - "Double": {"double", "Double", "Double", "Double", "DOUBLE"}, - "Utf8": {"String", "String", "Text", "String", "VARCHAR"}, - "String": {"byte[]", "byte[]", "Bytes", "Bytes", "BINARY"}, + "Bool": {"boolean", "Boolean", "Bool", "Boolean"}, + "Int8": {"byte", "Byte", "Int8", "Byte"}, + "Uint8": {"int", "Integer", "Uint8", "Int"}, + "Int16": {"short", "Short", "Int16", "Short"}, + "Uint16": {"int", "Integer", "Uint16", "Int"}, + "Int32": {"int", "Integer", "Int32", "Int"}, + "Uint32": {"long", "Long", "Uint32", "Long"}, + "Int64": {"long", "Long", "Int64", "Long"}, + "Uint64": {"long", "Long", "Uint64", "Long"}, + "Float": {"float", "Float", "Float", "Float"}, + "Double": {"double", "Double", "Double", "Double"}, + "Utf8": {"String", "String", "Text", "String"}, + "String": {"byte[]", "byte[]", "Bytes", "Bytes"}, } func typeInfo(t model.Type) (scalar, string, error) { @@ -227,6 +227,7 @@ func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { methods[method] = true row, _ := name(q.Name, true) row += "Row" + constant := method + "Sql" ret := "void" switch q.Command { case model.One, model.Many: @@ -247,7 +248,7 @@ func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { } params := []string{} paramNames := []string{} - seen := map[string]bool{"client": true, "_params": true, "_query": true, "_connection": true, "_statement": true, "_prepared": true, "_rows": true, "_items": true} + seen := map[string]bool{"client": true, constant: true, "_params": true, "_query": true, "_connection": true, "_statement": true, "_prepared": true, "_rows": true, "_items": true} for _, p := range q.Parameters { n, err := name(p.Name, false) if err != nil { @@ -264,7 +265,6 @@ func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { params = append(params, typ+" "+n) paramNames = append(paramNames, n) } - constant := method + "Sql" fmt.Fprintf(&b, "\n private static final String %s = %s;\n", constant, sqlLiteral(q.SQL)) throws := "" if o.Runtime == "jdbc" { diff --git a/internal/codegen/java/generator_test.go b/internal/codegen/java/generator_test.go index f646cb7..bdfbe44 100644 --- a/internal/codegen/java/generator_test.go +++ b/internal/codegen/java/generator_test.go @@ -97,6 +97,7 @@ func TestGenerateRejectsInvalidContracts(t *testing.T) { {"many_two_results", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Many, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "value", Type: utf8}}}, {Columns: []model.Column{{Name: "other", Type: utf8}}}}}}}, Options{}, "requires one nonempty result set"}, {"method_collision", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Get_User", Command: model.Exec}, {Name: "getUser", Command: model.Exec}}}, Options{}, "method name collision"}, {"parameter_collision", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, Parameters: []model.Parameter{{Name: "a-b", Type: utf8}, {Name: "a_b", Type: utf8}}}}}, Options{}, "parameter name collision"}, + {"parameter_shadows_sql_constant", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "GetAuthor", Command: model.Exec, Parameters: []model.Parameter{{Name: "get_author_sql", Type: utf8}}}}}, Options{}, "parameter name collision"}, {"field_collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "items", Columns: []model.Column{{Name: "a-b", Type: utf8}, {Name: "a_b", Type: utf8}}}}}}, Options{}, "field name collision"}, {"record_collision", &model.AnalysisResult{Catalog: model.Catalog{Tables: []model.Table{{Name: "get_author_row", Columns: []model.Column{{Name: "id", Type: utf8}}}}}, Queries: []model.AnalyzedQuery{{Name: "get_author", Command: model.One, ResultSets: []model.ResultSet{{Columns: []model.Column{{Name: "id", Type: utf8}}}}}}}, Options{}, "type name collision"}, {"invalid_utf8", &model.AnalysisResult{Queries: []model.AnalyzedQuery{{Name: "Bad", Command: model.Exec, SQL: string([]byte{0xff})}}}, Options{}, "must be valid UTF-8"}, diff --git a/internal/codegen/python/generator.go b/internal/codegen/python/generator.go index 42ac0c0..b9abb57 100644 --- a/internal/codegen/python/generator.go +++ b/internal/codegen/python/generator.go @@ -12,7 +12,7 @@ import ( ) type Options struct { - Package, Runtime string + Runtime string EmitSyncQuerier, EmitAsyncQuerier bool } @@ -29,9 +29,6 @@ func Generate(a *model.AnalysisResult, o Options) ([]model.File, error) { if o.Runtime != "ydb" && o.Runtime != "dbapi" && o.Runtime != "sqlalchemy" { return nil, fmt.Errorf("python generator: unsupported runtime %q", o.Runtime) } - if !o.EmitSyncQuerier && !o.EmitAsyncQuerier { - o.EmitSyncQuerier = true - } if o.EmitAsyncQuerier { return nil, fmt.Errorf("python generator: async querier is unsupported by the installed YDB Python runtimes; use a synchronous querier") } @@ -74,6 +71,9 @@ func validateQuery(q model.AnalyzedQuery) error { seenParams := map[string]bool{} for _, p := range q.Parameters { n := fieldName(p.Name) + if n == "self" { + return fmt.Errorf("python generator: query %q: parameter name %q conflicts with the generated method receiver", q.Name, n) + } if seenParams[n] { return fmt.Errorf("python generator: query %q: parameter name collision at %q", q.Name, n) } @@ -179,28 +179,6 @@ func writeTypeSignature(b *strings.Builder, t model.Type) { writeTypeSignature(b, *t.Key) b.WriteByte(']') } - if len(t.Items) > 0 { - b.WriteByte('(') - for i := range t.Items { - if i > 0 { - b.WriteByte(',') - } - writeTypeSignature(b, t.Items[i]) - } - b.WriteByte(')') - } - if len(t.Fields) > 0 { - b.WriteByte('{') - for i, f := range t.Fields { - if i > 0 { - b.WriteByte(',') - } - b.WriteString(f.Name) - b.WriteByte(':') - writeTypeSignature(b, f.Type) - } - b.WriteByte('}') - } } func renderModels(a *model.AnalysisResult) (string, error) { @@ -264,10 +242,6 @@ func renderQueries(a *model.AnalysisResult, o Options) (string, error) { } if o.Runtime == "sqlalchemy" { b.WriteString("import ydb\nfrom sqlalchemy import text\nfrom sqlalchemy.engine import Connection\n") - if o.EmitAsyncQuerier { - b.WriteString("from sqlalchemy.ext.asyncio import AsyncConnection\n") - } - b.WriteString("def _typed(value, typ):\n return (value, typ)\n\n") } b.WriteString("\n") for _, q := range a.Queries { @@ -282,33 +256,16 @@ func renderQueries(a *model.AnalysisResult, o Options) (string, error) { b.WriteString(constName(q.Name) + " = " + pySQLString(sql) + "\n\n") } if o.Runtime == "ydb" { - b.WriteString(ydbHelpers) - } else if o.Runtime == "dbapi" { - b.WriteString(dbapiHelpers) + b.WriteString(ydbTypedHelper) } else { - b.WriteString(sqlalchemyRowHelper) + b.WriteString(tupleTypedHelper) } - if o.EmitSyncQuerier { - renderClass(&b, a, o, false) - } - if o.EmitAsyncQuerier { - renderClass(&b, a, o, true) + if err := renderClass(&b, a, o); err != nil { + return "", err } return b.String(), nil } -const sqlalchemyRowHelper = ` -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - -` - func sqlalchemySQL(q model.AnalyzedQuery) (string, error) { allowed := map[string]bool{} for _, p := range q.Parameters { @@ -360,41 +317,19 @@ func sqlalchemySQL(q model.AnalyzedQuery) (string, error) { return out.String(), nil } -const ydbHelpers = ` -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - +const ydbTypedHelper = ` def _typed(value, typ): return ydb.TypedValue(value, typ) ` -const dbapiHelpers = ` +const tupleTypedHelper = ` def _typed(value, typ): return (value, typ) -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - ` -func renderClass(b *strings.Builder, a *model.AnalysisResult, o Options, async bool) { - n := "Querier" - if async { - n = "AsyncQuerier" - } - b.WriteString("\nclass " + n + ":\n") +func renderClass(b *strings.Builder, a *model.AnalysisResult, o Options) error { + b.WriteString("\nclass Querier:\n") if o.Runtime == "ydb" { b.WriteString(" def __init__(self, pool: ydb.QuerySessionPool):\n self._pool = pool\n\n") } @@ -402,18 +337,25 @@ func renderClass(b *strings.Builder, a *model.AnalysisResult, o Options, async b b.WriteString(" def __init__(self, connection):\n self._connection = connection\n\n") } if o.Runtime == "sqlalchemy" { - typ := "Connection" - if async { - typ = "AsyncConnection" - } - b.WriteString(" def __init__(self, connection: " + typ + "):\n self._connection = connection\n\n") + b.WriteString(" def __init__(self, connection: Connection):\n self._connection = connection\n\n") } for _, q := range a.Queries { - renderMethod(b, a, q, o, async) + if err := renderMethod(b, a, q, o); err != nil { + return err + } } + return nil } -func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQuery, o Options, async bool) { +func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQuery, o Options) error { + typeExprs := make([]string, len(q.Parameters)) + for i, parameter := range q.Parameters { + var err error + typeExprs[i], err = ydbTypeExpr(parameter.Type) + if err != nil { + return fmt.Errorf("python generator: query %q parameter %q: %w", q.Name, parameter.Name, err) + } + } p := "" for _, x := range q.Parameters { t, _ := pyType(x.Type) @@ -427,25 +369,18 @@ func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQ if q.Command == model.Many { ret = "Iterable[models." + row + "]" } - if q.Command == model.ExecRows { - ret = "int" - } - kw := "" - if async { - kw = "async " - } indent := " " if o.Runtime == "dbapi" { indent = " " } - b.WriteString(" " + kw + "def " + methodName(q.Name) + "(self" + p + ") -> " + ret + ":\n") + b.WriteString(" def " + methodName(q.Name) + "(self" + p + ") -> " + ret + ":\n") if o.Runtime == "ydb" { b.WriteString(" parameters = {") for i, x := range q.Parameters { if i > 0 { b.WriteString(",") } - value := "_typed(" + fieldName(x.Name) + ", " + ydbTypeExpr(x.Type) + ")" + value := "_typed(" + fieldName(x.Name) + ", " + typeExprs[i] + ")" b.WriteString(pyString("$"+x.Name) + ": " + value) } b.WriteString("}\n result_sets = self._pool.execute_with_retries(" + constName(q.Name) + ", parameters)\n") @@ -456,7 +391,7 @@ func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQ if i > 0 { b.WriteString(",") } - b.WriteString(pyString("$"+x.Name) + ": _typed(" + fieldName(x.Name) + ", " + ydbTypeExpr(x.Type) + ")") + b.WriteString(pyString("$"+x.Name) + ": _typed(" + fieldName(x.Name) + ", " + typeExprs[i] + ")") } b.WriteString("}\n cursor = self._connection.cursor()\n try:\n cursor.execute(" + constName(q.Name) + ", parameters)\n") } @@ -466,17 +401,14 @@ func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQ if i > 0 { b.WriteString(",") } - b.WriteString(pyString(x.Name) + ": _typed(" + fieldName(x.Name) + ", " + ydbTypeExpr(x.Type) + ")") + b.WriteString(pyString(x.Name) + ": _typed(" + fieldName(x.Name) + ", " + typeExprs[i] + ")") } b.WriteString("}\n result = ") - if async { - b.WriteString("await ") - } b.WriteString("self._connection.execute(text(" + constName(q.Name) + "), parameters)\n") } if q.Command == model.One || q.Command == model.Many { if o.Runtime == "ydb" { - b.WriteString(indent + "rows = result_sets[0].rows if result_sets else []\n") + b.WriteString(indent + "rows = result_sets[0].rows\n") } else if o.Runtime == "dbapi" { b.WriteString(indent + "rows = cursor.fetchall()\n") } else { @@ -485,24 +417,16 @@ func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQ if q.Command == model.One { b.WriteString(indent + "if not rows:\n" + indent + " return None\n" + indent + "row = rows[0]\n" + indent + "return models." + row + "(\n") for i, c := range q.ResultSets[0].Columns { - b.WriteString(indent + " " + fieldName(c.Name) + "=_row_value(row, " + pyString(c.Name) + ", " + strconv.Itoa(i) + "),\n") + b.WriteString(indent + " " + fieldName(c.Name) + "=" + rowValue(o.Runtime, c.Name, i) + ",\n") } b.WriteString(indent + ")\n") } else { b.WriteString(indent + "return (models." + row + "(\n") for i, c := range q.ResultSets[0].Columns { - b.WriteString(indent + " " + fieldName(c.Name) + "=_row_value(row, " + pyString(c.Name) + ", " + strconv.Itoa(i) + "),\n") + b.WriteString(indent + " " + fieldName(c.Name) + "=" + rowValue(o.Runtime, c.Name, i) + ",\n") } b.WriteString(indent + ") for row in rows)\n") } - } else if q.Command == model.ExecRows { - if o.Runtime == "dbapi" { - b.WriteString(indent + "return cursor.rowcount\n") - } else if o.Runtime == "sqlalchemy" { - b.WriteString(" return result.rowcount\n") - } else { - b.WriteString(" return 0\n") - } } else { if o.Runtime == "dbapi" { b.WriteString(indent + "return None\n") @@ -516,6 +440,18 @@ func renderMethod(b *strings.Builder, a *model.AnalysisResult, q model.AnalyzedQ b.WriteString(" finally:\n cursor.close()\n") } b.WriteString("\n") + return nil +} + +func rowValue(runtime, name string, index int) string { + switch runtime { + case "ydb": + return "row[" + pyString(name) + "]" + case "dbapi": + return "row[" + strconv.Itoa(index) + "]" + default: + return "row._mapping[" + pyString(name) + "]" + } } func queryRowClass(a *model.AnalysisResult, q model.AnalyzedQuery) string { @@ -553,34 +489,18 @@ func resultTypeSignature(t model.Type) string { } func pyType(t model.Type) (string, error) { - if t.Kind == "Optional" { + if t.IsOptional() { if t.Elem == nil { return "", fmt.Errorf("malformed Optional type") } x, e := pyType(*t.Elem) return "Optional[" + x + "]", e } - switch strings.ToLower(t.Kind) { - case "bool": - return "bool", nil - case "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64": - return "int", nil - case "float", "double": - return "float", nil - case "utf8": - return "str", nil - case "string": - return "bytes", nil - case "date", "date32": - return "date", nil - case "datetime", "datetime64", "timestamp", "timestamp64": - return "datetime", nil - case "interval", "interval64": - return "timedelta", nil - case "uuid": - return "UUID", nil - case "json", "jsondocument": - return "str", nil + kind := strings.ToLower(t.Kind) + if primitive, ok := pythonPrimitiveTypes[kind]; ok { + return primitive.python, nil + } + switch kind { case "list": if t.Elem == nil { return "", fmt.Errorf("List without element type") @@ -608,74 +528,81 @@ func pyType(t model.Type) (string, error) { } } -func ydbTypeExpr(t model.Type) string { - if t.IsOptional() && t.Elem != nil { - return "ydb.OptionalType(" + ydbTypeExpr(*t.Elem) + ")" - } - if strings.EqualFold(t.Kind, "list") && t.Elem != nil { - return "ydb.ListType(" + ydbTypeExpr(*t.Elem) + ")" - } - if strings.EqualFold(t.Kind, "set") && t.Elem != nil { - return "ydb.SetType(" + ydbTypeExpr(*t.Elem) + ")" - } - if strings.EqualFold(t.Kind, "dict") && t.Key != nil && t.Elem != nil { - return "ydb.DictType(" + ydbTypeExpr(*t.Key) + ", " + ydbTypeExpr(*t.Elem) + ")" - } - var n string - switch strings.ToLower(t.Kind) { - case "bool": - n = "Bool" - case "int8": - n = "Int8" - case "int16": - n = "Int16" - case "int32": - n = "Int32" - case "int64": - n = "Int64" - case "uint8": - n = "Uint8" - case "uint16": - n = "Uint16" - case "uint32": - n = "Uint32" - case "uint64": - n = "Uint64" - case "float": - n = "Float" - case "double": - n = "Double" - case "utf8": - n = "Utf8" - case "string": - n = "String" - case "date": - n = "Date" - case "date32": - n = "Date32" - case "datetime": - n = "Datetime" - case "datetime64": - n = "Datetime64" - case "timestamp": - n = "Timestamp" - case "timestamp64": - n = "Timestamp64" - case "interval": - n = "Interval" - case "interval64": - n = "Interval64" - case "uuid": - n = "UUID" - case "json": - n = "Json" - case "jsondocument": - n = "JsonDocument" - default: - return "ydb.PrimitiveType.Utf8" +func ydbTypeExpr(t model.Type) (string, error) { + if t.IsOptional() { + if t.Elem == nil { + return "", fmt.Errorf("malformed Optional type") + } + elem, err := ydbTypeExpr(*t.Elem) + return "ydb.OptionalType(" + elem + ")", err } - return "ydb.PrimitiveType." + n + kind := strings.ToLower(t.Kind) + if primitive, ok := pythonPrimitiveTypes[kind]; ok { + return "ydb.PrimitiveType." + primitive.ydb, nil + } + if kind == "list" || kind == "set" { + if t.Elem == nil { + return "", fmt.Errorf("%s without element type", t.Kind) + } + elem, err := ydbTypeExpr(*t.Elem) + if err != nil { + return "", err + } + constructor := "ListType" + if kind == "set" { + constructor = "SetType" + } + return "ydb." + constructor + "(" + elem + ")", nil + } + if kind == "dict" { + if t.Key == nil || t.Elem == nil { + return "", fmt.Errorf("Dict without key/value type") + } + key, err := ydbTypeExpr(*t.Key) + if err != nil { + return "", err + } + elem, err := ydbTypeExpr(*t.Elem) + if err != nil { + return "", err + } + return "ydb.DictType(" + key + ", " + elem + ")", nil + } + return "", fmt.Errorf("unsupported YQL type %q", t.Kind) +} + +type pythonPrimitiveType struct { + python string + ydb string } + +var pythonPrimitiveTypes = map[string]pythonPrimitiveType{ + "bool": {python: "bool", ydb: "Bool"}, + "int8": {python: "int", ydb: "Int8"}, + "int16": {python: "int", ydb: "Int16"}, + "int32": {python: "int", ydb: "Int32"}, + "int64": {python: "int", ydb: "Int64"}, + "uint8": {python: "int", ydb: "Uint8"}, + "uint16": {python: "int", ydb: "Uint16"}, + "uint32": {python: "int", ydb: "Uint32"}, + "uint64": {python: "int", ydb: "Uint64"}, + "float": {python: "float", ydb: "Float"}, + "double": {python: "float", ydb: "Double"}, + "utf8": {python: "str", ydb: "Utf8"}, + "string": {python: "bytes", ydb: "String"}, + "date": {python: "date", ydb: "Date"}, + "date32": {python: "date", ydb: "Date32"}, + "datetime": {python: "datetime", ydb: "Datetime"}, + "datetime64": {python: "datetime", ydb: "Datetime64"}, + "timestamp": {python: "datetime", ydb: "Timestamp"}, + "timestamp64": {python: "datetime", ydb: "Timestamp64"}, + "interval": {python: "timedelta", ydb: "Interval"}, + "interval64": {python: "timedelta", ydb: "Interval64"}, + "uuid": {python: "UUID", ydb: "UUID"}, + "json": {python: "str", ydb: "Json"}, + "jsondocument": {python: "str", ydb: "JsonDocument"}, +} + func pyString(s string) string { return strconv.Quote(s) } func pySQLString(s string) string { diff --git a/internal/codegen/python/generator_test.go b/internal/codegen/python/generator_test.go index f2f33db..f8279ee 100644 --- a/internal/codegen/python/generator_test.go +++ b/internal/codegen/python/generator_test.go @@ -98,6 +98,23 @@ func TestGenerateRejectsUnknownType(t *testing.T) { } } +func TestYDBTypeExpressionRejectsUnknownType(t *testing.T) { + if _, err := ydbTypeExpr(model.Type{Kind: "Any"}); err == nil || !strings.Contains(err.Error(), "unsupported YQL type") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRejectsParameterNamedSelf(t *testing.T) { + a := &model.AnalysisResult{Queries: []model.AnalyzedQuery{{ + Name: "delete_author", + Command: model.Exec, + Parameters: []model.Parameter{{Name: "self", Type: model.Type{Kind: "Uint64"}}}, + }}} + if _, err := Generate(a, Options{Runtime: "ydb"}); err == nil || !strings.Contains(err.Error(), `parameter name "self" conflicts with the generated method receiver`) { + t.Fatalf("unexpected error: %v", err) + } +} + func TestIdenticalTableProjectionsReuseRowModel(t *testing.T) { a := liveAnalysis("authors") if _, err := Generate(a, Options{Runtime: "dbapi"}); err != nil { @@ -315,7 +332,39 @@ class QuerySessionPool: pass `), 0600); err != nil { t.Fatal(err) } - script := fmt.Sprintf("import sys; sys.path.insert(0, %q); from db.queries import Querier; import ydb\nclass P:\n def __init__(self): self.calls=[]\n def execute_with_retries(self, sql, parameters):\n self.calls.append((sql, parameters)); return [ydb.ResultSet([ydb.Row(id=7, display_name=None)])]\np=P(); row=Querier(p).get_author(7); assert row.id == 7 and row.display_name is None; assert p.calls[0][1]['$id'].type == ydb.PrimitiveType.Uint64\n", dir) + script := fmt.Sprintf(`import sys +sys.path.insert(0, %q) +from db.queries import Querier +import ydb + +class Pool: + def __init__(self, results): + self.results = results + self.calls = [] + def execute_with_retries(self, sql, parameters): + self.calls.append((sql, parameters)) + return self.results + +pool = Pool([ydb.ResultSet([ydb.Row(id=7, display_name=None)])]) +row = Querier(pool).get_author(7) +assert row.id == 7 and row.display_name is None +assert pool.calls[0][1]['$id'].type == ydb.PrimitiveType.Uint64 +assert Querier(Pool([ydb.ResultSet([])])).get_author(7) is None + +try: + Querier(Pool([])).get_author(7) +except IndexError: + pass +else: + raise AssertionError("missing result set was masked as a missing row") + +try: + Querier(Pool([ydb.ResultSet([{0: 7, 1: None}])])).get_author(7) +except KeyError: + pass +else: + raise AssertionError("missing column names were masked by positional fallback") +`, dir) cmd := exec.Command("python3", "-c", script) cmd.Env = append(os.Environ(), "PYTHONPYCACHEPREFIX="+filepath.Join(dir, "pycache")) if out, err := cmd.CombinedOutput(); err != nil { @@ -360,14 +409,14 @@ func TestGeneratedSQLAlchemyClosesResultsWithMockAdapter(t *testing.T) { } script := fmt.Sprintf(`import sys sys.path.insert(0, %q) -from db.queries import Querier, _row_value -assert _row_value((3,), "count", 0) == 3 -assert _row_value((4,), "index", 0) == 4 +from db.queries import Querier +class Row: + def __init__(self, values): self._mapping = values class R: def __init__(self, fail): self.closed = False; self.fail = fail def fetchall(self): if self.fail: raise RuntimeError("fetch failure") - return [{'id': 7, 'display_name': None}] + return [Row({'id': 7, 'display_name': None})] def close(self): self.closed = True class C: def __init__(self): self.calls = []; self.fail = False @@ -411,7 +460,7 @@ func TestGeneratedDBAPIClosesCursorAndPreservesTransaction(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "ydb.py"), []byte("class PrimitiveType:\n Uint64 = 'Uint64'\nclass OptionalType:\n def __init__(self, item): self.item=item\n"), 0600); err != nil { t.Fatal(err) } - script := fmt.Sprintf("import sys; sys.path.insert(0, %q); from db.queries import Querier\nclass Cur:\n rowcount=1\n def execute(self, sql, params): self.params=params\n def fetchall(self): return [{'id': 7, 'display_name': None}]\n def close(self): self.closed=True\nclass C:\n def __init__(self): self.cur=Cur(); self.commits=0\n def cursor(self): return self.cur\nc=C(); row=Querier(c).get_author(7); assert row.id == 7 and c.cur.closed and c.commits == 0 and c.cur.params['$id'][0] == 7\n", dir) + script := fmt.Sprintf("import sys; sys.path.insert(0, %q); from db.queries import Querier\nclass Cur:\n rowcount=1\n def execute(self, sql, params): self.params=params\n def fetchall(self): return [(7, None)]\n def close(self): self.closed=True\nclass C:\n def __init__(self): self.cur=Cur(); self.commits=0\n def cursor(self): return self.cur\nc=C(); row=Querier(c).get_author(7); assert row.id == 7 and c.cur.closed and c.commits == 0 and c.cur.params['$id'][0] == 7\n", dir) cmd := exec.Command("python3", "-c", script) cmd.Env = append(os.Environ(), "PYTHONPYCACHEPREFIX="+filepath.Join(dir, "pycache")) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 91b2d62..a2baaad 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -44,11 +44,11 @@ type Go struct { } type Python struct { - Package string `yaml:"package"` - Out string `yaml:"out"` - Runtime string `yaml:"runtime"` - EmitSyncQuerier *bool `yaml:"emit_sync_querier"` - EmitAsyncQuerier bool `yaml:"emit_async_querier"` + Package yaml.Node `yaml:"package"` // Retained only to diagnose the formerly ignored option. + Out string `yaml:"out"` + Runtime string `yaml:"runtime"` + EmitSyncQuerier *bool `yaml:"emit_sync_querier"` + EmitAsyncQuerier bool `yaml:"emit_async_querier"` } type CPP struct { @@ -205,12 +205,12 @@ func Parse(data []byte) (*Config, error) { } } if p := s.Gen.Python; p != nil { + if p.Package.Kind != 0 { + return nil, fmt.Errorf("sql[%d].gen.python.package is unsupported; remove it: the Python package directory is selected with out", i) + } if p.Out == "" { return nil, fmt.Errorf("sql[%d].gen.python.out is required", i) } - if p.Package == "" { - p.Package = filepath.Base(filepath.Clean(p.Out)) - } if p.Runtime == "" { p.Runtime = "ydb" } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 41f0734..c877fba 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -37,6 +37,7 @@ func TestRejectUnsupportedConfiguration(t *testing.T) { {"codegen", base + " codegen: []\n", "migrate"}, {"unknown", base + " surprise: true\n", "field surprise"}, {"option", base + " gen:\n go:\n out: db\n emit_prepared_queries: true\n", "field emit_prepared_queries"}, + {"ignored Python package", base + " gen:\n python:\n out: py\n package: ignored\n", "Python package directory is selected with out"}, {"engine", strings.Replace(base, "ydb", "postgresql", 1), "engine must be ydb"}, {"documents", base + "---\nversion: '2'\n", "exactly one"}, {"empty path", strings.Replace(base, "s.sql", "''", 1), "non-empty path"}, diff --git a/internal/endtoend/golden_test.go b/internal/endtoend/golden_test.go index c621079..d6d4691 100644 --- a/internal/endtoend/golden_test.go +++ b/internal/endtoend/golden_test.go @@ -84,10 +84,9 @@ func copyFixture(t *testing.T, src, dst string) { } from, to := filepath.Join(src, e.Name()), filepath.Join(dst, e.Name()) if e.IsDir() { - if err := os.MkdirAll(to, 0755); err != nil { + if err := os.CopyFS(to, os.DirFS(from)); err != nil { t.Fatal(err) } - copyFixture(t, from, to) continue } data, err := os.ReadFile(from) @@ -111,30 +110,9 @@ func updateExpected(t *testing.T, fixture, dir string) { } else if err != nil { t.Fatal(err) } - copyTree(t, from, filepath.Join(expected, root)) - } -} - -func copyTree(t *testing.T, src, dst string) { - t.Helper() - entries, err := os.ReadDir(src) - if err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(dst, 0755); err != nil { - t.Fatal(err) - } - for _, e := range entries { - from, to := filepath.Join(src, e.Name()), filepath.Join(dst, e.Name()) - if e.IsDir() { - copyTree(t, from, to) - continue - } - data, err := os.ReadFile(from) - if err != nil { + if err := os.CopyFS(filepath.Join(expected, root), os.DirFS(from)); err != nil { t.Fatal(err) } - write(t, to, data) } } diff --git a/internal/endtoend/testdata/authors/expected/db/queries.sql.go b/internal/endtoend/testdata/authors/expected/db/queries.sql.go index 5dedf68..c0569da 100644 --- a/internal/endtoend/testdata/authors/expected/db/queries.sql.go +++ b/internal/endtoend/testdata/authors/expected/db/queries.sql.go @@ -7,12 +7,12 @@ import ( "database/sql" ) -const getAuthor = "-- name: GetAuthor :one\n" + +const queryGetAuthor = "-- name: GetAuthor :one\n" + "DECLARE $author_id AS Uint64;\n" + "SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;" -func (q *Queries) GetAuthor(ctx context.Context, author_id uint64) (GetAuthorRow, error) { +func (q *Queries) GetAuthor(ctx context.Context, arg uint64) (GetAuthorRow, error) { var row GetAuthorRow - err := q.db.QueryRowContext(ctx, getAuthor, sql.Named("author_id", author_id)).Scan(&row.ID, &row.Name, &row.Bio) + err := q.db.QueryRowContext(ctx, queryGetAuthor, sql.Named("author_id", arg)).Scan(&row.ID, &row.Name, &row.Bio) return row, err } diff --git a/internal/endtoend/testdata/authors/expected/py/queries.py b/internal/endtoend/testdata/authors/expected/py/queries.py index 049254e..4b26c43 100644 --- a/internal/endtoend/testdata/authors/expected/py/queries.py +++ b/internal/endtoend/testdata/authors/expected/py/queries.py @@ -9,15 +9,6 @@ SELECT `id`, `name`, `bio` FROM `authors` WHERE `id` = $author_id;""" -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) - def _typed(value, typ): return ydb.TypedValue(value, typ) @@ -29,12 +20,12 @@ def __init__(self, pool: ydb.QuerySessionPool): def get_author(self, author_id: int) -> Optional[models.Author]: parameters = {"$author_id": _typed(author_id, ydb.PrimitiveType.Uint64)} result_sets = self._pool.execute_with_retries(SQL_GET_AUTHOR, parameters) - rows = result_sets[0].rows if result_sets else [] + rows = result_sets[0].rows if not rows: return None row = rows[0] return models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), - bio=_row_value(row, "bio", 2), + id=row["id"], + name=row["name"], + bio=row["bio"], ) diff --git a/internal/endtoend/testdata/join_alias/expected/db/queries.sql.go b/internal/endtoend/testdata/join_alias/expected/db/queries.sql.go index e352ee2..e0e0127 100644 --- a/internal/endtoend/testdata/join_alias/expected/db/queries.sql.go +++ b/internal/endtoend/testdata/join_alias/expected/db/queries.sql.go @@ -4,16 +4,14 @@ package db import ( "context" - ydb "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/query" ) -const listAuthorBooks = `-- name: ListAuthorBooks :many +const queryListAuthorBooks = `-- name: ListAuthorBooks :many SELECT a.id AS author_id, a.name AS author_name, b.title AS book_title FROM authors AS a JOIN books AS b ON a.id = b.author_id;` func (q *Queries) ListAuthorBooks(ctx context.Context) ([]ListAuthorBooksRow, error) { - result, err := q.db.QueryResultSet(ctx, listAuthorBooks) + result, err := q.db.QueryResultSet(ctx, queryListAuthorBooks) if err != nil { return []ListAuthorBooksRow(nil), err } diff --git a/internal/endtoend/testdata/join_alias/expected/py/queries.py b/internal/endtoend/testdata/join_alias/expected/py/queries.py index f6e749e..d71ce2e 100644 --- a/internal/endtoend/testdata/join_alias/expected/py/queries.py +++ b/internal/endtoend/testdata/join_alias/expected/py/queries.py @@ -5,23 +5,14 @@ import ydb from sqlalchemy import text from sqlalchemy.engine import Connection -def _typed(value, typ): - return (value, typ) - SQL_LIST_AUTHOR_BOOKS = """-- name\\: ListAuthorBooks \\:many SELECT a.id AS author_id, a.name AS author_name, b.title AS book_title FROM authors AS a JOIN books AS b ON a.id = b.author_id;""" -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) +def _typed(value, typ): + return (value, typ) class Querier: @@ -36,7 +27,7 @@ def list_author_books(self) -> Iterable[models.ListAuthorBooksRow]: finally: result.close() return (models.ListAuthorBooksRow( - author_id=_row_value(row, "author_id", 0), - author_name=_row_value(row, "author_name", 1), - book_title=_row_value(row, "book_title", 2), + author_id=row._mapping["author_id"], + author_name=row._mapping["author_name"], + book_title=row._mapping["book_title"], ) for row in rows) diff --git a/internal/endtoend/testdata/migrations/expected/db/queries.sql.go b/internal/endtoend/testdata/migrations/expected/db/queries.sql.go index 9a35408..ffc049a 100644 --- a/internal/endtoend/testdata/migrations/expected/db/queries.sql.go +++ b/internal/endtoend/testdata/migrations/expected/db/queries.sql.go @@ -8,12 +8,12 @@ import ( "github.com/ydb-platform/ydb-go-sdk/v3/query" ) -const getAuthor = `-- name: GetAuthor :one +const queryGetAuthor = `-- name: GetAuthor :one DECLARE $id AS Uint64; SELECT * FROM authors WHERE id = $id;` -func (q *Queries) GetAuthor(ctx context.Context, id uint64) (GetAuthorRow, error) { - result, err := q.db.QueryRow(ctx, getAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$id").Uint64(id).Build())) +func (q *Queries) GetAuthor(ctx context.Context, arg uint64) (GetAuthorRow, error) { + result, err := q.db.QueryRow(ctx, queryGetAuthor, query.WithParameters(ydb.ParamsBuilder().Param("$id").Uint64(arg).Build())) if err != nil { return GetAuthorRow{}, err } diff --git a/internal/endtoend/testdata/migrations/expected/py/queries.py b/internal/endtoend/testdata/migrations/expected/py/queries.py index 5fc1fbb..e178034 100644 --- a/internal/endtoend/testdata/migrations/expected/py/queries.py +++ b/internal/endtoend/testdata/migrations/expected/py/queries.py @@ -5,23 +5,14 @@ import ydb from sqlalchemy import text from sqlalchemy.engine import Connection -def _typed(value, typ): - return (value, typ) - SQL_GET_AUTHOR = """-- name\\: GetAuthor \\:one DECLARE $id AS Uint64; SELECT * FROM authors WHERE id = :id;""" -def _row_value(row, name, index): - try: - return row[name] - except (KeyError, IndexError, TypeError): - try: - return row[index] - except (KeyError, IndexError, TypeError): - return getattr(row, name) +def _typed(value, typ): + return (value, typ) class Querier: @@ -39,6 +30,6 @@ def get_author(self, id: int) -> Optional[models.Author]: return None row = rows[0] return models.Author( - id=_row_value(row, "id", 0), - name=_row_value(row, "name", 1), + id=row._mapping["id"], + name=row._mapping["name"], ) diff --git a/internal/model/model.go b/internal/model/model.go index 61c073a..99b5904 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -25,12 +25,11 @@ func (d Diagnostic) Error() string { } // Type preserves YQL type identity, including nested containers and nullability. -// Kind is the canonical YQL spelling (e.g. Uint64, Utf8, Optional, List, Struct). +// Kind is the canonical YQL spelling (e.g. Uint64, Utf8, Optional, List). type Type struct { Kind string Elem *Type Key *Type - Fields []Field Items []Type Precision int Scale int @@ -45,10 +44,6 @@ func (t Type) UnwrapOptional() Type { return t } -type Field struct { - Name string - Type Type -} type Column struct { Name string Type Type diff --git a/scripts/release b/scripts/release new file mode 100755 index 0000000..f5c051b --- /dev/null +++ b/scripts/release @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly module_path='github.com/ydb-platform/sqlc-engine-ydb' +declare -a cleanup_paths=() + +cleanup() { + set +u + local path + for path in "${cleanup_paths[@]}"; do + [[ -n $path ]] && rm -rf -- "$path" + done +} +trap cleanup EXIT + +usage() { + cat >&2 <<'EOF' +usage: + scripts/release check [notes-file] + scripts/release build + scripts/release verify +EOF + exit 2 +} + +fail() { + echo "release: $*" >&2 + exit 1 +} + +version_from_tag() { + local tag=${1-} + if [[ ! $tag =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-rc(0|[1-9][0-9]*))?$ ]]; then + fail "tag must be a semantic version such as v1.2.3 or v1.2.3-rc0: ${tag:-}" + fi + printf '%s\n' "${tag#v}" +} + +archive_name() { + local version=$1 goos=$2 goarch=$3 + local extension=tar.gz + if [[ $goos == windows ]]; then + extension=zip + fi + printf 'sqlc-ydb_%s_%s_%s.%s\n' "$version" "$goos" "$goarch" "$extension" +} + +binary_name() { + if [[ $1 == windows ]]; then + printf 'sqlc-ydb.exe\n' + else + printf 'sqlc-ydb\n' + fi +} + +check_release() { + [[ $# -ge 1 && $# -le 2 ]] || usage + local tag=$1 notes=${2-} version changelog binary temporary= + version=$(version_from_tag "$tag") + changelog=${SQLC_YDB_CHANGELOG:-CHANGELOG.md} + binary=${SQLC_YDB_RELEASE_BINARY:-} + if [[ -z $binary ]]; then + temporary=$(mktemp -d) + cleanup_paths+=("$temporary") + binary="$temporary/sqlc-ydb" + local flags= + if [[ $version == *-rc* ]]; then + flags="-X ${module_path}/internal/cli.Version=${version}" + fi + CGO_ENABLED=0 go build -trimpath -ldflags "$flags" -o "$binary" ./cmd/sqlc-ydb + fi + [[ -x $binary ]] || fail "release binary is not executable: $binary" + local actual + actual=$("$binary" version) + [[ $actual == "$version" ]] || fail "tag $tag does not match CLI version $actual" + local notes_tmp + if [[ -n $notes ]]; then + notes_tmp=$notes + mkdir -p "$(dirname "$notes_tmp")" + else + notes_tmp=$(mktemp) + fi + python3 scripts/release-version.py notes --tag "$tag" --changelog "$changelog" --notes "$notes_tmp" + if [[ -z $notes ]]; then + rm -f "$notes_tmp" + fi + if [[ -n $temporary ]]; then + rm -rf "$temporary" + fi +} + +build_archive() { + [[ $# == 5 ]] || usage + local tag=$1 commit=$2 goos=$3 goarch=$4 dist=$5 + local version + version=$(version_from_tag "$tag") + [[ $commit =~ ^[0-9a-fA-F]{40}$ ]] || fail "commit must be a full 40-character Git object ID" + check_release "$tag" + case "$goos/$goarch" in + linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64|windows/arm64) ;; + *) fail "target is outside the release matrix: $goos/$goarch" ;; + esac + + mkdir -p "$dist" + local temporary base binary archive + temporary=$(mktemp -d) + cleanup_paths+=("$temporary") + base="sqlc-ydb_${version}_${goos}_${goarch}" + mkdir -p "$temporary/$base" + binary=$(binary_name "$goos") + GOOS=$goos GOARCH=$goarch CGO_ENABLED=0 \ + go build -trimpath -ldflags "-s -w -X ${module_path}/internal/cli.Version=${version} -X ${module_path}/internal/cli.Commit=${commit}" \ + -o "$temporary/$base/$binary" ./cmd/sqlc-ydb + cp LICENSE "$temporary/$base/LICENSE" + archive="$dist/$(archive_name "$version" "$goos" "$goarch")" + if [[ $goos == windows ]]; then + archive=$(cd "$(dirname "$archive")" && pwd)/$(basename "$archive") + (cd "$temporary" && zip -q -r "$archive" "$base") + else + tar -C "$temporary" -czf "$archive" "$base" + fi + rm -rf "$temporary" +} + +hash_file() { + local path=$1 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + else + shasum -a 256 "$path" | awk '{print $1}' + fi +} + +verify_archive() { + local archive=$1 base=$2 binary=$3 goos=$4 goarch=$5 commit=$6 temporary=$7 + local listing="$temporary/listing-$goos-$goarch" + if [[ $archive == *.zip ]]; then + unzip -Z1 "$archive" | sed '/\/$/d' | LC_ALL=C sort >"$listing" + else + tar -tzf "$archive" | sed '/\/$/d' | LC_ALL=C sort >"$listing" + fi + printf '%s\n%s\n' "$base/LICENSE" "$base/$binary" | LC_ALL=C sort >"$temporary/expected" + cmp -s "$temporary/expected" "$listing" || fail "unexpected archive contents: $archive" + local inspect="$temporary/inspect-$goos-$goarch" + mkdir -p "$inspect" + if [[ $archive == *.zip ]]; then + unzip -q "$archive" -d "$inspect" + else + tar -C "$inspect" -xzf "$archive" + fi + local metadata="$temporary/metadata-$goos-$goarch" + go version -m "$inspect/$base/$binary" >"$metadata" + grep -Fq $'\tpath\t'"${module_path}/cmd/sqlc-ydb" "$metadata" || fail "wrong Go command in $archive" + grep -Fq $'\tbuild\t'"GOOS=$goos" "$metadata" || fail "wrong GOOS metadata in $archive" + grep -Fq $'\tbuild\t'"GOARCH=$goarch" "$metadata" || fail "wrong GOARCH metadata in $archive" + grep -Fq $'\tbuild\tCGO_ENABLED=0' "$metadata" || fail "CGO is enabled in $archive" + grep -Fq $'\tbuild\t'"vcs.revision=$commit" "$metadata" || fail "wrong VCS revision in $archive" +} + +smoke_archive() { + local archive=$1 base=$2 binary=$3 version=$4 commit=$5 temporary=$6 + mkdir -p "$temporary/smoke" + if [[ $archive == *.zip ]]; then + unzip -q "$archive" -d "$temporary/smoke" + else + tar -C "$temporary/smoke" -xzf "$archive" + fi + local executable="$temporary/smoke/$base/$binary" + local actual verbose + actual=$("$executable" version) + [[ $actual == "$version" ]] || fail "host smoke returned version $actual, expected $version" + verbose=$("$executable" version --verbose) + [[ $verbose == *"$version"* && $verbose == *"$commit"* ]] || \ + fail "host verbose version does not contain version and commit" + "$executable" compile -f examples/authors/sqlc.yaml + "$executable" diff -f examples/authors/sqlc.yaml +} + +verify_release() { + [[ $# == 3 ]] || usage + local tag=$1 dist=$2 targets=$3 version commit_file commit host_os host_arch + version=$(version_from_tag "$tag") + [[ -d $dist ]] || fail "distribution directory not found: $dist" + [[ -f $targets ]] || fail "targets file not found: $targets" + commit_file="$dist/COMMIT" + [[ -f $commit_file ]] || fail "missing release commit file: $commit_file" + commit=$(cat "$commit_file") + [[ $commit =~ ^[0-9a-fA-F]{40}$ ]] || fail "COMMIT must contain one full Git object ID" + host_os=$(go env GOHOSTOS) + host_arch=$(go env GOHOSTARCH) + local temporary checksums count=0 + temporary=$(mktemp -d) + cleanup_paths+=("$temporary") + checksums="$temporary/SHA256SUMS" + : >"$checksums" + local goos goarch extra archive base binary hash + while read -r goos goarch extra; do + [[ -z ${goos:-} || $goos == \#* ]] && continue + [[ -z ${extra:-} ]] || fail "invalid targets line: $goos $goarch $extra" + case "$goos/$goarch" in + linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64|windows/arm64) ;; + *) fail "target is outside the release matrix: $goos/$goarch" ;; + esac + archive="$dist/$(archive_name "$version" "$goos" "$goarch")" + [[ -f $archive ]] || fail "missing archive: $(basename "$archive")" + base=${archive##*/sqlc-ydb_} + base=${base%.tar.gz} + base=${base%.zip} + base="sqlc-ydb_${base}" + binary=$(binary_name "$goos") + verify_archive "$archive" "$base" "$binary" "$goos" "$goarch" "$commit" "$temporary" + hash=$(hash_file "$archive") + printf '%s %s\n' "$hash" "$(basename "$archive")" >>"$checksums" + if [[ $goos == "$host_os" && $goarch == "$host_arch" ]]; then + smoke_archive "$archive" "$base" "$binary" "$version" "$commit" "$temporary" + fi + count=$((count + 1)) + done <"$targets" + [[ $count -gt 0 ]] || fail "targets file is empty" + local actual_count + actual_count=$(find "$dist" -maxdepth 1 -type f \( -name '*.tar.gz' -o -name '*.zip' \) | wc -l | tr -d ' ') + [[ $actual_count == "$count" ]] || fail "distribution has $actual_count archives, expected $count" + local generated_checksums="$temporary/SHA256SUMS.sorted" + LC_ALL=C sort -k2 "$checksums" >"$generated_checksums" + if [[ -f $dist/SHA256SUMS ]]; then + cmp -s "$dist/SHA256SUMS" "$generated_checksums" || fail "SHA256SUMS does not match the release archives" + else + cp "$generated_checksums" "$dist/SHA256SUMS" + fi + rm -rf "$temporary" +} + +command=${1-} +[[ -n $command ]] || usage +shift +case $command in + check) check_release "$@" ;; + build) build_archive "$@" ;; + verify) verify_release "$@" ;; + *) usage ;; +esac diff --git a/scripts/release-targets b/scripts/release-targets new file mode 100644 index 0000000..ee83ccf --- /dev/null +++ b/scripts/release-targets @@ -0,0 +1,6 @@ +linux amd64 +linux arm64 +darwin amd64 +darwin arm64 +windows amd64 +windows arm64 diff --git a/scripts/release-version.py b/scripts/release-version.py new file mode 100644 index 0000000..f529460 --- /dev/null +++ b/scripts/release-version.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Prepare release metadata and version files from the Unreleased changelog.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + + +CHANGELOG = Path("CHANGELOG.md") +VERSION_SOURCE = Path("internal/cli/cli.go") +STABLE_HEADING = re.compile(r"^## v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +VERSION_HEADING = re.compile(r"^## v\S+") +VERSION_DECLARATION = re.compile(r'^var Version = "(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)"$', re.MULTILINE) +CHANGE_HEADING = re.compile(r"^### (Added|Changed|Deprecated|Removed|Fixed|Security|Compatibility)$", re.MULTILINE) +BULLET = re.compile(r"^- .+\S", re.MULTILINE) + + +class ReleaseError(Exception): + pass + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + prepare = commands.add_parser("prepare") + prepare.add_argument("--part", required=True, choices=("PATCH", "MINOR", "MAJOR")) + prepare.add_argument("--rc", required=True, choices=("true", "false")) + prepare.add_argument("--notes", required=True, type=Path) + notes = commands.add_parser("notes") + notes.add_argument("--tag", required=True) + notes.add_argument("--changelog", required=True, type=Path) + notes.add_argument("--notes", required=True, type=Path) + return parser.parse_args() + + +def version_text(version: tuple[int, int, int]) -> str: + return ".".join(str(value) for value in version) + + +def bump(version: tuple[int, int, int], part: str) -> tuple[int, int, int]: + major, minor, patch = version + if part == "MAJOR": + return major + 1, 0, 0 + if part == "MINOR": + return major, minor + 1, 0 + return major, minor, patch + 1 + + +def section(text: str, heading: str) -> str: + lines = text.splitlines(keepends=True) + positions = [index for index, line in enumerate(lines) if line.rstrip("\r\n") == heading] + if len(positions) != 1: + raise ReleaseError(f"CHANGELOG must contain exactly one {heading} section") + start = positions[0] + 1 + end = next( + (index for index in range(start, len(lines)) if lines[index].startswith("## ")), + len(lines), + ) + return "".join(lines[start:end]).strip() + + +def validate_entries(content: str, label: str) -> None: + has_heading = False + has_entry = False + active_heading = False + for line in content.splitlines(): + if line.startswith("### "): + active_heading = CHANGE_HEADING.fullmatch(line) is not None + if not active_heading: + raise ReleaseError(f"{label} section has unsupported change heading: {line}") + has_heading = True + elif active_heading and BULLET.fullmatch(line): + has_entry = True + if not has_heading: + raise ReleaseError(f"{label} section has no supported change heading") + if not has_entry: + raise ReleaseError(f"{label} section has no change entries") + + +def parse_changelog(text: str) -> tuple[str, list[tuple[tuple[int, int, int], str]], str, str]: + lines = text.splitlines(keepends=True) + unreleased = [index for index, line in enumerate(lines) if line.rstrip("\r\n") == "## Unreleased"] + if len(unreleased) != 1: + raise ReleaseError("CHANGELOG must contain exactly one ## Unreleased section") + unreleased_index = unreleased[0] + for line in lines[:unreleased_index]: + if line.startswith("## "): + raise ReleaseError("## Unreleased must be the first level-two CHANGELOG section") + + stable: list[tuple[tuple[int, int, int], str]] = [] + next_section = len(lines) + for index in range(unreleased_index + 1, len(lines)): + heading = lines[index].rstrip("\r\n") + match = STABLE_HEADING.fullmatch(heading) + if match: + if next_section == len(lines): + next_section = index + stable.append((tuple(int(value) for value in match.groups()), heading)) + elif heading.startswith("## "): + if VERSION_HEADING.match(heading): + raise ReleaseError(f"invalid stable CHANGELOG heading: {heading}") + raise ReleaseError(f"unexpected CHANGELOG section: {heading}") + versions = [version for version, _ in stable] + if len(set(versions)) != len(versions): + raise ReleaseError("CHANGELOG contains duplicate stable version sections") + if any(older >= newer for newer, older in zip(versions, versions[1:])): + raise ReleaseError("stable CHANGELOG sections must be in descending version order") + + pending = "".join(lines[unreleased_index + 1 : next_section]).strip() + prefix = "".join(lines[: unreleased_index + 1]).rstrip() + "\n" + history = "".join(lines[next_section:]).lstrip("\r\n") + return pending, stable, prefix, history + + +def parse_source_version(text: str) -> tuple[tuple[int, int, int], re.Match[str]]: + matches = list(VERSION_DECLARATION.finditer(text)) + if len(matches) != 1: + raise ReleaseError('CLI source must contain exactly one var Version = "X.Y.Z" declaration') + match = matches[0] + return tuple(int(value) for value in match.groups()), match + + +def git_tags() -> list[str]: + result = subprocess.run( + ["git", "tag", "--list"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + message = result.stderr.strip() or "git tag --list failed" + raise ReleaseError(message) + return result.stdout.splitlines() + + +def stable_tags(tags: list[str]) -> set[tuple[int, int, int]]: + result: set[tuple[int, int, int]] = set() + for tag in tags: + match = re.fullmatch(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", tag) + if match: + result.add(tuple(int(value) for value in match.groups())) + return result + + +def target_version( + part: str, + source: tuple[int, int, int], + history: list[tuple[tuple[int, int, int], str]], +) -> tuple[int, int, int]: + if not history: + if part != "PATCH" or source != (0, 0, 1): + raise ReleaseError("the first release must be PATCH 0.0.1 with CLI Version 0.0.1") + return 0, 0, 1 + latest = history[0][0] + if source != latest: + raise ReleaseError( + f"CLI Version {version_text(source)} does not match latest stable CHANGELOG version {version_text(latest)}" + ) + return bump(latest, part) + + +def validate_latest_tag( + tags: set[tuple[int, int, int]], history: list[tuple[tuple[int, int, int], str]] +) -> None: + if not history: + if tags: + raise ReleaseError("first-release state cannot contain stable version tags") + return + latest = history[0][0] + if latest not in tags: + raise ReleaseError(f"latest stable CHANGELOG version v{version_text(latest)} has no Git tag") + higher = sorted(version for version in tags if version > latest) + if higher: + raise ReleaseError("stable Git tag is newer than CHANGELOG history: v" + version_text(higher[-1])) + + +def next_rc(tags: list[str], version: str) -> int: + pattern = re.compile(rf"v{re.escape(version)}-rc(0|[1-9][0-9]*)") + numbers = [int(match.group(1)) for tag in tags if (match := pattern.fullmatch(tag))] + return max(numbers, default=-1) + 1 + + +def stage_writes(changes: list[tuple[Path, str]]) -> None: + staged: list[tuple[Path, Path]] = [] + try: + for path, content in changes: + path.parent.mkdir(parents=True, exist_ok=True) + mode = path.stat().st_mode & 0o777 if path.exists() else 0o644 + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, mode) + staged.append((temporary, path)) + for temporary, path in staged: + os.replace(temporary, path) + finally: + for temporary, _ in staged: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def write_notes(changelog: Path, tag: str, notes: Path) -> None: + tag_match = re.fullmatch( + r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-rc(0|[1-9][0-9]*))?", + tag, + ) + if not tag_match: + raise ReleaseError(f"invalid release tag: {tag}") + if changelog.resolve() == notes.resolve(): + raise ReleaseError("notes path must differ from CHANGELOG") + text = changelog.read_text(encoding="utf-8") + source_heading = "## Unreleased" if tag_match.group(4) is not None else f"## {tag}" + content = section(text, source_heading) + validate_entries(content, source_heading.removeprefix("## ")) + stage_writes([(notes, f"## {tag}\n\n{content}\n")]) + + +def prepare_release(args: argparse.Namespace) -> dict[str, object]: + changelog_path = CHANGELOG.resolve() + source_path = VERSION_SOURCE.resolve() + notes_path = args.notes.resolve() + if notes_path in (changelog_path, source_path): + raise ReleaseError("notes path must differ from CHANGELOG and CLI source") + changelog = CHANGELOG.read_text(encoding="utf-8") + source = VERSION_SOURCE.read_text(encoding="utf-8") + pending, history, changelog_prefix, old_history = parse_changelog(changelog) + validate_entries(pending, "Unreleased") + source_version, source_match = parse_source_version(source) + tags = git_tags() + target = target_version(args.part, source_version, history) + target_text = version_text(target) + known_stable_tags = stable_tags(tags) + if target in known_stable_tags: + raise ReleaseError(f"stable tag v{target_text} already exists") + validate_latest_tag(known_stable_tags, history) + + release_candidate = args.rc == "true" + if release_candidate: + tag = f"v{target_text}-rc{next_rc(tags, target_text)}" + else: + tag = f"v{target_text}" + notes = f"## {tag}\n\n{pending}\n" + changes = [(notes_path, notes)] + if not release_candidate: + new_changelog = f"{changelog_prefix}\n## {tag}\n\n{pending}\n" + if old_history: + new_changelog += f"\n{old_history}" + new_source = source[: source_match.start(1)] + target_text + source[source_match.end(3) :] + changes.extend(((CHANGELOG, new_changelog), (VERSION_SOURCE, new_source))) + stage_writes(changes) + return {"tag": tag, "version": tag.removeprefix("v"), "release_candidate": release_candidate} + + +def main() -> int: + args = parse_args() + try: + if args.command == "notes": + write_notes(args.changelog, args.tag, args.notes) + else: + result = prepare_release(args) + print(json.dumps(result, separators=(",", ":"))) + return 0 + except (OSError, ReleaseError) as error: + print(f"release-version: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_release_version.py b/scripts/test_release_version.py new file mode 100644 index 0000000..d0d2437 --- /dev/null +++ b/scripts/test_release_version.py @@ -0,0 +1,248 @@ +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).with_name("release-version.py").resolve() +PENDING = "### Added\n\n- New release behavior." +OLD_SECTION = "## v1.2.3\n\n### Fixed\n\n- Previous fix.\n" + + +class ReleaseRepo: + def __init__(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + (self.root / "internal/cli").mkdir(parents=True) + self.git("init", "-q") + self.git("config", "user.email", "test@example.com") + self.git("config", "user.name", "Release Test") + self.write_state("0.0.1", PENDING) + self.git("add", ".") + self.git("commit", "-qm", "fixture") + + def close(self): + self.temporary.cleanup() + + def git(self, *args): + return subprocess.run( + ["git", *args], cwd=self.root, check=True, text=True, capture_output=True + ) + + def tag(self, *tags): + for tag in tags: + self.git("tag", tag) + + def clear_tags(self): + tags = self.git("tag", "--list").stdout.splitlines() + if tags: + self.git("tag", "-d", *tags) + + def write_state(self, version, pending, history="", intro="# Changelog\n\nIntro.\n"): + changelog = f"{intro}\n## Unreleased\n\n{pending}" + if history: + changelog += f"\n\n{history.lstrip()}" + if not changelog.endswith("\n"): + changelog += "\n" + (self.root / "CHANGELOG.md").write_text(changelog, encoding="utf-8") + (self.root / "internal/cli/cli.go").write_text( + f'package cli\n\nvar Version = "{version}"\nvar Commit = "unknown"\n', + encoding="utf-8", + ) + + def prepare(self, part="PATCH", rc="false", notes="notes.md"): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "prepare", + "--part", + part, + "--rc", + rc, + "--notes", + notes, + ], + cwd=self.root, + text=True, + capture_output=True, + ) + + def notes(self, tag, changelog="CHANGELOG.md", notes="notes.md"): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "notes", + "--tag", + tag, + "--changelog", + changelog, + "--notes", + notes, + ], + cwd=self.root, + text=True, + capture_output=True, + ) + + +class ReleaseVersionTest(unittest.TestCase): + def setUp(self): + self.repo = ReleaseRepo() + + def tearDown(self): + self.repo.close() + + def assert_success(self, result): + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, "") + + def assert_failure_without_writes(self, result, changelog, source): + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertEqual((self.repo.root / "CHANGELOG.md").read_text(), changelog) + self.assertEqual((self.repo.root / "internal/cli/cli.go").read_text(), source) + self.assertFalse((self.repo.root / "notes.md").exists()) + + def test_initial_stable_release_is_001_patch(self): + result = self.repo.prepare() + self.assert_success(result) + self.assertEqual( + json.loads(result.stdout), + {"tag": "v0.0.1", "version": "0.0.1", "release_candidate": False}, + ) + changelog = (self.repo.root / "CHANGELOG.md").read_text() + self.assertIn("## Unreleased\n\n## v0.0.1\n\n" + PENDING, changelog) + self.assertEqual(changelog.count("- New release behavior."), 1) + self.assertEqual((self.repo.root / "notes.md").read_text(), "## v0.0.1\n\n" + PENDING + "\n") + self.assertIn('var Version = "0.0.1"', (self.repo.root / "internal/cli/cli.go").read_text()) + + def test_first_release_rejects_other_parts_and_versions(self): + for part, version in (("MINOR", "0.0.1"), ("MAJOR", "0.0.1"), ("PATCH", "0.0.2")): + with self.subTest(part=part, version=version): + self.repo.clear_tags() + self.repo.write_state(version, PENDING) + before_changelog = (self.repo.root / "CHANGELOG.md").read_text() + before_source = (self.repo.root / "internal/cli/cli.go").read_text() + result = self.repo.prepare(part=part) + self.assertIn("first release", result.stderr) + self.assert_failure_without_writes(result, before_changelog, before_source) + + def test_patch_minor_and_major_bumps_reset_lower_parts(self): + expected = {"PATCH": "1.2.4", "MINOR": "1.3.0", "MAJOR": "2.0.0"} + for part, target in expected.items(): + with self.subTest(part=part): + self.repo.clear_tags() + self.repo.write_state("1.2.3", PENDING, OLD_SECTION) + self.repo.tag("v1.2.3") + result = self.repo.prepare(part=part) + self.assert_success(result) + self.assertEqual(json.loads(result.stdout)["version"], target) + self.assertIn(f'var Version = "{target}"', (self.repo.root / "internal/cli/cli.go").read_text()) + + def test_rc_uses_highest_numeric_suffix_and_leaves_inputs_untouched(self): + self.repo.write_state("1.2.3", PENDING, OLD_SECTION) + self.repo.tag("v1.2.3", "v1.2.4-rc0", "v1.2.4-rc2", "v1.2.4-rc10", "v1.2.4-rc03") + changelog = (self.repo.root / "CHANGELOG.md").read_bytes() + source = (self.repo.root / "internal/cli/cli.go").read_bytes() + result = self.repo.prepare(rc="true") + self.assert_success(result) + self.assertEqual( + json.loads(result.stdout), + {"tag": "v1.2.4-rc11", "version": "1.2.4-rc11", "release_candidate": True}, + ) + self.assertEqual((self.repo.root / "CHANGELOG.md").read_bytes(), changelog) + self.assertEqual((self.repo.root / "internal/cli/cli.go").read_bytes(), source) + self.assertEqual((self.repo.root / "notes.md").read_text(), "## v1.2.4-rc11\n\n" + PENDING + "\n") + + def test_stable_moves_pending_once_and_preserves_history(self): + self.repo.write_state("1.2.3", PENDING, OLD_SECTION) + self.repo.tag("v1.2.3") + result = self.repo.prepare(part="MINOR") + self.assert_success(result) + changelog = (self.repo.root / "CHANGELOG.md").read_text() + self.assertEqual(changelog.count(PENDING), 1) + self.assertIn("## Unreleased\n\n## v1.3.0\n\n" + PENDING, changelog) + self.assertTrue(changelog.endswith(OLD_SECTION)) + notes_result = self.repo.notes("v1.3.0", notes="stable-notes.md") + self.assert_success(notes_result) + self.assertEqual(notes_result.stdout, "") + self.assertEqual((self.repo.root / "stable-notes.md").read_text(), "## v1.3.0\n\n" + PENDING + "\n") + + def test_notes_command_extracts_unreleased_for_rc_without_git_or_source(self): + (self.repo.root / ".git").rename(self.repo.root / "git-away") + (self.repo.root / "internal/cli/cli.go").unlink() + result = self.repo.notes("v0.0.1-rc0") + self.assert_success(result) + self.assertEqual(result.stdout, "") + self.assertEqual((self.repo.root / "notes.md").read_text(), "## v0.0.1-rc0\n\n" + PENDING + "\n") + (self.repo.root / "CHANGELOG.md").write_text( + "# Changelog\n\n## Unreleased\n\n" + OLD_SECTION, + encoding="utf-8", + ) + stable = self.repo.notes("v1.2.3", notes="stable-notes.md") + self.assert_success(stable) + self.assertEqual(stable.stdout, "") + self.assertEqual( + (self.repo.root / "stable-notes.md").read_text(), + "## v1.2.3\n\n### Fixed\n\n- Previous fix.\n", + ) + + def test_empty_duplicate_and_malformed_changelog_are_rejected(self): + cases = { + "empty": "### Added", + "no heading": "- Entry without a category.", + "duplicate unreleased": PENDING + "\n\n## Unreleased\n\n" + PENDING, + "invalid version": PENDING + "\n\n## v01.2.3\n\n### Fixed\n\n- Old.", + "ascending history": PENDING + "\n\n## v1.2.3\n\n- A.\n\n## v1.3.0\n\n- B.", + "unexpected section": PENDING + "\n\n## Draft\n\n- Hidden.", + "unsupported later category": PENDING + "\n\n### Mystery\n\n- Hidden.", + } + for name, pending in cases.items(): + with self.subTest(name=name): + self.repo.clear_tags() + self.repo.write_state("0.0.1", pending) + before_changelog = (self.repo.root / "CHANGELOG.md").read_text() + before_source = (self.repo.root / "internal/cli/cli.go").read_text() + result = self.repo.prepare() + self.assert_failure_without_writes(result, before_changelog, before_source) + + self.repo.write_state( + "0.0.1", + PENDING, + intro="# Changelog\n\n## v0.0.0\n\n### Added\n\n- Misordered history.\n", + ) + before_changelog = (self.repo.root / "CHANGELOG.md").read_text() + before_source = (self.repo.root / "internal/cli/cli.go").read_text() + result = self.repo.prepare() + self.assertIn("must be the first level-two", result.stderr) + self.assert_failure_without_writes(result, before_changelog, before_source) + + def test_source_and_latest_tag_mismatches_are_rejected(self): + cases = (("1.2.2", ("v1.2.3",), "does not match"), ("1.2.3", (), "has no Git tag")) + for source_version, tags, message in cases: + with self.subTest(source_version=source_version, tags=tags): + self.repo.clear_tags() + self.repo.write_state(source_version, PENDING, OLD_SECTION) + self.repo.tag(*tags) + before_changelog = (self.repo.root / "CHANGELOG.md").read_text() + before_source = (self.repo.root / "internal/cli/cli.go").read_text() + result = self.repo.prepare() + self.assertIn(message, result.stderr) + self.assert_failure_without_writes(result, before_changelog, before_source) + + def test_existing_target_stable_tag_is_rejected(self): + self.repo.write_state("1.2.3", PENDING, OLD_SECTION) + self.repo.tag("v1.2.3", "v1.2.4") + before_changelog = (self.repo.root / "CHANGELOG.md").read_text() + before_source = (self.repo.root / "internal/cli/cli.go").read_text() + result = self.repo.prepare() + self.assertIn("stable tag v1.2.4 already exists", result.stderr) + self.assert_failure_without_writes(result, before_changelog, before_source) + + +if __name__ == "__main__": + unittest.main()