From 003fcd3c0fc9d52d6837610712c748ff3a6d6566 Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 24 Jul 2026 15:11:34 -0400 Subject: [PATCH 1/3] Generate and verify Go FFI header --- .github/workflows/test.yml | 9 +++---- AGENTS.md | 2 ++ Makefile | 12 ++++++++- sds-go/go/dd_sds.h | 51 ++++++++++++++++++++++++++------------ sds-go/go/rule.go | 6 ++--- sds-go/go/scanner.go | 12 ++++----- sds/cbindgen.toml | 9 +++++++ 7 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 sds/cbindgen.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 49c640fa..97c7384f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,8 +39,10 @@ jobs: steps: - uses: actions/checkout@v4 - run: source "$HOME/.cargo/env" - - name: "Build the SDS Go/Rust wrapper" - run: make build-sds-go + - name: "Install Go binding check dependencies" + run: cargo install cbindgen --version 0.29.2 --locked + - name: "Verify, build, and test the SDS Go bindings" + run: make check-sds-go-bindings - name: "Run the Go linter" run: | cd sds-go/go @@ -50,9 +52,6 @@ jobs: LD_LIBRARY_PATH=../../sds/target/release staticcheck ./... cd ../.. make format-go - - name: "Run the Go unit tests" - working-directory: sds-go/go - run: LD_LIBRARY_PATH=../../sds/target/release go test -v . - name: "Make sure the example builds and stays relevant" working-directory: sds-go/go run: | diff --git a/AGENTS.md b/AGENTS.md index d706a0ce..96f50113 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ This is a multi-language library providing core Sensitive Data Scanner (SDS) fun ## Build & Test Commands Use `make` to see available commands for building, testing, and formatting. +`make check-sds-go-bindings` requires `cbindgen` 0.29.2. ## Code Quality Requirements @@ -40,6 +41,7 @@ Key files: - `sds/src/native/` - Rust side of the Go FFI, enabled with the `dd_sds_go` feature - `sds/src/bindings_utils.rs` - Shared binding utilities, enabled with the `sds-bindings-utils` feature - `sds-go/go/scanner.go` - Main Go API +- `sds-go/go/dd_sds.h` - Generated C header for the Rust FFI ## Code Quality Standards diff --git a/Makefile b/Makefile index d060afee..cf472401 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ build-sds-go-no-third-party-active-checkers: ## Build the sds-go lib without thi @echo "Building sds-go lib without third-party active checkers" cargo build --manifest-path="sds/Cargo.toml" --release --no-default-features --features dd-sds,dd_sds_go +.PHONY: update-sds-go-header +update-sds-go-header: ## Regenerate the C header used by the Go bindings. + cd sds && cbindgen --quiet --config cbindgen.toml --crate dd-sensitive-data-scanner --output ../sds-go/go/dd_sds.h . + ##@ Formatting .PHONY: format-go @@ -43,7 +47,7 @@ format-all: format-rust format-go ## Format the rust lib and golang libs. .PHONY: test-go test-go: ## Test the golang lib. @echo "Testing golang lib" - cd sds-go/go && go test ./... + cd sds-go/go && LD_LIBRARY_PATH="$(ROOT_DIR)/sds/target/release" DYLD_LIBRARY_PATH="$(ROOT_DIR)/sds/target/release" go test -count=1 ./... .PHONY: test-rust test-rust: ## Test the rust lib. @@ -72,6 +76,12 @@ check-rust: ## Check the rust lib. @echo "Checking rust lib" bash ./scripts/rust_checks.sh +.PHONY: check-sds-go-bindings +check-sds-go-bindings: ## Verify, build, and test the Go bindings against the Rust FFI. + cd sds && cbindgen --quiet --config cbindgen.toml --crate dd-sensitive-data-scanner --verify --output ../sds-go/go/dd_sds.h . + $(MAKE) build-sds-go + $(MAKE) test-go + ##@ Licenses generation .PHONY: update-licenses diff --git a/sds-go/go/dd_sds.h b/sds-go/go/dd_sds.h index d2c2181c..343581f6 100644 --- a/sds-go/go/dd_sds.h +++ b/sds-go/go/dd_sds.h @@ -1,24 +1,43 @@ -// From the Go documentation, it's recommended to include stdlib.h if we need -// to use C.free. -#include +#ifndef DD_SDS_H +#define DD_SDS_H + +/* Generated from the Rust FFI. Run `make update-sds-go-header` to update. */ + +#include #include +#include +#include + +void append_rule_to_list(int64_t rule, int64_t list); + +int64_t create_regex_rule(const char *json_config); + +int64_t create_rule_list(void); + +int64_t create_scanner(int64_t rules, + const char *encoded_labels, + int32_t enable_debug_observability, + const char **error_out); + +void delete_scanner(int64_t scanner_id); -long create_regex_rule(const char* json_config); +void free_any_rule(int64_t rule_ptr); -long create_scanner(long rule_list, const char* encoded_labels, int enable_debug_observability, const char** error); -void delete_scanner(long scanner_id); +void free_rule_list(int64_t list); -// event is a non-null terminated TODO -// scan_metadata_json may be NULL, or a null-terminated UTF-8 JSON object string. -const char* scan(long scanner_id, const void* event, long event_size, long *retsize, long *retcap, const char** error, int with_validate_matching, const char* scan_metadata_json); +void free_string(const char *ptr); -void free_vec(const char* string, long len, long cap); -void free_string(const char* string); +void free_vec(const char *ptr, int64_t len, int64_t cap); -void free_any_rule(long rule_ptr); +const char *scan(int64_t scanner_id, + const void *event, + int64_t event_size, + int64_t *retsize, + int64_t *retcapacity, + const char **error_out, + int32_t with_validate_matching, + const char *scan_metadata_json); -long create_rule_list(); -void append_rule_to_list(long rule_ptr, long list_ptr); -void free_rule_list(long list_ptr); +const char *validate_regex(const char *regex, const char **error_out); -const char* validate_regex(const char* regex, const char** error_out); +#endif /* DD_SDS_H */ diff --git a/sds-go/go/rule.go b/sds-go/go/rule.go index 07293815..ec62f9ae 100644 --- a/sds-go/go/rule.go +++ b/sds-go/go/rule.go @@ -24,7 +24,7 @@ func CreateRuleFromRawPtr(ptr int64) Rule { // Delete deletes the native resources associated with this Rule. // It is safe to delete it while it is still being used by a scanner. func (r Rule) Delete() { - C.free_any_rule(C.long(r.nativeRulePtr)) + C.free_any_rule(C.int64_t(r.nativeRulePtr)) } type RuleList struct { @@ -40,11 +40,11 @@ func CreateRuleList() RuleList { } func (l RuleList) AppendRule(r *Rule) { - C.append_rule_to_list(C.long(r.nativeRulePtr), C.long(l.nativePtr)) + C.append_rule_to_list(C.int64_t(r.nativeRulePtr), C.int64_t(l.nativePtr)) } // Delete deletes the native resources associated with this RuleList. // It is safe to delete it while it is still being used by a scanner. func (r RuleList) Delete() { - C.free_rule_list(C.long(r.nativePtr)) + C.free_rule_list(C.int64_t(r.nativePtr)) } diff --git a/sds-go/go/scanner.go b/sds-go/go/scanner.go index 42cc57b6..c1a798df 100644 --- a/sds-go/go/scanner.go +++ b/sds-go/go/scanner.go @@ -90,12 +90,12 @@ func CreateScannerWithOptions(ruleConfigs []RuleConfig, options ScannerOptions) encodedLabelsJson := C.CString(string(labelsMarshalled)) defer C.free(unsafe.Pointer(encodedLabelsJson)) - var cEnableDebugObservability C.int + var cEnableDebugObservability C.int32_t if options.EnableDebugObservability { cEnableDebugObservability = 1 } var errorString *C.char - id := C.create_scanner(C.long(ruleList.nativePtr), encodedLabelsJson, cEnableDebugObservability, &errorString) + id := C.create_scanner(C.int64_t(ruleList.nativePtr), encodedLabelsJson, cEnableDebugObservability, &errorString) if id < 0 { switch id { @@ -131,7 +131,7 @@ func CreateScannerWithOptions(ruleConfigs []RuleConfig, options ScannerOptions) // Delete deletes the instance of the current Scanner. // The current Scanner should not be reused. func (s *Scanner) Delete() { - C.delete_scanner(C.long(s.Id)) + C.delete_scanner(C.int64_t(s.Id)) s.Id = 0 s.RuleConfigs = nil } @@ -144,7 +144,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s var retcap int64 var errorString *C.char - var cWithValidateMatching C.int + var cWithValidateMatching C.int32_t if withValidateMatching { cWithValidateMatching = 1 } else { @@ -161,7 +161,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s defer C.free(unsafe.Pointer(cScanMetadata)) } - rvdata := C.scan(C.long(s.Id), cdata, C.long(len(encodedEvent)), (*C.long)(unsafe.Pointer(&retsize)), (*C.long)(unsafe.Pointer(&retcap)), &errorString, cWithValidateMatching, cScanMetadata) + rvdata := C.scan(C.int64_t(s.Id), cdata, C.int64_t(len(encodedEvent)), (*C.int64_t)(unsafe.Pointer(&retsize)), (*C.int64_t)(unsafe.Pointer(&retcap)), &errorString, cWithValidateMatching, cScanMetadata) if errorString != nil { defer C.free_string(errorString) return nil, fmt.Errorf("internal panic: %v", C.GoString(errorString)) @@ -174,7 +174,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s // otherwise we received data initially owned by rust, once we've used it, // use `free_vec` to let know rust it can drop this memory. - defer C.free_vec(rvdata, C.long(retsize), C.long(retcap)) + defer C.free_vec(rvdata, C.int64_t(retsize), C.int64_t(retcap)) // Note that in the Go 1.21 documentation, GoBytes is part of: // > A few special functions convert between Go and C types by making copies of the data. diff --git a/sds/cbindgen.toml b/sds/cbindgen.toml new file mode 100644 index 00000000..f85229c9 --- /dev/null +++ b/sds/cbindgen.toml @@ -0,0 +1,9 @@ +language = "C" +include_guard = "DD_SDS_H" +autogen_warning = "/* Generated from the Rust FFI. Run `make update-sds-go-header` to update. */" +documentation = false +style = "type" +sort_by = "Name" + +[export] +item_types = ["functions"] From 8775283e93e3c3f9438cd4685ee23cc0baa276d4 Mon Sep 17 00:00:00 2001 From: Nathan Date: Fri, 24 Jul 2026 15:34:33 -0400 Subject: [PATCH 2/3] Document Go binding regeneration --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 4a2931f1..c10d83ff 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,18 @@ This contains: - An extensive library of secondary validators / checksums - Go bindings +## Updating the Go bindings + +The Go bindings use the generated C header at `sds-go/go/dd_sds.h`. When an +exported Rust function in `sds/src/native/` is added, removed, or its signature +changes, install `cbindgen` 0.29.2 and regenerate the header: + +```shell +cargo install cbindgen --version 0.29.2 --locked +make update-sds-go-header +``` + +Commit the regenerated header with the Rust and Go changes. CI runs +`make check-sds-go-bindings` to verify that the header is current and that the +Go bindings build and pass their tests against the Rust library. From 2f0da6433224fbd8ed4617e4707ecfb8c624dcfd Mon Sep 17 00:00:00 2001 From: Nathan Date: Mon, 27 Jul 2026 09:16:22 -0400 Subject: [PATCH 3/3] Check cbindgen before header generation --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cf472401..20839e37 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,16 @@ build-sds-go-no-third-party-active-checkers: ## Build the sds-go lib without thi @echo "Building sds-go lib without third-party active checkers" cargo build --manifest-path="sds/Cargo.toml" --release --no-default-features --features dd-sds,dd_sds_go +.PHONY: check-cbindgen +check-cbindgen: ## Ensure cbindgen is installed for Go binding header generation. + @if ! command -v cbindgen >/dev/null 2>&1; then \ + echo "error: cbindgen is not installed (required for Go binding header generation)." >&2; \ + echo "Install with: cargo install cbindgen --version 0.29.2 --locked" >&2; \ + exit 1; \ + fi + .PHONY: update-sds-go-header -update-sds-go-header: ## Regenerate the C header used by the Go bindings. +update-sds-go-header: check-cbindgen ## Regenerate the C header used by the Go bindings. cd sds && cbindgen --quiet --config cbindgen.toml --crate dd-sensitive-data-scanner --output ../sds-go/go/dd_sds.h . ##@ Formatting