Generate and verify Go FFI bindings

.github/workflows/test.yml

@@ -44,8 +44,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: source "$HOME/.cargo/env"
-      - name: "Build the SDS Go/Rust wrapper"
-        run: make build-sds-go
+      - name: "Install Go binding check dependencies"
+        run: cargo install cbindgen --version 0.29.2 --locked
+      - name: "Verify, build, and test the SDS Go bindings"
+        run: make check-sds-go-bindings
       - name: "Run the Go linter"
         run: |
           cd sds-go/go
@@ -55,9 +57,6 @@ jobs:
           LD_LIBRARY_PATH=../../sds/target/release staticcheck ./...
           cd ../..
           make format-go
-      - name: "Run the Go unit tests"
-        working-directory: sds-go/go
-        run: LD_LIBRARY_PATH=../../sds/target/release go test -v .
       - name: "Make sure the example builds and stays relevant"
         working-directory: sds-go/go
         run: |

AGENTS.md

@@ -7,6 +7,7 @@ This is a multi-language library providing core Sensitive Data Scanner (SDS) fun
 ## Build & Test Commands
 
 Use `make` to see available commands for building, testing, and formatting.
+`make check-sds-go-bindings` requires `cbindgen` 0.29.2.
 `make check-rust` requires `cargo-hack` 0.6.45. The Rust check and test targets require the
 Hyperscan development library so they can check every Cargo feature.
 
@@ -42,6 +43,7 @@ Key files:
 - `sds/src/native/` - Rust side of the Go FFI, enabled with the `dd_sds_go` feature
 - `sds/src/bindings_utils.rs` - Shared binding utilities, enabled with the `sds-bindings-utils` feature
 - `sds-go/go/scanner.go` - Main Go API
+- `sds-go/go/dd_sds.h` - Generated C header for the Rust FFI
 
 
 ## Code Quality Standards

Makefile

@@ -22,6 +22,18 @@ build-sds-go-no-third-party-active-checkers: ## Build the sds-go lib without thi
 	@echo "Building sds-go lib without third-party active checkers"
 	cd sds && cargo build --release --no-default-features --features dd_sds_go
 
+.PHONY: check-cbindgen
+check-cbindgen: ## Ensure cbindgen is installed for Go binding header generation.
+	@if ! command -v cbindgen >/dev/null 2>&1; then \
+		echo "error: cbindgen is not installed (required for Go binding header generation)." >&2; \
+		echo "Install with: cargo install cbindgen --version 0.29.2 --locked" >&2; \
+		exit 1; \
+	fi
+
+.PHONY: update-sds-go-header
+update-sds-go-header: check-cbindgen ## Regenerate the C header used by the Go bindings.
+	cd sds && cbindgen --quiet --config cbindgen.toml --crate dd-sensitive-data-scanner --output ../sds-go/go/dd_sds.h .
+
 ##@ Formatting
 
 .PHONY: format-go
@@ -43,7 +55,7 @@ format-all: format-rust format-go ## Format the rust lib and golang libs.
 .PHONY: test-go
 test-go: ## Test the golang lib.
 	@echo "Testing golang lib"
-	cd sds-go/go && go test ./...
+	cd sds-go/go && LD_LIBRARY_PATH="$(ROOT_DIR)/sds/target/release" DYLD_LIBRARY_PATH="$(ROOT_DIR)/sds/target/release" go test -count=1 ./...
 
 .PHONY: test-rust
 test-rust: ## Test the rust lib.
@@ -69,6 +81,12 @@ check-rust: ## Check the rust lib.
 	@echo "Checking rust lib"
 	bash ./scripts/rust_checks.sh
 
+.PHONY: check-sds-go-bindings
+check-sds-go-bindings: ## Verify, build, and test the Go bindings against the Rust FFI.
+	cd sds && cbindgen --quiet --config cbindgen.toml --crate dd-sensitive-data-scanner --verify --output ../sds-go/go/dd_sds.h .
+	$(MAKE) build-sds-go
+	$(MAKE) test-go
+
 ##@ Licenses generation
 
 .PHONY: update-licenses

README.md

@@ -11,4 +11,18 @@ This contains:
 - An extensive library of secondary validators / checksums
 - Go bindings
 
+## Updating the Go bindings
+
+The Go bindings use the generated C header at `sds-go/go/dd_sds.h`. When an
+exported Rust function in `sds/src/native/` is added, removed, or its signature
+changes, install `cbindgen` 0.29.2 and regenerate the header:
+
+```shell
+cargo install cbindgen --version 0.29.2 --locked
+make update-sds-go-header
+```
+
+Commit the regenerated header with the Rust and Go changes. CI runs
+`make check-sds-go-bindings` to verify that the header is current and that the
+Go bindings build and pass their tests against the Rust library.
 

sds-go/go/dd_sds.h

@@ -1,24 +1,43 @@
-// From the Go documentation, it's recommended to include stdlib.h if we need
-// to use C.free.
-#include <stdlib.h>
+#ifndef DD_SDS_H
+#define DD_SDS_H
+
+/* Generated from the Rust FFI. Run `make update-sds-go-header` to update. */
+
+#include <stdarg.h>
 #include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+void append_rule_to_list(int64_t rule, int64_t list);
+
+int64_t create_regex_rule(const char *json_config);
+
+int64_t create_rule_list(void);
+
+int64_t create_scanner(int64_t rules,
+                       const char *encoded_labels,
+                       int32_t enable_debug_observability,
+                       const char **error_out);
+
+void delete_scanner(int64_t scanner_id);
 
-long create_regex_rule(const char* json_config);
+void free_any_rule(int64_t rule_ptr);
 
-long create_scanner(long rule_list, const char* encoded_labels, int enable_debug_observability, const char** error);
-void delete_scanner(long scanner_id);
+void free_rule_list(int64_t list);
 
-// event is a non-null terminated TODO
-// scan_metadata_json may be NULL, or a null-terminated UTF-8 JSON object string.
-const char* scan(long scanner_id, const void* event, long event_size, long *retsize, long *retcap, const char** error, int with_validate_matching, const char* scan_metadata_json);
+void free_string(const char *ptr);
 
-void free_vec(const char* string, long len, long cap);
-void free_string(const char* string);
+void free_vec(const char *ptr, int64_t len, int64_t cap);
 
-void free_any_rule(long rule_ptr);
+const char *scan(int64_t scanner_id,
+                 const void *event,
+                 int64_t event_size,
+                 int64_t *retsize,
+                 int64_t *retcapacity,
+                 const char **error_out,
+                 int32_t with_validate_matching,
+                 const char *scan_metadata_json);
 
-long create_rule_list();
-void append_rule_to_list(long rule_ptr, long list_ptr);
-void free_rule_list(long list_ptr);
+const char *validate_regex(const char *regex, const char **error_out);
 
-const char* validate_regex(const char* regex, const char** error_out);
+#endif  /* DD_SDS_H */

sds-go/go/rule.go

@@ -24,7 +24,7 @@ func CreateRuleFromRawPtr(ptr int64) Rule {
 // Delete deletes the native resources associated with this Rule.
 // It is safe to delete it while it is still being used by a scanner.
 func (r Rule) Delete() {
-	C.free_any_rule(C.long(r.nativeRulePtr))
+	C.free_any_rule(C.int64_t(r.nativeRulePtr))
 }
 
 type RuleList struct {
@@ -40,11 +40,11 @@ func CreateRuleList() RuleList {
 }
 
 func (l RuleList) AppendRule(r *Rule) {
-	C.append_rule_to_list(C.long(r.nativeRulePtr), C.long(l.nativePtr))
+	C.append_rule_to_list(C.int64_t(r.nativeRulePtr), C.int64_t(l.nativePtr))
 }
 
 // Delete deletes the native resources associated with this RuleList.
 // It is safe to delete it while it is still being used by a scanner.
 func (r RuleList) Delete() {
-	C.free_rule_list(C.long(r.nativePtr))
+	C.free_rule_list(C.int64_t(r.nativePtr))
 }

sds-go/go/scanner.go

@@ -90,12 +90,12 @@ func CreateScannerWithOptions(ruleConfigs []RuleConfig, options ScannerOptions)
 	encodedLabelsJson := C.CString(string(labelsMarshalled))
 	defer C.free(unsafe.Pointer(encodedLabelsJson))
 
-	var cEnableDebugObservability C.int
+	var cEnableDebugObservability C.int32_t
 	if options.EnableDebugObservability {
 		cEnableDebugObservability = 1
 	}
 	var errorString *C.char
-	id := C.create_scanner(C.long(ruleList.nativePtr), encodedLabelsJson, cEnableDebugObservability, &errorString)
+	id := C.create_scanner(C.int64_t(ruleList.nativePtr), encodedLabelsJson, cEnableDebugObservability, &errorString)
 
 	if id < 0 {
 		switch id {
@@ -131,7 +131,7 @@ func CreateScannerWithOptions(ruleConfigs []RuleConfig, options ScannerOptions)
 // Delete deletes the instance of the current Scanner.
 // The current Scanner should not be reused.
 func (s *Scanner) Delete() {
-	C.delete_scanner(C.long(s.Id))
+	C.delete_scanner(C.int64_t(s.Id))
 	s.Id = 0
 	s.RuleConfigs = nil
 }
@@ -144,7 +144,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s
 	var retcap int64
 	var errorString *C.char
 
-	var cWithValidateMatching C.int
+	var cWithValidateMatching C.int32_t
 	if withValidateMatching {
 		cWithValidateMatching = 1
 	} else {
@@ -161,7 +161,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s
 		defer C.free(unsafe.Pointer(cScanMetadata))
 	}
 
-	rvdata := C.scan(C.long(s.Id), cdata, C.long(len(encodedEvent)), (*C.long)(unsafe.Pointer(&retsize)), (*C.long)(unsafe.Pointer(&retcap)), &errorString, cWithValidateMatching, cScanMetadata)
+	rvdata := C.scan(C.int64_t(s.Id), cdata, C.int64_t(len(encodedEvent)), (*C.int64_t)(unsafe.Pointer(&retsize)), (*C.int64_t)(unsafe.Pointer(&retcap)), &errorString, cWithValidateMatching, cScanMetadata)
 	if errorString != nil {
 		defer C.free_string(errorString)
 		return nil, fmt.Errorf("internal panic: %v", C.GoString(errorString))
@@ -174,7 +174,7 @@ func (s *Scanner) lowLevelScan(encodedEvent []byte, withValidateMatching bool, s
 
 	// otherwise we received data initially owned by rust, once we've used it,
 	// use `free_vec` to let know rust it can drop this memory.
-	defer C.free_vec(rvdata, C.long(retsize), C.long(retcap))
+	defer C.free_vec(rvdata, C.int64_t(retsize), C.int64_t(retcap))
 
 	// Note that in the Go 1.21 documentation, GoBytes is part of:
 	// > A few special functions convert between Go and C types by making copies of the data.

sds/cbindgen.toml

@@ -0,0 +1,9 @@
+language = "C"
+include_guard = "DD_SDS_H"
+autogen_warning = "/* Generated from the Rust FFI. Run `make update-sds-go-header` to update. */"
+documentation = false
+style = "type"
+sort_by = "Name"
+
+[export]
+item_types = ["functions"]