diff --git a/examples/cmd/benchmark_experimental.go b/examples/cmd/benchmark_chunked.go similarity index 65% rename from examples/cmd/benchmark_experimental.go rename to examples/cmd/benchmark_chunked.go index a944dfff3c..78c37dfabf 100644 --- a/examples/cmd/benchmark_experimental.go +++ b/examples/cmd/benchmark_chunked.go @@ -14,7 +14,7 @@ import ( "github.com/opentdf/platform/protocol/go/kas/kasconnect" "github.com/opentdf/platform/protocol/go/policy" - "github.com/opentdf/platform/sdk/experimental/tdf" + "github.com/opentdf/platform/sdk" "github.com/opentdf/platform/sdk/httputil" "github.com/spf13/cobra" ) @@ -27,10 +27,10 @@ var ( func init() { benchmarkCmd := &cobra.Command{ - Use: "benchmark-experimental-writer", - Short: "Benchmark experimental TDF writer speed", - Long: `Benchmark the experimental TDF writer with configurable payload size.`, - RunE: runExperimentalWriterBenchmark, + Use: "benchmark-chunked-writer", + Short: "Benchmark chunked TDF writer speed", + Long: `Benchmark the chunked TDF writer with configurable payload size.`, + RunE: runChunkedWriterBenchmark, } //nolint: mnd // no magic number, this is just default value for payload size benchmarkCmd.Flags().IntVar(&payloadSize, "payload-size", 1024*1024, "Payload size in bytes") // Default 1MB @@ -39,7 +39,7 @@ func init() { ExamplesCmd.AddCommand(benchmarkCmd) } -func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { +func runChunkedWriterBenchmark(_ *cobra.Command, _ []string) error { payload := make([]byte, payloadSize) _, err := rand.Read(payload) if err != nil { @@ -53,7 +53,6 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("failed to get public key from KAS: %w", err) } - var attrs []*policy.Value simpleyKey := &policy.SimpleKasKey{ KasUri: platformEndpoint, @@ -65,31 +64,43 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { }, } - attrs = append(attrs, &policy.Value{Fqn: testAttr, KasKeys: []*policy.SimpleKasKey{simpleyKey}, Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr}}) - writer, err := tdf.NewWriter(context.Background(), tdf.WithDefaultKASForWriter(simpleyKey), tdf.WithInitialAttributes(attrs)) + attrs := []*policy.Value{{ + Fqn: testAttr, + KasKeys: []*policy.SimpleKasKey{simpleyKey}, + Attribute: &policy.Attribute{Namespace: &policy.Namespace{Name: "example.com"}, Fqn: testAttr}, + }} + + // The package-level constructor rather than SDK.NewChunkedWriter: this + // benchmark talks to one KAS whose key it already fetched, so there is + // nothing for the platform to resolve and no reason to pay for a round trip + // to it inside the timed section. + writer, err := sdk.NewChunkedWriter(context.Background(), + sdk.WithChunkedDefaultKAS(simpleyKey), + sdk.WithChunkedInitialAttributes(attrs), + ) if err != nil { return fmt.Errorf("failed to create writer: %w", err) } - i := 0 - wg := sync.WaitGroup{} + segs := len(payload) / segmentChunk + errs := make([]error, segs) + wg := sync.WaitGroup{} wg.Add(segs) start := time.Now() - for i < segs { - segment := i + for segment := range segs { go func() { - start := i * segmentChunk - end := min(start+segmentChunk, len(payload)) - _, err = writer.WriteSegment(context.Background(), segment, payload[start:end]) - if err != nil { - fmt.Println(err) - panic(err) - } - wg.Done() + defer wg.Done() + lo := segment * segmentChunk + hi := min(lo+segmentChunk, len(payload)) + _, errs[segment] = writer.WriteSegment(context.Background(), segment, payload[lo:hi]) }() - i++ } wg.Wait() + for i, err := range errs { + if err != nil { + return fmt.Errorf("failed to write segment %d: %w", i, err) + } + } end := time.Now() result, err := writer.Finalize(context.Background()) @@ -98,7 +109,7 @@ func runExperimentalWriterBenchmark(_ *cobra.Command, _ []string) error { } totalTime := end.Sub(start) - fmt.Printf("# Benchmark Experimental TDF Writer Results:\n") + fmt.Printf("# Benchmark Chunked TDF Writer Results:\n") fmt.Printf("| Metric | Value |\n") fmt.Printf("|--------------------|--------------|\n") fmt.Printf("| Payload Size (B) | %d |\n", payloadSize) diff --git a/sdk/chunked_options.go b/sdk/chunked_options.go index b7272447ec..95a8b4d18f 100644 --- a/sdk/chunked_options.go +++ b/sdk/chunked_options.go @@ -9,12 +9,16 @@ import ( "github.com/opentdf/platform/protocol/go/policy" ) -// Each injection-seam option below rejects nil rather than storing it. -// A nil seam is not detectable later: the config field is -// indistinguishable from "not set", so NewChunkedWriter installs no -// default and the nil is dereferenced during writing -- for the -// splitter, not until Finalize, long after the caller has encrypted -// every segment. +// Every option below that takes a pointer or an interface rejects nil +// rather than storing it. A stored nil is not detectable later: the +// config field is indistinguishable from "not set". For an injection +// seam that means NewChunkedWriter installs no default and the nil is +// dereferenced during writing -- for the splitter, not until Finalize, +// long after the caller has encrypted every segment. For the default +// KAS it is worse than a panic, because nothing fails: key access +// silently falls back to the platform base key, and the caller learns +// their data went to a KAS they never named only when a reader cannot +// unwrap it. // The slice-valued options below clone what they are given. Each is retained // for the lifetime of the writer or of one Finalize call, and each determines @@ -72,8 +76,6 @@ func withChunkedClock(clock clock) ChunkedWriterOption { // WithChunkedInitialAttributes sets attribute values used by Finalize // when the Finalize call does not supply its own. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption { return func(c *chunkedWriterConfig) error { c.initialAttributes = slices.Clone(values) @@ -82,11 +84,13 @@ func WithChunkedInitialAttributes(values []*policy.Value) ChunkedWriterOption { } // WithChunkedDefaultKAS sets the default KAS used by Finalize when -// the Finalize call does not supply its own. -// -// Experimental: not part of the stable SDK API; may change or be removed. +// the Finalize call does not supply its own. The KAS must not be nil: +// omit the option to leave key access to be resolved some other way. func WithChunkedDefaultKAS(kas *policy.SimpleKasKey) ChunkedWriterOption { return func(c *chunkedWriterConfig) error { + if kas == nil { + return errors.New("chunked: default KAS must not be nil") + } c.initialDefaultKAS = kas return nil } @@ -96,14 +100,13 @@ func WithChunkedDefaultKAS(kas *policy.SimpleKasKey) ChunkedWriterOption { // [ChunkedWriter]. Callers with multi-KAS attribute grants should // inject a splitter that understands their grant model. The splitter // must not be nil. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedKeySplitter(splitter KeySplitter) ChunkedWriterOption { return func(c *chunkedWriterConfig) error { if splitter == nil { return errors.New("chunked: key splitter must not be nil") } c.splitter = splitter + c.splitterSet = true return nil } } @@ -120,12 +123,22 @@ func withChunkedRand(r io.Reader) ChunkedWriterOption { } } +// WithChunkedTDFOptions supplies the key access options — attributes, KAS +// information, preferred wrapping algorithm — that SDK.NewChunkedWriter +// resolves against the platform at Finalize. It has no effect on the +// package-level NewChunkedWriter, which has no platform to resolve against; +// use WithChunkedKeySplitter there. +func WithChunkedTDFOptions(opts ...TDFOption) ChunkedWriterOption { + return func(c *chunkedWriterConfig) error { + c.tdfOptions = append(c.tdfOptions, opts...) + return nil + } +} + // WithChunkedAssertions attaches signed assertions to the produced // TDF. Each assertion is bound to the payload's aggregate hash, so // they are signed at Finalize once every segment is in. Assertions // without their own SigningKey are signed with HS256 over the DEK. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { c.assertions = slices.Clone(assertions) @@ -143,8 +156,6 @@ func WithChunkedAssertions(assertions []AssertionConfig) ChunkedFinalizeOption { // silently would loosen the policy on the data, which is the one // mistake here that cannot be detected after the fact. Construct a // writer without WithChunkedInitialAttributes instead. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { c.attributes = slices.Clone(values) @@ -153,14 +164,13 @@ func WithChunkedAttributes(values []*policy.Value) ChunkedFinalizeOption { } // WithChunkedDefaultKASForFinalize overrides the writer's initial -// default KAS for this Finalize call. -// -// A nil argument reads as "not specified", so the writer's initial -// default KAS still applies; there is no way to unset it for one call. -// -// Experimental: not part of the stable SDK API; may change or be removed. +// default KAS for this Finalize call. The KAS must not be nil: omit +// the option to keep whatever WithChunkedDefaultKAS set. func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { + if kas == nil { + return errors.New("chunked: default KAS must not be nil") + } c.defaultKAS = kas return nil } @@ -169,8 +179,6 @@ func WithChunkedDefaultKASForFinalize(kas *policy.SimpleKasKey) ChunkedFinalizeO // WithChunkedEncryptedMetadata attaches AES-GCM-encrypted metadata to // every KAO in the TDF. The metadata is keyed on the split share and // only decryptable by a reader that has been granted access. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { c.encryptedMetadata = metadata @@ -181,8 +189,6 @@ func WithChunkedEncryptedMetadata(metadata string) ChunkedFinalizeOption { // WithChunkedTargetMode targets a specific TDF spec version, given as // a semver string such as "4.2.2". An empty mode selects the most // recently available target version (4.3.0). -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedTargetMode(mode string) ChunkedWriterOption { return func(c *chunkedWriterConfig) error { if mode == "" { @@ -201,8 +207,6 @@ func WithChunkedTargetMode(mode string) ChunkedWriterOption { } // WithChunkedMimeType records the payload MIME type in the manifest. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { c.mimeType = mimeType @@ -234,8 +238,6 @@ func WithChunkedMimeType(mimeType string) ChunkedFinalizeOption { // excludes from the manifest -- must still be appended by the caller // when assembling the final file. Skipping a dropped segment's bytes // produces an archive whose central directory offsets overshoot. -// -// Experimental: not part of the stable SDK API; may change or be removed. func WithChunkedSegments(indices []int) ChunkedFinalizeOption { return func(c *chunkedFinalizeConfig) error { // Cloned because segmentOrderLocked validates the live slice before diff --git a/sdk/chunked_test.go b/sdk/chunked_test.go index 50f1c5447f..b5b31ca07e 100644 --- a/sdk/chunked_test.go +++ b/sdk/chunked_test.go @@ -569,6 +569,15 @@ func newChunkedWriterForTest(ctx context.Context, t *testing.T, opts ...ChunkedW return w, kasBundle } +// PublicKey serves the wrapping key, so that a caller who names this KAS by URL +// alone can have its key fetched rather than having to supply the PEM. +func (k *chunkedFakeKAS) PublicKey(_ context.Context, _ *connect.Request[kaspb.PublicKeyRequest]) (*connect.Response[kaspb.PublicKeyResponse], error) { + return connect.NewResponse(&kaspb.PublicKeyResponse{ + PublicKey: k.publicPEM, + Kid: k.kid, + }), nil +} + // Rewrap unwraps every RSA-wrapped KAO under the KAS private key and // re-wraps under the caller's session public key. // @@ -927,12 +936,14 @@ func TestChunkedTargetModeInvalid(t *testing.T) { assert.Contains(t, err.Error(), "not-a-version") } -// TestChunkedOptionsRejectNil checks that the injection-seam options -// refuse a nil value instead of storing it. A stored nil is -// indistinguishable from an unset field, so no default gets installed -// and the nil surfaces as a panic partway through writing -- for the -// key splitter, not until Finalize, after the caller has already -// encrypted and uploaded every segment. +// TestChunkedOptionsRejectNil checks that every option taking a +// pointer or an interface refuses a nil value instead of storing it. A +// stored nil is indistinguishable from an unset field, so no default +// gets installed and the nil surfaces as a panic partway through +// writing -- for the key splitter, not until Finalize, after the caller +// has already encrypted and uploaded every segment. The default KAS +// fails more quietly still: it does not panic at all, it just sends the +// data to the platform base key. func TestChunkedOptionsRejectNil(t *testing.T) { ctx := context.Background() kasBundle := newChunkedFakeKAS(t) @@ -945,6 +956,7 @@ func TestChunkedOptionsRejectNil(t *testing.T) { {"archive writer factory", withChunkedArchiveWriterFactory(nil)}, {"cipher factory", withChunkedCipherFactory(nil)}, {"clock", withChunkedClock(nil)}, + {"default KAS", WithChunkedDefaultKAS(nil)}, {"key splitter", WithChunkedKeySplitter(nil)}, {"rand", withChunkedRand(nil)}, } { @@ -958,6 +970,18 @@ func TestChunkedOptionsRejectNil(t *testing.T) { assert.Nil(t, writer) }) } + + t.Run("default KAS for finalize", func(t *testing.T) { + writer, err := NewChunkedWriter(ctx, WithChunkedDefaultKAS(kasBundle.simpleKey())) + require.NoError(t, err) + + _, err = writer.WriteSegment(ctx, 0, []byte("hello")) + require.NoError(t, err) + + _, err = writer.Finalize(ctx, WithChunkedDefaultKASForFinalize(nil)) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not be nil") + }) } // TestChunkedIntegrityAlgorithmsAreFixed pins the only two algorithms @@ -2631,3 +2655,117 @@ func TestChunkedGetManifestDoesNotBlockWriteSegment(t *testing.T) { require.NoError(t, err) assert.Len(t, later.Segments, 2) } + +// TestSDKChunkedWriterResolvesKeyAccess covers SDK.NewChunkedWriter, whose +// whole point over the package-level constructor is that key access comes from +// the platform rather than from a KeySplitter the caller had to write. The KAS +// info here carries no public key, so passing the round trip means the writer +// went out and fetched it. +func TestSDKChunkedWriterResolvesKeyAccess(t *testing.T) { + ctx := context.Background() + kasBundle := newChunkedFakeKAS(t) + defer kasBundle.server.Close() + + s := newChunkedTestSDK(t) + + writer, err := s.NewChunkedWriter(ctx, WithChunkedTDFOptions( + // Autoconfigure off: this SDK is pointed at a bare KAS, with no policy + // service to ask which KAS grants which attribute. + WithAutoconfigure(false), + WithKasInformation(KASInfo{URL: kasBundle.url}), + )) + require.NoError(t, err) + + body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("sdk-"), []byte("chunked")}) + fin, err := writer.Finalize(ctx) + require.NoError(t, err) + require.Len(t, fin.Manifest.KeyAccessObjs, 1) + assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL) + assert.Equal(t, kasBundle.kid, fin.Manifest.KeyAccessObjs[0].KID) + + reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)), + WithKasAllowlist([]string{kasBundle.url}), + ) + require.NoError(t, err) + plain, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, []byte("sdk-chunked"), plain) +} + +// TestSDKChunkedWriterKeepsAnExplicitSplitter checks that a caller who supplies +// their own splitter keeps it. The TDF options name a KAS that does not exist, +// so platform resolution would fail loudly rather than quietly produce the same +// answer. +func TestSDKChunkedWriterKeepsAnExplicitSplitter(t *testing.T) { + ctx := context.Background() + kasBundle := newChunkedFakeKAS(t) + defer kasBundle.server.Close() + + s := newChunkedTestSDK(t) + + writer, err := s.NewChunkedWriter(ctx, + WithChunkedKeySplitter(DefaultKeySplitter()), + WithChunkedDefaultKAS(kasBundle.simpleKey()), + WithChunkedTDFOptions( + WithAutoconfigure(false), + WithKasInformation(KASInfo{URL: "https://kas.invalid"}), + ), + ) + require.NoError(t, err) + + body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("explicit splitter")}) + fin, err := writer.Finalize(ctx) + require.NoError(t, err) + require.Len(t, fin.Manifest.KeyAccessObjs, 1) + assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL) + + reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)), + WithKasAllowlist([]string{kasBundle.url}), + ) + require.NoError(t, err) + plain, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, []byte("explicit splitter"), plain) +} + +// TestSDKChunkedWriterFallsBackToBaseKey pins what happens when the caller +// names no KAS at all: SDK.NewChunkedWriter leaves autoconfigure on, finds no +// attribute grants, and wraps to the platform base key. Nothing errors, so this +// is the case that silently sends data somewhere the caller did not choose -- +// which is why WithChunkedDefaultKAS rejects nil rather than treating it as +// "unset", and why the constructor's doc comment spells the fallback out. +func TestSDKChunkedWriterFallsBackToBaseKey(t *testing.T) { + ctx := context.Background() + kasBundle := newChunkedFakeKAS(t) + defer kasBundle.server.Close() + + s := newChunkedTestSDK(t) + s.wellknownConfiguration = newMockWellKnownService(map[string]interface{}{ + baseKeyWellKnown: map[string]interface{}{ + "kas_uri": kasBundle.url, + baseKeyPublicKey: map[string]interface{}{ + baseKeyAlg: "rsa:2048", + "kid": kasBundle.kid, + "pem": kasBundle.publicPEM, + }, + }, + }, nil) + + writer, err := s.NewChunkedWriter(ctx) + require.NoError(t, err) + + body := writeChunkedSegments(ctx, t, writer, [][]byte{[]byte("base key")}) + fin, err := writer.Finalize(ctx) + require.NoError(t, err) + require.Len(t, fin.Manifest.KeyAccessObjs, 1) + assert.Equal(t, kasBundle.url, fin.Manifest.KeyAccessObjs[0].KasURL) + assert.Equal(t, kasBundle.kid, fin.Manifest.KeyAccessObjs[0].KID) + + reader, err := s.LoadTDF(bytes.NewReader(append(body, fin.Data...)), + WithKasAllowlist([]string{kasBundle.url}), + ) + require.NoError(t, err) + plain, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, []byte("base key"), plain) +} diff --git a/sdk/chunked_writer.go b/sdk/chunked_writer.go index fa39b3c48f..f3cee2da01 100644 --- a/sdk/chunked_writer.go +++ b/sdk/chunked_writer.go @@ -100,8 +100,6 @@ func defaultArchiveWriterFactory(c clock) zipstream.SegmentWriter { } // Sentinel errors returned by [ChunkedWriter]. -// -// Experimental: not part of the stable SDK API; may change or be removed. var ( // ErrChunkedAlreadyFinalized is returned when a ChunkedWriter // method is called after Finalize has already succeeded. @@ -188,8 +186,6 @@ var ( // off-thread or in parallel — then call Finalize to close the // archive. Contrast with SDK.CreateTDF, which requires the full // plaintext up front. -// -// Experimental: not part of the stable SDK API; may change or be removed. type ChunkedWriter interface { // Finalize completes TDF creation. Every option applies only to // this Finalize call; writer-level defaults set at NewChunked* @@ -266,8 +262,6 @@ type ChunkedWriter interface { // ChunkedSegmentResult carries the ZIP bytes for one segment plus its // integrity metadata. -// -// Experimental: not part of the stable SDK API; may change or be removed. type ChunkedSegmentResult struct { // EncryptedSize is the ciphertext byte length including nonce and // GCM tag. It is the size the manifest records for this segment, @@ -304,8 +298,6 @@ type ChunkedSegmentResult struct { // ChunkedFinalizeResult carries the finalized TDF's closing bytes and // metadata about what was written. -// -// Experimental: not part of the stable SDK API; may change or be removed. type ChunkedFinalizeResult struct { // Data is the ZIP closing bytes, in order: the payload's data // descriptor, the embedded manifest entry (its own local file @@ -398,6 +390,15 @@ type chunkedWriterConfig struct { // keyAccess is set. splitter KeySplitter + // splitterSet records whether WithChunkedKeySplitter was given, so + // that SDK.NewChunkedWriter can tell "left at the default" from + // "deliberately overridden" and only replace the former. + splitterSet bool + + // tdfOptions shape key access resolved against the platform. Only + // meaningful for SDK.NewChunkedWriter; see WithChunkedTDFOptions. + tdfOptions []TDFOption + // useHex hex-encodes segment, root, and assertion signatures // before base64, producing the doubly-encoded form that readers // older than 4.3.0 require. Set by WithChunkedTargetMode. @@ -441,13 +442,9 @@ type chunkedFinalizeConfig struct { // ChunkedWriterOption configures a ChunkedWriter at construction // time. -// -// Experimental: not part of the stable SDK API; may change or be removed. type ChunkedWriterOption func(*chunkedWriterConfig) error // ChunkedFinalizeOption configures a single Finalize call. -// -// Experimental: not part of the stable SDK API; may change or be removed. type ChunkedFinalizeOption func(*chunkedFinalizeConfig) error // segmentSlot is one entry in the writer's segment table. The slot @@ -536,10 +533,21 @@ type chunkedWriter struct { useHex bool } -// NewChunkedWriter constructs a per-segment TDF writer. WriteSegment -// may be called from several goroutines at once so long as each -// targets a distinct segment index; two concurrent calls for the same -// index are not allowed, and one of them will fail with +// defaultChunkedWriterConfig is the starting point both constructors apply +// options over. +func defaultChunkedWriterConfig() chunkedWriterConfig { + return chunkedWriterConfig{ + archiveFactory: defaultArchiveWriterFactory, + cipherFactory: defaultSegmentCipherFactory, + clock: systemClock{}, + rand: rand.Reader, + splitter: DefaultKeySplitter(), + } +} + +// WriteSegment may be called from several goroutines at once so long +// as each targets a distinct segment index; two concurrent calls for +// the same index are not allowed, and one of them will fail with // ErrChunkedSegmentAlreadyWritten rather than corrupt the archive. // // Finalize must happen-after every WriteSegment returns. The writer @@ -555,21 +563,49 @@ type chunkedWriter struct { // WithChunkedDefaultKAS) are supplied through options; the clock, // cipher, archive-writer and entropy seams are unexported test seams // and are not reachable from outside this package. -// -// Experimental: not part of the stable SDK API; may change or be removed. func NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWriter, error) { - cfg := chunkedWriterConfig{ - archiveFactory: defaultArchiveWriterFactory, - cipherFactory: defaultSegmentCipherFactory, - clock: systemClock{}, - rand: rand.Reader, - splitter: DefaultKeySplitter(), + cfg := defaultChunkedWriterConfig() + for _, opt := range opts { + if err := opt(&cfg); err != nil { + return nil, err + } } + return newChunkedWriter(cfg) +} + +// NewChunkedWriter creates a TDF from segments that may arrive in any order, +// with key access resolved against the platform this SDK is connected to. +// Attributes are run through the same autoconfigure path SDK.CreateTDF uses, so +// a caller gets multi-KAS attribute grants without implementing a KeySplitter. +// +// Options are the same as for the package-level [NewChunkedWriter]. Pass the +// TDFOptions that shape key access — [WithDataAttributes], [WithKasInformation], +// [WithWrappingKeyAlg] and so on — through [WithChunkedTDFOptions]; they are +// replayed at Finalize. Supplying [WithChunkedKeySplitter] opts out of platform +// resolution entirely and the given splitter is used as-is. +// +// Unlike SDK.CreateTDF, key access is resolved at Finalize rather than up front, +// because a chunked caller may still be adding attributes while segments are in +// flight. An unreachable KAS therefore surfaces at Finalize, after segments have +// already been handed back. +// +// Naming no KAS is not an error: with no [WithChunkedDefaultKAS] and no attribute +// that grants one, resolution falls through to the platform's base key, exactly as +// SDK.CreateTDF does. [WithKasInformation] does not change that — it fills +// kasInfoList but leaves autoconfigure on, so a platform with a base key configured +// overwrites it and logs "base key is enabled, overwriting kasInfoList with base key +// info". To pin key access to a KAS of your choosing, pass [WithChunkedDefaultKAS]; +// that is the only option here that turns autoconfigure off. +func (s SDK) NewChunkedWriter(_ context.Context, opts ...ChunkedWriterOption) (ChunkedWriter, error) { + cfg := defaultChunkedWriterConfig() for _, opt := range opts { if err := opt(&cfg); err != nil { return nil, err } } + if !cfg.splitterSet { + cfg.keyAccess = sdkKeyAccess{sdk: s, opts: cfg.tdfOptions} + } return newChunkedWriter(cfg) } diff --git a/sdk/experimental/tdf/doc.go b/sdk/experimental/tdf/doc.go index 4340fefb97..920c5b5593 100644 --- a/sdk/experimental/tdf/doc.go +++ b/sdk/experimental/tdf/doc.go @@ -1,13 +1,37 @@ -// Experimental: This package is EXPERIMENTAL and may change or be removed at any time -// Package tdf provides experimental streaming TDF (Trusted Data Format) creation capabilities. +// Package tdf provides streaming TDF (Trusted Data Format) creation capabilities. // -// # Experimental Status +// Deprecated: this package has graduated into the sdk package itself. Use +// [github.com/opentdf/platform/sdk.SDK.NewChunkedWriter], or the package-level +// [github.com/opentdf/platform/sdk.NewChunkedWriter] when there is no platform +// connection to resolve key access against. This package now forwards to that +// implementation and will be removed in a future release. // -// This package is EXPERIMENTAL and its API is subject to change in future releases. -// It is designed for advanced use cases requiring fine-grained control over TDF creation -// with streaming support for large datasets. +// # Migration // -// For most use cases, prefer the stable SDK-level TDF creation APIs. +// Writer options: +// +// tdf.NewWriter(ctx, opts...) -> sdk.NewChunkedWriter(ctx, opts...) +// client.NewChunkedWriter(ctx, opts...) +// tdf.WithInitialAttributes(vs) -> sdk.WithChunkedInitialAttributes(vs) +// tdf.WithDefaultKASForWriter(k) -> sdk.WithChunkedDefaultKAS(k) +// tdf.WithTargetMode(m) -> sdk.WithChunkedTargetMode(m) +// +// WithIntegrityAlgorithm and WithSegmentIntegrityAlgorithm have no +// counterpart: the stable writer always signs the root with HS256 and hashes +// segments with GMAC, which is the only combination either option accepted. +// +// Finalize options: +// +// tdf.WithAssertions(as) -> sdk.WithChunkedAssertions(as) +// tdf.WithAttributeValues(vs) -> sdk.WithChunkedAttributes(vs) +// tdf.WithDefaultKAS(k) -> sdk.WithChunkedDefaultKASForFinalize(k) +// tdf.WithEncryptedMetadata(m) -> sdk.WithChunkedEncryptedMetadata(m) +// tdf.WithExcludeVersionFromManifest() -> sdk.WithChunkedExcludeVersion() +// tdf.WithPayloadMimeType(m) -> sdk.WithChunkedMimeType(m) +// tdf.WithSegments(ix) -> sdk.WithChunkedSegments(ix) +// +// The manifest and assertion types this package exports are aliases of the sdk +// types, so values move across the boundary without conversion. // // # Overview // diff --git a/sdk/experimental/tdf/writer.go b/sdk/experimental/tdf/writer.go index 4c6fb2a1cf..2309f59ba9 100644 --- a/sdk/experimental/tdf/writer.go +++ b/sdk/experimental/tdf/writer.go @@ -133,13 +133,20 @@ func NewWriter(ctx context.Context, opts ...Option[*WriterConfig]) (*Writer, err if config.segmentIntegrityAlg != SegmentGMAC { return nil, fmt.Errorf("%w: %s", sdk.ErrUnsupportedSegmentIntegrityAlgorithm, config.segmentIntegrityAlg) } - - inner, err := sdk.NewChunkedWriter(ctx, + // A nil KAS means "unset" here -- WithDefaultKASForWriter has always + // accepted one -- but the stable option rejects nil, so leave it off + // rather than forwarding. The splitter reports the missing KAS at + // Finalize as ErrNoDefaultKAS, which is what callers already handle. + chunkedOpts := []sdk.ChunkedWriterOption{ sdk.WithChunkedInitialAttributes(config.initialAttributes), - sdk.WithChunkedDefaultKAS(config.initialDefaultKAS), sdk.WithChunkedKeySplitter(xorSplitter{}), sdk.WithChunkedTargetMode(config.targetMode), - ) + } + if config.initialDefaultKAS != nil { + chunkedOpts = append(chunkedOpts, sdk.WithChunkedDefaultKAS(config.initialDefaultKAS)) + } + + inner, err := sdk.NewChunkedWriter(ctx, chunkedOpts...) if err != nil { return nil, err } @@ -286,12 +293,17 @@ func finalizeOptions(opts []Option[*WriterFinalizeConfig]) []sdk.ChunkedFinalize for _, opt := range opts { opt(cfg) } - return []sdk.ChunkedFinalizeOption{ + out := []sdk.ChunkedFinalizeOption{ sdk.WithChunkedAttributes(cfg.attributes), - sdk.WithChunkedDefaultKASForFinalize(cfg.defaultKas), sdk.WithChunkedEncryptedMetadata(cfg.encryptedMetadata), sdk.WithChunkedMimeType(cfg.payloadMimeType), sdk.WithChunkedSegments(cfg.keepSegments), sdk.WithChunkedAssertions(cfg.assertions), } + // Omitted rather than forwarded as nil, for the reason given in + // NewWriter: the stable option rejects nil, this package's does not. + if cfg.defaultKas != nil { + out = append(out, sdk.WithChunkedDefaultKASForFinalize(cfg.defaultKas)) + } + return out } diff --git a/sdk/experimental/tdf/writer_test.go b/sdk/experimental/tdf/writer_test.go index 23da5cfcea..dc12973f49 100644 --- a/sdk/experimental/tdf/writer_test.go +++ b/sdk/experimental/tdf/writer_test.go @@ -681,6 +681,23 @@ func testErrorConditions(t *testing.T) { assert.Contains(t, err.Error(), "no default KAS", "Error should mention missing default KAS") }) + // A nil KAS means "unset" in this package, but the stable option it + // delegates to rejects nil outright. The writer has to drop the option + // rather than forward the nil, or construction and Finalize would fail + // with an option error instead of the ErrNoDefaultKAS callers handle. + t.Run("ExplicitNilKASStaysUnset", func(t *testing.T) { + writer, err := NewWriter(ctx, WithDefaultKASForWriter(nil)) + require.NoError(t, err, "a nil KAS at construction is not an error here") + + _, err = writer.WriteSegment(ctx, 0, []byte("test")) + require.NoError(t, err) + + _, err = writer.Finalize(ctx, WithDefaultKAS(nil)) + require.Error(t, err) + assert.Contains(t, err.Error(), "no default KAS", + "a nil KAS should surface as the splitter's error, not an option error") + }) + t.Run("SegmentsNamesUnwrittenIndex", func(t *testing.T) { writer, err := NewWriter(ctx) require.NoError(t, err) diff --git a/sdk/key_splitter.go b/sdk/key_splitter.go index 232e4802ff..7d4b471a1e 100644 --- a/sdk/key_splitter.go +++ b/sdk/key_splitter.go @@ -25,8 +25,6 @@ import ( // holding per-call state in a field rather than on the stack corrupts one of // the two manifests, and the damage is silent: the manifest is well-formed, // carries splits that do not reconstruct the DEK, and fails only at decrypt. -// -// Experimental: not part of the stable SDK API; may change or be removed. type KeySplitter interface { // Split evaluates the ABAC policy expressed by attrs, produces N // splits of dek per the resulting boolean expression, and returns @@ -42,8 +40,6 @@ type KeySplitter interface { // Split is one XOR share of the DEK bound to one or more KAS // servers. -// -// Experimental: not part of the stable SDK API; may change or be removed. type Split struct { // Data is this share of the DEK: the value that, XOR'd with every // other share in the result, reproduces the DEK. With a single @@ -66,8 +62,6 @@ type Split struct { // SplitResult is what KeySplitter.Split returns: the shares plus the // KAS wrapping keys needed to encrypt each share into a KeyAccess // object. -// -// Experimental: not part of the stable SDK API; may change or be removed. type SplitResult struct { // KASPublicKeys maps KAS URL to the wrapping key to use for that // URL. Populated for every URL referenced by any split. @@ -94,8 +88,6 @@ type SplitResult struct { // Validate sees only the shares, so it is half the contract: it can // confirm they agree with each other, but not that they agree with the // key. VerifyReconstruction is the other half. -// -// Experimental: not part of the stable SDK API; may change or be removed. func (r *SplitResult) Validate() error { if r == nil || len(r.Splits) == 0 { return errors.New("chunked: splitter returned no splits") @@ -160,8 +152,6 @@ func (r *SplitResult) Validate() error { // With a single split there is nothing to XOR against, so the share // must equal dek verbatim; the loop below expresses that as the // degenerate case rather than special-casing it. -// -// Experimental: not part of the stable SDK API; may change or be removed. func (r *SplitResult) VerifyReconstruction(dek []byte) error { if r == nil || len(r.Splits) == 0 { return errors.New("chunked: splitter returned no splits") @@ -194,8 +184,6 @@ func (r *SplitResult) VerifyReconstruction(dek []byte) error { } // KASPublicKey is the wrapping key resolved for one KAS URL. -// -// Experimental: not part of the stable SDK API; may change or be removed. type KASPublicKey struct { // Algorithm identifies the wrapping scheme. It must be one of the // values ocrypto.ParseKeyType accepts, e.g. ocrypto.RSA2048Key or @@ -252,19 +240,55 @@ var ErrSplitterUnsupportedAlgorithm = errors.New("chunked: unsupported KAS key a // Attributes are ignored; the entire DEK is bound to the caller's // default KAS. Callers with attribute-based key splits requirements // should inject their own splitter via WithChunkedKeySplitter. -// -// Experimental: not part of the stable SDK API; may change or be removed. func DefaultKeySplitter() KeySplitter { return &singleKASSplitter{} } -// singleKASSplitter binds the full DEK to a single KAS. Attributes -// are ignored; splitting into multi-KAS OR-of-AND shares is beyond -// this default's scope. +// singleKASSplitter binds the full DEK to a single KAS. Splitting into +// multi-KAS OR-of-AND shares is beyond this default's scope. +// +// Attributes are only carried into the policy document, never consulted for +// key placement -- which is correct as long as they name no KAS of their own. +// An attribute that does carry a grant is asking for the key to be somewhere +// this splitter will not put it, and Split refuses rather than quietly +// ignoring the request; see ErrSplitterIgnoresGrants. type singleKASSplitter struct{} +// ErrSplitterIgnoresGrants is returned by the default key splitter when an +// attribute value carries a KAS grant of its own. +// +// Honoring such a grant is the job of a multi-KAS splitter. This one would +// wrap the whole DEK to the default KAS instead, producing a TDF that looks +// correct everywhere it is inspected -- the policy names the attribute, the +// manifest names a KAS, the round trip succeeds -- while the key sits at a KAS +// the attribute never authorized and never reaches the one it did. Nothing +// downstream compares the two, so the only place it can be caught is here. +// +// Supply a splitter that understands the grant model via +// [WithChunkedKeySplitter]. +var ErrSplitterIgnoresGrants = errors.New("chunked: the default key splitter cannot honor attribute KAS grants") + // Split returns one split covering the full DEK, addressed to // defaultKAS. Errors when defaultKAS is nil, has no public key or -// URI, or names an algorithm this SDK cannot wrap for. -func (s *singleKASSplitter) Split(_ context.Context, _ []*policy.Value, dek []byte, defaultKAS *policy.SimpleKasKey) (*SplitResult, error) { +// URI, names an algorithm this SDK cannot wrap for, or when any +// attribute carries a KAS grant this splitter cannot honor. +func (s *singleKASSplitter) Split(_ context.Context, attrs []*policy.Value, dek []byte, defaultKAS *policy.SimpleKasKey) (*SplitResult, error) { + // Checked before the default KAS, so a caller with real grants is told + // what is actually wrong rather than being sent to configure a default + // KAS that would not have been the right answer anyway. + // + // Grants are inherited, so all three levels count: a value's own, its + // attribute definition's, and its namespace's. KasKeys is the newer + // spelling of the same intent and is checked alongside Grants at each. + for i, v := range attrs { + switch { + case len(v.GetGrants()) > 0 || len(v.GetKasKeys()) > 0: + return nil, fmt.Errorf("%w: value %q (index %d) names its own KAS", ErrSplitterIgnoresGrants, v.GetFqn(), i) + case len(v.GetAttribute().GetGrants()) > 0 || len(v.GetAttribute().GetKasKeys()) > 0: + return nil, fmt.Errorf("%w: the attribute definition of %q (index %d) names its own KAS", ErrSplitterIgnoresGrants, v.GetFqn(), i) + case len(v.GetAttribute().GetNamespace().GetGrants()) > 0 || len(v.GetAttribute().GetNamespace().GetKasKeys()) > 0: + return nil, fmt.Errorf("%w: the namespace of %q (index %d) names its own KAS", ErrSplitterIgnoresGrants, v.GetFqn(), i) + } + } + if defaultKAS == nil || defaultKAS.GetPublicKey() == nil || defaultKAS.GetPublicKey().GetPem() == "" { return nil, ErrSplitterRequiresDefaultKAS } @@ -350,6 +374,52 @@ func (r staticKeyAccess) resolve(_ context.Context, _ []byte, _ *chunkedFinalize return r.policy, r.kaos, nil } +// sdkKeyAccess resolves key access through the platform, the way SDK.CreateTDF +// does: the attributes settled by Finalize are run through autoconfigure to +// find the KAS servers that grant them, and the DEK is split across the +// resulting plan. +// +// This is what SDK.NewChunkedWriter installs in place of DefaultKeySplitter, +// which is single-KAS and attribute-blind. Resolution is deferred to Finalize +// rather than done at construction because a chunked caller may still be adding +// attributes while segments are in flight. +type sdkKeyAccess struct { + // sdk is the platform connection used to resolve grants and fetch KAS + // public keys. + sdk SDK + + // opts are the TDFOptions given to SDK.NewChunkedWriter. They are replayed + // on each Finalize so that resolution sees the attributes as of that call. + opts []TDFOption +} + +func (r sdkKeyAccess) resolve(ctx context.Context, dek []byte, cfg *chunkedFinalizeConfig) (string, []KeyAccess, error) { + opts := r.opts + if len(cfg.attributes) > 0 { + opts = append(slices.Clone(opts), WithDataAttributeValues(cfg.attributes...)) + } + tdfConfig, err := newTDFConfig(opts...) + if err != nil { + return "", nil, err + } + tdfConfig.metaData = cfg.encryptedMetadata + + // A caller-named KAS is a decision, not a hint: honor it instead of asking + // the platform which KAS the attributes point at. Autoconfigure has to go + // off for that, since initKAOTemplate refuses to run both. + if cfg.defaultKAS != nil { + tdfConfig.autoconfigure = false + if err := populateKasInfoFromBaseKey(cfg.defaultKAS, tdfConfig); err != nil { + return "", nil, err + } + } + + if err := tdfConfig.initKAOTemplate(ctx, r.sdk); err != nil { + return "", nil, err + } + return r.sdk.resolveKeyAccess(ctx, tdfConfig, dek) +} + // splitterKeyAccess adapts a public KeySplitter to keyAccessResolver. type splitterKeyAccess struct { // splitter maps attributes plus the DEK onto KAS-addressed shares. diff --git a/sdk/key_splitter_test.go b/sdk/key_splitter_test.go index c05ea576c1..1995a7a6be 100644 --- a/sdk/key_splitter_test.go +++ b/sdk/key_splitter_test.go @@ -433,3 +433,92 @@ func TestDefaultKeySplitterResultValidates(t *testing.T) { require.NoError(t, err) require.NoError(t, res.Validate()) } + +// TestSingleKASSplitterRejectsAttributeGrants checks that the default splitter +// refuses attributes naming a KAS of their own rather than binding the key to +// the default one anyway. +// +// This is the failure the guard exists for: the resulting TDF is well-formed +// at every layer that inspects it -- the policy names the attribute, the +// manifest names a KAS, the round trip succeeds -- while the key sits at a KAS +// the attribute never authorized. Nothing downstream compares the grant to the +// placement, so creation time is the only place it can be caught. +func TestSingleKASSplitterRejectsAttributeGrants(t *testing.T) { + const fqn = "https://example.com/attr/classification/value/secret" + grant := []*policy.KeyAccessServer{{Uri: "https://grants.example.com"}} + kasKeys := []*policy.SimpleKasKey{{KasUri: "https://grants.example.com"}} + + // A default KAS good enough to succeed on, so a passing case proves the + // grant is what was rejected rather than a missing default. + defaultKAS := &policy.SimpleKasKey{ + KasUri: "https://kas.example.com", + PublicKey: &policy.SimpleKasPublicKey{ + Algorithm: policy.Algorithm_ALGORITHM_RSA_2048, + Kid: "k1", + Pem: splitterTestPEM, + }, + } + + for _, tc := range []struct { + name string + value *policy.Value + want string + }{ + { + "value grant", + &policy.Value{Fqn: fqn, Grants: grant}, + "names its own KAS", + }, + { + "value kas keys", + &policy.Value{Fqn: fqn, KasKeys: kasKeys}, + "names its own KAS", + }, + { + // Grants are inherited, so a definition-level one reaches the + // value even though the value itself declares nothing. + "attribute definition grant", + &policy.Value{Fqn: fqn, Attribute: &policy.Attribute{Grants: grant}}, + "attribute definition", + }, + { + "attribute definition kas keys", + &policy.Value{Fqn: fqn, Attribute: &policy.Attribute{KasKeys: kasKeys}}, + "attribute definition", + }, + { + "namespace grant", + &policy.Value{Fqn: fqn, Attribute: &policy.Attribute{ + Namespace: &policy.Namespace{Grants: grant}, + }}, + "namespace", + }, + { + "namespace kas keys", + &policy.Value{Fqn: fqn, Attribute: &policy.Attribute{ + Namespace: &policy.Namespace{KasKeys: kasKeys}, + }}, + "namespace", + }, + } { + t.Run(tc.name, func(t *testing.T) { + res, err := DefaultKeySplitter().Split(context.Background(), + []*policy.Value{tc.value}, []byte("0123456789abcdef"), defaultKAS) + + require.ErrorIs(t, err, ErrSplitterIgnoresGrants) + require.ErrorContains(t, err, tc.want, "the error must name which level carried the grant") + require.ErrorContains(t, err, fqn) + assert.Nil(t, res) + }) + } + + // The control: the same attribute without any grant is exactly what this + // splitter is for, and must still work. + t.Run("grantless attribute is accepted", func(t *testing.T) { + res, err := DefaultKeySplitter().Split(context.Background(), + []*policy.Value{{Fqn: fqn}}, []byte("0123456789abcdef"), defaultKAS) + require.NoError(t, err) + require.Len(t, res.Splits, 1) + assert.Equal(t, []string{"https://kas.example.com"}, res.Splits[0].KASURLs) + }) +}