diff --git a/otdfctl/cmd/tdf/decrypt.go b/otdfctl/cmd/tdf/decrypt.go index f6cd7beded..864374fed1 100644 --- a/otdfctl/cmd/tdf/decrypt.go +++ b/otdfctl/cmd/tdf/decrypt.go @@ -2,14 +2,14 @@ package tdf import ( "errors" - "fmt" + "io" "os" "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/otdfctl/cmd/common" "github.com/opentdf/platform/otdfctl/pkg/cli" + "github.com/opentdf/platform/otdfctl/pkg/handlers" "github.com/opentdf/platform/otdfctl/pkg/man" - "github.com/opentdf/platform/otdfctl/pkg/utils" "github.com/spf13/cobra" ) @@ -43,55 +43,60 @@ func decryptRun(cmd *cobra.Command, args []string) { sessionKeyAlgorithm = ocrypto.RSA2048Key } - // check for piped input - piped := readPipedStdin() - - // Prefer file argument over piped input over default filename - bytesToDecrypt := piped + // Prefer the file argument over piped input. var tdfFile string - var err error if len(args) > 0 { tdfFile = args[0] - bytesToDecrypt, err = utils.ReadBytesFromFile(tdfFile, MaxFileSize) + } + in, closeIn, err := openSeekableInput(tdfFile) + if errors.Is(err, errNoInput) { + cli.ExitWithError("Must provide ONE of the following to decrypt: [file argument, stdin input]", err) + } else if err != nil { + cli.ExitWithError("Failed to read file:", err) + } + defer closeIn() + + // Resolve the destination before decrypting, so the plaintext streams + // straight to it rather than accumulating in memory first. + var dest io.Writer = os.Stdout + var outFile *outputFile + if output != "" { + outFile, err = newOutputFile(output) if err != nil { - cli.ExitWithError("Failed to read file:", err) + closeIn() + cli.ExitWithError("Failed to write decrypted data to file", err) } + defer outFile.Cleanup() + dest = outFile } - if len(bytesToDecrypt) == 0 { - cli.ExitWithError("Must provide ONE of the following to decrypt: [file argument, stdin input]", errors.New("no input provided")) + // cli.ExitWithError calls os.Exit, which skips deferred functions, so both + // the spooled input and the partial output have to be discarded first. + fail := func(msg string, err error) { + closeIn() + if outFile != nil { + outFile.Cleanup() + } + cli.ExitWithError(msg, err) } ignoreAllowlist := len(kasAllowList) == 1 && kasAllowList[0] == "*" - decrypted, err := h.DecryptBytes( - c.Context(), - bytesToDecrypt, - assertionVerification, - disableAssertionVerification, - sessionKeyAlgorithm, - kasAllowList, - ignoreAllowlist, - nil, - ) + err = h.Decrypt(c.Context(), dest, in, handlers.DecryptOptions{ + AssertionVerificationKeysFile: assertionVerification, + DisableAssertionCheck: disableAssertionVerification, + SessionKeyAlgorithm: sessionKeyAlgorithm, + KASAllowList: kasAllowList, + IgnoreAllowlist: ignoreAllowlist, + }) if err != nil { - cli.ExitWithError("Failed to decrypt file", err) + fail("Failed to decrypt file", err) } - if output == "" { - //nolint:forbidigo // printing decrypted content to stdout - fmt.Print(decrypted.String()) - return - } - // Here 'output' is the filename given with -o - f, err := os.Create(output) - if err != nil { - cli.ExitWithError("Failed to write decrypted data to file", err) - } - defer f.Close() - _, err = f.Write(decrypted.Bytes()) - if err != nil { - cli.ExitWithError("Failed to write decrypted data to file", err) + if outFile != nil { + if err := outFile.Commit(); err != nil { + fail("Failed to write decrypted data to file", err) + } } } diff --git a/otdfctl/cmd/tdf/encrypt.go b/otdfctl/cmd/tdf/encrypt.go index 3935825f28..28ce436906 100644 --- a/otdfctl/cmd/tdf/encrypt.go +++ b/otdfctl/cmd/tdf/encrypt.go @@ -1,6 +1,8 @@ package tdf import ( + "bufio" + "errors" "io" "log/slog" "os" @@ -11,8 +13,8 @@ import ( "github.com/opentdf/platform/lib/ocrypto" "github.com/opentdf/platform/otdfctl/cmd/common" "github.com/opentdf/platform/otdfctl/pkg/cli" + "github.com/opentdf/platform/otdfctl/pkg/handlers" "github.com/opentdf/platform/otdfctl/pkg/man" - "github.com/opentdf/platform/otdfctl/pkg/utils" "github.com/spf13/cobra" ) @@ -24,6 +26,38 @@ var ( EncryptCmd = &encryptDoc.Command ) +// detectMimeType sniffs the payload's type from its head, and returns a reader +// that still yields the whole payload. +// +// Detection needs only the first megabyte, and Peek does not consume, so those +// bytes stay in the buffer and reach the encoder. The payload is never held in +// memory in full. Wrapping is uniform across files and pipes and needs no seek. +func detectMimeType(in io.Reader, fileExt string) (string, io.Reader, error) { + buffered, ok := in.(*bufio.Reader) + if !ok { + buffered = bufio.NewReaderSize(in, Size1MB) + in = buffered + } + + mimetype.SetLimit(Size1MB) // limit to 1MB + head, err := buffered.Peek(Size1MB) + // A payload shorter than the peek window is the common case, not an error. + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, bufio.ErrBufferFull) { + return "", in, err + } + + // defaults to application/octet-stream if nothing is recognized + detected := mimetype.Detect(head).String() + if detected == "application/octet-stream" && fileExt != "" { + // Lookup returns nil for an extension it does not know, which is not an + // error — octet-stream remains the right answer. + if byExt := mimetype.Lookup(fileExt); byExt != nil { + detected = byExt.String() + } + } + return detected, in, nil +} + func encryptRun(cmd *cobra.Command, args []string) { c := cli.New(cmd, args, cli.WithPrintJSON()) h := common.NewHandler(c) @@ -57,13 +91,13 @@ func encryptRun(cmd *cobra.Command, args []string) { wrappingKeyAlgorithm = ocrypto.RSA2048Key } - piped := readPipedStdin() + piped, hasPiped := stdinReader() inputCount := 0 if filePath != "" { inputCount++ } - if len(piped) > 0 { + if hasPiped { inputCount++ } @@ -77,70 +111,86 @@ func encryptRun(cmd *cobra.Command, args []string) { } // prefer filepath argument over stdin input - bytesSlice := piped - var err error + var in io.Reader = piped + inputSize := handlers.InputSizeUnknown if filePath != "" { - bytesSlice, err = utils.ReadBytesFromFile(filePath, MaxFileSize) + f, err := os.Open(filePath) if err != nil { cli.ExitWithError("Failed to read file:", err) } + defer f.Close() + // Declaring the length keeps the SDK off its unknown-size path, which + // must assume ZIP64 because the choice is baked into the header of the + // first segment. Without this, wrapping the file for MIME detection + // below would hide its Seeker and silently change the output format. + if fi, err := f.Stat(); err == nil && fi.Mode().IsRegular() { + inputSize = fi.Size() + } + in = f } // auto-detect mime type if not provided if fileMimeType == "" { slog.Debug("detecting mime type of file") - // get the mime type of the file - mimetype.SetLimit(Size1MB) // limit to 1MB - m := mimetype.Detect(bytesSlice) - // default to application/octet-stream if no mime type is detected - fileMimeType = m.String() - - if fileMimeType == "application/octet-stream" { - if fileExt != "" { - fileMimeType = mimetype.Lookup(fileExt).String() - } + var err error + fileMimeType, in, err = detectMimeType(in, fileExt) + if err != nil { + cli.ExitWithError("Failed to read file:", err) } } slog.Debug("encrypting file", - slog.Int("file_len", len(bytesSlice)), + slog.Int64("file_len", inputSize), slog.String("mime_type", fileMimeType), ) - // Do the encryption - encrypted, err := h.EncryptBytes( - tdfType, - bytesSlice, - attrValues, - fileMimeType, - kasURLPath, - assertions, - wrappingKeyAlgorithm, - targetMode, - ) - if err != nil { - cli.ExitWithError("Failed to encrypt", err) - } - - // Find the destination as the output flag filename or stdout - var dest *os.File + // Resolve the destination before encrypting, so that the payload streams + // straight to it rather than accumulating in memory first. + var dest io.Writer + var tdfFile *outputFile if out != "" { // make sure output ends in .tdf extension if !strings.HasSuffix(out, ".tdf") { out += ".tdf" } - tdfFile, err := os.Create(out) + var err error + tdfFile, err = newOutputFile(out) if err != nil { cli.ExitWithError("Failed to write encrypted file "+out, err) } - defer tdfFile.Close() + defer tdfFile.Cleanup() dest = tdfFile } else { dest = os.Stdout } - _, e := io.Copy(dest, encrypted) - if e != nil { - cli.ExitWithError("Failed to write encrypted data to stdout", e) + // cli.ExitWithError calls os.Exit, which skips deferred functions, so the + // partial output has to be discarded before handing off to it. + fail := func(msg string, err error) { + if tdfFile != nil { + tdfFile.Cleanup() + } + cli.ExitWithError(msg, err) + } + + // Do the encryption + err := h.Encrypt(c.Context(), dest, in, handlers.EncryptOptions{ + TDFType: tdfType, + Attributes: attrValues, + MimeType: fileMimeType, + KASURLPath: kasURLPath, + Assertions: assertions, + WrappingKeyAlgorithm: wrappingKeyAlgorithm, + TargetMode: targetMode, + InputSize: inputSize, + }) + if err != nil { + fail("Failed to encrypt", err) + } + + if tdfFile != nil { + if err := tdfFile.Commit(); err != nil { + fail("Failed to write encrypted file "+out, err) + } } } diff --git a/otdfctl/cmd/tdf/inspect.go b/otdfctl/cmd/tdf/inspect.go index a35a3e0d04..be192dfd86 100644 --- a/otdfctl/cmd/tdf/inspect.go +++ b/otdfctl/cmd/tdf/inspect.go @@ -42,17 +42,32 @@ func inspectRun(cmd *cobra.Command, args []string) { h := common.NewHandler(c) defer h.Close() - data := cli.ReadFromArgsOrPipe(args, nil) - if len(data) == 0 { - c.ExitWithError("must provide ONE of the following: [file argument, stdin input]", errors.New("no input provided")) + // Inspect reads only the manifest, so the TDF is never held in memory even + // when the payload dwarfs it. + var path string + if len(args) > 0 { + path = args[0] + } + in, closeIn, err := openSeekableInput(path) + if errors.Is(err, errNoInput) { + c.ExitWithError("must provide ONE of the following: [file argument, stdin input]", err) + } else if err != nil { + c.ExitWithError("failed to read input", err) + } + // Every exit below goes through os.Exit, which skips deferred functions, so + // a spooled pipe has to be discarded explicitly on each path. + defer closeIn() + fail := func(msg string, err error) { + closeIn() + c.ExitWithError(msg, err) } - result, errs := h.InspectTDF(data) + result, errs := h.InspectTDF(in) for _, err := range errs { if errors.Is(err, handlers.ErrTDFInspectFailNotValidTDF) { - c.ExitWithError("not a valid TDF", err) + fail("not a valid TDF", err) } else if errors.Is(err, handlers.ErrTDFInspectFailNotInspectable) { - c.ExitWithError("failed to inspect TDF", err) + fail("failed to inspect TDF", err) } } @@ -76,9 +91,10 @@ func inspectRun(cmd *cobra.Command, args []string) { Attributes: result.Attributes, } + closeIn() c.ExitWithJSON(m, cli.ExitCodeSuccess) } - c.ExitWithError("failed to inspect TDF", nil) + fail("failed to inspect TDF", nil) } func InitInspectCommand() { diff --git a/otdfctl/cmd/tdf/tdf.go b/otdfctl/cmd/tdf/tdf.go index 7f727af92b..3bf445ee7b 100644 --- a/otdfctl/cmd/tdf/tdf.go +++ b/otdfctl/cmd/tdf/tdf.go @@ -1,31 +1,161 @@ package tdf import ( + "bufio" + "errors" "io" "os" + "path/filepath" "github.com/opentdf/platform/otdfctl/pkg/cli" ) const ( - Size1MB = 1024 * 1024 - MaxFileSize = int64(10 * 1024 * 1024 * 1024) // 10 GB - TDF = "TDF" + Size1MB = 1024 * 1024 + TDF = "TDF" // GroupID is the group ID for TDF commands GroupID = TDF ) -func readPipedStdin() []byte { - stat, err := os.Stdin.Stat() +// stdinReader reports whether stdin is a pipe or redirect carrying at least one +// byte, and returns a reader over it. +// +// Presence is established with a one-byte Peek rather than a read, so the +// payload still reaches the caller intact and nothing is buffered beyond the +// reader's window. A terminal, or an empty redirect such as +// `otdfctl encrypt < /dev/null`, reports false — matching the behavior of the +// buffered implementation, which decided the same question by checking whether +// a full read of stdin came back empty. +// +// The buffer is sized so that a later Peek for MIME detection is served from +// it without a second allocation. +func stdinReader() (*bufio.Reader, bool) { + r, ok, err := pipeReader(os.Stdin) if err != nil { - cli.ExitWithError("Failed to read stat from stdin", err) + cli.ExitWithError("failed to scan bytes from stdin", err) } - if (stat.Mode() & os.ModeCharDevice) == 0 { - buf, err := io.ReadAll(os.Stdin) + return r, ok +} + +// pipeReader is the testable half of stdinReader. +func pipeReader(in *os.File) (*bufio.Reader, bool, error) { + stat, err := in.Stat() + if err != nil { + return nil, false, err + } + if (stat.Mode() & os.ModeCharDevice) != 0 { + return nil, false, nil + } + + r := bufio.NewReaderSize(in, Size1MB) + if _, err := r.Peek(1); err != nil { + if errors.Is(err, io.EOF) { + return nil, false, nil + } + return nil, false, err + } + return r, true, nil +} + +// spoolToTempFile copies r into a temporary file and rewinds it, giving a +// seekable view of a stream that has none. +// +// A TDF's manifest sits at the end of the archive, so decrypt and inspect have +// to seek and cannot consume a pipe directly. Spooling trades disk for the +// memory the old whole-payload read used, and needs a TMPDIR with room for the +// full TDF — it fails loudly if there isn't one. +// +// The returned cleanup must run on every path. cli.ExitWithError calls os.Exit +// and skips deferred functions, so deferring it alone is not enough. +func spoolToTempFile(r io.Reader) (*os.File, func(), error) { + f, err := os.CreateTemp("", "otdfctl-spool-*.tdf") + if err != nil { + return nil, func() {}, err + } + cleanup := func() { + f.Close() + os.Remove(f.Name()) + } + if _, err := io.Copy(f, r); err != nil { + cleanup() + return nil, func() {}, err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + cleanup() + return nil, func() {}, err + } + return f, cleanup, nil +} + +// openSeekableInput resolves a command's input to something seekable: the named +// file when one is given, otherwise piped stdin spooled to disk. It reports the +// same "no input" condition for an absent file argument and an empty pipe. +// +// The returned cleanup must run on every path, per spoolToTempFile. +func openSeekableInput(path string) (*os.File, func(), error) { + if path != "" { + f, err := os.Open(path) if err != nil { - cli.ExitWithError("failed to scan bytes from stdin", err) + return nil, func() {}, err } - return buf + return f, func() { f.Close() }, nil } + + piped, ok := stdinReader() + if !ok { + return nil, func() {}, errNoInput + } + return spoolToTempFile(piped) +} + +var errNoInput = errors.New("no input provided") + +// outputFile writes to a temporary file alongside the destination and renames +// it into place only once the write has succeeded, so an interrupted or failed +// run leaves no partial output where a complete file is expected. +// +// Note that cli.ExitWithError calls os.Exit, which does not run deferred +// functions. Cleanup must therefore be called explicitly on every error path, +// not only via defer. +type outputFile struct { + f *os.File + path string + committed bool +} + +// newOutputFile creates the temporary file in the destination's own directory, +// which keeps the final rename atomic — across filesystems it would degrade to +// a copy. +func newOutputFile(path string) (*outputFile, error) { + f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*") + if err != nil { + return nil, err + } + return &outputFile{f: f, path: path}, nil +} + +func (o *outputFile) Write(p []byte) (int, error) { return o.f.Write(p) } + +// Commit closes the temporary file and moves it onto the destination path. +func (o *outputFile) Commit() error { + if err := o.f.Close(); err != nil { + os.Remove(o.f.Name()) + return err + } + if err := os.Rename(o.f.Name(), o.path); err != nil { + os.Remove(o.f.Name()) + return err + } + o.committed = true return nil } + +// Cleanup discards the temporary file. It is a no-op after a successful +// Commit, so it is safe to both defer it and call it directly. +func (o *outputFile) Cleanup() { + if o.committed { + return + } + o.f.Close() + os.Remove(o.f.Name()) +} diff --git a/otdfctl/cmd/tdf/tdf_test.go b/otdfctl/cmd/tdf/tdf_test.go new file mode 100644 index 0000000000..2d7160f8de --- /dev/null +++ b/otdfctl/cmd/tdf/tdf_test.go @@ -0,0 +1,229 @@ +package tdf + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOutputFileCommitRenamesIntoPlace(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "out.tdf") + + o, err := newOutputFile(dest) + require.NoError(t, err) + + _, err = o.Write([]byte("payload")) + require.NoError(t, err) + + // Nothing is visible at the destination until Commit. + _, err = os.Stat(dest) + require.ErrorIs(t, err, os.ErrNotExist, "destination must not exist before Commit") + + require.NoError(t, o.Commit()) + + got, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "payload", string(got)) + assert.Empty(t, tempSiblings(t, dir, "out.tdf"), "temp file should be gone after Commit") +} + +func TestOutputFileCleanupLeavesNoPartialOutput(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "out.tdf") + + o, err := newOutputFile(dest) + require.NoError(t, err) + _, err = o.Write([]byte("partial")) + require.NoError(t, err) + + // Simulates the failure path: encryption died after some bytes were written. + o.Cleanup() + + _, err = os.Stat(dest) + require.ErrorIs(t, err, os.ErrNotExist, "a failed run must not leave a partial file") + assert.Empty(t, tempSiblings(t, dir, "out.tdf"), "a failed run must not leave a temp file") +} + +func TestOutputFileCleanupAfterCommitIsNoop(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "out.tdf") + + o, err := newOutputFile(dest) + require.NoError(t, err) + _, err = o.Write([]byte("payload")) + require.NoError(t, err) + require.NoError(t, o.Commit()) + + // Both deferred and explicit cleanup run on the success path. + o.Cleanup() + o.Cleanup() + + got, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "payload", string(got), "Cleanup after Commit must not delete the output") +} + +func TestOutputFileTempIsSiblingOfDestination(t *testing.T) { + dir := t.TempDir() + dest := filepath.Join(dir, "out.tdf") + + o, err := newOutputFile(dest) + require.NoError(t, err) + defer o.Cleanup() + + // A cross-filesystem temp dir would turn the rename into a copy and lose + // atomicity, so the temp file must live beside the destination. + assert.Equal(t, dir, filepath.Dir(o.f.Name())) +} + +func TestPipeReader(t *testing.T) { + for _, tc := range []struct { + name string + content string + wantOK bool + }{ + {name: "with data", content: "hello world", wantOK: true}, + {name: "empty pipe reports absent", content: "", wantOK: false}, + } { + t.Run(tc.name, func(t *testing.T) { + r, w, err := os.Pipe() + require.NoError(t, err) + defer r.Close() + + go func() { + defer w.Close() + _, _ = io.WriteString(w, tc.content) + }() + + got, ok, err := pipeReader(r) + require.NoError(t, err) + require.Equal(t, tc.wantOK, ok) + if !tc.wantOK { + return + } + + // Peek must not consume: the whole payload is still readable. + all, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, tc.content, string(all)) + }) + } +} + +func TestPipeReaderPreservesPayloadLargerThanBuffer(t *testing.T) { + content := strings.Repeat("a", Size1MB*2+7) + + r, w, err := os.Pipe() + require.NoError(t, err) + defer r.Close() + + go func() { + defer w.Close() + _, _ = io.WriteString(w, content) + }() + + got, ok, err := pipeReader(r) + require.NoError(t, err) + require.True(t, ok) + + all, err := io.ReadAll(got) + require.NoError(t, err) + assert.Len(t, all, len(content)) +} + +func TestPipeReaderOnTerminalReportsAbsent(t *testing.T) { + // A regular file is not a char device, so use os.Stdin's actual mode only + // when it is one; otherwise this assertion is vacuous and we skip. + stat, err := os.Stdin.Stat() + require.NoError(t, err) + if (stat.Mode() & os.ModeCharDevice) == 0 { + t.Skip("stdin is not a terminal under this test runner") + } + _, ok, err := pipeReader(os.Stdin) + require.NoError(t, err) + assert.False(t, ok) +} + +func TestSpoolToTempFileIsSeekableAndComplete(t *testing.T) { + // Larger than any plausible internal buffer, so a truncating copy shows up. + content := strings.Repeat("xyz", Size1MB) + + f, cleanup, err := spoolToTempFile(strings.NewReader(content)) + require.NoError(t, err) + defer cleanup() + + // The spool must be positioned at the head, not at the end of the copy. + pos, err := f.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos, "spool must be rewound for the caller") + + got, err := io.ReadAll(f) + require.NoError(t, err) + assert.Len(t, got, len(content)) + + // Seeking backwards is the whole reason for spooling: a TDF's manifest is at + // the end of the archive, so the reader has to be able to go back. + _, err = f.Seek(0, io.SeekStart) + require.NoError(t, err) + head := make([]byte, 3) + _, err = io.ReadFull(f, head) + require.NoError(t, err) + assert.Equal(t, "xyz", string(head)) +} + +func TestSpoolToTempFileCleanupRemovesTheFile(t *testing.T) { + f, cleanup, err := spoolToTempFile(strings.NewReader("payload")) + require.NoError(t, err) + + name := f.Name() + require.FileExists(t, name) + + cleanup() + assert.NoFileExists(t, name, "the spool must not outlive the command") +} + +func TestOpenSeekableInputReadsNamedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "in.tdf") + require.NoError(t, os.WriteFile(path, []byte("payload"), 0o600)) + + in, cleanup, err := openSeekableInput(path) + require.NoError(t, err) + defer cleanup() + + got, err := io.ReadAll(in) + require.NoError(t, err) + assert.Equal(t, "payload", string(got)) + + // A named file is opened directly, not copied through a spool. + assert.Equal(t, path, in.Name()) +} + +func TestOpenSeekableInputReportsMissingFile(t *testing.T) { + _, _, err := openSeekableInput(filepath.Join(t.TempDir(), "absent.tdf")) + require.Error(t, err) + // Distinguishable from "you gave me nothing", which callers report differently. + require.NotErrorIs(t, err, errNoInput) + require.ErrorIs(t, err, os.ErrNotExist) +} + +// tempSiblings returns any leftover temp files newOutputFile would have created +// for dest in dir. +func tempSiblings(t *testing.T, dir, dest string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + + var found []string + for _, e := range entries { + if strings.HasPrefix(e.Name(), "."+dest+".tmp-") { + found = append(found, e.Name()) + } + } + return found +} diff --git a/otdfctl/e2e/encrypt-decrypt.bats b/otdfctl/e2e/encrypt-decrypt.bats index 32b336e676..23a0358381 100755 --- a/otdfctl/e2e/encrypt-decrypt.bats +++ b/otdfctl/e2e/encrypt-decrypt.bats @@ -103,6 +103,102 @@ teardown_file(){ ./otdfctl decrypt --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $OUTFILE_TXT | grep "$SECRET_TEXT" } +@test "roundtrip TDF3, no attributes, file to stdout" { + ./otdfctl encrypt --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $INFILE_GO_MOD > $OUTFILE_GO_MOD + ./otdfctl decrypt -o $RESULTFILE_GO_MOD --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $OUTFILE_GO_MOD + diff $INFILE_GO_MOD $RESULTFILE_GO_MOD +} + +# The fully piped form is the one documented in docs/man/encrypt/_index.md, and the +# one that has no seekable input on either end. +@test "roundtrip TDF3, stdin to stdout, fully piped" { + run bash -c "echo '$SECRET_TEXT' | ./otdfctl encrypt --host $HOST --tls-no-verify $WITH_CREDS | ./otdfctl decrypt --host $HOST --tls-no-verify $WITH_CREDS" + assert_success + assert_output --partial "$SECRET_TEXT" +} + +# An empty redirect is 'no input', not 'a zero-byte payload'. +@test "encrypt rejects empty stdin" { + run bash -c "./otdfctl encrypt --host $HOST --tls-no-verify $WITH_CREDS < /dev/null" + assert_failure +} + +# Output is written to a temp sibling and renamed only on success, so a failed run +# must leave neither a partial .tdf nor the temp file behind. +@test "encrypt leaves no output behind when it fails" { + rm -f $OUTFILE_TXT .$OUTFILE_TXT.tmp-* + run bash -c "echo '$SECRET_TEXT' | ./otdfctl encrypt -o $OUT_TXT --host $HOST --tls-no-verify $WITH_CREDS -a 'https://testing-enc-dec.io/attr/attr1/value/does-not-exist'" + assert_failure + [ ! -f "$OUTFILE_TXT" ] + run bash -c "ls .$OUTFILE_TXT.tmp-* 2>/dev/null | wc -l" + assert_output "0" +} + +# A TDF's manifest is at the end of the archive, so decrypt spools a pipe to disk +# to get a seekable view. Verify it round-trips and cleans the spool up. +@test "roundtrip TDF3, decrypt reading the TDF from stdin" { + ./otdfctl encrypt -o $OUTFILE_GO_MOD --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $INFILE_GO_MOD + ./otdfctl decrypt --host $HOST --tls-no-verify $WITH_CREDS < $OUTFILE_GO_MOD > $RESULTFILE_GO_MOD + diff $INFILE_GO_MOD $RESULTFILE_GO_MOD + run bash -c "ls ${TMPDIR:-/tmp}/otdfctl-spool-* 2>/dev/null | wc -l" + assert_output "0" +} + +@test "decrypt rejects empty stdin" { + run bash -c "./otdfctl decrypt --host $HOST --tls-no-verify $WITH_CREDS < /dev/null" + assert_failure +} + +@test "decrypt leaves no output behind when it fails" { + ./otdfctl encrypt -o $OUTFILE_GO_MOD --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $INFILE_GO_MOD + rm -f $RESULTFILE_GO_MOD .$RESULTFILE_GO_MOD.tmp-* + # An allowlist with no entry for the platform KAS fails the rewrap. + run ./otdfctl decrypt -o $RESULTFILE_GO_MOD --host $HOST --tls-no-verify $WITH_CREDS --kas-allowlist "https://nowhere.example.com" $OUTFILE_GO_MOD + assert_failure + [ ! -f "$RESULTFILE_GO_MOD" ] + run bash -c "ls .$RESULTFILE_GO_MOD.tmp-* 2>/dev/null | wc -l" + assert_output "0" +} + +@test "inspect reads a TDF from a file and from stdin" { + ./otdfctl encrypt -o $OUTFILE_GO_MOD --host $HOST --tls-no-verify $DEBUG_LEVEL $WITH_CREDS $INFILE_GO_MOD + run bash -c "./otdfctl inspect --host $HOST --tls-no-verify $WITH_CREDS $OUTFILE_GO_MOD | jq -r '.manifest.protocol'" + assert_success + assert_output "zip" + run bash -c "./otdfctl inspect --host $HOST --tls-no-verify $WITH_CREDS < $OUTFILE_GO_MOD | jq -r '.manifest.protocol'" + assert_success + assert_output "zip" +} + +# The point of DSPX-4499: peak RSS is bounded by segment size, not payload size. +# Needs GNU time for 'Maximum resident set size'; BSD/shell time cannot report it. +@test "encrypt and decrypt peak memory stay bounded on a large payload" { + GNU_TIME=$(command -v gtime || command -v /usr/bin/time) + if [ -z "$GNU_TIME" ] || ! $GNU_TIME -v true 2>&1 | grep -q "Maximum resident set size"; then + skip "GNU time not available" + fi + + local big=big.bin + local bigtdf=big.bin.tdf + local bigout=big.out + # 1 GiB. The buffered implementation peaked around 3.6x this for both commands. + dd if=/dev/zero of=$big bs=1048576 count=1024 status=none + + $GNU_TIME -v -o enc.log ./otdfctl encrypt -o $bigtdf --host $HOST --tls-no-verify $WITH_CREDS $big + $GNU_TIME -v -o dec.log ./otdfctl decrypt -o $bigout --host $HOST --tls-no-verify $WITH_CREDS $bigtdf + diff $big $bigout + + local enc_kb=$(grep "Maximum resident set size" enc.log | grep -o '[0-9]*') + local dec_kb=$(grep "Maximum resident set size" dec.log | grep -o '[0-9]*') + rm -f $big $bigtdf $bigout enc.log dec.log + + echo "peak RSS: encrypt ${enc_kb} KB, decrypt ${dec_kb} KB" + # 512 MiB leaves generous headroom over the ~66 MiB a 1 MiB payload used, while + # still failing loudly on any return to whole-payload buffering. + [ "$enc_kb" -lt 524288 ] + [ "$dec_kb" -lt 524288 ] +} + @test "allow traversal with mapped key uses definition when value missing" { local attr_name="attr-allow-traversal-${RANDOM}" local kas_name="kas-allow-traversal-${RANDOM}" diff --git a/otdfctl/pkg/cli/pipe.go b/otdfctl/pkg/cli/pipe.go deleted file mode 100644 index 561c8ff19f..0000000000 --- a/otdfctl/pkg/cli/pipe.go +++ /dev/null @@ -1,45 +0,0 @@ -package cli - -import ( - "io" - "os" -) - -func ReadFromArgsOrPipe(args []string, pipe *os.File) []byte { - if len(args) > 0 { - return ReadFromFile(args[0]) - } - if pipe == nil { - pipe = os.Stdin - } - return ReadFromPipe(pipe) -} - -func ReadFromPipe(in *os.File) []byte { - stat, err := in.Stat() - if err != nil { - ExitWithError("failed to read stat from stdin", err) - } - if (stat.Mode() & os.ModeCharDevice) == 0 { - buf, err := io.ReadAll(in) - if err != nil { - ExitWithError("failed to scan bytes from stdin", err) - } - return buf - } - return nil -} - -func ReadFromFile(filePath string) []byte { - fileToEncrypt, err := os.Open(filePath) - if err != nil { - ExitWithError("Failed to git open file at path: "+filePath, err) - } - defer fileToEncrypt.Close() - - bytes, err := io.ReadAll(fileToEncrypt) - if err != nil { - ExitWithError("Failed to read bytes from file at path: "+filePath, err) - } - return bytes -} diff --git a/otdfctl/pkg/handlers/tdf.go b/otdfctl/pkg/handlers/tdf.go index d8a5b0d69f..28f50bf7fb 100644 --- a/otdfctl/pkg/handlers/tdf.go +++ b/otdfctl/pkg/handlers/tdf.go @@ -1,7 +1,6 @@ package handlers import ( - "bytes" "context" "crypto/rsa" "crypto/x509" @@ -38,51 +37,68 @@ type TDFInspect struct { UnencryptedMetadata []byte } -func (h Handler) EncryptBytes( - tdfType string, - unencrypted []byte, - attrValues []string, - mimeType string, - kasURLPath string, - assertions string, - wrappingKeyAlgorithm ocrypto.KeyType, - targetMode string, -) (*bytes.Buffer, error) { - var encrypted []byte - enc := bytes.NewBuffer(encrypted) +// InputSizeUnknown marks a payload whose length cannot be established up front, +// such as one arriving on a pipe. Encrypt passes it through to the SDK, which +// then measures the payload by reading it. +const InputSizeUnknown = int64(-1) - switch tdfType { +// EncryptOptions carries the non-stream inputs to Encrypt. +type EncryptOptions struct { + TDFType string + Attributes []string + MimeType string + KASURLPath string + Assertions string + WrappingKeyAlgorithm ocrypto.KeyType + TargetMode string + + // InputSize is the payload length in bytes, or InputSizeUnknown when it + // cannot be determined without consuming the reader. Supplying a known + // length lets the SDK size the archive up front rather than defaulting to + // ZIP64, which keeps output byte-comparable with a seekable reader. + InputSize int64 +} + +// Encrypt streams the plaintext from in to a TDF written to out. Memory use is +// bounded by the SDK's segment size rather than by the payload length, so in +// may be a pipe and the payload may be larger than RAM. +func (h Handler) Encrypt(ctx context.Context, out io.Writer, in io.Reader, o EncryptOptions) error { + switch o.TDFType { // Encrypt the data as a ZTDF case "", tdf.TypeTDF3, tdf.TypeZTDF: opts := []sdk.TDFOption{ - sdk.WithDataAttributes(attrValues...), + sdk.WithDataAttributes(o.Attributes...), sdk.WithKasInformation(sdk.KASInfo{ - URL: h.platformEndpoint + kasURLPath, + URL: h.platformEndpoint + o.KASURLPath, }), - sdk.WithMimeType(mimeType), - sdk.WithWrappingKeyAlg(wrappingKeyAlgorithm), //nolint:staticcheck // SDK option is deprecated but no replacement is available in this SDK version. + sdk.WithMimeType(o.MimeType), + sdk.WithWrappingKeyAlg(o.WrappingKeyAlgorithm), //nolint:staticcheck // SDK option is deprecated but no replacement is available in this SDK version. + } + + if o.InputSize >= 0 { + opts = append(opts, sdk.WithInputSize(o.InputSize)) } var assertionConfigs []sdk.AssertionConfig //nolint:nestif // nested its mainly for error catching and handling case of string vs file - if assertions != "" { - err := json.Unmarshal([]byte(assertions), &assertionConfigs) + if o.Assertions != "" { + err := json.Unmarshal([]byte(o.Assertions), &assertionConfigs) if err != nil { // if unable to marshal to json, interpret as file string and try to read from file - assertionBytes, err := utils.ReadBytesFromFile(assertions, MaxAssertionsFileSize) + assertionBytes, err := utils.ReadBytesFromFile(o.Assertions, MaxAssertionsFileSize) if err != nil { - return nil, fmt.Errorf("unable to read assertions file: %w", err) + return fmt.Errorf("unable to read assertions file: %w", err) } err = json.Unmarshal(assertionBytes, &assertionConfigs) if err != nil { - return nil, fmt.Errorf("unable to unmarshal assertions json: %w", err) + return fmt.Errorf("unable to unmarshal assertions json: %w", err) } } for i, config := range assertionConfigs { if !config.SigningKey.IsEmpty() { correctedKey, err := correctKeyType(config.SigningKey, false) if err != nil { - return nil, fmt.Errorf("error with assertion signing key: %w", err) + return fmt.Errorf("error with assertion signing key: %w", err) } assertionConfigs[i].SigningKey.Key = correctedKey } @@ -90,85 +106,103 @@ func (h Handler) EncryptBytes( opts = append(opts, sdk.WithAssertions(assertionConfigs...)) } - if targetMode != "" { - opts = append(opts, sdk.WithTargetMode(targetMode)) + if o.TargetMode != "" { + opts = append(opts, sdk.WithTargetMode(o.TargetMode)) } - _, err := h.sdk.CreateTDF(enc, bytes.NewReader(unencrypted), opts...) - return enc, err + _, err := h.sdk.CreateTDFContext(ctx, out, in, opts...) + return err default: - return nil, errors.New("unknown TDF type") + return errors.New("unknown TDF type") } } -func (h Handler) DecryptBytes( - ctx context.Context, - toDecrypt []byte, - assertionVerificationKeysFile string, - disableAssertionCheck bool, - sessionKeyAlgorithm ocrypto.KeyType, - kasAllowList []string, - ignoreAllowlist bool, - fulfillableObligations []string, -) (*bytes.Buffer, error) { - out := &bytes.Buffer{} - pt := io.Writer(out) - ec := bytes.NewReader(toDecrypt) - switch sdk.GetTdfType(ec) { +// DecryptOptions carries the non-stream inputs to Decrypt. +type DecryptOptions struct { + AssertionVerificationKeysFile string + DisableAssertionCheck bool + SessionKeyAlgorithm ocrypto.KeyType + KASAllowList []string + IgnoreAllowlist bool + FulfillableObligations []string +} + +// streamingCopy asserts that the SDK reader is copied one segment at a time. +// +// io.Copy prefers WriteTo when the source implements it, and sdk.Reader's +// WriteTo decrypts segment by segment. Without it, io.Copy would fall back to +// Read, which serves bytes from an internal buffer grown by ReadAt — putting +// the whole payload back in memory and silently undoing this change with no +// test failure to show for it. +var _ io.WriterTo = (*sdk.Reader)(nil) + +// Decrypt streams the plaintext of the TDF in in to out. Memory use is bounded +// by the SDK's segment size rather than by the payload length. +// +// in must be seekable because the TDF's manifest lives at the end of the +// archive; callers with a pipe need to spool it first. +func (h Handler) Decrypt(ctx context.Context, out io.Writer, in io.ReadSeeker, o DecryptOptions) error { + switch sdk.GetTdfType(in) { case sdk.Standard: opts := []sdk.TDFReaderOption{ - sdk.WithDisableAssertionVerification(disableAssertionCheck), - sdk.WithSessionKeyType(sessionKeyAlgorithm), - sdk.WithIgnoreAllowlist(ignoreAllowlist), - sdk.WithTDFFulfillableObligationFQNs(fulfillableObligations), + sdk.WithDisableAssertionVerification(o.DisableAssertionCheck), + sdk.WithSessionKeyType(o.SessionKeyAlgorithm), + sdk.WithIgnoreAllowlist(o.IgnoreAllowlist), + sdk.WithTDFFulfillableObligationFQNs(o.FulfillableObligations), } - if kasAllowList != nil { - opts = append(opts, sdk.WithKasAllowlist(kasAllowList)) + if o.KASAllowList != nil { + opts = append(opts, sdk.WithKasAllowlist(o.KASAllowList)) } var assertionVerificationKeys sdk.AssertionVerificationKeys - if assertionVerificationKeysFile != "" { + if o.AssertionVerificationKeysFile != "" { // read the file - assertionVerificationBytes, err := utils.ReadBytesFromFile(assertionVerificationKeysFile, MaxAssertionsFileSize) + assertionVerificationBytes, err := utils.ReadBytesFromFile(o.AssertionVerificationKeysFile, MaxAssertionsFileSize) if err != nil { - return nil, fmt.Errorf("unable to read assertions verification keys file: %w", err) + return fmt.Errorf("unable to read assertions verification keys file: %w", err) } err = json.Unmarshal(assertionVerificationBytes, &assertionVerificationKeys) if err != nil { - return nil, fmt.Errorf("unable to unmarshal assertion verification keys json: %w", err) + return fmt.Errorf("unable to unmarshal assertion verification keys json: %w", err) } for assertionName, key := range assertionVerificationKeys.Keys { correctedKey, err := correctKeyType(key, true) if err != nil { - return nil, fmt.Errorf("error with assertion signing key: %w", err) + return fmt.Errorf("error with assertion signing key: %w", err) } assertionVerificationKeys.Keys[assertionName] = sdk.AssertionKey{Alg: key.Alg, Key: correctedKey} } opts = append(opts, sdk.WithAssertionVerificationKeys(assertionVerificationKeys)) } - r, err := h.sdk.LoadTDF(ec, opts...) + r, err := h.sdk.LoadTDF(in, opts...) if err != nil { - return nil, err + return err } //nolint:errorlint // callers intended to test error equality directly - if _, err = io.Copy(pt, r); err != nil && err != io.EOF { - return nil, formatDecryptError(ctx, r.Obligations, err) + if _, err = io.Copy(out, r); err != nil && err != io.EOF { + return formatDecryptError(ctx, r.Obligations, err) } case sdk.Invalid: - return nil, errors.New("invalid TDF") + return errors.New("invalid TDF") default: - return nil, errors.New("unknown TDF type") + return errors.New("unknown TDF type") } - return out, nil + return nil } -func (h Handler) InspectTDF(toInspect []byte) (TDFInspect, []error) { - b := bytes.NewReader(toInspect) - switch sdk.GetTdfType(b) { +// InspectTDF reads a TDF's manifest without materializing its payload. +func (h Handler) InspectTDF(in io.ReadSeeker) (TDFInspect, []error) { + switch sdk.GetTdfType(in) { case sdk.Standard: // grouping errors so we don't impact the piping of the data errs := []error{} - tdfreader, err := h.sdk.LoadTDF(bytes.NewReader(toInspect)) + // GetTdfType restores the offset it started from, but LoadTDF expects to + // begin at the head of the archive regardless of where the caller was. + if _, err := in.Seek(0, io.SeekStart); err != nil { + return TDFInspect{}, []error{errors.Join(ErrTDFInspectFailNotInspectable, err)} + } + + tdfreader, err := h.sdk.LoadTDF(in) if err != nil { if strings.Contains(err.Error(), "zip: not a valid zip file") { return TDFInspect{}, []error{ErrTDFInspectFailNotInspectable} diff --git a/spec/DSPX-4499.md b/spec/DSPX-4499.md new file mode 100644 index 0000000000..b8c16c23c2 --- /dev/null +++ b/spec/DSPX-4499.md @@ -0,0 +1,257 @@ +--- +ticket: DSPX-4499 +title: otdfctl streaming encrypt/decrypt +status: draft +authors: [dmihalcik@virtru.com] +branches: [opentdf/platform:DSPX-4499-streaming-codec] +prs: [] +depends-on: [opentdf/platform#3782, opentdf/platform#3865] +created: 2026-08-25 +updated: 2026-08-25 +--- + +# otdfctl: stream encrypt/decrypt instead of buffering the whole payload in memory + +## Summary + +`otdfctl encrypt` and `otdfctl decrypt` hold the entire plaintext *and* the entire +ciphertext in memory at once. Peak RSS scales at roughly 3.6x the payload, so a 1 GiB file +costs ~3.7 GiB of RAM and a large enough file simply OOMs on a machine that has plenty of +disk for it. The SDK already supports streaming; this is purely a CLI-layer choice. + +> The branch is named `DSPX-4499-streaming-codec`, but no codec abstraction is involved +> and none is being introduced. The name is vestigial. + +## Problem / Motivation + +Measured on a GitHub `ubuntu-latest` runner by the xtest paired benchmark harness +([tests run 32507318075](https://github.com/opentdf/tests/actions/runs/32507318075), go +SDK, `--bench-payloads 1GiB`), reading `ru_maxrss` from `os.wait4` on the forked CLI +process: + +| cell | median peak RSS | max | +|---|---|---| +| encrypt 1 MiB | 66 MiB | 79 MiB | +| encrypt 1 GiB | 3754 MiB | 5119 MiB | +| decrypt 1 GiB | 3808 MiB | 5025 MiB | + +Growth is essentially all payload-proportional: 66 MiB → 3754 MiB for 1023 MiB more input. + +### Root cause + +Line references are against `github.com/opentdf/platform/otdfctl` at `main` as vendored +into xtest on 2026-08-13, using `platform/sdk v0.29.0`. + +**Encrypt** + +1. `pkg/cli/pipe.go:40` — `ReadFromFile` does `io.ReadAll(fileToEncrypt)`, slurping the + whole file into a `[]byte`. (`cmd/tdf/encrypt.go` uses `utils.ReadBytesFromFile`, the + same pattern with a size cap.) +2. `pkg/handlers/tdf.go:52` — `EncryptBytes` creates `enc := bytes.NewBuffer(encrypted)` + and at line 97 calls `h.sdk.CreateTDF(enc, bytes.NewReader(unencrypted), opts...)`. The + entire TDF accumulates in a second in-memory buffer. +3. `cmd/tdf/encrypt.go:141` — only then does `io.Copy(dest, encrypted)` touch the disk. + +**Decrypt** + +1. `cmd/tdf/decrypt.go` — the TDF is slurped to `bytesToDecrypt`. +2. `pkg/handlers/tdf.go:114-116` — `out := &bytes.Buffer{}`, `ec := bytes.NewReader(toDecrypt)`; + `io.Copy(pt, r)` at line 153 fills `out` with the whole plaintext. +3. `cmd/tdf/decrypt.go:83` — for stdout, `fmt.Print(decrypted.String())` makes a *third* + full copy, since `Buffer.String()` allocates a new string. The `-o` path uses + `decrypted.Bytes()` and avoids that one. + +So the live set at peak is ~2 GiB minimum for a 1 GiB encrypt (two whole-payload copies), +plus the transient while each buffer's incremental growth copies old to new, plus Go's +default `GOGC=100` letting the heap reach ~2x live before collecting. RSS is a high-water +mark and Go returns pages lazily. ~3.7 GiB is what that arithmetic predicts, and it is what +we measure. + +### Why the SDK is not the blocker + +The relevant SDK signatures already take streams: + +```go +func (s SDK) CreateTDF(writer io.Writer, reader io.ReadSeeker, opts ...TDFOption) (*TDFObject, error) +func (s SDK) LoadTDF(reader io.ReadSeeker, opts ...TDFReaderOption) (*Reader, error) +func GetTdfType(reader io.ReadSeeker) TdfType +``` + +An `*os.File` satisfies `io.ReadSeeker`, so file-to-file streaming at roughly constant +memory is available today. `otdfctl` hands these APIs a `bytes.Reader` and a `bytes.Buffer` +instead. + +## Proposed Solution + +Pass the open `*os.File` straight through for the file-in/file-out case, and write to the +destination `*os.File` rather than to a `bytes.Buffer`. `EncryptBytes`/`DecryptBytes` grow +stream-shaped variants that take an `io.ReadSeeker`/`io.Reader` and an `io.Writer` instead +of taking `[]byte` and returning `*bytes.Buffer`. + +### Dependency: stack on #3865 + +This work stacks on [#3865](https://github.com/opentdf/platform/pull/3865) +(`feat/DSPX-2604-createtdf-chunked`), itself stacked on +[#3782](https://github.com/opentdf/platform/pull/3782) (`feat/DSPX-2604`, which adds +`ChunkedWriter`). #3865 rewrites `CreateTDFContext` on top of `ChunkedWriter` and relaxes +the signature: + +```go +func (s SDK) CreateTDF(writer io.Writer, reader io.Reader, opts ...TDFOption) (*TDFObject, error) +``` + +`resolveInputSize` probes for `io.Seeker` and treats a failed probe as an unmeasurable +payload rather than an error — "os.Stdin on the end of a pipe is the common case" — then +measures by reading. **Encrypt therefore needs no seekable input and no spooling.** + +Both parent PRs are open; this cannot merge until they land. + +### Decisions + +| Question | Decision | +|---|---| +| Base branch | Stack on `feat/DSPX-2604-createtdf-chunked` (#3865) | +| Piped stdin on decrypt | Spool to a temp file | +| `MaxFileSize` 10 GB CLI cap | Remove entirely | +| `inspect` | Convert in the same pass | + +## Inputs / Outputs / Contracts + +New handler surface in `otdfctl/pkg/handlers/tdf.go`. Options structs, because the existing +parameter lists are already long: + +```go +type EncryptOptions struct { + TDFType string + Attributes []string + MimeType string + KASURLPath string + Assertions string + WrappingKeyAlgorithm ocrypto.KeyType + TargetMode string + InputSize int64 // -1 when unknown +} + +// Encrypt streams plaintext from in to a TDF on out. Memory is bounded by the SDK's +// segment size, not by payload length. +func (h Handler) Encrypt(ctx context.Context, out io.Writer, in io.Reader, o EncryptOptions) error + +type DecryptOptions struct { + AssertionVerificationKeysFile string + DisableAssertionCheck bool + SessionKeyAlgorithm ocrypto.KeyType + KASAllowList []string + IgnoreAllowlist bool + FulfillableObligations []string +} + +func (h Handler) Decrypt(ctx context.Context, out io.Writer, in io.ReadSeeker, o DecryptOptions) error + +func (h Handler) InspectTDF(in io.ReadSeeker) (TDFInspect, []error) +``` + +These replace `EncryptBytes`, `DecryptBytes`, and the `[]byte` form of `InspectTDF`. The +only callers are `cmd/tdf/{encrypt,decrypt,inspect}.go`; there are no test callers. + +**No CLI flags change.** User-visible behavior is unchanged except as noted under Edge +Cases. Worth knowing for future work: otdfctl flags are declared in YAML frontmatter of +embedded markdown (`docs/man/*/_index.md`, read via `doc.GetDocFlag`), not in Go literals. + +## Edge Cases & Constraints + +- **Mime detection needs the head of the file.** `cmd/tdf/encrypt.go:94` calls + `mimetype.Detect(bytesSlice)` after `mimetype.SetLimit(Size1MB)`, so it only ever + consumes the first 1 MB — but it currently gets that from the fully-buffered slice. The + streaming version wraps input in `bufio.NewReaderSize(in, Size1MB+1)` and uses `Peek`, + which does not consume, so the bytes still reach the encoder. This works uniformly for + files and pipes and needs no seek. + +- **ZIP64 divergence when the length is unknown.** `newTDFChunkedWriter` forces + `zipstream.Zip64Always` when `inputSize == inputSizeUnknown`, because the ZIP64 choice is + baked into the payload's local file header that goes out with segment 0 and cannot be + revisited. Wrapping a file in a `bufio.Reader` hides its `Seeker`, which would silently + flip file encrypts to ZIP64 and produce byte-different output. **Mitigation:** for a + regular file, `os.Stat` it and pass `sdk.WithInputSize(fi.Size())`, which wins over the + reader probe and restores `Zip64Auto`. Piped input is legitimately ZIP64 — an accepted, + documented difference. + +- **stdin/stdout are not seekable, and decrypt has no streaming reader.** #3782 ships no + reader ("No reader is included") and `LoadTDF` remains `io.ReadSeeker` in #3865. Encrypt + can take a pipe directly; decrypt cannot. Piped decrypt input is spooled to a temp file, + read back seekably, and removed. This keeps memory bounded on every path at the cost of a + disk write and a writable `TMPDIR`. + +- **`io.Copy` must select `WriteTo`.** The SDK `Reader` implements `WriteTo` + (`sdk/tdf.go:973`), the O(one-segment) streaming decrypt path. Its `Read` + (`sdk/tdf.go:937`) delegates to `ReadAt` (`sdk/tdf.go:1050`), which accumulates every + segment into a `bytes.Buffer`. If a future refactor drops `io.WriterTo`, decrypt silently + re-regresses to buffering with no test failure. Guard with a compile-time assertion. + +- **Output file on error.** Writing directly to the destination means a failed encrypt + leaves a partial `.tdf` behind where today it leaves nothing. Write to a temp file in the + destination directory and rename on success; remove it on failure. Same-directory temp + keeps the rename atomic. For stdout there is no rename and a partial stream is + unavoidable, as with any streaming CLI. + +- **Ordering.** The `.tdf` extension fixup and `os.Create` at `cmd/tdf/encrypt.go:126-135` + happen *after* encryption today; they must move ahead of it. + +- **Empty-vs-absent stdin.** The current exactly-one-input check keys off + `len(piped) > 0`, which requires having read stdin. Replacing it with a + `os.ModeCharDevice` presence test alone would change behavior for + `otdfctl encrypt < /dev/null` (piped but empty). Use the presence test plus a + `bufio.Peek(1)` to preserve today's semantics without an unbounded read. + +- **File size cap.** `MaxFileSize` (`cmd/tdf/tdf.go:12`, 10 GB) exists to bound RAM. + With streaming it is an arbitrary cliff below the SDK's real 64 GiB + `maxFileSizeSupported` (`sdk/tdf.go:34`), which now enforces the limit. Remove it. + +## Out of Scope + +- Building a streaming/chunked **reader** in the SDK. That would let decrypt take a pipe + without spooling, but it is a large SDK change well beyond this ticket's CLI-layer scope. +- Removing the dead `--tdf-type`/`-t` flag on `decrypt` (`cmd/tdf/decrypt.go:106-111`), + documented as deprecated and never read by `decryptRun`. Noted, left alone. +- Any change to `nanotdf` — no nanotdf implementation exists in this repo. +- Flag or man-page changes; none are required. + +## Acceptance Criteria + +- [x] Peak RSS for `encrypt` and `decrypt` of a 1 GiB file is bounded by a small multiple + of the segment size, not of the payload size — target well under 500 MiB, versus + ~3.7 GiB today. +- [x] RSS stays roughly flat from 1 GiB to 4 GiB payloads. +- [ ] Existing xtest cross-SDK roundtrip matrix still passes; output TDFs are + byte-comparable with the buffered implementation for a fixed seed. +- [x] stdin/stdout paths keep working, including the documented + `echo "hello world" | otdfctl encrypt | otdfctl decrypt | cat` + (`docs/man/encrypt/_index.md:72`). +- [x] A failed encrypt with `-o` leaves neither a partial `.tdf` nor a stray temp file. +- [x] `make lint` passes with 0 new issues; `make test` passes. + +### Measured result + +Against a local platform (macOS, `/usr/bin/time -l`, peak RSS): + +| payload | encrypt | decrypt | +|---|---|---| +| 1 GiB | 73 MiB | 76 MiB | +| 4 GiB | 78 MiB | 74 MiB | + +Flat in payload size, roughly 50x below the ~3.7 GiB the buffered implementation used at +1 GiB, and both round-trips are byte-identical. A file input still produces a non-ZIP64 +archive; piped input is ZIP64, as anticipated in *Edge Cases & Constraints*. + +The xtest criterion is the one item that cannot be checked from a local run; it is +verified by the workflow linked from the PR. + +## Side effect worth knowing about + +The xtest benchmark harness gates on `rss`. At 1 GiB it currently reports ratio 1.00004 +with CI [0.9996, 1.0038] — the tightest, most confident-looking PASS in the run — because +both arms allocate the same two whole-payload buffers and that high-water mark swamps +everything else. Any per-segment allocation change (for example the ~2 MiB-per-segment +saving claimed by [#3865](https://github.com/opentdf/platform/pull/3865)) is invisible +under it. This is the mirror image of the RSS floor-censoring the harness already guards +against: a ceiling rather than a floor. Fixing the CLI makes the harness's RSS metric able +to see SDK allocation changes at all.