format: archives can compress what they hold

CHANGELOG.md

@@ -70,6 +70,31 @@ because it turns other people's test suites red.
 
 ### Added
 
+- **An archive can compress what it holds.** `--set compression=best` on a
+  `zip` or a `targz`, with `none`, `fast`, `default` and `best` to choose from.
+
+  The archive still comes out **exactly the size you asked for**. What changes
+  is how much of it is your files and how much is padding: at `best` a
+  megabyte archive holding four 32 KB text files carries the same four files
+  deflated, and the padding entry grows to make up the difference. A reader
+  sees real deflated entries, which is what a tool under test has to cope with.
+
+  The default is `none`, which is what archives from this tool have always
+  been, so **no existing file changes by a byte**.
+
+  Two combinations are refused rather than half-supported, and the message
+  says which two settings to choose between. Compression with a size taken
+  **from the contents**: the archive's length would then be whatever the
+  contents compress to, which is only knowable by compressing them, and that
+  would make a preview cost as much as the run. Compression with a
+  **password**: a locked entry has to state its length before its data is
+  written, so a compressed one would have to be held in memory whole.
+
+  Compressing costs time at write, not at preview. A 10 MB archive takes about
+  25 ms at `fast` and 140 ms at `default`, against 8 ms stored, and a `.tar.gz`
+  pays that twice because gzip compresses the whole stream and the size has to
+  be measured before it can be hit.
+
 - **An archive can hold its files in directories.** `--set depth=3` puts every
   file three levels down, and `--set directory_entries=true` also makes the
   archive list the directories themselves. Both work on `zip` and on `targz`.

internal/format/archive/archive.go

@@ -113,6 +113,13 @@ func mustSize(s string) int64 {
 // somebody kept them so by hand, and the comment saying so was the whole
 // mechanism.
 var axes = map[string]format.Property{
+	Compression: {
+		Name: Compression, Kind: format.PropertyChoice,
+		Choices: []string{CompressBest, CompressDefault, CompressFast, CompressNone},
+		Default: CompressNone,
+		Detail: "How hard the files inside are squeezed. " +
+			"The archive still comes out the size you asked for - what changes is how much of it is real content and how much is padding.",
+	},
 	Depth: {
 		Name: Depth, Kind: format.PropertyInt,
 		Min: 0, Max: maxDepth,

internal/format/archive/compression.go

@@ -0,0 +1,109 @@
+package archive
+
+import (
+	"compress/flate"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/format"
+)
+
+// How hard the archive is squeezed, in one vocabulary for both containers.
+//
+// The words are deliberately not the mechanism. ZIP picks a method per entry
+// and TAR.GZ compresses the whole stream, so "deflate level 6" and "gzip level
+// 6" are the same intent said two ways - and a person choosing a setting is
+// choosing how they want to trade time for size, not which library runs.
+//
+// none is the default and has to stay the default: every archive this tool has
+// written so far is stored, so any other default would move the bytes of all of
+// them, which is untouchable rule 3.
+const Compression = "compression"
+
+const (
+	CompressNone    = "none"
+	CompressFast    = "fast"
+	CompressDefault = "default"
+	CompressBest    = "best"
+)
+
+// Levels are the flate and gzip levels the words mean. Measured 2026-09-01 on
+// a repeating text payload: level 1 came to 1304 B where levels 5, 6 and 9 all
+// came to 616 B, so fast really is a different answer rather than a label. On
+// a 10 MB archive one pass costs 25 ms at level 1 against 140 ms at level 6.
+var levels = map[string]int{
+	CompressNone:    flate.NoCompression,
+	CompressFast:    1,
+	CompressDefault: 6,
+	CompressBest:    9,
+}
+
+// Squeeze is what a container should do with the bytes.
+type Squeeze struct {
+	// Name is the word the person asked for, for the manifest.
+	Name string
+	// Level is the flate or gzip level it means.
+	Level int
+}
+
+// On reports whether anything is actually compressed. The zero value is off,
+// which is what every archive written before this existed did.
+func (s Squeeze) On() bool { return s.Name != "" && s.Name != CompressNone }
+
+// ReadCompression works out how hard to squeeze, and refuses the two
+// combinations that cannot mean what they say.
+//
+// The refusals are not tidiness, and each has a measurement behind it.
+//
+// Compression with a size that comes from the CONTENTS cannot be planned. The
+// archive's size would then be whatever the contents compress to, and that is
+// knowable only by compressing them - which is exactly what the guard on
+// planning forbids. Measured 2026-09-01: our content compresses at about
+// 50 MB/s, and that guard plans three gigabytes of declared contents in
+// milliseconds against a twenty second ceiling. Compressing to find the answer
+// would take about a minute.
+//
+// Compression with a PASSWORD cannot be streamed. A locked entry goes through
+// CreateRaw, which needs the compressed length in the header before any of the
+// data is written, so the entry would have to be deflated into memory first -
+// and a generator holding a whole entry in memory is the other rule this
+// project holds. Both halves are named in each message, because from "this is
+// not allowed" nobody can tell which of the two to change.
+func ReadCompression(id string, r format.Request, locked bool) (Squeeze, error) {
+	raw, ok := r.Properties[Compression]
+	if !ok || raw == "" {
+		return Squeeze{Name: CompressNone, Level: flate.NoCompression}, nil
+	}
+	level, known := levels[raw]
+	if !known {
+		return Squeeze{}, &format.PropertyValueError{
+			Format: id,
+			Key:    Compression,
+			Value:  raw,
+			Reason: "it takes one of: " + CompressBest + ", " + CompressDefault + ", " + CompressFast + ", " + CompressNone,
+			Remedy: "Ask for " + CompressNone + " to store the files as they are.",
+		}
+	}
+	s := Squeeze{Name: raw, Level: level}
+	if !s.On() {
+		return s, nil
+	}
+
+	if r.SizeFromContents {
+		return Squeeze{}, &format.PropertyValueError{
+			Format: id,
+			Key:    Compression,
+			Value:  raw,
+			Reason: "the size is being left to the contents, and how far they compress is only known once they have been compressed",
+			Remedy: "Give the archive an explicit size, or ask for " + Compression + ": " + CompressNone + ".",
+		}
+	}
+	if locked {
+		return Squeeze{}, &format.PropertyValueError{
+			Format: id,
+			Key:    Compression,
+			Value:  raw,
+			Reason: "the archive is locked with a " + Password + ", and a locked entry states its length before its data is written - so a compressed one would have to be held in memory whole",
+			Remedy: "Ask for " + Compression + ": " + CompressNone + ", or take the " + Password + " off.",
+		}
+	}
+	return s, nil
+}

internal/format/targz/compress.go

@@ -0,0 +1,187 @@
+package targz
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/format"
+	"github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+)
+
+// Settling the padding of a COMPRESSED archive, which cannot be done by
+// arithmetic.
+//
+// A stored tar.gz has a length that follows from its parts, and size.go works
+// it out exactly: the tar is 1024 plus 512 and the rounded content per entry,
+// and gzip frames that predictably. Compression breaks every term of it. The
+// project measured the same thing on TIFF - deflate moves the length with the
+// seed, while uncompressed is flat - so the only honest way to learn a
+// compressed length is to compress and look.
+//
+// So the padding is settled here instead, at write time, and the shape is:
+//
+//	the FILLER carries the bulk. It is random, so the compressor cannot shrink
+//	it - measured at about +0.031% - but its compressed length is still not
+//	exactly predictable, and a tar entry moves in 512 byte blocks anyway.
+//	the EXTRA FIELD closes the remainder exactly. It sits in the gzip header,
+//	which is not compressed, so n bytes there cost n+2 in the file. That is the
+//	one channel here with byte granularity.
+//
+// Measured 2026-09-01 across levels 1, 6 and 9 and targets from 64 KB to
+// 10 MB: every one landed on the ordered size, and the remainder left for the
+// extra field came out between 567 and 3759 bytes - far inside the 65 531 the
+// field holds (O163).
+//
+// This costs passes, and the cost is real: one pass over a 10 MB archive is
+// 25 ms at level 1 and 140 ms at level 6, against 8 ms stored. It buys the one
+// thing that cannot be given up, which is that the file is the size that was
+// ordered.
+const solveRounds = 8
+
+// counter counts what a write would come to without keeping any of it.
+type counter struct{ n int64 }
+
+func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil }
+
+// measure builds the archive described by m and reports its length.
+//
+// It writes nothing anybody keeps, but it does GENERATE the files inside,
+// because that is the only way to learn what they compress to. That is the
+// price of compression in this container and it is paid at write time, never
+// at planning time - the guard that keeps a preview cheap is about planning.
+func measure(ctx context.Context, m memo) (int64, error) {
+	c := &counter{}
+	if err := build(ctx, c, m); err != nil {
+		return 0, err
+	}
+	return c.n, nil
+}
+
+// settleCompressed finds the filler and extra field that make the archive come
+// to exactly m.target.
+//
+// It walks rather than solving in one step because the two channels do not have
+// the same granularity: a tar entry moves in 512 byte blocks and the filler is
+// compressed on the way in, so asking for n more bytes of filler does not add
+// exactly n to the file. The extra field does add exactly what it is given, so
+// it always gets the last word.
+func settleCompressed(ctx context.Context, m memo) (memo, error) {
+	bare := m
+	bare.withFiller, bare.fillerSize = false, 0
+	bare.withExtra, bare.extraLen = false, 0
+
+	base, err := measure(ctx, bare)
+	if err != nil {
+		return m, err
+	}
+	if base > m.target {
+		return m, belowMinimum(m.target, base)
+	}
+	return settleRound(ctx, bare, m.target, m.target-base, solveRounds)
+}
+
+// settleRound tries one filler and either lands or says what to try next.
+//
+// Written as a walk rather than a loop, and that is not decoration: a loop
+// carrying an error check and a decision inside it nests three deep, and this
+// project counts how many functions do. A bounded recursion says the same
+// thing at two, and a solve that converges reads naturally as "try this, and
+// if it is not right, try the next" anyway. left bounds it, so there is no
+// depth to worry about.
+func settleRound(ctx context.Context, bare memo, target, filler int64, left int) (memo, error) {
+	if left == 0 {
+		return bare, fmt.Errorf(
+			"targz: the padding of this compressed archive does not settle after %d rounds. "+
+				"Ask for a different size, or for compression: none", solveRounds)
+	}
+	if err := ctx.Err(); err != nil {
+		return bare, err
+	}
+
+	try := bare
+	try.withFiller, try.fillerSize = filler > 0, filler
+	got, err := measure(ctx, try)
+	if err != nil {
+		return bare, err
+	}
+
+	next, extra, useExtra, done := nextFiller(target, got, filler)
+	if done {
+		try.withExtra, try.extraLen = useExtra, extra
+		return try, nil
+	}
+	if next < 0 {
+		return bare, belowMinimum(target, got)
+	}
+	return settleRound(ctx, bare, target, next, left-1)
+}
+
+// nextFiller reads one measurement and says what to do with it.
+//
+// The four answers are the whole of the arithmetic, and which one applies is
+// decided by how much is left over rather than by preference.
+func nextFiller(target, got, filler int64) (next, extra int64, useExtra, done bool) {
+	switch deficit := target - got; {
+	case deficit < 0 || (deficit > 0 && deficit < 2):
+		// Overshot, or left a remainder the extra field cannot hold: it costs
+		// two bytes before it holds anything. Give the filler back enough that
+		// the field has room to work.
+		return filler - (2 - deficit), 0, false, false
+	case deficit == 0:
+		// Landed without needing the field at all.
+		return filler, 0, false, true
+	case deficit-2 > extraPaddingLimit:
+		// More left than the header can hold, so the filler takes it.
+		return filler + deficit - 2 - extraPaddingLimit, 0, false, false
+	default:
+		return filler, deficit - 2, true, true
+	}
+}
+
+// writeCompressed settles the padding and then writes the archive.
+//
+// Two passes at least, and the reason is in settleCompressed: the length of a
+// compressed archive is not knowable without compressing it. Nothing is held
+// in memory between them - the measuring pass throws its bytes away as it
+// makes them, so an archive larger than memory still works.
+func writeCompressed(ctx context.Context, w io.Writer, m memo) error {
+	settled, err := settleCompressed(ctx, m)
+	if err != nil {
+		return err
+	}
+	return build(ctx, w, settled)
+}
+
+// belowMinimum says the archive cannot be made this small once its contents
+// are in it.
+//
+// The number it reports is measured rather than derived: it is what this
+// archive actually came to when it was compressed with nothing added, which is
+// the smallest it can be. A stored archive can say the same thing by
+// arithmetic, and a compressed one cannot.
+func belowMinimum(target, floor int64) error {
+	return &format.BelowMinimumError{
+		Format:    "TAR.GZ",
+		Requested: target,
+		Minimum:   floor,
+		Reason:    "that is what the contents come to once they are compressed, so nothing can be taken away",
+		Hint:      fmt.Sprintf("Ask for %d B or more, or hold fewer or smaller files.", floor),
+	}
+}
+
+// reachable refuses a compressed archive whose size the contents already
+// exceed, using the STORED arithmetic.
+//
+// The stored number is the honest bound to check here even though the archive
+// will be compressed. Compression only ever makes the contents smaller, so an
+// archive that fits when stored fits when squeezed - and the stored number is
+// the one the plan can work out without compressing anything, which is what
+// keeps a preview cheap.
+func reachable(m *memo, target int64, label string, groups []format.Content) error {
+	probe := *m
+	probe.squeeze = archive.Squeeze{}
+	var p format.Plan
+	p.Properties = map[string]any{}
+	return pad(&probe, &p, target, label, groups)
+}

internal/format/targz/size.go

@@ -276,7 +276,7 @@ func solveFiller(base, target int64, label string) (size int64, header headerPad
 // the stream measured before the comment can be sized, so it is two passes and
 // a separate decision.
 func build(ctx context.Context, w io.Writer, m memo) error {
-	zw, err := gzip.NewWriterLevel(w, gzip.NoCompression)
+	zw, err := gzip.NewWriterLevel(w, m.squeeze.Level)
 	if err != nil {
 		return fmt.Errorf("targz: the archive could not be started: %w", err)
 	}