build: the tool is built with Go 1.27 now, and six formats change bytes

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ env:
   # The exact toolchain used for tests and releases. go.mod declares a
   # minimum - this is the pin. Raising it can change generated bytes, so the
   # byte stability guard has to be green before it moves.
-  GO_VERSION: "1.26.7"
+  GO_VERSION: "1.27.0"
 
 jobs:
   test:
@@ -324,7 +324,7 @@ jobs:
         # what makes it worth having: a scanner that lists every advisory
         # touching the module graph produces noise, and noise gets switched off.
         # Measured before switching it on, 2026-08-02: no vulnerabilities found.
-        run: go run golang.org/x/vuln/cmd/govulncheck@v1.1.4 ./...
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./...
 
   staticcheck:
     name: staticcheck
@@ -367,7 +367,7 @@ jobs:
         # both of them the word "Pillow" at the start of an error string, which
         # is the name of the library that refused the image rather than a
         # sentence. Zero findings with the config in place.
-        run: go run honnef.co/go/tools/cmd/staticcheck@v0.7.0 ./...
+        run: go run honnef.co/go/tools/cmd/staticcheck@v0.8.1 ./...
 
   lint:
     name: linters

.github/workflows/release.yml

@@ -29,7 +29,7 @@ concurrency:
 env:
   # The same pin CI carries. A release built on a different toolchain than the
   # one the byte stability guards ran under is a release nobody measured.
-  GO_VERSION: "1.26.7"
+  GO_VERSION: "1.27.0"
 
 jobs:
   check:

CHANGELOG.md

@@ -16,6 +16,40 @@ because it turns other people's test suites red.
 
 ### Breaking
 
+- **Six formats have different bytes, because the tool is built with Go 1.27
+  now.** Sizes are unchanged. Every size that worked before still works, every
+  reader that took these files still takes them, and the same sizes are
+  reachable. What changed is the compressed data inside them.
+
+  Affected: `targz`, `png`, `docx`, `xlsx`, `pptx`, `ico` when it holds a png,
+  and `zip` when you ask for compression. `zip` left alone is untouched,
+  because its default stores rather than compresses. The other seventeen
+  formats are byte for byte what they were.
+
+  Go 1.27 changed `compress/flate`, which is what all of those run through.
+  Below the default compression level the change is only how a stream is
+  closed, and above it the compressor itself behaves differently.
+
+  Two minimums moved with it. The smallest `png` is **74 B** rather than 73,
+  and sizes 75 to 82 and 85 are the ones it cannot produce. The smallest
+  `targz` is **1049 B** rather than 1052, the next size up is 1051, and 1050 is
+  the one it cannot produce. `tfg formats` prints the current numbers.
+
+  **A suite pinning hashes for those formats will go red once and then stay
+  green.** There is no switch back: staying on the old compiler was not a
+  choice this tool can offer, since the compiler comes from whoever builds it.
+
+- **`.tar.gz` could not be produced at all under Go 1.27 until this release.**
+  Every size was refused with an error saying the generator produced three
+  bytes fewer than planned. The size of a `.tar.gz` is worked out rather than
+  measured - compressing twice to learn a length would make a preview cost what
+  the run costs - and that arithmetic carried a number that turned out to
+  describe one release of Go.
+
+  It measures that number now, at first use, and checks its own answer before
+  trusting it. A later Go release can move these bytes again, but it can no
+  longer stop the format from being written.
+
 - **A generated `.tar.gz` has different bytes, because a lot of them could
   not be opened by a Go program.** Sizes are unchanged, every size that
   worked before still works, and every reader that took these files still

go.mod

@@ -28,7 +28,23 @@ go 1.26.5
 // here" and "green there" have to mean the same compiler. The byte stability
 // guards were run under 1.26.7 before this line moved and none of them
 // shifted, so D11 holds and no major version is owed.
-toolchain go1.26.7
+//
+// Raised to 1.27.0 on 2026-09-01, and this one IS owed a major version. Go
+// 1.27 changed compress/flate, so every format that puts bytes through deflate
+// produces different ones - eleven of the fifty one pinned cases moved, and all
+// seven of the pinned standard library paths. Sizes are unchanged and the same
+// sizes are reachable. Decision by the owner, and the reason was not that the
+// release is better: this machine builds other projects that are already on
+// 1.27, a toolchain setting belongs to the account rather than to a project, so
+// the two were taking it in turns. Staying meant a check before every command
+// forever. Written up in docs/GO-127-MIGRATION.md.
+//
+// TAR.GZ could not be produced at all under 1.27 until this move, because its
+// size arithmetic carried the gzip framing as a constant and the block that
+// closes a level zero stream went from five bytes to two. It measures the
+// framing now, so the next release moves the bytes again but does not stop the
+// format from being written.
+toolchain go1.27.0
 
 require github.com/goccy/go-yaml v1.19.2
 

internal/format/targz/compress.go

@@ -126,8 +126,34 @@ func nextFiller(target, got, filler int64) (next, extra int64, useExtra, done bo
 	case deficit < 0 || (deficit > 0 && deficit < 2):
 		// Overshot, or left a remainder the extra field cannot hold: it costs
 		// two bytes before it holds anything. Give the filler back enough that
-		// the field has room to work.
-		return filler - (2 - deficit), 0, false, false
+		// the field has room to work, and give back A WHOLE BLOCK MORE.
+		//
+		// The block is the point, and giving back only the overshoot is what
+		// used to happen and does not converge. The filler is a tar entry, and
+		// a tar entry is padded up to a whole 512 byte block - so handing back
+		// fewer than 512 bytes changes WHICH bytes the archive carries and not
+		// HOW MANY. What comes out the other side of gzip then wobbles by a
+		// byte or so either way, and the walk spends its rounds stepping five
+		// bytes at a time across a staircase, never landing.
+		//
+		// Measured 2026-09-01: a 256 KiB archive at compression best sat at
+		// 262 147 and 262 148 B for eight rounds while the filler came down
+		// from 258 481 to 258 447. This had been true all along and Go 1.27
+		// only moved which sizes land on a step edge, so it looked like the
+		// compiler broke it. Probe: tools/probes/targzsettle.
+		//
+		// Undershooting is safe and overshooting is not: the extra field adds
+		// exactly what it is given, up to 65 531 B, so a round that lands under
+		// the target finishes on the next pass. A whole block is well inside
+		// that.
+		next := filler - (2 - deficit) - tarBlock
+		if next < 0 && filler > 0 {
+			// Try with no filler at all before deciding the size is out of
+			// reach. Only an archive that overshoots with nothing in it is
+			// genuinely too small.
+			next = 0
+		}
+		return next, 0, false, false
 	case deficit == 0:
 		// Landed without needing the field at all.
 		return filler, 0, false, true

internal/format/targz/framing.go

@@ -0,0 +1,146 @@
+// What gzip costs on top of the bytes it carries, measured rather than
+// written down.
+//
+// This file exists because a number that was written down turned out to be a
+// fact about one Go release. The size of a TAR.GZ is arithmetic - it has to
+// be, because the bytes pass through deflate and building the archive to
+// measure it would make a preview cost what the run costs. The arithmetic
+// needs to know what the gzip stream adds, and until 2026-09-01 that was three
+// constants in targz.go.
+//
+// Go 1.27.0 changed one of them. The block that closes a level zero stream
+// went from a five byte empty STORED block to a two byte one, so every archive
+// came out three bytes short of its plan and the format refused to write
+// anything at all - measured on every size tried, from 64 kB to 10 MB. The
+// engine was right to refuse. The arithmetic was describing Go 1.26.
+//
+// Bumping the constant would have worked until the next release. Measuring
+// asks the library that is actually linked, so it holds for the one after that
+// too. The model is the same under both releases and only the constants move:
+//
+//	overhead(n) = base + perBlock * ceil(n / storeBlock)
+//
+// Measured 2026-09-01, level zero: perBlock is 5 under both, base is 23 under
+// go1.26.7 and 20 under go1.27.0.
+//
+// One honest limit, because it would otherwise look like this file is proven
+// and it is only half proven. Replacing the measurement with today's two
+// constants written down would pass every test in this repository, today, on
+// this compiler - the mutation runner was pointed at exactly that and it
+// cannot go red. What the measurement buys is the NEXT release, and no test
+// that runs today can demonstrate that. The mutations here cover the
+// arithmetic being wrong. They cannot cover it being right for the wrong
+// reason. That is why this comment is long: it is the only thing standing
+// between a later reader and a tidy simplification back to the bug.
+package targz
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+)
+
+// framing is what a level zero gzip stream costs beyond its content.
+type framing struct {
+	// base is the header, the trailer, and the block that closes the stream.
+	base int64
+	// perBlock is what each stored block of content costs.
+	perBlock int64
+}
+
+// The framing, measured once before anything can ask for it.
+//
+// A package level variable rather than a sync.Once, and the guard against
+// stray concurrency is what pointed that out. The first version reached for
+// sync.OnceValues to keep the measurement lazy, and lazy bought nothing here:
+// the registry works out this format's minimum during init, which asks for the
+// framing anyway, so every program was paying for it before main started
+// whichever way it was written. Go initialises a package variable exactly once
+// and before any goroutine exists, so there is nothing here for a lock to
+// protect.
+var framingValue, framingErr = measureFraming()
+
+// measuredFraming hands back what was measured, in the shape the callers want.
+func measuredFraming() (framing, error) { return framingValue, framingErr }
+
+// measureFraming works the two constants out from three compressions, and then
+// checks the model against a fourth.
+//
+// Two points settle the line and the third says whether it is a line at all.
+// Without that check a change to the BLOCK SIZE - rather than to the cost of a
+// block - would be read as a change to the constants, and the arithmetic would
+// be quietly wrong instead of loudly refused. That is the failure this whole
+// file exists to stop happening a second time, so it is worth one more
+// compression of a buffer that is already in memory.
+func measureFraming() (framing, error) {
+	one, err := storedOverhead(storeBlock)
+	if err != nil {
+		return framing{}, err
+	}
+	two, err := storedOverhead(2 * storeBlock)
+	if err != nil {
+		return framing{}, err
+	}
+
+	f := framing{perBlock: two - one}
+	f.base = one - f.perBlock
+
+	// Two independent checks the two points above cannot make on their own: a
+	// third multiple of the block, and the empty stream, which is base alone.
+	three, err := storedOverhead(3 * storeBlock)
+	if err != nil {
+		return framing{}, err
+	}
+	empty, err := storedOverhead(0)
+	if err != nil {
+		return framing{}, err
+	}
+	if want := f.base + 3*f.perBlock; three != want {
+		return framing{}, fmt.Errorf(
+			"targz: this build of Go frames a gzip stream in a shape this tool does not understand. "+
+				"Three blocks of content cost %d B where the two measured before them predict %d B, "+
+				"so the size of an archive cannot be worked out without building it",
+			three, want)
+	}
+	if empty != f.base {
+		return framing{}, fmt.Errorf(
+			"targz: this build of Go frames an empty gzip stream at %d B where the measurement says %d B, "+
+				"so the size of an archive cannot be worked out without building it",
+			empty, f.base)
+	}
+	return f, nil
+}
+
+// storedOverhead is what gzip adds to n bytes at compression level zero.
+//
+// The content is zeros, and that is safe precisely because the level is zero:
+// stored blocks carry their input unchanged, so the framing does not depend on
+// what is in them. At any other level it would.
+func storedOverhead(n int64) (int64, error) {
+	var out bytes.Buffer
+	w, err := gzip.NewWriterLevel(&out, gzip.NoCompression)
+	if err != nil {
+		return 0, err
+	}
+	if n > 0 {
+		if _, err := io.CopyN(w, zeros{}, n); err != nil {
+			return 0, err
+		}
+	}
+	if err := w.Close(); err != nil {
+		return 0, err
+	}
+	return int64(out.Len()) - n, nil
+}
+
+// zeros is an endless run of zero bytes, so the measurement allocates one
+// small buffer rather than the megabyte it reads.
+type zeros struct{}
+
+func (zeros) Read(p []byte) (int, error) {
+	for i := range p {
+		p[i] = 0
+	}
+	return len(p), nil
+}

internal/format/targz/size.go

@@ -79,12 +79,19 @@ func tarLength(m memo) int64 {
 }
 
 // gzipFixed is the whole file except the comment.
+//
+// The framing comes from framing.go, which asks the gzip that is linked rather
+// than reading a number written here. If that measurement refused, this uses a
+// zero framing and the sizes are nonsense - which is safe only because Plan
+// returns the refusal before anything is written or announced. Nothing acts on
+// a size worked out from a framing this tool did not understand.
 func gzipFixed(tarLen int64) int64 {
+	f, _ := measuredFraming()
 	blocks := tarLen / storeBlock
 	if tarLen%storeBlock != 0 {
 		blocks++
 	}
-	return gzipFraming + storeBlockCost*(blocks+1) + tarLen
+	return f.base + f.perBlock*blocks + tarLen
 }
 
 // commentCost is what a comment adds to the file: its bytes plus the zero that

internal/format/targz/targz.go

@@ -72,14 +72,16 @@ const (
 	// storeBlock is the largest run of bytes gzip emits as one stored block at
 	// compression level zero. It decides the framing overhead, so it decides
 	// the arithmetic in size.go.
+	//
+	// This one stays written down because it is a fact about DEFLATE - a
+	// stored block carries a sixteen bit length - rather than about a release
+	// of Go. What each block COSTS, and what the header, trailer and closing
+	// block cost, moved to framing.go and are measured, because those did turn
+	// out to be facts about a release. framing.go checks this number too: if a
+	// build ever framed at a different block size, the model would stop
+	// predicting a third block and the format refuses rather than guessing.
 	storeBlock = 65535
 
-	// gzipFraming is the fixed ten byte header plus the eight byte trailer.
-	gzipFraming = 18
-
-	// storeBlockCost is what each stored block costs on top of its content.
-	storeBlockCost = 5
-
 	writeChunk = 32 * 1024
 )
 
@@ -188,6 +190,13 @@ type memo struct {
 }
 
 func (generator) Plan(r format.Request) (format.Plan, error) {
+	// Before anything else, because every size below is worked out from this
+	// and a framing this tool does not understand has to be a refusal rather
+	// than an archive of the wrong length. See framing.go.
+	if _, err := measuredFraming(); err != nil {
+		return format.Plan{}, err
+	}
+
 	groups, err := archive.Groups("targz", r)
 	if err != nil {
 		return format.Plan{}, err