format: a log can be six shapes, and it knows what time it is

.gitignore

@@ -56,6 +56,12 @@ files.jsonl
 /dist/
 /build/
 
+# What the tool itself writes. The window proposes this directory by default,
+# so running the program from a checkout fills it with generated files - and
+# this is a file generator, so its own repository fills up faster than any
+# other would.
+/tfg-out/
+
 # ---------------------------------------------------------------------------
 # Test and coverage artifacts
 # ---------------------------------------------------------------------------

CHANGELOG.md

@@ -16,6 +16,19 @@ because it turns other people's test suites red.
 
 ### Breaking
 
+- **A generated log now advances through time, so its bytes are different.**
+  Every entry used to carry the same instant. Ten thousand requests all landing
+  at one moment is not a log anybody can test a time window, a rate alert or a
+  rotation against, and it was obvious the moment you looked at a file.
+
+  Entries are now one second apart by default, and `rate` sets how many arrive
+  a second. The bytes of every log change, so a suite pinning their hashes will
+  go red.
+
+  **The way back is `--set timestamps=fixed`**, or `timestamps: fixed` on a
+  target in a recipe. That holds the clock still and writes the same bytes this
+  tool wrote before, to the byte - there is a pinned hash proving it.
+
 - **A generated GIF now moves, so its bytes are different.** A GIF is the one
   picture format here that can hold more than one frame, and a still one told
   you nothing about how the system under test treats an animation - whether it
@@ -34,6 +47,30 @@ because it turns other people's test suites red.
 
 ### Added
 
+- **A log can now be six shapes rather than one, and seven settings shape it.**
+  `tfg generate --format log --set entry_format=nginx` writes an nginx access
+  log. The others are `apache-combined` (the default, and what this format has
+  always written), `apache-common`, `syslog`, `plain` and `json-lines`.
+
+  Every template was taken from a real file rather than from a specification
+  remembered: a real nginx and a real Apache, and rsyslog on a real machine. Two
+  of them would have been wrong otherwise. An nginx line carries one more
+  quoted field than "combined" does, and Apache's own default is `common`, with
+  no referrer and no agent at all.
+
+  The rest of the settings: `timestamps` and `rate` for the clock, `methods` for
+  which verbs appear, `status_mix` for which response codes, `ip_version` to put
+  IPv6 addresses in front of a reader that may not expect them, and
+  `line_ending` for a log written by a Windows service.
+
+  **A setting that could not do anything is refused rather than ignored.**
+  Asking for `methods` beside `entry_format=syslog` is an error naming both,
+  because a syslog line carries no request - and a setting that silently does
+  nothing is worse than one that is not offered.
+
+  Every shape still hits the size to the byte, and every line is still a whole
+  entry. `tfg formats log` lists all of it.
+
 - **JPEG XL, the twenty fourth format.** One frame, 8 bit, RGB.
   `tfg generate --format jxl --size 300kb` writes a JPEG XL picture in the
   container the format defines for it. `width`, `height` and `quality` can be

internal/format/logfile/address.go

@@ -0,0 +1,168 @@
+// Addresses, and the rule that keeps the way back exact.
+//
+// Every draw here is made in the same order and from the same ranges the
+// generator used before entry formats existed. That is not tidiness: with
+// timestamps=fixed and the settings left alone, the file has to come out byte
+// for byte as it did, and a single extra call to the generator would shift
+// every entry after it.
+//
+// Which is why pick does not draw when there is only one thing to choose. A
+// list of one is not a choice, and asking for one costs a number out of the
+// stream that the old code never spent.
+package logfile
+
+import (
+	// nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used
+	"math/rand/v2"
+	"strconv"
+)
+
+// pick chooses one of a list, without spending a draw on a list of one.
+func pick[T any](rng *rand.Rand, xs []T) T {
+	if len(xs) == 1 {
+		return xs[0]
+	}
+	return xs[rng.IntN(len(xs))]
+}
+
+const (
+	// v4Longest is 255.255.255.255 and v6Longest is eight groups of four hex
+	// digits with seven colons. Both are the worst case, which is what the
+	// minimum has to be built from.
+	v4Longest = 15
+	v6Longest = 39
+)
+
+// address is one client address, drawn but not yet written.
+//
+// A value rather than a string, because a log of any size is millions of
+// entries and one string per entry is a multiple of the file in garbage. The
+// resource guard measures exactly that.
+type address struct {
+	v6    bool
+	parts [8]uint32
+}
+
+func drawAddress(rng *rand.Rand, o options) address {
+	v6 := o.ipv6
+	if o.ipMixed {
+		// One draw, so the stream stays predictable, and it is only ever
+		// reached when the settings asked for a mixture.
+		v6 = rng.IntN(2) == 1
+	}
+	var a address
+	a.v6 = v6
+	if !v6 {
+		// The same ranges, in the same order, as before entry formats
+		// existed. Nothing here may change or the way back stops being exact.
+		a.parts[0] = uint32(10 + rng.IntN(240))
+		a.parts[1] = uint32(rng.IntN(256))
+		a.parts[2] = uint32(rng.IntN(256))
+		a.parts[3] = uint32(1 + rng.IntN(254))
+		return a
+	}
+	// Documentation range, so a generated log never names somebody's real
+	// network. Groups are written the way a reader sees them, with leading
+	// zeros suppressed, which is why the length varies.
+	a.parts[0], a.parts[1] = 0x2001, 0x0db8
+	for i := 2; i < 8; i++ {
+		a.parts[i] = uint32(rng.IntN(0x10000))
+	}
+	return a
+}
+
+// length is how many bytes this address takes when written.
+func (a address) length() int {
+	if !a.v6 {
+		return decDigits(a.parts[0]) + decDigits(a.parts[1]) +
+			decDigits(a.parts[2]) + decDigits(a.parts[3]) + 3
+	}
+	n := 7 // the colons
+	for _, p := range a.parts {
+		n += hexDigits(p)
+	}
+	return n
+}
+
+func (a address) append(dst []byte) []byte {
+	if a.v6 {
+		return a.appendV6(dst)
+	}
+	return a.appendV4(dst)
+}
+
+// No leading zeros. Padding octets to three digits made the line length trivial
+// to predict and produced addresses no real log contains - and a leading zero
+// is read as octal by some address parsers, where 069 is not even valid octal.
+func (a address) appendV4(dst []byte) []byte {
+	for i := 0; i < 4; i++ {
+		if i > 0 {
+			dst = append(dst, '.')
+		}
+		dst = strconv.AppendInt(dst, int64(a.parts[i]), 10)
+	}
+	return dst
+}
+
+// Groups without leading zeros, which is how every reader shows them and why
+// the length of one of these varies.
+func (a address) appendV6(dst []byte) []byte {
+	for i, p := range a.parts {
+		if i > 0 {
+			dst = append(dst, ':')
+		}
+		dst = strconv.AppendInt(dst, int64(p), 16)
+	}
+	return dst
+}
+
+// decDigits is how many characters a byte sized number takes.
+func decDigits(n uint32) int {
+	switch {
+	case n < 10:
+		return 1
+	case n < 100:
+		return 2
+	default:
+		return 3
+	}
+}
+
+// hexDigits is how many characters a sixteen bit group takes in hex, written
+// without leading zeros the way every reader shows it.
+func hexDigits(n uint32) int {
+	switch {
+	case n < 0x10:
+		return 1
+	case n < 0x100:
+		return 2
+	case n < 0x1000:
+		return 3
+	default:
+		return 4
+	}
+}
+
+// longestAddress is the worst case for the settings in force, which is what
+// the minimum entry has to leave room for.
+func longestAddress(o options) int {
+	if o.ipv6 || o.ipMixed {
+		return v6Longest
+	}
+	return v4Longest
+}
+
+func longestMethod(o options) int { return longest(o.methods) }
+func longestAgent() int           { return longest(agents) }
+func longestTag() int             { return longest(tags) }
+func longestLevel() int           { return longest(levels) }
+
+func longest(xs []string) int {
+	n := 0
+	for _, x := range xs {
+		if len(x) > n {
+			n = len(x)
+		}
+	}
+	return n
+}

internal/format/logfile/clock.go

@@ -0,0 +1,69 @@
+// The clock: what time each entry says it happened.
+//
+// Until 2026-08-31 every entry in every log this tool wrote carried the same
+// instant, because the timestamp was a constant. That is a fidelity defect
+// rather than a missing setting, and docs/BACKLOG.md said so: a log where ten
+// thousand requests happen at one instant cannot be used to test a time window
+// query, a rate alert, or anything that rotates.
+//
+// It advances now, and that moves the bytes of every log, so `timestamps=fixed`
+// is kept as the way back and reproduces the old file exactly. Same shape as
+// `frames=1` for the still GIF, and there is a pinned hash for it too.
+package logfile
+
+import "time"
+
+// epoch is where every log starts. A constant, because D11 promises the same
+// bytes from the same seed and time.Now() would promise the opposite.
+//
+// It is the instant the fixed timestamp used to carry, so a run with
+// timestamps=fixed writes the bytes this tool wrote before this file existed.
+var epoch = time.Date(2026, time.August, 1, 12, 0, 0, 0, time.UTC)
+
+const (
+	// apacheTime is the layout the Apache family and nginx write, and
+	// isoTime is what rsyslog and application logs write. Both are fixed
+	// width for every instant with a four digit year, which is what lets an
+	// entry's length be known before it is built.
+	//
+	// Measured on a real nginx and a real rsyslog on 2026-08-31 rather than
+	// recalled - see docs/MVP-FORMATS.md.
+	apacheTime = "02/Jan/2006:15:04:05 -0700"
+	isoTime    = "2006-01-02T15:04:05.000000-07:00"
+)
+
+// clock hands out the instant for each entry in turn.
+//
+// It counts, so it is one of the builders core.Record.Discard exists for: the
+// filler builds one entry past the end to measure it and throws it away, and
+// without a way back the file would skip a tick. csv counts rows the same way.
+type clock struct {
+	// step is how far the clock moves between entries. Zero holds it still,
+	// which is what timestamps=fixed asks for.
+	step time.Duration
+	at   time.Time
+}
+
+func newClock(o options) clock {
+	c := clock{at: epoch}
+	if o.advancing {
+		// Entries per second into a gap between entries. Integer division
+		// rounds towards zero, so a rate above one second per entry still
+		// moves - a step of nought would silently reproduce fixed.
+		c.step = time.Second / time.Duration(o.rate)
+		if c.step <= 0 {
+			c.step = time.Nanosecond
+		}
+	}
+	return c
+}
+
+// tick returns the instant for this entry and moves on.
+func (c *clock) tick() time.Time {
+	at := c.at
+	c.at = c.at.Add(c.step)
+	return at
+}
+
+// back undoes one tick, for the entry that was built only to measure it.
+func (c *clock) back() { c.at = c.at.Add(-c.step) }