Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,23 @@ because it turns other people's test suites red.

### Added

- **A log can be made quiet, or full of errors.** `--set level_mix=errors` on
a `log`, with `realistic`, `quiet`, `errors` and `debug` to choose from. It
decides which severities appear, the way `status_mix` already decides which
response codes do.

Only the `plain` and `json-lines` entry formats carry a severity at all. Ask
for a mix beside one of the other four and the tool says so and stops,
naming both settings, rather than accepting a setting that would do nothing.

One thing worth knowing before you pick `quiet`: it draws only `INFO`, which
is a shorter word than `ERROR`, so the smallest log it can write is one byte
smaller than the other mixes. The tool tells you the floor for the settings
you gave it.

The default is `realistic`, the mix these logs have always had, so **no
existing file changes by a byte**.

- **An archive can compress what it holds.** `--set compression=best` on a
`zip` or a `targz`, with `none`, `fast`, `default` and `best` to choose from.

Expand Down
6 changes: 5 additions & 1 deletion internal/format/logfile/address.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,10 +152,14 @@ func longestAddress(o options) int {
return v4Longest
}

// longestLevel reads the SET IN FORCE rather than every level this format
// knows, because that is what the minimum entry has to leave room for. Reading
// the whole vocabulary instead would be wrong in the safe direction - a floor
// one byte too high - but it would announce a minimum the format can beat.
func longestLevel(o options) int { return longest(o.levels) }
func longestMethod(o options) int { return longest(o.methods) }
func longestAgent() int { return longest(agents) }
func longestTag() int { return longest(tags) }
func longestLevel() int { return longest(levels) }

func longest(xs []string) int {
n := 0
Expand Down
42 changes: 41 additions & 1 deletion internal/format/logfile/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ func properties() []format.Property {
Choices: []string{"realistic", "success", "client-errors", "server-errors"}, Default: "realistic",
Detail: "Which response codes appear. Realistic is mostly success with a tail of errors.",
},
{
Name: "level_mix", Kind: format.PropertyChoice,
Choices: levelMixIDs, Default: "realistic",
Detail: "Which severities appear. Only the plain and json-lines shapes carry one, and asking for it beside another shape is refused.",
},
{
Name: "ip_version", Kind: format.PropertyChoice,
Choices: []string{"v4", "v6", "mixed"}, Default: "v4",
Expand Down Expand Up @@ -169,6 +174,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
if opt.shape.web || opt.shape.id == "json-lines" {
p.Properties["status_mix"] = opt.statusMix
}
if opt.shape.levelled {
p.Properties["level_mix"] = opt.levelMix
}

m := memo{seed: r.Seed, opt: opt}
if r.Label {
Expand Down Expand Up @@ -277,4 +285,36 @@ const (

var tags = []string{"sshd", "cron", "systemd", "kernel", "nginx", "dockerd"}

var levels = []string{"INFO", "INFO", "INFO", "WARN", "ERROR", "DEBUG"}
// levelSets are the severity mixes, drawn from one vocabulary on purpose.
//
// Every set here is built from DEBUG, INFO, WARN and ERROR and nothing else,
// so the longest level in any of them is five bytes. That is not tidiness: the
// shortest entry a shape can write leaves room for the longest level it might
// draw, so a set carrying a longer word would move the minimum for plain and
// JSON lines. Adding CRITICAL later is allowed, it just has to be a decision
// about the minimum rather than a word slipped into a list.
//
// Repeats are the weighting. There is no separate share for each level because
// a list with three INFOs in it says the same thing and is what the draw
// already reads.
var levelSets = map[string][]string{
// realistic is the mix this format has always written, in the order it has
// always been in. D11 rests on that: the same seed has to draw the same
// levels it did before this setting existed, so this slice is the old
// variable moved rather than rewritten.
"realistic": {"INFO", "INFO", "INFO", "WARN", "ERROR", "DEBUG"},
// quiet is the boring baseline - a service with nothing to report. Useful
// as the control when a reader is being tested for what it does with the
// levels rather than for whether it parses.
"quiet": {"INFO"},
// errors is a service having a bad day, for a reader whose error handling
// is what is under test.
"errors": {"ERROR", "ERROR", "ERROR", "WARN", "WARN", "INFO"},
// debug is a build left verbose, which is how a log gets large in the first
// place.
"debug": {"DEBUG", "DEBUG", "DEBUG", "INFO", "INFO", "WARN"},
}

// levelMixIDs is the closed set the registry offers, in one order so that
// every surface lists them the same way.
var levelMixIDs = []string{"realistic", "quiet", "errors", "debug"}
35 changes: 34 additions & 1 deletion internal/format/logfile/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ type options struct {
statusMix string
statuses []int

levelMix string
levels []string

ipVersion string
ipv6 bool
ipMixed bool
Expand All @@ -63,6 +66,8 @@ func defaultOptions() options {
methods: methodSets["get"],
statusMix: "realistic",
statuses: statusSets["realistic"],
levelMix: "realistic",
levels: levelSets["realistic"],
ipVersion: "v4",
}
return o
Expand Down Expand Up @@ -94,7 +99,7 @@ func parseOptions(props map[string]string) (options, error) {
o := defaultOptions()
for _, read := range []func(map[string]string, *options) error{
readShape, readLineEnding, readTimestamps, readRate,
readMethods, readStatusMix, readIPVersion,
readMethods, readStatusMix, readLevelMix, readIPVersion,
} {
if err := read(props, &o); err != nil {
return options{}, err
Expand Down Expand Up @@ -205,6 +210,34 @@ func readStatusMix(props map[string]string, o *options) error {
return nil
}

// readLevelMix reads the severity mix, which only two shapes carry.
//
// Unlike status_mix, the set chosen here can move the MINIMUM: the shortest
// entry a shape can write has to leave room for the longest level it might
// draw, and quiet draws only INFO. That is why the set is settled here and
// read back out of options by longestLevel, rather than either of them
// reaching for the vocabulary directly.
func readLevelMix(props map[string]string, o *options) error {
v, ok := value(props, "level_mix")
if !ok {
return nil
}
set, known := levelSets[v]
if !known {
return badValue("level_mix", v, "it has to be realistic, quiet, errors or debug")
}
// Only a real choice can disagree with the shape - see asked. A window
// sends this key on every run, so refusing whenever it arrived would put
// the four shapes without a level out of reach from the window entirely,
// which is the defect reported from a screenshot on 2026-08-31.
if _, chosen := asked(props, "level_mix", "realistic"); chosen && !o.shape.levelled {
return conflict("level_mix and entry_format", v,
"the "+o.shape.id+" shape carries no severity")
}
o.levelMix, o.levels = v, set
return nil
}

func readIPVersion(props map[string]string, o *options) error {
v, ok := value(props, "ip_version")
if !ok {
Expand Down
17 changes: 11 additions & 6 deletions internal/format/logfile/shapes.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ type shape struct {
// if the method, status and address settings mean anything for it. A
// setting that would do nothing is refused rather than ignored.
web bool
// levelled says whether this shape carries a severity, which decides the
// same thing for level_mix. Separate from web rather than derived from it,
// because the two do not line up: no web shape carries a level, and of the
// three that are not web, only two do.
levelled bool
// appendTo writes one entry. want below zero means whatever length it
// comes out. Any other value is the exact length the line must have, its
// terminator included, and the stretchable field reaches it.
Expand Down Expand Up @@ -65,8 +70,8 @@ var shapes = map[string]*shape{
"apache-combined": {id: "apache-combined", web: true, appendTo: appendApacheCombined, shortest: shortestApacheCombined, label: hashLabel},
"nginx": {id: "nginx", web: true, appendTo: appendNginx, shortest: shortestNginx, label: hashLabel},
"syslog": {id: "syslog", web: false, appendTo: appendSyslog, shortest: shortestSyslog, label: hashLabel},
"plain": {id: "plain", web: false, appendTo: appendPlain, shortest: shortestPlain, label: hashLabel},
"json-lines": {id: "json-lines", web: false, appendTo: appendJSONLine, shortest: shortestJSONLine, label: jsonLabel},
"plain": {id: "plain", web: false, levelled: true, appendTo: appendPlain, shortest: shortestPlain, label: hashLabel},
"json-lines": {id: "json-lines", web: false, levelled: true, appendTo: appendJSONLine, shortest: shortestJSONLine, label: jsonLabel},
}

// shapeIDs is the closed set the registry offers, in one order so that every
Expand Down Expand Up @@ -253,7 +258,7 @@ func pidWidth(pid int) int {
// most home grown loggers write and the one with the least agreement about it,
// so this is the plainest reading of it.
func appendPlain(dst []byte, st *state, want int64) []byte {
level := pick(st.rng, levels)
level := pick(st.rng, st.opt.levels)
at := st.clock.tick()

base := int64(len(isoTime)+1+len(level)+1) + int64(len(st.opt.eol))
Expand All @@ -266,7 +271,7 @@ func appendPlain(dst []byte, st *state, want int64) []byte {
}

func shortestPlain(o options) int64 {
return int64(len(isoTime)+1+longestLevel()+1) + int64(len(o.eol)) + 1
return int64(len(isoTime)+1+longestLevel(o)+1) + int64(len(o.eol)) + 1
}

// JSON lines: one object a line, which is what makes it the one shape here
Expand All @@ -277,7 +282,7 @@ func shortestPlain(o options) int64 {
// nothing in it ever needs escaping - which matters, because an escape would
// make the line longer than the arithmetic said.
func appendJSONLine(dst []byte, st *state, want int64) []byte {
level := pick(st.rng, levels)
level := pick(st.rng, st.opt.levels)
status := pick(st.rng, st.opt.statuses)
at := st.clock.tick()

Expand All @@ -302,7 +307,7 @@ func appendJSONLine(dst []byte, st *state, want int64) []byte {

func shortestJSONLine(o options) int64 {
return int64(len(`{"time":"","level":"","status":,"msg":""}`)+
len(isoTime)+longestLevel()+statusWidth) + int64(len(o.eol)) + 1
len(isoTime)+longestLevel(o)+statusWidth) + int64(len(o.eol)) + 1
}

// appendMessage writes the sentence the message shapes end with, stretched to
Expand Down
Loading
Loading