diff --git a/CHANGELOG.md b/CHANGELOG.md index 2657860..7e25ccc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -150,6 +150,26 @@ because it turns other people's test suites red. ### Added +- **`csv` takes `columns`, so a table can be as narrow or as wide as the thing + you are testing.** Two to 32768, and six by default - which is the six + columns this tool has always written, so a `.csv` you already generate does + not change by a byte. + + Fewer than six drops them from the middle: `id` stays first and + `description` stays last, because that is the field stretched to reach the + exact size you asked for. More than six adds `field_7`, `field_8` and so on + in front of the description. + + **Above 16384 columns a spreadsheet quietly keeps the first 16384 and drops + the rest.** Measured with LibreOffice Calc: 16384 comes back whole, 16385 + comes back with one column missing and no warning anywhere. The ceiling here + is deliberately past that, so you can build the set either side of the line + rather than only the last file that survives it. + + The smallest file moves with the setting, the way it already does for row + endings and quoting - 36 B at two columns, 115 B at six, 5017 B at 256. + `tfg formats csv` prints what a given table needs. + - **A `targz` manifest says what its entries claim about themselves.** Two new keys on the file entry, `entry_mode` and `entry_owner`, written every run rather than only when you ask for them, so a harness never has to read a diff --git a/README.md b/README.md index f591081..fe1c8bc 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ recipe. `tfg formats ` prints the allowed range or list for each: | `docx` | `paragraphs` | | `xlsx` | `rows`, `columns` | | `pptx` | `slides` | -| `csv` | `delimiter`, `line_ending`, `header`, `quote_style` | +| `csv` | `delimiter`, `line_ending`, `header`, `quote_style`, `columns` | | `log` | `entry_format`, `timestamps`, `rate`, `methods`, `status_mix`, `level_mix`, `ip_version`, `line_ending` | | `json`, `xml`, `html`, `md`, `txt`, `svg` | none | diff --git a/internal/format/csvfile/csv.go b/internal/format/csvfile/csv.go index 3d95a2a..067d895 100644 --- a/internal/format/csvfile/csv.go +++ b/internal/format/csvfile/csv.go @@ -53,25 +53,51 @@ const ( // Bounding it rather than guessing is what lets the closing row reach its // length whatever row number it lands on. maxRowDigits = 19 - - // fixedBeforeEnding is every byte of a row except the row number, the name - // (which also forms the address), the description, the quotes and the row - // ending. A constant expression, so it cannot drift away from the template - // above. - // - // The five separators count one byte each, which is a fact about the - // separators offered rather than an assumption: every one of them is a - // single byte, and dialect.go says so where they are declared. - fixedBeforeEnding = 5 /* separators */ + len(emailDomain) + amountWidth + - len(createdDate) ) -// fixedWidth is fixedBeforeEnding plus the quotes and the row ending, both of -// which the dialect decides. A CRLF row costs one byte more than an LF one on -// every row, and quoting every field costs two bytes per column, which is why -// the minimum moves with either setting. +// widestColumn is the most bytes the column at this position can take, for any +// draw. The description is not here: it is always last and is empty in the row +// the floor is made of. +// +// Measured rather than guessed only in the sense that every number in it is +// read off the template above. The positions match columnNamesFor, and a +// column past the ones this format started with holds a word. +func widestColumn(at int) int { + switch at { + case 0: + return maxRowDigits + case 1: + return longestWord + case 2: + return longestWord + len(emailDomain) + case 3: + return amountWidth + case 4: + return len(createdDate) + default: + return longestWord + } +} + +// fixedWidth is the whole row except the description: every leading column at +// its widest, the separators between all of them, the quotes and the row +// ending. +// +// Three settings move it. A CRLF row costs one byte more than an LF one on +// every row, quoting every field costs two bytes a column, and the number of +// columns moves every term at once - which is why the minimum moves with any +// of the three. +// +// The separators count one byte each, which is a fact about the separators +// offered rather than an assumption: every one of them is a single byte, and +// dialect.go says so where they are declared. func fixedWidth(d dialect) int64 { - return int64(fixedBeforeEnding + d.quotes.quoteBytes() + len(d.eol)) + total := int64(d.columns-1) /* separators */ + int64(len(d.eol)) + + int64(d.quotes.quoteBytes(d.columns)) + for at := 0; at < d.columns-1; at++ { + total += int64(widestColumn(at)) + } + return total } func init() { @@ -150,7 +176,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { "separator": string(d.sep), "header": d.header, "quote_style": d.quotes.id, - "columns": len(columnNames), + "columns": d.columns, // Stated even though it is always false here, so a test can assert // on it without knowing which formats carry a label internally. format.PropertyLabelEmbedded: false, @@ -202,10 +228,10 @@ type rows struct { } // Shortest is the smallest row this builder can close a file with: the widest -// row number, the longest name in both the name and the address, and an empty -// description. It has to hold for every draw rather than for the lucky one. +// row number, the longest word wherever a word goes, and an empty description. +// It has to hold for every draw rather than for the lucky one. func (r *rows) Shortest() int64 { - return int64(maxRowDigits+2*longestWord) + fixedWidth(r.dia) + return fixedWidth(r.dia) } func (r *rows) Append(dst []byte, rng *rand.Rand) []byte { @@ -244,36 +270,51 @@ func (r *rows) append(dst []byte, rng *rand.Rand, want int64) []byte { sep := r.dia.sep q := r.dia.quotes - dst = q.mark(dst) - dst = strconv.AppendInt(dst, r.next, 10) - dst = q.mark(dst) - dst = append(dst, sep) - dst = q.mark(dst) - dst = append(dst, name...) - dst = q.mark(dst) - dst = append(dst, sep) - dst = q.mark(dst) - dst = append(dst, name...) - dst = append(dst, emailDomain...) - dst = q.mark(dst) - dst = append(dst, sep) - dst = q.mark(dst) - dst = strconv.AppendInt(dst, int64(whole), 10) - dst = append(dst, '.') - if cents < 10 { - dst = append(dst, '0') + // Every column but the description, in the order columnNamesFor names + // them. The draws above happen once and outside this loop, which is what + // keeps a six column table byte for byte what it always was: a narrower + // table draws exactly the same values and writes fewer of them, and a wider + // one draws its extra words only when it reaches them. + for at := 0; at < r.dia.columns-1; at++ { + dst = q.mark(dst) + dst = r.appendValue(dst, rng, at, name, whole, cents) + dst = q.mark(dst) + dst = append(dst, sep) } - dst = strconv.AppendInt(dst, int64(cents), 10) - dst = q.mark(dst) - dst = append(dst, sep) - dst = q.mark(dst) - dst = append(dst, createdDate...) - dst = q.mark(dst) - dst = append(dst, sep) return r.appendDescription(dst, rng, want, int64(len(dst)-start)) } +// appendValue writes the value of the column at this position. +// +// The name is drawn once by the caller and used twice, in the name column and +// inside the address, which is how this table has always read. The positions +// match widestColumn above, and the two are the pair that has to stay in step - +// a value wider than its column would make the closing row overshoot a length +// it was handed. +func (r *rows) appendValue(dst []byte, rng *rand.Rand, at int, name string, whole, cents int) []byte { + switch at { + case 0: + return strconv.AppendInt(dst, r.next, 10) + case 1: + return append(dst, name...) + case 2: + dst = append(dst, name...) + return append(dst, emailDomain...) + case 3: + dst = strconv.AppendInt(dst, int64(whole), 10) + dst = append(dst, '.') + if cents < 10 { + dst = append(dst, '0') + } + return strconv.AppendInt(dst, int64(cents), 10) + case 4: + return append(dst, createdDate...) + default: + return append(dst, words[rng.IntN(len(words))]...) + } +} + // appendDescription writes the last field and ends the row. // // want below zero means a natural row, any other value is the exact length the diff --git a/internal/format/csvfile/dialect.go b/internal/format/csvfile/dialect.go index 362aa84..8f1105e 100644 --- a/internal/format/csvfile/dialect.go +++ b/internal/format/csvfile/dialect.go @@ -15,6 +15,8 @@ package csvfile import ( "bytes" + "fmt" + "strconv" "strings" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -26,6 +28,37 @@ const ( LineEnding = "line_ending" Header = "header" QuoteStyle = "quote_style" + Columns = "columns" +) + +const ( + // minColumns is two, and it is a decision rather than a limit of the + // writer. A table of one column has no separator in it anywhere, so a file + // written with the wrong one is byte for byte a file written with the right + // one - our own structural check refuses such a file for exactly that + // reason, measured 2026-09-03. It would also leave the delimiter setting + // doing nothing at all, which this project treats as a refusal rather than + // a silence. And a single column of values is a txt file, which this tool + // already writes. + minColumns = 2 + + // defaultColumns is the six this format has always had. Leaving it alone + // produces the same bytes it always did. + defaultColumns = 6 + + // maxColumns is deliberately ABOVE the width of a spreadsheet, and that is + // the whole reason for the number. + // + // Measured 2026-09-03 with LibreOffice Calc 26.2.5.2 headless: a table of + // 16384 columns comes back whole, and one of 16385 comes back with 16384 - + // the last column is dropped without a word. 32768 columns came back as + // 16384 too, in three seconds, so the clamp is quiet and cheap rather than + // an error anybody would see. + // + // A tester needs to stand on BOTH sides of that line, because building the + // set around a boundary is what this tool is for. Stopping at 16384 would + // offer the last table that survives and not the first that does not. + maxColumns = 32768 ) // delimiters are the separators offered, by name rather than by character. @@ -118,12 +151,13 @@ func (q quoting) wraps(description []byte, sep byte) bool { // // That row has an empty description, and an empty field carries no separator, // so minimal leaves it bare and pays nothing. Only "all" pays, and it pays for -// every column. -func (q quoting) quoteBytes() int { +// every column - so the number of them is asked for rather than assumed to be +// the six this format started with. +func (q quoting) quoteBytes(columns int) int { if !q.everyField { return 0 } - return 2 * len(columnNames) + return 2 * columns } // mark writes the quote that wraps a plain field, which only "all" has. @@ -146,6 +180,8 @@ type dialect struct { header bool quotes quoting + + columns int } func defaultDialect() dialect { @@ -156,10 +192,11 @@ func defaultDialect() dialect { eol: "\n", header: true, quotes: quoteStyles["minimal"], + columns: defaultColumns, } } -// parseDialect reads the three settings. +// parseDialect reads the five settings. // // A value that is not in the declared set has already been refused by the // registry, which checks it against the declaration for every format at once. @@ -211,17 +248,70 @@ func parseDialect(props map[string]string) (dialect, error) { d.quotes = q } + n, err := columnsFrom(props, d.columns) + if err != nil { + return dialect{}, err + } + d.columns = n + return d, nil } +// columnsFrom reads the one setting here that is a number rather than a word. +// +// Its own function because the four above are each three lines and this is +// seven, and because parseDialect had reached the point where one more setting +// took it past the branching the code shape guard allows. Splitting it was the +// answer rather than raising that number, which is the rule this project keeps: +// the crowd counter only goes down. +func columnsFrom(props map[string]string, fallback int) (int, error) { + v, ok := props[Columns] + if !ok || v == "" { + return fallback, nil + } + n, err := strconv.Atoi(v) + if err != nil { + return 0, badValue(Columns, v, "it has to be a whole number") + } + if n < minColumns || n > maxColumns { + return 0, badValue(Columns, v, + fmt.Sprintf("it has to be between %d and %d", minColumns, maxColumns)) + } + return n, nil +} + func badValue(key, val, why string) error { return &format.PropertyValueError{Format: "csv", Key: key, Value: val, Reason: why} } -// columnNames are the columns this table has always had. The last one is the -// description, which is the field the closing row stretches to reach an exact -// length, so it stays last whatever else changes. -var columnNames = []string{"id", "name", "email", "amount", "created", "description"} +// baseColumns are the columns this table has always had, without the +// description. It is last in every table and is not in this list for that +// reason - it is the field the closing row stretches to reach an exact length, +// so it stays last whatever else changes. +var baseColumns = []string{"id", "name", "email", "amount", "created"} + +// columnNamesFor is the header of a table this wide. +// +// The leading names are taken from baseColumns in order and the description +// closes the row, so asking for fewer columns drops them from the MIDDLE +// rather than from either end. Six gives exactly the six this format has +// always written, which is what keeps the default byte for byte what it was. +// +// Past those, the names say where they are rather than what they hold. Column +// types - a name, an address, a telephone number - are a separate piece of +// work with its own place in the backlog, and inventing half of it here would +// be the harder half to take back. +func columnNamesFor(columns int) []string { + out := make([]string, 0, columns) + for i := 0; i < columns-1; i++ { + if i < len(baseColumns) { + out = append(out, baseColumns[i]) + continue + } + out = append(out, "field_"+strconv.Itoa(i+1)) + } + return append(out, "description") +} // headerLine is the first row, built from the dialect rather than written out, // so the separator in it cannot disagree with the separator in the rows below. @@ -233,12 +323,13 @@ var columnNames = []string{"id", "name", "email", "amount", "created", "descript // well, so a header left bare would be the one row disagreeing with the // setting that produced it. func (d dialect) headerLine() string { - names := columnNames + names := columnNamesFor(d.columns) if d.quotes.everyField { - names = make([]string, 0, len(columnNames)) - for _, column := range columnNames { - names = append(names, string(quoteMark)+column+string(quoteMark)) + quoted := make([]string, 0, len(names)) + for _, column := range names { + quoted = append(quoted, string(quoteMark)+column+string(quoteMark)) } + names = quoted } return strings.Join(names, string(d.sep)) + d.eol } @@ -276,5 +367,11 @@ func properties() []format.Property { Choices: quoteStyleIDs, Default: "minimal", Detail: "Which fields carry quotes. With none the description stops carrying separators as well, because an unquoted field cannot hold one.", }, + { + Name: Columns, Kind: format.PropertyInt, + Min: minColumns, Max: maxColumns, Unit: "columns", + Default: strconv.Itoa(defaultColumns), + Detail: "How many columns each row has. Above 16384 a spreadsheet may show only the first 16384 and drop the rest without a word.", + }, } } diff --git a/internal/guard/csvcolumns_test.go b/internal/guard/csvcolumns_test.go new file mode 100644 index 0000000..03ff43d --- /dev/null +++ b/internal/guard/csvcolumns_test.go @@ -0,0 +1,212 @@ +package guard + +import ( + "bytes" + "encoding/csv" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// csvColumnBounds is the range the registry offers, read from the declaration +// rather than written out here. A pair copied into a guard stops describing the +// thing it guards the moment somebody moves it. +func csvColumnBounds(t *testing.T) (min, max int64) { + t.Helper() + d, err := format.Get("csv") + if err != nil { + t.Fatal(err) + } + for _, p := range d.Properties { + if p.Name != "columns" { + continue + } + if p.Min <= 0 || p.Max <= p.Min { + t.Fatalf("csv declares columns as %d to %d, so this guard would walk nothing", p.Min, p.Max) + } + return p.Min, p.Max + } + t.Fatal("csv declares no columns setting, so this guard would walk nothing") + return 0, 0 +} + +// The table has the columns that were asked for, and the count moves the floor. +// +// What can be wrong here splits three ways, and only the first is what anybody +// would think to check. +// +// The count could be stored and ignored, which a size guard cannot see: a table +// of five columns where six were ordered is exactly as long, parses everywhere, +// and every row agrees with every other. So the FILE is counted, with a reader +// written in another language from the one that wrote it. +// +// The floor could miss that the count moves it. Every column adds its own width +// and a separator, so a floor worked out for six is wrong for every other +// number - and wrong in the SAFE direction going up, which nothing else would +// notice. That is asked by taking the floor the format announces and the byte +// below it, at each width. +// +// And the description could stop being last. It is the field the closing row +// stretches to reach an exact length, so a table that put it anywhere else +// would still be the right size while the padding landed in the middle of a +// row. Nothing about the length would change. +func TestTheCSVColumnCountIsInTheFileAndMovesTheFloor(t *testing.T) { + min, max := csvColumnBounds(t) + widths := []int64{min, min + 1, 5, 6, 7, 9, 40, max} + seen := map[int64]bool{} + + for _, n := range widths { + t.Run(fmt.Sprint(n), func(t *testing.T) { + props := map[string]string{"columns": fmt.Sprint(n)} + d, err := format.Get("csv") + if err != nil { + t.Fatal(err) + } + + floor := csvFloor(t, props) + seen[floor] = true + if _, err := d.Generator.Plan(format.Request{Bytes: floor, Seed: 1, Label: true, + Properties: props}); err != nil { + t.Errorf("announces %d B as its floor for %d columns and then refuses it: %v", floor, n, err) + } + if _, err := d.Generator.Plan(format.Request{Bytes: floor - 1, Seed: 1, Label: true, + Properties: props}); err == nil { + t.Errorf("took %d B, one below the %d B it calls its floor for %d columns", floor-1, floor, n) + } + + // Exact to the byte, at seeds rather than at sizes somebody picked. + // The closing row is stretched to reach the length and every column + // feeds into that arithmetic. + for seed := uint64(1); seed <= 4; seed++ { + for _, extra := range []int64{0, 1, 2, 733} { + writeCSV(t, floor+extra, seed, props) // fails inside on a miss + } + } + + // Counted with encoding/csv, which is a different implementation + // from the Python module the oracle uses and from the hand written + // checker beside it. + body := writeCSV(t, floor+733, 9, props) + rows, err := csv.NewReader(bytes.NewReader(body)).ReadAll() + if err != nil { + t.Fatalf("%d columns: the table does not parse: %v", n, err) + } + if len(rows) < 2 { + t.Fatalf("%d columns: the table holds %d row(s), so there is no data in it", n, len(rows)) + } + for i, row := range rows { + if int64(len(row)) != n { + t.Fatalf("row %d has %d fields and %d columns were ordered", i+1, len(row), n) + } + } + + // The description is last, and it is the one the padding went into. + if got := rows[0][len(rows[0])-1]; got != "description" { + t.Errorf("the last column of the header is %q rather than description, and that is the "+ + "field the closing row stretches - moved anywhere else, the padding lands in the "+ + "middle of a row and the size stays perfect", got) + } + closing := rows[len(rows)-1] + widest := 0 + for i, field := range closing { + if len(field) > len(closing[widest]) { + widest = i + } + } + if widest != len(closing)-1 { + t.Errorf("the longest field of the closing row is column %d of %d, so the padding did not "+ + "go into the description", widest+1, len(closing)) + } + }) + } + + // The floor is not one number wearing every hat. Without this the whole + // test above would pass on a build that worked the floor out for six + // columns and handed it to the rest. + if len(seen) != len(widths) { + t.Errorf("%d widths produced %d distinct floors, and every one of them should differ - each "+ + "column adds its own width and a separator", len(widths), len(seen)) + } +} + +// The six columns this format started with are still those six, in that order. +// +// Separate from everything above because it is the anchor rather than a +// property: the pinned bytes say the default file has not moved, and this says +// WHY in words, so a failure names the column that changed instead of handing +// over two hashes that differ. +func TestTheDefaultCSVStillHasItsSixOriginalColumns(t *testing.T) { + body := writeCSV(t, 8192, 9, map[string]string{}) + header, _, ok := strings.Cut(string(body), "\n") + if !ok { + t.Fatal("the file has no first line") + } + const want = "id,name,email,amount,created,description" + if header != want { + t.Errorf("the default header is %q and it has always been %q. Changing it is a breaking change "+ + "under D11, so it needs the owner, a major version and a changelog entry", header, want) + } +} + +// Every width is well formed, judged by the checker rather than by us. +// +// The checker is TOLD the width, which is what makes this worth running beside +// the counting above. Counting can only ask whether the rows agree with each +// other, and a table that wrote the wrong number of columns agrees with itself +// perfectly. Being told, it can refuse - and the last case here feeds it a +// width that is not the file's, because a checker that accepted that would make +// every pass above worth nothing. +func TestEveryCSVWidthIsWellFormed(t *testing.T) { + min, max := csvColumnBounds(t) + dir := t.TempDir() + + check := func(t *testing.T, body []byte, name string, settings ...string) oracle.Result { + t.Helper() + path := filepath.Join(dir, name+".csv") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + return oracle.Strict("csv", path, settings...) + } + + ran := 0 + widths := []int64{min, 3, 6, 12, 40, max} + for _, n := range widths { + t.Run(fmt.Sprint(n), func(t *testing.T) { + props := map[string]string{"columns": fmt.Sprint(n)} + body := writeCSV(t, csvFloor(t, props)+733, 9, props) + res := check(t, body, fmt.Sprintf("w%d", n), fmt.Sprintf("columns=%d", n)) + if !res.Available { + t.Skip("the structural check needs python") + } + ran++ + if res.Err != nil { + t.Errorf("%d columns is not well formed: %v", n, res.Err) + } + }) + } + if ran == 0 { + t.Skip("the structural check never ran, so nothing here was judged") + } + if ran != len(widths) { + t.Errorf("%d of %d widths reached the checker", ran, len(widths)) + } + + body := writeCSV(t, 8192, 4, map[string]string{"columns": "9"}) + for _, wrong := range []string{"columns=8", "columns=10"} { + res := check(t, body, "wrong_"+wrong, wrong) + if !res.Available { + t.Skip("the structural check needs python") + } + if res.Err == nil { + t.Errorf("a nine column table called %s passed the checker, so being told the width made it "+ + "a rubber stamp: %s", wrong, firstLineOf(res.Output)) + } + } +} diff --git a/internal/guard/csvdialect_test.go b/internal/guard/csvdialect_test.go index d88298a..e45c159 100644 --- a/internal/guard/csvdialect_test.go +++ b/internal/guard/csvdialect_test.go @@ -67,6 +67,27 @@ func writeCSV(t *testing.T, size int64, seed uint64, props map[string]string) [] return buf.Bytes() } +// csvPlannedColumns is how many columns this dialect produces, asked of the +// plan rather than counted in the guard. The plan is what the manifest carries, +// so a guard reading it is reading the number a user would. +func csvPlannedColumns(t *testing.T, props map[string]string) int { + t.Helper() + d, err := format.Get("csv") + if err != nil { + t.Fatal(err) + } + p, err := d.Generator.Plan(format.Request{Bytes: 8192, Seed: 1, Label: true, Properties: props}) + if err != nil { + t.Fatalf("planning to read the column count with %v: %v", props, err) + } + n, ok := p.Properties["columns"].(int) + if !ok || n < 2 { + t.Fatalf("the plan reports columns as %v, and this guard counts separators against it", + p.Properties["columns"]) + } + return n +} + // csvFloor is the smallest file this dialect will take, asked of the format // rather than worked out here. Repeating the arithmetic in the guard would only // prove the guard agrees with itself. @@ -251,8 +272,11 @@ func csvDialectCase(t *testing.T, seen map[int64]bool, delim, eol, header, style // csvPaddingCarriesTheSeparator asks whether the padded row puts the separator // inside the quotes, which is the only reason that column is ever quoted. // -// Asked by counting: a row has five separators between its six fields, so any -// row carrying more than five has them inside the quotes. +// Asked by counting: a row of N fields has N-1 separators between them, so any +// row carrying more is holding some inside its quotes. The number is asked of +// the registry rather than written as five - it WAS five until columns arrived +// on 2026-09-03, and a guard carrying the old constant would have gone on +// passing while counting the wrong thing. // // Over a BAND of sizes rather than one, and that is the whole reason this is a // function of its own. The closing row is whatever length was left over, and a @@ -265,12 +289,16 @@ func csvDialectCase(t *testing.T, seen map[int64]bool, delim, eol, header, style func csvPaddingCarriesTheSeparator(t *testing.T, props map[string]string, sep, delim, style, eol string) { t.Helper() const band = 24 + // How many separators hold the row together, as opposed to sitting inside a + // field. Read off the plan rather than assumed, so this keeps counting the + // right thing when a caller sets a width. + columns := csvPlannedColumns(t, props) carried, widest := 0, 0 for size := int64(8192); size < 8192+band; size++ { body := string(writeCSV(t, size, 9, props)) rows := strings.Split(strings.TrimSuffix(body, eol), eol) last := rows[len(rows)-1] - if inside := strings.Count(last, sep) - 5; inside > 0 { + if inside := strings.Count(last, sep) - (columns - 1); inside > 0 { carried++ } if len(last) > widest { @@ -280,8 +308,8 @@ func csvPaddingCarriesTheSeparator(t *testing.T, props map[string]string, sep, d if style == "none" { if carried != 0 { - t.Errorf("%d of %d closing rows carry a %s beyond the five that separate the fields, and "+ - "quote_style none has no quotes to hide one in", carried, band, delim) + t.Errorf("%d of %d closing rows carry a %s beyond the %d that separate the fields, and "+ + "quote_style none has no quotes to hide one in", carried, band, delim, columns-1) } return } @@ -403,13 +431,16 @@ func TestTheCSVManifestRecordsTheDialectAsFacts(t *testing.T) { eol string head bool style string + cols int }{ - {map[string]string{}, ",", "lf", true, "minimal"}, - {map[string]string{"delimiter": "semicolon"}, ";", "lf", true, "minimal"}, - {map[string]string{"delimiter": "tab", "line_ending": "crlf"}, "\t", "crlf", true, "minimal"}, - {map[string]string{"delimiter": "pipe", "header": "false"}, "|", "lf", false, "minimal"}, - {map[string]string{"quote_style": "all"}, ",", "lf", true, "all"}, - {map[string]string{"quote_style": "none", "delimiter": "tab"}, "\t", "lf", true, "none"}, + {map[string]string{}, ",", "lf", true, "minimal", 6}, + {map[string]string{"delimiter": "semicolon"}, ";", "lf", true, "minimal", 6}, + {map[string]string{"delimiter": "tab", "line_ending": "crlf"}, "\t", "crlf", true, "minimal", 6}, + {map[string]string{"delimiter": "pipe", "header": "false"}, "|", "lf", false, "minimal", 6}, + {map[string]string{"quote_style": "all"}, ",", "lf", true, "all", 6}, + {map[string]string{"quote_style": "none", "delimiter": "tab"}, "\t", "lf", true, "none", 6}, + {map[string]string{"columns": "2"}, ",", "lf", true, "minimal", 2}, + {map[string]string{"columns": "41", "quote_style": "all"}, ",", "lf", true, "all", 41}, } for _, c := range cases { t.Run(fmt.Sprint(c.props), func(t *testing.T) { @@ -432,6 +463,11 @@ func TestTheCSVManifestRecordsTheDialectAsFacts(t *testing.T) { if got := p.Properties["quote_style"]; got != c.style { t.Errorf("quote_style is %v, wanted %q", got, c.style) } + // The width the run actually used, not the one this format started + // with. A script reading the manifest sizes its assertions off it. + if got := p.Properties["columns"]; got != c.cols { + t.Errorf("columns is %v, wanted %d", got, c.cols) + } // And it stays JSON a script can read, rather than something that // only looks right in a Go printout. if _, err := json.Marshal(p.Properties); err != nil { diff --git a/internal/guard/generatorbytes_test.go b/internal/guard/generatorbytes_test.go index 6621343..5b575b7 100644 --- a/internal/guard/generatorbytes_test.go +++ b/internal/guard/generatorbytes_test.go @@ -228,6 +228,14 @@ func goldenCases() map[string]engine.Target { Properties: map[string]string{"quote_style": "all"}}, "csv_8kib_quote_none": {ID: "g", Format: "csv", Sizes: engine.Uniform(1, 8192), Label: true, Properties: map[string]string{"quote_style": "none"}}, + + // A width that is not the six this format started with. Every column + // carries its own value and its own separator, so this pins the whole + // row template rather than a count in a header - and it is the case + // that would move if somebody reordered the columns while keeping their + // number, which no size or count check would see. + "csv_8kib_seventeen_columns": {ID: "g", Format: "csv", Sizes: engine.Uniform(1, 8192), Label: true, + Properties: map[string]string{"columns": "17"}}, "json_8kib": {ID: "g", Format: "json", Sizes: engine.Uniform(1, 8192), Label: true}, "xml_8kib": {ID: "g", Format: "xml", Sizes: engine.Uniform(1, 8192), Label: true}, "html_8kib": {ID: "g", Format: "html", Sizes: engine.Uniform(1, 8192), Label: true}, diff --git a/internal/guard/parity_test.go b/internal/guard/parity_test.go index 94618ff..ce4fa30 100644 --- a/internal/guard/parity_test.go +++ b/internal/guard/parity_test.go @@ -99,6 +99,7 @@ var reachableFromTheWindow = []string{ // gains a property gains its field with no window code. // TestTheWindowDrawsAFieldForEveryDeclaredProperty. "property:bmp.height", + "property:csv.columns", "property:csv.delimiter", "property:csv.header", "property:csv.line_ending", diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 568497f..504af8b 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -251,6 +251,10 @@ "csv_8kib_quote_none": { "bytes": 8192, "sha256": "d596eccde18c924fe0245b21523a6fc47c90f2d322131e0b5bcb03dc8a5d8b93" + }, + "csv_8kib_seventeen_columns": { + "bytes": 8192, + "sha256": "23d2fc7e93456ed828159d7d5199faaed2129d164640e0743c5aaf37bb9b6143" } }, "remeasured": [ @@ -304,6 +308,13 @@ "csv_8kib_quote_all", "csv_8kib_quote_none" ] + }, + { + "on": "2026-09-03", + "why": "csv learned columns, and NOTHING here moved because of it. The three csv cases above are byte for byte what they were an hour earlier, which is the whole claim this setting makes: the default is the six columns this format has always written, so a table nobody asked to reshape is the table it was. The new case is the other half - a width of seventeen, pinned from the day it exists rather than after somebody notices it never was. It covers what a count cannot: every column carries its own value and its own separator, so reordering the columns while keeping their number would move these bytes and leave every size and width check green. The floor moves with the width and is measured, not pinned here: 36 B at two columns, 115 B at six, 5017 B at 256 and 709793 B at the ceiling of 32768.", + "files": [ + "csv_8kib_seventeen_columns" + ] } ] } diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index dd39192..b6376fd 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -305,6 +305,12 @@ def check_csv(data, settings=None): style = settings.get("quote_style", "minimal") if style not in ("minimal", "all", "none"): fail(f"quote_style {style!r} is not one of minimal, all, none") + # How many columns there should be, when the caller knows. Told rather than + # counted, because counting can only ask whether the rows AGREE with each + # other - and a table that wrote five columns where six were ordered agrees + # with itself perfectly, at exactly the right size. + wanted = settings.get("columns") + wanted = int(wanted) if wanted is not None else None try: text = data.decode("utf-8") @@ -371,6 +377,8 @@ def check_csv(data, settings=None): fail(f"the table holds {len(rows)} row(s), so there is no data to check") columns = len(rows[0]) + if wanted is not None and columns != wanted: + fail(f"the first row has {columns} column(s) and {wanted} were ordered") if columns < 2: fail(f"the first row has {columns} column(s), so nothing is separated by " f"the {settings.get('delimiter', 'comma')} this file is meant to use") diff --git a/web/public/formats/index.html b/web/public/formats/index.html index f50616d..67a8624 100644 --- a/web/public/formats/index.html +++ b/web/public/formats/index.html @@ -360,6 +360,11 @@

Settings each format accepts

quote_style all, minimal, none + + + columns + 2 - 32768 columns + docx paragraphs diff --git a/web/public/pl/formaty/index.html b/web/public/pl/formaty/index.html index 8af92be..208b717 100644 --- a/web/public/pl/formaty/index.html +++ b/web/public/pl/formaty/index.html @@ -360,6 +360,11 @@

Ustawienia, które przyjmuje każdy format

quote_style all, minimal, none + + + columns + 2 - 32768 kolumn + docx paragraphs