Add LIMIT support to interactive SELECT queries

ARCHITECTURE.md

@@ -243,8 +243,9 @@ The current code already uses state functions and file positions to support
 large dumps with limited memory. These areas merit measurement or refinement
 as the tool evolves:
 
-- [scanner.go](scanner.go) allocates channels for 2,000,000 tokens. Token emission
-  copies source text into strings.
+- [scanner.go](scanner.go) still allocates 2,000,000-token channels for full scans
+  and user-query parsing. Data-section scans use a 64-token queue. Token emission
+  still copies source text into strings. Queue byte limits need further work.
 - [cmd/sqlread/main.go](cmd/sqlread/main.go) configures a 100,000,000-byte reader
   buffer. Its capacity needs a measured justification against the memory budget.
 - [summary.go](summary.go) retains all data-region references in memory.
@@ -253,8 +254,9 @@ as the tool evolves:
   values still need a bounded representation.
 - [cmd/sqlread/readlinewrap.go](cmd/sqlread/readlinewrap.go) connects parser reads
   directly to terminal input.
-- [cmd/sqlread/query.go](cmd/sqlread/query.go) keeps execution in the command
-  package. Worker completion and output errors need a consistent owner.
+- [execute.go](execute.go) streams query results outside the command package.
+  It owns the active lexer worker. It waits for completion on limits and errors.
+  Other scanning paths still need the same lifecycle review.
 
 Refinements must preserve the tool's ability to process very large dumps with
 limited memory. Keep the state-function foundation and the file as the backing

README.md

@@ -28,6 +28,27 @@ go install github.com/donatj/sqlread/cmd/sqlread@latest
 
 Available on the [Releases](https://github.com/donatj/sqlread/releases) page currently for Linux and macOS.
 
+## User query grammar
+
+Square brackets (`[]`) mark optional clauses. Braces (`{}`) require one choice.
+A vertical bar (`|`) separates choices. An ellipsis (`...`) marks a repeated item.
+
+```
+SELECT { * | column_name } [, { * | column_name }] ...
+    FROM table_name
+    [INTO OUTFILE string]
+    [LIMIT { row_count | offset, row_count }];
+
+SHOW TABLES;
+SHOW COLUMNS FROM table_name;
+SHOW CREATE TABLE table_name;
+{ EXIT | QUIT };
+```
+
+Keywords are case-insensitive. Table names and column names may use backticks.
+`string` values may use single or double quotes. `offset` and `row_count` must be
+non-negative integers. Expressions, `WHERE`, and `ORDER BY` are not supported.
+
 ## Example usage
 
 ```
@@ -50,13 +71,9 @@ zip_code,varchar
 deleted,tinyint
 2017/12/22 12:24:08 restarting lexer
 
-> SELECT building_id, title FROM buildings;
+> SELECT building_id, title FROM buildings LIMIT 2;
 2,Home Building
 190,Test Building (demo)
-192,Donat Building
-194,Other Building
-201,Sample Building (demo)
-205,Johnson Building
 2017/12/22 12:24:44 restarting lexer
 
 > SELECT `building_id`, `title` FROM `buildings` INTO OUTFILE "dump.csv";

cmd/sqlread/interactive_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestInteractiveQueries(t *testing.T) {
+	file := queryFixture(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestInteractiveProcess$", file.Name())
+	cmd.Env = append(os.Environ(), "SQLREAD_TEST_INTERACTIVE=1")
+	var diagnostics bytes.Buffer
+	cmd.Stderr = &diagnostics
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		stdin.Close()
+		cancel()
+		cmd.Wait()
+	}()
+
+	reader := bufio.NewReader(stdout)
+	for _, test := range []struct {
+		query string
+		rows  []string
+	}{
+		{"SELECT id FROM other;", []string{"99"}},
+		{"SELECT id FROM records LIMIT 1, 2;", []string{"2", "3"}},
+		{"SELECT id FROM records LIMIT 0;\nSELECT id FROM other;", []string{"99"}},
+	} {
+		if _, err := fmt.Fprintln(stdin, test.query); err != nil {
+			t.Fatal(err)
+		}
+		// Keep stdin open. Each query must complete without another terminator or EOF.
+		for _, want := range test.rows {
+			line, err := reader.ReadString('\n')
+			if err != nil {
+				t.Fatalf("query %q did not produce %q: %v", test.query, want, err)
+			}
+			if strings.TrimSpace(line) != want {
+				t.Fatalf("query %q: output = %q; want %q", test.query, line, want)
+			}
+		}
+	}
+	if _, err := fmt.Fprintln(stdin, "QUIT;"); err != nil {
+		t.Fatal(err)
+	}
+	if err := cmd.Wait(); err != nil {
+		t.Fatalf("interactive process: %v\n%s", err, diagnostics.String())
+	}
+}
+
+func TestInteractiveProcess(t *testing.T) {
+	if os.Getenv("SQLREAD_TEST_INTERACTIVE") != "1" {
+		return
+	}
+	main()
+	os.Exit(0)
+}

cmd/sqlread/query.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"log"
 	"sort"
 
 	"github.com/donatj/sqlread"
@@ -12,63 +11,16 @@ import (
 type DataWriter interface {
 	Write(record []string) error
 	Flush()
+	Error() error
 }
 
 func execQuery(tree sqlread.SummaryTree, qry sqlread.Query, buff io.ReaderAt, w DataWriter) error {
-	tbl, tok := tree[qry.Table]
-	if !tok {
-		return fmt.Errorf("table `%s` not found", qry.Table)
-	}
-	colind := []int{}
-	for _, col := range qry.Columns {
-		found := false
-		for tci, tcol := range tbl.Cols {
-			if col == "*" || col == tcol.Name {
-				found = true
-				colind = append(colind, tci)
-			}
-		}
-
-		if !found {
-			return fmt.Errorf("column `%s` not found", col)
-		}
-	}
-	for _, loc := range tbl.DataLocs {
-		start := loc.Start.Pos
-		end := loc.End.Pos
-
-		sl, sli := sqlread.LexSection(buff, start, end-start+1)
-		go func() {
-			sl.Run(sqlread.StartState)
-		}()
-
-		sp := sqlread.NewInsertDetailParser()
-
-		spr := sqlread.Parse(sli)
-		go func() {
-			err := spr.Run(sp.ParseStart)
-			if err != nil {
-				log.Fatal(err)
-			}
-		}()
-
-		for {
-			row, ok := <-sp.Out
-			if !ok {
-				w.Flush()
-				break
-			}
-
-			out := make([]string, len(colind))
-			for i, ci := range colind {
-				out[i] = row[ci]
-			}
-
-			w.Write(out)
-		}
+	err := sqlread.ExecuteQuery(tree, qry, buff, w.Write)
+	w.Flush()
+	if err != nil {
+		return err
 	}
-
-	return nil
+	return w.Error()
 }
 
 func showColumns(tree sqlread.SummaryTree, sctbl string, w DataWriter) error {

cmd/sqlread/query_bench_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"encoding/csv"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/donatj/sqlread"
+)
+
+func BenchmarkExecQuery(b *testing.B) {
+	input := "CREATE TABLE `records` (`id` int);\nINSERT INTO `records` VALUES " +
+		strings.Repeat("(1),", 20000) + "(2);"
+	source := strings.NewReader(input)
+	l, items := sqlread.Lex(source)
+	done := make(chan struct{})
+	go func() { defer close(done); l.Run(sqlread.StartState) }()
+	defer func() { l.Stop(); <-done }()
+	summary := sqlread.NewSummaryParser()
+	if err := sqlread.Parse(items).Run(summary.ParseStart); err != nil {
+		b.Fatal(err)
+	}
+	for _, test := range []struct {
+		name  string
+		limit *sqlread.QueryLimit
+	}{{"all_rows", nil}, {"limit_10", &sqlread.QueryLimit{Count: 10}}} {
+		b.Run(test.name, func(b *testing.B) {
+			qry := sqlread.Query{Table: "records", Columns: []string{"*"}, Limit: test.limit}
+			writer := csv.NewWriter(io.Discard)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				if err := execQuery(summary.Tree, qry, source, writer); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}