From 2e9ff581739f1d6325fec29011a5889d51eac217 Mon Sep 17 00:00:00 2001 From: Walter Pinto Date: Sun, 4 Oct 2026 15:19:17 +0100 Subject: [PATCH 1/2] refactor(wit): discover worlds and package with the AST parser DiscoverWorlds and DiscoverPackage scanned lines with regexes that missed versioned packages and tripped on comments. They now use the shared lenient AST parser from tools/common/wit/ast, and the branch's copy of the parser is removed. Item 3 of #66. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 13 + docs/wit/architecture.md | 2 +- tools/please_wit/ast/BUILD | 21 - tools/please_wit/ast/ast.go | 152 ---- tools/please_wit/ast/ast_test.go | 178 ----- tools/please_wit/ast/parser.go | 814 --------------------- tools/please_wit/ast/parser_test.go | 643 ---------------- tools/please_wit/generate/BUILD | 4 +- tools/please_wit/generate/cpp.go | 2 +- tools/please_wit/generate/cpp_test.go | 2 +- tools/please_wit/generate/generate.go | 75 +- tools/please_wit/generate/generate_test.go | 86 ++- tools/please_wit/generate/generator.go | 2 +- tools/please_wit/generate/golang.go | 2 +- tools/please_wit/generate/golang_test.go | 2 +- tools/please_wit/generate/kotlin.go | 2 +- tools/please_wit/generate/kotlin_test.go | 4 +- tools/please_wit/generate/python.go | 2 +- tools/please_wit/generate/python_test.go | 2 +- tools/please_wit/generate/rust.go | 2 +- tools/please_wit/generate/rust_test.go | 2 +- tools/please_wit/generate/swift.go | 2 +- tools/please_wit/generate/swift_test.go | 2 +- tools/please_wit/generate/ts.go | 2 +- tools/please_wit/generate/ts_test.go | 2 +- 25 files changed, 106 insertions(+), 1914 deletions(-) delete mode 100644 tools/please_wit/ast/BUILD delete mode 100644 tools/please_wit/ast/ast.go delete mode 100644 tools/please_wit/ast/ast_test.go delete mode 100644 tools/please_wit/ast/parser.go delete mode 100644 tools/please_wit/ast/parser_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f95b0c0..44bbcf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- World and package discovery (`DiscoverWorlds`, `DiscoverPackage`) read the + WIT with the AST parser instead of line-based regular expressions, so + versioned packages (`package wasi:cli@0.2.0;`), commented-out declarations, + several declarations on one line and `use`, `variant` or `flags` declarations + are handled. A file that does not parse is now an error naming the file + instead of being skipped silently. +- The AST parser moved to `tools/common/wit/ast` (shared on `main`, so other + language plugins can use it). + ## [0.3.1] - 2026-09-20 ### Added diff --git a/docs/wit/architecture.md b/docs/wit/architecture.md index 8f4d3ac..c14013e 100644 --- a/docs/wit/architecture.md +++ b/docs/wit/architecture.md @@ -30,7 +30,7 @@ access. ## 2. Standalone Go AST Parser Architecture -`please_wit` embeds a lightweight lexer and parser in `tools/please_wit/ast`: +`please_wit` embeds a lightweight lexer and parser in `tools/common/wit/ast`: ```mermaid flowchart LR diff --git a/tools/please_wit/ast/BUILD b/tools/please_wit/ast/BUILD deleted file mode 100644 index f9b4b7f..0000000 --- a/tools/please_wit/ast/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -subinclude("///go//build_defs:go") - -go_library( - name = "ast", - srcs = [ - "ast.go", - "parser.go", - ], - visibility = ["//tools/please_wit/..."], -) - -go_test( - name = "ast_test", - srcs = [ - "ast_test.go", - "parser_test.go", - ], - deps = [ - ":ast", - ], -) diff --git a/tools/please_wit/ast/ast.go b/tools/please_wit/ast/ast.go deleted file mode 100644 index 0a7b92b..0000000 --- a/tools/please_wit/ast/ast.go +++ /dev/null @@ -1,152 +0,0 @@ -package ast - -import ( - "fmt" - "strings" -) - -// TypeKind distinguishes between primitives, compound types, and user references. -type TypeKind int - -const ( - KindPrimitive TypeKind = iota - KindList - KindOption - KindResult - KindTuple - KindNamed -) - -// TypeRef represents a type reference in WIT. -type TypeRef struct { - Kind TypeKind - Name string // e.g. "s32", "string", or user-defined type name - TypeArgs []*TypeRef // For list, option, result, tuple<...> -} - -func (t *TypeRef) String() string { - if t == nil { - return "unit" - } - switch t.Kind { - case KindPrimitive, KindNamed: - return t.Name - case KindList: - if len(t.TypeArgs) > 0 { - return fmt.Sprintf("list<%s>", t.TypeArgs[0].String()) - } - return "list" - case KindOption: - if len(t.TypeArgs) > 0 { - return fmt.Sprintf("option<%s>", t.TypeArgs[0].String()) - } - return "option" - case KindResult: - if len(t.TypeArgs) == 2 { - return fmt.Sprintf("result<%s, %s>", t.TypeArgs[0].String(), t.TypeArgs[1].String()) - } else if len(t.TypeArgs) == 1 { - return fmt.Sprintf("result<%s>", t.TypeArgs[0].String()) - } - return "result" - case KindTuple: - var inner []string - for _, arg := range t.TypeArgs { - inner = append(inner, arg.String()) - } - return fmt.Sprintf("tuple<%s>", strings.Join(inner, ", ")) - } - return t.Name -} - -// Param represents a parameter in a WIT function. -type Param struct { - Name string - Type *TypeRef -} - -// Function represents a function declared in a WIT interface. -type Function struct { - Name string - Doc string - Params []Param - Results *TypeRef // nil means void / unit -} - -// Field represents a field in a WIT record. -type Field struct { - Name string - Type *TypeRef -} - -// Record represents a record structure declared in a WIT interface. -type Record struct { - Name string - Doc string - Fields []Field -} - -// EnumCase represents a single case in an enum. -type EnumCase struct { - Name string -} - -// Enum represents an enum declared in a WIT interface. -type Enum struct { - Name string - Doc string - Cases []EnumCase -} - -// TypeDef represents a type alias declared in WIT. -type TypeDef struct { - Name string - Type *TypeRef -} - -// Resource represents a resource declaration in a WIT interface. -type Resource struct { - Name string - Doc string - Constructor *Function - Methods []Function - Static []Function -} - -// Interface represents an interface block in WIT. -type Interface struct { - Name string - Doc string - Functions []Function - Records []Record - Enums []Enum - TypeDefs []TypeDef - Resources []Resource -} - -// World represents a world definition in WIT. -type World struct { - Name string - Doc string - Imports []string - Exports []string -} - -// Package represents a WIT package containing interfaces and worlds. -type Package struct { - Namespace string - Name string - Version string - Interfaces []Interface - Worlds []World -} - -// FullName returns the canonical package identifier (e.g. "namespace:name"). -func (p *Package) FullName() string { - if p.Namespace != "" && p.Name != "" { - return fmt.Sprintf("%s:%s", p.Namespace, p.Name) - } - if p.Name != "" { - return p.Name - } - return p.Namespace -} diff --git a/tools/please_wit/ast/ast_test.go b/tools/please_wit/ast/ast_test.go deleted file mode 100644 index 015f92e..0000000 --- a/tools/please_wit/ast/ast_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package ast - -import ( - "os" - "path/filepath" - "testing" -) - -func TestParseSimpleWit(t *testing.T) { - input := ` -// Package comment -package test:structures@0.1.0; - -interface two-sum { - solve: func(nums: list, target: s32) -> list; - reset: func(); -} - -world two-sum-world { - export two-sum; -} -` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if pkg.Namespace != "test" { - t.Errorf("expected namespace 'test', got %q", pkg.Namespace) - } - if pkg.Name != "structures" { - t.Errorf("expected name 'structures', got %q", pkg.Name) - } - if pkg.Version != "0.1.0" { - t.Errorf("expected version '0.1.0', got %q", pkg.Version) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - iface := pkg.Interfaces[0] - if iface.Name != "two-sum" { - t.Errorf("expected interface 'two-sum', got %q", iface.Name) - } - - if len(iface.Functions) != 2 { - t.Fatalf("expected 2 functions, got %d", len(iface.Functions)) - } - - fn1 := iface.Functions[0] - if fn1.Name != "solve" { - t.Errorf("expected fn 'solve', got %q", fn1.Name) - } - if len(fn1.Params) != 2 { - t.Fatalf("expected 2 params, got %d", len(fn1.Params)) - } - if fn1.Params[0].Name != "nums" || fn1.Params[0].Type.Kind != KindList { - t.Errorf("unexpected param 0: %+v", fn1.Params[0]) - } - if fn1.Params[1].Name != "target" || fn1.Params[1].Type.Name != "s32" { - t.Errorf("unexpected param 1: %+v", fn1.Params[1]) - } - if fn1.Results == nil || fn1.Results.Kind != KindList { - t.Errorf("unexpected results: %+v", fn1.Results) - } - - fn2 := iface.Functions[1] - if fn2.Name != "reset" || fn2.Results != nil { - t.Errorf("unexpected fn2: %+v", fn2) - } - - if len(pkg.Worlds) != 1 { - t.Fatalf("expected 1 world, got %d", len(pkg.Worlds)) - } - world := pkg.Worlds[0] - if world.Name != "two-sum-world" { - t.Errorf("expected world 'two-sum-world', got %q", world.Name) - } - if len(world.Exports) != 1 || world.Exports[0] != "two-sum" { - t.Errorf("unexpected exports: %v", world.Exports) - } -} - -func TestParseRecordsAndEnums(t *testing.T) { - input := ` -package test:types; - -interface data-types { - /* Multi line comment - about point - */ - record point { - x: f64, - y: f64, - } - - enum color { - red, - green, - blue, - } - - type point-alias = point; - - get-color: func(p: point) -> option; - calc: func(a: s32) -> result; -} -` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - iface := pkg.Interfaces[0] - - if len(iface.Records) != 1 { - t.Fatalf("expected 1 record, got %d", len(iface.Records)) - } - rec := iface.Records[0] - if rec.Name != "point" || len(rec.Fields) != 2 { - t.Errorf("unexpected record: %+v", rec) - } - - if len(iface.Enums) != 1 { - t.Fatalf("expected 1 enum, got %d", len(iface.Enums)) - } - enm := iface.Enums[0] - if enm.Name != "color" || len(enm.Cases) != 3 { - t.Errorf("unexpected enum: %+v", enm) - } - - if len(iface.TypeDefs) != 1 { - t.Fatalf("expected 1 type def, got %d", len(iface.TypeDefs)) - } - - if len(iface.Functions) != 2 { - t.Fatalf("expected 2 functions, got %d", len(iface.Functions)) - } - - fn1 := iface.Functions[0] - if fn1.Results == nil || fn1.Results.Kind != KindOption { - t.Errorf("expected option result, got %+v", fn1.Results) - } - - fn2 := iface.Functions[1] - if fn2.Results == nil || fn2.Results.Kind != KindResult { - t.Errorf("expected result result, got %+v", fn2.Results) - } -} - -func TestParsePath(t *testing.T) { - tmpDir := t.TempDir() - witFile := filepath.Join(tmpDir, "test.wit") - content := ` -package test:structures; - -interface disjoint-set { - find: func(x: s32) -> s32; -} -` - if err := os.WriteFile(witFile, []byte(content), 0644); err != nil { - t.Fatal(err) - } - - pkg, err := ParsePath(tmpDir) - if err != nil { - t.Fatalf("ParsePath failed: %v", err) - } - if pkg.FullName() != "test:structures" { - t.Errorf("expected test:structures, got %q", pkg.FullName()) - } - if len(pkg.Interfaces) != 1 || pkg.Interfaces[0].Name != "disjoint-set" { - t.Errorf("unexpected interfaces: %+v", pkg.Interfaces) - } -} diff --git a/tools/please_wit/ast/parser.go b/tools/please_wit/ast/parser.go deleted file mode 100644 index eab2399..0000000 --- a/tools/please_wit/ast/parser.go +++ /dev/null @@ -1,814 +0,0 @@ -package ast - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "unicode" -) - -// TokenType represents lexical token categories in WIT. -type TokenType int - -const ( - TokenEOF TokenType = iota - TokenIdent - TokenColon - TokenSemicolon - TokenComma - TokenLBrace - TokenRBrace - TokenLParen - TokenRParen - TokenLAngle - TokenRAngle - TokenArrow - TokenEquals - TokenAt -) - -type Token struct { - Type TokenType - Value string - Line int -} - -// Lexer tokenizes WIT source code. -type Lexer struct { - src []rune - pos int - length int - line int -} - -func NewLexer(input string) *Lexer { - src := []rune(input) - return &Lexer{ - src: src, - pos: 0, - length: len(src), - line: 1, - } -} - -func (l *Lexer) peek() rune { - if l.pos >= l.length { - return 0 - } - return l.src[l.pos] -} - -func (l *Lexer) next() rune { - if l.pos >= l.length { - return 0 - } - r := l.src[l.pos] - l.pos++ - if r == '\n' { - l.line++ - } - return r -} - -func (l *Lexer) skipWhitespaceAndComments() { - for l.pos < l.length { - r := l.peek() - if unicode.IsSpace(r) { - l.next() - continue - } - if r == '/' && l.pos+1 < l.length { - nextR := l.src[l.pos+1] - if nextR == '/' { - // Single line comment - for l.pos < l.length && l.peek() != '\n' { - l.next() - } - continue - } else if nextR == '*' { - // Multi line comment - l.next() // consume '/' - l.next() // consume '*' - for l.pos < l.length { - if l.peek() == '*' && l.pos+1 < l.length && l.src[l.pos+1] == '/' { - l.next() - l.next() - break - } - l.next() - } - continue - } - } - break - } -} - -func (l *Lexer) NextToken() Token { - l.skipWhitespaceAndComments() - - if l.pos >= l.length { - return Token{Type: TokenEOF, Value: "", Line: l.line} - } - - startLine := l.line - r := l.next() - - switch r { - case ':': - return Token{Type: TokenColon, Value: ":", Line: startLine} - case ';': - return Token{Type: TokenSemicolon, Value: ";", Line: startLine} - case ',': - return Token{Type: TokenComma, Value: ",", Line: startLine} - case '{': - return Token{Type: TokenLBrace, Value: "{", Line: startLine} - case '}': - return Token{Type: TokenRBrace, Value: "}", Line: startLine} - case '(': - return Token{Type: TokenLParen, Value: "(", Line: startLine} - case ')': - return Token{Type: TokenRParen, Value: ")", Line: startLine} - case '<': - return Token{Type: TokenLAngle, Value: "<", Line: startLine} - case '>': - return Token{Type: TokenRAngle, Value: ">", Line: startLine} - case '=': - return Token{Type: TokenEquals, Value: "=", Line: startLine} - case '@': - return Token{Type: TokenAt, Value: "@", Line: startLine} - case '-': - if l.peek() == '>' { - l.next() - return Token{Type: TokenArrow, Value: "->", Line: startLine} - } - } - - // Identifiers (including kebab-case, dots, and colons in package paths or versions) - if unicode.IsLetter(r) || r == '_' || r == '%' { - var val []rune - val = append(val, r) - for l.pos < l.length { - peeked := l.peek() - if unicode.IsLetter(peeked) || unicode.IsDigit(peeked) || peeked == '_' || peeked == '-' { - val = append(val, l.next()) - } else { - break - } - } - return Token{Type: TokenIdent, Value: string(val), Line: startLine} - } - - // Digits (e.g. in versions) - if unicode.IsDigit(r) { - var val []rune - val = append(val, r) - for l.pos < l.length { - peeked := l.peek() - if unicode.IsDigit(peeked) || peeked == '.' { - val = append(val, l.next()) - } else { - break - } - } - return Token{Type: TokenIdent, Value: string(val), Line: startLine} - } - - return Token{Type: TokenIdent, Value: string(r), Line: startLine} -} - -// Parser parses WIT tokens into an AST. -type Parser struct { - tokens []Token - pos int -} - -func NewParser(tokens []Token) *Parser { - return &Parser{tokens: tokens, pos: 0} -} - -func (p *Parser) peek() Token { - if p.pos >= len(p.tokens) { - return Token{Type: TokenEOF} - } - return p.tokens[p.pos] -} - -func (p *Parser) next() Token { - tok := p.peek() - if p.pos < len(p.tokens) { - p.pos++ - } - return tok -} - -func (p *Parser) expect(tt TokenType) (Token, error) { - tok := p.next() - if tok.Type != tt { - return tok, fmt.Errorf("line %d: expected token type %v, got %v (%q)", tok.Line, tt, tok.Type, tok.Value) - } - return tok, nil -} - -func (p *Parser) expectIdent() (string, error) { - tok := p.next() - if tok.Type != TokenIdent { - return "", fmt.Errorf("line %d: expected identifier, got %q", tok.Line, tok.Value) - } - return tok.Value, nil -} - -// Parse parses the whole WIT file into a Package AST. -func (p *Parser) Parse() (*Package, error) { - pkg := &Package{} - - for p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "package": - p.next() - if err := p.parsePackage(pkg); err != nil { - return nil, err - } - case "interface": - p.next() - iface, err := p.parseInterface() - if err != nil { - return nil, err - } - pkg.Interfaces = append(pkg.Interfaces, *iface) - case "world": - p.next() - w, err := p.parseWorld() - if err != nil { - return nil, err - } - pkg.Worlds = append(pkg.Worlds, *w) - default: - // Skip unexpected token at top level - p.next() - } - } else { - p.next() - } - } - - return pkg, nil -} - -func (p *Parser) parsePackage(pkg *Package) error { - ns, err := p.expectIdent() - if err != nil { - return err - } - pkg.Namespace = ns - - if p.peek().Type == TokenColon { - p.next() - name, err := p.expectIdent() - if err != nil { - return err - } - pkg.Name = name - } else { - pkg.Name = pkg.Namespace - pkg.Namespace = "" - } - - if p.peek().Type == TokenAt { - p.next() - ver, err := p.expectIdent() - if err == nil { - pkg.Version = ver - } - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - return nil -} - -func (p *Parser) parseInterface() (*Interface, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - iface := &Interface{Name: name} - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "record": - p.next() - rec, err := p.parseRecord() - if err != nil { - return nil, err - } - iface.Records = append(iface.Records, *rec) - case "enum": - p.next() - enm, err := p.parseEnum() - if err != nil { - return nil, err - } - iface.Enums = append(iface.Enums, *enm) - case "type": - p.next() - tdef, err := p.parseTypeDef() - if err != nil { - return nil, err - } - iface.TypeDefs = append(iface.TypeDefs, *tdef) - case "resource": - p.next() - res, err := p.parseResource() - if err != nil { - return nil, err - } - iface.Resources = append(iface.Resources, *res) - default: - // Function declaration: : func(...) [-> ...] ; - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - iface.Functions = append(iface.Functions, *fn) - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - - return iface, nil -} - -func (p *Parser) parseFunction() (*Function, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - - funcKeyword, err := p.expectIdent() - if err != nil || funcKeyword != "func" { - return nil, fmt.Errorf("expected 'func', got %q", funcKeyword) - } - - if _, err := p.expect(TokenLParen); err != nil { - return nil, err - } - - var params []Param - for p.peek().Type != TokenRParen && p.peek().Type != TokenEOF { - pName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - pType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - params = append(params, Param{Name: pName, Type: pType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRParen); err != nil { - return nil, err - } - - var resultType *TypeRef - if p.peek().Type == TokenArrow { - p.next() - rt, err := p.parseTypeRef() - if err != nil { - return nil, err - } - resultType = rt - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - - return &Function{ - Name: name, - Params: params, - Results: resultType, - }, nil -} - -func (p *Parser) parseTypeRef() (*TypeRef, error) { - tok := p.peek() - if tok.Type != TokenIdent { - return nil, fmt.Errorf("line %d: expected type name, got %q", tok.Line, tok.Value) - } - p.next() - typeName := tok.Value - - switch typeName { - case "list": - if p.peek().Type == TokenLAngle { - p.next() - inner, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - return &TypeRef{Kind: KindList, Name: "list", TypeArgs: []*TypeRef{inner}}, nil - } - return &TypeRef{Kind: KindList, Name: "list"}, nil - - case "option": - if p.peek().Type == TokenLAngle { - p.next() - inner, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - return &TypeRef{Kind: KindOption, Name: "option", TypeArgs: []*TypeRef{inner}}, nil - } - return &TypeRef{Kind: KindOption, Name: "option"}, nil - - case "result": - var typeArgs []*TypeRef - if p.peek().Type == TokenLAngle { - p.next() - // Handle result<_, E> or result or result - if p.peek().Type == TokenIdent && p.peek().Value == "_" { - p.next() - typeArgs = append(typeArgs, &TypeRef{Kind: KindPrimitive, Name: "_"}) - } else { - t1, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t1) - } - - if p.peek().Type == TokenComma { - p.next() - t2, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t2) - } - - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - } - return &TypeRef{Kind: KindResult, Name: "result", TypeArgs: typeArgs}, nil - - case "tuple": - var typeArgs []*TypeRef - if p.peek().Type == TokenLAngle { - p.next() - for p.peek().Type != TokenRAngle && p.peek().Type != TokenEOF { - t, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t) - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - } - return &TypeRef{Kind: KindTuple, Name: "tuple", TypeArgs: typeArgs}, nil - - case "u8", "u16", "u32", "u64", "s8", "s16", "s32", "s64", "f32", "f64", "char", "bool", "string": - return &TypeRef{Kind: KindPrimitive, Name: typeName}, nil - - default: - return &TypeRef{Kind: KindNamed, Name: typeName}, nil - } -} - -func (p *Parser) parseRecord() (*Record, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - var fields []Field - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - fName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - fType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - fields = append(fields, Field{Name: fName, Type: fType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return &Record{Name: name, Fields: fields}, nil -} - -func (p *Parser) parseEnum() (*Enum, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - var cases []EnumCase - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - cName, err := p.expectIdent() - if err != nil { - return nil, err - } - cases = append(cases, EnumCase{Name: cName}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return &Enum{Name: name, Cases: cases}, nil -} - -func (p *Parser) parseTypeDef() (*TypeDef, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenEquals); err != nil { - return nil, err - } - tRef, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if p.peek().Type == TokenSemicolon { - p.next() - } - return &TypeDef{Name: name, Type: tRef}, nil -} - -func (p *Parser) parseResource() (*Resource, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - res := &Resource{Name: name} - - if p.peek().Type == TokenSemicolon { - p.next() - return res, nil - } - - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "constructor": - p.next() - fn, err := p.parseConstructor() - if err != nil { - return nil, err - } - res.Constructor = fn - case "static": - p.next() - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - res.Static = append(res.Static, *fn) - default: - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - res.Methods = append(res.Methods, *fn) - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - - return res, nil -} - -func (p *Parser) parseConstructor() (*Function, error) { - if _, err := p.expect(TokenLParen); err != nil { - return nil, err - } - - var params []Param - for p.peek().Type != TokenRParen && p.peek().Type != TokenEOF { - pName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - pType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - params = append(params, Param{Name: pName, Type: pType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRParen); err != nil { - return nil, err - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - - return &Function{ - Name: "constructor", - Params: params, - }, nil -} - -func (p *Parser) parseWorld() (*World, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - w := &World{Name: name} - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - if tok.Value == "export" { - p.next() - item, err := p.expectIdent() - if err == nil { - w.Exports = append(w.Exports, item) - } - if p.peek().Type == TokenSemicolon { - p.next() - } - } else if tok.Value == "import" { - p.next() - item, err := p.expectIdent() - if err == nil { - w.Imports = append(w.Imports, item) - } - if p.peek().Type == TokenSemicolon { - p.next() - } - } else { - p.next() - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return w, nil -} - -// ParseContent parses WIT content from a string. -func ParseContent(content string) (*Package, error) { - lexer := NewLexer(content) - var tokens []Token - for { - tok := lexer.NextToken() - tokens = append(tokens, tok) - if tok.Type == TokenEOF { - break - } - } - parser := NewParser(tokens) - return parser.Parse() -} - -// ParseFile parses a single .wit file. -func ParseFile(path string) (*Package, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return ParseContent(string(data)) -} - -// ParsePath parses .wit files in a file or directory, merging into a single Package AST. -func ParsePath(witPath string) (*Package, error) { - var files []string - fi, err := os.Stat(witPath) - if err != nil { - return nil, err - } - - if fi.IsDir() { - err := filepath.Walk(witPath, func(path string, info os.FileInfo, err error) error { - if err == nil && info != nil && !info.IsDir() && strings.HasSuffix(path, ".wit") { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - } else { - files = append(files, witPath) - } - - merged := &Package{} - for _, file := range files { - pkg, err := ParseFile(file) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", file, err) - } - if merged.Namespace == "" && pkg.Namespace != "" { - merged.Namespace = pkg.Namespace - } - if merged.Name == "" && pkg.Name != "" { - merged.Name = pkg.Name - } - if merged.Version == "" && pkg.Version != "" { - merged.Version = pkg.Version - } - merged.Interfaces = append(merged.Interfaces, pkg.Interfaces...) - merged.Worlds = append(merged.Worlds, pkg.Worlds...) - } - - return merged, nil -} diff --git a/tools/please_wit/ast/parser_test.go b/tools/please_wit/ast/parser_test.go deleted file mode 100644 index f4bca1e..0000000 --- a/tools/please_wit/ast/parser_test.go +++ /dev/null @@ -1,643 +0,0 @@ -package ast - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLexer_Punctuation(t *testing.T) { - input := ": ; , { } ( ) < > = @ ->" - lexer := NewLexer(input) - - expected := []struct { - tokType TokenType - val string - }{ - {TokenColon, ":"}, - {TokenSemicolon, ";"}, - {TokenComma, ","}, - {TokenLBrace, "{"}, - {TokenRBrace, "}"}, - {TokenLParen, "("}, - {TokenRParen, ")"}, - {TokenLAngle, "<"}, - {TokenRAngle, ">"}, - {TokenEquals, "="}, - {TokenAt, "@"}, - {TokenArrow, "->"}, - {TokenEOF, ""}, - } - - for i, exp := range expected { - tok := lexer.NextToken() - if tok.Type != exp.tokType { - t.Errorf("step %d: expected token type %v, got %v (%q)", i, exp.tokType, tok.Type, tok.Value) - } - if tok.Value != exp.val { - t.Errorf("step %d: expected value %q, got %q", i, exp.val, tok.Value) - } - } -} - -func TestLexer_IdentifiersAndNumbers(t *testing.T) { - input := "identifier kebab-case_name _private %interface 123 0.2.1" - lexer := NewLexer(input) - - expected := []string{ - "identifier", - "kebab-case_name", - "_private", - "%interface", - "123", - "0.2.1", - } - - for _, exp := range expected { - tok := lexer.NextToken() - if tok.Type != TokenIdent { - t.Errorf("expected TokenIdent for %q, got %v", exp, tok.Type) - } - if tok.Value != exp { - t.Errorf("expected value %q, got %q", exp, tok.Value) - } - } - - last := lexer.NextToken() - if last.Type != TokenEOF { - t.Errorf("expected EOF, got %v", last.Type) - } -} - -func TestLexer_CommentsAndWhitespace(t *testing.T) { - input := ` - // Single line comment - pkg1 - // Another comment with symbols: -> = {} - pkg2 - /* Multi - line - comment */ - pkg3 - /*** Star comment ***/ - pkg4 - ` - lexer := NewLexer(input) - - expected := []string{"pkg1", "pkg2", "pkg3", "pkg4"} - for _, exp := range expected { - tok := lexer.NextToken() - if tok.Type != TokenIdent || tok.Value != exp { - t.Errorf("expected ident %q, got %+v", exp, tok) - } - } - - if tok := lexer.NextToken(); tok.Type != TokenEOF { - t.Errorf("expected EOF, got %+v", tok) - } -} - -func TestLexer_HyphenMinus(t *testing.T) { - input := "- ->" - lexer := NewLexer(input) - - tok1 := lexer.NextToken() - if tok1.Type != TokenIdent || tok1.Value != "-" { - t.Errorf("expected '-' ident, got %+v", tok1) - } - - tok2 := lexer.NextToken() - if tok2.Type != TokenArrow || tok2.Value != "->" { - t.Errorf("expected '->' arrow, got %+v", tok2) - } -} - -func TestParser_PackageSyntax(t *testing.T) { - tests := []struct { - name string - input string - expectedNS string - expectedPkg string - expectedVer string - }{ - { - name: "full package with version", - input: "package my-org:my-pkg@1.2.3;", - expectedNS: "my-org", - expectedPkg: "my-pkg", - expectedVer: "1.2.3", - }, - { - name: "package without version", - input: "package org:pkg;", - expectedNS: "org", - expectedPkg: "pkg", - expectedVer: "", - }, - { - name: "single identifier package", - input: "package standalone;", - expectedNS: "", - expectedPkg: "standalone", - expectedVer: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pkg, err := ParseContent(tt.input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - if pkg.Namespace != tt.expectedNS { - t.Errorf("expected namespace %q, got %q", tt.expectedNS, pkg.Namespace) - } - if pkg.Name != tt.expectedPkg { - t.Errorf("expected name %q, got %q", tt.expectedPkg, pkg.Name) - } - if pkg.Version != tt.expectedVer { - t.Errorf("expected version %q, got %q", tt.expectedVer, pkg.Version) - } - }) - } -} - -func TestParser_EmptyInterface(t *testing.T) { - input := ` - package test:pkg; - interface empty-api {} - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - if pkg.Interfaces[0].Name != "empty-api" { - t.Errorf("expected empty-api, got %q", pkg.Interfaces[0].Name) - } - if len(pkg.Interfaces[0].Functions) != 0 { - t.Errorf("expected 0 functions, got %d", len(pkg.Interfaces[0].Functions)) - } -} - -func TestParser_FunctionVariants(t *testing.T) { - input := ` - package test:pkg; - - interface api { - no-args-no-ret: func(); - no-args-with-ret: func() -> string; - single-arg-no-ret: func(msg: string); - multiple-args-with-ret: func(a: s32, b: s32, flag: bool) -> result; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - if len(iface.Functions) != 4 { - t.Fatalf("expected 4 functions, got %d", len(iface.Functions)) - } - - // 1. no-args-no-ret - fn0 := iface.Functions[0] - if fn0.Name != "no-args-no-ret" || len(fn0.Params) != 0 || fn0.Results != nil { - t.Errorf("unexpected fn0: %+v", fn0) - } - - // 2. no-args-with-ret - fn1 := iface.Functions[1] - if fn1.Name != "no-args-with-ret" || len(fn1.Params) != 0 || fn1.Results == nil || fn1.Results.Name != "string" { - t.Errorf("unexpected fn1: %+v", fn1) - } - - // 3. single-arg-no-ret - fn2 := iface.Functions[2] - if fn2.Name != "single-arg-no-ret" || len(fn2.Params) != 1 || fn2.Params[0].Name != "msg" || fn2.Results != nil { - t.Errorf("unexpected fn2: %+v", fn2) - } - - // 4. multiple-args-with-ret - fn3 := iface.Functions[3] - if fn3.Name != "multiple-args-with-ret" || len(fn3.Params) != 3 { - t.Errorf("unexpected fn3: %+v", fn3) - } - if fn3.Results == nil || fn3.Results.Kind != KindResult || len(fn3.Results.TypeArgs) != 2 { - t.Errorf("unexpected fn3 results: %+v", fn3.Results) - } -} - -func TestParser_CompoundAndPrimitiveTypes(t *testing.T) { - input := ` - package test:types; - - interface type-matrix { - test-primitives: func( - p1: u8, p2: u16, p3: u32, p4: u64, - p5: s8, p6: s16, p7: s32, p8: s64, - p9: f32, p10: f64, p11: char, p12: bool, p13: string - ); - - test-bare-containers: func(l: list, o: option, r: result, tup: tuple); - - test-tuples: func(t1: tuple); - - test-result-blank: func() -> result<_, string>; - test-result-single: func() -> result; - - test-nested: func(matrix: list>, complex: list>>); - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - if len(iface.Functions) != 6 { - t.Fatalf("expected 6 functions, got %d", len(iface.Functions)) - } - - // Primitives - fnPrim := iface.Functions[0] - if len(fnPrim.Params) != 13 { - t.Fatalf("expected 13 primitive params, got %d", len(fnPrim.Params)) - } - for _, p := range fnPrim.Params { - if p.Type.Kind != KindPrimitive { - t.Errorf("expected primitive kind for %s (%s), got %v", p.Name, p.Type.Name, p.Type.Kind) - } - } - - // Bare containers - fnBare := iface.Functions[1] - if len(fnBare.Params) != 4 { - t.Fatalf("expected 4 bare params, got %d", len(fnBare.Params)) - } - if fnBare.Params[0].Type.Kind != KindList || len(fnBare.Params[0].Type.TypeArgs) != 0 { - t.Errorf("expected bare list, got %+v", fnBare.Params[0].Type) - } - if fnBare.Params[1].Type.Kind != KindOption || len(fnBare.Params[1].Type.TypeArgs) != 0 { - t.Errorf("expected bare option, got %+v", fnBare.Params[1].Type) - } - if fnBare.Params[2].Type.Kind != KindResult || len(fnBare.Params[2].Type.TypeArgs) != 0 { - t.Errorf("expected bare result, got %+v", fnBare.Params[2].Type) - } - if fnBare.Params[3].Type.Kind != KindTuple || len(fnBare.Params[3].Type.TypeArgs) != 0 { - t.Errorf("expected bare tuple, got %+v", fnBare.Params[3].Type) - } - - // Tuples - fnTup := iface.Functions[2] - tupType := fnTup.Params[0].Type - if tupType.Kind != KindTuple || len(tupType.TypeArgs) != 3 { - t.Errorf("expected tuple with 3 args, got %+v", tupType) - } - if tupType.String() != "tuple" { - t.Errorf("expected string representation 'tuple', got %q", tupType.String()) - } - - // result<_, E> - fnResBlank := iface.Functions[3] - if fnResBlank.Results == nil || fnResBlank.Results.Kind != KindResult || len(fnResBlank.Results.TypeArgs) != 2 { - t.Fatalf("expected result<_, string>, got %+v", fnResBlank.Results) - } - if fnResBlank.Results.TypeArgs[0].Name != "_" { - t.Errorf("expected first arg '_', got %q", fnResBlank.Results.TypeArgs[0].Name) - } - - // result (single argument) - fnResSingle := iface.Functions[4] - if fnResSingle.Results == nil || fnResSingle.Results.Kind != KindResult || len(fnResSingle.Results.TypeArgs) != 1 { - t.Fatalf("expected result, got %+v", fnResSingle.Results) - } - - // Nested types - fnNested := iface.Functions[5] - matParam := fnNested.Params[0].Type - if matParam.Kind != KindList || matParam.TypeArgs[0].Kind != KindList { - t.Errorf("expected nested list>, got %s", matParam.String()) - } -} - -func TestParser_RecordsAndEnumsDetailed(t *testing.T) { - input := ` - package app:models; - - interface entities { - record empty-rec {} - - record user { - id: u64, - username: string, - roles: list, - } - - enum empty-enum {} - - enum role { - admin, - viewer, - editor, - } - - type user-alias = user; - type id-alias = u64; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - - // Records - if len(iface.Records) != 2 { - t.Fatalf("expected 2 records, got %d", len(iface.Records)) - } - if iface.Records[0].Name != "empty-rec" || len(iface.Records[0].Fields) != 0 { - t.Errorf("unexpected empty record: %+v", iface.Records[0]) - } - userRec := iface.Records[1] - if userRec.Name != "user" || len(userRec.Fields) != 3 { - t.Fatalf("unexpected user record: %+v", userRec) - } - if userRec.Fields[0].Name != "id" || userRec.Fields[0].Type.Name != "u64" { - t.Errorf("field 0 mismatch: %+v", userRec.Fields[0]) - } - if userRec.Fields[2].Name != "roles" || userRec.Fields[2].Type.Kind != KindList { - t.Errorf("field 2 mismatch: %+v", userRec.Fields[2]) - } - - // Enums - if len(iface.Enums) != 2 { - t.Fatalf("expected 2 enums, got %d", len(iface.Enums)) - } - if iface.Enums[0].Name != "empty-enum" || len(iface.Enums[0].Cases) != 0 { - t.Errorf("unexpected empty enum: %+v", iface.Enums[0]) - } - roleEnum := iface.Enums[1] - if roleEnum.Name != "role" || len(roleEnum.Cases) != 3 { - t.Fatalf("unexpected role enum: %+v", roleEnum) - } - if roleEnum.Cases[0].Name != "admin" || roleEnum.Cases[1].Name != "viewer" || roleEnum.Cases[2].Name != "editor" { - t.Errorf("enum cases mismatch: %+v", roleEnum.Cases) - } - - // TypeDefs - if len(iface.TypeDefs) != 2 { - t.Fatalf("expected 2 type defs, got %d", len(iface.TypeDefs)) - } - if iface.TypeDefs[0].Name != "user-alias" || iface.TypeDefs[0].Type.Kind != KindNamed { - t.Errorf("unexpected typedef 0: %+v", iface.TypeDefs[0]) - } - if iface.TypeDefs[1].Name != "id-alias" || iface.TypeDefs[1].Type.Kind != KindPrimitive { - t.Errorf("unexpected typedef 1: %+v", iface.TypeDefs[1]) - } -} - -func TestParser_WorldImportsAndExports(t *testing.T) { - input := ` - package app:system; - - interface logger { - log: func(msg: string); - } - - interface handler { - handle: func(); - } - - world app-world { - import logger; - import config; - export handler; - export metrics; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if len(pkg.Worlds) != 1 { - t.Fatalf("expected 1 world, got %d", len(pkg.Worlds)) - } - - w := pkg.Worlds[0] - if w.Name != "app-world" { - t.Errorf("expected world name app-world, got %q", w.Name) - } - if len(w.Imports) != 2 || w.Imports[0] != "logger" || w.Imports[1] != "config" { - t.Errorf("unexpected imports: %v", w.Imports) - } - if len(w.Exports) != 2 || w.Exports[0] != "handler" || w.Exports[1] != "metrics" { - t.Errorf("unexpected exports: %v", w.Exports) - } -} - -func TestParser_TypeRefStringRepresentation(t *testing.T) { - var nilRef *TypeRef - if nilRef.String() != "unit" { - t.Errorf("expected 'unit' for nil TypeRef, got %q", nilRef.String()) - } - - listRef := &TypeRef{Kind: KindList, Name: "list", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "s32"}}} - if listRef.String() != "list" { - t.Errorf("expected 'list', got %q", listRef.String()) - } - - optRef := &TypeRef{Kind: KindOption, Name: "option", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "string"}}} - if optRef.String() != "option" { - t.Errorf("expected 'option', got %q", optRef.String()) - } - - res1Ref := &TypeRef{Kind: KindResult, Name: "result", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "s32"}}} - if res1Ref.String() != "result" { - t.Errorf("expected 'result', got %q", res1Ref.String()) - } - - res2Ref := &TypeRef{Kind: KindResult, Name: "result", TypeArgs: []*TypeRef{ - {Kind: KindPrimitive, Name: "s32"}, - {Kind: KindPrimitive, Name: "string"}, - }} - if res2Ref.String() != "result" { - t.Errorf("expected 'result', got %q", res2Ref.String()) - } - - tupRef := &TypeRef{Kind: KindTuple, Name: "tuple", TypeArgs: []*TypeRef{ - {Kind: KindPrimitive, Name: "u8"}, - {Kind: KindPrimitive, Name: "u16"}, - }} - if tupRef.String() != "tuple" { - t.Errorf("expected 'tuple', got %q", tupRef.String()) - } -} - -func TestParser_ErrorCases(t *testing.T) { - errorInputs := []struct { - name string - input string - }{ - {"missing interface name", "interface { }"}, - {"missing opening brace", "interface api }"}, - {"missing colon after func name", "interface api { my-fn func(); }"}, - {"missing func keyword", "interface api { my-fn: notfunc(); }"}, - {"unclosed param list", "interface api { my-fn: func(x: s32 ; }"}, - {"unclosed record brace", "interface api { record user { id: u64 }"}, - {"missing equals in type alias", "interface api { type my-type s32; }"}, - } - - for _, tt := range errorInputs { - t.Run(tt.name, func(t *testing.T) { - _, err := ParseContent(tt.input) - if err == nil { - t.Errorf("expected error for %q, but got nil", tt.input) - } - }) - } -} - -func TestParser_ParseFileAndPathErrors(t *testing.T) { - // Parse non-existent file - _, err := ParseFile("/non/existent/path/test.wit") - if err == nil { - t.Errorf("expected error for non-existent file, got nil") - } - - // ParsePath on non-existent path - _, err = ParsePath("/non/existent/path") - if err == nil { - t.Errorf("expected error for non-existent path, got nil") - } - - // ParsePath on directory with multiple WIT files - tmpDir := t.TempDir() - f1 := filepath.Join(tmpDir, "pkg.wit") - f2 := filepath.Join(tmpDir, "iface2.wit") - - c1 := ` - package multi:test@1.0.0; - interface one { - fn1: func(); - } - ` - c2 := ` - interface two { - fn2: func(); - } - world multi-world { - export one; - export two; - } - ` - if err := os.WriteFile(f1, []byte(c1), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(f2, []byte(c2), 0644); err != nil { - t.Fatal(err) - } - - pkg, err := ParsePath(tmpDir) - if err != nil { - t.Fatalf("ParsePath failed on multi-file directory: %v", err) - } - if pkg.FullName() != "multi:test" { - t.Errorf("expected package full name multi:test, got %q", pkg.FullName()) - } - if pkg.Version != "1.0.0" { - t.Errorf("expected version 1.0.0, got %q", pkg.Version) - } - if len(pkg.Interfaces) != 2 { - t.Errorf("expected 2 interfaces, got %d", len(pkg.Interfaces)) - } - if len(pkg.Worlds) != 1 { - t.Errorf("expected 1 world, got %d", len(pkg.Worlds)) - } - - // Single file directly to ParsePath - pkgSingle, err := ParsePath(f1) - if err != nil { - t.Fatalf("ParsePath failed on single file: %v", err) - } - if len(pkgSingle.Interfaces) != 1 || pkgSingle.Interfaces[0].Name != "one" { - t.Errorf("unexpected single file package: %+v", pkgSingle) - } - - // Malformed file in directory - badFile := filepath.Join(tmpDir, "bad.wit") - if err := os.WriteFile(badFile, []byte("interface broken {"), 0644); err != nil { - t.Fatal(err) - } - _, err = ParsePath(tmpDir) - if err == nil { - t.Errorf("expected error when directory contains malformed WIT, got nil") - } -} - -func TestParser_Resource(t *testing.T) { - content := ` - package babel:structures; - - interface disjoint-set { - resource disjoint-set { - constructor(); - make-set: func(x: s32); - find: func(x: s32) -> option; - union: func(x: s32, y: s32); - is-connected: func(x: s32, y: s32) -> bool; - static reset-all: func(); - } - - resource handle-only; - } - ` - - pkg, err := ParseContent(content) - if err != nil { - t.Fatalf("unexpected parse error: %v", err) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - - iface := pkg.Interfaces[0] - if len(iface.Resources) != 2 { - t.Fatalf("expected 2 resources, got %d", len(iface.Resources)) - } - - ds := iface.Resources[0] - if ds.Name != "disjoint-set" { - t.Errorf("expected resource name disjoint-set, got %q", ds.Name) - } - if ds.Constructor == nil { - t.Fatalf("expected constructor, got nil") - } - if len(ds.Constructor.Params) != 0 { - t.Errorf("expected 0 constructor params, got %d", len(ds.Constructor.Params)) - } - if len(ds.Methods) != 4 { - t.Fatalf("expected 4 methods, got %d", len(ds.Methods)) - } - if ds.Methods[0].Name != "make-set" || len(ds.Methods[0].Params) != 1 { - t.Errorf("unexpected method 0: %+v", ds.Methods[0]) - } - if ds.Methods[3].Name != "is-connected" || ds.Methods[3].Results.Name != "bool" { - t.Errorf("unexpected method 3: %+v", ds.Methods[3]) - } - if len(ds.Static) != 1 || ds.Static[0].Name != "reset-all" { - t.Errorf("expected 1 static method reset-all, got %+v", ds.Static) - } - - handle := iface.Resources[1] - if handle.Name != "handle-only" { - t.Errorf("expected resource handle-only, got %q", handle.Name) - } -} - diff --git a/tools/please_wit/generate/BUILD b/tools/please_wit/generate/BUILD index 6b212f9..d4c3f71 100644 --- a/tools/please_wit/generate/BUILD +++ b/tools/please_wit/generate/BUILD @@ -15,7 +15,7 @@ go_library( ], visibility = ["//tools/please_wit/..."], deps = [ - "//tools/please_wit/ast", + "//tools/common/wit/ast", ], ) @@ -33,7 +33,7 @@ go_test( ], deps = [ ":generate", - "//tools/please_wit/ast", + "//tools/common/wit/ast", ], ) diff --git a/tools/please_wit/generate/cpp.go b/tools/please_wit/generate/cpp.go index 712b0e3..f8a8613 100644 --- a/tools/please_wit/generate/cpp.go +++ b/tools/please_wit/generate/cpp.go @@ -4,7 +4,7 @@ import ( "fmt" "path/filepath" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // CppGenerator generates C++ headers and companion source files from WIT AST. diff --git a/tools/please_wit/generate/cpp_test.go b/tools/please_wit/generate/cpp_test.go index d41bf8b..87f0925 100644 --- a/tools/please_wit/generate/cpp_test.go +++ b/tools/please_wit/generate/cpp_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestCppGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/generate.go b/tools/please_wit/generate/generate.go index 60a598f..a8bebaa 100644 --- a/tools/please_wit/generate/generate.go +++ b/tools/please_wit/generate/generate.go @@ -1,14 +1,11 @@ package generate import ( - "bufio" "fmt" - "io" "os" "path/filepath" - "regexp" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) type Options struct { @@ -22,9 +19,6 @@ type Options struct { ModuleName string } -var worldRegex = regexp.MustCompile(`^\s*world\s+([a-zA-Z0-9_-]+)`) -var packageRegex = regexp.MustCompile(`(?m)^\s*package\s+([a-zA-Z0-9_:-]+);`) - // resolveWitFiles returns a slice of .wit file paths from a directory or single file path. func resolveWitFiles(witPath string) ([]string, error) { fi, err := os.Stat(witPath) @@ -49,7 +43,8 @@ func resolveWitFiles(witPath string) ([]string, error) { return files, nil } -// DiscoverWorlds scans all .wit files in a directory or file list and returns declared world names. +// DiscoverWorlds scans all .wit files in a directory or file list and returns declared world names, +// each once, in the order they are first declared. func DiscoverWorlds(witPath string) ([]string, error) { files, err := resolveWitFiles(witPath) if err != nil { @@ -59,39 +54,23 @@ func DiscoverWorlds(witPath string) ([]string, error) { var worlds []string seen := make(map[string]bool) for _, file := range files { - extracted := parseWorldsFromFile(file, seen) - worlds = append(worlds, extracted...) - } - - return worlds, nil -} - -func parseWorldsFromFile(file string, seen map[string]bool) []string { - f, err := os.Open(file) - if err != nil { - return nil - } - defer f.Close() - return parseWorldsFromReader(f, seen) -} - -func parseWorldsFromReader(r io.Reader, seen map[string]bool) []string { - var worlds []string - scanner := bufio.NewScanner(r) - for scanner.Scan() { - matches := worldRegex.FindStringSubmatch(scanner.Text()) - if len(matches) > 1 { - name := matches[1] - if !seen[name] { - seen[name] = true - worlds = append(worlds, name) + pkg, err := ast.ParseFileLenient(file) + if err != nil { + return nil, fmt.Errorf("%s: %w", file, err) + } + for _, w := range pkg.Worlds { + if !seen[w.Name] { + seen[w.Name] = true + worlds = append(worlds, w.Name) } } } - return worlds + + return worlds, nil } -// DiscoverPackage scans all .wit files in a directory or file list and returns the declared package name if found. +// DiscoverPackage scans all .wit files in a directory or file list and returns the declared package name +// ("namespace:name", without the version) if found. func DiscoverPackage(witPath string) (string, error) { files, err := resolveWitFiles(witPath) if err != nil { @@ -99,30 +78,18 @@ func DiscoverPackage(witPath string) (string, error) { } for _, file := range files { - if pkg := parsePackageFromFile(file); pkg != "" { - return pkg, nil + pkg, err := ast.ParseFileLenient(file) + if err != nil { + return "", fmt.Errorf("%s: %w", file, err) + } + if name := pkg.FullName(); name != "" { + return name, nil } } return "", nil } -func parsePackageFromFile(file string) string { - data, err := os.ReadFile(file) - if err != nil { - return "" - } - return parsePackageFromContent(data) -} - -func parsePackageFromContent(data []byte) string { - matches := packageRegex.FindSubmatch(data) - if len(matches) > 1 { - return string(matches[1]) - } - return "" -} - // splitIdentifier splits identifiers by common delimiters (_, -, :, .). func splitIdentifier(s string) []string { return strings.FieldsFunc(s, func(r rune) bool { diff --git a/tools/please_wit/generate/generate_test.go b/tools/please_wit/generate/generate_test.go index c850706..721e649 100644 --- a/tools/please_wit/generate/generate_test.go +++ b/tools/please_wit/generate/generate_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "strings" "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestDiscoverWorlds(t *testing.T) { @@ -692,24 +692,6 @@ func TestInternalHelpers(t *testing.T) { } }) - t.Run("parsePackageFromContent_various_forms", func(t *testing.T) { - cases := []struct { - content string - want string - }{ - {"package foo:bar;", "foo:bar"}, - {" package foo:bar-baz; ", "foo:bar-baz"}, - {"// package commented:out;\npackage actual:pkg;", "actual:pkg"}, - {"no package here", ""}, - } - for _, c := range cases { - got := parsePackageFromContent([]byte(c.content)) - if got != c.want { - t.Errorf("parsePackageFromContent(%q) = %q, want %q", c.content, got, c.want) - } - } - }) - t.Run("applyPackageOverride", func(t *testing.T) { pkg := &ast.Package{} applyPackageOverride(pkg, "ns:name") @@ -752,20 +734,6 @@ func TestInternalHelpers(t *testing.T) { } }) - t.Run("parseWorldsFromFile_nonexistent", func(t *testing.T) { - res := parseWorldsFromFile(filepath.Join(t.TempDir(), "missing.wit"), make(map[string]bool)) - if len(res) != 0 { - t.Errorf("expected empty result, got %v", res) - } - }) - - t.Run("parsePackageFromFile_nonexistent", func(t *testing.T) { - res := parsePackageFromFile(filepath.Join(t.TempDir(), "missing.wit")) - if res != "" { - t.Errorf("expected empty result, got %v", res) - } - }) - t.Run("writeOutputFiles_file_write_error", func(t *testing.T) { dir := t.TempDir() subDir := filepath.Join(dir, "output.txt") @@ -791,3 +759,55 @@ func TestInternalHelpers(t *testing.T) { } }) } + +// TestDiscoveryReadsWitStructure covers inputs a line-based regex scan got wrong. +func TestDiscoveryReadsWitStructure(t *testing.T) { + write := func(t *testing.T, content string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "x.wit") + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return file + } + + t.Run("versioned_package", func(t *testing.T) { + pkg, err := DiscoverPackage(write(t, "package wasi:cli@0.2.0;\nworld command {}\n")) + if err != nil || pkg != "wasi:cli" { + t.Errorf("pkg = %q, err = %v", pkg, err) + } + }) + + t.Run("commented_out_declarations_are_ignored", func(t *testing.T) { + file := write(t, "// package fake:one;\n/* world hidden {} */\npackage real:two;\n// world also-hidden {}\nworld shown {}\n") + pkg, _ := DiscoverPackage(file) + worlds, err := DiscoverWorlds(file) + if pkg != "real:two" || err != nil || len(worlds) != 1 || worlds[0] != "shown" { + t.Errorf("pkg = %q, worlds = %v, err = %v", pkg, worlds, err) + } + }) + + t.Run("two_worlds_on_one_line", func(t *testing.T) { + worlds, err := DiscoverWorlds(write(t, "package a:b;\nworld one {} world two {}\n")) + if err != nil || len(worlds) != 2 || worlds[0] != "one" || worlds[1] != "two" { + t.Errorf("worlds = %v, err = %v", worlds, err) + } + }) + + t.Run("declarations_the_ast_does_not_model", func(t *testing.T) { + src := "package a:b;\ninterface i {\n use other.{t};\n variant v { x, y(u32) }\n flags f { r, w }\n}\nworld first { use i.{t}; export i; }\nworld second {}\n" + file := write(t, src) + worlds, err := DiscoverWorlds(file) + if err != nil || len(worlds) != 2 || worlds[0] != "first" || worlds[1] != "second" { + t.Errorf("worlds = %v, err = %v", worlds, err) + } + }) + + t.Run("syntax_error_names_the_file", func(t *testing.T) { + file := write(t, "world {\n") + _, err := DiscoverWorlds(file) + if err == nil || !strings.Contains(err.Error(), file) { + t.Errorf("err = %v, want one naming %s", err, file) + } + }) +} diff --git a/tools/please_wit/generate/generator.go b/tools/please_wit/generate/generator.go index 9da2b6f..a3b81a5 100644 --- a/tools/please_wit/generate/generator.go +++ b/tools/please_wit/generate/generator.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // OutputFile represents a single generated output file. diff --git a/tools/please_wit/generate/golang.go b/tools/please_wit/generate/golang.go index 99bc26f..2c16fea 100644 --- a/tools/please_wit/generate/golang.go +++ b/tools/please_wit/generate/golang.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // GoGenerator generates Go interfaces, structs, and enums from WIT AST. diff --git a/tools/please_wit/generate/golang_test.go b/tools/please_wit/generate/golang_test.go index 091d47a..8ef0e23 100644 --- a/tools/please_wit/generate/golang_test.go +++ b/tools/please_wit/generate/golang_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestGoGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/kotlin.go b/tools/please_wit/generate/kotlin.go index 776b6da..483ec22 100644 --- a/tools/please_wit/generate/kotlin.go +++ b/tools/please_wit/generate/kotlin.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // KotlinGenerator generates idiomatic Kotlin interfaces and models from WIT AST. diff --git a/tools/please_wit/generate/kotlin_test.go b/tools/please_wit/generate/kotlin_test.go index 97705c7..7d59a87 100644 --- a/tools/please_wit/generate/kotlin_test.go +++ b/tools/please_wit/generate/kotlin_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestKotlinGenerator_MapWitType(t *testing.T) { @@ -160,7 +160,7 @@ public interface DataService { Name: "disjoint-set", Resources: []ast.Resource{ { - Name: "disjoint-set", + Name: "disjoint-set", Constructor: &ast.Function{Name: "constructor"}, Methods: []ast.Function{ { diff --git a/tools/please_wit/generate/python.go b/tools/please_wit/generate/python.go index 2aeb4f9..439c9b9 100644 --- a/tools/please_wit/generate/python.go +++ b/tools/please_wit/generate/python.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // PythonGenerator generates Python protocols, dataclasses, enums, and .pyi stubs from WIT AST. diff --git a/tools/please_wit/generate/python_test.go b/tools/please_wit/generate/python_test.go index 2fe5155..578074a 100644 --- a/tools/please_wit/generate/python_test.go +++ b/tools/please_wit/generate/python_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestPythonGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/rust.go b/tools/please_wit/generate/rust.go index a2bd2ec..45cf365 100644 --- a/tools/please_wit/generate/rust.go +++ b/tools/please_wit/generate/rust.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // RustGenerator generates Rust traits, structs, and enums from WIT AST. diff --git a/tools/please_wit/generate/rust_test.go b/tools/please_wit/generate/rust_test.go index 79d016e..c3b9ae4 100644 --- a/tools/please_wit/generate/rust_test.go +++ b/tools/please_wit/generate/rust_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestRustGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/swift.go b/tools/please_wit/generate/swift.go index 7cbef1c..397bde0 100644 --- a/tools/please_wit/generate/swift.go +++ b/tools/please_wit/generate/swift.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // SwiftGenerator generates Swift protocols, types, and module maps from WIT AST. diff --git a/tools/please_wit/generate/swift_test.go b/tools/please_wit/generate/swift_test.go index 83c51b0..f810dd6 100644 --- a/tools/please_wit/generate/swift_test.go +++ b/tools/please_wit/generate/swift_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestSwiftGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/ts.go b/tools/please_wit/generate/ts.go index f7a2539..89528f3 100644 --- a/tools/please_wit/generate/ts.go +++ b/tools/please_wit/generate/ts.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // TypeScriptGenerator generates TypeScript interface definitions from WIT AST. diff --git a/tools/please_wit/generate/ts_test.go b/tools/please_wit/generate/ts_test.go index 72f187a..91c76d1 100644 --- a/tools/please_wit/generate/ts_test.go +++ b/tools/please_wit/generate/ts_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestTypeScriptGenerator_MapWitType(t *testing.T) { From 2bedfc078cf07441414eaabf233a25951eba4249 Mon Sep 17 00:00:00 2001 From: Walter Pinto Date: Sun, 4 Oct 2026 15:19:33 +0100 Subject: [PATCH 2/2] docs(wit): format changelog entry Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44bbcf1..05f8dc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,12 @@ and this project adheres to ### Changed -- World and package discovery (`DiscoverWorlds`, `DiscoverPackage`) read the - WIT with the AST parser instead of line-based regular expressions, so - versioned packages (`package wasi:cli@0.2.0;`), commented-out declarations, - several declarations on one line and `use`, `variant` or `flags` declarations - are handled. A file that does not parse is now an error naming the file - instead of being skipped silently. +- World and package discovery (`DiscoverWorlds`, `DiscoverPackage`) read the WIT + with the AST parser instead of line-based regular expressions, so versioned + packages (`package wasi:cli@0.2.0;`), commented-out declarations, several + declarations on one line and `use`, `variant` or `flags` declarations are + handled. A file that does not parse is now an error naming the file instead of + being skipped silently. - The AST parser moved to `tools/common/wit/ast` (shared on `main`, so other language plugins can use it). @@ -24,9 +24,12 @@ and this project adheres to ### Added -- Support for WIT `resource` declarations in AST parser (`tools/please_wit/ast`). -- Support for `constructor`, instance methods, and `static` functions inside resource blocks. -- Resource interface generation across Kotlin, Swift, TypeScript, Python, Rust, Go, and C++ code generators. +- Support for WIT `resource` declarations in AST parser + (`tools/please_wit/ast`). +- Support for `constructor`, instance methods, and `static` functions inside + resource blocks. +- Resource interface generation across Kotlin, Swift, TypeScript, Python, Rust, + Go, and C++ code generators. ## [0.3.0] - 2026-09-19