diff --git a/CHANGELOG.md b/CHANGELOG.md index f95b0c0..05f8dc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- World and package discovery (`DiscoverWorlds`, `DiscoverPackage`) read the WIT + with the AST parser instead of line-based regular expressions, so versioned + packages (`package wasi:cli@0.2.0;`), commented-out declarations, several + declarations on one line and `use`, `variant` or `flags` declarations are + handled. A file that does not parse is now an error naming the file instead of + being skipped silently. +- The AST parser moved to `tools/common/wit/ast` (shared on `main`, so other + language plugins can use it). + ## [0.3.1] - 2026-09-20 ### Added -- Support for WIT `resource` declarations in AST parser (`tools/please_wit/ast`). -- Support for `constructor`, instance methods, and `static` functions inside resource blocks. -- Resource interface generation across Kotlin, Swift, TypeScript, Python, Rust, Go, and C++ code generators. +- Support for WIT `resource` declarations in AST parser + (`tools/please_wit/ast`). +- Support for `constructor`, instance methods, and `static` functions inside + resource blocks. +- Resource interface generation across Kotlin, Swift, TypeScript, Python, Rust, + Go, and C++ code generators. ## [0.3.0] - 2026-09-19 diff --git a/docs/wit/architecture.md b/docs/wit/architecture.md index 8f4d3ac..c14013e 100644 --- a/docs/wit/architecture.md +++ b/docs/wit/architecture.md @@ -30,7 +30,7 @@ access. ## 2. Standalone Go AST Parser Architecture -`please_wit` embeds a lightweight lexer and parser in `tools/please_wit/ast`: +`please_wit` embeds a lightweight lexer and parser in `tools/common/wit/ast`: ```mermaid flowchart LR diff --git a/tools/please_wit/ast/BUILD b/tools/please_wit/ast/BUILD deleted file mode 100644 index f9b4b7f..0000000 --- a/tools/please_wit/ast/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -subinclude("///go//build_defs:go") - -go_library( - name = "ast", - srcs = [ - "ast.go", - "parser.go", - ], - visibility = ["//tools/please_wit/..."], -) - -go_test( - name = "ast_test", - srcs = [ - "ast_test.go", - "parser_test.go", - ], - deps = [ - ":ast", - ], -) diff --git a/tools/please_wit/ast/ast.go b/tools/please_wit/ast/ast.go deleted file mode 100644 index 0a7b92b..0000000 --- a/tools/please_wit/ast/ast.go +++ /dev/null @@ -1,152 +0,0 @@ -package ast - -import ( - "fmt" - "strings" -) - -// TypeKind distinguishes between primitives, compound types, and user references. -type TypeKind int - -const ( - KindPrimitive TypeKind = iota - KindList - KindOption - KindResult - KindTuple - KindNamed -) - -// TypeRef represents a type reference in WIT. -type TypeRef struct { - Kind TypeKind - Name string // e.g. "s32", "string", or user-defined type name - TypeArgs []*TypeRef // For list, option, result, tuple<...> -} - -func (t *TypeRef) String() string { - if t == nil { - return "unit" - } - switch t.Kind { - case KindPrimitive, KindNamed: - return t.Name - case KindList: - if len(t.TypeArgs) > 0 { - return fmt.Sprintf("list<%s>", t.TypeArgs[0].String()) - } - return "list" - case KindOption: - if len(t.TypeArgs) > 0 { - return fmt.Sprintf("option<%s>", t.TypeArgs[0].String()) - } - return "option" - case KindResult: - if len(t.TypeArgs) == 2 { - return fmt.Sprintf("result<%s, %s>", t.TypeArgs[0].String(), t.TypeArgs[1].String()) - } else if len(t.TypeArgs) == 1 { - return fmt.Sprintf("result<%s>", t.TypeArgs[0].String()) - } - return "result" - case KindTuple: - var inner []string - for _, arg := range t.TypeArgs { - inner = append(inner, arg.String()) - } - return fmt.Sprintf("tuple<%s>", strings.Join(inner, ", ")) - } - return t.Name -} - -// Param represents a parameter in a WIT function. -type Param struct { - Name string - Type *TypeRef -} - -// Function represents a function declared in a WIT interface. -type Function struct { - Name string - Doc string - Params []Param - Results *TypeRef // nil means void / unit -} - -// Field represents a field in a WIT record. -type Field struct { - Name string - Type *TypeRef -} - -// Record represents a record structure declared in a WIT interface. -type Record struct { - Name string - Doc string - Fields []Field -} - -// EnumCase represents a single case in an enum. -type EnumCase struct { - Name string -} - -// Enum represents an enum declared in a WIT interface. -type Enum struct { - Name string - Doc string - Cases []EnumCase -} - -// TypeDef represents a type alias declared in WIT. -type TypeDef struct { - Name string - Type *TypeRef -} - -// Resource represents a resource declaration in a WIT interface. -type Resource struct { - Name string - Doc string - Constructor *Function - Methods []Function - Static []Function -} - -// Interface represents an interface block in WIT. -type Interface struct { - Name string - Doc string - Functions []Function - Records []Record - Enums []Enum - TypeDefs []TypeDef - Resources []Resource -} - -// World represents a world definition in WIT. -type World struct { - Name string - Doc string - Imports []string - Exports []string -} - -// Package represents a WIT package containing interfaces and worlds. -type Package struct { - Namespace string - Name string - Version string - Interfaces []Interface - Worlds []World -} - -// FullName returns the canonical package identifier (e.g. "namespace:name"). -func (p *Package) FullName() string { - if p.Namespace != "" && p.Name != "" { - return fmt.Sprintf("%s:%s", p.Namespace, p.Name) - } - if p.Name != "" { - return p.Name - } - return p.Namespace -} diff --git a/tools/please_wit/ast/ast_test.go b/tools/please_wit/ast/ast_test.go deleted file mode 100644 index 015f92e..0000000 --- a/tools/please_wit/ast/ast_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package ast - -import ( - "os" - "path/filepath" - "testing" -) - -func TestParseSimpleWit(t *testing.T) { - input := ` -// Package comment -package test:structures@0.1.0; - -interface two-sum { - solve: func(nums: list, target: s32) -> list; - reset: func(); -} - -world two-sum-world { - export two-sum; -} -` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if pkg.Namespace != "test" { - t.Errorf("expected namespace 'test', got %q", pkg.Namespace) - } - if pkg.Name != "structures" { - t.Errorf("expected name 'structures', got %q", pkg.Name) - } - if pkg.Version != "0.1.0" { - t.Errorf("expected version '0.1.0', got %q", pkg.Version) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - iface := pkg.Interfaces[0] - if iface.Name != "two-sum" { - t.Errorf("expected interface 'two-sum', got %q", iface.Name) - } - - if len(iface.Functions) != 2 { - t.Fatalf("expected 2 functions, got %d", len(iface.Functions)) - } - - fn1 := iface.Functions[0] - if fn1.Name != "solve" { - t.Errorf("expected fn 'solve', got %q", fn1.Name) - } - if len(fn1.Params) != 2 { - t.Fatalf("expected 2 params, got %d", len(fn1.Params)) - } - if fn1.Params[0].Name != "nums" || fn1.Params[0].Type.Kind != KindList { - t.Errorf("unexpected param 0: %+v", fn1.Params[0]) - } - if fn1.Params[1].Name != "target" || fn1.Params[1].Type.Name != "s32" { - t.Errorf("unexpected param 1: %+v", fn1.Params[1]) - } - if fn1.Results == nil || fn1.Results.Kind != KindList { - t.Errorf("unexpected results: %+v", fn1.Results) - } - - fn2 := iface.Functions[1] - if fn2.Name != "reset" || fn2.Results != nil { - t.Errorf("unexpected fn2: %+v", fn2) - } - - if len(pkg.Worlds) != 1 { - t.Fatalf("expected 1 world, got %d", len(pkg.Worlds)) - } - world := pkg.Worlds[0] - if world.Name != "two-sum-world" { - t.Errorf("expected world 'two-sum-world', got %q", world.Name) - } - if len(world.Exports) != 1 || world.Exports[0] != "two-sum" { - t.Errorf("unexpected exports: %v", world.Exports) - } -} - -func TestParseRecordsAndEnums(t *testing.T) { - input := ` -package test:types; - -interface data-types { - /* Multi line comment - about point - */ - record point { - x: f64, - y: f64, - } - - enum color { - red, - green, - blue, - } - - type point-alias = point; - - get-color: func(p: point) -> option; - calc: func(a: s32) -> result; -} -` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - iface := pkg.Interfaces[0] - - if len(iface.Records) != 1 { - t.Fatalf("expected 1 record, got %d", len(iface.Records)) - } - rec := iface.Records[0] - if rec.Name != "point" || len(rec.Fields) != 2 { - t.Errorf("unexpected record: %+v", rec) - } - - if len(iface.Enums) != 1 { - t.Fatalf("expected 1 enum, got %d", len(iface.Enums)) - } - enm := iface.Enums[0] - if enm.Name != "color" || len(enm.Cases) != 3 { - t.Errorf("unexpected enum: %+v", enm) - } - - if len(iface.TypeDefs) != 1 { - t.Fatalf("expected 1 type def, got %d", len(iface.TypeDefs)) - } - - if len(iface.Functions) != 2 { - t.Fatalf("expected 2 functions, got %d", len(iface.Functions)) - } - - fn1 := iface.Functions[0] - if fn1.Results == nil || fn1.Results.Kind != KindOption { - t.Errorf("expected option result, got %+v", fn1.Results) - } - - fn2 := iface.Functions[1] - if fn2.Results == nil || fn2.Results.Kind != KindResult { - t.Errorf("expected result result, got %+v", fn2.Results) - } -} - -func TestParsePath(t *testing.T) { - tmpDir := t.TempDir() - witFile := filepath.Join(tmpDir, "test.wit") - content := ` -package test:structures; - -interface disjoint-set { - find: func(x: s32) -> s32; -} -` - if err := os.WriteFile(witFile, []byte(content), 0644); err != nil { - t.Fatal(err) - } - - pkg, err := ParsePath(tmpDir) - if err != nil { - t.Fatalf("ParsePath failed: %v", err) - } - if pkg.FullName() != "test:structures" { - t.Errorf("expected test:structures, got %q", pkg.FullName()) - } - if len(pkg.Interfaces) != 1 || pkg.Interfaces[0].Name != "disjoint-set" { - t.Errorf("unexpected interfaces: %+v", pkg.Interfaces) - } -} diff --git a/tools/please_wit/ast/parser.go b/tools/please_wit/ast/parser.go deleted file mode 100644 index eab2399..0000000 --- a/tools/please_wit/ast/parser.go +++ /dev/null @@ -1,814 +0,0 @@ -package ast - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "unicode" -) - -// TokenType represents lexical token categories in WIT. -type TokenType int - -const ( - TokenEOF TokenType = iota - TokenIdent - TokenColon - TokenSemicolon - TokenComma - TokenLBrace - TokenRBrace - TokenLParen - TokenRParen - TokenLAngle - TokenRAngle - TokenArrow - TokenEquals - TokenAt -) - -type Token struct { - Type TokenType - Value string - Line int -} - -// Lexer tokenizes WIT source code. -type Lexer struct { - src []rune - pos int - length int - line int -} - -func NewLexer(input string) *Lexer { - src := []rune(input) - return &Lexer{ - src: src, - pos: 0, - length: len(src), - line: 1, - } -} - -func (l *Lexer) peek() rune { - if l.pos >= l.length { - return 0 - } - return l.src[l.pos] -} - -func (l *Lexer) next() rune { - if l.pos >= l.length { - return 0 - } - r := l.src[l.pos] - l.pos++ - if r == '\n' { - l.line++ - } - return r -} - -func (l *Lexer) skipWhitespaceAndComments() { - for l.pos < l.length { - r := l.peek() - if unicode.IsSpace(r) { - l.next() - continue - } - if r == '/' && l.pos+1 < l.length { - nextR := l.src[l.pos+1] - if nextR == '/' { - // Single line comment - for l.pos < l.length && l.peek() != '\n' { - l.next() - } - continue - } else if nextR == '*' { - // Multi line comment - l.next() // consume '/' - l.next() // consume '*' - for l.pos < l.length { - if l.peek() == '*' && l.pos+1 < l.length && l.src[l.pos+1] == '/' { - l.next() - l.next() - break - } - l.next() - } - continue - } - } - break - } -} - -func (l *Lexer) NextToken() Token { - l.skipWhitespaceAndComments() - - if l.pos >= l.length { - return Token{Type: TokenEOF, Value: "", Line: l.line} - } - - startLine := l.line - r := l.next() - - switch r { - case ':': - return Token{Type: TokenColon, Value: ":", Line: startLine} - case ';': - return Token{Type: TokenSemicolon, Value: ";", Line: startLine} - case ',': - return Token{Type: TokenComma, Value: ",", Line: startLine} - case '{': - return Token{Type: TokenLBrace, Value: "{", Line: startLine} - case '}': - return Token{Type: TokenRBrace, Value: "}", Line: startLine} - case '(': - return Token{Type: TokenLParen, Value: "(", Line: startLine} - case ')': - return Token{Type: TokenRParen, Value: ")", Line: startLine} - case '<': - return Token{Type: TokenLAngle, Value: "<", Line: startLine} - case '>': - return Token{Type: TokenRAngle, Value: ">", Line: startLine} - case '=': - return Token{Type: TokenEquals, Value: "=", Line: startLine} - case '@': - return Token{Type: TokenAt, Value: "@", Line: startLine} - case '-': - if l.peek() == '>' { - l.next() - return Token{Type: TokenArrow, Value: "->", Line: startLine} - } - } - - // Identifiers (including kebab-case, dots, and colons in package paths or versions) - if unicode.IsLetter(r) || r == '_' || r == '%' { - var val []rune - val = append(val, r) - for l.pos < l.length { - peeked := l.peek() - if unicode.IsLetter(peeked) || unicode.IsDigit(peeked) || peeked == '_' || peeked == '-' { - val = append(val, l.next()) - } else { - break - } - } - return Token{Type: TokenIdent, Value: string(val), Line: startLine} - } - - // Digits (e.g. in versions) - if unicode.IsDigit(r) { - var val []rune - val = append(val, r) - for l.pos < l.length { - peeked := l.peek() - if unicode.IsDigit(peeked) || peeked == '.' { - val = append(val, l.next()) - } else { - break - } - } - return Token{Type: TokenIdent, Value: string(val), Line: startLine} - } - - return Token{Type: TokenIdent, Value: string(r), Line: startLine} -} - -// Parser parses WIT tokens into an AST. -type Parser struct { - tokens []Token - pos int -} - -func NewParser(tokens []Token) *Parser { - return &Parser{tokens: tokens, pos: 0} -} - -func (p *Parser) peek() Token { - if p.pos >= len(p.tokens) { - return Token{Type: TokenEOF} - } - return p.tokens[p.pos] -} - -func (p *Parser) next() Token { - tok := p.peek() - if p.pos < len(p.tokens) { - p.pos++ - } - return tok -} - -func (p *Parser) expect(tt TokenType) (Token, error) { - tok := p.next() - if tok.Type != tt { - return tok, fmt.Errorf("line %d: expected token type %v, got %v (%q)", tok.Line, tt, tok.Type, tok.Value) - } - return tok, nil -} - -func (p *Parser) expectIdent() (string, error) { - tok := p.next() - if tok.Type != TokenIdent { - return "", fmt.Errorf("line %d: expected identifier, got %q", tok.Line, tok.Value) - } - return tok.Value, nil -} - -// Parse parses the whole WIT file into a Package AST. -func (p *Parser) Parse() (*Package, error) { - pkg := &Package{} - - for p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "package": - p.next() - if err := p.parsePackage(pkg); err != nil { - return nil, err - } - case "interface": - p.next() - iface, err := p.parseInterface() - if err != nil { - return nil, err - } - pkg.Interfaces = append(pkg.Interfaces, *iface) - case "world": - p.next() - w, err := p.parseWorld() - if err != nil { - return nil, err - } - pkg.Worlds = append(pkg.Worlds, *w) - default: - // Skip unexpected token at top level - p.next() - } - } else { - p.next() - } - } - - return pkg, nil -} - -func (p *Parser) parsePackage(pkg *Package) error { - ns, err := p.expectIdent() - if err != nil { - return err - } - pkg.Namespace = ns - - if p.peek().Type == TokenColon { - p.next() - name, err := p.expectIdent() - if err != nil { - return err - } - pkg.Name = name - } else { - pkg.Name = pkg.Namespace - pkg.Namespace = "" - } - - if p.peek().Type == TokenAt { - p.next() - ver, err := p.expectIdent() - if err == nil { - pkg.Version = ver - } - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - return nil -} - -func (p *Parser) parseInterface() (*Interface, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - iface := &Interface{Name: name} - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "record": - p.next() - rec, err := p.parseRecord() - if err != nil { - return nil, err - } - iface.Records = append(iface.Records, *rec) - case "enum": - p.next() - enm, err := p.parseEnum() - if err != nil { - return nil, err - } - iface.Enums = append(iface.Enums, *enm) - case "type": - p.next() - tdef, err := p.parseTypeDef() - if err != nil { - return nil, err - } - iface.TypeDefs = append(iface.TypeDefs, *tdef) - case "resource": - p.next() - res, err := p.parseResource() - if err != nil { - return nil, err - } - iface.Resources = append(iface.Resources, *res) - default: - // Function declaration: : func(...) [-> ...] ; - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - iface.Functions = append(iface.Functions, *fn) - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - - return iface, nil -} - -func (p *Parser) parseFunction() (*Function, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - - funcKeyword, err := p.expectIdent() - if err != nil || funcKeyword != "func" { - return nil, fmt.Errorf("expected 'func', got %q", funcKeyword) - } - - if _, err := p.expect(TokenLParen); err != nil { - return nil, err - } - - var params []Param - for p.peek().Type != TokenRParen && p.peek().Type != TokenEOF { - pName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - pType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - params = append(params, Param{Name: pName, Type: pType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRParen); err != nil { - return nil, err - } - - var resultType *TypeRef - if p.peek().Type == TokenArrow { - p.next() - rt, err := p.parseTypeRef() - if err != nil { - return nil, err - } - resultType = rt - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - - return &Function{ - Name: name, - Params: params, - Results: resultType, - }, nil -} - -func (p *Parser) parseTypeRef() (*TypeRef, error) { - tok := p.peek() - if tok.Type != TokenIdent { - return nil, fmt.Errorf("line %d: expected type name, got %q", tok.Line, tok.Value) - } - p.next() - typeName := tok.Value - - switch typeName { - case "list": - if p.peek().Type == TokenLAngle { - p.next() - inner, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - return &TypeRef{Kind: KindList, Name: "list", TypeArgs: []*TypeRef{inner}}, nil - } - return &TypeRef{Kind: KindList, Name: "list"}, nil - - case "option": - if p.peek().Type == TokenLAngle { - p.next() - inner, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - return &TypeRef{Kind: KindOption, Name: "option", TypeArgs: []*TypeRef{inner}}, nil - } - return &TypeRef{Kind: KindOption, Name: "option"}, nil - - case "result": - var typeArgs []*TypeRef - if p.peek().Type == TokenLAngle { - p.next() - // Handle result<_, E> or result or result - if p.peek().Type == TokenIdent && p.peek().Value == "_" { - p.next() - typeArgs = append(typeArgs, &TypeRef{Kind: KindPrimitive, Name: "_"}) - } else { - t1, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t1) - } - - if p.peek().Type == TokenComma { - p.next() - t2, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t2) - } - - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - } - return &TypeRef{Kind: KindResult, Name: "result", TypeArgs: typeArgs}, nil - - case "tuple": - var typeArgs []*TypeRef - if p.peek().Type == TokenLAngle { - p.next() - for p.peek().Type != TokenRAngle && p.peek().Type != TokenEOF { - t, err := p.parseTypeRef() - if err != nil { - return nil, err - } - typeArgs = append(typeArgs, t) - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - if _, err := p.expect(TokenRAngle); err != nil { - return nil, err - } - } - return &TypeRef{Kind: KindTuple, Name: "tuple", TypeArgs: typeArgs}, nil - - case "u8", "u16", "u32", "u64", "s8", "s16", "s32", "s64", "f32", "f64", "char", "bool", "string": - return &TypeRef{Kind: KindPrimitive, Name: typeName}, nil - - default: - return &TypeRef{Kind: KindNamed, Name: typeName}, nil - } -} - -func (p *Parser) parseRecord() (*Record, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - var fields []Field - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - fName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - fType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - fields = append(fields, Field{Name: fName, Type: fType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return &Record{Name: name, Fields: fields}, nil -} - -func (p *Parser) parseEnum() (*Enum, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - var cases []EnumCase - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - cName, err := p.expectIdent() - if err != nil { - return nil, err - } - cases = append(cases, EnumCase{Name: cName}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return &Enum{Name: name, Cases: cases}, nil -} - -func (p *Parser) parseTypeDef() (*TypeDef, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenEquals); err != nil { - return nil, err - } - tRef, err := p.parseTypeRef() - if err != nil { - return nil, err - } - if p.peek().Type == TokenSemicolon { - p.next() - } - return &TypeDef{Name: name, Type: tRef}, nil -} - -func (p *Parser) parseResource() (*Resource, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - - res := &Resource{Name: name} - - if p.peek().Type == TokenSemicolon { - p.next() - return res, nil - } - - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - switch tok.Value { - case "constructor": - p.next() - fn, err := p.parseConstructor() - if err != nil { - return nil, err - } - res.Constructor = fn - case "static": - p.next() - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - res.Static = append(res.Static, *fn) - default: - fn, err := p.parseFunction() - if err != nil { - return nil, err - } - res.Methods = append(res.Methods, *fn) - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - - return res, nil -} - -func (p *Parser) parseConstructor() (*Function, error) { - if _, err := p.expect(TokenLParen); err != nil { - return nil, err - } - - var params []Param - for p.peek().Type != TokenRParen && p.peek().Type != TokenEOF { - pName, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenColon); err != nil { - return nil, err - } - pType, err := p.parseTypeRef() - if err != nil { - return nil, err - } - params = append(params, Param{Name: pName, Type: pType}) - - if p.peek().Type == TokenComma { - p.next() - } else { - break - } - } - - if _, err := p.expect(TokenRParen); err != nil { - return nil, err - } - - if p.peek().Type == TokenSemicolon { - p.next() - } - - return &Function{ - Name: "constructor", - Params: params, - }, nil -} - -func (p *Parser) parseWorld() (*World, error) { - name, err := p.expectIdent() - if err != nil { - return nil, err - } - if _, err := p.expect(TokenLBrace); err != nil { - return nil, err - } - - w := &World{Name: name} - - for p.peek().Type != TokenRBrace && p.peek().Type != TokenEOF { - tok := p.peek() - if tok.Type == TokenIdent { - if tok.Value == "export" { - p.next() - item, err := p.expectIdent() - if err == nil { - w.Exports = append(w.Exports, item) - } - if p.peek().Type == TokenSemicolon { - p.next() - } - } else if tok.Value == "import" { - p.next() - item, err := p.expectIdent() - if err == nil { - w.Imports = append(w.Imports, item) - } - if p.peek().Type == TokenSemicolon { - p.next() - } - } else { - p.next() - } - } else { - p.next() - } - } - - if _, err := p.expect(TokenRBrace); err != nil { - return nil, err - } - return w, nil -} - -// ParseContent parses WIT content from a string. -func ParseContent(content string) (*Package, error) { - lexer := NewLexer(content) - var tokens []Token - for { - tok := lexer.NextToken() - tokens = append(tokens, tok) - if tok.Type == TokenEOF { - break - } - } - parser := NewParser(tokens) - return parser.Parse() -} - -// ParseFile parses a single .wit file. -func ParseFile(path string) (*Package, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return ParseContent(string(data)) -} - -// ParsePath parses .wit files in a file or directory, merging into a single Package AST. -func ParsePath(witPath string) (*Package, error) { - var files []string - fi, err := os.Stat(witPath) - if err != nil { - return nil, err - } - - if fi.IsDir() { - err := filepath.Walk(witPath, func(path string, info os.FileInfo, err error) error { - if err == nil && info != nil && !info.IsDir() && strings.HasSuffix(path, ".wit") { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, err - } - } else { - files = append(files, witPath) - } - - merged := &Package{} - for _, file := range files { - pkg, err := ParseFile(file) - if err != nil { - return nil, fmt.Errorf("failed to parse %s: %w", file, err) - } - if merged.Namespace == "" && pkg.Namespace != "" { - merged.Namespace = pkg.Namespace - } - if merged.Name == "" && pkg.Name != "" { - merged.Name = pkg.Name - } - if merged.Version == "" && pkg.Version != "" { - merged.Version = pkg.Version - } - merged.Interfaces = append(merged.Interfaces, pkg.Interfaces...) - merged.Worlds = append(merged.Worlds, pkg.Worlds...) - } - - return merged, nil -} diff --git a/tools/please_wit/ast/parser_test.go b/tools/please_wit/ast/parser_test.go deleted file mode 100644 index f4bca1e..0000000 --- a/tools/please_wit/ast/parser_test.go +++ /dev/null @@ -1,643 +0,0 @@ -package ast - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLexer_Punctuation(t *testing.T) { - input := ": ; , { } ( ) < > = @ ->" - lexer := NewLexer(input) - - expected := []struct { - tokType TokenType - val string - }{ - {TokenColon, ":"}, - {TokenSemicolon, ";"}, - {TokenComma, ","}, - {TokenLBrace, "{"}, - {TokenRBrace, "}"}, - {TokenLParen, "("}, - {TokenRParen, ")"}, - {TokenLAngle, "<"}, - {TokenRAngle, ">"}, - {TokenEquals, "="}, - {TokenAt, "@"}, - {TokenArrow, "->"}, - {TokenEOF, ""}, - } - - for i, exp := range expected { - tok := lexer.NextToken() - if tok.Type != exp.tokType { - t.Errorf("step %d: expected token type %v, got %v (%q)", i, exp.tokType, tok.Type, tok.Value) - } - if tok.Value != exp.val { - t.Errorf("step %d: expected value %q, got %q", i, exp.val, tok.Value) - } - } -} - -func TestLexer_IdentifiersAndNumbers(t *testing.T) { - input := "identifier kebab-case_name _private %interface 123 0.2.1" - lexer := NewLexer(input) - - expected := []string{ - "identifier", - "kebab-case_name", - "_private", - "%interface", - "123", - "0.2.1", - } - - for _, exp := range expected { - tok := lexer.NextToken() - if tok.Type != TokenIdent { - t.Errorf("expected TokenIdent for %q, got %v", exp, tok.Type) - } - if tok.Value != exp { - t.Errorf("expected value %q, got %q", exp, tok.Value) - } - } - - last := lexer.NextToken() - if last.Type != TokenEOF { - t.Errorf("expected EOF, got %v", last.Type) - } -} - -func TestLexer_CommentsAndWhitespace(t *testing.T) { - input := ` - // Single line comment - pkg1 - // Another comment with symbols: -> = {} - pkg2 - /* Multi - line - comment */ - pkg3 - /*** Star comment ***/ - pkg4 - ` - lexer := NewLexer(input) - - expected := []string{"pkg1", "pkg2", "pkg3", "pkg4"} - for _, exp := range expected { - tok := lexer.NextToken() - if tok.Type != TokenIdent || tok.Value != exp { - t.Errorf("expected ident %q, got %+v", exp, tok) - } - } - - if tok := lexer.NextToken(); tok.Type != TokenEOF { - t.Errorf("expected EOF, got %+v", tok) - } -} - -func TestLexer_HyphenMinus(t *testing.T) { - input := "- ->" - lexer := NewLexer(input) - - tok1 := lexer.NextToken() - if tok1.Type != TokenIdent || tok1.Value != "-" { - t.Errorf("expected '-' ident, got %+v", tok1) - } - - tok2 := lexer.NextToken() - if tok2.Type != TokenArrow || tok2.Value != "->" { - t.Errorf("expected '->' arrow, got %+v", tok2) - } -} - -func TestParser_PackageSyntax(t *testing.T) { - tests := []struct { - name string - input string - expectedNS string - expectedPkg string - expectedVer string - }{ - { - name: "full package with version", - input: "package my-org:my-pkg@1.2.3;", - expectedNS: "my-org", - expectedPkg: "my-pkg", - expectedVer: "1.2.3", - }, - { - name: "package without version", - input: "package org:pkg;", - expectedNS: "org", - expectedPkg: "pkg", - expectedVer: "", - }, - { - name: "single identifier package", - input: "package standalone;", - expectedNS: "", - expectedPkg: "standalone", - expectedVer: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pkg, err := ParseContent(tt.input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - if pkg.Namespace != tt.expectedNS { - t.Errorf("expected namespace %q, got %q", tt.expectedNS, pkg.Namespace) - } - if pkg.Name != tt.expectedPkg { - t.Errorf("expected name %q, got %q", tt.expectedPkg, pkg.Name) - } - if pkg.Version != tt.expectedVer { - t.Errorf("expected version %q, got %q", tt.expectedVer, pkg.Version) - } - }) - } -} - -func TestParser_EmptyInterface(t *testing.T) { - input := ` - package test:pkg; - interface empty-api {} - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - if pkg.Interfaces[0].Name != "empty-api" { - t.Errorf("expected empty-api, got %q", pkg.Interfaces[0].Name) - } - if len(pkg.Interfaces[0].Functions) != 0 { - t.Errorf("expected 0 functions, got %d", len(pkg.Interfaces[0].Functions)) - } -} - -func TestParser_FunctionVariants(t *testing.T) { - input := ` - package test:pkg; - - interface api { - no-args-no-ret: func(); - no-args-with-ret: func() -> string; - single-arg-no-ret: func(msg: string); - multiple-args-with-ret: func(a: s32, b: s32, flag: bool) -> result; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - if len(iface.Functions) != 4 { - t.Fatalf("expected 4 functions, got %d", len(iface.Functions)) - } - - // 1. no-args-no-ret - fn0 := iface.Functions[0] - if fn0.Name != "no-args-no-ret" || len(fn0.Params) != 0 || fn0.Results != nil { - t.Errorf("unexpected fn0: %+v", fn0) - } - - // 2. no-args-with-ret - fn1 := iface.Functions[1] - if fn1.Name != "no-args-with-ret" || len(fn1.Params) != 0 || fn1.Results == nil || fn1.Results.Name != "string" { - t.Errorf("unexpected fn1: %+v", fn1) - } - - // 3. single-arg-no-ret - fn2 := iface.Functions[2] - if fn2.Name != "single-arg-no-ret" || len(fn2.Params) != 1 || fn2.Params[0].Name != "msg" || fn2.Results != nil { - t.Errorf("unexpected fn2: %+v", fn2) - } - - // 4. multiple-args-with-ret - fn3 := iface.Functions[3] - if fn3.Name != "multiple-args-with-ret" || len(fn3.Params) != 3 { - t.Errorf("unexpected fn3: %+v", fn3) - } - if fn3.Results == nil || fn3.Results.Kind != KindResult || len(fn3.Results.TypeArgs) != 2 { - t.Errorf("unexpected fn3 results: %+v", fn3.Results) - } -} - -func TestParser_CompoundAndPrimitiveTypes(t *testing.T) { - input := ` - package test:types; - - interface type-matrix { - test-primitives: func( - p1: u8, p2: u16, p3: u32, p4: u64, - p5: s8, p6: s16, p7: s32, p8: s64, - p9: f32, p10: f64, p11: char, p12: bool, p13: string - ); - - test-bare-containers: func(l: list, o: option, r: result, tup: tuple); - - test-tuples: func(t1: tuple); - - test-result-blank: func() -> result<_, string>; - test-result-single: func() -> result; - - test-nested: func(matrix: list>, complex: list>>); - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - if len(iface.Functions) != 6 { - t.Fatalf("expected 6 functions, got %d", len(iface.Functions)) - } - - // Primitives - fnPrim := iface.Functions[0] - if len(fnPrim.Params) != 13 { - t.Fatalf("expected 13 primitive params, got %d", len(fnPrim.Params)) - } - for _, p := range fnPrim.Params { - if p.Type.Kind != KindPrimitive { - t.Errorf("expected primitive kind for %s (%s), got %v", p.Name, p.Type.Name, p.Type.Kind) - } - } - - // Bare containers - fnBare := iface.Functions[1] - if len(fnBare.Params) != 4 { - t.Fatalf("expected 4 bare params, got %d", len(fnBare.Params)) - } - if fnBare.Params[0].Type.Kind != KindList || len(fnBare.Params[0].Type.TypeArgs) != 0 { - t.Errorf("expected bare list, got %+v", fnBare.Params[0].Type) - } - if fnBare.Params[1].Type.Kind != KindOption || len(fnBare.Params[1].Type.TypeArgs) != 0 { - t.Errorf("expected bare option, got %+v", fnBare.Params[1].Type) - } - if fnBare.Params[2].Type.Kind != KindResult || len(fnBare.Params[2].Type.TypeArgs) != 0 { - t.Errorf("expected bare result, got %+v", fnBare.Params[2].Type) - } - if fnBare.Params[3].Type.Kind != KindTuple || len(fnBare.Params[3].Type.TypeArgs) != 0 { - t.Errorf("expected bare tuple, got %+v", fnBare.Params[3].Type) - } - - // Tuples - fnTup := iface.Functions[2] - tupType := fnTup.Params[0].Type - if tupType.Kind != KindTuple || len(tupType.TypeArgs) != 3 { - t.Errorf("expected tuple with 3 args, got %+v", tupType) - } - if tupType.String() != "tuple" { - t.Errorf("expected string representation 'tuple', got %q", tupType.String()) - } - - // result<_, E> - fnResBlank := iface.Functions[3] - if fnResBlank.Results == nil || fnResBlank.Results.Kind != KindResult || len(fnResBlank.Results.TypeArgs) != 2 { - t.Fatalf("expected result<_, string>, got %+v", fnResBlank.Results) - } - if fnResBlank.Results.TypeArgs[0].Name != "_" { - t.Errorf("expected first arg '_', got %q", fnResBlank.Results.TypeArgs[0].Name) - } - - // result (single argument) - fnResSingle := iface.Functions[4] - if fnResSingle.Results == nil || fnResSingle.Results.Kind != KindResult || len(fnResSingle.Results.TypeArgs) != 1 { - t.Fatalf("expected result, got %+v", fnResSingle.Results) - } - - // Nested types - fnNested := iface.Functions[5] - matParam := fnNested.Params[0].Type - if matParam.Kind != KindList || matParam.TypeArgs[0].Kind != KindList { - t.Errorf("expected nested list>, got %s", matParam.String()) - } -} - -func TestParser_RecordsAndEnumsDetailed(t *testing.T) { - input := ` - package app:models; - - interface entities { - record empty-rec {} - - record user { - id: u64, - username: string, - roles: list, - } - - enum empty-enum {} - - enum role { - admin, - viewer, - editor, - } - - type user-alias = user; - type id-alias = u64; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - iface := pkg.Interfaces[0] - - // Records - if len(iface.Records) != 2 { - t.Fatalf("expected 2 records, got %d", len(iface.Records)) - } - if iface.Records[0].Name != "empty-rec" || len(iface.Records[0].Fields) != 0 { - t.Errorf("unexpected empty record: %+v", iface.Records[0]) - } - userRec := iface.Records[1] - if userRec.Name != "user" || len(userRec.Fields) != 3 { - t.Fatalf("unexpected user record: %+v", userRec) - } - if userRec.Fields[0].Name != "id" || userRec.Fields[0].Type.Name != "u64" { - t.Errorf("field 0 mismatch: %+v", userRec.Fields[0]) - } - if userRec.Fields[2].Name != "roles" || userRec.Fields[2].Type.Kind != KindList { - t.Errorf("field 2 mismatch: %+v", userRec.Fields[2]) - } - - // Enums - if len(iface.Enums) != 2 { - t.Fatalf("expected 2 enums, got %d", len(iface.Enums)) - } - if iface.Enums[0].Name != "empty-enum" || len(iface.Enums[0].Cases) != 0 { - t.Errorf("unexpected empty enum: %+v", iface.Enums[0]) - } - roleEnum := iface.Enums[1] - if roleEnum.Name != "role" || len(roleEnum.Cases) != 3 { - t.Fatalf("unexpected role enum: %+v", roleEnum) - } - if roleEnum.Cases[0].Name != "admin" || roleEnum.Cases[1].Name != "viewer" || roleEnum.Cases[2].Name != "editor" { - t.Errorf("enum cases mismatch: %+v", roleEnum.Cases) - } - - // TypeDefs - if len(iface.TypeDefs) != 2 { - t.Fatalf("expected 2 type defs, got %d", len(iface.TypeDefs)) - } - if iface.TypeDefs[0].Name != "user-alias" || iface.TypeDefs[0].Type.Kind != KindNamed { - t.Errorf("unexpected typedef 0: %+v", iface.TypeDefs[0]) - } - if iface.TypeDefs[1].Name != "id-alias" || iface.TypeDefs[1].Type.Kind != KindPrimitive { - t.Errorf("unexpected typedef 1: %+v", iface.TypeDefs[1]) - } -} - -func TestParser_WorldImportsAndExports(t *testing.T) { - input := ` - package app:system; - - interface logger { - log: func(msg: string); - } - - interface handler { - handle: func(); - } - - world app-world { - import logger; - import config; - export handler; - export metrics; - } - ` - pkg, err := ParseContent(input) - if err != nil { - t.Fatalf("ParseContent failed: %v", err) - } - - if len(pkg.Worlds) != 1 { - t.Fatalf("expected 1 world, got %d", len(pkg.Worlds)) - } - - w := pkg.Worlds[0] - if w.Name != "app-world" { - t.Errorf("expected world name app-world, got %q", w.Name) - } - if len(w.Imports) != 2 || w.Imports[0] != "logger" || w.Imports[1] != "config" { - t.Errorf("unexpected imports: %v", w.Imports) - } - if len(w.Exports) != 2 || w.Exports[0] != "handler" || w.Exports[1] != "metrics" { - t.Errorf("unexpected exports: %v", w.Exports) - } -} - -func TestParser_TypeRefStringRepresentation(t *testing.T) { - var nilRef *TypeRef - if nilRef.String() != "unit" { - t.Errorf("expected 'unit' for nil TypeRef, got %q", nilRef.String()) - } - - listRef := &TypeRef{Kind: KindList, Name: "list", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "s32"}}} - if listRef.String() != "list" { - t.Errorf("expected 'list', got %q", listRef.String()) - } - - optRef := &TypeRef{Kind: KindOption, Name: "option", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "string"}}} - if optRef.String() != "option" { - t.Errorf("expected 'option', got %q", optRef.String()) - } - - res1Ref := &TypeRef{Kind: KindResult, Name: "result", TypeArgs: []*TypeRef{{Kind: KindPrimitive, Name: "s32"}}} - if res1Ref.String() != "result" { - t.Errorf("expected 'result', got %q", res1Ref.String()) - } - - res2Ref := &TypeRef{Kind: KindResult, Name: "result", TypeArgs: []*TypeRef{ - {Kind: KindPrimitive, Name: "s32"}, - {Kind: KindPrimitive, Name: "string"}, - }} - if res2Ref.String() != "result" { - t.Errorf("expected 'result', got %q", res2Ref.String()) - } - - tupRef := &TypeRef{Kind: KindTuple, Name: "tuple", TypeArgs: []*TypeRef{ - {Kind: KindPrimitive, Name: "u8"}, - {Kind: KindPrimitive, Name: "u16"}, - }} - if tupRef.String() != "tuple" { - t.Errorf("expected 'tuple', got %q", tupRef.String()) - } -} - -func TestParser_ErrorCases(t *testing.T) { - errorInputs := []struct { - name string - input string - }{ - {"missing interface name", "interface { }"}, - {"missing opening brace", "interface api }"}, - {"missing colon after func name", "interface api { my-fn func(); }"}, - {"missing func keyword", "interface api { my-fn: notfunc(); }"}, - {"unclosed param list", "interface api { my-fn: func(x: s32 ; }"}, - {"unclosed record brace", "interface api { record user { id: u64 }"}, - {"missing equals in type alias", "interface api { type my-type s32; }"}, - } - - for _, tt := range errorInputs { - t.Run(tt.name, func(t *testing.T) { - _, err := ParseContent(tt.input) - if err == nil { - t.Errorf("expected error for %q, but got nil", tt.input) - } - }) - } -} - -func TestParser_ParseFileAndPathErrors(t *testing.T) { - // Parse non-existent file - _, err := ParseFile("/non/existent/path/test.wit") - if err == nil { - t.Errorf("expected error for non-existent file, got nil") - } - - // ParsePath on non-existent path - _, err = ParsePath("/non/existent/path") - if err == nil { - t.Errorf("expected error for non-existent path, got nil") - } - - // ParsePath on directory with multiple WIT files - tmpDir := t.TempDir() - f1 := filepath.Join(tmpDir, "pkg.wit") - f2 := filepath.Join(tmpDir, "iface2.wit") - - c1 := ` - package multi:test@1.0.0; - interface one { - fn1: func(); - } - ` - c2 := ` - interface two { - fn2: func(); - } - world multi-world { - export one; - export two; - } - ` - if err := os.WriteFile(f1, []byte(c1), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(f2, []byte(c2), 0644); err != nil { - t.Fatal(err) - } - - pkg, err := ParsePath(tmpDir) - if err != nil { - t.Fatalf("ParsePath failed on multi-file directory: %v", err) - } - if pkg.FullName() != "multi:test" { - t.Errorf("expected package full name multi:test, got %q", pkg.FullName()) - } - if pkg.Version != "1.0.0" { - t.Errorf("expected version 1.0.0, got %q", pkg.Version) - } - if len(pkg.Interfaces) != 2 { - t.Errorf("expected 2 interfaces, got %d", len(pkg.Interfaces)) - } - if len(pkg.Worlds) != 1 { - t.Errorf("expected 1 world, got %d", len(pkg.Worlds)) - } - - // Single file directly to ParsePath - pkgSingle, err := ParsePath(f1) - if err != nil { - t.Fatalf("ParsePath failed on single file: %v", err) - } - if len(pkgSingle.Interfaces) != 1 || pkgSingle.Interfaces[0].Name != "one" { - t.Errorf("unexpected single file package: %+v", pkgSingle) - } - - // Malformed file in directory - badFile := filepath.Join(tmpDir, "bad.wit") - if err := os.WriteFile(badFile, []byte("interface broken {"), 0644); err != nil { - t.Fatal(err) - } - _, err = ParsePath(tmpDir) - if err == nil { - t.Errorf("expected error when directory contains malformed WIT, got nil") - } -} - -func TestParser_Resource(t *testing.T) { - content := ` - package babel:structures; - - interface disjoint-set { - resource disjoint-set { - constructor(); - make-set: func(x: s32); - find: func(x: s32) -> option; - union: func(x: s32, y: s32); - is-connected: func(x: s32, y: s32) -> bool; - static reset-all: func(); - } - - resource handle-only; - } - ` - - pkg, err := ParseContent(content) - if err != nil { - t.Fatalf("unexpected parse error: %v", err) - } - - if len(pkg.Interfaces) != 1 { - t.Fatalf("expected 1 interface, got %d", len(pkg.Interfaces)) - } - - iface := pkg.Interfaces[0] - if len(iface.Resources) != 2 { - t.Fatalf("expected 2 resources, got %d", len(iface.Resources)) - } - - ds := iface.Resources[0] - if ds.Name != "disjoint-set" { - t.Errorf("expected resource name disjoint-set, got %q", ds.Name) - } - if ds.Constructor == nil { - t.Fatalf("expected constructor, got nil") - } - if len(ds.Constructor.Params) != 0 { - t.Errorf("expected 0 constructor params, got %d", len(ds.Constructor.Params)) - } - if len(ds.Methods) != 4 { - t.Fatalf("expected 4 methods, got %d", len(ds.Methods)) - } - if ds.Methods[0].Name != "make-set" || len(ds.Methods[0].Params) != 1 { - t.Errorf("unexpected method 0: %+v", ds.Methods[0]) - } - if ds.Methods[3].Name != "is-connected" || ds.Methods[3].Results.Name != "bool" { - t.Errorf("unexpected method 3: %+v", ds.Methods[3]) - } - if len(ds.Static) != 1 || ds.Static[0].Name != "reset-all" { - t.Errorf("expected 1 static method reset-all, got %+v", ds.Static) - } - - handle := iface.Resources[1] - if handle.Name != "handle-only" { - t.Errorf("expected resource handle-only, got %q", handle.Name) - } -} - diff --git a/tools/please_wit/generate/BUILD b/tools/please_wit/generate/BUILD index 6b212f9..d4c3f71 100644 --- a/tools/please_wit/generate/BUILD +++ b/tools/please_wit/generate/BUILD @@ -15,7 +15,7 @@ go_library( ], visibility = ["//tools/please_wit/..."], deps = [ - "//tools/please_wit/ast", + "//tools/common/wit/ast", ], ) @@ -33,7 +33,7 @@ go_test( ], deps = [ ":generate", - "//tools/please_wit/ast", + "//tools/common/wit/ast", ], ) diff --git a/tools/please_wit/generate/cpp.go b/tools/please_wit/generate/cpp.go index 712b0e3..f8a8613 100644 --- a/tools/please_wit/generate/cpp.go +++ b/tools/please_wit/generate/cpp.go @@ -4,7 +4,7 @@ import ( "fmt" "path/filepath" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // CppGenerator generates C++ headers and companion source files from WIT AST. diff --git a/tools/please_wit/generate/cpp_test.go b/tools/please_wit/generate/cpp_test.go index d41bf8b..87f0925 100644 --- a/tools/please_wit/generate/cpp_test.go +++ b/tools/please_wit/generate/cpp_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestCppGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/generate.go b/tools/please_wit/generate/generate.go index 60a598f..a8bebaa 100644 --- a/tools/please_wit/generate/generate.go +++ b/tools/please_wit/generate/generate.go @@ -1,14 +1,11 @@ package generate import ( - "bufio" "fmt" - "io" "os" "path/filepath" - "regexp" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) type Options struct { @@ -22,9 +19,6 @@ type Options struct { ModuleName string } -var worldRegex = regexp.MustCompile(`^\s*world\s+([a-zA-Z0-9_-]+)`) -var packageRegex = regexp.MustCompile(`(?m)^\s*package\s+([a-zA-Z0-9_:-]+);`) - // resolveWitFiles returns a slice of .wit file paths from a directory or single file path. func resolveWitFiles(witPath string) ([]string, error) { fi, err := os.Stat(witPath) @@ -49,7 +43,8 @@ func resolveWitFiles(witPath string) ([]string, error) { return files, nil } -// DiscoverWorlds scans all .wit files in a directory or file list and returns declared world names. +// DiscoverWorlds scans all .wit files in a directory or file list and returns declared world names, +// each once, in the order they are first declared. func DiscoverWorlds(witPath string) ([]string, error) { files, err := resolveWitFiles(witPath) if err != nil { @@ -59,39 +54,23 @@ func DiscoverWorlds(witPath string) ([]string, error) { var worlds []string seen := make(map[string]bool) for _, file := range files { - extracted := parseWorldsFromFile(file, seen) - worlds = append(worlds, extracted...) - } - - return worlds, nil -} - -func parseWorldsFromFile(file string, seen map[string]bool) []string { - f, err := os.Open(file) - if err != nil { - return nil - } - defer f.Close() - return parseWorldsFromReader(f, seen) -} - -func parseWorldsFromReader(r io.Reader, seen map[string]bool) []string { - var worlds []string - scanner := bufio.NewScanner(r) - for scanner.Scan() { - matches := worldRegex.FindStringSubmatch(scanner.Text()) - if len(matches) > 1 { - name := matches[1] - if !seen[name] { - seen[name] = true - worlds = append(worlds, name) + pkg, err := ast.ParseFileLenient(file) + if err != nil { + return nil, fmt.Errorf("%s: %w", file, err) + } + for _, w := range pkg.Worlds { + if !seen[w.Name] { + seen[w.Name] = true + worlds = append(worlds, w.Name) } } } - return worlds + + return worlds, nil } -// DiscoverPackage scans all .wit files in a directory or file list and returns the declared package name if found. +// DiscoverPackage scans all .wit files in a directory or file list and returns the declared package name +// ("namespace:name", without the version) if found. func DiscoverPackage(witPath string) (string, error) { files, err := resolveWitFiles(witPath) if err != nil { @@ -99,30 +78,18 @@ func DiscoverPackage(witPath string) (string, error) { } for _, file := range files { - if pkg := parsePackageFromFile(file); pkg != "" { - return pkg, nil + pkg, err := ast.ParseFileLenient(file) + if err != nil { + return "", fmt.Errorf("%s: %w", file, err) + } + if name := pkg.FullName(); name != "" { + return name, nil } } return "", nil } -func parsePackageFromFile(file string) string { - data, err := os.ReadFile(file) - if err != nil { - return "" - } - return parsePackageFromContent(data) -} - -func parsePackageFromContent(data []byte) string { - matches := packageRegex.FindSubmatch(data) - if len(matches) > 1 { - return string(matches[1]) - } - return "" -} - // splitIdentifier splits identifiers by common delimiters (_, -, :, .). func splitIdentifier(s string) []string { return strings.FieldsFunc(s, func(r rune) bool { diff --git a/tools/please_wit/generate/generate_test.go b/tools/please_wit/generate/generate_test.go index c850706..721e649 100644 --- a/tools/please_wit/generate/generate_test.go +++ b/tools/please_wit/generate/generate_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "strings" "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestDiscoverWorlds(t *testing.T) { @@ -692,24 +692,6 @@ func TestInternalHelpers(t *testing.T) { } }) - t.Run("parsePackageFromContent_various_forms", func(t *testing.T) { - cases := []struct { - content string - want string - }{ - {"package foo:bar;", "foo:bar"}, - {" package foo:bar-baz; ", "foo:bar-baz"}, - {"// package commented:out;\npackage actual:pkg;", "actual:pkg"}, - {"no package here", ""}, - } - for _, c := range cases { - got := parsePackageFromContent([]byte(c.content)) - if got != c.want { - t.Errorf("parsePackageFromContent(%q) = %q, want %q", c.content, got, c.want) - } - } - }) - t.Run("applyPackageOverride", func(t *testing.T) { pkg := &ast.Package{} applyPackageOverride(pkg, "ns:name") @@ -752,20 +734,6 @@ func TestInternalHelpers(t *testing.T) { } }) - t.Run("parseWorldsFromFile_nonexistent", func(t *testing.T) { - res := parseWorldsFromFile(filepath.Join(t.TempDir(), "missing.wit"), make(map[string]bool)) - if len(res) != 0 { - t.Errorf("expected empty result, got %v", res) - } - }) - - t.Run("parsePackageFromFile_nonexistent", func(t *testing.T) { - res := parsePackageFromFile(filepath.Join(t.TempDir(), "missing.wit")) - if res != "" { - t.Errorf("expected empty result, got %v", res) - } - }) - t.Run("writeOutputFiles_file_write_error", func(t *testing.T) { dir := t.TempDir() subDir := filepath.Join(dir, "output.txt") @@ -791,3 +759,55 @@ func TestInternalHelpers(t *testing.T) { } }) } + +// TestDiscoveryReadsWitStructure covers inputs a line-based regex scan got wrong. +func TestDiscoveryReadsWitStructure(t *testing.T) { + write := func(t *testing.T, content string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "x.wit") + if err := os.WriteFile(file, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return file + } + + t.Run("versioned_package", func(t *testing.T) { + pkg, err := DiscoverPackage(write(t, "package wasi:cli@0.2.0;\nworld command {}\n")) + if err != nil || pkg != "wasi:cli" { + t.Errorf("pkg = %q, err = %v", pkg, err) + } + }) + + t.Run("commented_out_declarations_are_ignored", func(t *testing.T) { + file := write(t, "// package fake:one;\n/* world hidden {} */\npackage real:two;\n// world also-hidden {}\nworld shown {}\n") + pkg, _ := DiscoverPackage(file) + worlds, err := DiscoverWorlds(file) + if pkg != "real:two" || err != nil || len(worlds) != 1 || worlds[0] != "shown" { + t.Errorf("pkg = %q, worlds = %v, err = %v", pkg, worlds, err) + } + }) + + t.Run("two_worlds_on_one_line", func(t *testing.T) { + worlds, err := DiscoverWorlds(write(t, "package a:b;\nworld one {} world two {}\n")) + if err != nil || len(worlds) != 2 || worlds[0] != "one" || worlds[1] != "two" { + t.Errorf("worlds = %v, err = %v", worlds, err) + } + }) + + t.Run("declarations_the_ast_does_not_model", func(t *testing.T) { + src := "package a:b;\ninterface i {\n use other.{t};\n variant v { x, y(u32) }\n flags f { r, w }\n}\nworld first { use i.{t}; export i; }\nworld second {}\n" + file := write(t, src) + worlds, err := DiscoverWorlds(file) + if err != nil || len(worlds) != 2 || worlds[0] != "first" || worlds[1] != "second" { + t.Errorf("worlds = %v, err = %v", worlds, err) + } + }) + + t.Run("syntax_error_names_the_file", func(t *testing.T) { + file := write(t, "world {\n") + _, err := DiscoverWorlds(file) + if err == nil || !strings.Contains(err.Error(), file) { + t.Errorf("err = %v, want one naming %s", err, file) + } + }) +} diff --git a/tools/please_wit/generate/generator.go b/tools/please_wit/generate/generator.go index 9da2b6f..a3b81a5 100644 --- a/tools/please_wit/generate/generator.go +++ b/tools/please_wit/generate/generator.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // OutputFile represents a single generated output file. diff --git a/tools/please_wit/generate/golang.go b/tools/please_wit/generate/golang.go index 99bc26f..2c16fea 100644 --- a/tools/please_wit/generate/golang.go +++ b/tools/please_wit/generate/golang.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // GoGenerator generates Go interfaces, structs, and enums from WIT AST. diff --git a/tools/please_wit/generate/golang_test.go b/tools/please_wit/generate/golang_test.go index 091d47a..8ef0e23 100644 --- a/tools/please_wit/generate/golang_test.go +++ b/tools/please_wit/generate/golang_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestGoGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/kotlin.go b/tools/please_wit/generate/kotlin.go index 776b6da..483ec22 100644 --- a/tools/please_wit/generate/kotlin.go +++ b/tools/please_wit/generate/kotlin.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // KotlinGenerator generates idiomatic Kotlin interfaces and models from WIT AST. diff --git a/tools/please_wit/generate/kotlin_test.go b/tools/please_wit/generate/kotlin_test.go index 97705c7..7d59a87 100644 --- a/tools/please_wit/generate/kotlin_test.go +++ b/tools/please_wit/generate/kotlin_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestKotlinGenerator_MapWitType(t *testing.T) { @@ -160,7 +160,7 @@ public interface DataService { Name: "disjoint-set", Resources: []ast.Resource{ { - Name: "disjoint-set", + Name: "disjoint-set", Constructor: &ast.Function{Name: "constructor"}, Methods: []ast.Function{ { diff --git a/tools/please_wit/generate/python.go b/tools/please_wit/generate/python.go index 2aeb4f9..439c9b9 100644 --- a/tools/please_wit/generate/python.go +++ b/tools/please_wit/generate/python.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // PythonGenerator generates Python protocols, dataclasses, enums, and .pyi stubs from WIT AST. diff --git a/tools/please_wit/generate/python_test.go b/tools/please_wit/generate/python_test.go index 2fe5155..578074a 100644 --- a/tools/please_wit/generate/python_test.go +++ b/tools/please_wit/generate/python_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestPythonGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/rust.go b/tools/please_wit/generate/rust.go index a2bd2ec..45cf365 100644 --- a/tools/please_wit/generate/rust.go +++ b/tools/please_wit/generate/rust.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // RustGenerator generates Rust traits, structs, and enums from WIT AST. diff --git a/tools/please_wit/generate/rust_test.go b/tools/please_wit/generate/rust_test.go index 79d016e..c3b9ae4 100644 --- a/tools/please_wit/generate/rust_test.go +++ b/tools/please_wit/generate/rust_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestRustGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/swift.go b/tools/please_wit/generate/swift.go index 7cbef1c..397bde0 100644 --- a/tools/please_wit/generate/swift.go +++ b/tools/please_wit/generate/swift.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // SwiftGenerator generates Swift protocols, types, and module maps from WIT AST. diff --git a/tools/please_wit/generate/swift_test.go b/tools/please_wit/generate/swift_test.go index 83c51b0..f810dd6 100644 --- a/tools/please_wit/generate/swift_test.go +++ b/tools/please_wit/generate/swift_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestSwiftGenerator_MapWitType(t *testing.T) { diff --git a/tools/please_wit/generate/ts.go b/tools/please_wit/generate/ts.go index f7a2539..89528f3 100644 --- a/tools/please_wit/generate/ts.go +++ b/tools/please_wit/generate/ts.go @@ -3,7 +3,7 @@ package generate import ( "fmt" "strings" - "tools/please_wit/ast" + "tools/common/wit/ast" ) // TypeScriptGenerator generates TypeScript interface definitions from WIT AST. diff --git a/tools/please_wit/generate/ts_test.go b/tools/please_wit/generate/ts_test.go index 72f187a..91c76d1 100644 --- a/tools/please_wit/generate/ts_test.go +++ b/tools/please_wit/generate/ts_test.go @@ -2,7 +2,7 @@ package generate import ( "testing" - "tools/please_wit/ast" + "tools/common/wit/ast" ) func TestTypeScriptGenerator_MapWitType(t *testing.T) {