Staging

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88 # black formatting

.github/workflows/publish.yml

@@ -0,0 +1,27 @@
+name: "Publish"
+
+on:
+  push:
+    tags:
+      # Publish on any tag starting with a `v`, e.g., v0.1.0
+      - v*
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Install Python 3.13
+        run: uv python install 3.13
+      - name: Build
+        run: uv build
+      - name: Publish
+        run: uv publish

.github/workflows/test_publish.yml

@@ -22,8 +22,11 @@ jobs:
       - name: Install Python 3.13
         run: uv python install 3.13
       - name: Build
-        run: uv build
+        run: |
+          uv version --bump patch --bump dev=$RANDOM # random version id
+          uv build
       - name: Publish
         env:
           TEST_PYPI_SECRET_KEY: ${{ secrets.TEST_PYPI_SECRET_KEY }}
-        run: uv publish --index testpypi --token $TEST_PYPI_SECRET_KEY
+        run: |
+          uv publish --index testpypi --token $TEST_PYPI_SECRET_KEY

compilertoolkit/ast.py

@@ -101,7 +101,7 @@ def __init__(self, tokens: T):
     def set_parent(self, parent: Self):
         self.parent = parent
 
-    def walk[S](self, func: Callable[[Self], S]) -> list[S]:
+    def walk[S](self, func: Callable[["AbstractAstNode"], S]) -> list[S]:
         """Walk the syntax tree and run a function"""
         return [
             func(self),

compilertoolkit/exceptions.py

@@ -1,7 +1,7 @@
 """Compilation Exceptions.
 You pass a source position to these kind of exceptions"""
 
-from .tokens import SourcePosition
+from compilertoolkit.tokens import SourcePosition
 
 
 class CompilerError(Exception):
@@ -10,4 +10,18 @@ def __init__(self, positions: list[SourcePosition] | SourcePosition, msg: str):
         super().__init__(msg)
 
 
+class ParserError(Exception):
+    """Errors in parser construction or execution \
+    related to internal failure"""
+
+    pass
+
+
+class ParserNotFound(ParserError):
+    """Parser not found"""
+
+    def __init__(self):
+        super().__init__("Parser Not Found")
+
+
 __all__ = ["CompilerError"]

compilertoolkit/parsing.py

@@ -1,9 +1,10 @@
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Final, Generator, Self, Sequence, Type
+from collections.abc import MutableSequence
+from typing import TYPE_CHECKING, Any, Final, Generator, Self, Sequence, Type, overload
 
+from compilertoolkit.exceptions import ParserNotFound
 from compilertoolkit.tokens import TokenEnum, TokenType
 
-
 if TYPE_CHECKING:
     from compilertoolkit.ast import AbstractAstNode
 
@@ -21,17 +22,28 @@ def __init__(self):
 
     @abstractmethod
     def eval(
-        self, token: TokenEnum, index: int, precedence, pattern_precedence=None, parser=None
+        self,
+        token: TokenEnum,
+        index: int,
+        precedence,
+        pattern_precedence=None,
+        parser=None,
     ) -> bool:
         pass
 
     def __set_name__(self, owner, name):
         self.idx = owner._IDX
         owner._IDX += 1
 
-    def __get__(self, inst, owner) -> TokenEnum[T]:
+    @overload
+    def __get__(self, inst: "ParsingPattern", owner: Any) -> TokenEnum[T]: ...
+
+    @overload
+    def __get__(self, inst: Any, owner: Any) -> Self: ...
+
+    def __get__(self, inst: "ParsingPattern | Any", owner: Any) -> TokenEnum[T] | Self:
         if inst is None:
-            return self  # type: ignore
+            return self
         return inst._children[self.idx]
 
     def __set__(self, value, inst):
@@ -45,7 +57,12 @@ def __init__(self, typ: TokenType):
         self.typ = typ
 
     def eval(
-        self, token: TokenEnum, index: int, precedence, pattern_precedence=None, parser=None
+        self,
+        token: TokenEnum,
+        index: int,
+        precedence,
+        pattern_precedence=None,
+        parser=None,
     ) -> bool:
         return token.typ == self.typ
 
@@ -57,7 +74,12 @@ def __init__(self, typ: Type | tuple[Type]):
         self.typ = typ
 
     def eval(
-        self, token: TokenEnum, index: int, precedence, pattern_precedence=None, parser=None
+        self,
+        token: TokenEnum,
+        index: int,
+        precedence,
+        pattern_precedence=None,
+        parser=None,
     ) -> bool:
         return isinstance(token.typ, self.typ)
 
@@ -69,22 +91,41 @@ def __init__(self, val: Any):
         self.val = val
 
     def eval(
-        self, token: TokenEnum, index: int, precedence, pattern_precedence=None, parser=None
+        self,
+        token: TokenEnum,
+        index: int,
+        precedence,
+        pattern_precedence=None,
+        parser=None,
     ) -> bool:
         return token.value == self.val
 
 
 class ParseThenCheck(TokenPattern):
     __slots__ = "node"
 
+    node: TokenPattern
+
     def __init__(self, node: TokenPattern):
         self.node = node
 
     def eval(
-        self, token: TokenEnum, index: int, precedence, pattern_precedence=None, parser=None
+        self,
+        token: TokenEnum,
+        index: int,
+        precedence,
+        pattern_precedence=None,
+        parser=None,
     ) -> bool:
+        if parser is None:
+            raise ParserNotFound()
         # TODO: verify node we are checking is fully parsed
-        return self.node.eval(parser(self.idx, pattern_precedence)[index], precedence, pattern_precedence, parser)
+        return self.node.eval(
+            parser(self.idx, pattern_precedence)[index],
+            precedence,
+            pattern_precedence,
+            parser,
+        )
 
 
 class ParsingPatternMeta(type):
@@ -120,11 +161,13 @@ def __init_subclass__(
             if isinstance(value, TokenPattern)
         }
 
-    def __init__(self, items: list[TokenEnum]):
-        self._children = items
+    def __init__(self, items: MutableSequence[TokenEnum]):
+        self._children = list(items)
 
     @classmethod
-    def eval(cls, tokens: list[TokenEnum], index: int, precedence, parser) -> bool:
+    def eval(
+        cls, tokens: MutableSequence[TokenEnum], index: int, precedence, parser
+    ) -> bool:
         if cls._PATTERN_PRECEDENCE is not None and precedence > cls._PATTERN_PRECEDENCE:
             return False
         return all(
@@ -137,18 +180,19 @@ def __iter__(self) -> Generator[TokenEnum, None, None]:
 
     def set_parents(self, parent: "AbstractAstNode"):
         from compilertoolkit.ast import AbstractAstNode
+
         for child in self._children:
             if child.value is not None and isinstance(child.value, AbstractAstNode):
                 child.value.set_parent(parent)
 
 
-class Parser[T]:
+class Parser:
     __slots__ = ("rules", "eof")
 
     rules: list[Type[ParsingPattern]]
-    eof: Final[T]
+    eof: Final[TokenEnum[None]]
 
-    def __init__(self, EOF_token: T):
+    def __init__(self, EOF_token: TokenEnum[None]):
         self.rules = []
         self.eof = EOF_token
 
@@ -160,25 +204,33 @@ def add_rules(self, rules: list[Type[ParsingPattern]]) -> Self:
         self.rules += rules
         return self
 
-    def get_tokens(self, tokens: list[TokenEnum[Any]], start: int, end: int):
+    def get_tokens(self, tokens: MutableSequence[TokenEnum[Any]], start: int, end: int):
         if start > len(tokens):
             return [self.eof] * (end - start)
-        return tokens[start: min(end, len(tokens))] + (
+        return list(tokens[start : min(end, len(tokens))]) + (
             [self.eof] * (max(end, len(tokens)) - min(end, len(tokens)))
         )
 
     def parse(
-        self, tokens: Sequence[TokenEnum[Any]], offset: int, precedence: int
+        self, _tokens: Sequence[TokenEnum[Any]], offset: int, precedence: int
     ) -> list[TokenEnum[Any]]:
         """Notice: the tokens list will be edited"""
+        # relies on editing a singular list instance-
+        #  we must not reinstantiate the list
+        if not isinstance(_tokens, list):
+            tokens = list(_tokens)
+        else:
+            tokens = _tokens
 
         def parser(n_offset, n_precedence):
             return self.parse(tokens, offset + n_offset, n_precedence)
 
         for rule in self.rules:
             rule_tokens = self.get_tokens(tokens, offset, offset + len(rule._PATTERNS))
             if rule.eval(rule_tokens, offset, precedence, parser):
-                tok = rule._OWNER(rule(self.get_tokens(tokens, offset, offset + len(rule._PATTERNS))))
+                tok = rule._OWNER(
+                    rule(self.get_tokens(tokens, offset, offset + len(rule._PATTERNS)))
+                )
                 for _ in range(len(rule._PATTERNS) - 1):
                     del tokens[offset]
                 tokens[offset] = rule._TOKEN_TYPE(tok.position, tok)

compilertoolkit/tokens.py

@@ -28,7 +28,8 @@ class SourcePosition(NamedTuple):
     end_line: int
     """The line the token ends on"""
     end_column: int
-    """The ending index of the last character of the token on the last line of the position"""
+    """The ending index of the last character of the token \
+    on the last line of the position"""
 
     source: Source
 
@@ -110,17 +111,19 @@ def __repr__(self) -> str:
         return f"{self.typ.name}({self.position}, {repr(self.value)})"
 
 
-class Lexer[T: TokenEnum](rplyLexer):
+class Lexer[T: TokenEnum]():
     """A wrapper around the rply lexer.
     Returns your custom Token types when lexing"""
 
-    __slots__ = "_TokenType"
+    __slots__ = "_TokenType", "inner_lexer"
 
     _TokenType: Type[T]
+    inner_lexer: rplyLexer
+    '''Internal rply lexer used'''
 
     def __init__(self, rules, ignore_rules, token_type):
         self._TokenType = token_type
-        super().__init__(rules, ignore_rules)
+        self.inner_lexer = rplyLexer(rules, ignore_rules)
 
     def _fix_position(
         self, source: Source, position: rplySourcePosition, value
@@ -130,14 +133,16 @@ def _fix_position(
         )
 
     def _fix_token(self, source: Source, token: rplyToken) -> T:
+        if token.source_pos is None:
+            raise ValueError(token.source_pos)
         return self._TokenType(
             self._fix_position(source, token.source_pos, token.value),
             token.name,
             token.name.initializer(token.value),
         )
 
     def lex(self, source: Source) -> list[T]:
-        output = super().lex(source.contents)
+        output = self.inner_lexer.lex(source.contents)
         return [self._fix_token(source, token) for token in output]
 
 

mockup.py

@@ -40,7 +40,7 @@ class AstNode(AbstractAstNode):
     __slots__ = ()
 
     @abstractcompilationstep(0)
-    def analyze_types(self, ctx) -> type:
+    def analyze_types(self, ctx):
         ...
 
     @abstractcompilationstep(1)
@@ -124,10 +124,13 @@ def compile(self, ctx):
             add_rule(NumberLiteral.ParserPattern).\
             add_rule(SumNode.ParserPattern)
 
+print(lexed_data)
+
 parsed = parser.parse(lexed_data, 0, 0)
+parsed = parser.parse(parsed, 0, 0)
 
 if len(parsed) != 1:
-    raise Exception()
+    raise Exception(parsed)
 
 
 ast = parsed[0].value

pyproject.toml

@@ -50,10 +50,20 @@ Issues = "https://github.com/summersweet-software/compilertoolkit/issues"
 
 
 [dependency-groups]
-dev = ["mypy>=1.20.1", "sphinx>=9.1.0", "sphinx-autobuild>=2025.8.25"]
+dev = [
+    "black>=26.3.1",
+    "mypy>=1.20.1",
+    "sphinx>=9.1.0",
+    "sphinx-autobuild>=2025.8.25",
+]
 
 [[tool.uv.index]]
 name = "testpypi"
 url = "https://test.pypi.org/simple/"
 publish-url = "https://test.pypi.org/legacy/"
 explicit = true
+
+[tool.setuptools]
+packages = ["compilertoolkit"]
+
+[tool.black]