diff --git a/.gitignore b/.gitignore index 248d031..e70838a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ uv.lock .ropeproject node_modules mutants +AGENTS.md diff --git a/README.md b/README.md index 01eebd4..48fab90 100644 --- a/README.md +++ b/README.md @@ -168,9 +168,14 @@ def change_add(node: Add, context: Context): # <- The function takes a second a ) ``` -The context object has two main fields and one useful method: - -- `coordinate` with fields `start_line: int`, `start_column: int`, `end_line: int`, `end_column: int` and some others — identifies the current location in the code. +The frozen `Context` dataclass provides the following attributes and method: + +- `position: SourcePosition` — the node’s position in the original source. It provides: + - `coordinate` with fields `start_line: int`, `start_column: int`, `end_line: int`, `end_column: int` and some others — identifies the current syntactic location in the code; + - `source` — the complete original source passed to `Changer`; + - `node_range` — the node’s [`WhitespaceInclusivePositionProvider`](https://libcst.readthedocs.io/en/latest/metadata.html#libcst.metadata.WhitespaceInclusivePositionProvider) range, including whitespace owned by that node; + - `start_offset: int` and `end_offset: int` — the lazily computed inclusive start and exclusive end indices of that range in Python characters; + - `code_before` and `code_after` — the lazily computed source text before and after that range. - `comment` — the comment on the node’s first line, if there is one, without the leading `#`, or `None` if there is no comment. - `get_metacodes(key: Union[str, List[str]]) -> List[ParsedComment]` — a method that returns a list of parsed comments in [metacode format](https://github.com/mutating/metacode) associated with the current line of code. diff --git a/cstvis/changer.py b/cstvis/changer.py index 0a5fb3f..5ba1538 100644 --- a/cstvis/changer.py +++ b/cstvis/changer.py @@ -15,6 +15,7 @@ from cstvis.collector import Collector from cstvis.dto import Context, Coordinate +from cstvis.source_offsets import SourceOffsetResolver from cstvis.transformers.super_transformer import SuperTransformer from cstvis.visitors.bloodhound import Bloodhound from cstvis.visitors.comments_aggregator import CommentsAggregator @@ -66,12 +67,14 @@ def converter(self, function: Optional[Union[Callable[[CSTNode], CSTNode], Calla def iterate_coordinates(self) -> Generator[Coordinate, None, None]: wrapper = metadata.MetadataWrapper(self.module) - printer = Bloodhound(self.converters_by_types, self._comments_by_lines, self.filters_by_types) + node_ranges = wrapper.resolve(metadata.WhitespaceInclusivePositionProvider) + printer = Bloodhound(self.converters_by_types, self._comments_by_lines, self.filters_by_types, SourceOffsetResolver(wrapper.module, self.source, node_ranges.values())) wrapper.visit(printer) yield from printer.coordinates def apply_coordinate(self, coordinate: Coordinate) -> str: wrapper = metadata.MetadataWrapper(self.module) - modified = wrapper.visit(SuperTransformer(coordinate, self.converters_by_types, self._comments_by_lines, set())) + node_ranges = wrapper.resolve(metadata.WhitespaceInclusivePositionProvider) + modified = wrapper.visit(SuperTransformer(coordinate, self.converters_by_types, self._comments_by_lines, set(), SourceOffsetResolver(wrapper.module, self.source, node_ranges.values()))) return modified.code diff --git a/cstvis/dto.py b/cstvis/dto.py index eef4ac8..f8bb612 100644 --- a/cstvis/dto.py +++ b/cstvis/dto.py @@ -1,8 +1,11 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +from functools import cached_property from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from libcst.metadata import CodeRange from metacode import ParsedComment, parse +from printo import describe_call @dataclass @@ -15,12 +18,96 @@ class Coordinate: end_column: int converter_id: Optional[str] = None -@dataclass -class Context: + +@dataclass(frozen=True, repr=False) +class SourcePosition: + """ + Describe and lazily partition a node's span in the original source. + + ``coordinate`` is the ordinary syntactic position, while ``node_range`` is + the node's whitespace-inclusive LibCST range. The required resolver maps + that range to absolute Python-character offsets in ``source``. It is + normally shared by every position produced during one traversal, so the + first access performs one alignment pass and later positions reuse its + results. + + The range identifies a contextual source span rather than + ``Module.code_for_node()`` output, which may lose ambient indentation or + add a final newline. For the exact source slice ``node_source_span``:: + + code_before + node_source_span + code_after == source + + Offsets and the two surrounding source slices are cached independently. + """ + coordinate: Coordinate + source: str + node_range: CodeRange + offset_resolver: Callable[[CodeRange], Tuple[int, int]] = field(repr=False, compare=False) + + @cached_property + def start_offset(self) -> int: + """Lazily resolve and cache the node span's starting source index.""" + return self.offset_resolver(self.node_range)[0] + + @cached_property + def end_offset(self) -> int: + """Lazily resolve and cache the node span's ending source index.""" + return self.offset_resolver(self.node_range)[1] + + @cached_property + def code_before(self) -> str: + """Lazily slice and cache the exact source prefix before node_range.""" + return self.source[:self.start_offset] + + @cached_property + def code_after(self) -> str: + """Lazily slice and cache the exact source suffix after node_range.""" + return self.source[self.end_offset:] + + def __repr__(self) -> str: + """Describe public source data and lazy offsets with an item limit of 80.""" + return describe_call( + type(self), + [], + { + 'coordinate': self.coordinate, + 'source': self.source, + 'node_range': self.node_range, + 'start_offset': self.start_offset, + 'end_offset': self.end_offset, + }, + item_limit=80, + ) + + +@dataclass(frozen=True, repr=False) +class Context: + """ + Describe a callback invocation and its position in the original source. + + Source-related data and lazy fragments are exposed through ``position``. + The dataclass is frozen, although a dictionary supplied as ``meta`` remains + mutable. + """ + + position: SourcePosition comment: Optional[str] meta: Optional[Dict[str, Any]] = None + def __repr__(self) -> str: + """Describe public callback data with an item limit of 80.""" + return describe_call( + type(self), + [], + { + 'position': self.position, + 'comment': self.comment, + 'meta': self.meta, + }, + item_limit=80, + ) + def get_metacodes(self, key: Union[str, List[str]]) -> List[ParsedComment]: if self.comment is None: return [] diff --git a/cstvis/source_offsets.py b/cstvis/source_offsets.py new file mode 100644 index 0000000..9720d70 --- /dev/null +++ b/cstvis/source_offsets.py @@ -0,0 +1,101 @@ +from typing import Dict, Iterable, Optional, Set, Tuple + +from libcst import Module +from libcst._nodes.internal import CodegenState +from libcst.metadata import CodePosition, CodeRange + + +class _SourceOffsetCodegenState(CodegenState): + """ + Align LibCST code-generation positions with the original source string. + + Generated positions can diverge from original-source offsets when LibCST + omits source text such as an initial BOM, form-feed prefixes, or explicit + line continuations. Token-by-token alignment preserves those characters as + well as Unicode and mixed LF, CRLF, and CR line endings in the resulting + source partition. + """ + + def __init__(self, module: Module, source: str, target_positions: Set[CodePosition]) -> None: + super().__init__(default_indent=module.default_indent, default_newline=module.default_newline) + self.source = source + self.source_offset = int(source.startswith('\ufeff')) + self.line = 1 + self.column = 0 + self.target_positions = target_positions + self.right_position_offsets: Dict[CodePosition, int] = {} + + def _record_right_position(self, source_offset: int) -> None: + position = CodePosition(self.line, self.column) + if position in self.target_positions: + self.right_position_offsets[position] = source_offset + + def _consume_source_token(self, value: str, search_forward: bool) -> Optional[int]: + if self.source.startswith(value, self.source_offset): + match_start = self.source_offset + elif search_forward: + match_start = self.source.find(value, self.source_offset) + else: + match_start = -1 + + if match_start >= 0: + self.source_offset = match_start + len(value) + return match_start + return None + + def _add_generated_token(self, value: str, search_forward: bool) -> None: + source_start = self._consume_source_token(value, search_forward) + aligned_source_offset = self.source_offset if source_start is None else source_start + self._record_right_position(aligned_source_offset) + + cursor = 0 + while cursor < len(value): + if value[cursor] == '\r' and cursor + 1 < len(value) and value[cursor + 1] == '\n': + cursor += 2 + self.line += 1 + self.column = 0 + elif value[cursor] in {'\r', '\n'}: + cursor += 1 + self.line += 1 + self.column = 0 + else: + cursor += 1 + self.column += 1 + + position_source_offset = self.source_offset if source_start is None else source_start + cursor + self._record_right_position(position_source_offset) + + def add_indent_tokens(self) -> None: + for token in self.indent_tokens: + self._add_generated_token(token, search_forward=False) + self.tokens.extend(self.indent_tokens) + + def add_token(self, value: str) -> None: + self._add_generated_token(value, search_forward=True) + self.tokens.append(value) + + +class SourceOffsetResolver: + """ + Lazily map registered whitespace-inclusive ranges to source offsets. + + The first requested range triggers one shared code-generation pass for all + positions registered at construction. Later ranges reuse the cached + character offsets. Alignment reads the original source directly without + copying it in full or splitting it into lines. + """ + + def __init__(self, module: Module, source: str, node_ranges: Iterable[CodeRange]) -> None: + self.module = module + self.source = source + self.target_positions = {position for node_range in node_ranges for position in (node_range.start, node_range.end)} + self._offsets: Dict[CodePosition, int] = {} + + def __call__(self, node_range: CodeRange) -> Tuple[int, int]: + """Return character offsets for a range registered at construction.""" + if not self._offsets: + state = _SourceOffsetCodegenState(self.module, self.source, self.target_positions) + self.module._codegen(state) + state._record_right_position(state.source_offset) + self._offsets.update(state.right_position_offsets) + return self._offsets[node_range.start], self._offsets[node_range.end] diff --git a/cstvis/transformers/super_transformer.py b/cstvis/transformers/super_transformer.py index f95bdb4..bc3d715 100644 --- a/cstvis/transformers/super_transformer.py +++ b/cstvis/transformers/super_transformer.py @@ -9,7 +9,8 @@ leave, ) -from cstvis.dto import Context, Coordinate +from cstvis.dto import Context, Coordinate, SourcePosition +from cstvis.source_offsets import SourceOffsetResolver from cstvis.wrapper import CallableWrapper @@ -34,7 +35,15 @@ def leave_all(function: Callable[[Any, CSTNode, CSTNode], CSTNode]) -> Callable[ class SuperTransformer(MatcherDecoratableTransformer): - METADATA_DEPENDENCIES = (metadata.PositionProvider,) + """ + Apply one conversion with positions from the original node. + + Public coordinates use ``PositionProvider``; contextual ranges use the + additional ``WhitespaceInclusivePositionProvider`` metadata pass. Reading + ``Context.position`` later resolves the original node's source offsets. + """ + + METADATA_DEPENDENCIES = (metadata.PositionProvider, metadata.WhitespaceInclusivePositionProvider) def __init__( self, @@ -42,11 +51,13 @@ def __init__( nodes_mapping: Dict[Type[CSTNode], List[CallableWrapper[CSTNode]]], comments: Dict[int, str], nodes_ids: Set[int], + source_offsets: SourceOffsetResolver, ): self.target_coordinate = target_coordinate self.nodes_mapping = nodes_mapping self.comments = comments self.nodes_ids = nodes_ids + self.source_offsets = source_offsets super().__init__() @@ -56,6 +67,10 @@ def leave(self, original_node, updated_node): # type: ignore[no-untyped-def] return updated_node self.nodes_ids.add(id(original_node)) + converters = self.nodes_mapping.get(type(original_node), []) + self.nodes_mapping.get(CSTNode, []) # type: ignore[type-abstract] + if not converters: + return updated_node + position = self.get_metadata(metadata.PositionProvider, original_node) coordinate = Coordinate( file=None, @@ -74,11 +89,13 @@ def leave(self, original_node, updated_node): # type: ignore[no-untyped-def] end_column=self.target_coordinate.end_column, ) - converters = self.nodes_mapping.get(type(original_node), []) + self.nodes_mapping.get(CSTNode, []) # type: ignore[type-abstract] - - if coordinate == target_coordinate_without_converter_id and converters: - context = Context(coordinate, self.comments.get(coordinate.start_line)) + if coordinate == target_coordinate_without_converter_id: for converter in converters: # pragma: no branch if converter.get_function_id() == self.target_coordinate.converter_id: + node_range = self.get_metadata(metadata.WhitespaceInclusivePositionProvider, original_node) + context = Context( + SourcePosition(coordinate, self.source_offsets.source, node_range, self.source_offsets), + self.comments.get(coordinate.start_line), + ) return converter(updated_node, context) return updated_node diff --git a/cstvis/visitors/bloodhound.py b/cstvis/visitors/bloodhound.py index bd3e9ca..d272f9e 100644 --- a/cstvis/visitors/bloodhound.py +++ b/cstvis/visitors/bloodhound.py @@ -1,27 +1,42 @@ -from copy import deepcopy +from dataclasses import replace from typing import Dict, List, Type from libcst import CSTNode, CSTVisitor, metadata -from cstvis.dto import Context, Coordinate +from cstvis.dto import Context, Coordinate, SourcePosition +from cstvis.source_offsets import SourceOffsetResolver from cstvis.wrapper import CallableWrapper class Bloodhound(CSTVisitor): - METADATA_DEPENDENCIES = (metadata.PositionProvider,) + """ + Discover coordinates and source ranges for registered conversions. + + Public coordinates use ``PositionProvider``. Contextual node ranges use + ``WhitespaceInclusivePositionProvider``, which requires an additional + metadata pass and includes whitespace owned by each node. + """ + + METADATA_DEPENDENCIES = (metadata.PositionProvider, metadata.WhitespaceInclusivePositionProvider) def __init__( self, nodes_mapping: Dict[Type[CSTNode], List[CallableWrapper[CSTNode]]], comments: Dict[int, str], filters: Dict[Type[CSTNode], List[CallableWrapper[bool]]], + source_offsets: SourceOffsetResolver, ) -> None: self.coordinates: List[Coordinate] = [] self.nodes_mapping = nodes_mapping self.comments = comments self.filters = filters + self.source_offsets = source_offsets def on_visit(self, node: CSTNode) -> bool: + converters = self.nodes_mapping.get(type(node), []) + self.nodes_mapping.get(CSTNode, []) # type: ignore[type-abstract] + if not converters: + return True + position = self.get_metadata(metadata.PositionProvider, node) coordinate = Coordinate( file=None, @@ -32,20 +47,17 @@ def on_visit(self, node: CSTNode) -> bool: end_column=position.end.column, ) - converters = self.nodes_mapping.get(type(node), []) + self.nodes_mapping.get(CSTNode, []) # type: ignore[type-abstract] - - if converters: - filters = self.filters.get(type(node), []) + self.filters.get(CSTNode, []) # type: ignore[type-abstract] - context = Context(coordinate, self.comments.get(coordinate.start_line)) - if filters: - for filter_function in filters: - if not filter_function(node, context): - return True - for converter_id in set([x.get_function_id() for x in converters]): - emitting_coordinate = deepcopy(coordinate) - emitting_coordinate.converter_id = converter_id - self.coordinates.append( - emitting_coordinate, - ) + filters = self.filters.get(type(node), []) + self.filters.get(CSTNode, []) # type: ignore[type-abstract] + if filters: + node_range = self.get_metadata(metadata.WhitespaceInclusivePositionProvider, node) + context = Context( + SourcePosition(coordinate, self.source_offsets.source, node_range, self.source_offsets), + self.comments.get(coordinate.start_line), + ) + for filter_function in filters: + if not filter_function(node, context): + return True + for converter_id in {converter.get_function_id() for converter in converters}: + self.coordinates.append(replace(coordinate, converter_id=converter_id)) return True diff --git a/cstvis/wrapper.py b/cstvis/wrapper.py index f44d2a1..14d6a9a 100644 --- a/cstvis/wrapper.py +++ b/cstvis/wrapper.py @@ -1,3 +1,4 @@ +from dataclasses import replace from typing import ( Any, Callable, @@ -42,8 +43,8 @@ def __init__(self, function: Union[Callable[[CSTNode], FilterOrConverterReturnVa def __call__(self, node: CSTNode, context: Context) -> FilterOrConverterReturnValue: if PossibleCallMatcher('..').match(self.function): - context.meta = self.meta.copy() if isinstance(self.meta, dict) else None - return self.function(node, context) # type: ignore[call-arg] + callback_context = replace(context, meta=self.meta.copy() if isinstance(self.meta, dict) else None) + return self.function(node, callback_context) # type: ignore[call-arg] return self.function(node) # type: ignore[call-arg] diff --git a/docs/plans/1.md b/docs/plans/1.md new file mode 100644 index 0000000..a139d9c --- /dev/null +++ b/docs/plans/1.md @@ -0,0 +1,184 @@ +# Ленивые `Context.position.code_before` и `Context.position.code_after` + +## Summary + +Добавить в `Context` обязательный замороженный `SourcePosition` с независимо и +лениво вычисляемыми свойствами: + +```python +context.position.code_before: str +context.position.code_after: str +``` + +Текст ноды определяется как точный contextual source-span по `WhitespaceInclusivePositionProvider`: + +```python +context.position.code_before + node_source_span + context.position.code_after == context.position.source +``` + +Первым изменением репозитория сохранить этот план в `docs/plans/1.md`. + +## Публичный API + +Обратную совместимость старого конструктора не сохранять: + +```python +@dataclass(frozen=True, repr=False) +class SourcePosition: + coordinate: Coordinate + source: str + node_range: CodeRange + offset_resolver: Callable[[CodeRange], Tuple[int, int]] = field( + repr=False, + compare=False, + ) + + +@dataclass(frozen=True, repr=False) +class Context: + position: SourcePosition + comment: Optional[str] + meta: Optional[Dict[str, Any]] = None +``` + +- `position` обязателен; `coordinate`, `source`, `node_range`, offsets и fragments доступны без прокси через `context.position`. +- Resolver обязателен при создании `SourcePosition`, исключён из repr/equality и обычно разделяется всеми позициями одного traversal. +- `source` и `node_range` участвуют в equality `SourcePosition`; derived cache — нет. +- Не использовать `slots`, чтобы `cached_property` мог сохранять результат в `__dict__`. +- `start_offset`, `end_offset`, `code_before` и `code_after` реализовать независимыми `cached_property` в `SourcePosition`. +- Frozen dataclass исключает устаревание кэша после переприсваивания source/range. +- Заморозка неглубокая: содержимое словаря `meta` остаётся изменяемым. +- Для `repr` использовать `printo.describe_call`, а не `reprlib` и не ручную конкатенацию. +- Передавать поля как keyword arguments в порядке конструктора и устанавливать `item_limit=80`. Ограничение применяется средствами `printo` к представлению каждого поля. +- `coordinate` сохраняет существующую семантику, но переносится в `SourcePosition`; `comment`, `meta` и `get_metacodes()` остаются в `Context`. + +Установленная минимальная версия `printo==0.0.29` поддерживает `item_limit` в `describe_call`, но не в декораторе `repred`, поэтому для Context и SourcePosition нужен собственный `__repr__`, делегирующий форматирование в `describe_call`. + +## Передача `meta` + +Текущую мутацию `context.meta` в `CallableWrapper` заменить на frozen-копию: + +```python +callback_context = dataclasses.replace( + context, + meta=self.meta.copy() if isinstance(self.meta, dict) else None, +) +``` + +- Каждый двухаргументный callback получает отдельный Context и свежую копию wrapper meta. +- `dataclasses.replace` сохраняет тот же `SourcePosition`, включая resolver и уже вычисленные кэши. +- Мутация `meta` внутри одного callback не влияет на следующие вызовы, decorator dict, wrapper или исходный Context. +- При `wrapper.meta is None` callback получает `context.meta is None`, даже если у базового Context meta была непустой. +- Для одноаргументного callback поведение остаётся прежним. +- Не фиксировать тестами применение именно `dataclasses.replace` или identity передаваемого Context — это детали реализации. + +## Границы и offsets + +- `PositionProvider` продолжает формировать публичный `position.coordinate`. +- `WhitespaceInclusivePositionProvider` формирует обязательный `node_range`. +- `Bloodhound` и `SuperTransformer` получают исходный `Changer.source` и оба metadata ranges. +- В трансформере ranges брать у `original_node`. +- Все внутренние вызовы создают `SourcePosition(source, node_range, shared_resolver)` и передают его в `Context`. + +Публичного whitespace-inclusive character-offset provider в LibCST нет: + +- `ByteSpanPositionProvider` возвращает готовые байтовые offsets, но исключает owned whitespace; +- `WhitespaceInclusivePositionProvider` возвращает нужные границы только как `line`/`column`; +- `Module.code_for_node()` и поиск текста не дают универсально корректных границ. + +Добавить внутреннее преобразование `CodePosition` в абсолютный символьный индекс, которое: + +- поддерживает `\n`, `\r\n`, одиночный `\r` и их смешение; +- считает Python-символы, а не UTF-8-байты; +- работает для `line > 1` после Unicode и разных newline; +- поддерживает виртуальную EOF-позицию при отсутствии финального newline; +- поддерживает `start == end`; +- не копирует весь source и не строит список всех строк; +- учитывает начальные символы, которые LibCST исключает из codegen-позиций: BOM `U+FEFF` и валидный leading form-feed prefix; +- вызывается только при чтении соответствующего cached property. + +## Правила тестирования + +- Общие test-helper функции для проверки разбиения не создавать. +- Exact-ассерты дублировать непосредственно в каждом correctness-тесте. +- Ожидаемый node span задавать строковым литералом, а не вычислять production-helper’ом или `Module.code_for_node()`. +- Каждый correctness-тест напрямую проверяет: + +```python +assert context.position.code_before == expected_before +assert context.position.node_range == expected_range +assert context.position.code_after == expected_after +assert ( + context.position.code_before + + expected_node_source_span + + context.position.code_after + == context.position.source +) +``` + +- Каждый новый тест получает докстрингу с описанием контракта и значимых деталей сценария. +- Ленивость доказывать наблюдением реальных операций `source[...]`, а не только вызовов offset-helper. +- Локальный инструментальный класс внутри отдельного теста допустим; общим helper или fixture его не делать. + +## Подробный тестовый план + +Все перечисленные тесты новые; существующие тесты изменять не требуется. + +| Название теста | Суть | Подготовка | Конкретные проверки | +|---|---|---|---| +| `test_context_requires_source_position_with_resolver` | Новый конструктор требует полный source context. | Создать Coordinate/CodeRange/resolver; вызвать полные и неполные конструкторы. | Context требует position; SourcePosition требует source/range/resolver; поля доступны публично. | +| `test_context_is_frozen` | Source и range нельзя заменить после создания. | Попытаться присвоить новые source и node_range. | Оба присваивания дают `FrozenInstanceError`; значения сохраняются. | +| `test_context_equality_includes_fields_but_ignores_derived_caches` | Публичные данные входят в equality, resolver/cache — нет. | Создать равные Context с разными resolver и варианты с другим source/range; вычислить properties только у одного. | Равенство не меняется от resolver и кэша; другой source или range делает объекты неравными. | +| `test_context_repr_uses_printo_item_limit` | Repr делегируется `printo` и ограничивает большие поля. | Создать Context с коротким и длинным source; обернуть `describe_call` spy с сохранением поведения. | `describe_call` вызван с `item_limit=80`; короткий source виден полностью; длинный сокращён; полный длинный source отсутствует; порядок полей корректен. | +| `test_callable_wrapper_passes_meta_without_mutating_frozen_context` | Wrapper сохраняет публичную meta-семантику без мутации. | Базовый Context с собственной meta и wrapper с другой meta. | Callback видит wrapper meta; базовый Context и decorator dict не изменены; callback meta не алиасит wrapper meta. | +| `test_callable_wrapper_creates_fresh_meta_for_each_invocation` | Повторные вызовы wrapper не разделяют mutable meta. | Вызвать один wrapper дважды; в первом callback мутировать meta. | Второй вызов получает исходное значение; словари первого и второго вызовов различны; wrapper/decorator/base meta не изменены. | +| `test_callable_wrapper_with_no_meta_passes_none` | `meta=None` продолжает перекрывать meta базового Context. | Передать wrapper без meta базовый Context с непустой meta. | Callback получает `context.meta is None`; исходный Context сохраняет свою meta. | +| `test_source_position_offsets_and_fragments_are_lazy_and_cached` | Проверяется реальная материализация offsets и строковых срезов. | Внутри теста определить `TrackingString(str)` со счётчиком операций; создать SourcePosition для непустого среднего диапазона. | Конструктор, equality и старый API не вычисляют offsets/срезы; offsets и fragments кэшируются независимо. | +| `test_filter_context_exposes_whitespace_inclusive_fragments` | Filter получает inclusive range при неизменном coordinate. | `'left = 1 +\t2 # tail\r\nright = 3\n'`, identity converter и filter для Add. | Coordinate покрывает `+`; range — `' +\t'`; exact before/after и полный инвариант. | +| `test_converter_context_exposes_original_fragments` | Converter Context описывает исходную ноду. | Add заменить на Subtract и сохранить Context. | Exact parts восстанавливают исходник с `+`; результат применения содержит `-`. | +| `test_context_pipeline_does_not_materialize_unread_fragments` | Оба traversal-пути не создают срезы без чтения properties. | Параметризовать filter/converter; использовать TrackingString, обнулить parser baseline; callbacks читают только coordinate. | После iterate/apply новых fragment slices нет; функциональный результат корректен. | +| `test_node_only_callbacks_do_not_materialize_context_fragments` | Callback без Context также не вызывает eager fragment slicing. | Одноаргументные filter/converter с TrackingString и parser baseline. | После полного pipeline fragment-срезов нет; callbacks работают штатно. | +| `test_context_pipeline_materializes_only_requested_fragment` | Независимая ленивость работает в обоих lifecycle-путях. | Параметризовать filter/converter × before/after; дважды прочитать только выбранное свойство. | Создан ровно один соответствующий slice; противоположный не создан; значение точное. | +| `test_context_fragments_can_be_read_after_traversal` | Context остаётся самодостаточным после traversal. | Сохранить Context без чтения, завершить filter/converter traversal, затем прочитать properties. | До чтения slices нет; после traversal exact before/range/after и round-trip корректны. | +| `test_code_fragments_cover_single_node_and_file_boundaries` | Начало и конец файла дают пустые внешние части. | Integer в `'1'`, первая Name и последний Integer в `'x = 1'`. | Для каждого случая exact range/before/after и полный инвариант. | +| `test_code_fragments_partition_zero_width_parameters` | Zero-width range корректно делит непустой source. | Source `'def f():\n pass\n'`; выбрать пустой `Parameters` с range `(1,6)–(1,6)`. | `before == 'def f('`; span `''`; `after == '):\n pass\n'`; start равен end; round-trip точен. | +| `test_code_fragments_handle_later_statement_without_final_newline` | Виртуальный EOF работает не только на первой строке. | Source `'head = 0\r\nx = 1'`; выбрать последнюю `SimpleStatementLine`. | Before содержит первую CRLF-строку; span `'x = 1'`; after пуст; отсутствующий newline не добавлен. | +| `test_code_fragments_preserve_semicolon_owned_trivia` | `; ` принадлежит первой statement-ноде. | `'x=1; y=2\n'`, первый Assign. | Exact before `''`, span `'x=1; '`, after `'y=2\n'`, range и round-trip. | +| `test_code_fragments_preserve_decorators_and_surrounding_comments` | FunctionDef включает декораторы, но не внешние комментарии. | Header, `@dec`, функция и footer. | Exact многострочные before/span/after, range и round-trip. | +| `test_code_fragments_preserve_nested_indentation_and_comments` | Вложенная statement сохраняет contextual indentation. | `if` с leading comment, multiline assignment, inline comment и footer. | Exact indentation/comments в каждой части, range и round-trip. | +| `test_code_fragments_preserve_multiline_parenthesized_expression` | Скобки и continuation whitespace входят в span. | Выбрать BinaryOperation внутри multiline assignment. | Exact span от `(` до `)`, before/after, range и round-trip. | +| `test_code_fragments_select_correct_repeated_occurrence` | Повторяющийся код разделяется по metadata range. | Две одинаковые строки `'foo = foo + foo\n'`; оба Add. | Span обоих равен `' + '`, before/after различаются ожидаемо; каждый round-trip точен. | +| `test_code_fragments_resolve_node_after_mixed_newlines` | Offset для later-line node учитывает CRLF, bare CR и LF. | `'first=0\r\nsecond=1\rthird = 2 +\t3\nlast=4'`; выбрать Add на третьей строке. | Before содержит точную последовательность CRLF/CR; span `' +\t'`; after `'3\nlast=4'`; range `(3,9)–(3,13)`; round-trip точен. | +| `test_code_fragments_preserve_unicode_before_later_node` | Unicode предыдущих и текущей строк влияет на абсолютный и column offset. | Поместить emoji/non-BMP и combining sequence до Add, target — на второй или последующей строке. | Exact Unicode before/span/after, range и посимвольный round-trip. | +| `test_code_fragments_handle_bom_before_node_on_later_line` | BOM не добавляется к later-line offset повторно. | `'\ufeffheader = "🙂e\u0301"\nvalue = 1 +\t2\ntail = 3\n'`; Add на второй строке. | Before начинается с BOM; span `' +\t'`; after содержит suffix и tail; обе части непусты; round-trip точен. | +| `test_module_fragments_cover_empty_and_complete_source` | Module покрывает пустой и обычный source целиком. | `''` и `'x=1\n'`; выбрать Module. | Before/after пусты; span равен source; exact range и round-trip. | +| `test_module_range_leaves_bom_in_code_before` | BOM не входит в range Module. | `'\ufeffx=1\n'`; выбрать Module. | Before `'\ufeff'`; span `'x=1\n'`; after `''`; round-trip сохраняет BOM. | +| `test_code_fragments_preserve_leading_form_feed_prefix` | Валидный leading `\f`, удаляемый LibCST codegen, не теряется. | `'\fresult = 1 + 2\n'`; выбрать Add. | Before `'\fresult = 1'`; span `' + '`; after `'2\n'`; exact range и полный исходник восстановлены. | +| `test_multiple_filters_receive_isolated_meta_contexts` | Frozen-контексты нескольких filters не переносят meta друг другу. | Два filter с разными meta; первый мутирует свою meta; оба Context сохранить после traversal. | Каждый видит свою meta; мутация первого не влияет на второго и decorator dict; после traversal значения остаются изолированными; fragment invariant проходит в обоих. | +| `test_multiple_converter_ids_receive_equal_original_fragments` | Разные converter choices получают эквивалентный исходный контекст. | Add отдельно заменить на Subtract и Multiply. | Source/range/before/after равны; оба round-trip восстанавливают исходник; результаты содержат разные операторы. | + +## Документация и проверка + +README должен компактно описывать публичный `Context.position` и его поля. Детали +ленивого alignment, contextual source-span, отличия от `Module.code_for_node()`, +metadata cost и сохранения codegen-omitted prefixes документируются в +докстрингах соответствующих классов. + +После реализации выполнить: + +1. Полный `pytest`. +2. Coverage и branch coverage с существующим порогом 100%. +3. `ruff check cstvis tests`. +4. `mypy --strict cstvis` и `mypy tests`. +5. Проверить, что изменены только `docs/plans/1.md`, реализация, тесты и README; версию пакета и release-конфигурацию не менять. + +## Assumptions + +- Обратная совместимость старого конструктора намеренно не сохраняется. +- `Context` и `SourcePosition` — frozen и не slotted. +- Resolver `SourcePosition` обязателен и разделяется между позициями traversal. +- `meta` остаётся mutable dict, но каждый callback получает свежую копию wrapper meta. +- Before/after всегда доступны через `Context.position` и относятся к исходному `Changer.source`. +- Общих test helpers не создаётся. +- Каждый correctness-тест напрямую проверяет before, range, явный node span, after и round-trip. +- Repr строится через `printo.describe_call` с `item_limit=80`. diff --git a/pyproject.toml b/pyproject.toml index dae7b66..43a5a16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cstvis" -version = "0.0.9" +version = "0.0.10" authors = [{ name = "Evgeniy Blinov", email = "zheni-b@yandex.ru" }] description = 'Incremental change of CST' readme = "README.md" diff --git a/requirements_dev.txt b/requirements_dev.txt index c617929..8fb631c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,6 @@ pytest==8.3.5 -coverage==7.6.1 +coverage==7.6.1; python_version < "3.15" +coverage==7.15.3; python_version >= "3.15" build==1.2.2.post1 twine==6.1.0 mypy==1.14.1 diff --git a/tests/test_changer.py b/tests/test_changer.py index c0343d6..d5579fa 100644 --- a/tests/test_changer.py +++ b/tests/test_changer.py @@ -1,14 +1,16 @@ -# ruff: noqa: ARG001 - -from typing import Any, Union +from typing import Any, List, Union +from unittest.mock import patch import pytest from full_match import match from libcst import Add, CSTNode, Multiply, SimpleString, Subtract +from libcst.metadata import CodePosition, CodeRange from metacode import ParsedComment from sigmatch import SignatureMismatchError -from cstvis import Changer, Collector, Context +import cstvis.source_offsets as source_offsets_module +from cstvis import Changer, Collector, Context, Coordinate +from cstvis.source_offsets import SourceOffsetResolver @pytest.mark.parametrize( @@ -34,11 +36,11 @@ def test_just_iterate_add_coordinates(file, with_context, unfold): if with_context: @unfold(changer.converter) - def name_changer(node: Add, context: Context): + def name_changer(node: Add, context: Context): # noqa: ARG001 return True else: @unfold(changer.converter) - def name_changer(node: Add): + def name_changer(node: Add): # noqa: ARG001 return True coordinates = list(changer.iterate_coordinates()) @@ -84,8 +86,8 @@ def name_changer(node: Add): return node @changer.filter - def filter_second_add(node: Add, context: Context) -> bool: - return context.coordinate.start_line == 2 + def filter_second_add(node: Add, context: Context) -> bool: # noqa: ARG001 + return context.position.coordinate.start_line == 2 coordinates = list(changer.iterate_coordinates()) @@ -115,7 +117,7 @@ def test_apply_one_change(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_add_to_sub(node: Add, context: Context): + def change_add_to_sub(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -155,7 +157,7 @@ def test_apply_two_changes_at_same_line(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_add_to_sub(node: Add, context: Context): + def change_add_to_sub(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -199,14 +201,14 @@ def test_to_different_changers_to_same_line(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_add_to_sub(node: Add, context: Context): + def change_add_to_sub(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.converter) - def change_sub_to_add(node: Subtract, context: Context): + def change_sub_to_add(node: Subtract, context: Context): # noqa: ARG001 return Add( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -256,7 +258,7 @@ def test_changing_function_with_wrong_number_of_parameters(file, unfold): with pytest.raises(SignatureMismatchError, match=match('A function that takes a CST node and a context is expected.')): @unfold(changer.converter) - def changing_function_1(node: Add, context: Context, something_else: str): + def changing_function_1(node: Add, context: Context, something_else: str): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -354,14 +356,14 @@ def test_filter_any_on(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Any, context: Context) -> bool: + def filter_something(node: Any, context: Context) -> bool: # noqa: ARG001 return True else: @@ -373,7 +375,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Any) -> bool: + def filter_something(node: Any) -> bool: # noqa: ARG001 return True results = [] @@ -403,14 +405,14 @@ def test_filter_any_off(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Any, context: Context) -> bool: + def filter_something(node: Any, context: Context) -> bool: # noqa: ARG001 return False else: @@ -422,7 +424,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Any) -> bool: + def filter_something(node: Any) -> bool: # noqa: ARG001 return False results = [] @@ -451,14 +453,14 @@ def test_filter_cstnode_on(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: CSTNode, context: Context) -> bool: + def filter_something(node: CSTNode, context: Context) -> bool: # noqa: ARG001 return True else: @@ -470,7 +472,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: CSTNode) -> bool: + def filter_something(node: CSTNode) -> bool: # noqa: ARG001 return True results = [] @@ -500,14 +502,14 @@ def test_filter_cstnode_off(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: CSTNode, context: Context) -> bool: + def filter_something(node: CSTNode, context: Context) -> bool: # noqa: ARG001 return False else: @@ -519,7 +521,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: CSTNode) -> bool: + def filter_something(node: CSTNode) -> bool: # noqa: ARG001 return False results = [] @@ -548,14 +550,14 @@ def test_filter_node_on(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Add, context: Context) -> bool: + def filter_something(node: Add, context: Context) -> bool: # noqa: ARG001 return True else: @@ -567,7 +569,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Add) -> bool: + def filter_something(node: Add) -> bool: # noqa: ARG001 return True results = [] @@ -597,14 +599,14 @@ def test_filter_node_off(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Add, context: Context) -> bool: + def filter_something(node: Add, context: Context) -> bool: # noqa: ARG001 return False else: @@ -616,7 +618,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Add) -> bool: + def filter_something(node: Add) -> bool: # noqa: ARG001 return False results = [] @@ -645,14 +647,14 @@ def test_filter_other_node_on(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Subtract, context: Context) -> bool: + def filter_something(node: Subtract, context: Context) -> bool: # noqa: ARG001 return True else: @@ -664,7 +666,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Subtract) -> bool: + def filter_something(node: Subtract) -> bool: # noqa: ARG001 return True results = [] @@ -694,14 +696,14 @@ def test_filter_other_node_off(file, with_context, unfold): if with_context: @unfold(changer.converter) - def change_something(node: Add, context: Context): + def change_something(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.filter) - def filter_something(node: Subtract, context: Context) -> bool: + def filter_something(node: Subtract, context: Context) -> bool: # noqa: ARG001 return False else: @@ -713,7 +715,7 @@ def change_something(node: Add): ) @unfold(changer.filter) - def filter_something(node: Subtract) -> bool: + def filter_something(node: Subtract) -> bool: # noqa: ARG001 return False results = [] @@ -735,7 +737,7 @@ def test_converter_with_no_annotation(with_context, unfold): if with_context: @unfold(changer.converter) - def converter_func(node, context): + def converter_func(node, context): # noqa: ARG001 return node else: @unfold(changer.converter) @@ -755,7 +757,7 @@ def test_converter_with_any_annotation(with_context, unfold): if with_context: @unfold(changer.converter) - def converter_func(node: Any, context): + def converter_func(node: Any, context): # noqa: ARG001 return node else: @unfold(changer.converter) @@ -775,7 +777,7 @@ def test_converter_with_cstnode_annotation_restriction(with_context, unfold): if with_context: @unfold(changer.converter) - def converter_func(node: CSTNode, context): + def converter_func(node: CSTNode, context): # noqa: ARG001 return node else: @unfold(changer.converter) @@ -797,7 +799,7 @@ def test_convert_str(with_context, unfold): if with_context: @unfold(changer.converter) - def converter_func(node: str, context: Context): + def converter_func(node: str, context: Context): # noqa: ARG001 nodes.append(node) return node else: @@ -823,7 +825,7 @@ def test_convert_float(with_context, unfold): if with_context: @unfold(changer.converter) - def converter_func(node: float, context: Context): + def converter_func(node: float, context: Context): # noqa: ARG001 return node.with_changes(value=repr(node.evaluated_value + 1)) # type: ignore[attr-defined] else: @unfold(changer.converter) @@ -846,7 +848,7 @@ def test_filter_with_wrong_number_of_parameters(unfold): with pytest.raises(SignatureMismatchError, match=match('A function that takes a CST node and a context is expected.')): @unfold(changer.filter) - def filter_func(node: Add, context: Context, extra_param: str): + def filter_func(node: Add, context: Context, extra_param: str): # noqa: ARG001 return True with pytest.raises(SignatureMismatchError, match=match('A function that takes a CST node and a context is expected.')): @@ -868,7 +870,7 @@ class SomeClass: with pytest.raises(TypeError, match=match('The type annotation for the first argument of the function must be descended from the libcst.CSTNode class.')): @unfold(changer.filter) - def filter_func(node: SomeClass, context: Context): + def filter_func(node: SomeClass, context: Context): # noqa: ARG001 return True @@ -882,14 +884,14 @@ def test_two_converters_for_same_node(with_context, unfold): if with_context: @unfold(changer.converter) - def converter1(node: Add, context: Context): + def converter1(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(changer.converter) - def converter2(node: Add, context: Context): + def converter2(node: Add, context: Context): # noqa: ARG001 return Multiply( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -923,7 +925,7 @@ def test_use_collector_for_converter(with_context, unfold): if with_context: @unfold(collector.converter) - def some_converter(node: Add, context: Context): + def some_converter(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -953,14 +955,14 @@ def test_use_collector_for_converter_and_filter(with_context, unfold): if with_context: @unfold(collector.converter) - def some_converter(node: Add, context: Context): + def some_converter(node: Add, context: Context): # noqa: ARG001 return Subtract( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, ) @unfold(collector.filter) - def some_filter(node: Add, context: Context): + def some_filter(node: Add, context: Context): # noqa: ARG001 return filters_value else: @@ -972,7 +974,7 @@ def some_converter(node: Add): ) @unfold(collector.filter) - def some_filter(node: Add): + def some_filter(node: Add): # noqa: ARG001 return filters_value changer = Changer('a = 5 + 5', collector=collector) @@ -994,7 +996,7 @@ def test_union_with_csts(with_context, unfold): if with_context: @unfold(changer.converter) - def some_converter(node: Union[Add, Subtract], context: Context): + def some_converter(node: Union[Add, Subtract], context: Context): # noqa: ARG001 return Multiply( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -1020,7 +1022,7 @@ def test_union_with_union_with_csts(with_context, unfold): if with_context: @unfold(changer.converter) - def some_converter(node: Union[Add, Union[Multiply, Subtract]], context: Context): + def some_converter(node: Union[Add, Union[Multiply, Subtract]], context: Context): # noqa: ARG001 return Multiply( whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after, @@ -1046,7 +1048,7 @@ def test_convert_plus_one(with_context, unfold): if with_context: @unfold(changer.converter) - def convert_ints(node: int, context): + def convert_ints(node: int, context): # noqa: ARG001 return node.with_changes(value=repr(node.evaluated_value + 1)) # type: ignore[attr-defined] else: @unfold(changer.converter) @@ -1068,7 +1070,7 @@ def test_converter_for_any(with_context, unfold): if with_context: @unfold(changer.converter) - def do_something(node: Any, context): + def do_something(node: Any, context): # noqa: ARG001 nodes.append(nodes) return node else: @@ -1091,7 +1093,7 @@ def test_if_node_is_not_exist_nothing_changed(with_context, unfold): if with_context: @unfold(changer.converter) - def do_something(node: float, context): + def do_something(node: float, context): # noqa: ARG001 return node else: @unfold(changer.converter) @@ -1110,18 +1112,18 @@ def test_get_function_id_from_itself(unfold): changer = Changer('5 - 5 + 5') @unfold(changer.converter) - def do_something(node: float, context): + def do_something(node: float, context): # noqa: ARG001 return node @unfold(changer.filter) - def filter_something(node: float, context): + def filter_something(node: float, context): # noqa: ARG001 return False converter = list(changer.converters_by_types.values())[0][0] # noqa: RUF015 filter = list(changer.filters_by_types.values())[0][0] # noqa: RUF015, A001 - assert converter.get_function_id() == 'tests.test_changer:do_something:1112' - assert filter.get_function_id() == 'tests.test_changer:filter_something:1116' + assert converter.get_function_id() == 'tests.test_changer:do_something:1114' + assert filter.get_function_id() == 'tests.test_changer:filter_something:1118' def test_wrong_converter_and_wrong_filter(unfold): @@ -1149,7 +1151,7 @@ def filter_something_1(): with pytest.raises(SignatureMismatchError, match=match('A function that takes a CST node and a context is expected.')): @unfold(changer.filter) - def filter_something_2(a, b, c): + def filter_something_2(a, b, c): # noqa: ARG001 return False @@ -1195,7 +1197,7 @@ def some_converter(node: Add): ) @changer.filter(meta=meta) - def filter_something(node: Add, context: Context): + def filter_something(node: Add, context: Context): # noqa: ARG001 bread_crumbs.append(context.meta) return False @@ -1248,7 +1250,7 @@ def some_converter(node: Add): ) @collector.filter(meta=meta) - def filter_something(node: Add, context: Context): + def filter_something(node: Add, context: Context): # noqa: ARG001 bread_crumbs.append(context.meta) return False @@ -1279,3 +1281,384 @@ def change_something(node: Add, context=None): assert len(contexts) == 1 assert isinstance(contexts[0], Context) + + +def test_filter_context_exposes_whitespace_inclusive_fragments(): + """ + Context.position exposes an Add's whitespace-inclusive range, offsets, and fragments to filters. + + The ordinary coordinate remains limited to the '+' token. + """ + source = 'left = 1 +\t2 # tail\r\nright = 3\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.coordinate == Coordinate(None, 'Add', 1, 10, 1, 11) + assert context.position.node_range == CodeRange(CodePosition(1, 8), CodePosition(1, 12)) + assert context.position.start_offset == 8 + assert context.position.end_offset == 12 + assert context.position.code_before == 'left = 1' + assert context.position.code_after == '2 # tail\r\nright = 3\n' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_converter_context_exposes_original_fragments(): + """ + A converter's Context.position retains the original Add's range, offsets, and fragments after replacement. + """ + source = 'left = 1 +\t2 # tail\r\nright = 3\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add, context: Context) -> Subtract: + contexts.append(context) + return Subtract(whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after) + + coordinate = next(changer.iterate_coordinates()) + transformed_source = changer.apply_coordinate(coordinate) + + assert transformed_source == 'left = 1 -\t2 # tail\r\nright = 3\n' + assert len(contexts) == 1 + context = contexts[0] + assert context.position.coordinate == Coordinate(None, 'Add', 1, 10, 1, 11) + assert context.position.node_range == CodeRange(CodePosition(1, 8), CodePosition(1, 12)) + assert context.position.start_offset == 8 + assert context.position.end_offset == 12 + assert context.position.code_before == 'left = 1' + assert context.position.code_after == '2 # tail\r\nright = 3\n' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize('callback_kind', ['filter', 'converter']) +def test_context_pipeline_does_not_materialize_unread_fragments(callback_kind): # noqa: C901 + """ + Coordinate-only filter and converter callbacks neither resolve fragment offsets nor slice the source. + """ + class TrackingString(str): + __slots__ = ('alignment_operations', 'slices') + alignment_operations: List[str] + slices: List[slice] + + def __new__(cls, value: str): # type: ignore[no-untyped-def] + instance = super().__new__(cls, value) + instance.alignment_operations = [] + instance.slices = [] + return instance + + def __getitem__(self, key): # type: ignore[no-untyped-def] + if isinstance(key, slice): + self.slices.append(key) + return super().__getitem__(key) + + def find(self, substring, *args): # type: ignore[no-untyped-def] + self.alignment_operations.append('find') + return super().find(substring, *args) + + def startswith(self, prefix, *args): # type: ignore[no-untyped-def] + self.alignment_operations.append('startswith') + return super().startswith(prefix, *args) + + source = TrackingString('x = 1 + 2\n') + changer = Changer(source) + source.alignment_operations.clear() + source.slices.clear() + observed_coordinates = [] + + if callback_kind == 'filter': + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def inspect_coordinate(node: Add, context: Context) -> bool: # noqa: ARG001 + observed_coordinates.append(context.position.coordinate) + return False + else: + @changer.converter + def convert(node: Add, context: Context) -> Subtract: + observed_coordinates.append(context.position.coordinate) + return Subtract(whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after) + + alignment_ranges = [] + resolve_offsets = SourceOffsetResolver.__call__ + + def track_alignment(resolver, node_range): # type: ignore[no-untyped-def] + alignment_ranges.append(node_range) + return resolve_offsets(resolver, node_range) + + with patch.object(SourceOffsetResolver, '__call__', track_alignment): + coordinates = list(changer.iterate_coordinates()) + if callback_kind == 'filter': + assert coordinates == [] + else: + assert len(coordinates) == 1 + assert changer.apply_coordinate(coordinates[0]) == 'x = 1 - 2\n' + + assert observed_coordinates == [Coordinate(None, 'Add', 1, 6, 1, 7)] + assert alignment_ranges == [] + assert source.alignment_operations == [] + assert source.slices == [] + + +@pytest.mark.parametrize('callback_kind', ['filter', 'converter']) +def test_node_only_callbacks_do_not_materialize_context_fragments(callback_kind): + """ + Node-only filter and converter callbacks neither resolve fragment offsets nor slice the source. + """ + class TrackingString(str): + __slots__ = ('slices',) + slices: List[slice] + + def __new__(cls, value: str): # type: ignore[no-untyped-def] + instance = super().__new__(cls, value) + instance.slices = [] + return instance + + def __getitem__(self, key): # type: ignore[no-untyped-def] + if isinstance(key, slice): + self.slices.append(key) + return super().__getitem__(key) + + source = TrackingString('x = 1 + 2\n') + changer = Changer(source) + source.slices.clear() + converter_inputs = [] + filter_inputs = [] + alignment_ranges = [] + + @changer.converter + def convert(node: Add) -> Subtract: + converter_inputs.append(node) + return Subtract(whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after) + + if callback_kind == 'filter': + @changer.filter + def reject(node: Add) -> bool: + filter_inputs.append(node) + return False + + resolve_offsets = SourceOffsetResolver.__call__ + + def track_alignment(resolver, node_range): # type: ignore[no-untyped-def] + alignment_ranges.append(node_range) + return resolve_offsets(resolver, node_range) + + with patch.object(SourceOffsetResolver, '__call__', track_alignment): + coordinates = list(changer.iterate_coordinates()) + if callback_kind == 'converter': + assert changer.apply_coordinate(coordinates[0]) == 'x = 1 - 2\n' + + if callback_kind == 'filter': + assert coordinates == [] + assert len(filter_inputs) == 1 + assert isinstance(filter_inputs[0], Add) + assert converter_inputs == [] + else: + assert len(coordinates) == 1 + assert len(converter_inputs) == 1 + assert isinstance(converter_inputs[0], Add) + assert alignment_ranges == [] + assert source.slices == [] + + +@pytest.mark.parametrize('callback_kind', ['filter', 'converter']) +@pytest.mark.parametrize('fragment_name', ['code_before', 'code_after']) +def test_context_pipeline_materializes_only_requested_fragment(callback_kind, fragment_name): + """ + Filter and converter paths resolve and slice only the requested fragment once despite repeated reads. + """ + class TrackingString(str): + __slots__ = ('slices',) + slices: List[slice] + + def __new__(cls, value: str): # type: ignore[no-untyped-def] + instance = super().__new__(cls, value) + instance.slices = [] + return instance + + def __getitem__(self, key): # type: ignore[no-untyped-def] + if isinstance(key, slice): + self.slices.append(key) + return super().__getitem__(key) + + source = TrackingString('x = 1 + 2\n') + changer = Changer(source) + fragment_values = [] + contexts = [] + source.slices.clear() + + if callback_kind == 'filter': + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def read_fragment(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + fragment_values.extend([getattr(context.position, fragment_name), getattr(context.position, fragment_name)]) + return True + else: + @changer.converter + def convert(node: Add, context: Context) -> Add: + contexts.append(context) + fragment_values.extend([getattr(context.position, fragment_name), getattr(context.position, fragment_name)]) + return node + + alignment_ranges = [] + resolve_offsets = SourceOffsetResolver.__call__ + + def track_alignment(resolver, node_range): # type: ignore[no-untyped-def] + alignment_ranges.append(node_range) + return resolve_offsets(resolver, node_range) + + with patch.object(SourceOffsetResolver, '__call__', track_alignment): + coordinates = list(changer.iterate_coordinates()) + if callback_kind == 'converter': + changer.apply_coordinate(coordinates[0]) + + expected_fragment = 'x = 1' if fragment_name == 'code_before' else '2\n' + expected_slice = slice(None, 5) if fragment_name == 'code_before' else slice(8, None) + other_fragment_name = 'code_after' if fragment_name == 'code_before' else 'code_before' + assert len(contexts) == 1 + assert fragment_name in contexts[0].position.__dict__ + assert other_fragment_name not in contexts[0].position.__dict__ + assert fragment_values == [expected_fragment, expected_fragment] + assert alignment_ranges == [CodeRange(CodePosition(1, 5), CodePosition(1, 8))] + assert source.slices == [expected_slice] + + +@pytest.mark.parametrize('callback_kind', ['filter', 'converter']) +def test_context_fragments_can_be_read_after_traversal(callback_kind): + """ + A captured Context.position can resolve unread fragments after traversal and reconstruct the source. + """ + class TrackingString(str): + __slots__ = ('slices',) + slices: List[slice] + + def __new__(cls, value: str): # type: ignore[no-untyped-def] + instance = super().__new__(cls, value) + instance.slices = [] + return instance + + def __getitem__(self, key): # type: ignore[no-untyped-def] + if isinstance(key, slice): + self.slices.append(key) + return super().__getitem__(key) + + source = TrackingString('x = 1 + 2\n') + changer = Changer(source) + contexts = [] + source.slices.clear() + + if callback_kind == 'filter': + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + else: + @changer.converter + def convert(node: Add, context: Context) -> Add: + contexts.append(context) + return node + + coordinates = list(changer.iterate_coordinates()) + if callback_kind == 'converter': + changer.apply_coordinate(coordinates[0]) + + assert source.slices == [] + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 5), CodePosition(1, 8)) + assert context.position.code_before == 'x = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +def test_multiple_filters_receive_contexts_with_isolated_meta(): + """ + Filter Context copies share one SourcePosition while their mutable meta copies remain isolated. + """ + source = 'x = 1 + 2\n' + changer = Changer(source) + first_filter_meta = {'filter': 1} + second_filter_meta = {'filter': 2} + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter(meta=first_filter_meta) + def first_filter(node: Add, context: Context) -> bool: # noqa: ARG001 + assert context.meta is not None + context.meta['changed'] = True + contexts.append(context) + return True + + @changer.filter(meta=second_filter_meta) + def second_filter(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 2 + assert contexts[0].position is contexts[1].position + assert contexts[0].meta == {'filter': 1, 'changed': True} + assert contexts[1].meta == {'filter': 2} + assert first_filter_meta == {'filter': 1} + assert second_filter_meta == {'filter': 2} + with patch('cstvis.source_offsets._SourceOffsetCodegenState', wraps=source_offsets_module._SourceOffsetCodegenState) as create_state: + for context in contexts: + assert context.position.node_range == CodeRange(CodePosition(1, 5), CodePosition(1, 8)) + assert context.position.code_before == 'x = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + assert create_state.call_count == 1 + + +def test_multiple_converters_receive_equal_original_fragments(): + """ + Different Add replacements receive equal original fragments and produce distinct transformed sources. + """ + source = 'x = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def subtract(node: Add, context: Context) -> Subtract: + contexts.append(context) + return Subtract(whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after) + + @changer.converter + def multiply(node: Add, context: Context) -> Multiply: + contexts.append(context) + return Multiply(whitespace_before=node.whitespace_before, whitespace_after=node.whitespace_after) + + transformed_sources = {changer.apply_coordinate(coordinate) for coordinate in changer.iterate_coordinates()} + + assert transformed_sources == {'x = 1 - 2\n', 'x = 1 * 2\n'} + assert len(contexts) == 2 + for context in contexts: + assert context.position.source == source + assert context.position.node_range == CodeRange(CodePosition(1, 5), CodePosition(1, 8)) + assert context.position.code_before == 'x = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source diff --git a/tests/test_dto.py b/tests/test_dto.py new file mode 100644 index 0000000..25aaf9b --- /dev/null +++ b/tests/test_dto.py @@ -0,0 +1,348 @@ +import sys +from dataclasses import FrozenInstanceError +from typing import List +from unittest.mock import patch + +import pytest +from full_match import match +from libcst import parse_module +from libcst.metadata import CodePosition, CodeRange +from metacode import ParsedComment +from printo import describe_call as printo_describe_call + +from cstvis import Context, Coordinate +from cstvis.dto import SourcePosition +from cstvis.source_offsets import SourceOffsetResolver + + +def test_context_requires_source_position_with_resolver(): + """ + Context requires SourcePosition, which requires coordinate, source, node_range, and a resolver. + + Constructor TypeErrors include the class name starting with Python 3.10. + """ + coordinate = Coordinate(None, 'Add', 1, 2, 1, 3) + node_range = CodeRange(CodePosition(1, 1), CodePosition(1, 4)) + resolver = SourceOffsetResolver(parse_module('1 + 2'), '1 + 2', [node_range]) + position = SourcePosition(coordinate, '1 + 2', node_range, resolver) + + context = Context(position, None) + + assert context.position is position + assert position.coordinate == coordinate + assert position.source == '1 + 2' + assert position.node_range == node_range + assert position.offset_resolver is resolver + assert context.meta is None + context_init = 'Context.__init__' if sys.version_info >= (3, 10) else '__init__' + source_position_init = 'SourcePosition.__init__' if sys.version_info >= (3, 10) else '__init__' + with pytest.raises(TypeError, match=match(f"{context_init}() missing 1 required positional argument: 'position'")): + Context(comment=None) # type: ignore[call-arg] + with pytest.raises(TypeError, match=match(f"{source_position_init}() missing 1 required positional argument: 'offset_resolver'")): + SourcePosition(coordinate, '1 + 2', node_range) # type: ignore[call-arg] + with pytest.raises(TypeError, match=match(f"{source_position_init}() missing 2 required positional arguments: 'node_range' and 'offset_resolver'")): + SourcePosition(coordinate, '1 + 2') # type: ignore[call-arg] + + +def test_context_is_frozen(): + """ + Context and SourcePosition are frozen to protect lazy caches, while meta remains mutable. + """ + coordinate = Coordinate(None, 'Add', 1, 2, 1, 3) + node_range = CodeRange(CodePosition(1, 1), CodePosition(1, 4)) + resolver = SourceOffsetResolver(parse_module('1 + 2'), '1 + 2', [node_range]) + position = SourcePosition(coordinate, '1 + 2', node_range, resolver) + context = Context(position, None, {'key': 'value'}) + + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'source'")): + position.source = '3 + 4' # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'node_range'")): + position.node_range = CodeRange(CodePosition(1, 0), CodePosition(1, 1)) # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'offset_resolver'")): + position.offset_resolver = resolver # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'position'")): + context.position = position # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'coordinate'")): + position.coordinate = Coordinate(None, 'Add', 1, 0, 1, 1) # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'comment'")): + context.comment = 'replacement' # type: ignore[misc] + with pytest.raises(FrozenInstanceError, match=match("cannot assign to field 'meta'")): + context.meta = {'replacement': True} # type: ignore[misc] + + assert position.source == '1 + 2' + assert position.node_range == node_range + assert position.offset_resolver is resolver + assert position.coordinate == coordinate + assert context.comment is None + assert context.meta is not None + context.meta['key'] = 'changed' + assert context.meta == {'key': 'changed'} + + +def test_context_equality_includes_fields_but_ignores_derived_caches(): + """ + Equality includes Context and SourcePosition data but ignores resolvers and derived caches. + """ + coordinate = Coordinate(None, 'Add', 1, 2, 1, 3) + node_range = CodeRange(CodePosition(1, 1), CodePosition(1, 4)) + resolver = SourceOffsetResolver(parse_module('1 + 2'), '1 + 2', [node_range]) + other_resolver = SourceOffsetResolver(parse_module('1 + 2'), '1 + 2', [node_range]) + position = SourcePosition(coordinate, '1 + 2', node_range, resolver) + equal_position = SourcePosition(coordinate, '1 + 2', node_range, other_resolver) + context = Context(position, None) + equal_context = Context(equal_position, None) + + assert context == equal_context + assert context.position.code_before == '1' + assert context == equal_context + assert context.position.code_after == '2' + assert context == equal_context + assert context != Context(SourcePosition(coordinate, '3 + 4', node_range, resolver), None) + different_range = CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + assert context != Context(SourcePosition(coordinate, '1 + 2', different_range, resolver), None) + different_coordinate = Coordinate(None, 'Add', 1, 0, 1, 1) + assert context != Context(SourcePosition(different_coordinate, '1 + 2', node_range, resolver), None) + assert context != Context(position, 'comment') + assert context != Context(position, None, {'key': 'value'}) + assert position != object() + + +def test_context_repr_uses_printo_item_limit(): + """ + Nested reprs delegate ordered public data and lazy offsets to printo with item_limit=80. + + Repr truncates a long source and leaves both surrounding fragments lazy. + """ + coordinate = Coordinate(None, 'Add', 1, 2, 1, 3) + node_range = CodeRange(CodePosition(1, 1), CodePosition(1, 4)) + comment = 'keep this comment' + meta = {'mode': 'test'} + short_resolver = SourceOffsetResolver(parse_module('1 + 2'), '1 + 2', [node_range]) + short_position = SourcePosition(coordinate, '1 + 2', node_range, short_resolver) + short_context = Context(short_position, comment, meta) + long_source = 'x' * 200 + long_resolver = SourceOffsetResolver(parse_module(long_source), long_source, [node_range]) + long_position = SourcePosition(coordinate, long_source, node_range, long_resolver) + long_context = Context(long_position, comment, meta) + + with patch('cstvis.dto.describe_call', wraps=printo_describe_call) as describe_call: + short_position_repr = repr(short_position) + long_position_repr = repr(long_position) + short_repr = repr(short_context) + long_repr = repr(long_context) + + assert "source='1 + 2'" in short_position_repr + assert 'start_offset=1' in short_position_repr + assert 'end_offset=4' in short_position_repr + assert long_source not in long_position_repr + assert 'source=' in long_position_repr + assert 'position=SourcePosition(' in short_repr + assert 'position=SourcePosition(' in long_repr + assert short_position.__dict__['start_offset'] == 1 + assert short_position.__dict__['end_offset'] == 4 + assert long_position.__dict__['start_offset'] == 1 + assert long_position.__dict__['end_offset'] == 4 + assert 'code_before' not in short_position.__dict__ + assert 'code_after' not in short_position.__dict__ + assert 'code_before' not in long_position.__dict__ + assert 'code_after' not in long_position.__dict__ + context_calls = [call for call in describe_call.call_args_list if call.args[0] is Context] + position_calls = [call for call in describe_call.call_args_list if call.args[0] is SourcePosition] + assert len(context_calls) == 2 + assert len(position_calls) == 4 + short_context_call, long_context_call = context_calls + short_position_call, long_position_call = position_calls[:2] + assert short_context_call.args == ( + Context, + [], + { + 'position': short_position, + 'comment': comment, + 'meta': meta, + }, + ) + assert long_context_call.args == ( + Context, + [], + { + 'position': long_position, + 'comment': comment, + 'meta': meta, + }, + ) + assert short_position_call.args == ( + SourcePosition, + [], + { + 'coordinate': coordinate, + 'source': '1 + 2', + 'node_range': node_range, + 'start_offset': 1, + 'end_offset': 4, + }, + ) + assert long_position_call.args == ( + SourcePosition, + [], + { + 'coordinate': coordinate, + 'source': long_source, + 'node_range': node_range, + 'start_offset': 1, + 'end_offset': 4, + }, + ) + assert list(short_context_call.args[2]) == ['position', 'comment', 'meta'] + assert list(long_context_call.args[2]) == ['position', 'comment', 'meta'] + assert list(short_position_call.args[2]) == ['coordinate', 'source', 'node_range', 'start_offset', 'end_offset'] + assert list(long_position_call.args[2]) == ['coordinate', 'source', 'node_range', 'start_offset', 'end_offset'] + for call in describe_call.call_args_list: + assert call.kwargs == {'item_limit': 80} + + +def test_source_position_offsets_and_fragments_are_lazy_and_cached(): # noqa: C901, PLR0915 + """ + Construction, equality, coordinate access, and metacode parsing resolve neither offsets nor fragments. + + Reading either offset performs one shared alignment without slicing, copying + the complete source, or splitting it into lines. Fragment properties then + cache only their own source slices. + """ + class TrackingString(str): + __slots__ = ('alignment_operations', 'full_iterations', 'line_materializations', 'slices', 'string_conversions') + alignment_operations: List[str] + full_iterations: int + line_materializations: List[str] + slices: List[slice] + string_conversions: int + + def __new__(cls, value: str): # type: ignore[no-untyped-def] + instance = super().__new__(cls, value) + instance.alignment_operations = [] + instance.full_iterations = 0 + instance.line_materializations = [] + instance.slices = [] + instance.string_conversions = 0 + return instance + + def __getitem__(self, key): # type: ignore[no-untyped-def] + if isinstance(key, slice): + self.slices.append(key) + return super().__getitem__(key) + + def __iter__(self): # type: ignore[no-untyped-def] + self.full_iterations += 1 + return super().__iter__() + + def __str__(self) -> str: + self.string_conversions += 1 + return super().__str__() + + def find(self, substring, *args): # type: ignore[no-untyped-def] + self.alignment_operations.append('find') + return super().find(substring, *args) + + def startswith(self, prefix, *args): # type: ignore[no-untyped-def] + self.alignment_operations.append('startswith') + return super().startswith(prefix, *args) + + def split(self, separator=None, maxsplit=-1): # type: ignore[no-untyped-def] + self.line_materializations.append('split') + return super().split(separator, maxsplit) + + def splitlines(self, keepends=False): # type: ignore[no-untyped-def] + self.line_materializations.append('splitlines') + return super().splitlines(keepends) + + source = TrackingString('header\nleft + right\n') + coordinate = Coordinate(None, 'Add', 2, 5, 2, 6) + node_range = CodeRange(CodePosition(2, 4), CodePosition(2, 7)) + resolver = SourceOffsetResolver(parse_module(source), source, [node_range]) + alignment_ranges = [] + resolve_offsets = SourceOffsetResolver.__call__ + + def track_alignment(source_offsets, requested_range): # type: ignore[no-untyped-def] + alignment_ranges.append(requested_range) + return resolve_offsets(source_offsets, requested_range) + + with patch.object(SourceOffsetResolver, '__call__', track_alignment): + position = SourcePosition(coordinate, source, node_range, resolver) + equal_position = SourcePosition(coordinate, source, node_range, resolver) + context = Context(position, 'key: action') + equal_context = Context(equal_position, 'key: action') + + assert context == equal_context + assert context.position.coordinate == coordinate + assert context.comment == 'key: action' + assert context.get_metacodes('key') == [ParsedComment(key='key', command='action', arguments=[])] + assert alignment_ranges == [] + assert source.alignment_operations == [] + assert source.full_iterations == 0 + assert source.line_materializations == [] + assert source.slices == [] + assert source.string_conversions == 0 + assert 'start_offset' not in position.__dict__ + assert 'end_offset' not in position.__dict__ + assert 'code_before' not in position.__dict__ + assert 'code_after' not in position.__dict__ + + assert context.position.start_offset == 11 + assert alignment_ranges == [node_range] + assert source.alignment_operations + assert source.slices == [] + assert position.__dict__['start_offset'] == 11 + assert 'end_offset' not in position.__dict__ + + assert context.position.end_offset == 14 + assert alignment_ranges == [node_range, node_range] + assert source.slices == [] + assert position.__dict__['end_offset'] == 14 + + assert context.position.code_before == 'header\nleft' + assert alignment_ranges == [node_range, node_range] + assert source.slices == [slice(None, 11)] + assert context.position.code_before == 'header\nleft' + assert alignment_ranges == [node_range, node_range] + assert source.slices == [slice(None, 11)] + assert 'code_after' not in position.__dict__ + + assert context.position.code_after == 'right\n' + assert alignment_ranges == [node_range, node_range] + assert source.full_iterations == 0 + assert source.line_materializations == [] + assert source.slices == [slice(None, 11), slice(14, None)] + assert source.string_conversions == 0 + assert context.position.code_after == 'right\n' + assert alignment_ranges == [node_range, node_range] + assert source.slices == [slice(None, 11), slice(14, None)] + + +def test_source_position_maps_code_range_without_matching_node(): + """ + An in-bounds CodeRange partitions source without a matching CST node. + """ + node_range = CodeRange(CodePosition(1, 1), CodePosition(1, 2)) + coordinate = Coordinate(None, 'manual', 1, 1, 1, 2) + resolver = SourceOffsetResolver(parse_module('abc'), 'abc', [node_range]) + position = SourcePosition(coordinate, 'abc', node_range, resolver) + + assert position.code_before == 'a' + assert position.node_range == node_range + assert position.code_after == 'c' + assert position.code_before + 'b' + position.code_after == position.source + + +def test_source_position_maps_multiline_range_across_codegen_omitted_prefix(): + """ + A manual multiline range maps across CRLF and a form feed omitted by LibCST code generation. + """ + source = 'a=1\r\n\fvalue=2\n' + node_range = CodeRange(CodePosition(1, 1), CodePosition(2, 2)) + coordinate = Coordinate(None, 'manual', 1, 1, 2, 2) + resolver = SourceOffsetResolver(parse_module(source), source, [node_range]) + position = SourcePosition(coordinate, source, node_range, resolver) + + assert position.code_before == 'a' + assert position.node_range == node_range + assert position.code_after == 'lue=2\n' + assert position.code_before + '=1\r\n\fva' + position.code_after == position.source diff --git a/tests/test_source_offsets.py b/tests/test_source_offsets.py new file mode 100644 index 0000000..ebd2aa3 --- /dev/null +++ b/tests/test_source_offsets.py @@ -0,0 +1,925 @@ +from unittest.mock import patch + +import pytest +from libcst import ( + Add, + Assign, + BinaryOperation, + FunctionDef, + Integer, + Module, + Name, + Parameters, + SimpleStatementLine, + SimpleString, + parse_module, +) +from libcst.metadata import ( + CodePosition, + CodeRange, + MetadataWrapper, + WhitespaceInclusivePositionProvider, +) + +import cstvis.source_offsets as source_offsets_module +from cstvis import Changer, Context +from cstvis.dto import SourcePosition +from cstvis.source_offsets import SourceOffsetResolver + + +@pytest.mark.parametrize( + ('source', 'position', 'expected_offset'), + [ + pytest.param('', CodePosition(1, 0), 0, id='empty-source'), + pytest.param('x=1', CodePosition(2, 0), 3, id='virtual-eof'), + pytest.param('x=1\n', CodePosition(2, 0), 4, id='physical-eof'), + pytest.param('x=1\n\f', CodePosition(3, 0), 5, id='form-feed-after-code'), + pytest.param('\ufeff', CodePosition(1, 0), 1, id='bom-only'), + pytest.param('\ufeffx=1\n', CodePosition(2, 0), 5, id='bom-and-code'), + ], +) +def test_source_offset_resolver_returns_exact_eof_offsets(source, position, expected_offset): + """ + EOF positions resolve exactly to len(source), including virtual newlines and an omitted initial BOM. + """ + node_range = CodeRange(position, position) + module = parse_module(source) + resolver = SourceOffsetResolver(module, source, [node_range]) + + assert expected_offset == len(source) + assert resolver(node_range) == (expected_offset, expected_offset) + + +def test_source_offset_resolver_batches_unique_ranges_and_caches_codegen(): + """ + One lazy codegen pass resolves every unique registered endpoint and serves later ranges from cache. + """ + source = 'a = 1 + 2\nb = 3 * 4\n' + module = parse_module(source) + add_range = CodeRange(CodePosition(1, 5), CodePosition(1, 8)) + multiply_range = CodeRange(CodePosition(2, 5), CodePosition(2, 8)) + registered_ranges = (node_range for node_range in [add_range, add_range, multiply_range]) + + with patch('cstvis.source_offsets._SourceOffsetCodegenState', wraps=source_offsets_module._SourceOffsetCodegenState) as create_state: + resolver = SourceOffsetResolver(module, source, registered_ranges) + + assert resolver.target_positions == {add_range.start, add_range.end, multiply_range.start, multiply_range.end} + assert create_state.call_count == 0 + assert resolver(multiply_range) == (15, 18) + assert create_state.call_count == 1 + assert set(resolver._offsets) == resolver.target_positions + assert resolver(add_range) == (5, 8) + assert resolver(multiply_range) == (15, 18) + assert create_state.call_count == 1 + + +@pytest.mark.parametrize( + 'source', + [ + pytest.param( + '@decorator\ndef function(value: int = 1):\n\ttext = f"{value}"\n\treturn (text, value + 1)\n', + id='grammar-and-tab-indent', + ), + pytest.param( + '\ufeffhead = "🙂e\u0301"\r\nvalue = 1 + 2\r', + id='bom-unicode-and-mixed-newlines', + ), + pytest.param( + 'if True:\n x=0\n\\\nresult = 1 + 2\n', + id='codegen-omitted-continuation', + ), + ], +) +def test_source_offset_resolver_preserves_global_range_invariants(source): + """ + Every provider range stays bounded, ordered, monotone, shared-position consistent, and batch/single equivalent. + """ + wrapper = MetadataWrapper(parse_module(source)) + node_ranges = list(wrapper.resolve(WhitespaceInclusivePositionProvider).values()) + forward_resolver = SourceOffsetResolver(wrapper.module, source, node_ranges) + reverse_resolver = SourceOffsetResolver(wrapper.module, source, reversed(node_ranges)) + + for node_range in reversed(node_ranges): + reverse_resolver(node_range) + + position_offsets = {} + for node_range in node_ranges: + start_offset, end_offset = forward_resolver(node_range) + assert 0 <= start_offset <= end_offset <= len(source) + assert reverse_resolver(node_range) == (start_offset, end_offset) + + for position, offset in ((node_range.start, start_offset), (node_range.end, end_offset)): + if position in position_offsets: + assert position_offsets[position] == offset + else: + position_offsets[position] = offset + zero_width_range = CodeRange(position, position) + single_resolver = SourceOffsetResolver(parse_module(source), source, [zero_width_range]) + assert single_resolver(zero_width_range) == (offset, offset) + + ordered_offsets = [ + position_offsets[position] + for position in sorted(position_offsets, key=lambda position: (position.line, position.column)) + ] + assert ordered_offsets == sorted(ordered_offsets) + + +def test_code_fragments_cover_single_node_and_file_boundaries(): + """ + Nodes spanning or touching file boundaries leave the corresponding surrounding fragments empty. + """ + single_node_changer = Changer('1') + single_node_contexts = [] + + @single_node_changer.converter + def convert_single_node(node: Integer) -> Integer: + return node + + @single_node_changer.filter + def capture_single_node(node: Integer, context: Context) -> bool: # noqa: ARG001 + single_node_contexts.append(context) + return True + + assert len(list(single_node_changer.iterate_coordinates())) == 1 + single_node_context = single_node_contexts[0] + assert single_node_context.position.code_before == '' + assert single_node_context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + assert single_node_context.position.code_after == '' + assert single_node_context.position.code_before + '1' + single_node_context.position.code_after == single_node_context.position.source + + boundary_changer = Changer('x = 1') + first_node_contexts = [] + last_node_contexts = [] + + @boundary_changer.converter + def convert_first_node(node: Name) -> Name: + return node + + @boundary_changer.converter + def convert_last_node(node: Integer) -> Integer: + return node + + @boundary_changer.filter + def capture_first_node(node: Name, context: Context) -> bool: # noqa: ARG001 + first_node_contexts.append(context) + return True + + @boundary_changer.filter + def capture_last_node(node: Integer, context: Context) -> bool: # noqa: ARG001 + last_node_contexts.append(context) + return True + + assert len(list(boundary_changer.iterate_coordinates())) == 2 + first_node_context = first_node_contexts[0] + assert first_node_context.position.code_before == '' + assert first_node_context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + assert first_node_context.position.code_after == ' = 1' + assert first_node_context.position.code_before + 'x' + first_node_context.position.code_after == first_node_context.position.source + + last_node_context = last_node_contexts[0] + assert last_node_context.position.code_before == 'x = ' + assert last_node_context.position.node_range == CodeRange(CodePosition(1, 4), CodePosition(1, 5)) + assert last_node_context.position.code_after == '' + assert last_node_context.position.code_before + '1' + last_node_context.position.code_after == last_node_context.position.source + + +def test_code_fragments_partition_zero_width_parameters(): + """ + A zero-width Parameters node partitions non-empty source without shifting either fragment. + """ + source = 'def f():\n pass\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Parameters) -> Parameters: + return node + + @changer.filter + def capture(node: Parameters, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 6), CodePosition(1, 6)) + assert context.position.code_before == 'def f(' + assert context.position.code_after == '):\n pass\n' + assert context.position.code_before + '' + context.position.code_after == context.position.source + + +def test_code_fragments_handle_later_statement_without_final_newline(): + """ + At virtual EOF, a statement keeps the preceding CRLF and Context adds no final newline, unlike code_for_node(). + """ + source = 'head = 0\r\nx = 1' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: SimpleStatementLine) -> SimpleStatementLine: + return node + + @changer.filter + def capture(node: SimpleStatementLine, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 2 + context = contexts[1] + assert context.position.node_range == CodeRange(CodePosition(2, 0), CodePosition(3, 0)) + assert context.position.code_before == 'head = 0\r\n' + assert context.position.code_after == '' + assert context.position.code_before + 'x = 1' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_semicolon_owned_trivia(): + """ + The first Assign owns its trailing semicolon and space, excluding them from code_after. + """ + source = 'x=1; y=2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Assign) -> Assign: + return node + + @changer.filter + def capture(node: Assign, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 2 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(1, 5)) + assert context.position.code_before == '' + assert context.position.code_after == 'y=2\n' + assert context.position.code_before + 'x=1; ' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_decorators_and_surrounding_comments(): + """ + FunctionDef's span includes decorators, an indented body, and newlines but excludes surrounding comments. + """ + source = '# h\n@dec\ndef f(x: int = 1):\n return x\n# f\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: FunctionDef) -> FunctionDef: + return node + + @changer.filter + def capture(node: FunctionDef, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(2, 0), CodePosition(5, 0)) + assert context.position.code_before == '# h\n' + assert context.position.code_after == '# f\n' + assert context.position.code_before + '@dec\ndef f(x: int = 1):\n return x\n' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_nested_indentation_and_comments(): + """ + A nested statement span retains ambient indentation that Module.code_for_node() would omit. + + The exact partition also preserves its leading and inline comments and multiline assignment. + """ + source = '# header\nif True:\n # lead\n value = (\n 1 + 2 # inner\n )\n# footer\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: SimpleStatementLine) -> SimpleStatementLine: + return node + + @changer.filter + def capture(node: SimpleStatementLine, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(3, 0), CodePosition(7, 0)) + assert context.position.code_before == '# header\nif True:\n' + assert context.position.code_after == '# footer\n' + assert context.position.code_before + ' # lead\n value = (\n 1 + 2 # inner\n )\n' + context.position.code_after == context.position.source + + +def test_codegen_indent_does_not_search_forward_in_source(): + """ + A generated suite indent cannot jump to matching spaces later in source after an omitted continuation. + """ + source = 'if True:\n x=0\n\\\nresult = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Name) -> Name: + return node + + @changer.filter + def capture(node: Name, context: Context) -> bool: + if node.value != 'result': + return False + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(3, 4), CodePosition(3, 10)) + assert context.position.code_before == 'if True:\n x=0\n\\\n' + assert context.position.code_after == ' = 1 + 2\n' + assert context.position.code_before + 'result' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + ('indent', 'expected_range'), + [ + pytest.param('\t', CodeRange(CodePosition(2, 11), CodePosition(2, 14)), id='tab'), + pytest.param(' ', CodeRange(CodePosition(2, 12), CodePosition(2, 15)), id='two-spaces'), + ], +) +def test_code_fragments_follow_module_default_indent(indent, expected_range): + """ + Nested Add fragments use LibCST's inferred tab or two-space default indent instead of four hard-coded spaces. + """ + source = f'if True:\n{indent}result = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == expected_range + assert context.position.code_before == f'if True:\n{indent}result = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + 'newline', + [ + pytest.param('\r\n', id='crlf'), + pytest.param('\r', id='bare-cr'), + ], +) +def test_code_fragments_follow_module_default_newline_at_node_boundary(newline): + """ + An Assign ending immediately before CRLF or bare CR leaves the complete inferred newline in code_after. + """ + source = f'head=0{newline}x=1' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Assign) -> Assign: + return node + + @changer.filter + def capture(node: Assign, context: Context) -> bool: # noqa: ARG001 + if context.position.coordinate.start_line != 1: + return False + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(1, 6)) + assert context.position.code_before == '' + assert context.position.code_after == f'{newline}x=1' + assert context.position.code_before + 'head=0' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_multiline_parenthesized_expression(): + """ + A multiline BinaryOperation span includes its parentheses, continuation indentation, and inline comment. + """ + source = '# header\nif True:\n value = (\n 1 + 2 # inner\n )\n# footer\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: BinaryOperation) -> BinaryOperation: + return node + + @changer.filter + def capture(node: BinaryOperation, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(3, 12), CodePosition(5, 5)) + assert context.position.code_before == '# header\nif True:\n value = ' + assert context.position.code_after == '\n# footer\n' + assert context.position.code_before + '(\n 1 + 2 # inner\n )' + context.position.code_after == context.position.source + + +def test_code_fragments_select_correct_repeated_occurrence(): + """ + Distinct ranges partition identical Add occurrences instead of using the first textual match. + """ + source = 'foo = foo + foo\nfoo = foo + foo\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 2 + first_context, second_context = contexts + assert first_context.position.node_range == CodeRange(CodePosition(1, 9), CodePosition(1, 12)) + assert first_context.position.code_before == 'foo = foo' + assert first_context.position.code_after == 'foo\nfoo = foo + foo\n' + assert first_context.position.code_before + ' + ' + first_context.position.code_after == first_context.position.source + assert second_context.position.node_range == CodeRange(CodePosition(2, 9), CodePosition(2, 12)) + assert second_context.position.code_before == 'foo = foo + foo\nfoo = foo' + assert second_context.position.code_after == 'foo\n' + assert second_context.position.code_before + ' + ' + second_context.position.code_after == second_context.position.source + + +def test_code_fragments_map_later_node_across_mixed_newlines(): + """ + A third-line Add maps across preceding CRLF and bare CR while preserving the following LF. + """ + source = 'first=0\r\nsecond=1\rthird = 2 +\t3\nlast=4' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(3, 9), CodePosition(3, 13)) + assert context.position.code_before == 'first=0\r\nsecond=1\rthird = 2' + assert context.position.code_after == '3\nlast=4' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_unicode_before_later_node(): + """ + Non-BMP and combining characters before a later Add count as Python characters, not UTF-8 bytes. + """ + source = 'header = "🙂e\u0301"\nvalue = π +\t2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(2, 9), CodePosition(2, 13)) + assert context.position.code_before == 'header = "🙂e\u0301"\nvalue = π' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_code_fragments_count_same_line_unicode_as_python_characters(): + """ + Non-BMP and combining characters on Add's line each occupy one LibCST column and one source index. + """ + source = 'header = "🙂e\u0301"; value = π +\t2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 25), CodePosition(1, 29)) + assert context.position.code_before == 'header = "🙂e\u0301"; value = π' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_bom_before_later_node(): + """ + Absolute offsets count an initial BOM and Unicode header before a second-line Add. + + The BOM remains in code_before rather than shifting the Add's contextual span. + """ + source = '\ufeffheader = "🙂e\u0301"\nvalue = 1 +\t2\ntail = 3\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(2, 9), CodePosition(2, 13)) + assert context.position.start_offset == 25 + assert context.position.end_offset == 29 + assert context.position.code_before == '\ufeffheader = "🙂e\u0301"\nvalue = 1' + assert context.position.code_after == '2\ntail = 3\n' + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_internal_bom_does_not_shift_initial_source_offset(): + """ + A BOM inside a string does not trigger the one-character offset reserved for an initial BOM. + """ + source = 'prefix = "\ufeff"\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Name) -> Name: + return node + + @changer.filter + def capture(node: Name, context: Context) -> bool: + if node.value != 'prefix': + return False + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(1, 6)) + assert context.position.code_before == '' + assert context.position.code_after == ' = "\ufeff"\n' + assert context.position.code_before + 'prefix' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize('source', ['', 'x=1\n'], ids=['empty-source', 'ordinary-source']) +def test_module_fragments_cover_empty_and_nonempty_sources_completely(source): + """ + Module spans empty and non-empty sources, including an empty range, without surrounding fragments. + """ + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Module, context: Context) -> Module: + contexts.append(context) + return node + + coordinate = next(changer.iterate_coordinates()) + assert changer.apply_coordinate(coordinate) == source + assert len(contexts) == 1 + context = contexts[0] + expected_end_position = CodePosition(1, 0) if not source else CodePosition(2, 0) + assert context.position.node_range == CodeRange(CodePosition(1, 0), expected_end_position) + assert context.position.code_before == '' + assert context.position.code_after == '' + assert context.position.code_before + source + context.position.code_after == context.position.source + + +def test_module_range_leaves_bom_in_code_before(): + """ + LibCST excludes an initial U+FEFF BOM from Module's range; SourcePosition keeps it in code_before. + """ + source = '\ufeffx=1\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Module, context: Context) -> Module: + contexts.append(context) + return node + + coordinate = next(changer.iterate_coordinates()) + changer.apply_coordinate(coordinate) + assert len(contexts) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(2, 0)) + assert context.position.code_before == '\ufeff' + assert context.position.code_after == '' + assert context.position.code_before + 'x=1\n' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + ('source', 'expected_before', 'expected_module_span'), + [ + pytest.param('\f', '', '\f', id='form-feed-only'), + pytest.param(' \f', '', ' \f', id='space-and-form-feed'), + pytest.param('\ufeff\f', '\ufeff', '\f', id='bom-and-form-feed'), + ], +) +def test_module_fragments_keep_eof_form_feed_in_module_span(source, expected_before, expected_module_span): + """ + An EOF form feed remains in Module's span for whitespace-only files, with or without a BOM. + """ + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Module, context: Context) -> Module: + contexts.append(context) + return node + + coordinate = next(changer.iterate_coordinates()) + changer.apply_coordinate(coordinate) + assert len(contexts) == 1 + context = contexts[0] + assert context.position.code_before == expected_before + assert context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(2, 0)) + assert context.position.code_after == '' + assert context.position.code_before + expected_module_span + context.position.code_after == context.position.source + + +@pytest.mark.parametrize('prefix', [pytest.param('\f', id='form-feed'), pytest.param('\t\f', id='tab-and-form-feed')]) +def test_code_fragments_preserve_leading_form_feed_prefix(prefix): + """ + A codegen-omitted form-feed prefix remains in code_before without shifting Add's span, with or without a tab. + """ + source = f'{prefix}result = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 10), CodePosition(1, 14)) + assert context.position.code_before == f'{prefix}result = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + 'prefix', + [ + pytest.param('\\\n', id='lf-continuation'), + pytest.param('\\\r', id='cr-continuation'), + pytest.param('\\\r\n', id='crlf-continuation'), + pytest.param('\f\\\n', id='form-feed-and-continuation'), + pytest.param('\ufeff\\\n', id='bom-and-continuation'), + pytest.param('\\\n\t\f', id='continued-tab-and-form-feed'), + ], +) +def test_code_fragments_preserve_leading_line_continuation_prefix(prefix): + """ + Codegen-omitted LF, CR, and CRLF continuations stay in code_before across BOM, form-feed, and indentation variants. + """ + source = f'{prefix}result = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 10), CodePosition(1, 14)) + assert context.position.code_before == f'{prefix}result = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + 'prefix', + [ + pytest.param('\n\f', id='blank-line-before-form-feed'), + pytest.param('# lead\n \f', id='comment-before-form-feed'), + pytest.param('\n\\\n', id='blank-line-before-continuation'), + pytest.param('# lead\r\n\\\r\n', id='crlf-comment-before-continuation'), + ], +) +def test_code_fragments_preserve_codegen_omitted_prefix_after_blank_or_comment_line(prefix): + """ + An omitted form feed or continuation stays in code_before after a blank or comment line without shifting the later Add. + """ + source = f'{prefix}result = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(2, 10), CodePosition(2, 14)) + assert context.position.code_before == f'{prefix}result = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +def test_code_fragments_preserve_nested_form_feed_indentation(): + """ + A form feed in suite indentation affects LibCST columns without misaligning nested fragments. + """ + source = 'if True:\n\f result = 1 + 2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(2, 15), CodePosition(2, 19)) + assert context.position.code_before == 'if True:\n\f result = 1' + assert context.position.code_after == '2\n' + assert context.position.code_before + ' + ' + context.position.code_after == context.position.source + + +@pytest.mark.parametrize( + ('source', 'expected_range', 'expected_before', 'expected_after'), + [ + pytest.param( + 'value = (\n\f1 +\t2\n)\n', + CodeRange(CodePosition(2, 2), CodePosition(2, 6)), + 'value = (\n\f1', + '2\n)\n', + id='form-feed-in-parenthesized-expression', + ), + pytest.param( + 'value=(\n\\\n1 +\t2\n)\n', + CodeRange(CodePosition(3, 1), CodePosition(3, 5)), + 'value=(\n\\\n1', + '2\n)\n', + id='continuation-in-parenthesized-expression', + ), + pytest.param( + 's = f"""\n\f{1 +\t2}\n"""\n', + CodeRange(CodePosition(2, 3), CodePosition(2, 7)), + 's = f"""\n\f{1', + '2}\n"""\n', + id='form-feed-in-fstring', + ), + pytest.param( + 's = """\n\\\ntext\n"""\nresult=π +\t2\n', + CodeRange(CodePosition(5, 8), CodePosition(5, 12)), + 's = """\n\\\ntext\n"""\nresult=π', + '2\n', + id='continuation-in-preceding-triple-string', + ), + pytest.param( + 'if True:\n x = 0\n\\\nresult = 1 +\t2\n', + CodeRange(CodePosition(3, 14), CodePosition(3, 18)), + 'if True:\n x = 0\n\\\nresult = 1', + '2\n', + id='continuation-after-indented-suite', + ), + ], +) +def test_code_fragments_preserve_alignment_across_parser_contexts(source, expected_range, expected_before, expected_after): + """ + Exact fragments verify Add alignment across form feeds and continuations inside parentheses and f-strings, after a triple-quoted string, and after an indented suite. + """ + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: Add) -> Add: + return node + + @changer.filter + def capture(node: Add, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == expected_range + assert context.position.code_before == expected_before + assert context.position.code_after == expected_after + assert context.position.code_before + ' +\t' + context.position.code_after == context.position.source + + +def test_code_fragments_align_nested_string_matching_enclosing_fstring_quotes(): + """ + A nested SimpleString sharing its f-string's quotes includes enclosing parentheses but excludes the closing brace. + """ + source = 'value = f"""{("""inner\n\fbody""")}\ntail"""\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: SimpleString) -> SimpleString: + return node + + @changer.filter + def capture(node: SimpleString, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 1 + context = contexts[0] + assert context.position.node_range == CodeRange(CodePosition(1, 13), CodePosition(2, 9)) + assert context.position.code_before == 'value = f"""{' + assert context.position.code_after == '}\ntail"""\n' + assert context.position.code_before + '("""inner\n\fbody""")' + context.position.code_after == context.position.source + + +def test_pipeline_and_direct_source_positions_partition_form_feed_identically(): + """ + Pipeline and directly constructed SourcePositions partition an omitted form feed identically. + """ + source = 'x=1\n\fresult=2\n' + changer = Changer(source) + contexts = [] + + @changer.converter + def convert(node: SimpleStatementLine) -> SimpleStatementLine: + return node + + @changer.filter + def capture(node: SimpleStatementLine, context: Context) -> bool: # noqa: ARG001 + contexts.append(context) + return True + + assert len(list(changer.iterate_coordinates())) == 2 + first_context, second_context = contexts + assert first_context.position.node_range == CodeRange(CodePosition(1, 0), CodePosition(2, 0)) + assert first_context.position.code_before == '' + assert first_context.position.code_after == 'result=2\n' + assert first_context.position.code_before + 'x=1\n\f' + first_context.position.code_after == first_context.position.source + direct_resolver = SourceOffsetResolver( + parse_module(first_context.position.source), + first_context.position.source, + [first_context.position.node_range], + ) + direct_position = SourcePosition( + first_context.position.coordinate, + first_context.position.source, + first_context.position.node_range, + direct_resolver, + ) + assert direct_position == first_context.position + assert direct_position.code_before == first_context.position.code_before + assert direct_position.code_after == first_context.position.code_after + assert second_context.position.node_range == CodeRange(CodePosition(2, 0), CodePosition(3, 0)) + assert second_context.position.code_before == 'x=1\n\f' + assert second_context.position.code_after == '' + assert second_context.position.code_before + 'result=2\n' + second_context.position.code_after == second_context.position.source diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index a9099a7..dc0c520 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -1,3 +1,8 @@ +from libcst import Integer +from libcst.metadata import CodePosition, CodeRange + +from cstvis import Context, Coordinate +from cstvis.dto import SourcePosition from cstvis.wrapper import CallableWrapper @@ -12,3 +17,87 @@ def function(a, b): assert repr(CallableWrapper(function)) == 'CallableWrapper(function)' assert repr(CallableWrapper(function, meta={'kek': 1234})) == "CallableWrapper(function, {'kek': 1234})" + + +def test_callable_wrapper_copies_meta_without_mutating_base_context(): + """ + A context-aware callback receives copied decorator meta without changing the base Context or caller-owned dictionary. + """ + decorator_meta = {'mode': 'wrapped'} + base_meta = {'mode': 'base'} + coordinate = Coordinate(None, 'Integer', 1, 0, 1, 1) + node_range = CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + position = SourcePosition(coordinate, '1', node_range, lambda _node_range: (0, 1)) + base_context = Context(position, None, base_meta) + received_contexts = [] + + def callback(node: Integer, context: Context) -> Integer: + received_contexts.append(context) + return node + + wrapper = CallableWrapper(callback, decorator_meta) + node = Integer('1') + + assert wrapper(node, base_context) is node + assert len(received_contexts) == 1 + callback_context = received_contexts[0] + assert callback_context.meta == decorator_meta + assert callback_context.meta is not decorator_meta + assert callback_context.position is position + assert base_context.meta is base_meta + assert base_meta == {'mode': 'base'} + assert wrapper.meta is decorator_meta + assert decorator_meta == {'mode': 'wrapped'} + + +def test_callable_wrapper_creates_fresh_meta_for_each_invocation(): + """ + Each call receives fresh meta, isolating mutations from later calls and wrapper state. + """ + decorator_meta = {'value': 'original'} + coordinate = Coordinate(None, 'Integer', 1, 0, 1, 1) + node_range = CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + position = SourcePosition(coordinate, '1', node_range, lambda _node_range: (0, 1)) + base_context = Context(position, None, {'base': True}) + received_meta = [] + + def callback(node: Integer, context: Context) -> Integer: + assert context.meta is not None + received_meta.append(context.meta) + if len(received_meta) == 1: + context.meta['value'] = 'changed' + return node + + wrapper = CallableWrapper(callback, decorator_meta) + wrapper(Integer('1'), base_context) + wrapper(Integer('1'), base_context) + + first_callback_meta, second_callback_meta = received_meta + assert first_callback_meta is not second_callback_meta + assert first_callback_meta is not decorator_meta + assert second_callback_meta is not decorator_meta + assert first_callback_meta == {'value': 'changed'} + assert second_callback_meta == {'value': 'original'} + assert decorator_meta == {'value': 'original'} + assert wrapper.meta == {'value': 'original'} + assert base_context.meta == {'base': True} + + +def test_callable_wrapper_with_no_meta_passes_none(): + """ + A wrapper without decorator meta passes meta=None without mutating the base Context. + """ + coordinate = Coordinate(None, 'Integer', 1, 0, 1, 1) + node_range = CodeRange(CodePosition(1, 0), CodePosition(1, 1)) + position = SourcePosition(coordinate, '1', node_range, lambda _node_range: (0, 1)) + base_context = Context(position, None, {'base': True}) + received_meta = [] + + def callback(node: Integer, context: Context) -> Integer: + received_meta.append(context.meta) + return node + + CallableWrapper(callback)(Integer('1'), base_context) + + assert received_meta == [None] + assert base_context.meta == {'base': True}