From d4dce70d2a52ad7fee8599bc8b8c433de37d04f8 Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Thu, 20 Aug 2026 08:47:44 +0100 Subject: [PATCH 01/12] Add an opt-in LLM engine for the reference models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A third sequence-model engine alongside wapiti and delft, reachable only through one of three opt-in profiles: llm_reference_segmenter, llm_citation, or llm_references for both. One profile per model so that a failing run says which model produced it. The shipped default stays grobid_crf, and a default install acquires no network dependency, credential requirement or new failure mode — asserted by test rather than intended, including that every other model in each profile stays on wapiti. No text reaches a document that was not in the source. The segmenter returns line numbers into numbered input and never text at all. The citation model returns field values, which the engine locates back in the token sequence: a value that cannot be found, or whose tokens an earlier field already claimed, raises. Either way the existing seam in Model independently rejects any result whose tokens are not the input tokens. The two shapes are opposite on purpose. Whatever the model must emit verbatim is what breaks at scale, so the shape follows the ratio of output to input: a citation response is shorter than its input, a segmentation response quoting reference text is as long as it. response_shape is therefore configuration, so comparing shapes is defining a second profile rather than building a second evaluation route. An invalid response raises rather than falling back to a CRF engine, since a score is only meaningful if every label came from the model under test. A response cut off at the output limit raises its own error naming finish_reason and the completion token count, because "truncated" and "malformed" want different fixes and reporting the first as the second sends you looking for a parsing bug. Every request pins zero data retention and fails closed, and a :free model id is refused at load because that tier requires allowing training on prompts. Three details that came from measurement rather than taste. The line_status column is resolved by name through the task's own data generator, because the feature layout differs per model and a wrong column produces plausible output with no exception. Predicted boundaries snap onto a preceding bare-label line, because models point at the line where reference text starts while the training convention puts the boundary on the line holding the number — and stating that in the prompt did not fix it. Values are matched on word tokens at the earliest unclaimed occurrence rather than in document order, because models write "Treble-Barna" where the tokeniser emits three tokens, and emit date out of position often enough that requiring order rejects values the source contains. The citation label vocabulary is read from TRAINING_XML_ELEMENT_PATH_BY_LABEL rather than restated, so it cannot drift from what the extractor understands. Checked end to end against qwen3.5-9b: 33 references against a gold of 33 over 1502 tokens for the segmenter, and 0.865 token accuracy over 223 tokens for citation, with tokens returned unchanged in both. --- doc/llm_engine.md | 79 +++++++++ sciencebeam_parser/models/llm/__init__.py | 0 sciencebeam_parser/models/llm/client.py | 140 ++++++++++++++++ sciencebeam_parser/models/llm/config.py | 67 ++++++++ sciencebeam_parser/models/llm/decode.py | 147 +++++++++++++++++ sciencebeam_parser/models/llm/features.py | 47 ++++++ sciencebeam_parser/models/llm/model_impl.py | 129 +++++++++++++++ sciencebeam_parser/models/llm/prompt.py | 25 +++ .../models/llm/prompts/citation/values-v1.md | 36 ++++ .../prompts/reference_segmenter/lines-v1.md | 12 ++ sciencebeam_parser/models/llm/tasks.py | 15 ++ sciencebeam_parser/models/llm/values.py | 115 +++++++++++++ .../models/model_impl_factory.py | 15 +- .../resources/default_config/config.yml | 48 ++++++ tests/models/llm/__init__.py | 0 tests/models/llm/client_test.py | 44 +++++ tests/models/llm/config_test.py | 53 ++++++ tests/models/llm/decode_test.py | 128 +++++++++++++++ tests/models/llm/model_impl_test.py | 154 ++++++++++++++++++ tests/models/llm/shipped_config_test.py | 84 ++++++++++ tests/models/llm/values_test.py | 117 +++++++++++++ tests/models/model_impl_factory_test.py | 26 ++- 22 files changed, 1479 insertions(+), 2 deletions(-) create mode 100644 doc/llm_engine.md create mode 100644 sciencebeam_parser/models/llm/__init__.py create mode 100644 sciencebeam_parser/models/llm/client.py create mode 100644 sciencebeam_parser/models/llm/config.py create mode 100644 sciencebeam_parser/models/llm/decode.py create mode 100644 sciencebeam_parser/models/llm/features.py create mode 100644 sciencebeam_parser/models/llm/model_impl.py create mode 100644 sciencebeam_parser/models/llm/prompt.py create mode 100644 sciencebeam_parser/models/llm/prompts/citation/values-v1.md create mode 100644 sciencebeam_parser/models/llm/prompts/reference_segmenter/lines-v1.md create mode 100644 sciencebeam_parser/models/llm/tasks.py create mode 100644 sciencebeam_parser/models/llm/values.py create mode 100644 tests/models/llm/__init__.py create mode 100644 tests/models/llm/client_test.py create mode 100644 tests/models/llm/config_test.py create mode 100644 tests/models/llm/decode_test.py create mode 100644 tests/models/llm/model_impl_test.py create mode 100644 tests/models/llm/shipped_config_test.py create mode 100644 tests/models/llm/values_test.py diff --git a/doc/llm_engine.md b/doc/llm_engine.md new file mode 100644 index 00000000..02db251b --- /dev/null +++ b/doc/llm_engine.md @@ -0,0 +1,79 @@ +# LLM engine (experimental) + +A third sequence-model engine alongside `wapiti` and `delft`, serving the `reference_segmenter` and +`citation` models. It is **opt-in**: the shipped default profile stays `grobid_crf`, and a default +install acquires no network dependency and no credential requirement. + +## Using it + +```sh +export OPENROUTER_API_KEY=... # or SCIENCEBEAM_LLM_API_KEY +export SCIENCEBEAM_PARSER__PROFILE=llm_reference_segmenter +``` + +Override the profile by environment rather than editing `profile:` in `config.yml`. The shipped +default is asserted by test, so changing it there fails the build. Env keys use the +`SCIENCEBEAM_PARSER__` prefix with `__` between levels, so a single setting can be overridden the +same way — for example +`SCIENCEBEAM_PARSER__SEQUENCE_MODEL_PROFILES__LLM_REFERENCE_SEGMENTER__CITATION__MODEL`. + +Three profiles, all extending `grobid_crf_0_9_0` so every other model stays on wapiti: + +| profile | replaces | +| --- | --- | +| `llm_reference_segmenter` | `reference_segmenter` | +| `llm_citation` | `citation` | +| `llm_references` | both | + +One per model matters for attribution: when a run fails, the per-model profiles say which model did +it without having to read a stack trace. + +## Configuration + +```yaml +reference_segmenter: + engine: 'llm' + task: 'reference_segmenter' # selects the prompt and the feature layout + response_shape: 'lines' # line numbers where each reference begins + model: 'qwen/qwen3.5-9b' + provider: 'siliconflow' # pinned; routing fails closed without a match + prompt_version: 'lines-v1' # sciencebeam_parser/models/llm/prompts//.md + reasoning: 'off' # models that think by default must be told not to +citation: + engine: 'llm' + task: 'citation' + response_shape: 'values' # field values, located back in the token sequence + model: 'qwen/qwen3.5-9b' + provider: 'siliconflow' + prompt_version: 'values-v1' + reasoning: 'off' +``` + +`response_shape` is configuration rather than a fixed choice, because the best shape differs by +model and by task and moves with each new checkpoint. Comparing shapes is therefore defining a +second profile and running the benchmark, not building a second evaluation route. + +Also accepted: `endpoint` (any OpenAI-compatible base URL, so a self-hosted vLLM works), +`temperature`, `timeout_seconds`, `max_output_tokens`, `max_attempts`, `extra_body`. + +## What it guarantees + +No text reaches a document that was not in the source. Under `lines` the model returns line numbers +and never text at all. Under `values` it returns text, and every value is located back in the token +sequence — one that cannot be found, or that a previous field already claimed, raises. Either way +`Model._iter_flat_label_model_data_lists_to` independently rejects any result whose tokens are not +the input tokens. + +The `citation` label vocabulary is read from the model's own label map rather than restated in the +prompt source, so it cannot drift from the labels the extractor understands. + +Every request enforces zero data retention — `zdr`, `data_collection: deny`, +`allow_fallbacks: false`, `require_parameters: true`, and `only: [provider]` when pinned. A `:free` +model id is refused at load, because that tier requires allowing training on prompts. + +A response that cannot be decoded raises. There is no fallback to a CRF engine and no partial +labelling: a score is only meaningful if every label came from the model under test. + +A response cut off at the output limit raises `LlmTruncatedResponseError` naming +`finish_reason`, the completion token count and the task, rather than surfacing as a JSON parse +error. Raise `max_output_tokens`, or send less per request. diff --git a/sciencebeam_parser/models/llm/__init__.py b/sciencebeam_parser/models/llm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sciencebeam_parser/models/llm/client.py b/sciencebeam_parser/models/llm/client.py new file mode 100644 index 00000000..220fed77 --- /dev/null +++ b/sciencebeam_parser/models/llm/client.py @@ -0,0 +1,140 @@ +import logging +import time +from typing import Any, Dict, Mapping, Protocol + +import httpx + +from sciencebeam_parser.models.llm.config import LlmEngineConfig, get_api_key + + +LOGGER = logging.getLogger(__name__) + +RETRY_STATUS_CODES = frozenset({408, 429, 500, 502, 503, 504}) + + +class LlmRequestError(RuntimeError): + pass + + +class LlmCompletionClient(Protocol): + def validate_configuration(self) -> None: + ... + + def get_completion( + self, prompt: str, response_schema: Mapping[str, Any] + ) -> Mapping[str, Any]: + ... + + +class LlmClient: + def __init__(self, config: LlmEngineConfig): + self.config = config + + def _headers(self) -> Dict[str, str]: + return {'Authorization': f'Bearer {get_api_key()}'} + + def _request_body(self, prompt: str, response_schema: Mapping[str, Any]) -> Dict[str, Any]: + body: Dict[str, Any] = { + 'model': self.config.model, + 'temperature': self.config.temperature, + 'max_tokens': self.config.max_output_tokens, + 'messages': [{'role': 'user', 'content': prompt}], + 'response_format': { + 'type': 'json_schema', + 'json_schema': { + 'name': 'sciencebeam_labels', + 'strict': True, + 'schema': response_schema, + }, + }, + 'provider': self.config.provider_routing, + **self.config.extra_body, + } + if self.config.reasoning == 'off': + body['reasoning'] = {'enabled': False} + return body + + def validate_configuration(self) -> None: + """Fails at load rather than at first request, and spends no tokens.""" + url = f'{self.config.endpoint.rstrip("/")}/models' + try: + with httpx.Client(timeout=self.config.timeout_seconds) as client: + response = client.get(url, headers=self._headers()) + except httpx.HTTPError as exc: + raise LlmRequestError(f'{self.config.endpoint} is not reachable: {exc}') from exc + if response.status_code != 200: + raise LlmRequestError( + f'{url} returned {response.status_code}: {response.text[:200]}' + ) + model_ids = { + entry.get('id') for entry in response.json().get('data', []) + if isinstance(entry, dict) + } + if model_ids and self.config.model not in model_ids: + raise LlmRequestError( + f'{self.config.endpoint} does not offer model {self.config.model!r}' + ) + LOGGER.info( + 'llm engine configured: model=%r provider=%r prompt=%r shape=%r', + self.config.model, self.config.provider, self.config.prompt_version, + self.config.response_shape + ) + + def get_completion(self, prompt: str, response_schema: Mapping[str, Any]) -> Mapping[str, Any]: + return self._post_with_retry(prompt, response_schema) + + def _post_with_retry( + self, prompt: str, response_schema: Mapping[str, Any] + ) -> Mapping[str, Any]: + url = f'{self.config.endpoint.rstrip("/")}/chat/completions' + body = self._request_body(prompt, response_schema) + last_error = '' + for attempt in range(self.config.max_attempts): + if attempt: + time.sleep(2 ** attempt) + try: + with httpx.Client(timeout=self.config.timeout_seconds) as client: + response = client.post(url, headers=self._headers(), json=body) + except httpx.HTTPError as exc: + last_error = f'{type(exc).__name__}: {exc}' + continue + if response.status_code in RETRY_STATUS_CODES: + last_error = f'http {response.status_code}: {response.text[:200]}' + continue + if response.status_code != 200: + raise LlmRequestError( + f'http {response.status_code}: {response.text[:200]}' + ) + return response.json() + raise LlmRequestError( + f'giving up after {self.config.max_attempts} attempts: {last_error}' + ) + + +class LlmTruncatedResponseError(LlmRequestError): + pass + + +def get_response_content(response_json: Mapping[str, Any]) -> str: + choices = response_json.get('choices') + if not choices: + raise LlmRequestError(f'response has no choices: {str(response_json)[:200]}') + choice = choices[0] + content = choice.get('message', {}).get('content') + finish_reason = choice.get('finish_reason') or choice.get('native_finish_reason') + if finish_reason == 'length': + completion_tokens = ( + response_json.get('usage', {}).get('completion_tokens') + ) + raise LlmTruncatedResponseError( + 'response hit the output token limit' + f' (finish_reason={finish_reason!r},' + f' completion_tokens={completion_tokens},' + f' chars={len(content or "")});' + ' raise max_output_tokens or reduce the input per request' + ) + if not content: + raise LlmRequestError( + f'response content is empty (finish_reason={finish_reason!r})' + ) + return content diff --git a/sciencebeam_parser/models/llm/config.py b/sciencebeam_parser/models/llm/config.py new file mode 100644 index 00000000..d4449221 --- /dev/null +++ b/sciencebeam_parser/models/llm/config.py @@ -0,0 +1,67 @@ +import os +from dataclasses import dataclass, field, fields +from typing import Any, Dict, Mapping, Optional + + +API_KEY_ENV_NAMES = ('SCIENCEBEAM_LLM_API_KEY', 'OPENROUTER_API_KEY') + +DEFAULT_ENDPOINT = 'https://openrouter.ai/api/v1' + + +class LlmConfigError(ValueError): + pass + + +@dataclass(frozen=True) +class LlmEngineConfig: + task: str + model: str + prompt_version: str + response_shape: str = 'lines' + endpoint: str = DEFAULT_ENDPOINT + provider: Optional[str] = None + reasoning: str = '' + temperature: float = 0.0 + timeout_seconds: float = 300.0 + max_output_tokens: int = 8000 + max_attempts: int = 4 + extra_body: Dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_model_config(config: Mapping[str, Any]) -> 'LlmEngineConfig': + for required in ('task', 'model', 'prompt_version'): + if not config.get(required): + raise LlmConfigError(f'llm engine requires {required!r} in the model config') + model = config['model'] + if model.endswith(':free'): + raise LlmConfigError( + f'refusing model id {model!r}: the free tier requires allowing training on' + ' prompts, which the zero-retention requirement forbids' + ) + known = {field_.name for field_ in fields(LlmEngineConfig)} + return LlmEngineConfig(**{ + key: value for key, value in config.items() + if key in known + }) + + @property + def provider_routing(self) -> Dict[str, Any]: + routing: Dict[str, Any] = { + 'zdr': True, + 'data_collection': 'deny', + 'allow_fallbacks': False, + 'require_parameters': True, + } + if self.provider: + routing['only'] = [self.provider] + return routing + + +def get_api_key() -> str: + for name in API_KEY_ENV_NAMES: + value = os.environ.get(name) + if value: + return value + raise LlmConfigError( + 'no api key: set one of ' + ', '.join(API_KEY_ENV_NAMES) + ) diff --git a/sciencebeam_parser/models/llm/decode.py b/sciencebeam_parser/models/llm/decode.py new file mode 100644 index 00000000..31c5e686 --- /dev/null +++ b/sciencebeam_parser/models/llm/decode.py @@ -0,0 +1,147 @@ +import json +import re +from typing import Any, List, Mapping, Sequence, Tuple + + +LINE_START = 'LINESTART' + +LABEL_ONLY_LINE = re.compile(r'^[\[(]?\d{1,3}[\])]?[.)]?$') + +LINES_RESPONSE_SCHEMA: Mapping[str, Any] = { + 'type': 'object', + 'additionalProperties': False, + 'required': ['starts'], + 'properties': { + 'starts': { + 'type': 'array', + 'items': {'type': 'integer'}, + }, + }, +} + + +class LlmResponseError(ValueError): + pass + + +def get_line_numbers(line_status_values: Sequence[str]) -> List[int]: + line_numbers: List[int] = [] + current = -1 + for index, status in enumerate(line_status_values): + if status == LINE_START or index == 0: + current += 1 + line_numbers.append(current) + return line_numbers + + +def get_lines(tokens: Sequence[str], line_numbers: Sequence[int]) -> List[List[str]]: + lines: List[List[str]] = [] + for token, line_number in zip(tokens, line_numbers): + while len(lines) <= line_number: + lines.append([]) + lines[line_number].append(token) + return lines + + +def render_numbered_lines(tokens: Sequence[str], line_numbers: Sequence[int]) -> str: + return '\n'.join( + f'{number}\t' + ' '.join(line_tokens) + for number, line_tokens in enumerate(get_lines(tokens, line_numbers)) + ) + + +def parse_line_starts(content: str, line_count: int) -> List[int]: + try: + payload = json.loads(content) + except ValueError as exc: + raise LlmResponseError(f'response is not json: {exc}') from exc + if not isinstance(payload, dict) or 'starts' not in payload: + raise LlmResponseError('response has no "starts"') + starts = payload['starts'] + if not isinstance(starts, list) or not starts: + raise LlmResponseError('"starts" is empty or not a list') + resolved: List[int] = [] + for value in starts: + if isinstance(value, bool) or not isinstance(value, int): + raise LlmResponseError(f'line number is not an integer: {value!r}') + if not 0 <= value < line_count: + raise LlmResponseError( + f'line number {value} out of range for {line_count} lines' + ) + resolved.append(value) + if resolved != sorted(set(resolved)): + raise LlmResponseError(f'line numbers are not strictly ascending: {resolved}') + return resolved + + +def snap_starts_to_label_lines( + line_starts: Sequence[int], + lines: Sequence[Sequence[str]] +) -> List[int]: + existing = set(line_starts) + snapped: List[int] = [] + for start in line_starts: + previous = start - 1 + if ( + previous >= 0 + and previous not in existing + and previous not in snapped + and LABEL_ONLY_LINE.match(''.join(lines[previous])) + ): + snapped.append(previous) + continue + snapped.append(start) + return sorted(snapped) + + +def iter_labels_for_line_starts( + tokens: Sequence[str], + line_numbers: Sequence[int], + line_starts: Sequence[int] +) -> List[str]: + lines = get_lines(tokens, line_numbers) + starts = set(line_starts) + labels: List[str] = [] + started = False + for token_index, line_number in enumerate(line_numbers): + is_line_start = token_index == 0 or line_numbers[token_index - 1] != line_number + if line_number in starts and is_line_start: + started = True + if LABEL_ONLY_LINE.match(''.join(lines[line_number])): + labels.append('B-