From 68ef0a68ca5da58d6fc051d3947d65553c46f669 Mon Sep 17 00:00:00 2001
From: Daniel Ecer <d.ecer@elifesciences.org>
Date: Wed, 19 Aug 2026 15:20:05 +0100
Subject: [PATCH] Rebuild training data B-/I- from element boundaries

AbstractTrainingTeiParser assigned the entity prefix only when the label
changed, and skipped whitespace-only text nodes without touching the previous
label. Two sibling elements sharing a label, with only whitespace between
them, therefore came back as one entity: the second element's start was lost,
and generate_delft_data maps our B- to GROBID's I- start marker, so a start
the parser never emitted could not reappear in the artifact. Where a boundary
did survive it survived incidentally - any non-whitespace separator is
labelled O, which changed the previous label and restored the next start, so
two idno elements separated by a comma were kept apart and the same pair
separated by a space were not.

is_start on TeiTrainingText already means "first text run in this element",
including across <lb/>, so the guard is all that had to go. A whitespace-only
run no longer spends the flag either, which is what a single space before a
nested <label> used to cost.

The semantics is one entity per element occurrence: text returning to a
labelled parent after a labelled child continues the parent's entity. That is
the inverse of what the writer does, where a B- opens a new sibling element,
and it keeps <paragraph> whole across the markers the fulltext model nests
inside it. GROBID's SAX parsers declare begin inside writeField and so start a
field per text run, which would split a paragraph at every citation marker.

Over the committed generated corpus the reference segmenter gains 1786
reference starts - 624 to 1678 for ore, 911 to 1643 for
scielo_preprints-jats - and the header model 10. Citation,
affiliation-address and segmentation are unchanged, every label and every
document. On a collapsed document the delft output differs in the label column
alone: 152 <reference> become I-<reference> and no token moves.

The citation identifier test asserted the merged value as known behaviour; it
now round-trips both identifiers.
---
 sciencebeam_parser/models/training_data.py    | 12 ++---
 tests/models/citation/training_data_test.py   | 24 +++++++--
 tests/models/fulltext/training_data_test.py   | 17 +++++++
 .../reference_segmenter/training_data_test.py | 51 +++++++++++++++++++
 tests/models/training_data_test.py            | 45 ++++++++++++++++
 5 files changed, 138 insertions(+), 11 deletions(-)

diff --git a/sciencebeam_parser/models/training_data.py b/sciencebeam_parser/models/training_data.py
index aed4d56d..cbee105a 100644
--- a/sciencebeam_parser/models/training_data.py
+++ b/sciencebeam_parser/models/training_data.py
@@ -433,7 +433,8 @@ def _iter_flat_tei_training_text_from_element(
             path=current_path,
             is_start=is_start
         )
-        is_start = False
+        if not parent_element.text.isspace():
+            is_start = False
 
     for child_element in parent_element:
         if is_line_break_element(child_element):
@@ -451,7 +452,8 @@ def _iter_flat_tei_training_text_from_element(
                 path=current_path,
                 is_start=is_start
             )
-            is_start = False
+            if not child_element.tail.isspace():
+                is_start = False
 
 
 def _iter_tei_training_lines_from_element(
@@ -593,8 +595,6 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
                 )
             )
             LOGGER.debug('tei_training_lines: %r', tei_training_lines)
-            prefix = ''
-            prev_label = ''
             for line_index, line in enumerate(tei_training_lines):
                 line_meta = LayoutLineMeta(line_id=1 + line_index)
                 for text in line.text_list:
@@ -603,14 +603,12 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
                     token_count = 0
                     if text.path.element_list:
                         label = self.get_label_for_element_path(text.path, text=text.text)
-                        if prev_label != label:
-                            prefix = 'B-' if text.is_start else 'I-'
+                        prefix = 'B-' if text.is_start else 'I-'
                     else:
                         label = 'O'
                         prefix = ''
                     if label in OTHER_LABELS:
                         prefix = ''
-                    prev_label = label
                     for token_text in get_tokenized_tokens(text.text):
                         yield LabeledLayoutToken(
                             label=prefix + label,
diff --git a/tests/models/citation/training_data_test.py b/tests/models/citation/training_data_test.py
index 93055373..3127b87e 100644
--- a/tests/models/citation/training_data_test.py
+++ b/tests/models/citation/training_data_test.py
@@ -390,6 +390,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<title>')
         ]]
 
+    def test_should_start_a_new_entity_for_sibling_identifiers_separated_by_whitespace(self):
+        tei_root = _get_training_tei_with_references([
+            TEI_E('bibl', *[
+                TEI_E('idno', TOKEN_1),
+                ' ',
+                TEI_E('idno', TOKEN_2, TEI_E('lb')),
+                '\n'
+            ])
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<pubnum>'),
+            (TOKEN_2, 'B-<pubnum>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_references([
             TEI_E('bibl', *[
@@ -567,9 +584,7 @@ def test_should_round_trip_multiple_identifiers_of_one_reference(self):
             (SemanticExternalIdentifierTypes.PMID, '1234567')
         ]
 
-    def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
-        # the parser reconstructs B-/I- from label changes rather than element boundaries,
-        # so two idno elements with only whitespace between them come back as one identifier
+    def test_should_round_trip_directly_adjacent_identifiers_of_one_reference(self):
         label_and_layout_line_list = [
             (IDENTIFIER_LABEL, get_next_layout_line_for_text('10.1234/test')),
             (IDENTIFIER_LABEL, get_next_layout_line_for_text('PMID: 1234567'))
@@ -585,7 +600,8 @@ def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
         references = get_semantic_references_for_training_tei_xml(xml_root)
         assert len(references) == 1
         assert get_external_identifier_types_and_values(references[0]) == [
-            (SemanticExternalIdentifierTypes.DOI, '10.1234/testPMID:1234567')
+            (SemanticExternalIdentifierTypes.DOI, '10.1234/test'),
+            (SemanticExternalIdentifierTypes.PMID, '1234567')
         ]
 
     @pytest.mark.parametrize(
diff --git a/tests/models/fulltext/training_data_test.py b/tests/models/fulltext/training_data_test.py
index 27af74a7..cd214b3f 100644
--- a/tests/models/fulltext/training_data_test.py
+++ b/tests/models/fulltext/training_data_test.py
@@ -360,6 +360,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<paragraph>')
         ]]
 
+    def test_should_continue_the_paragraph_after_a_nested_citation_marker(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, ' ', E('ref', {'type': 'biblio'}, TOKEN_2), ' ', TOKEN_3, E('lb')),
+            '\n',
+            E('p', TOKEN_4, E('lb')),
+            '\n'
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'B-<citation_marker>'),
+            (TOKEN_3, 'I-<paragraph>'),
+            (TOKEN_4, 'B-<paragraph>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_text([
             E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')),
diff --git a/tests/models/reference_segmenter/training_data_test.py b/tests/models/reference_segmenter/training_data_test.py
index bc95bfda..4d13cf7d 100644
--- a/tests/models/reference_segmenter/training_data_test.py
+++ b/tests/models/reference_segmenter/training_data_test.py
@@ -295,6 +295,57 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<reference>')
         ]]
 
+    def test_should_start_a_new_reference_for_each_bibl_without_a_label(self):
+        tei_root = _get_training_tei_with_references([
+            E('bibl', TOKEN_1, E('lb')),
+            '\n',
+            E('bibl', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<reference>'),
+            (TOKEN_2, 'B-<reference>')
+        ]]
+
+    def test_should_start_the_reference_after_whitespace_preceding_the_label(self):
+        tei_root = _get_training_tei_with_references([E(
+            'bibl',
+            ' ',
+            E('label', TOKEN_1),
+            ' ',
+            TOKEN_2,
+            E('lb'),
+            '\n'
+        )])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<label>'),
+            (TOKEN_2, 'B-<reference>')
+        ]]
+
+    def test_should_keep_a_reference_spanning_multiple_lines_as_one_entity(self):
+        tei_root = _get_training_tei_with_references([E(
+            'bibl',
+            TOKEN_1,
+            E('lb'),
+            '\n',
+            TOKEN_2,
+            E('lb'),
+            '\n'
+        )])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<reference>'),
+            (TOKEN_2, 'I-<reference>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_references([E(
             'bibl',
diff --git a/tests/models/training_data_test.py b/tests/models/training_data_test.py
index d4ddb323..ef5823a5 100644
--- a/tests/models/training_data_test.py
+++ b/tests/models/training_data_test.py
@@ -171,6 +171,51 @@ def test_should_raise_error_for_element_with_more_than_one_matching_attribute(se
                 tei_root
             )
 
+    def test_should_start_a_new_entity_for_each_sibling_element_with_the_same_label(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb')),
+            '\n',
+            E('p', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'B-<paragraph>')
+        ]]
+
+    @pytest.mark.parametrize('separator', ['\n', ' ', '\n  ', ', '])
+    def test_should_start_a_new_entity_for_a_sibling_element_whatever_separates_them(
+        self, separator: str
+    ):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb')),
+            separator,
+            E('p', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        label_by_token = dict(tag_result[0])
+        assert label_by_token[TOKEN_1] == 'B-<paragraph>'
+        assert label_by_token[TOKEN_2] == 'B-<paragraph>'
+
+    def test_should_keep_an_element_spanning_multiple_lines_as_one_entity(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'I-<paragraph>')
+        ]]
+
     def test_unannotated_tokens_after_annotated_span_use_O(self):
         # training_data.py emits plain 'O' for all unannotated tokens regardless of
         # position. The I-<other> GROBID convention is applied later by