Rebuild training data B-/I- from element boundaries

sciencebeam_parser/models/training_data.py

@@ -433,7 +433,8 @@ def _iter_flat_tei_training_text_from_element(
             path=current_path,
             is_start=is_start
         )
-        is_start = False
+        if not parent_element.text.isspace():
+            is_start = False
 
     for child_element in parent_element:
         if is_line_break_element(child_element):
@@ -451,7 +452,8 @@ def _iter_flat_tei_training_text_from_element(
                 path=current_path,
                 is_start=is_start
             )
-            is_start = False
+            if not child_element.tail.isspace():
+                is_start = False
 
 
 def _iter_tei_training_lines_from_element(
@@ -593,8 +595,6 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
                 )
             )
             LOGGER.debug('tei_training_lines: %r', tei_training_lines)
-            prefix = ''
-            prev_label = ''
             for line_index, line in enumerate(tei_training_lines):
                 line_meta = LayoutLineMeta(line_id=1 + line_index)
                 for text in line.text_list:
@@ -603,14 +603,12 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
                     token_count = 0
                     if text.path.element_list:
                         label = self.get_label_for_element_path(text.path, text=text.text)
-                        if prev_label != label:
-                            prefix = 'B-' if text.is_start else 'I-'
+                        prefix = 'B-' if text.is_start else 'I-'
                     else:
                         label = 'O'
                         prefix = ''
                     if label in OTHER_LABELS:
                         prefix = ''
-                    prev_label = label
                     for token_text in get_tokenized_tokens(text.text):
                         yield LabeledLayoutToken(
                             label=prefix + label,

tests/models/citation/training_data_test.py

@@ -390,6 +390,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<title>')
         ]]
 
+    def test_should_start_a_new_entity_for_sibling_identifiers_separated_by_whitespace(self):
+        tei_root = _get_training_tei_with_references([
+            TEI_E('bibl', *[
+                TEI_E('idno', TOKEN_1),
+                ' ',
+                TEI_E('idno', TOKEN_2, TEI_E('lb')),
+                '\n'
+            ])
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<pubnum>'),
+            (TOKEN_2, 'B-<pubnum>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_references([
             TEI_E('bibl', *[
@@ -567,9 +584,7 @@ def test_should_round_trip_multiple_identifiers_of_one_reference(self):
             (SemanticExternalIdentifierTypes.PMID, '1234567')
         ]
 
-    def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
-        # the parser reconstructs B-/I- from label changes rather than element boundaries,
-        # so two idno elements with only whitespace between them come back as one identifier
+    def test_should_round_trip_directly_adjacent_identifiers_of_one_reference(self):
         label_and_layout_line_list = [
             (IDENTIFIER_LABEL, get_next_layout_line_for_text('10.1234/test')),
             (IDENTIFIER_LABEL, get_next_layout_line_for_text('PMID: 1234567'))
@@ -585,7 +600,8 @@ def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
         references = get_semantic_references_for_training_tei_xml(xml_root)
         assert len(references) == 1
         assert get_external_identifier_types_and_values(references[0]) == [
-            (SemanticExternalIdentifierTypes.DOI, '10.1234/testPMID:1234567')
+            (SemanticExternalIdentifierTypes.DOI, '10.1234/test'),
+            (SemanticExternalIdentifierTypes.PMID, '1234567')
         ]
 
     @pytest.mark.parametrize(

tests/models/fulltext/training_data_test.py

@@ -360,6 +360,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<paragraph>')
         ]]
 
+    def test_should_continue_the_paragraph_after_a_nested_citation_marker(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, ' ', E('ref', {'type': 'biblio'}, TOKEN_2), ' ', TOKEN_3, E('lb')),
+            '\n',
+            E('p', TOKEN_4, E('lb')),
+            '\n'
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'B-<citation_marker>'),
+            (TOKEN_3, 'I-<paragraph>'),
+            (TOKEN_4, 'B-<paragraph>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_text([
             E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')),

tests/models/reference_segmenter/training_data_test.py

@@ -295,6 +295,57 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
             (TOKEN_2, 'B-<reference>')
         ]]
 
+    def test_should_start_a_new_reference_for_each_bibl_without_a_label(self):
+        tei_root = _get_training_tei_with_references([
+            E('bibl', TOKEN_1, E('lb')),
+            '\n',
+            E('bibl', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<reference>'),
+            (TOKEN_2, 'B-<reference>')
+        ]]
+
+    def test_should_start_the_reference_after_whitespace_preceding_the_label(self):
+        tei_root = _get_training_tei_with_references([E(
+            'bibl',
+            ' ',
+            E('label', TOKEN_1),
+            ' ',
+            TOKEN_2,
+            E('lb'),
+            '\n'
+        )])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<label>'),
+            (TOKEN_2, 'B-<reference>')
+        ]]
+
+    def test_should_keep_a_reference_spanning_multiple_lines_as_one_entity(self):
+        tei_root = _get_training_tei_with_references([E(
+            'bibl',
+            TOKEN_1,
+            E('lb'),
+            '\n',
+            TOKEN_2,
+            E('lb'),
+            '\n'
+        )])
+        tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<reference>'),
+            (TOKEN_2, 'I-<reference>')
+        ]]
+
     def test_should_parse_single_label_with_multiple_lines(self):
         tei_root = _get_training_tei_with_references([E(
             'bibl',

tests/models/training_data_test.py

@@ -171,6 +171,51 @@ def test_should_raise_error_for_element_with_more_than_one_matching_attribute(se
                 tei_root
             )
 
+    def test_should_start_a_new_entity_for_each_sibling_element_with_the_same_label(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb')),
+            '\n',
+            E('p', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'B-<paragraph>')
+        ]]
+
+    @pytest.mark.parametrize('separator', ['\n', ' ', '\n  ', ', '])
+    def test_should_start_a_new_entity_for_a_sibling_element_whatever_separates_them(
+        self, separator: str
+    ):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb')),
+            separator,
+            E('p', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        label_by_token = dict(tag_result[0])
+        assert label_by_token[TOKEN_1] == 'B-<paragraph>'
+        assert label_by_token[TOKEN_2] == 'B-<paragraph>'
+
+    def test_should_keep_an_element_spanning_multiple_lines_as_one_entity(self):
+        tei_root = _get_training_tei_with_text([
+            E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')),
+            '\n'
+        ])
+        tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result(
+            tei_root
+        )
+        assert tag_result == [[
+            (TOKEN_1, 'B-<paragraph>'),
+            (TOKEN_2, 'I-<paragraph>')
+        ]]
+
     def test_unannotated_tokens_after_annotated_span_use_O(self):
         # training_data.py emits plain 'O' for all unannotated tokens regardless of
         # position. The I-<other> GROBID convention is applied later by