diff --git a/sciencebeam_parser/models/training_data.py b/sciencebeam_parser/models/training_data.py index aed4d56d..cbee105a 100644 --- a/sciencebeam_parser/models/training_data.py +++ b/sciencebeam_parser/models/training_data.py @@ -433,7 +433,8 @@ def _iter_flat_tei_training_text_from_element( path=current_path, is_start=is_start ) - is_start = False + if not parent_element.text.isspace(): + is_start = False for child_element in parent_element: if is_line_break_element(child_element): @@ -451,7 +452,8 @@ def _iter_flat_tei_training_text_from_element( path=current_path, is_start=is_start ) - is_start = False + if not child_element.tail.isspace(): + is_start = False def _iter_tei_training_lines_from_element( @@ -593,8 +595,6 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens( ) ) LOGGER.debug('tei_training_lines: %r', tei_training_lines) - prefix = '' - prev_label = '' for line_index, line in enumerate(tei_training_lines): line_meta = LayoutLineMeta(line_id=1 + line_index) for text in line.text_list: @@ -603,14 +603,12 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens( token_count = 0 if text.path.element_list: label = self.get_label_for_element_path(text.path, text=text.text) - if prev_label != label: - prefix = 'B-' if text.is_start else 'I-' + prefix = 'B-' if text.is_start else 'I-' else: label = 'O' prefix = '' if label in OTHER_LABELS: prefix = '' - prev_label = label for token_text in get_tokenized_tokens(text.text): yield LabeledLayoutToken( label=prefix + label, diff --git a/tests/models/citation/training_data_test.py b/tests/models/citation/training_data_test.py index 93055373..3127b87e 100644 --- a/tests/models/citation/training_data_test.py +++ b/tests/models/citation/training_data_test.py @@ -390,6 +390,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self): (TOKEN_2, 'B-') ]] + def test_should_start_a_new_entity_for_sibling_identifiers_separated_by_whitespace(self): + tei_root = _get_training_tei_with_references([ + TEI_E('bibl', *[ + TEI_E('idno', TOKEN_1), + ' ', + TEI_E('idno', TOKEN_2, TEI_E('lb')), + '\n' + ]) + ]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<pubnum>'), + (TOKEN_2, 'B-<pubnum>') + ]] + def test_should_parse_single_label_with_multiple_lines(self): tei_root = _get_training_tei_with_references([ TEI_E('bibl', *[ @@ -567,9 +584,7 @@ def test_should_round_trip_multiple_identifiers_of_one_reference(self): (SemanticExternalIdentifierTypes.PMID, '1234567') ] - def test_should_merge_directly_adjacent_identifiers_of_one_reference(self): - # the parser reconstructs B-/I- from label changes rather than element boundaries, - # so two idno elements with only whitespace between them come back as one identifier + def test_should_round_trip_directly_adjacent_identifiers_of_one_reference(self): label_and_layout_line_list = [ (IDENTIFIER_LABEL, get_next_layout_line_for_text('10.1234/test')), (IDENTIFIER_LABEL, get_next_layout_line_for_text('PMID: 1234567')) @@ -585,7 +600,8 @@ def test_should_merge_directly_adjacent_identifiers_of_one_reference(self): references = get_semantic_references_for_training_tei_xml(xml_root) assert len(references) == 1 assert get_external_identifier_types_and_values(references[0]) == [ - (SemanticExternalIdentifierTypes.DOI, '10.1234/testPMID:1234567') + (SemanticExternalIdentifierTypes.DOI, '10.1234/test'), + (SemanticExternalIdentifierTypes.PMID, '1234567') ] @pytest.mark.parametrize( diff --git a/tests/models/fulltext/training_data_test.py b/tests/models/fulltext/training_data_test.py index 27af74a7..cd214b3f 100644 --- a/tests/models/fulltext/training_data_test.py +++ b/tests/models/fulltext/training_data_test.py @@ -360,6 +360,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self): (TOKEN_2, 'B-<paragraph>') ]] + def test_should_continue_the_paragraph_after_a_nested_citation_marker(self): + tei_root = _get_training_tei_with_text([ + E('p', TOKEN_1, ' ', E('ref', {'type': 'biblio'}, TOKEN_2), ' ', TOKEN_3, E('lb')), + '\n', + E('p', TOKEN_4, E('lb')), + '\n' + ]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<paragraph>'), + (TOKEN_2, 'B-<citation_marker>'), + (TOKEN_3, 'I-<paragraph>'), + (TOKEN_4, 'B-<paragraph>') + ]] + def test_should_parse_single_label_with_multiple_lines(self): tei_root = _get_training_tei_with_text([ E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')), diff --git a/tests/models/reference_segmenter/training_data_test.py b/tests/models/reference_segmenter/training_data_test.py index bc95bfda..4d13cf7d 100644 --- a/tests/models/reference_segmenter/training_data_test.py +++ b/tests/models/reference_segmenter/training_data_test.py @@ -295,6 +295,57 @@ def test_should_parse_single_token_labelled_training_tei_lines(self): (TOKEN_2, 'B-<reference>') ]] + def test_should_start_a_new_reference_for_each_bibl_without_a_label(self): + tei_root = _get_training_tei_with_references([ + E('bibl', TOKEN_1, E('lb')), + '\n', + E('bibl', TOKEN_2, E('lb')), + '\n' + ]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<reference>'), + (TOKEN_2, 'B-<reference>') + ]] + + def test_should_start_the_reference_after_whitespace_preceding_the_label(self): + tei_root = _get_training_tei_with_references([E( + 'bibl', + ' ', + E('label', TOKEN_1), + ' ', + TOKEN_2, + E('lb'), + '\n' + )]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<label>'), + (TOKEN_2, 'B-<reference>') + ]] + + def test_should_keep_a_reference_spanning_multiple_lines_as_one_entity(self): + tei_root = _get_training_tei_with_references([E( + 'bibl', + TOKEN_1, + E('lb'), + '\n', + TOKEN_2, + E('lb'), + '\n' + )]) + tag_result = get_training_tei_parser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<reference>'), + (TOKEN_2, 'I-<reference>') + ]] + def test_should_parse_single_label_with_multiple_lines(self): tei_root = _get_training_tei_with_references([E( 'bibl', diff --git a/tests/models/training_data_test.py b/tests/models/training_data_test.py index d4ddb323..ef5823a5 100644 --- a/tests/models/training_data_test.py +++ b/tests/models/training_data_test.py @@ -171,6 +171,51 @@ def test_should_raise_error_for_element_with_more_than_one_matching_attribute(se tei_root ) + def test_should_start_a_new_entity_for_each_sibling_element_with_the_same_label(self): + tei_root = _get_training_tei_with_text([ + E('p', TOKEN_1, E('lb')), + '\n', + E('p', TOKEN_2, E('lb')), + '\n' + ]) + tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<paragraph>'), + (TOKEN_2, 'B-<paragraph>') + ]] + + @pytest.mark.parametrize('separator', ['\n', ' ', '\n ', ', ']) + def test_should_start_a_new_entity_for_a_sibling_element_whatever_separates_them( + self, separator: str + ): + tei_root = _get_training_tei_with_text([ + E('p', TOKEN_1, E('lb')), + separator, + E('p', TOKEN_2, E('lb')), + '\n' + ]) + tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result( + tei_root + ) + label_by_token = dict(tag_result[0]) + assert label_by_token[TOKEN_1] == 'B-<paragraph>' + assert label_by_token[TOKEN_2] == 'B-<paragraph>' + + def test_should_keep_an_element_spanning_multiple_lines_as_one_entity(self): + tei_root = _get_training_tei_with_text([ + E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')), + '\n' + ]) + tag_result = SampleTrainingTeiParser().parse_training_tei_to_tag_result( + tei_root + ) + assert tag_result == [[ + (TOKEN_1, 'B-<paragraph>'), + (TOKEN_2, 'I-<paragraph>') + ]] + def test_unannotated_tokens_after_annotated_span_use_O(self): # training_data.py emits plain 'O' for all unannotated tokens regardless of # position. The I-<other> GROBID convention is applied later by