diff --git a/sciencebeam_parser/models/training_data.py b/sciencebeam_parser/models/training_data.py
index aed4d56d..cbee105a 100644
--- a/sciencebeam_parser/models/training_data.py
+++ b/sciencebeam_parser/models/training_data.py
@@ -433,7 +433,8 @@ def _iter_flat_tei_training_text_from_element(
path=current_path,
is_start=is_start
)
- is_start = False
+ if not parent_element.text.isspace():
+ is_start = False
for child_element in parent_element:
if is_line_break_element(child_element):
@@ -451,7 +452,8 @@ def _iter_flat_tei_training_text_from_element(
path=current_path,
is_start=is_start
)
- is_start = False
+ if not child_element.tail.isspace():
+ is_start = False
def _iter_tei_training_lines_from_element(
@@ -593,8 +595,6 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
)
)
LOGGER.debug('tei_training_lines: %r', tei_training_lines)
- prefix = ''
- prev_label = ''
for line_index, line in enumerate(tei_training_lines):
line_meta = LayoutLineMeta(line_id=1 + line_index)
for text in line.text_list:
@@ -603,14 +603,12 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
token_count = 0
if text.path.element_list:
label = self.get_label_for_element_path(text.path, text=text.text)
- if prev_label != label:
- prefix = 'B-' if text.is_start else 'I-'
+ prefix = 'B-' if text.is_start else 'I-'
else:
label = 'O'
prefix = ''
if label in OTHER_LABELS:
prefix = ''
- prev_label = label
for token_text in get_tokenized_tokens(text.text):
yield LabeledLayoutToken(
label=prefix + label,
diff --git a/tests/models/citation/training_data_test.py b/tests/models/citation/training_data_test.py
index 93055373..3127b87e 100644
--- a/tests/models/citation/training_data_test.py
+++ b/tests/models/citation/training_data_test.py
@@ -390,6 +390,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
(TOKEN_2, 'B-
')
]]
+ def test_should_start_a_new_entity_for_sibling_identifiers_separated_by_whitespace(self):
+ tei_root = _get_training_tei_with_references([
+ TEI_E('bibl', *[
+ TEI_E('idno', TOKEN_1),
+ ' ',
+ TEI_E('idno', TOKEN_2, TEI_E('lb')),
+ '\n'
+ ])
+ ])
+ tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+ tei_root
+ )
+ assert tag_result == [[
+ (TOKEN_1, 'B-'),
+ (TOKEN_2, 'B-')
+ ]]
+
def test_should_parse_single_label_with_multiple_lines(self):
tei_root = _get_training_tei_with_references([
TEI_E('bibl', *[
@@ -567,9 +584,7 @@ def test_should_round_trip_multiple_identifiers_of_one_reference(self):
(SemanticExternalIdentifierTypes.PMID, '1234567')
]
- def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
- # the parser reconstructs B-/I- from label changes rather than element boundaries,
- # so two idno elements with only whitespace between them come back as one identifier
+ def test_should_round_trip_directly_adjacent_identifiers_of_one_reference(self):
label_and_layout_line_list = [
(IDENTIFIER_LABEL, get_next_layout_line_for_text('10.1234/test')),
(IDENTIFIER_LABEL, get_next_layout_line_for_text('PMID: 1234567'))
@@ -585,7 +600,8 @@ def test_should_merge_directly_adjacent_identifiers_of_one_reference(self):
references = get_semantic_references_for_training_tei_xml(xml_root)
assert len(references) == 1
assert get_external_identifier_types_and_values(references[0]) == [
- (SemanticExternalIdentifierTypes.DOI, '10.1234/testPMID:1234567')
+ (SemanticExternalIdentifierTypes.DOI, '10.1234/test'),
+ (SemanticExternalIdentifierTypes.PMID, '1234567')
]
@pytest.mark.parametrize(
diff --git a/tests/models/fulltext/training_data_test.py b/tests/models/fulltext/training_data_test.py
index 27af74a7..cd214b3f 100644
--- a/tests/models/fulltext/training_data_test.py
+++ b/tests/models/fulltext/training_data_test.py
@@ -360,6 +360,23 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
(TOKEN_2, 'B-')
]]
+ def test_should_continue_the_paragraph_after_a_nested_citation_marker(self):
+ tei_root = _get_training_tei_with_text([
+ E('p', TOKEN_1, ' ', E('ref', {'type': 'biblio'}, TOKEN_2), ' ', TOKEN_3, E('lb')),
+ '\n',
+ E('p', TOKEN_4, E('lb')),
+ '\n'
+ ])
+ tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+ tei_root
+ )
+ assert tag_result == [[
+ (TOKEN_1, 'B-'),
+ (TOKEN_2, 'B-'),
+ (TOKEN_3, 'I-'),
+ (TOKEN_4, 'B-')
+ ]]
+
def test_should_parse_single_label_with_multiple_lines(self):
tei_root = _get_training_tei_with_text([
E('p', TOKEN_1, E('lb'), '\n', TOKEN_2, E('lb')),
diff --git a/tests/models/reference_segmenter/training_data_test.py b/tests/models/reference_segmenter/training_data_test.py
index bc95bfda..4d13cf7d 100644
--- a/tests/models/reference_segmenter/training_data_test.py
+++ b/tests/models/reference_segmenter/training_data_test.py
@@ -295,6 +295,57 @@ def test_should_parse_single_token_labelled_training_tei_lines(self):
(TOKEN_2, 'B-')
]]
+ def test_should_start_a_new_reference_for_each_bibl_without_a_label(self):
+ tei_root = _get_training_tei_with_references([
+ E('bibl', TOKEN_1, E('lb')),
+ '\n',
+ E('bibl', TOKEN_2, E('lb')),
+ '\n'
+ ])
+ tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+ tei_root
+ )
+ assert tag_result == [[
+ (TOKEN_1, 'B-'),
+ (TOKEN_2, 'B-')
+ ]]
+
+ def test_should_start_the_reference_after_whitespace_preceding_the_label(self):
+ tei_root = _get_training_tei_with_references([E(
+ 'bibl',
+ ' ',
+ E('label', TOKEN_1),
+ ' ',
+ TOKEN_2,
+ E('lb'),
+ '\n'
+ )])
+ tag_result = get_training_tei_parser().parse_training_tei_to_tag_result(
+ tei_root
+ )
+ assert tag_result == [[
+ (TOKEN_1, 'B-