One identifier label for the citation model

benchmarks/analyze_field_regressions/_models.py

@@ -2,6 +2,8 @@
 
 from typing import Dict, List, Optional
 
+from sciencebeam_parser.models.citation.labels import IDENTIFIER_LABEL
+
 _REFERENCE_MODEL_LABELS: Dict[str, frozenset] = {
     'segmentation':        frozenset({'<references>'}),
     'reference-segmenter': frozenset({'<reference>'}),
@@ -13,7 +15,7 @@
 
 MODEL_RELEVANT_LABELS: Dict[str, Dict[str, frozenset]] = {
     'reference_doi':        {**_REFERENCE_MODEL_LABELS,
-                             'citation': frozenset({'<pubnum>', '<web>'})},
+                             'citation': frozenset({IDENTIFIER_LABEL, '<web>'})},
     'reference_title':      {**_REFERENCE_MODEL_LABELS, 'citation': frozenset({'<title>'})},
     'first_reference_text': _REFERENCE_MODEL_LABELS,
     'title':                {**_HEADER_MODEL_LABELS, 'header': frozenset({'<title>'})},

sciencebeam_parser/models/citation/extract.py

@@ -25,6 +25,7 @@
     T_SemanticContentFactory
 )
 from sciencebeam_parser.document.layout_document import LayoutBlock
+from sciencebeam_parser.models.citation.labels import IDENTIFIER_LABEL, NOTE_CITATION_LABELS
 from sciencebeam_parser.models.extract import SimpleModelSemanticExtractor
 
 
@@ -181,7 +182,10 @@ def get_invalid_reference(ref: SemanticReference) -> SemanticInvalidReference:
 
 class CitationSemanticExtractor(SimpleModelSemanticExtractor):
     def __init__(self):
-        super().__init__(semantic_content_class_by_tag=SIMPLE_SEMANTIC_CONTENT_CLASS_BY_TAG)
+        super().__init__(
+            semantic_content_class_by_tag=SIMPLE_SEMANTIC_CONTENT_CLASS_BY_TAG,
+            expected_note_tags=NOTE_CITATION_LABELS
+        )
 
     def get_semantic_content_for_entity_name(  # pylint: disable=too-many-return-statements
         self,
@@ -192,7 +196,7 @@ def get_semantic_content_for_entity_name(  # pylint: disable=too-many-return-sta
             return parse_page_range(layout_block)
         if name == '<web>':
             return parse_web(layout_block)
-        if name == '<pubnum>':
+        if name == IDENTIFIER_LABEL:
             return parse_pubnum(layout_block)
         if name == '<date>':
             return parse_date(layout_block)

sciencebeam_parser/models/citation/labels.py

@@ -0,0 +1,49 @@
+from typing import AbstractSet, FrozenSet
+
+
+# The label set the citation model is trained and served against, matching GROBID's
+# citation corpus.
+#
+# Every identifier carries IDENTIFIER_LABEL, whatever kind it is; the kind is detected
+# from the text at extraction, so no label depends on a regex having matched. This is
+# what GROBID does: TEICitationSaxParser maps both idno and pubnum to <pubnum>, and
+# CitationParser types the prediction afterwards via BiblioItem.checkIdentifier(). Its
+# training TEI carries the kind as an idno attribute, spelled type="arXiv" and
+# type="PMC" and also covering ISSN and ISBN, which is why the parser accepts any
+# idno type rather than the ones detection can produce here.
+IDENTIFIER_LABEL = '<pubnum>'
+
+OTHER_LABEL = '<other>'
+
+CITATION_LABELS: FrozenSet[str] = frozenset({
+    '<author>',
+    '<booktitle>',
+    '<collaboration>',
+    '<date>',
+    '<editor>',
+    '<institution>',
+    '<issue>',
+    '<journal>',
+    '<location>',
+    '<note>',
+    OTHER_LABEL,
+    '<pages>',
+    '<publisher>',
+    '<series>',
+    '<tech>',
+    '<title>',
+    '<volume>',
+    '<web>',
+    IDENTIFIER_LABEL
+})
+
+# Labels with no semantic counterpart, which CitationSemanticExtractor keeps as notes.
+# Anything else in CITATION_LABELS has to map to semantic content.
+NOTE_CITATION_LABELS: AbstractSet[str] = frozenset({
+    '<booktitle>',
+    '<collaboration>',
+    '<institution>',
+    '<note>',
+    '<series>',
+    '<tech>'
+})

sciencebeam_parser/models/citation/training_data.py

@@ -3,25 +3,19 @@
 
 from lxml import etree
 
-from sciencebeam_parser.document.semantic_document import SemanticExternalIdentifierTypes
 from sciencebeam_parser.document.tei.common import TEI_NS_PREFIX, tei_xpath
 from sciencebeam_parser.models.training_data import (
     AbstractTeiTrainingDataGenerator,
-    AbstractTrainingTeiParser
+    AbstractTrainingTeiParser,
+    TeiTrainingElementPath
 )
 from sciencebeam_parser.models.citation.extract import (
     get_detected_external_identifier_type_for_text
 )
+from sciencebeam_parser.models.citation.labels import IDENTIFIER_LABEL
 from sciencebeam_parser.utils.xml import get_text_content
 
-# get_post_processed_xml_root tags <idno> elements with any detected identifier type
-# (not just DOI); all but DOI should still resolve back to the <pubnum> label
-_PUBNUM_EXTERNAL_IDENTIFIER_TYPES = (
-    SemanticExternalIdentifierTypes.ARXIV,
-    SemanticExternalIdentifierTypes.PII,
-    SemanticExternalIdentifierTypes.PMCID,
-    SemanticExternalIdentifierTypes.PMID,
-)
+_IDNO_TAG = TEI_NS_PREFIX + 'idno'
 
 # Matches "388 - 412", "281-282", "1199 -1207" etc.
 _PAGE_RANGE_RE = re.compile(r'^(\S+)\s*[-–]\s*(\S+)$')
@@ -52,12 +46,15 @@
     '<location>': ROOT_TRAINING_XML_ELEMENT_PATH + ['pubPlace'],
     '<tech>': ROOT_TRAINING_XML_ELEMENT_PATH + ['note[@type="report"]'],
     '<web>': ROOT_TRAINING_XML_ELEMENT_PATH + ['ptr[@type="web"]'],
-    '<idno>': ROOT_TRAINING_XML_ELEMENT_PATH + ['idno[@type="DOI"]'],
-    '<pubnum>': ROOT_TRAINING_XML_ELEMENT_PATH + ['idno'],
+    IDENTIFIER_LABEL: ROOT_TRAINING_XML_ELEMENT_PATH + ['idno'],
     '<note>': ROOT_TRAINING_XML_ELEMENT_PATH + ['note']
 }
 
 
+def _is_idno_path_step(path_step: str) -> bool:
+    return path_step == _IDNO_TAG or path_step.startswith(_IDNO_TAG + '[')
+
+
 class CitationTeiTrainingDataGenerator(AbstractTeiTrainingDataGenerator):
     DEFAULT_TEI_FILENAME_SUFFIX = '.references.tei.xml'
 
@@ -101,7 +98,13 @@ def __init__(self) -> None:
             ),
             use_tei_namespace=True
         )
-        for external_identifier_type in _PUBNUM_EXTERNAL_IDENTIFIER_TYPES:
-            self.label_by_relative_element_path_map[
-                ('{}idno[@type="{}"]'.format(TEI_NS_PREFIX, external_identifier_type),)
-            ] = '<pubnum>'
+
+    def get_label_for_element_path(
+        self,
+        tei_training_element_path: TeiTrainingElementPath,
+        text: str
+    ) -> str:
+        element_path = tei_training_element_path.get_path()
+        if element_path and _is_idno_path_step(element_path[-1]):
+            return IDENTIFIER_LABEL
+        return super().get_label_for_element_path(tei_training_element_path, text=text)

sciencebeam_parser/models/extract.py

@@ -1,14 +1,15 @@
 from abc import ABC, abstractmethod
 import logging
 import re
-from typing import Iterable, Mapping, Optional, Tuple
+from typing import AbstractSet, Iterable, Mapping, Optional, Set, Tuple
 
 from sciencebeam_parser.document.semantic_document import (
     SemanticContentWrapper,
     SemanticNote,
     T_SemanticContentFactory
 )
 from sciencebeam_parser.document.layout_document import EMPTY_BLOCK, LayoutBlock, LayoutTokensText
+from sciencebeam_parser.utils.labels import OTHER_LABELS
 
 
 LOGGER = logging.getLogger(__name__)
@@ -57,10 +58,28 @@ def __init__(
         self,
         semantic_content_class_by_tag: Optional[
             Mapping[str, T_SemanticContentFactory]
-        ] = None
+        ] = None,
+        *,
+        expected_note_tags: Optional[AbstractSet[str]] = None
     ):
         super().__init__()
         self.semantic_content_class_by_tag = semantic_content_class_by_tag or {}
+        self.expected_note_tags = expected_note_tags
+        self._reported_unexpected_note_tags: Set[str] = set()
+
+    def _report_unexpected_note_tag(self, name: str) -> None:
+        if self.expected_note_tags is None:
+            return
+        if name in self.expected_note_tags or name in OTHER_LABELS:
+            return
+        if name in self._reported_unexpected_note_tags:
+            return
+        self._reported_unexpected_note_tags.add(name)
+        LOGGER.warning(
+            'no semantic content for label %r, keeping it as a note'
+            ' (it will not appear in any extracted field)',
+            name
+        )
 
     def get_semantic_content_for_entity_name(
         self,
@@ -70,6 +89,7 @@ def get_semantic_content_for_entity_name(
         semantic_content_class = self.semantic_content_class_by_tag.get(name)
         if semantic_content_class:
             return semantic_content_class(layout_block=layout_block)
+        self._report_unexpected_note_tag(name)
         return SemanticNote(
             layout_block=layout_block,
             note_type=name

sciencebeam_parser/models/training_data.py

@@ -7,7 +7,7 @@
 from lxml.builder import ElementMaker
 
 from sciencebeam_parser.utils.xml_writer import XmlTreeWriter
-from sciencebeam_parser.utils.labels import get_split_prefix_label
+from sciencebeam_parser.utils.labels import OTHER_LABELS, get_split_prefix_label
 from sciencebeam_parser.utils.tokenizer import get_tokenized_tokens
 from sciencebeam_parser.document.tei.common import TEI_E, TEI_NS_PREFIX, tei_xpath
 from sciencebeam_parser.document.layout_document import (
@@ -30,9 +30,6 @@
 NO_NS_TEI_E = ElementMaker()
 
 
-OTHER_LABELS = {'<other>', 'O'}
-
-
 class ExtractInstruction:
     pass
 
@@ -568,7 +565,7 @@ def __init__(
         self.root_training_xml_xpath = './' + '/'.join(root_training_xml_element_path)
         self.line_as_token = line_as_token
 
-    def _get_label_for_element_path(
+    def get_label_for_element_path(
         self,
         tei_training_element_path: TeiTrainingElementPath,
         text: str
@@ -605,7 +602,7 @@ def iter_parse_training_tei_to_flat_labeled_layout_tokens(
                         continue
                     token_count = 0
                     if text.path.element_list:
-                        label = self._get_label_for_element_path(text.path, text=text.text)
+                        label = self.get_label_for_element_path(text.path, text=text.text)
                         if prev_label != label:
                             prefix = 'B-' if text.is_start else 'I-'
                     else:

sciencebeam_parser/training/cli/generate_data.py

@@ -41,6 +41,7 @@
     iter_data_lines_for_model_data_iterables,
     iter_labeled_layout_token_for_layout_model_label
 )
+from sciencebeam_parser.models.citation.labels import IDENTIFIER_LABEL
 from sciencebeam_parser.models.training_data import TeiTrainingDataGenerator
 from sciencebeam_parser.processors.fulltext.models import FullTextModels
 from sciencebeam_parser.resources.default_config import DEFAULT_CONFIG_FILE
@@ -1166,16 +1167,33 @@ def get_main_model(self, document_context: TrainingDataDocumentContext) -> Model
         return document_context.fulltext_models.citation_model
 
     def get_jats_label_fn(self) -> Optional[JatsLabelFn]:
+        previous_identifier: Optional[Tuple[int, Optional[str]]] = None
+
         def fn(
             annotated: JatsAnnotatedLayoutDocument,
             _seg_labels: Dict[int, str],
             md: LayoutModelData,
         ) -> Optional[str]:
+            nonlocal previous_identifier
             token = md.layout_token
             if not token:
                 return None
             sub_field = annotated.get_token_sub_field(token)
-            return CITATION_LABEL_BY_SUB_FIELD.get(sub_field or '') if sub_field else None
+            label = CITATION_LABEL_BY_SUB_FIELD.get(sub_field or '') if sub_field else None
+            if label != IDENTIFIER_LABEL:
+                previous_identifier = None
+                return label
+            # Identifiers of different kinds share one label, so without a B- prefix a DOI
+            # directly followed by a PMID would be written as a single <idno>, and typed as
+            # one identifier whose value carries both.
+            identifier = (annotated.get_token_instance(token), sub_field)
+            is_new_identifier = (
+                previous_identifier is not None
+                and previous_identifier[0] == identifier[0]
+                and previous_identifier[1] != identifier[1]
+            )
+            previous_identifier = identifier
+            return 'B-' + label if is_new_identifier else label
         return fn
 
     def iter_model_layout_documents(

sciencebeam_parser/training/jats/field_vocab.py

@@ -1,5 +1,7 @@
 from typing import Dict
 
+from sciencebeam_parser.models.citation.labels import IDENTIFIER_LABEL
+
 
 class JatsFieldNames:
     TITLE = 'title'
@@ -116,9 +118,9 @@ class JatsSubFieldNames:
     JatsSubFieldNames.REFERENCE_ISSUE:           '<issue>',
     JatsSubFieldNames.REFERENCE_FPAGE:           '<pages>',
     JatsSubFieldNames.REFERENCE_LPAGE:           '<pages>',
-    JatsSubFieldNames.REFERENCE_DOI:             '<idno>',
-    JatsSubFieldNames.REFERENCE_PMID:            '<pubnum>',
-    JatsSubFieldNames.REFERENCE_PMCID:           '<pubnum>',
+    JatsSubFieldNames.REFERENCE_DOI:             IDENTIFIER_LABEL,
+    JatsSubFieldNames.REFERENCE_PMID:            IDENTIFIER_LABEL,
+    JatsSubFieldNames.REFERENCE_PMCID:           IDENTIFIER_LABEL,
     JatsSubFieldNames.REFERENCE_WEB:             '<web>',
     JatsSubFieldNames.REFERENCE_LABEL:           '<note>',
     JatsSubFieldNames.REFERENCE_PUBLISHER_NAME:  '<publisher>',

sciencebeam_parser/utils/labels.py

@@ -1,4 +1,7 @@
-from typing import Tuple
+from typing import FrozenSet, Tuple
+
+
+OTHER_LABELS: FrozenSet[str] = frozenset({'<other>', 'O'})
 
 
 def strip_tag_prefix(tag: str) -> str:

tests/models/citation/labels_test.py

@@ -0,0 +1,59 @@
+import logging
+
+import pytest
+
+from sciencebeam_parser.document.layout_document import LayoutBlock
+from sciencebeam_parser.document.semantic_document import SemanticNote
+from sciencebeam_parser.models.citation.extract import CitationSemanticExtractor
+from sciencebeam_parser.models.citation.labels import (
+    CITATION_LABELS,
+    IDENTIFIER_LABEL,
+    NOTE_CITATION_LABELS,
+    OTHER_LABEL
+)
+from sciencebeam_parser.models.citation.training_data import (
+    TRAINING_XML_ELEMENT_PATH_BY_LABEL
+)
+from sciencebeam_parser.training.jats.field_vocab import CITATION_LABEL_BY_SUB_FIELD
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+TEXT_1 = 'text 1'
+
+
+class TestCitationLabels:
+    def test_should_declare_note_labels_within_the_label_set(self):
+        assert NOTE_CITATION_LABELS <= CITATION_LABELS
+
+    def test_should_declare_identifier_label_within_the_label_set(self):
+        assert IDENTIFIER_LABEL in CITATION_LABELS
+
+    def test_should_generate_training_tei_for_every_label_except_other(self):
+        assert set(TRAINING_XML_ELEMENT_PATH_BY_LABEL.keys()) == CITATION_LABELS - {OTHER_LABEL}
+
+    def test_should_only_use_known_labels_for_jats_sub_fields(self):
+        assert set(CITATION_LABEL_BY_SUB_FIELD.values()) <= CITATION_LABELS
+
+    def test_should_use_the_identifier_label_for_every_jats_identifier_sub_field(self):
+        identifier_labels = {
+            label
+            for sub_field, label in CITATION_LABEL_BY_SUB_FIELD.items()
+            if sub_field.endswith(('-doi', '-pmid', '-pmcid'))
+        }
+        assert identifier_labels == {IDENTIFIER_LABEL}
+
+    @pytest.mark.parametrize("label", sorted(CITATION_LABELS - {OTHER_LABEL}))
+    def test_should_extract_semantic_content_for_every_label_that_is_not_a_declared_note(
+        self,
+        label: str
+    ):
+        semantic_content = CitationSemanticExtractor().get_semantic_content_for_entity_name(
+            label, layout_block=LayoutBlock.for_text(TEXT_1)
+        )
+        LOGGER.debug('semantic_content: %r', semantic_content)
+        if label in NOTE_CITATION_LABELS:
+            assert isinstance(semantic_content, SemanticNote)
+        else:
+            assert not isinstance(semantic_content, SemanticNote)