Emit generated training data at GROBID's column layout

doc/training.md

@@ -80,6 +80,20 @@ It will do one of the following:
 - For models with only `tei` XML files (no layout feature), it will parse the `tei` and generate data using the data generator.
 - For models with additional layout data files, it will align the parsed `tei` with the layout data file and add the label to it.
 
+The output matches GROBID's column layout for the model, so it can be mixed with
+GROBID's own corpus. The expected layout per model is recorded in
+[`grobid_column_layout.yml`](../sciencebeam_parser/resources/grobid_column_layout.yml),
+and generating data for a model with no entry there fails rather than guessing.
+`python -m sciencebeam_parser.training.cli.check_grobid_column_layout` re-checks
+that file against GROBID's published corpora; it downloads them, so it is run by
+hand rather than in CI.
+
+Pass `--include-extra-columns` to also emit the columns this project adds on top
+of GROBID's layout. Today that is the `segmentation` model's `whole_line_text`,
+which `delft` models read as a text feature and `wapiti` templates do not
+reference. Training data generated with the flag cannot be mixed with GROBID's
+`segmentation` corpus, since it is one column wider.
+
 #### Example command for `segmentation` model
 
 ```bash

pyproject.toml

@@ -65,6 +65,7 @@ include = ["sciencebeam_parser*"]
 [tool.setuptools.package-data]
 "sciencebeam_parser" = [
   "py.typed",
+  "resources/*.yml",
   "resources/default_config/*.yml",
   "resources/xslt/*.xsl"
 ]

sciencebeam_parser/resources/grobid_column_layout.yml

@@ -0,0 +1,287 @@
+# The column layout of GROBID's own training data, per model.
+#
+# GROBID's FeaturesVector*.printVector() ends with a label slot:
+#
+#     if (label != null)
+#         res.append(" " + label + "\n");
+#     else
+#         res.append(" 0\n");
+#
+# label_slot records what GROBID's corpus writer does with it, which decides
+# what a training data line looks like:
+#
+#   filled    the writer sets features.label = tag, so the label occupies the
+#             slot: token, columns, label
+#   unfilled  the writer leaves it null and appends the tag, so the slot stays
+#             `0`: token, columns, 0, label
+#   absent    printVector has the block commented out: token, columns, label
+#
+# columns are the feature columns excluding the slot, starting with the token
+# itself, and match the data generator's feature_names position by position.
+# extra_columns are columns we add that GROBID has no counterpart for; they are
+# emitted only with --include-extra-columns.
+#
+# reference_training_corpus is the labelled corpus the recorded layout was
+# measured against, and what
+# `python -m sciencebeam_parser.training.cli.check_grobid_column_layout` reads.
+# It is training-specific: the inference vector is columns plus extra_columns
+# plus the slot, and has no published counterpart here.
+#
+# Models sharing a data generator share an entry, via a YAML anchor.
+
+models:
+
+  # whole_line_text is ours: GROBID has no counterpart for it, and it sits at
+  # the index GROBID uses for the label. delft models read it as a text
+  # feature; wapiti templates stop before it.
+  segmentation:
+    generator: SegmentationDataGenerator
+    label_slot: absent
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-segmentation.train.gz
+    columns:
+      - token_text
+      - second_token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - block_status
+      - page_status
+      - token_font_status
+      - token_font_size
+      - is_bold
+      - is_italic
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_proper_name
+      - is_common_name
+      - is_first_name
+      - is_year
+      - is_month
+      - is_email
+      - is_http
+      - relative_document_position
+      - relative_page_position
+      - punctuation_profile
+      - punctuation_profile_length
+      - block_relative_line_length
+      - is_bitmap_around
+      - is_vector_around
+      - is_repetitive_pattern
+      - is_first_repetitive_pattern
+      - is_main_area
+    extra_columns:
+      - whole_line_text
+
+  header:
+    generator: HeaderDataGenerator
+    label_slot: unfilled
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-header.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - block_status
+      - line_status
+      - alignment
+      - token_font_status
+      - token_font_size
+      - is_bold
+      - is_italic
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_proper_name
+      - is_common_name
+      - is_year
+      - is_month
+      - is_location_name
+      - is_email
+      - is_http
+      - punctuation_type
+      - is_largest_font
+      - is_smallest_font
+      - is_larger_than_average_font
+
+  # figure and table share this layout: GROBID generates their training data
+  # with the fulltext feature vector (FeaturesVectorFigure is unreferenced), so
+  # the fulltext corpus is the reference for all three. Their own published
+  # corpora are not: at 0.9.0 the figure one is 26 feature columns throughout
+  # and the table one mixes 26 across 11569 lines with 27 across 5837, the
+  # 27-column lines being those that carry is_superscript.
+  fulltext: &fulltext
+    generator: FullTextDataGenerator
+    label_slot: absent
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-fulltext.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - block_status
+      - line_status
+      - alignment
+      - token_font_status
+      - token_font_size
+      - is_bold
+      - is_italic
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - punctuation_type
+      - relative_document_position
+      - relative_page_position
+      - is_bitmap_around
+      - callout_type
+      - is_callout_known
+      - is_superscript
+
+  figure: *fulltext
+
+  table: *fulltext
+
+  reference_segmenter:
+    generator: ReferenceSegmenterDataGenerator
+    label_slot: unfilled
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-reference-segmenter.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - line_status
+      - alignment
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_proper_name
+      - is_common_name
+      - is_first_name
+      - is_location_name
+      - is_year
+      - is_month
+      - is_http
+      - punctuation_profile
+      - line_token_relative_position
+      - line_relative_length
+      - block_status
+      - truncated_punctuation_profile_length
+
+  citation:
+    generator: CitationDataGenerator
+    label_slot: filled
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-citation.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - line_status
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_proper_name
+      - is_common_name
+      - is_first_name
+      - is_last_name
+      - is_location_name
+      - is_year
+      - is_month
+      - is_http
+      - is_known_collaboration
+      - is_known_journal_title
+      - is_known_conference_title
+      - is_known_publisher
+      - is_known_identifier
+      - punctuation_type
+      - sentence_token_relative_position
+
+  affiliation_address:
+    generator: AffiliationAddressDataGenerator
+    label_slot: filled
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-affiliation-address.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - line_status
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_proper_name
+      - is_common_name
+      - is_first_name
+      - is_location_name
+      - is_country
+      - punctuation_type
+      - word_shape
+
+  name_header: &name
+    generator: NameDataGenerator
+    label_slot: filled
+    reference_training_corpus:
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-name-header.train.gz
+      - https://github.com/eLifePathways/sciencebeam-datasets/releases/download/grobid-0.9.0/delft-grobid-0.9.0-name-citation.train.gz
+    columns:
+      - token_text
+      - lower_token_text
+      - prefix_1
+      - prefix_2
+      - prefix_3
+      - prefix_4
+      - suffix_1
+      - suffix_2
+      - suffix_3
+      - suffix_4
+      - line_status
+      - capitalisation
+      - digit_status
+      - is_single_char
+      - is_common_name
+      - is_first_name
+      - is_last_name
+      - is_known_title
+      - is_known_suffix
+      - punctuation_type
+
+  name_citation: *name