From 53957e3c395eda7554044be3820795a3d8969f5a Mon Sep 17 00:00:00 2001 From: Chase Conching Date: Fri, 17 Jul 2026 00:50:51 -1000 Subject: [PATCH] Harden imports and release tooling --- .github/workflows/ci.yml | 44 ++++++++++++++++++++++++++++ CHANGELOG.md | 17 +++++++++++ README.md | 4 +-- SECURITY.md | 12 ++++++++ bin/build-zip.sh | 11 +++++++ config/README.md | 3 +- inc/import-lib.php | 38 +++++++++++++++++------- inc/importer.php | 37 +++++++++++++++-------- inc/kses.php | 32 ++++++++++++++++++++ lc-core.php | 4 +-- requirements.txt | 3 ++ tests/run.sh | 7 +++-- tests/test-import-lib.php | 3 +- tests/test-kses.php | 58 +++++++++++++++++++++++++++++++++++++ tests/test_normalize_e2e.py | 19 +++++++++++- tests/test_normalize_lib.py | 6 ++++ tools/normalize_lib.py | 11 +++++++ tools/normalize_workbook.py | 22 +++++++++++--- 18 files changed, 295 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 SECURITY.md create mode 100644 requirements.txt create mode 100644 tests/test-kses.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b10c43c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + php: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + php: ['7.4', '8.2', '8.4'] + steps: + - uses: actions/checkout@v4 + - uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + coverage: none + - name: PHP lint and unit tests + run: | + find . -name '*.php' -type f -not -path './dist/*' -not -path './.git/*' -print0 | xargs -0 -n1 php -l + php tests/test-import-lib.php + php tests/test-kses.php + + python: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.9', '3.12'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + - run: pip install -r requirements.txt + - run: python tests/test_normalize_lib.py + - run: python tests/test_normalize_e2e.py + - run: bash -n bin/build-zip.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a68dc73..094d0d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to LC Core. Format loosely follows Keep a Changelog; this plugin uses semantic-ish versioning. +## [0.3.0] — 2026-07-17 + +### Security and correctness + +- Restrict the global KSES input allowance to radio/checkbox types. +- Reject ambiguous CSV type signatures, sanitize/limit supplied import keys, count + creates/updates only after successful writes, and require an unambiguous exact + attachment filename match. +- Add an opt-in spreadsheet-safe CSV mode for formula-like cells. + +### Performance and delivery + +- Stream workbook rows after the bounded header scan instead of loading a whole + sheet into memory. +- Add Python dependency metadata, PHP/Python CI matrices, focused KSES tests, + and filename-bound SHA-256 release checksums. + ## [0.2.2] — 2026-07-16 ### Added (example per-site config) diff --git a/README.md b/README.md index 3d6bd3a..71a0e58 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A site-agnostic WordPress starter framework for Library Creative sites, deployed alongside `lc-bricks-mcp`. It ships the reusable machinery every LC site needs and **nothing site-specific**: the custom post types, taxonomies, ACF field groups, and import mappings for a given project live in that project's own **config layer**. -- **Version:** 0.1.0 +- **Version:** 0.3.0 - **License:** GPL-2.0-or-later - **Requires PHP:** 7.4 - **Text domain:** `lc-core` @@ -76,7 +76,7 @@ The config declares, per output CSV: the source sheet, the column order (with a ```bash tests/run.sh # php -l + python syntax check + standalone unit tests (no WordPress, no openpyxl) -bin/build-zip.sh # -> dist/lc-core-0.1.0.zip, honoring .distignore +bin/build-zip.sh # -> dist/lc-core-0.3.0.zip, honoring .distignore ``` ## What is NOT here (left in the predecessor plugin) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..735f9b2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security policy + +Report suspected vulnerabilities privately through this repository's GitHub +**Security > Report a vulnerability** flow. Do not open a public issue with +exploit details or production data. + +Include the affected LC Core version, reproduction steps, expected impact, and +sanitized sample input when relevant. Remove credentials, personal data, and +client content from all reports. + +Only the latest tagged release is supported. Import files should be treated as +untrusted input and reviewed before use on a production site. diff --git a/bin/build-zip.sh b/bin/build-zip.sh index dc30e14..94d1799 100755 --- a/bin/build-zip.sh +++ b/bin/build-zip.sh @@ -31,10 +31,21 @@ rm -f "$EXCLUDES" mkdir -p dist ZIP="$ROOT/dist/lc-core-${VERSION}.zip" +CHECKSUM="${ZIP}.sha256" rm -f "$ZIP" ( cd "$STAGE" && zip -rq "$ZIP" lc-core ) +if command -v shasum >/dev/null 2>&1; then + ( cd "$ROOT/dist" && shasum -a 256 "$(basename "$ZIP")" > "$(basename "$CHECKSUM")" ) +elif command -v sha256sum >/dev/null 2>&1; then + ( cd "$ROOT/dist" && sha256sum "$(basename "$ZIP")" > "$(basename "$CHECKSUM")" ) +else + echo "FATAL: shasum or sha256sum is required to produce the release checksum" >&2 + exit 1 +fi + echo echo "== Contents ==" unzip -l "$ZIP" echo "Built: dist/lc-core-${VERSION}.zip" +echo "Checksum: dist/lc-core-${VERSION}.zip.sha256" diff --git a/config/README.md b/config/README.md index 7b0ba53..1ccf8e7 100644 --- a/config/README.md +++ b/config/README.md @@ -27,7 +27,8 @@ ```php $config['my_cpt'] = array( // Header column(s) that uniquely identify this type in a CSV. All must be - // present for the type to be detected. First config entry that matches wins. + // present for the type to be detected. Signatures must be unique; multiple + // matches are rejected as ambiguous before any content is written. 'signature' => array( 'my_signature_column' ), // Header-title assertion: the file is REJECTED if any of these column titles diff --git a/inc/import-lib.php b/inc/import-lib.php index 5e663f3..420db01 100644 --- a/inc/import-lib.php +++ b/inc/import-lib.php @@ -91,24 +91,21 @@ function lc_core_assert_headers( $headers, $required ) { } endif; -if ( ! function_exists( 'lc_core_detect_type' ) ) : +if ( ! function_exists( 'lc_core_detect_types' ) ) : /** - * Detect a post type from a CSV header row using per-type signature columns. - * - * $config is the import config (see inc/config.php): keyed by post type, each - * entry may carry a 'signature' => string|string[] of header names that uniquely - * identify that type. The first type whose ALL signature headers are present wins - * (order = config order). Generalizes the predecessor plugin's hard-coded detect_type(). + * Return every configured post type whose signature matches the headers. * * @param string[] $headers Header cells. * @param array $config Import config keyed by post type. - * @return string Post type slug, or '' if none matched. + * @return string[] Matching post type slugs. */ - function lc_core_detect_type( $headers, $config ) { + function lc_core_detect_types( $headers, $config ) { $have = array(); foreach ( (array) $headers as $h ) { $have[ (string) $h ] = true; } + + $matches = array(); foreach ( (array) $config as $type => $def ) { if ( empty( $def['signature'] ) ) { continue; @@ -122,10 +119,29 @@ function lc_core_detect_type( $headers, $config ) { } } if ( $matched ) { - return (string) $type; + $matches[] = (string) $type; } } - return ''; + + return $matches; + } +endif; + +if ( ! function_exists( 'lc_core_detect_type' ) ) : + /** + * Detect a post type from a CSV header row using per-type signature columns. + * + * $config is the import config (see inc/config.php): keyed by post type, each + * entry may carry a 'signature' => string|string[] of header names that uniquely + * identify that type. Exactly one type must match; ambiguous signatures fail closed. + * + * @param string[] $headers Header cells. + * @param array $config Import config keyed by post type. + * @return string Post type slug, or '' if none matched. + */ + function lc_core_detect_type( $headers, $config ) { + $matches = lc_core_detect_types( $headers, $config ); + return 1 === count( $matches ) ? $matches[0] : ''; } endif; diff --git a/inc/importer.php b/inc/importer.php index ad8543e..fb3bf2d 100644 --- a/inc/importer.php +++ b/inc/importer.php @@ -77,11 +77,14 @@ function ( $x ) { $headers ); - $type = lc_core_detect_type( $headers, $config ); + $matching_types = lc_core_detect_types( $headers, $config ); + $type = 1 === count( $matching_types ) ? $matching_types[0] : ''; $report['type'] = $type; if ( ! $type ) { fclose( $fh ); - $report['error'] = 'Could not detect post type from headers. Check the `signature` columns in your import config. Header: ' . implode( ', ', $headers ); + $report['error'] = count( $matching_types ) > 1 + ? 'Ambiguous post type signatures matched: ' . implode( ', ', $matching_types ) . '. Make each signature unique before importing.' + : 'Could not detect post type from headers. Check the `signature` columns in your import config. Header: ' . implode( ', ', $headers ); return $report; } if ( ! post_type_exists( $type ) ) { @@ -124,9 +127,12 @@ function ( $x ) { } // import_key from the CSV, else derived from the TITLE ONLY. - $key = ( isset( $row['import_key'] ) && '' !== $row['import_key'] ) - ? $row['import_key'] - : lc_core_derive_import_key( $title ); + if ( isset( $row['import_key'] ) && '' !== $row['import_key'] ) { + $clean_key = sanitize_text_field( $row['import_key'] ); + $key = function_exists( 'mb_substr' ) ? mb_substr( $clean_key, 0, 191, 'UTF-8' ) : substr( $clean_key, 0, 191 ); + } else { + $key = lc_core_derive_import_key( $title ); + } $status = ( isset( $row['post_status'] ) && in_array( $row['post_status'], array( 'publish', 'draft' ), true ) ) ? $row['post_status'] @@ -193,16 +199,19 @@ function ( $x ) { if ( $existing ) { $postarr['ID'] = $existing; $post_id = wp_update_post( $postarr, true ); - $report['updated']++; } else { $post_id = wp_insert_post( $postarr, true ); - $report['created']++; } if ( is_wp_error( $post_id ) ) { $report['skipped']++; $report['notes'][] = "Error on '{$title}': " . $post_id->get_error_message(); continue; } + if ( $existing ) { + $report['updated']++; + } else { + $report['created']++; + } $claimed[] = (int) $post_id; if ( $key ) { update_post_meta( $post_id, $key_meta, $key ); @@ -357,14 +366,18 @@ function lc_core_ensure_term( $name_or_slug, $tax, &$report, $slug = '' ) { */ function lc_core_find_attachment( $filename ) { global $wpdb; - $name = pathinfo( $filename, PATHINFO_FILENAME ); - $id = $wpdb->get_var( + $name = sanitize_file_name( basename( (string) $filename ) ); + if ( '' === $name ) { + return 0; + } + $ids = $wpdb->get_col( $wpdb->prepare( - "SELECT post_id FROM {$wpdb->postmeta} WHERE meta_key='_wp_attached_file' AND meta_value LIKE %s LIMIT 1", - '%' . $wpdb->esc_like( $name ) . '%' + "SELECT post_id FROM {$wpdb->postmeta} WHERE meta_key='_wp_attached_file' AND (meta_value = %s OR meta_value LIKE %s) ORDER BY post_id ASC LIMIT 2", + $name, + '%/' . $wpdb->esc_like( $name ) ) ); - return $id ? (int) $id : 0; + return 1 === count( $ids ) ? (int) $ids[0] : 0; } /* ------------------------------------------------------------------------- diff --git a/inc/kses.php b/inc/kses.php index 762caef..e4c73d9 100644 --- a/inc/kses.php +++ b/inc/kses.php @@ -22,6 +22,7 @@ } add_filter( 'wp_kses_allowed_html', 'lc_core_kses_allow_form_controls', 10, 2 ); +add_filter( 'pre_kses', 'lc_core_kses_restrict_input_types', 10, 3 ); function lc_core_kses_allow_form_controls( $tags, $context ) { if ( ! apply_filters( 'lc_core_kses_enable', true ) ) { @@ -67,3 +68,34 @@ function lc_core_kses_allow_form_controls( $tags, $context ) { return (array) apply_filters( 'lc_core_kses_allowed_html', $tags, $context ); } + +/** + * Remove input types outside the inert CSS-only contract before KSES runs. + * + * KSES allowlists attribute names, not an enum of allowed attribute values. The + * post-context allowance therefore needs this value-level guard so it cannot be + * reused for password, file, submit, or other interactive controls. + * + * @param string $content Content before KSES filtering. + * @param array $allowed_html Effective allowed tag/attribute map. + * @param array $allowed_protocols Allowed URL protocols (unused). + * @return string Filtered content. + */ +function lc_core_kses_restrict_input_types( $content, $allowed_html, $allowed_protocols = array() ) { // phpcs:ignore Generic.CodeAnalysis.UnusedFunctionParameter.FoundAfterLastUsed + if ( ! isset( $allowed_html['input'] ) ) { + return $content; + } + + $allowed_types = (array) apply_filters( 'lc_core_kses_input_types', array( 'radio', 'checkbox' ) ); + return (string) preg_replace_callback( + '/]*>/i', + function ( $match ) use ( $allowed_types ) { + if ( 1 !== preg_match( '/\btype\s*=\s*(["\']?)([^\s"\'>]+)\1/i', $match[0], $type_match ) ) { + return ''; + } + $type = strtolower( $type_match[2] ); + return in_array( $type, $allowed_types, true ) ? $match[0] : ''; + }, + (string) $content + ); +} diff --git a/lc-core.php b/lc-core.php index 437db89..9312565 100644 --- a/lc-core.php +++ b/lc-core.php @@ -2,7 +2,7 @@ /** * Plugin Name: LC Core * Description: Site-agnostic starter framework for Library Creative WordPress sites — a config-driven content importer, a KSES allowlist module, and generic Bricks query-var helpers. Per-site CPTs / taxonomies / ACF field groups / import mappings live in a separate config layer (see config/example-config.php). Deploy alongside lc-bricks-mcp. - * Version: 0.2.2 + * Version: 0.3.0 * Requires PHP: 7.4 * Author: Library Creative * License: GPL-2.0-or-later @@ -18,7 +18,7 @@ exit; } -define( 'LC_CORE_VERSION', '0.2.2' ); +define( 'LC_CORE_VERSION', '0.3.0' ); define( 'LC_CORE_DIR', plugin_dir_path( __FILE__ ) ); define( 'LC_CORE_URL', plugin_dir_url( __FILE__ ) ); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b77157a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# Runtime dependencies for tools/normalize_workbook.py. +openpyxl>=3.1,<4 +PyYAML>=6,<7 diff --git a/tests/run.sh b/tests/run.sh index 592fe83..8d40c2b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -7,6 +7,8 @@ set -u cd "$(dirname "$0")/.." FAIL=0 +PYCACHE_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/lc-core-pycache.XXXXXX") +trap 'rm -rf "$PYCACHE_ROOT"' EXIT echo "=== php -l ===" while IFS= read -r -d '' f; do @@ -16,7 +18,7 @@ done < <(find . -name '*.php' -not -path './dist/*' -not -path './.git/*' -print echo echo "=== python3 syntax check ===" for f in tools/normalize_workbook.py tools/normalize_lib.py tests/test_normalize_lib.py tests/test_normalize_e2e.py; do - if python3 -m py_compile "$f"; then + if PYTHONPYCACHEPREFIX="$PYCACHE_ROOT" python3 -m py_compile "$f"; then echo "No syntax errors detected in $f" else FAIL=1 @@ -26,13 +28,14 @@ done echo echo "=== PHP unit tests (inc/import-lib.php) ===" php tests/test-import-lib.php || FAIL=1 +php tests/test-kses.php || FAIL=1 echo echo "=== Python unit tests (tools/normalize_lib.py) ===" python3 tests/test_normalize_lib.py || FAIL=1 echo -echo "=== Python e2e test (tools/normalize_workbook.py; skips without openpyxl) ===" +echo "=== Python e2e test (tools/normalize_workbook.py) ===" python3 tests/test_normalize_e2e.py || FAIL=1 echo diff --git a/tests/test-import-lib.php b/tests/test-import-lib.php index 7866380..4c20f3f 100644 --- a/tests/test-import-lib.php +++ b/tests/test-import-lib.php @@ -62,7 +62,8 @@ function check( $label, $actual, $expected ) { ); check( 'single signature match', lc_core_detect_type( array( 'post_title', 'vendor_category' ), $config ), 'mrfm_vendor' ); check( 'second type', lc_core_detect_type( array( 'post_title', 'event_date' ), $config ), 'mrfm_event' ); -check( 'first full match wins (config order)', lc_core_detect_type( array( 'vendor_category', 'event_date' ), $config ), 'mrfm_vendor' ); +check( 'ambiguous signatures fail closed', lc_core_detect_type( array( 'vendor_category', 'event_date' ), $config ), '' ); +check( 'all ambiguous matches are reported', lc_core_detect_types( array( 'vendor_category', 'event_date' ), $config ), array( 'mrfm_vendor', 'mrfm_event', 'mrfm_both' ) ); check( 'no match -> empty', lc_core_detect_type( array( 'post_title' ), $config ), '' ); check( 'multi-column signature needs ALL', lc_core_detect_type( array( 'event_date' ), array( 'x' => array( 'signature' => array( 'event_date', 'venue' ) ) ) ), '' ); check( 'empty config -> empty', lc_core_detect_type( array( 'post_title' ), array() ), '' ); diff --git a/tests/test-kses.php b/tests/test-kses.php new file mode 100644 index 0000000..293bdbe --- /dev/null +++ b/tests/test-kses.php @@ -0,0 +1,58 @@ + array( 'type' => true, 'id' => true ) ); +kses_check( + 'radio input survives', + lc_core_kses_restrict_input_types( '', $allowed ), + '' +); +kses_check( + 'checkbox input survives', + lc_core_kses_restrict_input_types( "", $allowed ), + "" +); +kses_check( + 'password input is removed', + lc_core_kses_restrict_input_types( '

A

B

', $allowed ), + '

A

B

' +); +kses_check( + 'missing type is removed', + lc_core_kses_restrict_input_types( '', $allowed ), + '' +); +kses_check( + 'other KSES contexts remain untouched', + lc_core_kses_restrict_input_types( '', array() ), + '' +); + +echo "\n{$assertions} assertions, {$failures} failure(s)\n"; +exit( $failures ? 1 : 0 ); diff --git a/tests/test_normalize_e2e.py b/tests/test_normalize_e2e.py index 5c04c33..5b3d7b0 100644 --- a/tests/test_normalize_e2e.py +++ b/tests/test_normalize_e2e.py @@ -58,6 +58,8 @@ def build_workbook(path): ws.append([2, "Blue Barn Bakery", "", "", "Sat"]) # no URL -> draft ws.append([3, "Maple Ridge Honey", "https://example.org/honey2", "", ""]) # title drift -> merges ws.append([4, "Fiddlehead Farm", "fiddlehead.example", "", "Blursday"]) # bad URL + unknown day + ws.append([5, "Formula Farm", "=HYPERLINK(\"https://bad.example\")", "", "Sat"]) + ws["C7"].data_type = "s" # text beginning '=' rather than an evaluated workbook formula wb.save(path) @@ -92,7 +94,7 @@ def check(label, cond, detail=""): for r in csv.DictReader(f): rows[r["post_title"]] = r - check("title-drift rows merged (3 unique vendors)", len(rows) == 3, str(sorted(rows))) + check("title-drift rows merged (4 unique vendors)", len(rows) == 4, str(sorted(rows))) honey = rows.get("Maple Ridge Honey", {}) check("first URL kept on merge", honey.get("url") == "https://example.org/honey", str(honey)) check("key derived from title only", honey.get("import_key") == __import__("hashlib").md5(b"maple ridge honey").hexdigest()[:12], str(honey.get("import_key"))) @@ -106,6 +108,21 @@ def check(label, cond, detail=""): check("malformed URL warned", "MALFORMED URL" in warns, warns) check("unmatched tax value warned", "Blursday" in warns, warns) + # --- explicit spreadsheet-safe export path --- + safe_dir = os.path.join(tmp, "safe-out") + res_safe = subprocess.run( + [sys.executable, TOOL, wb_path, cfg_path, "--out", safe_dir, "--spreadsheet-safe"], + capture_output=True, text=True, + ) + check("spreadsheet-safe export exits 0", res_safe.returncode == 0, res_safe.stdout + res_safe.stderr) + with open(os.path.join(safe_dir, "vendors.csv"), encoding="utf-8") as f: + safe_rows = {r["post_title"]: r for r in csv.DictReader(f)} + check( + "formula-like cell is escaped for spreadsheet opening", + safe_rows.get("Formula Farm", {}).get("url", "").startswith("'="), + str(safe_rows.get("Formula Farm")), + ) + # --- header-assertion path: config expects a column the sheet lacks --- bad_cfg = json.loads(json.dumps(CONFIG)) bad_cfg["sheets"][0]["columns"]["post_title"]["from"] = "Inserted Column" diff --git a/tests/test_normalize_lib.py b/tests/test_normalize_lib.py index c25835c..136935c 100644 --- a/tests/test_normalize_lib.py +++ b/tests/test_normalize_lib.py @@ -22,6 +22,7 @@ norm, resolve, spec_required_titles, + spreadsheet_safe, upsert_record, ) @@ -37,6 +38,11 @@ def test_curly_quotes_fold_to_okina(self): def test_none_is_empty(self): self.assertEqual(norm(None), "") + def test_spreadsheet_safe_formula_prefixes(self): + self.assertEqual(spreadsheet_safe("=1+1"), "'=1+1") + self.assertEqual(spreadsheet_safe("+18085551212"), "'+18085551212") + self.assertEqual(spreadsheet_safe("ordinary"), "ordinary") + class TestHkey(unittest.TestCase): """Identity = TITLE ONLY (the 2026-07-09 26-duplicate incident).""" diff --git a/tools/normalize_lib.py b/tools/normalize_lib.py index efefefa..530faa4 100644 --- a/tools/normalize_lib.py +++ b/tools/normalize_lib.py @@ -22,6 +22,17 @@ DEFAULT_SPLIT_RE = r"[;,/]| and |\+|\|" +def spreadsheet_safe(value): + """Escape a CSV cell that spreadsheet apps could evaluate as a formula. + + This is intentionally opt-in at the CLI because the normal output is consumed + directly by WordPress and a leading apostrophe would become imported data. + """ + if not isinstance(value, str) or not value: + return value + return "'" + value if value[0] in ("=", "+", "-", "@") else value + + def norm(s): """Normalize a cell: NFC, curly quotes -> ʻokina, collapse whitespace, trim.""" if s is None: diff --git a/tools/normalize_workbook.py b/tools/normalize_workbook.py index bb0f9e7..d847298 100755 --- a/tools/normalize_workbook.py +++ b/tools/normalize_workbook.py @@ -26,6 +26,7 @@ import argparse import csv import json +import itertools import os import sys @@ -38,6 +39,7 @@ map_row, norm, spec_required_titles, + spreadsheet_safe, upsert_record, ) @@ -117,6 +119,11 @@ def main(argv=None): ap.add_argument("workbook", help="client .xlsx workbook") ap.add_argument("config", help="per-site normalize config (.yml/.yaml/.json)") ap.add_argument("--out", default=None, help="output dir (default: ./data next to this script)") + ap.add_argument( + "--spreadsheet-safe", + action="store_true", + help="prefix formula-like CSV cells for safe human opening (do not use for direct WordPress import)", + ) args = ap.parse_args(argv) cfg = load_config(args.config) @@ -148,13 +155,15 @@ def main(argv=None): columns = sheet_cfg["columns"] required = spec_required_titles(columns) - rows = list(rows_of(ws)) + rows = rows_of(ws) + header_scan = list(itertools.islice(rows, 20)) # MANDATORY header-title assertion — HeaderError is a SystemExit (loud, non-zero). - header_idx, header_map = find_header_row(rows, required) + header_idx, header_map = find_header_row(header_scan, required) print("[%s] header validated (%d source columns) at row %d" % (sheet_name, len(required), header_idx + 1)) records = {} - for row_cells in rows[header_idx + 1 :]: + data_rows = itertools.chain(header_scan[header_idx + 1 :], rows) + for row_cells in data_rows: record, warnings = map_row(row_cells, header_map, columns, lookup, split_re) record = apply_row_rules(record, sheet_cfg, warnings) for w in warnings: @@ -167,7 +176,12 @@ def main(argv=None): w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") w.writeheader() for record in sorted(records.values(), key=lambda r: r.get("post_title", "").lower()): - w.writerow(record) + output_record = ( + {key: spreadsheet_safe(value) for key, value in record.items()} + if args.spreadsheet_safe + else record + ) + w.writerow(output_record) print("[%s] wrote %d rows -> %s" % (sheet_name, len(records), out_path)) all_warnings = list(dict.fromkeys(all_warnings)) # dedupe, keep order