fix(dtypes,strings): never abort cleaning on an unusual object cell #137
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Batch profile + clean every fixture on each PR. Fails if any dataset errors. | |
| name: Freshdata automation | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| concurrency: | |
| group: automation-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| automate: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| - name: Install | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run fixture automation | |
| run: | | |
| python scripts/automate_freshdata.py \ | |
| --output-dir scripts/.automation_out \ | |
| --report scripts/.automation_out/summary.json | |
| - name: Job summary | |
| if: always() | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| summary_path = Path("scripts/.automation_out/summary.json") | |
| if not summary_path.exists(): | |
| print("No automation summary found.") | |
| raise SystemExit(0) | |
| data = json.loads(summary_path.read_text(encoding="utf-8")) | |
| lines = [ | |
| "## Freshdata automation", | |
| "", | |
| f"- **Datasets:** {data.get('total', 0)}", | |
| f"- **OK:** {data.get('ok', 0)}", | |
| f"- **Failed:** {data.get('fail', 0)}", | |
| f"- **Steps:** `{', '.join(data.get('steps', []))}`", | |
| "", | |
| ] | |
| out_dir = Path(data.get("output_dir", "scripts/.automation_out")) | |
| for report_name in data.get("reports", []): | |
| report = json.loads((out_dir / report_name).read_text(encoding="utf-8")) | |
| ds = report["dataset"] | |
| before = report.get("before_shape") | |
| after = report.get("after_shape") | |
| err = report.get("error") | |
| if err: | |
| lines.append(f"- ❌ `{ds}` — {err}") | |
| else: | |
| lines.append(f"- ✅ `{ds}` {before} → {after} ({report.get('seconds', '?')}s)") | |
| body = "\n".join(lines) | |
| print(body) | |
| summary_file = os.environ.get("GITHUB_STEP_SUMMARY") | |
| if summary_file: | |
| Path(summary_file).write_text(body + "\n", encoding="utf-8") | |
| PY | |
| - name: Upload automation artifacts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: freshdata-automation | |
| path: scripts/.automation_out/ | |
| if-no-files-found: warn |