diff --git a/.env.example b/.env.example index df87eee..dff0bf2 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,9 @@ HARVARD_DATAVERSE_GUESTBOOK_JSON = '{"email":Replace with your email address,"institution":Replace with your institution}' +# INGEST_ROOT and SCRATCH_ROOT take any fsspec URI. A local root must be an absolute path +# that already exists; a gs:// root needs application-default credentials. +# INGEST_ROOT = '/absolute/path/to/ingest' INGEST_ROOT = 'gs://cornerstone-ingest-us-central1' NUMBER_OF_DASK_WORKERS = 12 +# SCRATCH_ROOT = '/absolute/path/to/scratch' SCRATCH_ROOT = 'gs://cornerstone-scratch-us-central1' USDA_NASS_API_KEY = 'Replace me with value from https://quickstats.nass.usda.gov/api' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6518c7c..32cc5b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,6 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.3.2 - with: - enable-cache: true - name: Install dependencies run: uv sync - uses: actions/cache@v6 @@ -28,9 +26,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.3.2 - with: - enable-cache: true - name: Install dependencies run: uv sync - name: pytest (unit only) - run: uv run pytest jdluc/__tests__/ + run: uv run pytest diff --git a/.gitignore b/.gitignore index 38d6436..91815b4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ __pycache__/ # Env .env + +.cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb0ef25..e2863e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,19 +18,30 @@ repos: - id: mypy name: mypy language: system - entry: uv run mypy jdluc + entry: uv run mypy jdluc validation pass_filenames: false - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.11.26 hooks: - id: uv-lock + - repo: https://github.com/hukkin/mdformat + rev: 1.0.0 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-gfm + entry: mdformat --number --wrap=no - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: check-added-large-files + - id: check-ast + - id: check-json - id: check-merge-conflict - - id: check-yaml - id: check-toml + - id: check-yaml - id: debug-statements - id: end-of-file-fixer + - id: pretty-format-json + args: [--autofix] - id: trailing-whitespace diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 833a1ed..c0225da 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -1,4 +1,4 @@ -Attribute all emissions factors or other data built on this pipeline as: "Cornerstone Sustainability Data Initiative, http://doi.org/DOI", using the DOI for the data version. If no published DOI exists, link to the relevant code release on GitHub, or simply https://github.com/cornerstone-data/jdluc/. +Attribute all emissions factors or other data built on this pipeline as: "Cornerstone Sustainability Data Initiative, http://doi.org/DOI", using the DOI for the data version. If no published DOI exists, link to the relevant code release on GitHub, or simply https://github.com/cornerstone-data/luc/. You must prominently display this attribution in public-facing models, tools, datasets or any other applications that are generated from this code. This includes reimplementations of the code that are built primarily by passing the methodology and/or technical specs to a coding agent. diff --git a/LICENSE.txt b/LICENSE.txt index 9fd02c9..f511db0 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -7,7 +7,7 @@ License The licensor licenses you to do everything with this software that would otherwise infringe the licensor’s copyright in it, or any patent claims the licensor can license or becomes able to license. If you make any written claim that the software infringes or contributes to infringement of any patent, your license ends immediately. Notices on Software -If you provide a copy of the software to anyone else, or provide or display Produced Data to anyone else, you must provide a copy of this license or a link to https://github.com/cornerstone-data/jdluc/blob/main/LICENSE.txt. +If you provide a copy of the software to anyone else, or provide or display Produced Data to anyone else, you must provide a copy of this license or a link to https://github.com/cornerstone-data/luc/blob/main/LICENSE.txt. Notices on Produced Data If you calculate data (including summaries or reports of data) using this software, you must deliver an attribution, in a form to be specified by the licensor, in a reasonable and conspicuous manner to anyone to whom you deliver or display the data. diff --git a/README.md b/README.md index f3121e7..1bef6a3 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,10 @@ -# Cornerstone jdLUC +# Cornerstone LUC This repo contains an experimental methodology and data pipeline for estimating the land use change (LUC) related emissions associated with agricultural commodities. This methodology allocates LUC emissions to crops in proportion to their displacement of natural ecosystems, based on high-resolution satellite imagery. It's primarily intended for use in corporate GHG inventories, and follows the new GHGP Land Sector and Removals Standard. It supports two attribution methodologies over a shared per-pixel emissions core: the high-resolution "jurisdictional direct land use change" (jdLUC) calculation where detailed crop maps exist, and a coarser-resolution statistical approach for regions where they don't. -As a proof of concept, the jurisdictional-direct leg focuses on the main row crops grown in the United States (corn, soy, and wheat). Although U.S. land use change emissions are relatively modest contributors to global totals, the U.S. agricultural sector is well studied and has strong data infrastructure, which makes it a good place to start testing methods. The statistical leg extends the same per-pixel emissions core to global crop coverage, using coarser sub-national statistics (IFPRI MapSPAM) where high-resolution crop maps aren't available. +As a proof of concept, the jurisdictional-direct leg covers eleven crops grown in the United States (barley, dry beans, cotton, maize, potato, rice, sorghum, soybean, sugarbeet, sugarcane and wheat). Although U.S. land use change emissions are relatively modest contributors to global totals, the U.S. agricultural sector is well studied and has strong data infrastructure, which makes it a good place to start testing methods. The statistical leg extends the same per-pixel emissions core to global crop coverage, using coarser sub-national statistics (IFPRI MapSPAM) where high-resolution crop maps aren't available. -> **Assessing LSRS conformance?** The [executive summary](docs/executive_summary.md) gives a -> concise account of how this methodology maps to the GHGP Land Sector and Removals Standard — -> which requirements it meets, the key modeling choices behind them, and where it deviates or -> remains a work in progress. +> **Assessing LSRS conformance?** The [executive summary](docs/executive_summary.md) gives a concise account of how this methodology maps to the GHGP Land Sector and Removals Standard — which requirements it meets, the key modeling choices behind them, and where it deviates or remains a work in progress. ![Land conversion and soy expansion drive LUC emissions in Matopiba, Brazil](docs/figures/soy-brazil-matopiba-methodology.png) @@ -25,22 +22,16 @@ The methodology and technical decisions in this repo are intended as a starting ## Data access -The pipeline publishes three artifacts — see `docs/data.md` for each artifact's full schema, grid, and variable/column reference. - -Harmonized inputs and per-pixel emissions (zarr): +The pipeline's published artifact is the emissions-factor table: one `emissions-factors.parquet` per data version deposited in a public data archive that mints a DOI per version. The archive is still being set up. Each version's git tag names the commit that produced the table. ```python ->>> import xarray ->>> harmonized = xarray.open_zarr("gs://cornerstone-luc/v3-jdluc-sluc-south-america/harmonize.zarr", consolidated=False) ->>> emissions = xarray.open_zarr("gs://cornerstone-luc/v3-jdluc-sluc-south-america/emit.zarr", consolidated=False) +>>> import pandas +>>> emission_factors = pandas.read_parquet("emissions-factors.parquet") ``` -Emissions factors (parquet): +See `docs/data.md` for the full column reference and `docs/coverage.md` for the countries and crops a version covers. -```python ->>> import pandas ->>> emission_factors = pandas.read_parquet("gs://cornerstone-luc/v3-jdluc-sluc-south-america/emissions-factors.parquet") -``` +The harmonized inputs and per-pixel emissions behind that table are pipeline outputs rather than published artifacts — zarr stores far too large to deposit alongside it. Reproduce them with the steps under [Running the pipeline](#running-the-pipeline); sharing the underlying maps is open work. The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Please follow the latest attribution guidance in ATTRIBUTION.md. @@ -48,26 +39,18 @@ The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Once you're ready to look under the hood: -- `docs/executive_summary.md` — the short, non-technical overview: what jdLUC is, why LUC - emissions matter, and the headline takeaways. Start here if you're new. -- `docs/methodology.md` — the high-level overview of the methodology: the datasets behind - it, how we quantify per-pixel emissions, and how the jurisdictional-direct and statistical - attribution legs produce emissions factors. The *what* we compute and *why*, from a - scientific standpoint. -- `docs/architecture.md` — the system architecture and the rationale behind it: why we - build on the [Pangeo](https://pangeo.io/) stack, how the five-stage pipeline is - structured, and the tooling, storage, and caching choices that make it reproducible on a - single host. The *how* it's built and *why those choices*. -- `docs/data.md` — the published data products: storage locations, grids, and full schemas - for the harmonized inputs, per-pixel emissions, and emissions-factor table. -- `docs/coverage.md` — the full list of countries and territories the pipeline produces - emissions factors for, by ISO 3166-1 alpha-3 code. +- `docs/executive_summary.md` — the short, non-technical overview: what jdLUC is, why LUC emissions matter, and the headline takeaways. Start here if you're new. +- `docs/methodology.md` — the high-level overview of the methodology: the datasets behind it, how we quantify per-pixel emissions, and how the jurisdictional-direct and statistical attribution legs produce emissions factors. The *what* we compute and *why*, from a scientific standpoint. +- `docs/architecture.md` — the system architecture and the rationale behind it: why we build on the [Pangeo](https://pangeo.io/) stack, how the five-stage pipeline is structured, and the tooling, storage, and caching choices that make it reproducible on a single host. The *how* it's built and *why those choices*. +- `docs/data.md` — how to get the data products, grids, and full schemas for the harmonized inputs, per-pixel emissions, and emissions-factor table. +- `docs/coverage.md` — the full list of countries and crops the pipeline produces emissions factors for, by ISO 3166-1 alpha-3 code. +- `docs/validation.md` — how the pipeline is measured against external datasets: how the (country, crop) targets are chosen, which relationships are held as regression controls, and what such a comparison can and cannot conclude. The tooling lives in `validation/` and reports against the gaps in `docs/further_research.md`. ## Running and contributing ### Getting set up -You'll need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager, plus a GCP project with GCS access. +You'll need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager. Storage is either local directories or a GCS bucket: `INGEST_ROOT` and `SCRATCH_ROOT` in `.env` take any fsspec-supported URI, and only a `gs://` root needs a GCP project. Every source dataset is fetched over https, so ingest needs no cloud account either way. ```bash # Copy the example env file and fill in your values. @@ -76,39 +59,54 @@ cp .env.example .env # Sync Python dependencies into the project venv. uv sync +``` + +**Local roots.** Point `INGEST_ROOT` and `SCRATCH_ROOT` at absolute paths and create them up front: +```bash +mkdir -p "${HOME}/luc/ingest" "${HOME}/luc/scratch" +``` + +Budget the disk: the two zarr stores dominate, at 12 TiB for the continent-scale benchmark in [`docs/architecture.md`](docs/architecture.md#single-host-by-design). + +**A `gs://` root** additionally needs application-default credentials, which gcsfs finds on its own and GDAL's `/vsigs` reader does not: + +```bash # Authenticate gcloud application-default credentials (for GCS access). gcloud auth application-default login --project "${GCP_PROJECT}" + +# Point GDAL at those credentials. +export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/.config/gcloud/application_default_credentials.json" ``` ### Running the pipeline -The pipeline runs as a sequence of per-stage entry points. Stages 1–3 operate on a **continent** (`AFRICA`, `ASIA`, `EUROPE`, `NORTH_AMERICA`, `OCEANIA`, `RUSSIA`, `SOUTH_AMERICA`, `UNCLASSIFIED`); stages 4–5 operate on one or more **ISO 3166 country codes** plus a `--methodology-name` (`STATISTICAL` or `JURISDICTIONAL_DIRECT`). The example below reproduces the United States via the statistical leg: +Every stage is scoped by one or more **ISO 3166 alpha-3 country codes**, or by `--backfill` for a global run. Stages 4–5 additionally take a `--methodology-name` (`STATISTICAL` or `JURISDICTIONAL_DIRECT`). The example below reproduces Honduras via the statistical leg. Honduras is the cheapest end-to-end run in the corpus: the unit of work is a 10° tile, and it falls inside a single one (`20N_090W`). Swap in `USA` for the jurisdictional-direct leg, which needs the US-only CDL and NASS inputs: ```bash -# 1. Ingest each source dataset for the continent (positional: continent, dataset). +# 1. Ingest each source dataset for the countries (positional: dataset, then ISO codes). # Repeat per DATASET in the inventory (see docs/methodology.md). -uv run python -m jdluc.ingest NORTH_AMERICA GLAD_GLCLUC +uv run python -m jdluc.ingest GLAD_GLCLUC HND -# 2. Harmonize the continent's tiles onto the common grid (--grid-name defaults to GLAD ~30 m). -uv run python jdluc/harmonize.py NORTH_AMERICA +# 2. Harmonize the countries' tiles onto the common grid (--grid-name defaults to GLAD ~30 m). +uv run python jdluc/harmonize.py HND # 3. Compute per-pixel land-conversion emissions (writes zarr). -uv run python jdluc/emit.py NORTH_AMERICA +uv run python jdluc/emit.py HND # 4. Attribute emissions to crops for one or more countries (writes the rollup parquet). -uv run python jdluc/attribute.py USA --methodology-name STATISTICAL +uv run python jdluc/attribute.py HND --methodology-name STATISTICAL # 5. Reduce the rollup to the emissions-factor table (writes parquet). -uv run python jdluc/trace.py USA --methodology-name STATISTICAL +uv run python jdluc/trace.py HND --methodology-name STATISTICAL ``` -Each stage pulls its cached upstreams, so re-running a later stage recomputes only what's missing. The artifacts these stages write are documented in `docs/data.md`. +Stages 1–3 derive their tiles from the country boundaries exactly as stage 4 does, so pre-warming them covers precisely what attribution will request. Each stage pulls its cached upstreams, so re-running a later stage recomputes only what's missing. The artifacts these stages write are documented in `docs/data.md`. ### Running tests ```bash -uv run pytest jdluc +uv run pytest ``` ### Linting the codebase diff --git a/docs/architecture.md b/docs/architecture.md index 6ff235a..86474d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,72 +1,36 @@ # Cornerstone LUC: system architecture and rationale -This document describes how the LUC pipeline is built and, more importantly, **why it is -built this way** — the tooling, storage, and structural decisions behind the methodology in -`methodology.md`. It is written for collaborators who bring domain expertise and/or -software and data-engineering experience, so that they can get oriented quickly and match -the style and philosophy of the repository. +This document describes how the LUC pipeline is built and, more importantly, **why it is built this way** — the tooling, storage, and structural decisions behind the methodology in `methodology.md`. It is written for collaborators who bring domain expertise and/or software and data-engineering experience, so that they can get oriented quickly and match the style and philosophy of the repository. -The methodology itself — how emissions are quantified and attributed to crops — lives in -`methodology.md`. This document is about the machine that runs it. +The methodology itself — how emissions are quantified and attributed to crops — lives in `methodology.md`. This document is about the machine that runs it. ## Design ethos A single principle runs through the choices below: -> **Start simple, allow validation at every step, choose designs that are easy to change, -> and keep our options open for cloud and data-orchestration infrastructure.** +> **Start simple, allow validation at every step, choose designs that are easy to change, and keep our options open for cloud and data-orchestration infrastructure.** Two habits follow from it: -- **Separation of concerns.** Geospatial ingestion, harmonization, and the methodological - logic (detecting conversions, quantifying emissions, attributing them, and reducing them to - factors) are different kinds of work with different failure modes. Keeping them in separate - stages helps ensure that complexity — and the bugs that come with it — stays contained rather - than leaking across boundaries. Ingestion in particular tends to require iteration and manual - inspection, and we prefer to keep that work separate from the science. -- **Minimal structure, abstracted only when deep.** Following *A Philosophy of Software - Design*, we tend to introduce an abstraction only once it becomes a *deep* module — a narrow - interface hiding substantial complexity. Early on we kept structure minimal so that - refactoring stayed cheap, and we favor flat layouts because they make onboarding and - collaboration easier. +- **Separation of concerns.** Geospatial ingestion, harmonization, and the methodological logic (detecting conversions, quantifying emissions, attributing them, and reducing them to factors) are different kinds of work with different failure modes. Keeping them in separate stages helps ensure that complexity — and the bugs that come with it — stays contained rather than leaking across boundaries. Ingestion in particular tends to require iteration and manual inspection, and we prefer to keep that work separate from the science. +- **Minimal structure, abstracted only when deep.** Following *A Philosophy of Software Design*, we tend to introduce an abstraction only once it becomes a *deep* module — a narrow interface hiding substantial complexity. Early on we kept structure minimal so that refactoring stayed cheap, and we favor flat layouts because they make onboarding and collaboration easier. ## Why Pangeo, not Earth Engine -The most consequential decision was to build on the [Pangeo](https://pangeo.io/) stack -(xarray + dask + rasterio/rioxarray + GDAL, over object storage) rather than Google Earth -Engine. There is no Earth Engine and no BigQuery anywhere in the pipeline: all raster work -runs locally (or on any dask-capable host) and all outputs land in GCS as zarr and parquet. - -We seriously considered staying on Earth Engine — it is the most *convenient* option, it -offers the fastest wall-clock compute, and it is widely accepted in the field. We moved to -Pangeo anyway, for reasons that all trace back to the ethos above: - -- **Separation of concerns.** GEE merges the data catalog, ingestion, compute, and - visualization into one system. That convenience is also its cost: complexity (and bugs) - can span every layer at once. A staged Pangeo pipeline isolates ingestion from - harmonization from methodology, so a problem in one stage stays in one stage. -- **Inspectability.** Being able to download every intermediate artifact and open it in - QGIS, GDAL, or Python was essential to tracking down bugs and verifying their fixes. - Exporting rasters out of Earth Engine is slow and expensive, which makes that - tight inspect-fix-verify loop painful. -- **Local reproducibility and portability.** Running locally means results are quick to - reproduce, execution is portable — the same code can be hoisted into a cloud VM or a data - orchestrator — and deterministic, isolated unit tests can be written. GEE's hosted - compute model does not allow that. -- **Keeping our options open.** Earth Engine is a great and complete tool, but adopting it - for the catalog, ingestion, compute, *and* visualization forecloses experimentation with - other tools that may be cheaper, better integrated, or more featureful. Pangeo keeps - those doors open. - -The trade-off we accept is some wall-clock speed and the convenience of a ready-made catalog, -in exchange for isolation, inspectability, reproducibility, and flexibility which each help -development more forwards faster and more confidently. +The most consequential decision was to build on the [Pangeo](https://pangeo.io/) stack (xarray + dask + rasterio/rioxarray + GDAL, over object storage) rather than Google Earth Engine. There is no Earth Engine and no BigQuery anywhere in the pipeline: all raster work runs locally (or on any dask-capable host) and all outputs land as zarr and parquet under the configured roots — local directories or a GCS bucket. + +We seriously considered staying on Earth Engine — it is the most *convenient* option, it offers the fastest wall-clock compute, and it is widely accepted in the field. We moved to Pangeo anyway, for reasons that all trace back to the ethos above: + +- **Separation of concerns.** GEE merges the data catalog, ingestion, compute, and visualization into one system. That convenience is also its cost: complexity (and bugs) can span every layer at once. A staged Pangeo pipeline isolates ingestion from harmonization from methodology, so a problem in one stage stays in one stage. +- **Inspectability.** Being able to download every intermediate artifact and open it in QGIS, GDAL, or Python was essential to tracking down bugs and verifying their fixes. Exporting rasters out of Earth Engine is slow and expensive, which makes that tight inspect-fix-verify loop painful. +- **Local reproducibility and portability.** Running locally means results are quick to reproduce, execution is portable — the same code can be hoisted into a cloud VM or a data orchestrator — and deterministic, isolated unit tests can be written. GEE's hosted compute model does not allow that. +- **Keeping our options open.** Earth Engine is a great and complete tool, but adopting it for the catalog, ingestion, compute, *and* visualization forecloses experimentation with other tools that may be cheaper, better integrated, or more featureful. Pangeo keeps those doors open. + +The trade-off we accept is some wall-clock speed and the convenience of a ready-made catalog, in exchange for isolation, inspectability, reproducibility, and flexibility which each help development more forwards faster and more confidently. ## The pipeline -The pipeline is a linear chain of five stages, each consuming the previous stage's output. -The linearity is deliberate: it mirrors the separation of concerns and makes the dependency -graph trivial to reason about. +The pipeline is a linear chain of five stages, each consuming the previous stage's output. The linearity is deliberate: it mirrors the separation of concerns and makes the dependency graph trivial to reason about. ```mermaid flowchart LR @@ -76,192 +40,108 @@ flowchart LR attribute --> trace["trace
emissions factors"] ``` -- **Ingest** — download every external source dataset into GCS as tiled Cloud-Optimized - GeoTIFFs (COGs, raster), FlatGeobuf (vector), or parquet (tabular), tagged with provenance - metadata. -- **Harmonize** — mosaic the ingested tiles and warp them onto a single common grid, - yielding one lazy `xarray.Dataset` backed by a zarr. -- **Emit** — compute per-pixel land conversion events and emissions as a lazy dask/xarray - graph and persist to zarr. -- **Attribute** — clip per-pixel emissions to jurisdiction polygons and crop masks, rolling - up to per-(jurisdiction, crop) totals via the jurisdictional-direct or statistical leg. -- **Trace** — join the attribution rollups to production (NASS yields for the direct leg, - MapSPAM for the statistical leg) to produce the final emissions-factor table. +- **Ingest** — download every external source dataset into `INGEST_ROOT` as tiled Cloud-Optimized GeoTIFFs (COGs, raster), FlatGeobuf (vector), or parquet (tabular), tagged with provenance metadata, and with minimal changes. +- **Harmonize** — align the ingested sources onto a common grid, one 10° tile at a time, yielding one lazy `xarray.Dataset` per tile backed by a zarr. +- **Emit** — compute per-pixel land conversion events and emissions as a lazy dask/xarray graph and persist to zarr. +- **Attribute** — clip per-pixel emissions to jurisdiction polygons and crop masks, rolling up to per-(jurisdiction, crop) totals via the jurisdictional-direct or statistical leg. +- **Trace** — join the attribution rollups to production (NASS yields for the direct leg, MapSPAM for the statistical leg) to produce the final emissions-factor table. ## Compute platform and libraries -Raster computation is expressed as lazy `xarray.Dataset` / `dask.array` graphs built with -rioxarray and rasterio; GDAL VRTs do the mosaicking and grid warping. Nothing materializes -until a stage writes its result, at which point dask streams the computation tile-by-tile -into the output store. +Raster computation is expressed as lazy `xarray.Dataset` / `dask.array` graphs built with rioxarray and rasterio; GDAL VRTs do the grid alignment and resampling. Nothing materializes until a stage writes its result, at which point dask streams the computation tile-by-tile into the output store. Each library earns its place: -- **dask** — careful chunking of the compute DAG is what makes it possible to process - global 30 m datasets *on a laptop*. Chunked compute-and-write means no specialized - hardware is ever needed. -- **xarray** — expressive, performant, and convenient for compute, validation, and testing. - Labeled dimensions make the methodology code read close to how it is described. -- **rasterio / rioxarray** — Python entry points into most of the GDAL functionality that - is needed (without having to become an expert in GDAL!). +- **dask** — careful chunking of the compute DAG is what makes it possible to process global 30 m datasets *on a laptop*. Chunked compute-and-write means no specialized hardware is ever needed. +- **xarray** — expressive, performant, and convenient for compute, validation, and testing. Labeled dimensions make the methodology code read close to how it is described. +- **rasterio / rioxarray** — Python entry points into most of the GDAL functionality that is needed (without having to become an expert in GDAL!). -Together the Pangeo stack delivers high performance *and* good development ergonomics -without leaving the Python ecosystem. +Together the Pangeo stack delivers high performance *and* good development ergonomics without leaving the Python ecosystem. -Configuration comes from a `.env` file, loaded once via `config.Config.from_dot_env()`. -GCS access uses application-default credentials (`gcloud auth application-default login`). +Configuration comes from a `.env` file, loaded once via `config.Config.from_dot_env()`. `INGEST_ROOT` and `SCRATCH_ROOT` are URIs resolved through fsspec, so local directories and `gs://` buckets are interchangeable; `storage.to_gdal_path` rewrites `gs://` to `/vsigs/` for the GDAL readers and passes a local path through untouched. A `gs://` root needs application-default credentials (`gcloud auth application-default login`), with `GOOGLE_APPLICATION_CREDENTIALS` exported to the credentials file gcloud writes — gcsfs finds it without help, but GDAL's `/vsigs` reader does not. A local root needs absolute paths that already exist: `storage.open_zarr_to_dask_dataset` asserts the store it reopens is the URI it asked for, and xarray normalizes local paths to absolute, while parquet writes fail on a missing parent directory. ### No direct GDAL (rasterio only) -One restriction we adopted was to avoid using any GDAL tools directly, and instead rely on the -subset of functionality exposed by the rasterio library. This costs us some capability and -performance, but it avoids the overhead of installing GDAL, which often requires conda or a -specialized Docker image. Instead, we can use simple virtual environments and keep our setup -lean, fast, and portable. +One restriction we adopted was to avoid using any GDAL tools directly, and instead rely on the subset of functionality exposed by the rasterio library. This costs us some capability and performance, but it avoids the overhead of installing GDAL, which often requires conda or a specialized Docker image. Instead, we can use simple virtual environments and keep our setup lean, fast, and portable. ## Storage and formats -The unifying theme is **chunked formats** — every store can be read and written in pieces, -which is what keeps memory bounded end-to-end. +The unifying theme is **chunked formats** — every store can be read and written in pieces, which is what keeps memory bounded end-to-end. -- **zarr** for raster outputs. Together with dask it lets the pipeline chunk both compute - and write, so large rasters never have to fit in RAM. *(Visualization will likely use - different formats down the line, e.g. XYZ-tiled COGs — a rendering concern, distinct from - the analytical store.)* -- **parquet** for tabular outputs. Also chunked; it carries its own schema and interoperates - with a wide range of tools. +- **zarr** for raster outputs. Together with dask it lets the pipeline chunk both compute and write, so large rasters never have to fit in RAM. *(Visualization will likely use different formats down the line, e.g. XYZ-tiled COGs — a rendering concern, distinct from the analytical store.)* +- **parquet** for tabular outputs. Also chunked; it carries its own schema and interoperates with a wide range of tools. - **FlatGeobuf** for vector data — a chunked, streamable vector format. -- **Cloud-Optimized GeoTIFF (COG)** for ingested rasters — the GDAL-native raster - interchange format, trivially inspectable in QGIS/GDAL (which matters for the ingestion - inspect-fix-verify loop), and structured for efficient windowed/range reads directly from - object storage. -- **GCS** as the backing object store. This is a necessity, not a preference: the pipeline - needs somewhere that can cheaply absorb and store its TiBs of data and serve it back at - high throughput. +- **Cloud-Optimized GeoTIFF (COG)** for ingested rasters — the GDAL-native raster interchange format, trivially inspectable in QGIS/GDAL (which matters for the ingestion inspect-fix-verify loop), and structured for efficient windowed/range reads directly from object storage. +- **GCS** as the backing object store for the runs we publish from. At global scale this is a necessity rather than a preference: the pipeline needs somewhere that can cheaply absorb and store its TiBs of data and serve it back at high throughput. Nothing in the code requires it — the storage layer is fsspec-backed, and a country-scale run fits comfortably in local directories. ## Geospatial grid and harmonization ### The common grid -All rasters are harmonized onto the **GLAD GLCLUC native grid**: global `EPSG:4326` at -0.00025° (~30 m). We chose GLAD as the common grid because most of the source datasets are -already provided in it, so warping *to* it minimizes resampling of the very layers that -define land-cover transitions. It is also an intuitive grid and compatible with a tiled -processing approach, with the working grid constructed from the requested tile set. +All rasters are harmonized onto the **GLAD GLCLUC native grid**: global `EPSG:4326` at 0.00025° (~30 m). We chose GLAD as the common grid because most of the source datasets are already provided in it, so warping *to* it minimizes resampling of the very layers that define land-cover transitions. It is also an intuitive grid and compatible with a tiled processing approach: the unit of work is a single 10° tile, and every grid is anchored on the absolute 10° graticule rather than on the extent of whatever was requested. + +A second, coarser grid (the ~10 km MapSPAM resolution) exists for the statistical attribution leg, which downsamples per-pixel emissions to match the resolution of the MapSPAM crop statistics rather than implying a precision the crop data does not have. + +### One VRT per band per tile — chunked end to end + +Harmonization is **one GDAL VRT per source band per tile**. Each VRT declares the destination tile's grid — `EPSG:4326`, with a `` anchored on the absolute 10° graticule — and names the ingested COG as a single `` whose `SrcRect`, `DstRect`, and `resampling` attribute rescale the source onto that grid. + +There is no reprojection step and no mosaic step: -A second, coarser grid (the ~10 km MapSPAM resolution) exists for the statistical -attribution leg, which downsamples per-pixel emissions to match the resolution of the -MapSPAM crop statistics rather than implying a precision the crop data does not have. +- **No reprojection.** Every ingested COG is already `EPSG:4326` — `geo.validate_geotiff` asserts it at ingest — so the VRT only ever rescales and offsets within a single CRS. An earlier design added a second VRT pass (`rasterio.vrt.WarpedVRT`) to reproject; it was removed once the 4326-at-ingest invariant was enforced. (A `WarpedVRT` does survive in `geo.py`, where the statistical leg downsamples to the MapSPAM grid; it too is a same-CRS rescale.) +- **No mosaic.** The unit of work is one tile, so there is never more than one source per band to combine. Two source layouts are handled: ten-degree-tiled datasets (GLAD, GFW peatlands, Harris AGB, Huang BGB, SoilGrids, CDL, TCL) map a whole source tile onto the whole destination tile, while whole-world datasets (IPCC climate zones, MapSPAM) use `SrcRect` to window the tile's footprint out of the global raster. -### Mosaic then warp — chunked end to end +A tile whose source was never published is tolerated when the caller opts in: the VRT is emitted with a band header and no source, which GDAL reads as all-nodata. A minority of tiles need this, and the emissions core coerces those nodatas to zero, so that "no data here" does not propagate as NaN. -Harmonization is two GDAL VRT steps per source band: the ingested tiles are **mosaicked** -onto the common grid, then **warped** to `EPSG:4326` at the grid's transform and resolution -(via `rasterio.vrt.WarpedVRT`). Using VRTs rather than materializing intermediates is central -to the design: the pipeline must be **fully chunked from end to end**, so that at no point is -an entire raster dumped into RAM — doing so would cause massive slowdowns and out-of-memory -failures. The warped VRTs are opened with rioxarray and assembled into one `xarray.Dataset`, -one variable per fully-qualified source band (`{source}:{product}:{band}`). +Using VRTs rather than materializing intermediates is central to the design: the pipeline must be **fully chunked from end to end**, so that at no point is an entire raster dumped into RAM — doing so would cause massive slowdowns and out-of-memory failures. The VRTs are opened with rioxarray and assembled into one `xarray.Dataset`, one variable per fully-qualified source band (`{source}:{product}:{band}`). ### Per-dataset nodata, dtype, and resampling -Resampling is a per-dataset property: nearest-neighbor/mode for categorical layers (GLAD -land classes, CDL, peatland mask, and climate zones) and bilinear/average for intensive layers -(SoilGrids, Huang BGB, and Harris AGB). Extensive layers would be summed when downsampling, but -this functionality isn't currently implemented. The nodata, dtype, and resampling conventions vary -across the external datasets and must be **manually specified and validated** per dataset — these -details matter, and getting them wrong silently corrupts downstream results, which is another -reason ingestion is isolated and independently inspectable. +Resampling is a per-dataset property: nearest-neighbor/mode for categorical layers (GLAD land classes, CDL, peatland mask, and climate zones) and bilinear/average for intensive layers (SoilGrids, Huang BGB, and Harris AGB). Extensive layers would be summed when downsampling, but this functionality isn't currently implemented. The nodata, dtype, and resampling conventions vary across the external datasets and must be **manually specified and validated** per dataset — these details matter, and getting them wrong silently corrupts downstream results, which is another reason ingestion is isolated and independently inspectable. ## Caching, versioning, and provenance ### Stage-output caching (`storage.cache_*`) -Stage outputs are cached in GCS by a pair of decorators in `storage.py`: -`@storage.cache_to_zarr(version)` for `xarray.Dataset` results and -`@storage.cache_to_parquet(version)` for `pandas.DataFrame` results. The cache key is a SHA1 -over the function's module path, qualified name, the integer `version`, and the (non-ignored) -call arguments; each decorator also accepts `ignored_args` for arguments that must not affect -the key. On a hit the stage deserializes the stored result; on a miss it runs and serializes. - -This is a deliberately **lightweight** way to get one of the nicest developer-experience -features of a data orchestrator — cached, addressable stage outputs — *without committing to -one*. When an operation is expensive its result should be cached; it should also be possible -to manually inspect a materialization of that result and reference it via a static URI. The -decorators additionally allow stages to be explicitly linked together (so a required upstream -is generated when needed) without a lot of glue. - -**Versioning is intentionally manual.** The cache key hashes the call arguments plus an -integer `version`; the expectation is that whenever a function's *logic* or its *input -datasets* change, the `version` is bumped. We accept the small risk of a forgotten bump in -exchange for avoiding needless recomputation: - -- The *ideal* would be content-based hashing of the arguments, the input datasets, and the - function's logic. A promising path to that (without materializing results) would be to - pass input datasets *by reference* and fold them into the hash, which would also narrow - the scope of each cache version. -- Hashing on the git SHA would guarantee that stale results are never accidentally reused, but - it would waste time and compute by invalidating on every commit, including cosmetic ones. - That trade is a bad deal. +Stage outputs are cached under `SCRATCH_ROOT` by a pair of decorators in `storage.py`: `@storage.cache_to_zarr(version)` for `xarray.Dataset` results and `@storage.cache_to_parquet(version)` for `pandas.DataFrame` results. The cache key is a SHA1 over the function's module path, qualified name, the integer `version`, and the (non-ignored) call arguments; each decorator also accepts `ignored_args` for arguments that must not affect the key. On a hit the stage deserializes the stored result; on a miss it runs and serializes. + +This is a deliberately **lightweight** way to get one of the nicest developer-experience features of a data orchestrator — cached, addressable stage outputs — *without committing to one*. When an operation is expensive its result should be cached; it should also be possible to manually inspect a materialization of that result and reference it via a static URI. The decorators additionally allow stages to be explicitly linked together (so a required upstream is generated when needed) without a lot of glue. + +**Versioning is intentionally manual.** The cache key hashes the call arguments plus an integer `version`; the expectation is that whenever a function's *logic* or its *input datasets* change, the `version` is bumped. We accept the small risk of a forgotten bump in exchange for avoiding needless recomputation: + +- The *ideal* would be content-based hashing of the arguments, the input datasets, and the function's logic. A promising path to that (without materializing results) would be to pass input datasets *by reference* and fold them into the hash, which would also narrow the scope of each cache version. +- Hashing on the git SHA would guarantee that stale results are never accidentally reused, but it would waste time and compute by invalidating on every commit, including cosmetic ones. That trade is a bad deal. ### Ingest-layer provenance -Provenance for *ingested* data is handled differently, and on purpose. Ingested datasets -are best thought of as minimally-modified, analysis-ready copies of the external sources; -they change rarely, so a more expensive and manual explicit approach is worthwhile. Each -ingested COG carries metadata tags (`watershed-data-version`, -`watershed-processing-version` = git SHA, `watershed-processing-time`, -`watershed-source-name`, `watershed-product-name`, `watershed-remote-url`), and tiles are -stored under a deterministic prefix `{source}/{product}/{version}/{partitioning}/{tile_id}`. -The goal is that anyone who stumbles onto an artifact can understand how it was created and -where to find more context. +Provenance for *ingested* data is handled differently, and on purpose. Ingested datasets are best thought of as minimally-modified, analysis-ready copies of the external sources; they change rarely, so a more expensive and manual explicit approach is worthwhile. Each ingested COG carries metadata tags (`watershed-data-version`, `watershed-processing-version` = git SHA, `watershed-processing-time`, `watershed-source-name`, `watershed-product-name`, `watershed-remote-url`), and tiles are stored under a deterministic prefix `{source}/{product}/{version}/{partitioning}/{tile_id}`. The goal is that anyone who stumbles onto an artifact can understand how it was created and where to find more context. -The contrast is deliberate: **explicit provenance for slow-changing ingested sources; -light, implicit cache keys for the internal logic that changes often.** +The contrast is deliberate: **explicit provenance for slow-changing ingested sources; light, implicit cache keys for the internal logic that changes often.** ## Testing -Tests live in `jdluc/__tests__/`. Unit tests cover pure logic (land-class mapping, grid -math, chunking, transition/span weighting) with **no network** — a direct payoff of the -local, deterministic execution model that motivated the move off Earth Engine. The aim -is for coverage to be high for core logic where logic resides and can change; much of the -imperative shell is not automatically tested. +Tests live in `jdluc/__tests__/` for the pipeline and `validation/__tests__/` for the validation tooling, and CI runs both. Unit tests cover pure logic (land-class mapping, grid math, chunking, transition/span weighting) against data the test itself supplies — a direct payoff of the local, deterministic execution model that motivated the move off Earth Engine. The validation suite works the same way, on invented rows in the user-assigned `XA` ISO range, so the report's failure paths are exercised without waiting for a real run to regress. The aim is for coverage to be high for core logic where logic resides and can change; much of the imperative shell is not automatically tested. ## Scale and scope ### Single host, by design -The pipeline runs on a single dask-capable host — no distributed cluster. We started on a -single host because it is the simplest thing that works, and the tooling choices above mean -the pipeline is not *stuck* there: much of the computation is parallelizable, and the same -code is portable to a cloud VM or any of a number of data orchestrators (which would also -bring their own versioning, metadata, caching, and monitoring). Staying local keeps that -option open rather than closing it. Because the entire pipeline is chunkable end to end, -there is no known ceiling: memory stays bounded regardless of tile set (WHOLE_WORLD included), -so scaling up is a question of wall-clock time and where it is run — not of whether the host -can hold the data. - -The benchmark below is an **M4 MacBook (48 GB RAM)** with the default configuration, for the -51 tile North America continent: - -| Stage | Wall-clock | Cached output | Output size | -|---|---|---|---| -| Ingest | ~1 h | source tiles (COG) | 36.4 GiB | -| Harmonize | ~1.5 h | common-grid `xarray.Dataset` (zarr) | 4.1 TiB | -| Emit | ~2 h | per-pixel emissions (zarr) | 8.1 TiB | -| Attribute (Statistical ) | ~3 h | downscale (zarr) | 3.5 MiB | -| Attribute (Statistical ) | ~15 m | rollup (parquet) | < 1 MiB | -| Attribute (Jurisdictional Direct) | ~15 m | rollup (parquet) | < 1 MiB | -| Trace | ~1 m | EF table (parquet) | < 1 MiB | - -End-to-end ≈ **8 h** for North America. Harmonize, emit, and attribute are the cost centers; -storage is dominated by the two zarr stores. Attribute is compute-heavy despite its -kilobyte-scale output because it streams the full per-pixel zarr through the clip/mask/rollup. +The pipeline runs on a single dask-capable host — no distributed cluster. We started on a single host because it is the simplest thing that works, and the tooling choices above mean the pipeline is not *stuck* there: much of the computation is parallelizable, and the same code is portable to a cloud VM or any of a number of data orchestrators (which would also bring their own versioning, metadata, caching, and monitoring). Staying local keeps that option open rather than closing it. Because the entire pipeline is chunkable end to end, there is no known ceiling: the working set is a single 10° tile no matter how many tiles are requested, so scaling up is a question of wall-clock time and where it is run — not of whether the host can hold the data. + +The benchmark below is an **M4 MacBook (48 GB RAM)** with the default configuration, covering the 51 tiles of North America. It was measured on an earlier code path that built one region-sized grid per stage — which is why the rows are regional totals rather than per-tile. The per-tile path processes the same pixels and writes a comparable volume, but has not yet been re-benchmarked end to end, and tiles are currently processed serially with a dask `LocalCluster` spun up per tile. Treat the wall-clock column as the historical measurement it is, not as a current one: + +| Stage | Wall-clock | Cached output | Output size | +| ----------------------------------------- | ---------- | ----------------------------------- | ----------- | +| Ingest | ~1 h | source tiles (COG) | 36.4 GiB | +| Harmonize | ~1.5 h | common-grid `xarray.Dataset` (zarr) | 4.1 TiB | +| Emit | ~2 h | per-pixel emissions (zarr) | 8.1 TiB | +| Attribute (Statistical, downscale) | ~3 h | downscale (zarr) | 3.5 MiB | +| Attribute (Statistical, rollup) | ~15 m | rollup (parquet) | < 1 MiB | +| Attribute (Jurisdictional Direct, rollup) | ~15 m | rollup (parquet) | < 1 MiB | +| Trace | ~1 m | EF table (parquet) | < 1 MiB | + +End-to-end ≈ **8 h** for North America. Harmonize, emit, and attribute are the cost centers; storage is dominated by the two zarr stores. Attribute is compute-heavy despite its kilobyte-scale output because it streams the full per-pixel zarr through the clip/mask/rollup. ### Why the United States first -The proof of concept targets US row crops for an architectural reason as well as a -methodological one: the US has high-quality open data (notably CDL and NASS), which lends -itself to a genuinely free and open implementation that is also accurate. Crucially, good -input data isolates any discrepancies to the **methodology** rather than to data -quality — exactly the kind of clean signal a proof of concept needs. +The proof of concept targets US crops for an architectural reason as well as a methodological one: the US has high-quality open data (notably CDL and NASS), which lends itself to a genuinely free and open implementation that is also accurate. Crucially, good input data isolates any discrepancies to the **methodology** rather than to data quality — exactly the kind of clean signal a proof of concept needs. diff --git a/docs/cdl_glad_comparison_supplement.md b/docs/cdl_glad_comparison_supplement.md index 0b1c34c..abdb0b0 100644 --- a/docs/cdl_glad_comparison_supplement.md +++ b/docs/cdl_glad_comparison_supplement.md @@ -1,89 +1,58 @@ # GLAD GLC vs CDL row crop comparison -A supplement to [`methodology.md`](methodology.md), assessing whether restricting the -jurisdictional-direct leg to GLAD-identified cropland biases the resulting emissions -factors. +A supplement to [`methodology.md`](methodology.md), assessing whether restricting the jurisdictional-direct leg to GLAD-identified cropland biases the resulting emissions factors. -The methodology calculates emissions for pixels identified by GLAD GLC as cropland, using -CDL to allocate among crops. This means that any CDL pixels not identified as cropland by -GLAD GLC 2020 are excluded from both emissions and production. +The methodology calculates emissions for pixels identified by GLAD GLC as cropland, using CDL to allocate among crops. This means that any CDL pixels not identified as cropland by GLAD GLC 2020 are excluded from both emissions and production. -To test whether this exclusion likely biases our EFs, we computed confusion matrices -crossing CDL row crop classification against GLAD GLC cropland (pixel value 244) for all 48 -CONUS states + DC, comparing CDL 2020 × GLAD GLC 2020. +To test whether this exclusion likely biases our EFs, we computed confusion matrices crossing CDL row crop classification against GLAD GLC cropland (pixel value 244) for all 48 CONUS states + DC, comparing CDL 2020 × GLAD GLC 2020. -| | GLAD GLC cropland | GLAD GLC non-cropland | Total | -|---|---:|---:|---:| -| **CDL row crop** | 99,475,866 ha | 10,248,055 ha | 109,723,921 ha | -| **CDL not row crop** | 37,238,049 ha | 631,957,357 ha | 669,195,406 ha | -| **Total** | 136,713,915 ha | 642,205,412 ha | 778,919,327 ha | +| | GLAD GLC cropland | GLAD GLC non-cropland | Total | +| -------------------- | ----------------: | --------------------: | -------------: | +| **CDL row crop** | 99,475,866 ha | 10,248,055 ha | 109,723,921 ha | +| **CDL not row crop** | 37,238,049 ha | 631,957,357 ha | 669,195,406 ha | +| **Total** | 136,713,915 ha | 642,205,412 ha | 778,919,327 ha | 90.7% of CDL row crop pixels are also identified as crops by GLAD. -The 9.3% of "lost" CDL row crops are concentrated in regions with smaller, more fragmented -fields: +The 9.3% of "lost" CDL row crops are concentrated in regions with smaller, more fragmented fields: -| Region | GLAD GLC confirmation rate | -|---|---| -| Corn Belt (IA, IL, IN, NE, OH) | 91–95% | -| Great Plains (ND, SD, KS, MT) | 91–95% | -| Southeast (AL, FL, GA, SC) | 75–82% | -| Northeast (CT, MA, RI, PA) | 39–77% | +| Region | GLAD GLC confirmation rate | +| ------------------------------ | -------------------------- | +| Corn Belt (IA, IL, IN, NE, OH) | 91–95% | +| Great Plains (ND, SD, KS, MT) | 91–95% | +| Southeast (AL, FL, GA, SC) | 75–82% | +| Northeast (CT, MA, RI, PA) | 39–77% | -These pixels fall into two cases, which we analyzed for 10 key agricultural states, giving -what we'd expect to be a representative picture for CONUS overall: +These pixels fall into two cases, which we analyzed for 10 key agricultural states, giving what we'd expect to be a representative picture for CONUS overall: **GLC changed 2000→2020** (24% of excluded pixels) -| GLC 2000 source → GLC 2020 destination | Area (ha) | % of disagreement | Emissions relevance | -|---|---:|---:|---| -| Cropland → non-cropland | 1,130,028 | 21% | None | -| Short veg → non-cropland | 99,962 | 2% | Grassland conversion emissions | -| Forest → non-cropland | 32,392 | <1% | Forest conversion emissions | -| Wetland → non-cropland | 16,720 | <1% | Grassland/forest | -| Water/bare/other → non-cropland | 6,790 | <1% | Negligible | +| GLC 2000 source → GLC 2020 destination | Area (ha) | % of disagreement | Emissions relevance | +| -------------------------------------- | --------: | ----------------: | ------------------------------ | +| Cropland → non-cropland | 1,130,028 | 21% | None | +| Short veg → non-cropland | 99,962 | 2% | Grassland conversion emissions | +| Forest → non-cropland | 32,392 | \<1% | Forest conversion emissions | +| Wetland → non-cropland | 16,720 | \<1% | Grassland/forest | +| Water/bare/other → non-cropland | 6,790 | \<1% | Negligible | **GLC stable 2000=2020** (76% of excluded pixels) -| GLC 2020 class (stable since 2000) | Area (ha) | % of disagreement | Emissions relevance | -|---|---:|---:|---| -| Built-up | 2,419,875 | 44% | None — farmsteads, grain bins, rural infrastructure. Zero biomass. | -| Short vegetation | 1,622,525 | 30% | Would generate grassland→cropland emissions if misclassified. | -| Forest | 77,956 | 1% | Would generate forest emissions if misclassified. | -| Wetland | 66,092 | 1% | Would generate forest or grassland emissions. | -| Water/bare/other | 8,429 | <1% | Negligible | - -Because lost pixels are excluded from both the emissions numerator and the production -denominator, what matters for EF accuracy is whether they are systematically different from -the in-scope population. If so, it appears that it's in a way that biases the EF slightly -upward rather than downward: only ~3% of lost pixels have a GLC history showing conversion -from a non-crop source, compared with the ~6.5% conversion-from-non-crop rate observed -among the in-scope CDL row crop population. The dropped pixels are, if anything, biased -toward stable, non-converting land. +| GLC 2020 class (stable since 2000) | Area (ha) | % of disagreement | Emissions relevance | +| ---------------------------------- | --------: | ----------------: | ------------------------------------------------------------------ | +| Built-up | 2,419,875 | 44% | None — farmsteads, grain bins, rural infrastructure. Zero biomass. | +| Short vegetation | 1,622,525 | 30% | Would generate grassland→cropland emissions if misclassified. | +| Forest | 77,956 | 1% | Would generate forest emissions if misclassified. | +| Wetland | 66,092 | 1% | Would generate forest or grassland emissions. | +| Water/bare/other | 8,429 | \<1% | Negligible | + +Because lost pixels are excluded from both the emissions numerator and the production denominator, what matters for EF accuracy is whether they are systematically different from the in-scope population. If so, it appears that it's in a way that biases the EF slightly upward rather than downward: only ~3% of lost pixels have a GLC history showing conversion from a non-crop source, compared with the ~6.5% conversion-from-non-crop rate observed among the in-scope CDL row crop population. The dropped pixels are, if anything, biased toward stable, non-converting land. ## The statistical leg -The comparison above concerns the jurisdictional-direct leg, where the GLAD-cropland -restriction is symmetric: the same per-pixel crop mask gates both the emissions numerator -and the crop-area denominator, so a lost pixel drops out of both together. - -The statistical leg applies the same GLAD-2020-cropland restriction, but only to the -emissions numerator (`get_downscaled_luc_emissions` in `statistical.py`). This is -deliberate, not an oversight. Its denominator is MapSPAM crop production — a crop quantity, -and therefore zero wherever there is no cropland — so the denominator is already -cropland-restricted by construction, to MapSPAM's delineation of cropland rather than -GLAD's. Masking the numerator to GLAD-2020 cropland is what brings the two into alignment. -Without it, conversion emissions whose destination is non-agricultural (e.g. forest or -grassland lost to built-up) would enter the emissions pool and be spread across crops by -expansion share — charging, say, a town's expansion to soybeans. - -The only residual inconsistency is that GLAD's and MapSPAM's cropland footprints do not -perfectly coincide: a pixel MapSPAM credits with crop area but GLAD does not call cropland -in 2020 sits in the denominator yet is excluded from the numerator. This is the same -dataset-delineation disagreement quantified above for CDL, now between GLAD and MapSPAM. At -the ~10 km MapSPAM resolution the two footprints cannot be co-registered any more tightly, -and the mismatch is not removable by dropping the mask — that would only add the -non-agricultural emissions described above. - -See [`analyses/cdl_glad_glc_comparison.ipynb`](../analyses/cdl_glad_glc_comparison.ipynb) -for the full analysis. +The comparison above concerns the jurisdictional-direct leg, where the GLAD-cropland restriction is symmetric: the same per-pixel crop mask gates both the emissions numerator and the crop-area denominator, so a lost pixel drops out of both together. + +The statistical leg applies the same GLAD-2020-cropland restriction, but only to the emissions numerator (`get_downscaled_luc_emissions` in `statistical.py`). This is deliberate, not an oversight. Its denominator is MapSPAM crop production — a crop quantity, and therefore zero wherever there is no cropland — so the denominator is already cropland-restricted by construction, to MapSPAM's delineation of cropland rather than GLAD's. Masking the numerator to GLAD-2020 cropland is what brings the two into alignment. Without it, conversion emissions whose destination is non-agricultural (e.g. forest or grassland lost to built-up) would enter the emissions pool and be spread across crops by expansion share — charging, say, a town's expansion to soybeans. + +The only residual inconsistency is that GLAD's and MapSPAM's cropland footprints do not perfectly coincide: a pixel MapSPAM credits with crop area but GLAD does not call cropland in 2020 sits in the denominator yet is excluded from the numerator. This is the same dataset-delineation disagreement quantified above for CDL, now between GLAD and MapSPAM. At the ~10 km MapSPAM resolution the two footprints cannot be co-registered any more tightly, and the mismatch is not removable by dropping the mask — that would only add the non-agricultural emissions described above. + +See [`analyses/cdl_glad_glc_comparison.ipynb`](../analyses/cdl_glad_glc_comparison.ipynb) for the full analysis. diff --git a/docs/coverage.md b/docs/coverage.md index 1fd5a69..0fdc538 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -2,230 +2,236 @@ ## Countries -The pipeline produces emissions factors for the following 220 countries and territories, identified by ISO 3166-1 alpha-3 code. +The pipeline produces emissions factors for the following countries and territories, identified by ISO 3166-1 alpha-3 code. -| Country | ISO 3166-1 alpha-3 | -| --- | --- | -| Afghanistan | AFG | -| Åland Islands | ALA | -| Albania | ALB | -| Algeria | DZA | -| American Samoa | ASM | -| Andorra | AND | -| Angola | AGO | -| Anguilla | AIA | -| Antigua and Barbuda | ATG | -| Argentina | ARG | -| Armenia | ARM | -| Aruba | ABW | -| Australia | AUS | -| Austria | AUT | -| Azerbaijan | AZE | -| Bahamas | BHS | -| Bahrain | BHR | -| Bangladesh | BGD | -| Barbados | BRB | -| Belarus | BLR | -| Belgium | BEL | -| Belize | BLZ | -| Benin | BEN | -| Bermuda | BMU | -| Bhutan | BTN | -| Bolivia | BOL | -| Bonaire, Sint Eustatius and Saba | BES | -| Bosnia and Herzegovina | BIH | -| Botswana | BWA | -| Brazil | BRA | -| Brunei Darussalam | BRN | -| Bulgaria | BGR | -| Burkina Faso | BFA | -| Burundi | BDI | -| Cambodia | KHM | -| Cameroon | CMR | -| Canada | CAN | -| Cayman Islands | CYM | -| Central African Republic | CAF | -| Chad | TCD | -| Chile | CHL | -| China | CHN | -| Colombia | COL | -| Comoros | COM | -| Congo | COG | -| Congo, The Democratic Republic of the | COD | -| Costa Rica | CRI | -| Côte d'Ivoire | CIV | -| Croatia | HRV | -| Cuba | CUB | -| Curaçao | CUW | -| Cyprus | CYP | -| Czechia | CZE | -| Denmark | DNK | -| Djibouti | DJI | -| Dominica | DMA | -| Dominican Republic | DOM | -| Ecuador | ECU | -| Egypt | EGY | -| El Salvador | SLV | -| Equatorial Guinea | GNQ | -| Eritrea | ERI | -| Estonia | EST | -| Eswatini | SWZ | -| Ethiopia | ETH | -| Fiji | FJI | -| Finland | FIN | -| France | FRA | -| French Guiana | GUF | -| French Southern Territories | ATF | -| Gabon | GAB | -| Gambia | GMB | -| Georgia | GEO | -| Germany | DEU | -| Ghana | GHA | -| Greece | GRC | -| Grenada | GRD | -| Guadeloupe | GLP | -| Guatemala | GTM | -| Guernsey | GGY | -| Guinea | GIN | -| Guinea-Bissau | GNB | -| Guyana | GUY | -| Haiti | HTI | -| Honduras | HND | -| Hong Kong | HKG | -| Hungary | HUN | -| Iceland | ISL | -| India | IND | -| Indonesia | IDN | -| Iran | IRN | -| Iraq | IRQ | -| Ireland | IRL | -| Isle of Man | IMN | -| Israel | ISR | -| Italy | ITA | -| Jamaica | JAM | -| Japan | JPN | -| Jersey | JEY | -| Jordan | JOR | -| Kazakhstan | KAZ | -| Kenya | KEN | -| Kiribati | KIR | -| Kosovo | XKX | -| Kuwait | KWT | -| Kyrgyzstan | KGZ | -| Laos | LAO | -| Latvia | LVA | -| Lebanon | LBN | -| Lesotho | LSO | -| Liberia | LBR | -| Libya | LBY | -| Liechtenstein | LIE | -| Lithuania | LTU | -| Luxembourg | LUX | -| Macao | MAC | -| Madagascar | MDG | -| Malawi | MWI | -| Malaysia | MYS | -| Maldives | MDV | -| Mali | MLI | -| Malta | MLT | -| Martinique | MTQ | -| Mauritania | MRT | -| Mauritius | MUS | -| Mayotte | MYT | -| Mexico | MEX | -| Micronesia, Federated States of | FSM | -| Moldova | MDA | -| Mongolia | MNG | -| Montenegro | MNE | -| Montserrat | MSR | -| Morocco | MAR | -| Mozambique | MOZ | -| Myanmar | MMR | -| Namibia | NAM | -| Nepal | NPL | -| Netherlands | NLD | -| New Caledonia | NCL | -| New Zealand | NZL | -| Nicaragua | NIC | -| Niger | NER | -| Nigeria | NGA | -| Norfolk Island | NFK | -| North Korea | PRK | -| North Macedonia | MKD | -| Norway | NOR | -| Oman | OMN | -| Pakistan | PAK | -| Palau | PLW | -| Palestine, State of | PSE | -| Panama | PAN | -| Papua New Guinea | PNG | -| Paraguay | PRY | -| Peru | PER | -| Philippines | PHL | -| Poland | POL | -| Portugal | PRT | -| Puerto Rico | PRI | -| Qatar | QAT | -| Réunion | REU | -| Romania | ROU | -| Russian Federation | RUS | -| Rwanda | RWA | -| Saint Kitts and Nevis | KNA | -| Saint Lucia | LCA | -| Saint Pierre and Miquelon | SPM | -| Saint Vincent and the Grenadines | VCT | -| Samoa | WSM | -| San Marino | SMR | -| Sao Tome and Principe | STP | -| Saudi Arabia | SAU | -| Senegal | SEN | -| Serbia | SRB | -| Seychelles | SYC | -| Sierra Leone | SLE | -| Singapore | SGP | -| Sint Maarten (Dutch part) | SXM | -| Slovakia | SVK | -| Slovenia | SVN | -| Solomon Islands | SLB | -| Somalia | SOM | -| South Africa | ZAF | -| South Korea | KOR | -| South Sudan | SSD | -| Spain | ESP | -| Sri Lanka | LKA | -| Sudan | SDN | -| Suriname | SUR | -| Svalbard and Jan Mayen | SJM | -| Sweden | SWE | -| Switzerland | CHE | -| Syria | SYR | -| Tajikistan | TJK | -| Tanzania | TZA | -| Thailand | THA | -| Timor-Leste | TLS | -| Togo | TGO | -| Tonga | TON | -| Trinidad and Tobago | TTO | -| Tunisia | TUN | -| Türkiye | TUR | -| Turkmenistan | TKM | -| Turks and Caicos Islands | TCA | -| Uganda | UGA | -| Ukraine | UKR | -| United Arab Emirates | ARE | -| United Kingdom | GBR | -| United States | USA | -| Uruguay | URY | -| Uzbekistan | UZB | -| Vanuatu | VUT | -| Venezuela | VEN | -| Vietnam | VNM | -| Virgin Islands, British | VGB | -| Virgin Islands, U.S. | VIR | -| Wallis and Futuna | WLF | -| Yemen | YEM | -| Zambia | ZMB | -| Zimbabwe | ZWE | +### Territorial extent + +A country's footprint is its **full national boundary** (World Bank admin-0) intersected with the ten-degree tiles the source datasets actually publish. A country appears below when it touches at least one published tile *and* carries crop production for the pipeline to attribute emissions to. + +The first half is a property of the source data rather than a curated list, so the set can shift when an upstream dataset is refreshed. The second half is a list: eight countries touch a published tile but grow nothing the pipeline can produce a factor for — Saint Barthélemy, Gibraltar, Greenland, Saint Martin, Monaco, Nauru, Tuvalu and the Holy See — and no property of the geometry separates them, so they are named explicitly as `worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S` and excluded there — by `get_all_iso_3166s`, which is what `--backfill` runs, and by `tools/build-tiled-countries.py`, which reads the same constant. The table below and `validation/data/tiled_iso_3166s.json` are the same 220 codes; regenerating that file is what keeps them so. + +| Country | ISO 3166-1 alpha-3 | +| ------------------------------------- | ------------------ | +| Afghanistan | AFG | +| Åland Islands | ALA | +| Albania | ALB | +| Algeria | DZA | +| American Samoa | ASM | +| Andorra | AND | +| Angola | AGO | +| Anguilla | AIA | +| Antigua and Barbuda | ATG | +| Argentina | ARG | +| Armenia | ARM | +| Aruba | ABW | +| Australia | AUS | +| Austria | AUT | +| Azerbaijan | AZE | +| Bahamas | BHS | +| Bahrain | BHR | +| Bangladesh | BGD | +| Barbados | BRB | +| Belarus | BLR | +| Belgium | BEL | +| Belize | BLZ | +| Benin | BEN | +| Bermuda | BMU | +| Bhutan | BTN | +| Bolivia | BOL | +| Bonaire, Sint Eustatius and Saba | BES | +| Bosnia and Herzegovina | BIH | +| Botswana | BWA | +| Brazil | BRA | +| Brunei Darussalam | BRN | +| Bulgaria | BGR | +| Burkina Faso | BFA | +| Burundi | BDI | +| Cambodia | KHM | +| Cameroon | CMR | +| Canada | CAN | +| Cayman Islands | CYM | +| Central African Republic | CAF | +| Chad | TCD | +| Chile | CHL | +| China | CHN | +| Colombia | COL | +| Comoros | COM | +| Congo | COG | +| Congo, The Democratic Republic of the | COD | +| Costa Rica | CRI | +| Côte d'Ivoire | CIV | +| Croatia | HRV | +| Cuba | CUB | +| Curaçao | CUW | +| Cyprus | CYP | +| Czechia | CZE | +| Denmark | DNK | +| Djibouti | DJI | +| Dominica | DMA | +| Dominican Republic | DOM | +| Ecuador | ECU | +| Egypt | EGY | +| El Salvador | SLV | +| Equatorial Guinea | GNQ | +| Eritrea | ERI | +| Estonia | EST | +| Eswatini | SWZ | +| Ethiopia | ETH | +| Fiji | FJI | +| Finland | FIN | +| France | FRA | +| French Guiana | GUF | +| French Southern Territories | ATF | +| Gabon | GAB | +| Gambia | GMB | +| Georgia | GEO | +| Germany | DEU | +| Ghana | GHA | +| Greece | GRC | +| Grenada | GRD | +| Guadeloupe | GLP | +| Guatemala | GTM | +| Guernsey | GGY | +| Guinea | GIN | +| Guinea-Bissau | GNB | +| Guyana | GUY | +| Haiti | HTI | +| Honduras | HND | +| Hong Kong | HKG | +| Hungary | HUN | +| Iceland | ISL | +| India | IND | +| Indonesia | IDN | +| Iran | IRN | +| Iraq | IRQ | +| Ireland | IRL | +| Isle of Man | IMN | +| Israel | ISR | +| Italy | ITA | +| Jamaica | JAM | +| Japan | JPN | +| Jersey | JEY | +| Jordan | JOR | +| Kazakhstan | KAZ | +| Kenya | KEN | +| Kiribati | KIR | +| Kosovo | XKX | +| Kuwait | KWT | +| Kyrgyzstan | KGZ | +| Laos | LAO | +| Latvia | LVA | +| Lebanon | LBN | +| Lesotho | LSO | +| Liberia | LBR | +| Libya | LBY | +| Liechtenstein | LIE | +| Lithuania | LTU | +| Luxembourg | LUX | +| Macao | MAC | +| Madagascar | MDG | +| Malawi | MWI | +| Malaysia | MYS | +| Maldives | MDV | +| Mali | MLI | +| Malta | MLT | +| Martinique | MTQ | +| Mauritania | MRT | +| Mauritius | MUS | +| Mayotte | MYT | +| Mexico | MEX | +| Micronesia, Federated States of | FSM | +| Moldova | MDA | +| Mongolia | MNG | +| Montenegro | MNE | +| Montserrat | MSR | +| Morocco | MAR | +| Mozambique | MOZ | +| Myanmar | MMR | +| Namibia | NAM | +| Nepal | NPL | +| Netherlands | NLD | +| New Caledonia | NCL | +| New Zealand | NZL | +| Nicaragua | NIC | +| Niger | NER | +| Nigeria | NGA | +| Norfolk Island | NFK | +| North Korea | PRK | +| North Macedonia | MKD | +| Norway | NOR | +| Oman | OMN | +| Pakistan | PAK | +| Palau | PLW | +| Palestine, State of | PSE | +| Panama | PAN | +| Papua New Guinea | PNG | +| Paraguay | PRY | +| Peru | PER | +| Philippines | PHL | +| Poland | POL | +| Portugal | PRT | +| Puerto Rico | PRI | +| Qatar | QAT | +| Réunion | REU | +| Romania | ROU | +| Russian Federation | RUS | +| Rwanda | RWA | +| Saint Kitts and Nevis | KNA | +| Saint Lucia | LCA | +| Saint Pierre and Miquelon | SPM | +| Saint Vincent and the Grenadines | VCT | +| Samoa | WSM | +| San Marino | SMR | +| Sao Tome and Principe | STP | +| Saudi Arabia | SAU | +| Senegal | SEN | +| Serbia | SRB | +| Seychelles | SYC | +| Sierra Leone | SLE | +| Singapore | SGP | +| Sint Maarten (Dutch part) | SXM | +| Slovakia | SVK | +| Slovenia | SVN | +| Solomon Islands | SLB | +| Somalia | SOM | +| South Africa | ZAF | +| South Korea | KOR | +| South Sudan | SSD | +| Spain | ESP | +| Sri Lanka | LKA | +| Sudan | SDN | +| Suriname | SUR | +| Svalbard and Jan Mayen | SJM | +| Sweden | SWE | +| Switzerland | CHE | +| Syria | SYR | +| Tajikistan | TJK | +| Tanzania | TZA | +| Thailand | THA | +| Timor-Leste | TLS | +| Togo | TGO | +| Tonga | TON | +| Trinidad and Tobago | TTO | +| Tunisia | TUN | +| Türkiye | TUR | +| Turkmenistan | TKM | +| Turks and Caicos Islands | TCA | +| Uganda | UGA | +| Ukraine | UKR | +| United Arab Emirates | ARE | +| United Kingdom | GBR | +| United States | USA | +| Uruguay | URY | +| Uzbekistan | UZB | +| Vanuatu | VUT | +| Venezuela | VEN | +| Vietnam | VNM | +| Virgin Islands, British | VGB | +| Virgin Islands, U.S. | VIR | +| Wallis and Futuna | WLF | +| Yemen | YEM | +| Zambia | ZMB | +| Zimbabwe | ZWE | ## Crops @@ -233,55 +239,55 @@ The pipeline produces emissions factors for the following 220 countries and terr The statistical pipeline covers the following 32 crops, identified by IFPRI MapSPAM crop code. -| Crop | MapSPAM code | -| --- | --- | -| ARABICA_COFFEE | ACOF | -| BANANA | BANA | -| BARLEY | BARL | -| BEAN | BEAN | -| CASSAVE | CASS | -| CHICKPEA | CHIC | -| COCONUT | CNUT | -| COTTON | COTT | -| COWPEA | COWP | -| GROUNDNUT | GROU | -| LENTIL | LENT | -| MAIZE | MAIZ | -| OILPALM | OILP | -| OTHER_OILCROPS | OOIL | -| OTHER_PULSES | OPUL | -| PEARL_MILLET | PMIL | -| PIGEONPEA | PIGE | -| PLANTAIN | PLNT | -| POTATO | POTA | -| RAPESEED | RAPE | -| RICE | RICE | -| ROBUSTA_COFFEE | RCOF | -| SESAME_SEED | SESA | -| SMALL_MILLET | SMIL | -| SORGHUM | SORG | -| SOYBEAN | SOYB | -| SUGARBEET | SUGB | -| SUGARCANE | SUGC | -| SUNFLOWER | SUNF | -| SWEET_POTATO | SWPO | -| WHEAT | WHEA | -| YAM | YAMS | +| Crop | MapSPAM code | +| -------------- | ------------ | +| ARABICA_COFFEE | ACOF | +| BANANA | BANA | +| BARLEY | BARL | +| BEAN | BEAN | +| CASSAVA | CASS | +| CHICKPEA | CHIC | +| COCONUT | CNUT | +| COTTON | COTT | +| COWPEA | COWP | +| GROUNDNUT | GROU | +| LENTIL | LENT | +| MAIZE | MAIZ | +| OILPALM | OILP | +| OTHER_OILCROPS | OOIL | +| OTHER_PULSES | OPUL | +| PEARL_MILLET | PMIL | +| PIGEONPEA | PIGE | +| PLANTAIN | PLNT | +| POTATO | POTA | +| RAPESEED | RAPE | +| RICE | RICE | +| ROBUSTA_COFFEE | RCOF | +| SESAME_SEED | SESA | +| SMALL_MILLET | SMIL | +| SORGHUM | SORG | +| SOYBEAN | SOYB | +| SUGARBEET | SUGB | +| SUGARCANE | SUGC | +| SUNFLOWER | SUNF | +| SWEET_POTATO | SWPO | +| WHEAT | WHEA | +| YAM | YAMS | ### jdLUC (jurisdictional-direct) The jurisdictional-direct pipeline (USA only) covers the following 11 crops, mapped to USDA NASS CDL crop classes. -| Crop | USDA NASS CDL class(es) | -| --- | --- | -| BARLEY | BARLEY | -| BEAN | DRY_BEANS | -| COTTON | COTTON | -| MAIZE | CORN | -| POTATO | POTATOES | -| RICE | RICE | -| SORGHUM | SORGHUM | -| SOYBEAN | SOYBEANS | -| SUGARBEET | SUGARBEETS | -| SUGARCANE | SUGARCANE | -| WHEAT | DURUM_WHEAT, SPRING_WHEAT, WINTER_WHEAT | +| Crop | USDA NASS CDL class(es) | +| --------- | --------------------------------------- | +| BARLEY | BARLEY | +| BEAN | DRY_BEANS | +| COTTON | COTTON | +| MAIZE | CORN | +| POTATO | POTATOES | +| RICE | RICE | +| SORGHUM | SORGHUM | +| SOYBEAN | SOYBEANS | +| SUGARBEET | SUGARBEETS | +| SUGARCANE | SUGARCANE | +| WHEAT | DURUM_WHEAT, SPRING_WHEAT, WINTER_WHEAT | diff --git a/docs/data.md b/docs/data.md index 9c15067..f2355b7 100644 --- a/docs/data.md +++ b/docs/data.md @@ -1,107 +1,102 @@ # Cornerstone LUC: data products -This document is the reference for the artifacts the pipeline publishes: their storage -locations, grids, and full schemas. For *what* the numbers mean see `methodology.md`; for -*how* they are produced see `architecture.md`; to reproduce them yourself see the -["Running the pipeline"](../README.md#running-the-pipeline) section of the README. +This document is the reference for the artifacts the pipeline produces: how to get them, grids, and full schemas. For *what* the numbers mean see `methodology.md`; for *how* they are produced see `architecture.md`; to reproduce them yourself see the ["Running the pipeline"](../README.md#running-the-pipeline) section of the README. -The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/); please -follow the attribution guidance in `ATTRIBUTION.md`. +The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/); please follow the attribution guidance in `ATTRIBUTION.md`. + +## Access + +Of the three artifacts below, one is published and two are not. + +**The emissions-factor table is published as an archived dataset.** Each data version is deposited as `emissions-factors.parquet` in a public data archive that mints a DOI per version, and the matching git tag names the commit behind it. The archive is still being set up; cite a version by its DOI, or follow the fallback in `ATTRIBUTION.md`. + +**The two raster artifacts are pipeline outputs.** They are zarr stores far too large to deposit, so they exist only where a run writes them: under the `SCRATCH_ROOT` of whoever ran the pipeline, keyed by the cache hash `storage.py` derives (see [`architecture.md`](architecture.md#caching-versioning-and-provenance)). Sharing the underlying maps in a form that can be read remotely is open work — the schemas below are documented so a run's own output is legible in the meantime. ## Conventions Unless noted otherwise, every raster artifact shares these conventions: -| Property | Value | -|---|---| -| CRS | `EPSG:4326` | +| Property | Value | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| CRS | `EPSG:4326` | | Grid / resolution | GLAD native — `0.00025°` (~30 m). The statistical leg additionally works on a coarser ~10 km MapSPAM grid (`0.0833°`); see `methodology.md`. | -| Dimensions | `(y, x)` per variable, one variable per band | -| Dtype | `float32` (all variables, including categorical codes) | -| No-data | `NaN` | -| Format | zarr (rasters) / parquet (tables), chunked for out-of-core reads | +| Dimensions | `(y, x)` per variable, one variable per band | +| Dtype | `float32` (all variables, including categorical codes) | +| No-data | `NaN` | +| Format | zarr (rasters) / parquet (tables), chunked for out-of-core reads | -Emission quantities use two unit conventions: `t CO₂e/ha` (tonnes CO₂e per hectare, on the -raster layers) and `t CO₂e` (metric tonnes, on the tabular rollups — the `*_mt` columns). -Areas are hectares (`ha`), production is kilograms (`kg`). +Emission quantities use two unit conventions: `t CO₂e/ha` (tonnes CO₂e per hectare, on the raster layers) and `t CO₂e` (metric tonnes, on the tabular rollups — the `*_mt` columns). Areas are hectares (`ha`), production is kilograms (`kg`). ## Artifacts ### 1. Harmonized inputs (zarr) -`gs://cornerstone-luc/v3-jdluc-sluc-south-america/harmonize.zarr` +Written by the `harmonize` stage; a pipeline output, not a published artifact (see [Access](#access)). -Every source raster reprojected and warped onto the common grid — the input to the -emissions core. One variable per fully-qualified source band, named `{source}:{product}:{band}`. +Every source raster reprojected and warped onto the common grid — the input to the emissions core. One variable per fully-qualified source band, named `{source}:{product}:{band}`. -| Variable | Type | Units | Description | -|---|---|---|---| -| `glad:glcluc:year=2000` … `glad:glcluc:year=2020` | float32 | GLCLUC code | GLAD land-cover/land-use class, one variable per year (2000, 2005, 2010, 2015, 2020) | -| `gfw:harris-agb:aboveground-biomass-mg-per-ha` | float32 | Mg/ha | Forest above-ground woody biomass (2000) | -| `huang:bgb:belowground-biomass-mg-per-ha` | float32 | Mg/ha | Forest below-ground (root) biomass | -| `soilgrids:organic-carbon-stocks:organic-soil-carbon-mg-per-ha` | float32 | Mg/ha | Soil organic carbon stock, 0–30 cm | -| `gfw:global-peatlands:is-peatland` | float32 | 0/1 | Binary peatland mask | -| `ipcc:climate-zones:climate-zone` | float32 | zone code | IPCC climate domain per pixel | +| Variable | Type | Units | Description | +| --------------------------------------------------------------- | ------- | ----------- | ------------------------------------------------------------------------------------ | +| `glad:glcluc:year=2000` … `glad:glcluc:year=2020` | float32 | GLCLUC code | GLAD land-cover/land-use class, one variable per year (2000, 2005, 2010, 2015, 2020) | +| `gfw:harris-agb:aboveground-biomass-mg-per-ha` | float32 | Mg/ha | Forest above-ground woody biomass (2000) | +| `huang:bgb:belowground-biomass-mg-per-ha` | float32 | Mg/ha | Forest below-ground (root) biomass | +| `soilgrids:organic-carbon-stocks:organic-soil-carbon-mg-per-ha` | float32 | Mg/ha | Soil organic carbon stock, 0–30 cm | +| `gfw:global-peatlands:is-peatland` | float32 | 0/1 | Binary peatland mask | +| `ipcc:climate-zones:climate-zone` | float32 | zone code | IPCC climate domain per pixel | ```python import xarray -harmonized = xarray.open_zarr("gs://cornerstone-luc/v3-jdluc-sluc-south-america/harmonize.zarr", consolidated=False) +# written by the harmonize stage +harmonized = xarray.open_zarr("/.zarr", consolidated=False) ``` ### 2. Per-pixel emissions (zarr) -`gs://cornerstone-luc/v3-jdluc-sluc-south-america/emit.zarr` +Written by the `emit` stage; a pipeline output, not a published artifact (see [Access](#access)). -The crop-agnostic, per-pixel, per-span LUC emissions produced by the `emit` stage. Spans -are the four consecutive GLAD intervals `2000-2005`, `2005-2010`, `2010-2015`, `2015-2020`; -per-year layers cover `2000, 2005, 2010, 2015, 2020`. +The crop-agnostic, per-pixel, per-span LUC emissions produced by the `emit` stage. Spans are the four consecutive GLAD intervals `2000-2005`, `2005-2010`, `2010-2015`, `2015-2020`; per-year layers cover `2000, 2005, 2010, 2015, 2020`. -| Variable | Type | Units | Description | -|---|---|---|---| -| `land-class:{year}` | float32 | LandClass code | Per-year land class (Forest, Grassland, Cropland, Built-up, Water, Snow/ice, Ocean), stored as a float code | -| `vegetation-emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span vegetation-carbon loss (above-ground, below-ground, dead organic matter, grassland) | -| `soil-emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span soil-carbon loss (mineral stock change + peatland drainage pulse) | -| `emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span total (vegetation + soil), before temporal discounting | -| `peatland-occupation:tco2e-per-ha` | float32 | t CO₂e/ha | Current-year annual peatland-occupation emissions | -| `emissions-per-hectare:tco2e-per-ha` | float32 | t CO₂e/ha | 20-year linearly-discounted LUC emissions + peatland occupation — the headline per-hectare layer | -| `hectares-per-pixel:ha` | float32 | ha | Pixel area (varies with latitude) | +| Variable | Type | Units | Description | +| ---------------------------------------------------- | ------- | -------------- | ----------------------------------------------------------------------------------------------------------- | +| `land-class:{year}` | float32 | LandClass code | Per-year land class (Forest, Grassland, Cropland, Built-up, Water, Snow/ice, Ocean), stored as a float code | +| `vegetation-emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span vegetation-carbon loss (above-ground, below-ground, dead organic matter, grassland) | +| `soil-emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span soil-carbon loss (mineral stock change + peatland drainage pulse) | +| `emissions:tco2e-per-ha:{before}-{after}` | float32 | t CO₂e/ha | Per-span total (vegetation + soil), before temporal discounting | +| `peatland-occupation:tco2e-per-ha` | float32 | t CO₂e/ha | Current-year annual peatland-occupation emissions | +| `emissions-per-hectare:tco2e-per-ha` | float32 | t CO₂e/ha | 20-year linearly-discounted LUC emissions + peatland occupation — the headline per-hectare layer | +| `hectares-per-pixel:ha` | float32 | ha | Pixel area (varies with latitude) | ```python import xarray -emissions = xarray.open_zarr("gs://cornerstone-luc/v3-jdluc-sluc-south-america/emit.zarr", consolidated=False) +# written by the emit stage +emissions = xarray.open_zarr("/.zarr", consolidated=False) ``` ### 3. Emissions factors (parquet) -`gs://cornerstone-luc/v3-jdluc-sluc-south-america/emissions-factors.parquet` - -The final per-(jurisdiction, crop) table produced by the `trace` stage. Indexed by -`(admin_level, crop_name, jurisdiction_name, methodology)` — all strings — where `admin_level` is `PROVINCIAL` (World Bank admin-1) or `NATIONAL` (admin-0, summed from provincial rows) and `methodology` is `JURISDICTIONAL_DIRECT` or `STATISTICAL`. - -| Column | Type | Units | Description | -|---|---|---|---| -| `admin_id` | string | — | World Bank admin id (e.g. `USA008`); the ISO-A3 code on national rows | -| `crop_hectares` | float64 | ha | Crop area in the jurisdiction (land occupation) | -| `peatland_crop_hectares` | float64 | ha | Crop area on peatland | -| `emissions_mt` | float64 | t CO₂e | Total allocated LUC emissions | -| `peatland_occupation_emissions_mt` | float64 | t CO₂e | Annual peatland-occupation emissions on crop pixels | -| `forest_emissions_mt` | float64 | t CO₂e | Forest-conversion emissions | -| `peatland_conversion_emissions_mt` | float64 | t CO₂e | Peatland-conversion (drainage-pulse) emissions | -| `production_kg` | float64 | kg | Crop production (NASS yield × area for the direct leg; MapSPAM for the statistical leg) | -| `yield_kg_per_ha` | float64 | kg/ha | `production_kg / crop_hectares` | -| `emissions_factor_kgco2e_per_kg` | float64 | kg CO₂e / kg | `1000 × emissions_mt / production_kg` — the headline factor | -| `peatland_occupation_fraction` | float64 | ratio (0–1) | `peatland_occupation_emissions_mt / emissions_mt` | +Deposited per data version as `emissions-factors.parquet` (see [Access](#access)). + +The final per-(jurisdiction, crop) table produced by the `trace` stage. Indexed by `(admin_level, admin_id, crop_name, methodology)` — all strings — where `admin_level` is `PROVINCIAL` (World Bank admin-1) or `NATIONAL` (admin-0, summed from provincial rows) and `methodology` is `JURISDICTIONAL_DIRECT` or `STATISTICAL`. + +| Column | Type | Units | Description | +| ---------------------------------- | ------- | ------------ | -------------------------------------------------------------------------------------------- | +| `jurisdiction_name` | string | — | Display name (e.g. `United States of America \| Iowa`) | +| `crop_hectares` | float64 | ha | Crop area in the jurisdiction (land occupation), on production's discounted 2000–2020 window | +| `peatland_crop_hectares` | float64 | ha | Crop area on peatland | +| `emissions_mt` | float64 | t CO₂e | Total allocated LUC emissions | +| `peatland_occupation_emissions_mt` | float64 | t CO₂e | Annual peatland-occupation emissions on crop pixels | +| `forest_emissions_mt` | float64 | t CO₂e | Forest-conversion emissions | +| `peatland_conversion_emissions_mt` | float64 | t CO₂e | Peatland-conversion (drainage-pulse) emissions | +| `production_kg` | float64 | kg | Crop production (NASS yield × area for the direct leg; MapSPAM for the statistical leg) | +| `yield_kg_per_ha` | float64 | kg/ha | `production_kg / crop_hectares` | +| `emissions_factor_kgco2e_per_kg` | float64 | kg CO₂e / kg | `1000 × emissions_mt / production_kg` — the headline factor | +| `peatland_occupation_fraction` | float64 | ratio (0–1) | `peatland_occupation_emissions_mt / emissions_mt` | Both attribution legs emit the same columns. ```python import pandas -emission_factors = pandas.read_parquet("gs://cornerstone-luc/v3-jdluc-sluc-south-america/emissions-factors.parquet") +emission_factors = pandas.read_parquet("emissions-factors.parquet") ``` -The `attribute` stage writes an intermediate rollup parquet with the same index and the -non-derived subset of these columns (`crop_hectares`, `peatland_crop_hectares`, -`emissions_mt`, `peatland_occupation_emissions_mt`, `forest_emissions_mt`, -`peatland_conversion_emissions_mt`, and — statistical only — `production_mt`); `trace` joins -production and appends the ratio columns (`production_kg`, `yield_kg_per_ha`, -`emissions_factor_kgco2e_per_kg`, `peatland_occupation_fraction`). +The `attribute` stage writes an intermediate rollup parquet with the same index and the non-derived subset of these columns (`crop_hectares`, `peatland_crop_hectares`, `emissions_mt`, `peatland_occupation_emissions_mt`, `forest_emissions_mt`, `peatland_conversion_emissions_mt`, and — statistical only — `production_mt`); `trace` joins production and appends the ratio columns (`production_kg`, `yield_kg_per_ha`, `emissions_factor_kgco2e_per_kg`, `peatland_occupation_fraction`). diff --git a/docs/executive_summary.md b/docs/executive_summary.md index 7321d5a..12d5659 100644 --- a/docs/executive_summary.md +++ b/docs/executive_summary.md @@ -1,34 +1,34 @@ # Executive summary -This repository implements an open-source, satellite-derived methodology for estimating the land-use-change (LUC) emissions of agricultural commodities, built for corporate GHG inventories under the GHG Protocol **Land Sector and Removals Standard (LSRS)**. It allocates LUC emissions to crops in proportion to their displacement of natural ecosystems, using a high-resolution **jurisdictional-direct (jdLUC)** calculation where detailed crop maps exist (currently the United States) and a coarser **statistical (sLUC)** calculation for the rest of the world. Both attribution approaches use the same underlying global land conversion and emissions map. The full crop and country lists are in [`coverage.md`](coverage.md). +This repository implements an open-source, satellite-derived methodology for estimating the land-use-change (LUC) emissions of agricultural commodities, built for corporate GHG inventories under the GHG Protocol **Land Sector and Removals Standard (LSRS)**. It allocates LUC emissions to crops in proportion to their displacement of natural ecosystems, using a high-resolution **jurisdictional-direct (jdLUC)** calculation where detailed crop maps exist (currently the United States) and a coarser **statistical (sLUC)** calculation for the rest of the world. Both attribution approaches use the same underlying global land conversion and emissions map. The full crop and country lists are in [`coverage.md`](coverage.md). This document is aimed at readers assessing how the methodology maps to the LSRS. The deeper *what* and *why* live in [`methodology.md`](methodology.md). The methodology is a proof of concept intended as a starting point for discussion and collaboration. ## LSRS conformance at a glance -| LSRS requirement | How this methodology addresses it | Status | -|---|---|---| -| **Land-use-change emissions in scope** | Detects high-carbon → low-carbon land-cover transitions across forests, grasslands and peatlands per ~30 m pixel across five GLAD epochs (2000–2020) and quantifies the carbon lost. | Met | -| **20-year assessment period, linearly discounted (§7.2.1)** | All fluxes are allocated over a 20-year linearly-discounted lookback; more recent conversions are weighted more heavily. GLAD's 5-year spans are aggregated into one discount weight per span. | Met | -| **Carbon pools (vegetation + soil)** | Includes key carbon pools described on the LSRS as follows: Above-ground biomass (Harris 2021), below-ground/root biomass (Huang 2021), dead organic matter (CDM AR-TOOL-12, forest only), mineral soil organic carbon (IPCC 2019 Tier 1 stock-change, 0–30 cm) and complements with peatland emissions (newly drained + long term drained). | Met | -| **Jurisdictional direct vs. statistical land use change** | Two legs share one emissions core: jdLUC via spatial intersection with high-resolution crop maps (USDA CDL), and sLUC via each crop's share of local cropland expansion (IFPRI MapSPAM). | Met | -| **Allocation to commodities** | Product expansion approach is used for sLUC calculation. | Met | -| **Land management emissions (peatland)** | Peatland drainage is modeled in two parts — a LUC drainage pulse (discounted as a transition) plus an ongoing land-management occupation emission. | Extension — LSRS gives no peatland-specific method | -| **Carbon removals** | Not tracked; reverse (low- → high-carbon) transitions are excluded. Consistent with the LSRS treatment of LUC emissions; removals would require separate accounting. | By design — not covered | -| **Jurisdictional reporting** | Per-pixel emissions roll up to World Bank administrative jurisdictions (admin-0/1/2). | Met | +| LSRS requirement | How this methodology addresses it | Status | +| ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| **Land-use-change emissions in scope** | Detects high-carbon → low-carbon land-cover transitions across forests, grasslands and peatlands per ~30 m pixel across five GLAD epochs (2000–2020) and quantifies the carbon lost. | Met | +| **20-year assessment period, linearly discounted (§7.2.1)** | All fluxes are allocated over a 20-year linearly-discounted lookback; more recent conversions are weighted more heavily. GLAD's 5-year spans are aggregated into one discount weight per span. | Met | +| **Carbon pools (vegetation + soil)** | Includes key carbon pools described on the LSRS as follows: Above-ground biomass (Harris 2021), below-ground/root biomass (Huang 2021), dead organic matter (CDM AR-TOOL-12, forest only), mineral soil organic carbon (IPCC 2019 Tier 1 stock-change, 0–30 cm) and complements with peatland emissions (newly drained + long term drained). | Met | +| **Jurisdictional direct vs. statistical land use change** | Two legs share one emissions core: jdLUC via spatial intersection with high-resolution crop maps (USDA CDL), and sLUC via each crop's share of local cropland expansion (IFPRI MapSPAM). | Met | +| **Allocation to commodities** | Product expansion approach is used for sLUC calculation. | Met | +| **Land management emissions (peatland)** | Peatland drainage is modeled in two parts — a LUC drainage pulse (discounted as a transition) plus an ongoing land-management occupation emission. | Extension — LSRS gives no peatland-specific method | +| **Carbon removals** | Not tracked; reverse (low- → high-carbon) transitions are excluded. Consistent with the LSRS treatment of LUC emissions; removals would require separate accounting. | By design — not covered | +| **Jurisdictional reporting** | Per-pixel emissions roll up to World Bank administrative jurisdictions (admin-0/1/2). | Met | ## Data output -The published deliverable is a single per-(jurisdiction, crop) table — the **emissions-factors parquet** — indexed by `(admin_level, crop_name, jurisdiction_name, methodology)`, where `admin_level` is `NATIONAL` or `PROVINCIAL` and `methodology` is `JURISDICTIONAL_DIRECT` or `STATISTICAL`. Each row carries the headline emissions factor and the quantities behind it; both attribution legs (jdLUC and sLUC) emit the same columns. The full schema is in [`data.md`](data.md#3-emissions-factors-parquet). +The published deliverable is a single per-(jurisdiction, crop) table — the **emissions-factors parquet** — indexed by `(admin_level, admin_id, crop_name, methodology)`, where `admin_level` is `NATIONAL` or `PROVINCIAL`, `admin_id` is the ISO 3166 alpha-3 code nationally and the World Bank admin-1 code provincially, and `methodology` is `JURISDICTIONAL_DIRECT` or `STATISTICAL`. Each row carries the headline emissions factor and the quantities behind it; both attribution legs (jdLUC and sLUC) emit the same columns. The full schema is in [`data.md`](data.md#3-emissions-factors-parquet). -| Column | Units | What it is | -|---|---|---| +| Column | Units | What it is | +| -------------------------------- | ------------ | ----------------------------------------------------------- | | `emissions_factor_kgco2e_per_kg` | kg CO₂e / kg | The headline factor — `1000 × emissions_mt / production_kg` | -| `emissions_mt` | t CO₂e | Total allocated LUC emissions | -| `production_kg` | kg | Crop production (NASS × area for jdLUC; MapSPAM for sLUC) | -| `crop_hectares` | ha | Crop area in the jurisdiction | -| `yield_kg_per_ha` | kg/ha | `production_kg / crop_hectares` | -| `peatland_occupation_fraction` | ratio (0–1) | Share of emissions from ongoing peatland occupation | +| `emissions_mt` | t CO₂e | Total allocated LUC emissions | +| `production_kg` | kg | Crop production (NASS × area for jdLUC; MapSPAM for sLUC) | +| `crop_hectares` | ha | Crop area in the jurisdiction, over the 2000–2020 window | +| `yield_kg_per_ha` | kg/ha | `production_kg / crop_hectares` | +| `peatland_occupation_fraction` | ratio (0–1) | Share of emissions from ongoing peatland occupation | ## Known deviations and open items diff --git a/docs/further_research.md b/docs/further_research.md index bb12a74..5bd8617 100644 --- a/docs/further_research.md +++ b/docs/further_research.md @@ -1,25 +1,138 @@ # Areas for further research -A supplement to [`methodology.md`](methodology.md): areas where further research could -improve the methodology. +A supplement to [`methodology.md`](methodology.md): areas where further research could improve the methodology. This methodology is a first draft. We see a number of areas where further research could lead to additional improvements. These are listed in our estimated rough sense of impact/priority, though that certainly could be debated. Contributions would be particularly welcome on these topics. +The tooling described in [`validation.md`](validation.md) measures the entries below against external datasets rather than maintaining a second list: a finding it reports carries one of the headings on this page, verbatim, so a measurement lands on an existing entry. Where it finds a gap with no entry here, the gap should be proposed back to this document. + +## Woody perennial crops fall outside GLAD's cropland class + +**Issue**: GLAD GLCLUC's cropland class covers annual *and perennial herbaceous* crops; perennial **woody** crops are excluded by construction — "Perennial woody crops, permanent pastures and shifting cultivation are excluded from the definition" (Potapov et al., 2022b) and appear instead as tree cover (values 25–48 and 125–148), which `emit.get_land_class` maps to `LandClass.FOREST`. Four consequences follow. The first three compound for peat under oil palm and acacia, the land uses carrying the highest IPCC drained-organic-soil emission factors; the fourth is the largest, and is not about peat at all: + +1. `get_soil_emissions` treats a transition as emissive only where the destination is *not* forest or short vegetation, so peat swamp forest → plantation never fires the 621 t CO₂e/ha drainage pulse. +2. `get_peatland_occupation_emissions` lists `FOREST` among the undrained classes, so an actively drained plantation on peat is charged no annual occupation emissions either. +3. Both attribution legs restrict the emissions numerator to pixels GLAD calls cropland in 2020, which filters out whatever did survive — while MapSPAM keeps the corresponding oil palm production in the emission-factor denominator. +4. Clearing forest for plantation is not registered as conversion at all: the destination class is forest, so no emissive transition fires and the frontier's emissions never enter any pool, peat or mineral. This is the dominant term, and it is measured below. + +Sampled over 1°×1° windows at 30 m: in Riau, Sumatra, GLAD calls 0.13% of the window cropland while 78% is tree cover, and 74% of the window reads as forest in both 2000 and 2020; in Central Kalimantan the figures are 2.2% cropland, 53% tree cover, and 47% forest-to-forest. Both are among the most intensively drained peat landscapes in the world. + +**Measured, and detection is the dominant term.** The share-free forest-conversion pool each country's cropland contains, against WRI's national total: Indonesia **0.020×**, Malaysia **0.001×**, Papua New Guinea 0.002×, Colombia 0.013×, Brazil 0.137× — against 0.42–0.67× for Argentina, Canada, Bolivia and South Africa. No crop allocation enters that comparison, so it isolates detection from attribution: the tropics are short by 50–760× before a single share is computed. This is the dominant term in the tropical factors: it sets them directly, and no allocation choice downstream can recover conversion the pipeline never detected. + +**No cropland mask tested recovers the destination class.** Against Descals' 2019 closed-canopy palm map (Descals et al., 2021) in a Riau window that is 45.1% palm, palm recall runs GLC_FCS30D 25.7% — its tree/shrub orchard class alone 21.4% — then Impact Observatory LULC (Karra et al., 2021) 12.5%, GACED30 (Liao et al., 2026) 1.0%, and GLAD GAC and GLCLUC both 0.0%; GLCLUC calls 91.6% of that palm forest. GACED30 is built to the FAO *Cropland* aggregate and so admits permanent crops by definition, and that definitional fix does work on open canopy — on Puglia's olive groves it exceeds GLCLUC by 15 percentage points — but it does not cross the dense tropical canopy boundary. The mask is therefore the second-order term and the missing transition the first: see "GLAD GLC vs Hansen TCL forest-detection globally", where TCL recalls 97.10% of the same loss. + +**The gap crops are enumerable and rankable.** A 5-arcmin screen of GLAD's 3 km cropland percent against all 46 MapSPAM 2020 physical-area layers ranks each crop by the share of its area the mask can account for: oil palm 6.8%, cocoa 14.9%, coconut 16.7%, rubber 18.6%, robusta coffee 24.4%, arabica coffee 26.6%, rising to sugar beet 90.9%. The ranking recovers FAO's permanent-woody list unprompted, and surfaces a second mode alongside it — cassava 32.8%, other roots 30.2%, yams 41.1%, sweet potato 43.9%, other vegetables 55.6%, together 40.6 Mha unbacked — which is the shifting-cultivation exclusion and the four-year fallow rule rather than missing woody crops, and is addressed by relaxing the fallow assumption rather than by a new mask. Joined to WRI's `LD_2020` as `LD × unbacked_share`, the crops in the gap carry 1,071 Mt CO₂e of WRI's 2,683 Mt global total, oil palm alone 352 Mt; coffee and tea have no open global extent map, and rubber and citrus have no WRI factor at all. The screen is an upper bound on recall rather than recall — it assumes the most favourable assignment of GLAD's hectares to mapped crop area, and measured palm recall in the Riau window was 0.00% against the screen's 6.8% — and it ceilings near 90% from 10 km mixing, so it is a ranking and not a level. + +**Potential impact:** Potentially the largest single gap for the global extension. The tropical acacia and oil palm emission factors (78.6 and 45.0 t CO₂e ha⁻¹ yr⁻¹) motivate the front-loaded shape of the entire peatland model, yet the pipeline can attribute close to zero peat emissions to those crops. Note that the reassurance offered under "GLAD GLC built-up classification is over-inclusive vs. NLCD" — that per-crop factors are protected because the numerator only counts pixels that are cropland in 2020 — does not hold for woody perennials, where that same restriction is the mechanism of the undercount. This is distinct from, though related to, the detection question raised under "GLAD GLC vs Hansen TCL forest-detection globally": that entry asks whether GLAD *sees* clearing it classifies as clearing, this one asks about clearing whose destination class it never treats as converted. + +**Potential improvement path:** Ingest a dedicated perennial-plantation layer — Descals et al. (2024) Global Oil Palm "Year of Plantation" is the obvious first candidate, and usefully supplies a plantation year as well as an extent, which would let the peat pulse be dated directly rather than inferred from GLAD spans. Treat mapped plantation on peat as drained for the occupation term, admit it to the emissions numerator alongside GLAD cropland, and reconcile the MapSPAM woody-perennial crop list against whichever extent layer is adopted. That list is woody crops only, which is narrower than "perennial": sugarcane, banana and plantain are perennial but herbaceous, so GLAD's definition admits them. The classifier does not honor that admission where the canopy is dense — the screen above puts sugarcane at 64.2% coverage, banana at 33.9% and plantain at 28.6% — which brings those 71 (country, crop) pairs into scope alongside the woody crops. A general woody-cropland upgrade is also available: GLC_FCS30D (Zhang et al., 2024) carries a separable tree/shrub orchard cropland class, annual for 2000–2022, which would admit woody cropland to the numerator under a different carbon treatment than annual cropland — something GACED30's binary cropland class cannot support. Establish its per-class accuracy for that class first, since the published global overall accuracy of 80.88% spans 35 classes and says nothing about it, along with whether the 25.7% palm recall holds outside Riau and whether the 24% of that window it calls shrubland is immature palm. Adopting it also forces a regrid: its 30 m lattice does not nest into GLAD's 0.00025°, on which TCL, SDPT v2 and Global Pasture Watch are all native. + ## GLAD GLC vs Hansen TCL forest-detection globally -Although forest conversion emissions are very small in the US, they will become the dominant source of emissions as we expand globally. Our preliminary analysis (offline, not yet published in this repo) shows significant differences in the US in forest detections between the GLAD GLC layer we are using and the widely used Global Forest Watch tree cover loss dataset designed specifically for this purpose. There are few enough conversion events in the US that these disagreements could be noise. But knowing we want to extend globally over time, and with our land cover layer as the single most important methodology choice, we should do more investigation of this issue early. +Although forest conversion emissions are very small in the US, they become the dominant source as the methodology extends globally. GLAD GLCLUC and Hansen Global Forest Change tree-cover loss (Hansen et al., 2013) disagree on forest detection, and with the land cover layer as the single most important methodology choice, the size and direction of that disagreement sets the tropical factors directly. + +In the US the disagreement may be noise: there are few enough conversion events that offline analysis cannot separate signal from noise either way. On tropical plantation frontiers it is neither small nor ambiguous. Over Descals' palm-driven primary-forest loss in a 1°×1° Riau window (n = 453,569 pixels at 10 m), TCL `lossyear` recalls **97.10%** of the loss — 99.75% within ±3 pixels of registration tolerance — and dates 82.76% of it to the same year and 89.30% to within one year. GLCLUC agrees that 98.71% of those pixels were forest in 2000, and still classifies **84.46% of them as forest in 2020**. In Ghana's Western Region, TCL detects loss on 17.38% of the window and GLCLUC calls **86.30%** of that forest in 2020. -**Potential impact:** Minimal for US row-crop EFs; potentially substantial in other geographies. +The disagreement runs in both directions, so TCL is not a superset: of GLCLUC's own forest-to-non-forest pixels in the Ghana window (28,005 ha), TCL corroborates **53.24%**. -**Potential improvement path:** Localize where GLAD misses oil-palm-driven forest loss and benchmark GLAD forest detection against Global Forest Watch tree-cover-loss on those frontiers before relying on the global expansion. +Palm frontiers belong to this entry and to "Woody perennial crops fall outside GLAD's cropland class" together, not to one or the other. Detection and destination fail on the same pixels — GLCLUC misses the clearing, and of the 15.54% it does register as leaving forest it assigns cropland to 0.03% — so neither failure alone accounts for the gap and fixing either alone recovers nothing. + +**Potential impact:** Minimal for US row-crop EFs; the dominant term in the tropics. On the Riau palm frontier a TCL-triggered transition addresses roughly 97% of the forest loss where the best available cropland-mask upgrade reaches about 26%, so it is worth roughly 4× the mask route on the same pixels. + +**Potential improvement path:** Prototype a TCL-triggered conversion with a plantation layer supplying the destination class, and measure it against the current pipeline on the Riau and Ghana windows. Hansen GFC-2025-v1.13 `lossyear` is already ingested (`jdluc/datasets/gfw_tcl.py`), is annual for 2001–2025, and is native to the GLAD grid, so the work is in the emissions logic rather than the ingest; SDPT v2 is the obvious destination layer, being on the same grid and carrying per-polygon species and carbon removal factors. Two limits bound where TCL can be trusted alone. It detects loss rather than land use, so it cannot supply a destination class by itself. And shaded cocoa can be established by thinning under retained canopy, which is legitimate continuity for both a canopy-height classifier and a loss detector, so TCL is not the safe fallback for cocoa that it is for palm — attribution on the Ghana window is further confounded by galamsey gold mining and logging, which register as loss and are not cocoa. ## sLUC grassland attribution (expansion-share over-attribution) -**Issue**: The statistical (sLUC) leg attributes each coarse cell's conversion emissions to crops by their share of crop *expansion* within the cell (`statistical.py`; see §3, Statistical). A post-refactor validation found this hands corn/soy/wheat substantially more grassland conversion than the per-pixel jurisdictional-direct (jdLUC) leg does on the *same* emissions layer (grassland EF: corn 0.032 vs 0.012, soy 0.126 vs 0.054 kg CO₂e/kg), driving the US corn+soy+wheat total to ~76 MtCO₂e — the highest of the satellite EF models and above jdLUC (61). Grassland (30.9 Mt) is the largest single US source and the biggest change from the refactor, yet it has **no reliable external anchor** since WRI is forest-only. The one weak reference — EPA's whole-cropland Grassland→Cropland (16.3 Mt, *all* crops) — sits *below* the sLUC corn+soy+wheat *subset* (30.9), i.e. a subset nearly doubling the whole, though the comparison is muddied by a temporal-basis mismatch (EPA single-year 2022 annual vs sLUC 20-yr committed/discounted). Whether expansion-share allocation over-attributes grassland — or the per-pixel CDL leg under-attributes it — is unresolved. +**Issue**: The statistical (sLUC) leg attributes each coarse cell's conversion emissions to crops by their share of crop *expansion* within the cell (`statistical.py`; see §3, Statistical). A post-refactor validation found this hands corn/soy/wheat substantially more grassland conversion than the per-pixel jurisdictional-direct (jdLUC) leg does on the *same* emissions layer (grassland EF: corn 0.032 vs 0.012, soy 0.126 vs 0.054 kg CO₂e/kg), driving the US corn+soy+wheat total to ~76 MtCO₂e — the highest of the satellite EF models and above jdLUC (61). Grassland (30.9 Mt) is the largest single US source and the biggest change from the refactor, yet it has **no reliable external anchor** since WRI is forest-only. The one weak reference — EPA's whole-cropland Grassland→Cropland, **10.6 Mt for 2020** — sits *below* the sLUC corn+soy+wheat *subset* (30.9): a three-crop subset nearly **tripling** the whole-cropland total. Both sides are annual, so this is not muddied by a temporal basis. The GHGP per-year weights integrate to exactly 1.0 across the 20-year window, so `SPAN_TO_LINEAR_DISCOUNT_WEIGHT` allocates each conversion's emissions to a *single sourcing year* rather than accumulating twenty of them, and the production denominator is a weighted mean over the same spans — an annual numerator over an annual denominator. The one real mismatch is scope, and it runs against us: widening sLUC from three crops to all cropland can only raise 30.9. Whether expansion-share allocation over-attributes grassland — or the per-pixel CDL leg under-attributes it — is unresolved. + +**Not the share-denominator asymmetry.** The expansion-share denominator is the obvious suspect and is not the cause: eliminating it leaves the US sLUC/jdLUC total-factor ratio at 1.15–1.75× and the grassland ratio at 0.97–2.34×. The figures quoted above reproduce from the current parquets (corn 0.0317 vs 0.0131, soy 0.1255 vs 0.0576), so the divergence is stable and unexplained. **Potential impact:** Grassland is the dominant driver of US row-crop LUC emissions and the largest sLUC pool, so this directly sets the headline US emission factors and totals; resolving it could move the US corn+soy+wheat total by tens of MtCO₂e. It is also the largest source of divergence between the two attribution legs (state-level sLUC/jdLUC spans 0.19–39×). -**Potential improvement path:** (1) Reconcile the time bases (annualize sLUC or accumulate the EPA annual figure) so the EPA grassland baseline becomes a like-for-like check rather than a subset-vs-whole comparison. (2) Anchor grassland — and peat, which is equally unanchored (WRI forest-only) — externally and non-circularly: biome-stratified grassland carbon from Spawn et al. (2019) / IPCC, and IPCC drained-organic-soil (peat) emission factors from the 2013 Wetlands Supplement. This is the only route to a validation that does not lean on sLUC's own carbon densities. (3) Stress-test the expansion-share rule itself — in particular its behavior on cells where the converted pixel's CDL destination is non-crop — against the jdLUC state-level head-to-head. +**Potential improvement path:** (1) Match the scope: compute the sLUC grassland figure over all cropland rather than the corn-soy-wheat subset, which is what stands between this and a like-for-like check. The time bases already agree and need no adjustment on either side. (2) Anchor grassland — and peat, which is equally unanchored (WRI forest-only) — externally and non-circularly: biome-stratified grassland carbon from Spawn et al. (2019) / IPCC, and IPCC drained-organic-soil (peat) emission factors from the 2013 Wetlands Supplement. This is the only route to a validation that does not lean on sLUC's own carbon densities. (3) Stress-test the expansion-share rule itself — in particular its behavior on cells where the converted pixel's CDL destination is non-crop — against the jdLUC state-level head-to-head. + +## The statistical leg under-allocates oil palm relative to WRI + +**Issue**: With WRI rebased to national production the emission-factor denominator cancels, so the sLUC/WRI factor ratio is exactly the product of two independent terms — whether the two methods see the same forest-conversion total for the jurisdiction, and whether they split that total the same way across crops. For Indonesian oil palm the ratio is 0.008, and it decomposes as `0.020 × 0.38`: + +| term | value | meaning | +| ---------------- | ----: | --------------------------------------------------------------- | +| total ratio | 0.020 | sLUC sees 2% of the forest conversion WRI reports for Indonesia | +| allocation ratio | 0.38 | sLUC gives oil palm 12% of that total where WRI gives 33% | + +The first term is the detection gap. It is much the larger, and it belongs entirely to "Woody perennial crops fall outside GLAD's cropland class", where it is derived and measured; this entry is about the second term alone. That term is separate and unexplained: the expansion-share rule hands oil palm a smaller slice of Indonesia's forest conversion than WRI's allocation does. Either per-crop expansion clipping understates a woody perennial whose siblings churn in the same cells, or WRI's allocation overstates oil palm, or the two disagree before either rule runs: oil palm enters the 2000→2005 span only through the `OOIL` decomposition, split within the group per pixel where WRI gap-fills each crop by regression against a jurisdiction total, so the 6.25% of the numerator that span carries is not built from the same oil palm area on both sides. WRI shares this leg's MapSPAM expansion-share family, so it is not an independent arbiter of that question — and where the two differ is the denominator rather than the rule: WRI divides a cell's crop expansion by the expansion of cropland *plus* cultivated pasture and subsistence area (Fitts et al., 2025a, Equation 1 and endnote 21), where this leg divides by crop expansion alone and instead masks the numerator to 2020 cropland. + +**Potential impact:** Compounding, in the same direction as the detection gap, on the crop with the largest global land-use footprint. A factor of ~2.6 on Indonesian oil palm, against ~50 from detection — worth resolving, but only after detection. + +**Potential improvement path:** Compare the two allocations directly on a jurisdiction where both see a similar forest total, so the allocation term is isolated from the detection term — Argentina (0.42×), Bolivia (0.60×) or South Africa (0.67×) rather than the tropics. If sLUC's crop shares track WRI's there, the tropical divergence is a consequence of detection rather than an independent allocation defect. + +## MapSPAM's 2000 release carries area its successor does not account for + +**Issue**: The 2000 snapshot is a different MapSPAM release (v3.0.7) from the ones that follow (v3.2), and its unmodeled crop vocabulary is disjoint from 2005's: a catch-all `OTHE` holds 560 Mha where the nine crops that replace it hold 161 Mha, correlating at r = 0.17 per pixel. The 2000 release simply accounts for ~400 Mha that 2005 does not, and no mapping between the two would be honest. + +The consequence is that the nine 2005 crops read as expansion from zero across the 2000→2005 span, whose clipped unmodeled term is +77 Mha. Restricting that span to crops both snapshots report — the treatment applied to the 2005-onward boundaries, where the vocabularies do correspond — destroys 96% of the term rather than stabilizing it, so it is deliberately not applied there. + +**Potential impact:** Small and bounded. The affected span carries 6.3% of the linear discount weight, the least of the three, and the error is one-directional: a denominator inflated by area the earlier release did not name depresses modeled crops' shares and cannot over-attribute. + +**Potential improvement path:** Nothing within MapSPAM. Reconciling the two releases needs an independent estimate of what `OTHE` contained, which is the same external-extent-layer dependency the woody-perennials entry describes. Alternatively accept the 2000 span as structurally coarser than the others, which is what this entry does. + +## Emissions attributed to no crop are discarded rather than renormalized + +**Issue**: Shares sum to at most one, and the shortfall — the fourteen MapSPAM crops this leg does not model — is charged to nobody. Those crops get no row in the emissions-factor table, so a fraction of every cell's emissions leaves no trace in the output. Both share paths carry it, and the second is worse: + +| window | conversion shares | peatland-occupation shares | +| --------- | ----------------: | -------------------------: | +| Global | ~13% | ~14% | +| Indonesia | ~19% | **~24%** | + +Indonesia's conversion figure is the firmer of the two: measured against the forest-conversion pool its cropland actually contains, the leg attributes 81% and discards the rest. The others are expansion-weighted estimates from the MapSPAM snapshots, which understate any effect concentrated in low-expansion cells — so treat them as floors. | Brazil | 5.2% | 5.2% | | US windows | 1.5–1.8% | 0.8–0.9% | + +Indonesia's occupation figure is the largest of these, and peat occupation is Indonesia's dominant emissions source. Neither of the two share-denominator fixes reaches it: `get_crop_to_area_share` divides each crop's 2020 area by the 2020 total over all 46 crops, so the unmodeled fourteen dilute occupation exactly as they dilute expansion, with no clipping or cross-year comparison involved. + +The choice to discard is defensible and deliberately conservative: a crop should not be charged for a neighbor's expansion, and each modeled crop's own factor is unaffected. Renormalizing so shares sum to one would charge modeled crops for conversion that unmodeled crops drove, inflating every factor. The point is that this is a real methodological fork currently settled by omission rather than by argument. + +Two consequences are worth stating plainly. **Summing the emissions-factor table does not give a jurisdiction's LUC total**, and in the tropics the gap is not a rounding error. And the biggest single component cannot be closed from MapSPAM: rubber is the largest unmodeled crop in Indonesia at **4.92 of its 10.93 Mha** of unmodeled area, and MapSPAM reports rubber **only in 2020**, so there is no earlier area from which to build its expansion. Of Indonesia's unmodeled area only about a third sits in crops that are both reported across the window and an actual commodity — `REST` is a catch-all, not something one publishes a factor for. + +**Potential impact:** Nothing changes for a single crop's emission factor, which is what most consumers use. It matters for any use that sums across crops, and it governs how much of the tropical peat and forest signal reaches the output at all — nearly a quarter of Indonesia's peatland-occupation emissions currently reach no row. + +**Potential improvement path:** Emit an explicit unattributed row per jurisdiction, so the discard is visible in the output rather than inferable only from a shortfall nobody sees. That is cheap and makes every future change to this logic auditable. Attributing rather than merely reporting it means modeling the missing crops, and MapSPAM does not readily support any of them. None of the fourteen has a 2000 band, so none is a `SHARED_CROP_NAMES` member or a group constituent, and `get_harmonized_quantity` has no 2000 quantity to decompose for them — adding one as a `Crop` fails the module assert outright. The only in-MapSPAM route is to declare `OTHE` a seventh group with the nine as its constituents, which would hand each a share of 560 Mha against their actual 161 and inflate their 2000 area roughly 3.5×. Cocoa is the best candidate on coverage (1.53 Mha in Indonesia, 7.99 Mha globally, reported in every snapshot from 2005) and rubber the most valuable, but both need an external extent layer to supply the years MapSPAM does not. + +WRI reaches the same fork and answers it without an external layer, which makes it a precedent worth weighing (Fitts et al., 2025a, pp. 36–37): for crops MapSPAM maps from 2005 onward it estimates the missing 2000 area by linear regression on jurisdiction-level area, then divides within the FAOSTAT commodity group proportionally, constrained to preserve that data year's total crop area. That is how it publishes 42 crops against our 32 — cocoa, tea, tobacco and the fruit and vegetable groups among them. Two limits keep it from being a recipe. Its guidebook does not say how the gap-filled area is spatialized, and a jurisdiction-grain estimate fabricates within-jurisdiction presence in the same way an even split fabricates within-pixel presence. And a regression needs two points, so it stops exactly where we stop: rubber is mapped only in 2020, WRI publishes **no rubber factor at all** (Fitts et al., 2025b), and the crops new in SPAM 2020 v2 (citrus, onion, rubber, tomato) get yield factors and no emission factor. Rubber therefore has no external anchor either, so closing it here would also mean giving up the check on it. + +## MapSPAM group decomposition falls back to an even split + +**Issue**: The 2000 MapSPAM snapshot uses a coarser crop taxonomy than later years, so six "group" crops must be decomposed into their constituents using within-group shares pooled from the 2005/2010/2020 snapshots (see "Reconciling MapSPAM's crop taxonomy" in `methodology.md`). Where no reference year places the group in a pixel there is no basis for a split at all, and only two of the six groups — OOIL and OPUL — have a catch-all constituent (`GROUP_TO_RESIDUAL_NAME`) to absorb the unattributable remainder. For the other four (BANP bananas/plantains, COFF arabica/robusta, MILL pearl/small millet, SWPY sweet potato/yam), `get_harmonized_quantity` divides the group total evenly across named siblings. An even split fabricates presence: it asserts a crop exists in a pixel on no evidence beyond its group's total, which is how a tropical crop comes to be credited to a temperate country. + +A holdout test compounds the concern. Holding out a snapshot year and scoring each rule by the fraction of the group's 2000 mass it misallocates (total-variation distance between predicted and actual within-group shares, restricted to pixels all three years place the group), pooling is *worse* than using the nearest year alone on the extrapolation case — the one that resembles estimating 2000: + +| holdout target | nearest-year | pooled | pooled vs nearest | +| ----------------------------------------------------------- | ------------------: | -----: | ----------------: | +| 2005 — outside its predictors' range, the 2000 analogue | 9.58% of valid mass | 10.66% | **+11.3%** | +| 2010 — bracketed by its predictors, an easier interpolation | 9.58% | 10.39% | +8.4% | + +(Production leg; the area leg gives +11.5% and +0.6%.) Pooling is nonetheless preferred because it wins outright on *coverage* — the restriction to pixels all three years agree on covers only about half the group mass, and pooling gives a basis for a split in many pixels where 2005 alone gives none. The trade is deliberate: better coverage at slightly worse accuracy per covered pixel. + +MILL is the worst group on both counts. It has no catch-all, and pooling misallocates 3.3× what the nearest year does (0.41 → 1.36 Mkg of production), because small millet's reported yield rises ~58% between the 2010 and 2020 snapshots — the largest single distortion pooling introduces. + +**Potential impact:** The fallback is common, but its reach is bounded. Measured directly from the MapSPAM snapshots, it divides **18.1 Mt of 2000 production and 4.84 Mha of area** — about 7.7% of the four affected groups — evenly among named siblings. It is worst for bananas and plantains (10.9 Mt globally, 11.6% of the group) rather than millet, and worst of all for coffee in India, where **43%** of the 2000 crop has no reference-year basis for an arabica/robusta split. It fires in countries already processed: 0.84 Mt of banana/plantain and 0.31 Mt of coffee in Brazil, 0.71 Mt of banana/plantain in Mexico. + +What bounds it is that the decomposition is the only consumer of the 2000 snapshot, and that snapshot reaches results with little weight — 6.25% of the emissions numerator, via the 2000→2005 expansion share, and 3.13% of the production denominator, one term halved in the span average. Since the split redistributes *within* a group and conserves its total, a sibling can be mis-credited by at most half the fabricated mass, which works out below 0.5% on an affected crop's factor in most places and about 1.3% for Indian coffee. The pooling penalty above is smaller again: a 1.08-percentage-point increase in misallocated group mass, carried at 6.25%, is under 0.1% on a finished factor. + +Note for anyone re-measuring this: the even-split *fingerprint* — byte-identical production across siblings in the output — badly understates it, finding four national rows and 11 kg. Final production is a discount-weighted average over four spans in which the 2000 term is one contribution at 3.13%, averaged with 2005, so siblings only come out byte-identical when the crop is absent from every later snapshot. The fingerprint detects that corner and nothing else; measure the fallback against the snapshots instead. + +**Potential improvement path:** Route the unattributable remainder to the sLUC residual for all six groups rather than only the two that happen to have a named catch-all — the residual already exists as a destination and needs no new constituent. Failing that, fall back to a coarser spatial prior (within-group shares aggregated to the admin unit or the country over the reference years) before resorting to an even split, so the rule degrades from pixel evidence to regional evidence rather than straight to uniform. A per-group choice between pooled and nearest-year shares, informed by the holdout above, would also recover most of the MILL penalty. + +## NASS yield coverage gaps and product-form conversions + +**Issue**: The jurisdictional-direct leg divides emissions by `crop_hectares x yield_kg_per_ha`, where the yield is a USDA NASS survey statistic and the hectares come from the CDL. Every one of the eleven crops `jurisdictional_direct.Crop` models carries a yield, so every one produces an emissions factor. Three properties of how that denominator is assembled are worth recording, because each is an assumption rather than a measurement. + +1. **Unmatched (admin, crop) pairs are still only a log line.** NASS suppresses small and identifying cells under Title 7 U.S.C. and CIPSEA, and does not survey every crop in every state. Where the CDL sees a crop in a state that NASS does not publish a yield for, that state contributes hectares and emissions to the national rollup but no production, so the national factor is a ratio over the reporting states alone. The count of such rows is logged and then discarded. +2. **Cotton is converted between product forms by a single constant.** NASS reports cotton yield as ginned lint, while MapSPAM, FAOSTAT and WRI all carry seed cotton — see the `ItemCode` docstring in `faostat_production`, which pins item 328 over ginned lint's 767 on the grounds that either mistake scales a comparison by the milling yield. `usda_nass_quickstats.Series.reported_fraction` divides the lint yield back up to the unginned crop at 0.36. That figure is what FAOSTAT's US seed-cotton yield implies against the NASS series over 2011–2020, and it agrees with the conventional ~35% gin turnout, but the same comparison implies ~0.403 over 2000–2010. The constant is therefore tied to the window `trace.NASS_YIELD_YEARS` currently averages, and a national constant hides whatever provincial variation in gin turnout exists. +3. **Two series are not quite the crop the CDL scores.** NASS renamed its rolled-up dry-bean class in 2019 and publishes no `ALL CLASSES` to fall back on, so spanning a multi-year window takes both names; the pre-2019 aggregate includes chickpeas, which the CDL scores separately as `CHICK_PEAS`. Sugarcane publishes no `ALL UTILIZATION PRACTICES` row either, so its total is `SUGAR & SEED` and includes seed cane. + +**Potential impact:** Modest overall, and concentrated in one crop. Cotton is the pair whose assumption moves a published number materially: carried as lint rather than seed cotton its factor would read roughly 2.8x higher. The eight crops outside maize, soybean and wheat carry 9% of US jurisdictional-direct emissions between them, so an error in their denominators is bounded by that share. The residual coverage gap is concentrated in minor producing states for the smaller crops, so it biases a national factor by whatever those states' yields differ from the surveyed ones — likely small for maize and soybean, larger for crops NASS surveys in only a handful of states. The chickpea contamination affects half the current averaging window for one crop carrying 0.627 Mt. + +**Potential improvement path:** Carry the unmatched count into the reported output rather than the log, so a factor computed over partial state coverage is visible to whoever reads it — the same treatment `validation.md` gives its other coverage measures. For cotton, derive the lint fraction per state and year from NASS's own cottonseed production rather than a single national constant, which would also make the pre-2011 drift moot. Re-measure the constant if `NASS_YIELD_YEARS` ever moves earlier. ## Higher resolution yield data @@ -31,8 +144,8 @@ Although forest conversion emissions are very small in the US, they will become **Issue**: Two subissues, but closely related: (a) extending conversion-loss estimates from 30cm to 1m depth, and (b) switching 0–30cm SOC from current-state SoilGrids to Sanderman 2017 pre-disturbance reconstruction. -1. **30cm-vs-1m depth.** The POC currently follows IPCC's Tier 1 calculation method: SoilGrids 0–30cm stock × Table 5.5 F_LU factors that were calibrated against paired-plot data to a 30cm depth (Annex 5A.1). Tier 1 explicitly "assumes management practice influences stocks to a depth of 30 cm" (§5.2.3.2), but acknowledges that sub-30cm losses are real and material, citing Angers et al. that "including soil C stock data below the depth of tillage is necessary to provide an accurate estimate." Sanderman 2017 and Spawn 2019 both find that the deeper-layer share of total cultivation loss is substantial — Sanderman reports global SOC losses of 37, 75, and 133 Pg C to 0.3 m, 1 m, and 2 m respectively (so the sub-30cm layer accounts for ~51% of cultivation loss in the 0–1 m column). To capture this loss we'd need to either (i) establish a defensible simple F_LU value at 1 m or (ii) adopt a Spawn-style carbon-response function over a depth-resolved soil map. Complex, but potentially worth it given that SOC represents the majority of carbon loss for US grassland conversion. Unfortunately, the simpler-looking shortcut of plugging a 1m SoilGrids stock into the current 30cm-calibrated F_LU is expressly prohibited by IPCC Vol 4 Ch 2 §2.3.3.1, which requires the SOC reference value and the stock-change factors to share a depth basis. -2. **Current-state vs pre-disturbance stock.** The POC reads SoilGrids `ocs_mean` at the converted pixel — a post-cultivation value, not the native stock IPCC Tier 1 imagines. Sanderman 2017 NoLU is a global 10 km raster of "what each pixel's SOC would be today if never cultivated", at the IPCC-canonical 0–30cm depth, and runs ~30% above POC-current across the Plains states. It could be a superior alternative, though requires further review. +1. **30cm-vs-1m depth.** The POC currently follows IPCC's Tier 1 calculation method: SoilGrids 0–30cm stock × Table 5.5 F_LU factors that were calibrated against paired-plot data to a 30cm depth (Annex 5A.1). Tier 1 explicitly "assumes management practice influences stocks to a depth of 30 cm" (§5.2.3.2), but acknowledges that sub-30cm losses are real and material, citing Angers & Eriksen-Hamel that "including soil C stock data below the depth of tillage is necessary to provide an accurate estimate." Sanderman 2017 and Spawn 2019 both find that the deeper-layer share of total cultivation loss is substantial — Sanderman reports global SOC losses of 37, 75, and 133 Pg C to 0.3 m, 1 m, and 2 m respectively (so the sub-30cm layer accounts for ~51% of cultivation loss in the 0–1 m column). To capture this loss we'd need to either (i) establish a defensible simple F_LU value at 1 m or (ii) adopt a Spawn-style carbon-response function over a depth-resolved soil map. Complex, but potentially worth it given that SOC represents the majority of carbon loss for US grassland conversion. Unfortunately, the simpler-looking shortcut of plugging a 1m SoilGrids stock into the current 30cm-calibrated F_LU is expressly prohibited by IPCC Vol 4 Ch 2 §2.3.3.1, which requires the SOC reference value and the stock-change factors to share a depth basis. +2. **Current-state vs pre-disturbance stock.** The POC reads SoilGrids `ocs_mean` at the converted pixel — a post-cultivation value, not the native stock IPCC Tier 1 imagines. Sanderman 2017 NoLU is a global 10 km raster of "what each pixel's SOC would be today if never cultivated", at the IPCC-canonical 0–30cm depth, and runs ~30% above POC-current across the Plains states. It could be a superior alternative, though requires further review. **Potential impact:** If we adopted both changes, the expected combined effect on grassland per-event ΔC could be 2–3×, bringing CONUS pipelines area-average from ~68 tCO₂/ha toward Spawn's 190. We expect this would increase overall EFs for corn and soy by ~1.4×, and wheat by ~2.4× (since wheat has less peat-LM to dilute out the grassland impact). @@ -40,17 +153,17 @@ Although forest conversion emissions are very small in the US, they will become **Issue**: We use the GFW Global Peatlands raster composite. For CONUS, GFW uses Xu PEATMAP above 40°N; below 40°N it falls back to Gumbricht et al. (2017), a tropical-tuned hydrological model. This cutoff may be in the wrong spot for the temperate US — it affects Delaware, the mid-Atlantic, the Southeast, southern California, Arizona, New Mexico, and most of Texas. -**Potential impact:** National-level impact is likely modest because the Corn Belt sits above 40°N where Xu is the active source anyway; the issue concentrates in Southeast and mid-Atlantic states. +**Potential impact:** National-level impact is likely modest because the Corn Belt sits above 40°N where Xu is the active source anyway; the issue concentrates in Southeast and mid-Atlantic states. **Potential improvement path:** The most obvious potential fix would be to ingest Xu PEATMAP directly and build our own hybrid that uses it farther south than GFW, but there may be other better options. ## Short vegetation carbon stock for shrubland -**Issue:** GLAD GLC's "short vegetation" category (values 1–24) encodes only vegetation cover fraction (~7% to 100% cover), not vegetation type (Potapov et al., 2022). It does not distinguish grassland from shrubland. In the face of this limitation, the current methodology uses simple climate-zone-stratified carbon stock from the Houghton/BLUE bookkeeping parameterization for these pixels. The Houghton/BLUE values seem reasonably well-calibrated for herbaceous grassland (the dominant short vegetation type in US cropland conversion areas), but undercount AGB for woody shrubland: sagebrush steppe has 3–4 tC/ha (Fusco et al., 2019), and mature California chaparral has 17–28 tC/ha (Bohlman et al., 2018). Because most US short vegetation → cropland conversion occurs on Great Plains grassland rather than shrubland, the impact on national-level emission factors is likely small, but the undercount could be material for state-level factors in shrubland-heavy states, and when the methodology is extended globally. +**Issue:** GLAD GLC's "short vegetation" category (values 1–24) encodes only vegetation cover fraction (~7% to 100% cover), not vegetation type (Potapov et al., 2022a). It does not distinguish grassland from shrubland. In the face of this limitation, the current methodology uses simple climate-zone-stratified carbon stock from the Houghton/BLUE bookkeeping parameterization for these pixels. The Houghton/BLUE values seem reasonably well-calibrated for herbaceous grassland (the dominant short vegetation type in US cropland conversion areas), but undercount AGB for woody shrubland: sagebrush steppe has 3–4 tC/ha (Fusco et al., 2019), and mature California chaparral has 17–28 tC/ha (Bohlman et al., 2018). Because most US short vegetation → cropland conversion occurs on Great Plains grassland rather than shrubland, the impact on national-level emission factors is likely small, but the undercount could be material for state-level factors in shrubland-heavy states, and when the methodology is extended globally. **Potential impact:** Grassland conversion is the dominant driver of emissions for row crops in the United States. Even if shrubland is only 10-20% of the "short vegetation" conversions, the underestimate on those pixels could be large enough to matter. -**Potential improvement path**: Two approaches, in increasing order of sophistication: (1) Overlay an auxiliary classification that distinguishes shrubland from grassland (e.g., ESA WorldCover at 10m or NLCD Shrub/Scrub class) and apply differentiated literature-based carbon densities (~6–12 tC/ha for sagebrush, ~17–28 tC/ha for chaparral, vs. the current 5–7 tC/ha for all short vegetation). (2) Replace the static lookup table entirely with satellite-derived, spatially explicit AGB estimates for non-forest vegetation, using a product like IB-AGC (Li et al., 2025) at 25km resolution, subtracting known forest and crop biomass contributions, and distributing the residual across 30m grassland/shrubland pixels as a continuous function of woody fractional cover from the Copernicus Global Land Service. +**Potential improvement path**: Two approaches, in increasing order of sophistication: (1) Overlay an auxiliary classification that distinguishes shrubland from grassland and apply differentiated literature-based carbon densities (~6–12 tC/ha for sagebrush, ~17–28 tC/ha for chaparral, vs. the current 5–7 tC/ha for all short vegetation). Global Pasture Watch (Parente et al., 2024) is the strongest candidate: it separates cultivated grassland from natural and semi-natural grassland and from open shrubland, is annual for 2000–2022, and is native to the GLAD 0.00025° grid, so it can type both ends of a transition with no regrid. The alternatives are narrower — ESA WorldCover at 10 m covers 2020 and 2021 only, so it cannot type a 2000–2020 transition, and the NLCD Shrub/Scrub class is US-only, which does not carry to the global extension. The same three-way split bears on "sLUC grassland attribution (expansion-share over-attribution)" as well: separating conversion of natural rangeland from conversion of land already in cultivated pasture is a candidate explanation for the unresolved gap between the sLUC grassland figure and EPA's, and a route to anchoring grassland without leaning on the leg's own carbon densities. (2) Replace the static lookup table entirely with satellite-derived, spatially explicit AGB estimates for non-forest vegetation, using a product like IB-AGC (Li et al., 2025) at 25km resolution, subtracting known forest and crop biomass contributions, and distributing the residual across 30m grassland/shrubland pixels as a continuous function of woody fractional cover from the Copernicus Global Land Service. ## Within-year double-cropping @@ -66,7 +179,7 @@ Although forest conversion emissions are very small in the US, they will become ## Dead organic matter mismatch to US-specific data -**Issue**: The CDM AR-TOOL-12 DOM factors for tropical forests appear to be an underestimate relative to US FIA field measurements, which show a national average of ~20 tC/ha total DOM (~10 tC/ha dead wood + ~10 tC/ha litter; Domke et al., 2016; Woodall et al., 2008) -- ~2.4x the 8.2 tC/ha typical value we calculated above. We've chosen to stick with the well-standardized and peer reviewed CDM AR-TOOL-12 approach for now, but it would be good to investigate and understand this difference. One early hypothesis is a definitional difference: FIA "forest floor" may include duff/humus (partially decomposed organic material above mineral soil), that is classified as soil rather than litter in the CDM/IPCC framework. Another factor could be that the 8.2 tC/ha "typical" value quoted above is below the area-weighted US average, which could be pulled up by outliers. +**Issue**: The CDM AR-TOOL-12 DOM factors for tropical forests appear to be an underestimate relative to US FIA field measurements, which show a national average of ~20 tC/ha total DOM (~10 tC/ha dead wood + ~10 tC/ha litter; Domke et al., 2016; Woodall et al., 2008) -- ~2.4x the 8.2 tC/ha typical value we calculated above. We've chosen to stick with the well-standardized and peer reviewed CDM AR-TOOL-12 approach for now, but it would be good to investigate and understand this difference. One early hypothesis is a definitional difference: FIA "forest floor" may include duff/humus (partially decomposed organic material above mineral soil), that is classified as soil rather than litter in the CDM/IPCC framework. Another factor could be that the 8.2 tC/ha "typical" value quoted above is below the area-weighted US average, which could be pulled up by outliers. **Potential impact:** The ~2.4x gap between CDM AR-TOOL-12 (~8.2 tC/ha) and FIA measurements (~20 tC/ha) translates to ~43 tCO2/ha missing per forest pixel. That's roughly 8-12% of typical per-pixel forest conversion emissions. If forest conversion emissions are only 10% of total row crop emissions, that's ~1% of total emissions. But the gap may be partly definitional (see above), so the real impact could be considerably smaller. Worth investigating but uncertain. @@ -84,7 +197,7 @@ Although forest conversion emissions are very small in the US, they will become ## GLAD GLC built-up classification is over-inclusive vs. NLCD -**Issue**: The GLAD GLCLUC v2 built-up class (value 250) is derived by a U-Net CNN trained on OpenStreetMap building and road data; its published validation (Potapov et al. 2022, Table 6) reports user's accuracy 63.7% ± 5 and producer's accuracy 39.1% ± 19.5 for stable built-up globally, indicating substantial omission of existing urban. CONUS-wide the over-inclusiveness is sharp in the opposite direction: GLAD 2020 built-up covers ~78.3 Mha vs. NLCD 2021 developed (classes 21–24) at ~31.5 Mha — a 2.5× mismatch. The GLAD signal appears to pick up rural infrastructure, small roads, and sub-pixel impervious that NLCD's developed classes exclude. This shows up directly in our pipeline as inflated `forest → built-up` and `short-veg → built-up` transition area (in Delaware, GLAD records 5.7× more forest→built-up than our prior Hansen-loss + CDL classification did, with ~60% of those "new built-up" pixels still reading as forest in 2023 per the legacy classifier). +**Issue**: The GLAD GLCLUC v2 built-up class (value 250) is derived by a U-Net CNN trained on OpenStreetMap building and road data; its published validation (Potapov et al. 2022a, Table 6) reports user's accuracy 63.7% ± 5 and producer's accuracy 39.1% ± 19.5 for stable built-up globally, indicating substantial omission of existing urban. CONUS-wide the over-inclusiveness is sharp in the opposite direction: GLAD 2020 built-up covers ~78.3 Mha vs. NLCD 2021 developed (classes 21–24) at ~31.5 Mha — a 2.5× mismatch. The GLAD signal appears to pick up rural infrastructure, small roads, and sub-pixel impervious that NLCD's developed classes exclude. This shows up directly in our pipeline as inflated `forest → built-up` and `short-veg → built-up` transition area (in Delaware, GLAD records 5.7× more forest→built-up than our prior Hansen-loss + CDL classification did, with ~60% of those "new built-up" pixels still reading as forest in 2023 per the legacy classifier). **Potential impact:** State and national total LUC emissions are likely overstated by this effect, probably modest (built-up is a small fraction of total transitions) but systematic. Per-crop emissions factors are unaffected because the crop-EF numerator only counts allocated emissions on pixels that are cropland in 2020 — forest→built-up pixels don't enter the numerator. The direct impact is on state-total allocated LUC reporting and on any consumer of the `forest_to_built_up` or `short_veg_to_built_up` rows in the summary table. Worth cross-checking against NLCD in future validation work. @@ -92,18 +205,37 @@ Although forest conversion emissions are very small in the US, they will become **Issue**: The two-phase peatland model excludes peat fire emissions entirely; episodic fires can dwarf annual oxidative losses in fire years. -**Potential impact:** Peat fires are quite rare in the US -- maybe a few hectares per decade. Therefore this issue likely has negligible impact on US emission factors. For global extension (especially Indonesia), this would jump to the top of the list. +**Potential impact:** Peat fires are quite rare in the US -- maybe a few hectares per decade. Therefore this issue likely has negligible impact on US emission factors. For global extension (especially Indonesia), this would jump to the top of the list. + +## Repeated peatland drainage pulses + +**Issue**: `emit.get_soil_emissions` evaluates each five-year GLAD span independently, so the one-time drainage pulse (`PEATLAND_EMISSIONS_PULSE_TCO2E_PER_HA`, 621 t CO₂e/ha) is charged again every time a pixel leaves the forest and short-vegetation classes. A pixel that converts, reverts — fallow, pasture rotation, or classifier noise between GLAD cropland (244) and short vegetation (0–24) — and then re-converts within 2000–2020 is charged two or three pulses, although drained peat releases its labile carbon only once. The model intends a single pulse dated to the first drainage event, followed by the flat annual occupation term. + +**Potential impact:** Small, and largely suppressed by the GLAD-2020-cropland restriction on the emissions numerator: a pixel must return to forest or short vegetation mid-window to re-fire, and such pixels mostly do not read as cropland in 2020. Among pixels that are GLAD cropland in 2020 and have at least one emissive transition, those with two or more are 0.04% in the Dutch/German peat polders, 0.20% in the Everglades Agricultural Area, 1.7% in Riau, and 6.3% in Central Kalimantan — inflating the 621 t/ha pulse term by +0.1%, +0.3%, +2.1% and +9.2% respectively. The annual occupation term is unaffected, so the effect on a finished emission factor is smaller again. Run without the 2020-cropland restriction the same windows inflate by +1.1%, +26.8%, +1.7% and +15.5%, so the suppression is real but incidental rather than designed. + +**Potential improvement path:** Carry a cumulative "already drained" mask across spans in `emit.workflow` so the peat pulse fires only on a pixel's earliest emissive span. The mineral-soil branch has the same structure and would benefit from the same treatment, although its per-event stock change is roughly an order of magnitude smaller than 621 t/ha. ## References - Angers, D.A. & Eriksen-Hamel, N.S. (2008). Full-inversion tillage and organic carbon distribution in soil profiles: a meta-analysis. Soil Science Society of America Journal 72(5), 1370–1374. https://doi.org/10.2136/sssaj2007.0342 - Bohlman, G.N., Underwood, E.C. & Safford, H.D. (2018). Estimating biomass in California's chaparral and coastal sage scrub shrublands. Madroño 65, 28–46. +- Descals, A., Wich, S., Meijaard, E., Gaveau, D.L.A., Peedell, S. & Szantoi, Z. (2021). High-resolution global map of smallholder and industrial closed-canopy oil palm plantations. Earth System Science Data 13(3), 1211–1231. https://doi.org/10.5194/essd-13-1211-2021 +- Descals, A. (2024). Global oil palm extent and planting year from 1990 to 2021 [Data set]. Zenodo. https://doi.org/10.5281/zenodo.13379129 - Domke, G.M. et al. (2016). Estimating litter carbon stocks on forest land in the United States. Science of the Total Environment 557–558, 469–478. https://doi.org/10.1016/j.scitotenv.2016.03.090 -- Fusco, E.J., Finn, J.T., Abatzoglou, J.T., Balch, J.K., Dadashi, S. & Bradley, B.A. (2019). Accounting for aboveground carbon storage in shrubland and woodland ecosystems in the Great Basin. Ecosphere 10(8), e02821. https://doi.org/10.1002/ecs2.2821 +- Fitts, L.A., James, O., Gibbs, D. et al. (2025a). Geospatial Methods for Corporate GHG Accounting of Deforestation and Land Occupation. Guidebook, version 1. World Resources Institute, Washington, DC. https://doi.org/10.46830/wrigb.22.00158 \[Table 3, Table 4 and Equation 1 are the reporting-year, discounting and allocation mechanics; pp. 36–37 are the crop disaggregation. See `validation.md`.\] +- Fitts, L.A., James, O., Gibbs, D. et al. (2025b). Statistical Land Use Change Emissions from Deforestation and Land Occupation for 42 Agricultural Crop Categories. Technical note. World Resources Institute and Quantis. Dataset: https://github.com/wri/GCSC \[The anchor `validation/pull.py` pins by commit.\] +- Fusco, E.J., Rau, B.M., Falkowski, M., Filippelli, S. & Bradley, B.A. (2019). Accounting for aboveground carbon storage in shrubland and woodland ecosystems in the Great Basin. Ecosphere 10(8), e02821. https://doi.org/10.1002/ecs2.2821 - Gumbricht, T., Roman-Cuesta, R.M., Verchot, L., Herold, M., Wittmann, F., Householder, E., Herold, N. & Murdiyarso, D. (2017). An expert system model for mapping tropical wetlands and peatlands reveals South America as the largest contributor. Global Change Biology 23(9), 3581–3599. https://doi.org/10.1111/gcb.13689 +- Hansen, M.C., Potapov, P.V., Moore, R. et al. (2013). High-Resolution Global Maps of 21st-Century Forest Cover Change. Science 342(6160), 850–853. https://doi.org/10.1126/science.1244693 +- Harris, N.L., Gibbs, D.A., Baccini, A. et al. (2021). Global maps of twenty-first century forest carbon fluxes. Nature Climate Change 11(3), 234–240. https://doi.org/10.1038/s41558-020-00976-6 - IPCC (2014). 2013 Supplement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories: Wetlands (Hiraishi, T. et al., eds.). IPCC, Switzerland. [Drained organic (peat) soil emission factors.] +- Karra, K., Kontgis, C., Statman-Weil, Z., Mazzariello, J.C., Mathis, M. & Brumby, S.P. (2021). Global land use/land cover with Sentinel-2 and deep learning. IGARSS 2021, 4704–4707. https://doi.org/10.1109/IGARSS47720.2021.9553499 - Li, X., Ciais, P., Frappart, F. et al. (2025). IB-AGC: Annual 25 km global live biomass carbon product from SMOS L-band passive microwave vegetation optical depth. Scientific Data 12, 1156. https://doi.org/10.1038/s41597-025-05470-2 -- Potapov, P., Hansen, M.C., Pickens, A. et al. (2022). The Global 2000–2020 Land Cover and Land Use Change Dataset Derived From the Landsat Archive: First Results. Frontiers in Remote Sensing 3, 856903. https://doi.org/10.3389/frsen.2022.856903 +- Liao, Y., Chen, S., Bai, Y., Wang, J. & Gong, P. (2026). Global 30-m annual cropland extent dynamics (2000–2024) [preprint]. Earth System Science Data Discussions. https://doi.org/10.5194/essd-2025-838 +- Parente, L., Sloat, L., Mesquita, V. et al. (2024). Annual 30-m maps of global grassland class and extent (2000–2022) based on spatiotemporal Machine Learning. Scientific Data 11, 1303. https://doi.org/10.1038/s41597-024-04139-6 +- Potapov, P., Hansen, M.C., Pickens, A. et al. (2022a). The Global 2000–2020 Land Cover and Land Use Change Dataset Derived From the Landsat Archive: First Results. Frontiers in Remote Sensing 3, 856903. https://doi.org/10.3389/frsen.2022.856903 +- Potapov, P., Turubanova, S., Hansen, M.C. et al. (2022b). Global maps of cropland extent and change show accelerated cropland expansion in the twenty-first century. Nature Food 3, 19–28. https://doi.org/10.1038/s43016-021-00429-z - Sanderman, J., Hengl, T. & Fiske, G.J. (2017). Soil carbon debt of 12,000 years of human land use. Proceedings of the National Academy of Sciences 114(36), 9575–9580. https://doi.org/10.1073/pnas.1706103114 - Spawn, S.A., Lark, T.J. & Gibbs, H.K. (2019). Carbon emissions from cropland expansion in the United States. Environmental Research Letters 14, 045009. https://doi.org/10.1088/1748-9326/ab0399 - Woodall, C.W., Heath, L.S. & Smith, J.E. (2008). National inventories of down and dead woody material forest carbon stocks in the United States: Challenges and opportunities. Forest Ecology and Management 256(3), 221–228. https://doi.org/10.1016/j.foreco.2008.04.003 +- Zhang, X., Zhao, T., Xu, H. et al. (2024). GLC_FCS30D: the first global 30 m land-cover dynamics monitoring product with a fine classification system for the period from 1985 to 2022. Earth System Science Data 16(3), 1353–1381. https://doi.org/10.5194/essd-16-1353-2024 diff --git a/docs/methodology.md b/docs/methodology.md index 6ac082e..3ee66ef 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -1,36 +1,23 @@ # Cornerstone LUC: methodology and approach -This document is a high-level overview of the datasets and emissions methodology -implemented in this repository — *what* is computed and *why*, from a scientific standpoint. -It is intended to be read on its own, and links out to the deeper references where they -exist: - -- `architecture.md` — the system architecture and the rationale behind the tooling, - storage, and pipeline design (*how* it is built and *why those choices*). -- `data.md` — the published data products: storage locations, grids, and schemas for the - harmonized inputs, per-pixel emissions, and emissions-factor table. +This document is a high-level overview of the datasets and emissions methodology implemented in this repository — *what* is computed and *why*, from a scientific standpoint. It is intended to be read on its own, and links out to the deeper references where they exist: + +- `architecture.md` — the system architecture and the rationale behind the tooling, storage, and pipeline design (*how* it is built and *why those choices*). +- `data.md` — how to get the data products, grids, and full schemas for the harmonized inputs, per-pixel emissions, and emissions-factor table. - `peatland_methodology_supplement.md` — the two-part peatland emissions model. - `cdl_glad_comparison_supplement.md` — validation of the GLAD-cropland restriction against CDL row crops. - `further_research.md` — known limitations and areas for further research. -The pipeline quantifies the land-use-change (LUC) emissions associated with agricultural -commodities and allocates them to specific crops. It supports two attribution -methodologies that share a single per-pixel emissions core: +The pipeline quantifies the land-use-change (LUC) emissions associated with agricultural commodities and allocates them to specific crops. It supports two attribution methodologies that share a single per-pixel emissions core: -- **Jurisdictional direct** — high-resolution crop maps link emissions to crops by - spatial intersection. Currently **United States only** (USDA Cropland Data Layer). -- **Statistical** — where only coarse crop statistics exist, emissions are attributed in - proportion to each crop's share of local cropland expansion. This leg is **global** - (IFPRI MapSPAM). +- **Jurisdictional direct** — high-resolution crop maps link emissions to crops by spatial intersection. Currently **United States only** (USDA Cropland Data Layer). +- **Statistical** — where only coarse crop statistics exist, emissions are attributed in proportion to each crop's share of local cropland expansion. This leg is **global** (IFPRI MapSPAM). -Both legs follow the GHGP Land Sector and Removals Standard's 20-year linearly-discounted -lookback, and both roll up to World Bank administrative jurisdictions. +Both legs follow the GHGP Land Sector and Removals Standard's 20-year linearly-discounted lookback, and both roll up to World Bank administrative jurisdictions. ### At a glance -At a high level, the pipeline detects land-cover transitions and quantifies their -emissions per pixel, then attributes those emissions to crops and jurisdictions to -produce emissions factors — the three stages detailed in §§1–3 below. +At a high level, the pipeline detects land-cover transitions and quantifies their emissions per pixel, then attributes those emissions to crops and jurisdictions to produce emissions factors — the three stages detailed in §§1–3 below. ```mermaid graph TD @@ -81,251 +68,153 @@ A worked example of that flow on real data — soy in Matopiba, Brazil: *Matopiba, Brazil, showing the 2000 and 2020 endpoints of the five-epoch GLAD series (2000/2005/2010/2015/2020) for conciseness. **Top:** GLAD land cover before/after — cropland (amber) expands into forest and Cerrado. **Bottom:** MapSPAM soybean area before/after — soy floods the same belts. **Right:** the resulting 20-year-discounted per-hectare LUC emissions (crop-agnostic). The statistical leg attributes these emissions to soy in proportion to its share of local cropland expansion.* ---- +______________________________________________________________________ ## 1. Datasets and grids ### Source datasets -Every input is ingested from its upstream publisher into cloud storage as tiled -Cloud-Optimized GeoTIFFs (COGs, raster), FlatGeobuf (vector), or parquet (tabular), tagged -with provenance metadata. One module per dataset lives under `jdluc/datasets/`. The ingestion -and harmonization machinery that produces these analysis-ready copies is described in `architecture.md`. - -| Dataset | Role | Source | Kind | -|---|---|---|---| -| **GLAD GLCLUC v2** | Land cover / land-use time series (2000–2020, 5-year spans) — the backbone | GLAD/Hansen GeoTIFFs | raster | -| **GFW Harris AGB (2000)** | Forest above-ground woody biomass | GFW data-api (WHRC AGB v1.4) | raster | -| **Huang BGB** | Forest below-ground (root) biomass | Figshare (doi:10.6084/m9.figshare.12199637) | raster | -| **SoilGrids OCS (0–30 cm)** | Soil organic carbon stock | ISRIC SoilGrids (WCS) | raster | -| **GFW Global Peatlands** | Binary peatland mask | GFW data-api (`gfw_peatlands` v20230315) | raster | -| **IPCC climate zones** | Climate domain per pixel (drives IPCC factors) | Zenodo (doi:10.5281/zenodo.7303808) | raster | -| **USDA NASS CDL** | US per-pixel crop identity (jurisdictional-direct leg) | USDA Cropland Data Layer | raster | -| **IFPRI MapSPAM** | Global per-crop physical area + production, 2000/2005/2010/2020 (statistical leg) | Harvard Dataverse | raster | -| **USDA NASS QuickStats** | State-level crop yields (jurisdictional-direct production) | NASS QuickStats API | tabular | -| **World Bank Official Boundaries** | Admin-0/1/2 jurisdiction polygons | World Bank | vector | - -The first six rasters feed the per-pixel emissions core. CDL and MapSPAM feed the two -attribution legs respectively. NASS yields and World Bank boundaries are joined downstream -when building emissions factors. - -MapSPAM is the newest addition and the one that makes the statistical leg possible: it -downscales sub-national crop statistics to a ~10 km (5 arc-minute) grid via a -cross-entropy allocation, publishing physical area and production per crop for 2000, 2005, -2010, and 2020 (note: **no 2015 snapshot**). Its crop taxonomy is coarser in earlier years -(the 2000 snapshot reports only 20, partly grouped, crops), which the statistical leg -reconciles to a common per-crop taxonomy before use (see §3). +Every input is ingested from its upstream publisher into cloud storage as tiled Cloud-Optimized GeoTIFFs (COGs, raster), FlatGeobuf (vector), or parquet (tabular), tagged with provenance metadata. One module per dataset lives under `jdluc/datasets/`. The ingestion and harmonization machinery that produces these analysis-ready copies is described in `architecture.md`. + +| Dataset | Role | Source | Kind | +| ---------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------- | ------- | +| **GLAD GLCLUC v2** | Land cover / land-use time series (2000–2020, 5-year spans) — the backbone | GLAD/Hansen GeoTIFFs | raster | +| **GFW Harris AGB (2000)** | Forest above-ground woody biomass | GFW data-api (WHRC AGB v1.4) | raster | +| **Huang BGB** | Forest below-ground (root) biomass | Figshare (doi:10.6084/m9.figshare.12199637) | raster | +| **SoilGrids OCS (0–30 cm)** | Soil organic carbon stock | ISRIC SoilGrids (WCS) | raster | +| **GFW Global Peatlands** | Binary peatland mask | GFW data-api (`gfw_peatlands` v20230315) | raster | +| **IPCC climate zones** | Climate domain per pixel (drives IPCC factors) | Zenodo (doi:10.5281/zenodo.7303808) | raster | +| **USDA NASS CDL** | US per-pixel crop identity (jurisdictional-direct leg) | USDA Cropland Data Layer | raster | +| **IFPRI MapSPAM** | Global per-crop physical area + production, 2000/2005/2010/2020 (statistical leg) | Harvard Dataverse | raster | +| **USDA NASS QuickStats** | State-level crop yields (jurisdictional-direct production) | NASS QuickStats API | tabular | +| **World Bank Official Boundaries** | Admin-0/1/2 jurisdiction polygons | World Bank | vector | +| **FAOSTAT Production** | National crop production and harvested area (validation yardstick) | FAO bulk download | tabular | +| **GFW Hansen tree-cover loss** | Annual gross tree-cover loss, 2001-2025 (forest-detection comparison) | GFW/Hansen GFC-2025-v1.13 | raster | + +The first six rasters feed the per-pixel emissions core. CDL and MapSPAM feed the two attribution legs respectively. NASS yields and World Bank boundaries are joined downstream when building emissions factors. + +The last two are ingested but feed neither leg, and are listed so the inventory matches `datasets.DatasetName`. FAOSTAT is the national production yardstick the validation tooling measures against — the global analogue of NASS QuickStats, and deliberately outside the attribution path so that no leg can consume the number it is measured by (see `validation.md`). Hansen tree-cover loss is held for the GLAD forest-detection comparison described in `further_research.md`. + +MapSPAM is the newest addition and the one that makes the statistical leg possible: it downscales sub-national crop statistics to a ~10 km (5 arc-minute) grid via a cross-entropy allocation, publishing physical area and production per crop for 2000, 2005, 2010, and 2020 (note: **no 2015 snapshot**). Its crop taxonomy is coarser in earlier years (the 2000 snapshot reports only 21, partly grouped, crops), which the statistical leg reconciles to a common per-crop taxonomy before use (see §3). ### Two grids -All rasters are reprojected to `EPSG:4326` and warped onto one of two common grids, -depending on which stage consumes them: +Every source is ingested in `EPSG:4326` and resampled onto one of two common grids, depending on which stage consumes them: -- **GLAD grid** — the GLAD GLCLUC native resolution of ~30 m. Everything in the emissions - core and the jurisdictional-direct leg lives here. -- **MapSPAM grid** — the coarser ~10 km MapSPAM resolution. The statistical leg downsamples - per-pixel emissions to this grid to match the resolution of the MapSPAM crop statistics, - rather than implying a 30 m precision the underlying crop data does not have. +- **GLAD grid** — the GLAD GLCLUC native resolution of ~30 m. Everything in the emissions core and the jurisdictional-direct leg lives here. +- **MapSPAM grid** — the coarser ~10 km MapSPAM resolution. The statistical leg downsamples per-pixel emissions to this grid to match the resolution of the MapSPAM crop statistics, rather than implying a 30 m precision the underlying crop data does not have. -The choice of the GLAD grid as the common backbone, and the mechanics of mosaicking and -warping every source onto it, are covered in `architecture.md`. +The choice of the GLAD grid as the common backbone, and the mechanics of resampling every source onto it one 10° tile at a time, are covered in `architecture.md`. ---- +______________________________________________________________________ ## 2. Quantifying emissions -The emissions core (`emit.py`) computes **per-pixel, per-span LUC emissions** on the GLAD -30 m grid, independent of any crop. This single layer feeds both attribution legs. +The emissions core (`emit.py`) computes **per-pixel, per-span LUC emissions** on the GLAD 30 m grid, independent of any crop. This single layer feeds both attribution legs. ### Cataloguing transitions -For every ~30 m pixel a land-cover time series is built across the five GLAD epochs -(2000 → 2005 → 2010 → 2015 → 2020) from the GLAD GLCLUC v2 maps (Potapov et al., 2022). -GLAD's 8-bit codes are collapsed into a 7-member `LandClass` enum (Forest, Grassland, -Cropland, Built-up, Water, Snow/ice, Ocean; see `datasets/glad_glcluc.py`). Consecutive -epochs are compared and each transition's source and destination class is recorded. +For every ~30 m pixel a land-cover time series is built across the five GLAD epochs (2000 → 2005 → 2010 → 2015 → 2020) from the GLAD GLCLUC v2 maps (Potapov et al., 2022). GLAD's 8-bit codes are collapsed into a 7-member `LandClass` enum (Forest, Grassland, Cropland, Built-up, Water, Snow/ice, Ocean; see `datasets/glad_glcluc.py`). Consecutive epochs are compared and each transition's source and destination class is recorded. -Only transitions from a **higher-carbon** state (e.g. forest, grassland) to a -**lower-carbon** state (e.g. cropland, built-up) generate emissions. Reverse transitions -represent removals, which this methodology does not currently count. Two static -per-pixel attributes are also catalogued: a binary **peatland** flag and the **IPCC -climate domain**. +Only transitions from a **higher-carbon** state (e.g. forest, grassland) to a **lower-carbon** state (e.g. cropland, built-up) generate emissions. Reverse transitions represent removals, which this methodology does not currently count. Two static per-pixel attributes are also catalogued: a binary **peatland** flag and the **IPCC climate domain**. ### Carbon stocks and fluxes -For each emissive transition, emissions are the sum of carbon lost from vegetation and -soil. Each carbon pool is either read from a harmonized source layer or looked up from -published factors; refer to `emit.py` for the specific values. +For each emissive transition, emissions are the sum of carbon lost from vegetation and soil. Each carbon pool is either read from a harmonized source layer or looked up from published factors; refer to `emit.py` for the specific values. 1. **Vegetation carbon** - - Above-ground biomass — Harris et al. (2021) for forests; a climate-domain lookup - derived from the BLUE bookkeeping model (Hansis et al., 2015) for grassland/shrubland. - - Root (below-ground) biomass — Huang et al. (2021), with a root-to-shoot-ratio - fallback where Huang data is missing. - - Dead organic matter (dead wood + litter) — forests only, estimated as a fraction of - above-ground biomass following UNFCCC CDM AR-TOOL-12. IPCC Tier 1 treats non-forest - dead organic matter as zero, so it is excluded for grassland/shrubland. - - Forest conversions differentiate all three vegetation pools; grassland/shrubland use a - single combined vegetation-carbon value. Forest → grassland loses only the difference - in vegetation carbon between the two states, whereas forest → cropland or built-up - loses the full forest vegetation carbon. + + - Above-ground biomass — Harris et al. (2021) for forests; a climate-domain lookup derived from the BLUE bookkeeping model (Hansis et al., 2015) for grassland/shrubland. + - Root (below-ground) biomass — Huang et al. (2021), with a root-to-shoot-ratio fallback where Huang data is missing. + - Dead organic matter (dead wood + litter) — forests only, estimated as a fraction of above-ground biomass following UNFCCC CDM AR-TOOL-12. IPCC Tier 1 treats non-forest dead organic matter as zero, so it is excluded for grassland/shrubland. + + Forest conversions differentiate all three vegetation pools; grassland/shrubland use a single combined vegetation-carbon value. Forest → grassland loses only the difference in vegetation carbon between the two states, whereas forest → cropland or built-up loses the full forest vegetation carbon. + 2. **Soil organic carbon** - - Mineral soils — the IPCC 2019 Tier 1 stock-change method (Vol 4, Ch 5) applied to the - SoilGrids 0–30 cm stock, using land-use-change factors keyed on the destination land - class and climate domain (built-up reuses the cropland factor as a proxy). - - Peatland — a two-part model calibrated to the IPCC 2013 Wetlands Supplement: an - initial drainage **pulse** (discounted over time as a transition) plus a flat annual - **occupation** emission that continues for as long as the peatland stays under - cultivation. The derivation is in `peatland_methodology_supplement.md`. + + - Mineral soils — the IPCC 2019 Tier 1 stock-change method (Vol 4, Ch 5) applied to the SoilGrids 0–30 cm stock, using land-use-change factors keyed on the destination land class and climate domain (built-up reuses the cropland factor as a proxy). + - Peatland — a two-part model calibrated to the IPCC 2013 Wetlands Supplement: an initial drainage **pulse** (discounted over time as a transition) plus a flat annual **occupation** emission that continues for as long as the peatland stays under cultivation. The derivation is in `peatland_methodology_supplement.md`. ### Allocation to crop years (GHGP linear discounting) -All fluxes are first computed as if the transition were instantaneous, then allocated over -time using the 20-year linearly-discounted lookback prescribed by the GHGP Land Sector and -Removals Standard (§7.2.1): the more recently a conversion occurred, the more heavily its -emissions are weighted. Because GLAD resolves transitions only to 5-year spans, the -GHGP's per-year weights are aggregated into one weight per span as the unbiased mean of -that span's candidate conversion years (`SPAN_TO_LINEAR_DISCOUNT_WEIGHT` in `emit.py`). +All fluxes are first computed as if the transition were instantaneous, then allocated over time using the 20-year linearly-discounted lookback prescribed by the GHGP Land Sector and Removals Standard (§7.2.1): the more recently a conversion occurred, the more heavily its emissions are weighted. Because GLAD resolves transitions only to 5-year spans, the GHGP's per-year weights are aggregated into one weight per span as the unbiased mean of that span's candidate conversion years (`SPAN_TO_LINEAR_DISCOUNT_WEIGHT` in `emit.py`). -The per-pixel result is a discounted sum of span transition emissions plus the current -year's peatland occupation emissions, scaled by pixel area. This same temporal ramp is -shared by both attribution methodologies. +The per-pixel result is a discounted sum of span transition emissions plus the current year's peatland occupation emissions, scaled by pixel area. This same temporal ramp is shared by both attribution methodologies. ---- +______________________________________________________________________ ## 3. Attribution and emissions factors -Attribution turns the crop-agnostic per-pixel emissions layer into per-(jurisdiction, -crop) totals, and then into emissions factors. `attribute.py` dispatches each country to -one of two methodologies (clustering countries by continent so they share tile sets; see -[`coverage.md`](coverage.md) for the full country list), and `trace.py` converts the rollups -into the final emissions-factor table. Both legs clip to provincial (World Bank admin-1) -polygons and restrict to pixels that GLAD classifies as cropland in 2020. +Attribution turns the crop-agnostic per-pixel emissions layer into per-(jurisdiction, crop) totals, and then into emissions factors. `attribute.py` dispatches each country to one of two methodologies, fanning out over the ten-degree tiles that country touches and summing the per-tile partials (see [`coverage.md`](coverage.md) for the full country list and the territorial extent rule), and `trace.py` converts the rollups into the final emissions-factor table. Both legs clip to provincial (World Bank admin-1) polygons and restrict to pixels that GLAD classifies as cropland in 2020. ### Shared framing -Both legs quantify and allocate LUC emissions to a crop from a specific region in the same -way. The crop's sourcing year is specified and a 20-year lookback window is defined, then -that window is split into spans for which land use is known at both the start and end. -Within each span, conversions from high- to low-carbon-density states are identified and -the associated carbon emissions are quantified (§2), attributing them to all production -within the lookback window. To more strongly penalize production that follows land -conversion more quickly, emissions are discounted using a linear temporal ramp. Because -this step operates independently on every pixel, it applies globally and is shared between -the direct and statistical methodologies. +Both legs quantify and allocate LUC emissions to a crop from a specific region in the same way. The crop's sourcing year is specified and a 20-year lookback window is defined, then that window is split into spans for which land use is known at both the start and end. Within each span, conversions from high- to low-carbon-density states are identified and the associated carbon emissions are quantified (§2), attributing them to all production within the lookback window. To more strongly penalize production that follows land conversion more quickly, emissions are discounted using a linear temporal ramp. Because this step operates independently on every pixel, it applies globally and is shared between the direct and statistical methodologies. ### Jurisdictional direct (`jurisdictional_direct.py`) -Direct attribution is possible when the spatial resolution is high enough to unambiguously -link emissions to specific crop production through spatial intersection. Where such -traceability is available, the emissions factor is reduced by summing attributed emissions -within the traced region and dividing by total production in that region — an aggregation -that works for individual fields as well as district-, provincial-, and national-level -jurisdictions. +Direct attribution is possible when the spatial resolution is high enough to unambiguously link emissions to specific crop production through spatial intersection. Where such traceability is available, the emissions factor is reduced by summing attributed emissions within the traced region and dividing by total production in that region — an aggregation that works for individual fields as well as district-, provincial-, and national-level jurisdictions. Concretely, for each (jurisdiction, crop): 1. Clip the per-pixel emissions to the admin-1 polygon. -2. Mask to the crop's CDL codes (via the `Crop` enum), optionally intersected with GLAD - 2020 cropland. -3. Sum emissions (including the forest- and peatland-conversion components), crop hectares, - peatland crop hectares, and peatland-occupation emissions over the masked pixels. +2. Mask to the crop's CDL codes (via the `Crop` enum), optionally intersected with GLAD 2020 cropland. +3. Sum emissions (including the forest- and peatland-conversion components), crop hectares, peatland crop hectares, and peatland-occupation emissions over the masked pixels. -Because direct attribution relies only on **current** production, production is later -computed as crop area × NASS QuickStats yield (a multi-year state-level mean), converting -NASS's reported bushels/acre to kg/ha with the USDA 7 CFR 810 bushel weights. This leg -currently asserts `iso_3166 == "USA"`, since CDL is US-only. +Because direct attribution relies only on **current** production, production is later computed as crop area × NASS QuickStats yield (a multi-year state-level mean), converting NASS's reported yield per acre to kg/ha. NASS publishes that yield in four units — bushels, pounds, hundredweight and short tons per acre — and which one applies is a property of the commodity, so `usda_nass_quickstats.CropSeries` carries the unit and its weight per crop. The bushel weights are marketing bushels, from table 6 of USDA Agricultural Handbook 697 (ERS, June 1992), rather than the grading test weights of 7 CFR 810, which are a different quantity. Cotton is the one crop whose published yield is not the harvested crop: NASS reports ginned lint where MapSPAM and FAOSTAT carry seed cotton, so the lint yield is divided by a lint fraction of 0.36. This leg currently asserts `iso_3166 == "USA"`, since CDL is US-only. ### Statistical (`statistical.py`) -When crop production is only available at low spatial resolution, spatial intersection is -ambiguous and a statistical model is needed instead. The assumption is that emissions are -driven by crop expansion: within a coarse spatial cell, emissions are attributed to each -crop in proportion to its share of total expansion (retractions are ignored). Whereas -direct attribution relies only on current production, the statistical leg is estimated from -multiple spans — the emissions factor is reduced over a traceability region by summing -attributed emissions across spans and dividing by the sum of linearly discounted -production across those same spans, again aggregating to district-, provincial-, or -national-level jurisdictions. +When crop production is only available at low spatial resolution, spatial intersection is ambiguous and a statistical model is needed instead. The assumption is that emissions are driven by crop expansion: within a coarse spatial cell, emissions are attributed to each crop in proportion to its share of total expansion (retractions are ignored). Whereas direct attribution relies only on current production, the statistical leg is estimated from multiple spans — the emissions factor is reduced over a traceability region by summing attributed emissions across spans and dividing by the sum of linearly discounted production across those same spans, again aggregating to district-, provincial-, or national-level jurisdictions. Concretely: -1. Downsample the per-pixel GLAD emissions (per-span conversion emissions, forest vs. - peatland-conversion split, and peatland occupation) to the MapSPAM ~10 km grid. -2. For each MapSPAM span, compute each crop's **expansion** (positive change in physical - area) and its **share** of total crop expansion in that cell. Because MapSPAM has no - 2015 snapshot, the 2010→2015 and 2015→2020 GLAD spans both use the 2010→2020 MapSPAM - expansion. -3. Attribute each span's emissions to crops by that span's expansion share, weighting by - the same GHGP span discount weights. -4. Compute each crop's production denominator as a discount-weighted average over the - lookback window (described below), rather than a single current-year snapshot. - -Note that this GLAD-2020-cropland restriction is applied to the emissions numerator only; -the MapSPAM production denominator is a crop quantity and so is already cropland-restricted -by construction. - -This leg is global; MapSPAM crops are modeled with a broader `Crop` enum (maize, soybean, -wheat, rice, oil palm, coffee, and more; see `statistical.py`). - -**Reconciling MapSPAM's crop taxonomy.** Because MapSPAM's crop list is coarser in 2000 -than in later years, any crop that appears only in the finer later-year taxonomy must be -recovered from its 2000 group. The 2000 group total is decomposed into its constituent -crops, assuming each constituent's within-group share matches its share of the group's -physical area in the 2005 reference year (`DECOMPOSITION_REFERENCE_YEAR`); if the group is -absent in 2005, the total is split evenly across constituents. The same decomposition is -applied both to the physical-area layers that drive the expansion shares and to the -production layers that drive the denominator, so all spans speak a single crop taxonomy. - -**Windowed production denominator (a departure from WRI).** The emissions numerator is -time-resolved: it links conversion emissions to each crop's *historical* expansion, -weighted by recency. Dividing that by a single *current-year* production snapshot — as in -WRI's methodology — would place the numerator and denominator on inconsistent time bases. -A crop that expanded early in the window and then contracted would carry real conversion -emissions against a shrunken (or zero) present-day production, yielding an inflated or -undefined emissions factor. Instead, production is reduced over the same 20-year lookback -window using the same linear temporal ramp applied to emissions: each span's production is -the mean of its two MapSPAM snapshot years, weighted by that span's discount weight and -normalized by the sum of the weights. This ties emission allocation and production to the -same years with the same recency weighting. When production is flat across the window the -result reduces exactly to the current-production value; it diverges only when production -actually changed. This consistent linkage is a material improvement over attributing -historical-expansion emissions to present-day production alone. +1. Downsample the per-pixel GLAD emissions (per-span conversion emissions, forest vs. peatland-conversion split, and peatland occupation) to the MapSPAM ~10 km grid. +2. For each MapSPAM span, compute each crop's **expansion** (positive change in physical area) and its **share** of total crop expansion in that cell. Because MapSPAM has no 2015 snapshot, the 2010→2015 and 2015→2020 GLAD spans both use the 2010→2020 MapSPAM expansion. The total is built by clipping each modeled crop's own movement, exactly as its numerator is, so the shares sum to **at most** one. +3. Attribute each span's emissions to crops by that span's expansion share, weighting by the same GHGP span discount weights. +4. Compute each crop's production denominator as a discount-weighted average over the lookback window (described below), rather than a single current-year snapshot. + +Note that this GLAD-2020-cropland restriction is applied to the emissions numerator only; the MapSPAM production denominator is a crop quantity and so is already cropland-restricted by construction. + +**The unattributed remainder is discarded, not reassigned.** Because the shares sum to at most one, part of each cell's conversion emissions corresponds to expansion by MapSPAM crops this leg does not model — citrus, cocoa, rubber, vegetables and ten others. Those crops get no row, so that fraction is simply dropped: it is charged to nobody. Two consequences follow. Each modeled crop's factor is unaffected, which is the point — a crop is not charged for a neighbor's expansion. But the emissions-factor table accounts for less than the whole of a landscape's conversion emissions, and **summing its rows does not give a jurisdiction's LUC total**. Measured over the spans carrying most of the discount weight, the discarded fraction is about a fifth of the forest-conversion pool in Indonesia, measured against the pool that country's cropland actually contains. Peatland-occupation shares are formed the same way and discard a comparable fraction. See "MapSPAM crops that first appear in a later snapshot read as expansion from zero" in `further_research.md`, which is the largest single contributor to it. + +This leg is global; MapSPAM crops are modeled with a broader `Crop` enum (maize, soybean, wheat, rice, oil palm, coffee, and more; see `statistical.py`). + +**Reconciling MapSPAM's crop taxonomy.** Because MapSPAM's crop list is coarser in 2000 than in later years, any crop that appears only in the finer later-year taxonomy must be recovered from its 2000 group. The 2000 group total is decomposed into its constituent crops, assuming each constituent's within-group share matches its share of the group pooled over the later snapshots (`DECOMPOSITION_REFERENCE_YEARS`, 2005/2010/2020). Pooling rather than deferring to the nearest year is deliberate: a pixel the 2000 snapshot places a group in but 2005 does not is an inconsistency between MapSPAM's own releases, not a crop that arrived later. + +Each quantity is split by its own distribution — production by the pooled production share, physical area by the pooled area share — and both are used, so all spans speak a single crop taxonomy. Splitting production by area would assume every constituent of a group yields the same in that pixel, which they do not; bananas outyield plantains several-fold. + +Where no reference year places the group in a pixel there is no basis for a split at all. Two of the six 2000 groups have a catch-all constituent (`GROUP_TO_RESIDUAL_NAME`: "other oil crops" and "other pulses"), which absorbs the whole unattributable remainder rather than handing a slice to every named sibling. The other four have none, so the total is split evenly across constituents as a last resort. That last resort is not rare: measured across the MapSPAM snapshots it divides 18.1 Mt of 2000 production and 4.84 Mha of area — about 7.7% of the four affected groups — among crops that may not grow in the pixel at all. Its effect on a finished factor is nonetheless bounded well under a percent, because the 2000 snapshot is the only thing the decomposition touches and it carries 6.25% of the emissions numerator and 3.13% of the production denominator. See "MapSPAM group decomposition falls back to an even split" in `further_research.md`. + +**Tropical woody-perennial factors are lower bounds.** GLAD's cropland class excludes perennial woody crops, and the emissions numerator is restricted to pixels GLAD calls cropland in 2020, so the conversion emissions available to oil palm, coffee, coconut and rubber are a small fraction of what external references report — 2% of WRI's figure for Indonesia, 0.1% for Malaysia. Until that is addressed these crops' factors understate by roughly that factor and should be read as lower bounds, not estimates. `further_research.md` covers the mechanism, and separately a smaller allocation gap that compounds it for oil palm specifically. + +**Windowed production denominator (a departure from WRI).** The emissions numerator is time-resolved: it links conversion emissions to each crop's *historical* expansion, weighted by recency. Dividing that by a single *current-year* production snapshot — as in WRI's methodology — would place the numerator and denominator on inconsistent time bases. A crop that expanded early in the window and then contracted would carry real conversion emissions against a shrunken (or zero) present-day production, yielding an inflated or undefined emissions factor. Instead, production is reduced over the same 20-year lookback window using the same linear temporal ramp applied to emissions: each span's production is the mean of its two MapSPAM snapshot years, weighted by that span's discount weight and normalized by the sum of the weights. This ties emission allocation and production to the same years with the same recency weighting. When production is flat across the window the result reduces exactly to the current-production value; it diverges only when production actually changed. This consistent linkage is a material improvement over attributing historical-expansion emissions to present-day production alone. ### From rollups to emissions factors (`trace.py`) -`trace.py` takes the attribution rollup and derives the emissions factor identically for -both methodologies — the only methodology-specific step is where production comes from -(NASS yield × area for direct; MapSPAM production for statistical): +`trace.py` takes the attribution rollup and derives the emissions factor identically for both methodologies — the only methodology-specific step is where production comes from (NASS yield × area for direct; MapSPAM production for statistical): ``` emissions_factor_kgco2e_per_kg = Σ emissions / Σ production peatland_occupation_fraction = peatland_occupation_emissions / total_emissions ``` -Provincial (admin-1) rows are then summed up to national (admin-0) totals. The final -output is a `pandas.DataFrame` indexed by (admin level, crop, jurisdiction) with crop -hectares, peatland crop hectares, emissions (with the forest- and peatland-conversion -components broken out), production, the emissions factor, and the peatland-occupation -fraction; see `data.md` for the full column schema. The full stage-by-stage pipeline that -produces this table, and how its outputs are cached, is described in `architecture.md`. +Provincial (admin-1) rows are then summed up to national (admin-0) totals. The final output is a `pandas.DataFrame` indexed by `(admin_level, admin_id, crop_name, methodology)` with crop hectares, peatland crop hectares, emissions (with the forest- and peatland-conversion components broken out), production, the emissions factor, and the peatland-occupation fraction; see `data.md` for the full column schema. The full stage-by-stage pipeline that produces this table, and how its outputs are cached, is described in `architecture.md`. ---- +______________________________________________________________________ ## Supporting documents -Primary datasets and standards this methodology depends on. Exact values, factors, and -codes referenced above live in the source code. +Primary datasets and standards this methodology depends on. Exact values, factors, and codes referenced above live in the source code. **Land cover & carbon-stock datasets** -- GLAD GLCLUC v2 land cover — Potapov et al. (2022). -- Forest above-ground biomass — Harris et al. (2021). -- Forest root biomass — Huang et al. (2021). +- GLAD GLCLUC v2 land cover — Potapov et al. (2022). https://doi.org/10.3389/frsen.2022.856903 +- Forest above-ground biomass — Harris et al. (2021). https://doi.org/10.1038/s41558-020-00976-6 +- Forest root biomass — Huang et al. (2021). https://doi.org/10.5194/essd-13-4263-2021 - Soil organic carbon stock — ISRIC SoilGrids. - Peatland extent — Global Forest Watch Global Peatlands. -- Climate domain — Lewis (2022) raster, built from the IPCC 2019 Refinement decision tree. +- Climate domain — Lewis (2022) raster, built from the IPCC 2019 Refinement decision tree. https://doi.org/10.5281/zenodo.7303808 **Crop & jurisdiction datasets** @@ -340,5 +229,5 @@ codes referenced above live in the source code. - Mineral-soil stock change — 2019 Refinement to the 2006 IPCC Guidelines, Vol 4, Ch 5. - Peatland emissions — 2013 IPCC Wetlands Supplement. - Dead organic matter — UNFCCC CDM AR-TOOL-12. -- Grassland/shrubland vegetation carbon — BLUE bookkeeping model (Hansis et al., 2015). -- Yield unit conversion — USDA 7 CFR 810 bushel weights. +- Grassland/shrubland vegetation carbon — BLUE bookkeeping model (Hansis et al., 2015). https://doi.org/10.1002/2014GB004997 +- Yield unit conversion — USDA Agricultural Handbook 697, table 6 (marketing bushel weights). diff --git a/docs/peatland_methodology_supplement.md b/docs/peatland_methodology_supplement.md index d36814f..32a0629 100644 --- a/docs/peatland_methodology_supplement.md +++ b/docs/peatland_methodology_supplement.md @@ -29,9 +29,9 @@ This carbon store is maintained by high water levels. When the water table is at ### Complex carbon structure leads to extended decay -Peat organic matter is not homogeneous. It consists of a complex mix of organic components with widely varying susceptibilities to decomposition. Clymo (1984, 1998) first demonstrated that although approximately 90% of original plant biomass is lost in the upper peat layer within 100 years after drainage, selective decay continues in deeper layers even thereafter as more resistant compounds persist. +Peat organic matter is not homogeneous. It consists of a complex mix of organic components with widely varying susceptibilities to decomposition. Clymo (1984, 1992) first demonstrated that although approximately 90% of original plant biomass is lost in the upper peat layer within 100 years after drainage, selective decay continues in deeper layers even thereafter as more resistant compounds persist. -Leifeld et al. (2018) showed that the chemical composition of peat shifts progressively as drainage proceeds: the relative abundance of labile compounds (polysaccharides, simple carbohydrates) decreases while recalcitrant compounds (lignin, polyphenols) become proportionally more abundant. This selective depletion of labile carbon is the primary physical mechanism that produces declining emission rates over time — as the most easily decomposed components are consumed, the remaining peat becomes increasingly resistant to further oxidation. +Bader et al. (2018) showed that the chemical composition of peat shifts progressively as drainage proceeds: the relative abundance of labile compounds (polysaccharides, simple carbohydrates) decreases while recalcitrant compounds (lignin, polyphenols) become proportionally more abundant. This selective depletion of labile carbon is the primary physical mechanism that produces declining emission rates over time — as the most easily decomposed components are consumed, the remaining peat becomes increasingly resistant to further oxidation. More recently, McCalmont et al. (2021) carefully measured net ecosystem CO₂ exchange at two oil palm plantations of different ages on tropical peat in Malaysian Borneo, reconstructing a 12-year time series. They found that mean annual net emission for the newer plantation (137.8 Mg CO₂ ha⁻¹ yr⁻¹) was an order of magnitude higher than at the mature plantation (17.5 Mg CO₂ ha⁻¹ yr⁻¹), confirming that emissions were front-loaded, but that some emissions persist after long periods. @@ -47,7 +47,7 @@ Two principal methods are used to quantify carbon losses from drained peatlands, **Subsidence monitoring** measures the physical lowering of the peat surface over time and converts it to carbon loss using measured bulk density and carbon content. It integrates oxidative decomposition and dissolved organic carbon export, but can be confounded by physical compaction which lowers the peat surface without exporting CO₂, especially in the first years after drainage. -The result of the different boundaries from the two basic measurement approaches are results that vary significantly based on local circumstances: on established plantations where compaction has stabilised and root respiration is significant, subsidence-based estimates tend to exceed flux measurements because they capture DOC that flux instruments miss; on bare or recently drained peat where compaction is ongoing and roots are absent, this ordering can reverse (Marliana et al., 2023). It is possible to try to reconcile measurements, but it inevitably requires some inference or modeling. +The result of the different boundaries from the two basic measurement approaches are results that vary significantly based on local circumstances: on established plantations where compaction has stabilized and root respiration is significant, subsidence-based estimates tend to exceed flux measurements because they capture DOC that flux instruments miss; on bare or recently drained peat where compaction is ongoing and roots are absent, this ordering can reverse (Mos et al., 2023). It is possible to try to reconcile measurements, but it inevitably requires some inference or modeling. ### IPCC Tier 1 emission factors @@ -55,15 +55,15 @@ The IPCC 2013 Wetlands Supplement made an attempt to consolidate all the best ev The table below shows these emissions factors. -| Climate zone | Land use | CO₂ oxidation | Dissolved carbon | CH₄ | N₂O | **Total** | -|---|---|---|---|---|---|---| -| **Tropical** | Acacia | 73.3 | 3.0 | 1.3 | 1.0 | **78.6** | -| | Cropland | 51.3 | 3.0 | 1.4 | 2.1 | **57.9** | -| | Oil palm | 40.3 | 3.0 | 1.2 | 0.5 | **45.0** | -| **Temperate** | Cropland | 29.0 | 1.1 | 1.6 | 5.6 | **37.3** | -| | Pasture | 22.4 | 1.1 | 2.0 | 3.5 | **29.0** | -| **Boreal** | Cropland | 29.0 | 0.4 | 1.6 | 5.6 | **36.6** | -| | Pasture | 20.9 | 0.4 | 1.6 | 4.1 | **27.0** | +| Climate zone | Land use | CO₂ oxidation | Dissolved carbon | CH₄ | N₂O | **Total** | +| ------------- | -------- | ------------- | ---------------- | --- | --- | --------- | +| **Tropical** | Acacia | 73.3 | 3.0 | 1.3 | 1.0 | **78.6** | +| | Cropland | 51.3 | 3.0 | 1.4 | 2.1 | **57.9** | +| | Oil palm | 40.3 | 3.0 | 1.2 | 0.5 | **45.0** | +| **Temperate** | Cropland | 29.0 | 1.1 | 1.6 | 5.6 | **37.3** | +| | Pasture | 22.4 | 1.1 | 2.0 | 3.5 | **29.0** | +| **Boreal** | Cropland | 29.0 | 0.4 | 1.6 | 5.6 | **36.6** | +| | Pasture | 20.9 | 0.4 | 1.6 | 4.1 | **27.0** | _All values in t CO₂-eq ha⁻¹ yr⁻¹, converted from t C using × 3.667, and using AR6 GWP values_ @@ -109,7 +109,7 @@ This equation approximates the three pool model from Qiu et al. (2021) (active, Notably, the Swails et al. modeled tropical emissions curve is *lower* than the Qiu et al. temperate emissions curve at all years. This is surprising: there is a theoretical basis to suspect that emissions rates from peat should be higher in tropical regions than temperate and boreal regions — decay is highly temperature dependent, with most models suggesting organic compounds should degrade to CO₂ at least 2x faster for every 10 degrees of average temperature increase. The reversal of the expected relationship in the Qiu and Swails studies is partially explained by the fact that the Qiu curve represents gross peat decomposition, while the Swails curve represents net emissions (after vegetation carbon offsets). But this provides only a partial explanation. -The IPCC reference values for CO₂ emissions show a similar pattern. Although the absolute IPCC factors are higher for tropical regions, the difference is no larger than the gap that would be expected from age differences alone, if the general shape of the Swails and Qiu curves is correct. In other words, the IPCC data points also fail to show higher tropical emissions, once corrected for age of the measured sites. +The IPCC reference values for CO₂ emissions show a similar pattern. Although the absolute IPCC factors are higher for tropical regions, the difference is no larger than the gap that would be expected from age differences alone, if the general shape of the Swails and Qiu curves is correct. In other words, the IPCC data points also fail to show higher tropical emissions, once corrected for age of the measured sites. Given these observations, we do not attempt to build regional or crop specific curves. Rather we simply use a blended version of the Swails model, the Qiu model, and the IPCC values to construct a single reference CO₂ emissions curve for all climate zones and land use types. We generate this curve by: @@ -173,20 +173,22 @@ See the [accompanying python notebook](../analyses/peatland_emissions_modeling.i This catalogues a partial review of IPCC citations. We believe these to be representative (the IPCC Wetlands Supplement acknowledges the age difference between sites used for the published EFs), but completing this review is an area for refinement. -| IPCC Category | Papers reviewed | Sites with age data | Est. drainage age range | -|---|---|---|---| -| Tropical Acacia | Hooijer 2012, Jauhiainen 2012a, Basuki 2012, Hergoualc'h & Verchot 2011 | 125+ sites (Hooijer) + 12 plots (Basuki) | **5–10 yr** | -| Tropical Oil Palm | Hooijer 2012, Hergoualc'h & Verchot 2011, Melling 2005a/2007 | 42 sites (Hooijer) + 1 site (Melling) | **5–18 yr** | -| Tropical Cropland | Hergoualc'h & Verchot 2011 (indirect) | few | **8–15 yr** | -| Boreal/Temperate Cropland | Qiu et al. 2021 Table S2 (summarizing all 39 sites) | 24 of 39 with known ages | **26–300 yr** | -| Temperate Grassland | Schrier-Uijl et al. 2014 | 2 Dutch sites | **~200–800+ yr** (Dutch polders) | -| Boreal Grassland | - | - | **unknown** | +| IPCC Category | Papers reviewed | Sites with age data | Est. drainage age range | +| ------------------------- | ----------------------------------------------------------------------- | ---------------------------------------- | -------------------------------- | +| Tropical Acacia | Hooijer 2012, Jauhiainen 2012a, Basuki 2012, Hergoualc'h & Verchot 2011 | 125+ sites (Hooijer) + 12 plots (Basuki) | **5–10 yr** | +| Tropical Oil Palm | Hooijer 2012, Hergoualc'h & Verchot 2011, Melling 2005a/2007 | 42 sites (Hooijer) + 1 site (Melling) | **5–18 yr** | +| Tropical Cropland | Hergoualc'h & Verchot 2011 (indirect) | few | **8–15 yr** | +| Boreal/Temperate Cropland | Qiu et al. 2021 Table S2 (summarizing all 39 sites) | 24 of 39 with known ages | **26–300 yr** | +| Temperate Grassland | Schrier-Uijl et al. 2014 | 2 Dutch sites | **~200–800+ yr** (Dutch polders) | +| Boreal Grassland | - | - | **unknown** | ## References +- Bader, C., Müller, M., Schulin, R. & Leifeld, J. (2018). Peat decomposability in managed organic soils in relation to land use, organic matter composition and temperature. *Biogeosciences*, 15, 703–719. + - Clymo, R.S. (1984). The limits to peat bog growth. *Philosophical Transactions of the Royal Society of London B*, 303, 605–654. -- Clymo, R.S. (1998). Models of peat growth. *Suo*, 49, 75–80. +- Clymo, R.S. (1992). Models of peat growth. *Suo*, 43, 127–136. - Deshmukh, C.S., Susanto, A.P., Nardi, N., et al. (2023). Net greenhouse gas balance of fibre wood plantation on peat in Indonesia. *Nature*, 616, 740–746. @@ -198,19 +200,21 @@ This catalogues a partial review of IPCC citations. We believe these to be repre - Keck H, Meurer KHE, Jordan S, Kätterer T, Hadden D and Grelle A (2024) Setting-aside cropland did not reduce greenhouse gas emissions from a drained peat soil in Sweden. Front. Environ. Sci. 12:1386134. -- Leifeld, J., Klein, K. & Wüst-Galley, C. (2018). Peat decomposability in managed organic soils in relation to land use, organic matter composition and temperature. *Biogeosciences*, 15, 703–719. - -- Marliana, S.N., et al. (2023). Differences in CO₂ emissions on a bare-drained peat area in Sarawak, Malaysia, based on different measurement techniques. *Agriculture*, 13, 622. - - McCalmont, J., Kho, L.K., Teh, Y.A., et al. (2021). Short- and long-term carbon emissions from oil palm plantations converted from logged tropical peat swamp forest. *Global Change Biology*, 27, 2361–2376. +- Mos, H., Harun, M.H., Jantan, N.M., Hashim, Z., Ibrahim, A.S. & Yusup, Y. (2023). Differences in CO₂ emissions on a bare-drained peat area in Sarawak, Malaysia, based on different measurement techniques. *Agriculture*, 13, 622. + - Page, S.E., Rieley, J.O. & Banks, C.J. (2011a). Global and regional importance of the tropical peatland carbon pool. *Global Change Biology*, 17, 798–818. - Page, S.E., Morrison, R., Malins, C., Hooijer, A., Rieley, J.O. & Jauhiainen, J. (2011b). Review of peat surface greenhouse gas emissions from oil palm plantations in Southeast Asia. ICCT White Paper 15. - Qiu, C., Ciais, P., Zhu, D., et al. (2021). Large historical carbon emissions from cultivated northern peatlands. *Science Advances*, 7, eabf1332. -- Swails, E., Hergoualc'h, K., Deng, J., et al. (2022). Peat CO₂ and N₂O flux calibrations for the DNDC model in tropical oil palm plantations on peat. *European Journal of Soil Science*, 73, e13199. +- Schrier-Uijl, A.P., Kroon, P.S., Hendriks, D.M.D., et al. (2014). Agricultural peatlands: towards a greenhouse gas sink — a synthesis of a Dutch landscape study. *Biogeosciences*, 11, 4559–4576. + +- Swails, E., Hergoualc'h, K., Deng, J., Frolking, S. & Novita, N. (2022). How can process-based modeling improve peat CO₂ and N₂O emission factors for oil palm plantations? *Science of the Total Environment*, 839, 156153. + +- Tiemeyer, B., Freibauer, A., Albiac Borraz, E., et al. (2020). A new methodology for organic soils in national greenhouse gas inventories: Data synthesis, derivation and application. *Ecological Indicators*, 109, 105838. - Truskavetskii, R.S. (2014). Carbon budget of drained peat bogs in Ukrainian Polesie. *Eurasian Soil Science*, 47, 687–693. diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..1f45375 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,233 @@ +# Validating the pipeline against external datasets + +A supplement to [`methodology.md`](methodology.md) and [`architecture.md`](architecture.md). The methodology says how emissions are quantified and attributed; this describes the tooling in `validation/` that measures the result against datasets produced by other people, and — more importantly — **why it is shaped the way it is**. + +It is internal tooling with two jobs: to put measured magnitudes on the methodology gaps we already know about, and to surface ones we do not. It is not auditor-facing and it is not CI. + +**It extends [`further_research.md`](further_research.md) rather than forking it.** That document is already a gap register: entries ordered by estimated impact, each with a prose *Potential impact* and *Potential improvement path*. What it lacks is measurement. So a finding's `slug` is one of that document's headings, verbatim, and this tooling's contribution is a magnitude, a breadth and a confidence per entry — turning "potentially substantial in other geographies" into a number that can be ranked against the others. A gap with no entry there gets a new slug and should be proposed back to that document. + +______________________________________________________________________ + +## 1. What it answers + +Three questions, each narrowing the next. + +1. **Where do we disagree with the outside world?** Every target against every anchor. +2. **Which term carries the disagreement?** Intensity, yield, activity data, detection, extent, allocation, carbon density or discount. §4 answers most of this without touching a raster. +3. **What should we fix first?** Ranked by magnitude and confidence. + +These are questions, not components. A stage says *when code runs*; a question says *what is being asked*, and one stage serves several. + +Two constraints shape everything below. **Changing the methodology is out of scope** — every probe is report-only. And **the report's exit code never depends on its findings**: it exits 0 whenever it ran, however bad the news, because a report that fails on a bad result cannot be used to characterize bad results. It fails only on unreadable input or a missing join key. The generators under `tools/` are the deliberate exception — a build tool that emits a broken committed artifact exits nonzero, because a bad committed mapping is not a finding, it is a defect. + +## 2. Layout, and why it is split this way + +``` +tools/ offline generators; the output is the artifact + build-national-mappings.py GADM<->World Bank and Orbae<->World Bank maps, self-checking + build-tiled-countries.py the 220 countries E1 admits, self-checking +validation/ + schema.py vocabulary: enums, Finding, code_version, source identity + targets.py reads data/targets.json; Measure, Target, Control + pull.py anchor retrieval, the data-directory layout, sources.lock.json + prepare.py all transforms, eligibility, and the source-specific readers + report.py Markdown renderers; reads no files and computes no quantities + capture.py runs both pipeline legs and writes efs.parquet [runs the pipeline] + __main__.py the reporting entrypoint + data/ committed JSONs; .cache/ is gitignored +``` + +The split is **by what the code is, not by what it computes**. + +`tools/` needs geopandas and reads a 93 MiB GeoPackage. Nothing imports it — the hyphenated filenames make that literal — so its cost stays off the reporting path. It runs rarely, and what it produces is a committed artifact rather than console output, because a generator whose result evaporates cannot be read by anything downstream. + +`capture.py` is the only module that runs the pipeline itself, which is why it is run by hand rather than reached from `python -m validation`. Keeping it separate is what lets the report finish in seconds. + +`report.py` reads only the frames it is handed, and computes nothing. That constraint is load-bearing: it forces every filter, rescale, factorization and cross-walk into `prepare.py`, where each is named and testable, and leaves a report section as a pivot plus a caption. + +### `pull` is not `ingest` + +`jdluc.ingest` mirrors a registered dataset into the pipeline's own managed storage, normalized and tile-partitioned, so the pipeline can compute over it. `pull` retrieves an anchor exactly as published, to a local cache, pinned by revision and sha256 in a committed lock, so the report can compare against it. + +**The lock is committed; the bytes are not.** 10 MiB across the 128 WRI files reproduces from a 66 KiB lock, and the digests prove a fresh checkout got what the lock describes. The same rule decides what else gets committed: anything derivable from a pinned anchor is derived on each run rather than stored beside it. The eligible shortlist is the worked example — 864 pairs recomputed in about a tenth of a second from anchors the lock already pins, where a committed copy would be a second, unpinned record of the same numbers that goes stale the moment the pin moves. + +**One artifact does not reproduce from the lock: the capture.** `efs.parquet` and `forest_pools.parquet` are re-keyed extracts of the pipeline's own parquet outputs, so they are neither committed here nor re-fetchable; producing them means re-running `validation.capture`, which takes hours of pipeline. Everything upstream of a capture reproduces in seconds — eligibility, yields, the anchor tables, and `ORBAE_OVER_WRI`, whose two sides are both external. Only the FAOSTAT half of that needs more than the lock: `prepare.read_faostat_production` reads the ingested parquet rather than a second copy of the upstream archive, so a checkout that has never run the ingest fetches it once and caches it beside the pulled anchors, with `pull.record_digest` pinning what it got. Every measure with a term of ours does not, and neither does the conservation bound. A report carrying those numbers is reproducible only alongside the run that produced them, and `code_version` and `source_version` on every row name which run that is. + +FAOSTAT **is** an ingested dataset, because it is a plausible pipeline input: national tabular production statistics keyed on jdluc's own `admin_id` and `crop_name`, the global analogue of `usda_nass_quickstats`. WRI, EPA and Orbae stay outside `jdluc/`. Every pipeline module imports `jdluc.datasets`, so putting the yardstick there would let `statistical.py` consume the number it is measured against, with no contract preventing it. + +## 3. Choosing what to validate + +A target is a **(country, crop) pair**, not a triple with a methodology: the legs available are derivable. `jurisdictional_direct.workflow` asserts `iso_3166 == "USA"`, and all 11 jdLUC crops are a subset of sLUC's 32 — so a US target yields three comparisons, including **sLUC against jdLUC, which needs no external anchor at all**. That internal head-to-head is the cheapest comparison in the design and the only one where a disagreement proves one of our own legs wrong without an outside arbiter. + +Eligibility is mechanical, four filters, and re-derived on every run: + +| | Filter | Why | +| --- | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| E1 | the crop is modeled, and the country is one of the 220 in `coverage.md` | a pair we cannot compute is not a target | +| E2 | at least one anchor, for at least one measure | nothing to compare against, nothing to learn | +| E3 | WRI publishes provincial rows, and the key map covers >=50% of the country's land area | the provincial grain is the default | +| E4 | national production >= 100 kt | a factor that cannot move a companywide number is not worth capture time | + +**E2 is deliberately weak.** Requiring a WRI *emissions* anchor would exclude every grassland-dominated pair, since WRI is forest-only — exactly the regime with the least external validation and the largest unanchored pool. + +**E3 weights by land area because counting units misleads in both directions.** Brazil matches 87% of its provinces but 100% of its area, the four it misses totaling 12 km²; Algeria matches 96% of its provinces and only 74% of its area, missing two Saharan wilayas of 607,000 km². Counting would pass Algeria and fail Brazil. Area is still a proxy — the weight that matters is cropland — but cropland weighting needs a capture, which needs the target set, which needs E3. + +**E1 keeps woody perennials on purpose.** Oil palm, coconut and coffee rank at the top on WRI's deforestation while the pipeline attributes close to zero to them, because GLAD's cropland class excludes perennial *woody* crops by construction — it covers "annual and perennial herbaceous crops", and "perennial woody crops, permanent pastures and shifting cultivation are excluded from the definition" (Potapov et al., 2022, *Nature Food* 3, 19–28, [doi:10.1038/s43016-021-00429-z](https://doi.org/10.1038/s43016-021-00429-z)). The distinction is canopy, not lifespan: sugarcane is perennial and ratooned but herbaceous, and GLAD sees it. + +Keeping those pairs eligible is the finding, not a filtering mistake, and it is the single largest thing this tooling has measured: **54 eligible pairs hold 415 Mt of WRI-attributed deforestation between them**, led by Indonesian oil palm at 218 Mt. It needs no capture, because both sides come from anchors. + +The list is woody crops only, and that is a narrower thing than "perennial": sugarcane, banana and plantain are all perennial and all herbaceous, so GLAD's definition admits them and they are not part of this gap. An assertion in `prepare.py` names all three, because reading the list as "perennial" rather than "woody" is the mistake it invites. + +### What E3 costs, and why most of it cannot be recovered + +E3 rests on a key map between GADM's `GID_1` and the World Bank's `ADM1CD_c`, built offline by `tools/build-national-mappings.py` from WRI's own key file and the World Bank admin-1 GeoPackage. Roughly 63% of units match, plus a handful of hand-written overrides. Of the 244 countries in the layer, 53 are fully covered, 87 are partial but still clear E3, and 104 fall below it. + +Weighed by the anchor's own figure rather than by country count, the loss is small and concentrated. Against 2,662 Mt of WRI 2020 deforestation over all GFW-tiled countries, the countries E3 excludes hold **45 Mt — 1.7%**. The 87 partial countries keep their pairs and lose provinces at the margin; among the chosen targets the worst are Mozambique at 27% of provincial units unmapped, Malaysia 13%, Mexico 13% and Thailand 12%, while Brazil, Argentina, the USA, Bolivia and the DRC lose none. + +**Most of the excluded 45 Mt is not a matching failure and cannot be fixed by a better matcher.** Diagnosing each excluded country by whether its two sources share a vocabulary splits them cleanly: + +- **Different administrative levels — 41 Mt across 29 countries, 92% of the loss.** The two sources describe different things, so no name matching can succeed. The Philippines is the largest at 24.8 Mt: the World Bank gives it 17 *regions* where GADM gives 81 *provinces*, which nest inside them — **0 of the 81 names match, even fuzzily**. The Central African Republic (7 regions against 17 prefectures, 8.6 Mt), Sri Lanka (9 against 25, 4.0 Mt) and Malawi (4 against 28, 2.5 Mt) are the same failure at smaller scale. +- **Names partly overlap — 3.7 Mt across 61 countries, 8% of the loss.** Here a better matcher might help, and it is not worth building: the largest is South Sudan at 1.1 Mt, Guyana fuzzy-matches 9 of its 10 units for 0.4 Mt, and Russia — 83 provincial units, almost all unmapped — carries so little deforestation-linked production that its ~68 unmatched province names are not worth overriding. + +So the recoverable share of E3's cost is **8% of 1.7%**, and the rest needs a different mechanism rather than a better matcher. + +A country in the first group is carried as a **national ratio only** rather than dropped. `CAF MAIZE`, `PHL COCONUT` and `PHL RICE` are in the target set on that basis, off the eligible shortlist entirely. It keeps the anchor comparison available at the grain where it is defined, and records why the provincial grain is not. + +Carrying two pairs for one country buys something a single pair cannot. `PHL COCONUT` is a `gap` entry, expected to fail on the woody-perennial detection gap; `PHL RICE` is `ranked`, and rice is herbaceous, so GLAD's cropland class does see it. Holding the country fixed across the two separates "the Philippines is under-detected because its crops are perennial" from "the Philippines is under-detected because it is the Philippines" — the same construction as `IDN MAIZE` sitting beside `IDN OILPALM`. Between them they cover 12.9 of the country's 24.8 Mt. + +**The map is deliberately injective, and that is load-bearing rather than hygiene.** A many-to-one map is representable in the stored direction — 81 GADM provinces to 17 World Bank regions is a perfectly good `gadm_id -> world_bank_id` mapping — but `prepare.get_orbae_gadm_ids` inverts it to resolve Orbae's provinces onto GADM keys, and a dict inversion under a many-to-one map silently keeps whichever entry comes last. Eighty of the Philippines' eighty-one provinces would vanish without a word. So admitting many-to-one countries at the provincial grain is not a matter of relaxing a check: it needs the inversion to handle one-to-many, and the comparison to go through the intensity-times-area rollup, since an emissions factor is a ratio and cannot be summed up an administrative hierarchy. + +Finally, `area_coverage` weights by land area, which is a proxy for the cropland weighting that would actually matter — so these figures overstate the loss in arid countries, Algeria's missing Sahara being the standing example, and could understate it where the unmapped provinces are the agricultural ones. Correcting that needs a capture, which needs the target set, which needs E3. + +### There is no scoring + +Ranking the eligible set by a blend of weighted, percentile-ranked criteria does not work here, and the arithmetic is worth stating. One criterion of the four takes only three values, so percentile-ranking it turns a near-boolean into a swing that outweighs its nominal weight: such a blend's top 21 has **90% overlap with sorting on that one flag alone, and 24% with sorting on the anchor's own deforestation figure**. It drops `BRA SOYBEAN`, the largest pair in the set, in favor of pairs under 8 Mt. + +The formalism is not merely redundant, it conceals that one binary flag decides everything. So the set is chosen by a person and recorded in `data/targets.json` with a written reason per pair. Every pair can be asked "why are you here?", and the answer is a sentence rather than a score. + +A target is never dropped merely because it now agrees. That would bias the set toward disagreement, destroy the controls, and make every run look worse than the last. **Agreement is a result.** + +## 4. WRI's factor factorizes, which localizes most disagreements without a raster + +``` +EF [kgCO2/kg] = intensity [kgCO2/ha] x yield_factor_kg [ha/kg] +``` + +**The identity is checkable nationally, and only there.** Nationally WRI publishes deforestation emissions, production and the factor, so `EF = LD / production` has three independent terms — `LD` being WRI's column name for the emissions, short for *linearly discounted*, since they carry the GHGP 20-year discount already applied. It holds across 4,442 rows over 42 crops, worst relative error 0.05%, which is rounding in the published precision. Provincially WRI publishes only the factor, so `intensity = EF ÷ yield` has no third term that could contradict it. Nothing provincial can fail this check, and it must not be claimed as one. + +What *can* fail provincially is the join. It was measured when the provincial path was built — **54,655 of 55,000 provincial factors (99.37%) convert to an intensity**, once the taxonomy rename below is resolved — and the report does not recompute it: `ORBAE_OVER_WRI` is the only measure the provincial factors reach, and each of its rows carries the province count its correlation was taken over, so a join that degraded would show there as a thinner pair or none at all. + +Three consequences follow. + +**Intensity, not the per-kg factor, is what a provincial row supports.** Provincial per-kg factors span 0.004 to 8,993 because they are ratios over deforestation-linked production, so a province holding a sliver of it beside a large clearing yields a number meaningless on its own. Intensity removes the *production* denominator, and that is all it removes: because WRI's yield is a whole-jurisdiction figure while its factor's denominator is deforestation-linked production, `EF × yield` is emissions over WRI's deforestation-linked *area* rather than over the crop's. The identity is exact — `intensity × area == LD ÷ deforestation_share` — and it holds to a median 0.01% across the 2,312 pairs carrying deforestation emissions, a yield and a FAOSTAT area. + +**That identity is why intensity anchors no comparison of ours.** Carried on our crop area it compares us against WRI's own figure inflated by 1 ÷ `deforestation_share`: 6.55× at the median, 1.02× for `CIV COCO` and 1.23× for `IDN OILP`, but 1.78× for `BRA SOYB` and 18.96× for `USA MAIZ`. Smallest where deforestation dominates the footprint and largest in the temperate pairs, so it is neither a wash nor a constant a tolerance could absorb. Intensity therefore serves the one job where it is not a level against us — `ORBAE_OVER_WRI`, anchor against anchor — and `SLUC_OVER_WRI` compares tonnes against tonnes instead. + +**The split is exact and additive in logs**: `log(EF ratio) = log(intensity ratio) + log(yield ratio)`. An all-yield disagreement means a denominator problem; an all-intensity one means carbon density or destination-class treatment. Which it is decides whether raster work is worth running. + +**Provincial intensity is dominated by slivers, so nothing reads it as a level.** Derived intensity has a median of 2.2 tCO₂/ha, but 1.68% of rows exceed 1,000 tCO₂/ha — more than any biome holds — topping out at 4.1 million for a province with essentially no production of that crop. They are spread across nearly every crop rather than concentrated, so this is the sliver case and not a data defect. No distribution statistic over provincial intensity is reported for that reason, and `ORBAE_OVER_WRI`, the one measure built on it, is a rank correlation: a sliver holds a single position in an ordering however extreme its value, so it cannot pull a correlation the way it pulls a mean. + +### `SLUC_OVER_WRI` is national, and compares WRI's deforestation emissions against ours + +An emissions factor is a ratio, so provincial factors cannot be averaged up, and WRI publishes no provincial weights. A rollup on our own weights — `Σ(WRI_intensity × our_area) / Σ(our_production)` — was the obvious way round that, and it is the construction the identity above rules out: it compares us against `LD ÷ deforestation_share` rather than against the emissions themselves. + +So the measure is our forest emissions over WRI's published deforestation emissions, nationally, in tonnes, with no yield, area or production term on either side. Both are the same physical quantity in the same units, which is the strongest form this comparison can take, and it is the construction `further_research.md`'s oil-palm decomposition already used. + +**Six of seven control priors come back within their quoted precision.** The `inherited` figures in `data/targets.json` are from earlier work, and the first capture reproduces them under this construction: `BRA SOYBEAN` 0.190 against a prior of 0.19, `PRY SOYBEAN` 0.067 against 0.07, `ARG SOYBEAN` 0.423 against 0.43, `ARG MAIZE` 0.441 against 0.46, `COD MAIZE` 0.319 against 0.32 and `AGO MAIZE` 0.571 against 0.58. `CAF MAIZE` is the exception at 0.357 against 0.40, 11% under. That is independent of the algebra above and points the same way, since the priors were computed as emissions against emissions. + +Two consequences, one paid and one earned. **The provincial grain is lost for this measure, and WRI's release cannot give it back**: provincially it publishes a factor and no production, so no provincial emissions figure exists to compare against — intensity was the only route to one, and intensity is what carries the share. **The gas scope is now matched**: WRI publishes CO2e nationally and CO2 only provincially, so a provincial construction was pinned to CO2 against our CO2e, and this is CO2e on both sides. Every row is therefore `AS_PUBLISHED` with `coverage_fraction` 1.0 — nothing is joined, so nothing can be dropped. + +Note this narrows what E3 is buying. It still gates the provincial factors `ORBAE_OVER_WRI` needs, but a country whose key map is too thin for E3 can now be compared against WRI nationally — the 104 countries E3 excludes hold 45 Mt of WRI deforestation, and none of that is out of reach any more. Whether E3 should stay an eligibility filter or become a reported attribute is an open question, deliberately not settled here. + +### The reporting year is the window, not a vintage + +WRI publishes `{LD,production,EF}_{2020..2024}`, and the year is not a data vintage: it names where the LSRS 20-year assessment window sits. Reporting year 2020 covers loss years 2001–2020, 2024 covers 2005–2024, and each loss year carries the linear discounting factor for its age — 9.75% for the reporting year itself, falling 0.5 pt/yr to 0.25% nineteen years back, summing to 100% (Fitts et al., 2025a, Table 4, which takes the schedule from the draft LSRG). Those are the weights in `emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT`: WRI's five-year block sums are exactly 5× ours, because ours multiplies a span *total* by a span-average factor where WRI multiplies annual emissions by annual factors. + +**The window slides over real new loss data, so the five columns are a series.** Re-discounting a fixed 2001–2020 series would bound `LD_2024 / LD_2020` at 31/39 = 0.795 for any non-negative series; 62% of the 4,423 populated pairs exceed that bound, 864 rise monotonically and hold 49% of global `LD_2024`, and the global total is flat at 2,664 → 2,669 Mt where re-discounting would have to fall. The 19 pairs that decay to exactly zero are the kernel's fingerprint rather than a counterexample: `GEO LENT` and `Z07 OCER` both run 7 : 5 : 3 : 1 : 0, one loss year in 2004 and nothing since. The denominator slides too — `production_Y` is SPAM 2020 production rescaled by the FAOSTAT national ratio for year *Y* (Fitts et al., 2025a, step 5), which reproduces to a median 0.02% (2021), 0.05% (2022) and 0.24% (2023), then 6.2% for 2024, because WRI predicted 2024 with a per-(country, crop) random forest before FAOSTAT published it. It is not national production: over 1,133 pairs above 100 kt it is a median 12% of FAOSTAT's national figure — 52% for `BRA SOYB`, 5% for `USA MAIZ`, 93% for `CIV COCO` — which is the deforestation-linked denominator this section relies on, measured. + +So `REFERENCE_YEAR = 2020` matches windows rather than picking a convention, and the spread across the five is the price of leaving the year unpinned, not WRI disagreeing with itself. + +**The crop side does not move with the reporting year at all.** Crops enter only through the product allocation factor: per 10 km cell, a crop's physical-area expansion over the expansion of all agricultural land, with pasture from Global Pasture Watch and SPAM's subsistence area in the denominator only (Fitts et al., 2025a, Equation 1 and endnote 21). SPAM exists for 2000, 2005, 2010 and 2020, so the PAF is a step function — 2000→2005 allocates loss years 2001–2005, 2005→2010 allocates 2006–2010, and 2010→2020 allocates 2011 through **2024** (ibid., Table 3 and endnote 22), the same freeze as our `GLAD_TO_MAPSPAM_SPAN` reusing 2010→2020 for two spans. Weighted by the discount, that puts 6.25% / 18.75% / 75.00% of a 2020 factor on the three snapshots, and 0.25% / 8.75% / 91.00% of a 2024 one. + +Because the PAF is identical in every column, crop coverage is a property of the SPAM snapshots rather than of the reporting year: all 42 crops appear in all five columns, and only 27 of the 4,654 rows the lock pins flip between zero and non-zero — every one of those is loss timing, not crop coverage. + +**Crops missing from a snapshot get two different treatments, and neither is ours.** SPAM 2000 carries 21 crops against 2020 v2's 46, so WRI gap-fills the older years, and the instruction is worth quoting because it is the whole of the method: “to extrapolate or interpolate SPAM crop data for missing years (e.g., 2000), use a linear regression to estimate crop area in each jurisdiction and ensure that the total crop area expected for 2000 remains consistent”, then “proportionally divide the predicted SPAM crop area for disaggregated crop categories to missing data years using linear extrapolation” — preserving each data year's total area and staying “consistent with per-product physical area estimates at the national country level provided by FAOSTAT annually” (Fitts et al., 2025a, pp. 36–37, “Additional details for calculating the PAF for product expansion”). Their worked example is arabica and robusta coffee, one category in SPAM 2000 and two later. Crops that exist only in the newest snapshot are dropped instead, the published set being the 42 present in SPAM 2005, 2010 *and* 2020. So `CITR`, `ONIO`, `RUBB` and `TOMA` have yield factors but no emission factor — there is no `EF_ADM0_RUBB_*.csv` at the revision `pull.WRI_REVISION` pins, meaning **WRI publishes no factor for rubber** — and the factor files keep the 2005-era `ACOF`/`SMIL` codes rather than 2020's `COFF`/`MILL`. + +Ours decomposes rather than gap-fills, per pixel and by within-group shares pooled over the later snapshots — described in [`methodology.md`](methodology.md) and bounded in [`further_research.md`](further_research.md), so it is not restated here. Both hold a group's total and redistribute inside it, so the two agree on a group aggregate; what differs is the evidence each split runs on and the grain it runs at. Ours reads the pixel's own composition in the later snapshots. WRI's is a jurisdiction-level regression through the year series, held consistent with FAOSTAT's per-product national area, and the guidebook does not say how that estimate is spatialized. So an individual sibling — banana against plantain, arabica against robusta, coconut against oil palm — need not agree in any given cell, and a sibling-level disagreement can be laid at the 2000 crop split. **What bounds it is weight rather than agreement**: the 2000 snapshot carries the same 6.25% of a 2020 factor's numerator for both methods, and neither treatment touches a crop SPAM 2000 maps individually, which is 28 of the 35 pairs in the target set. The seven it does touch are the `OOIL` constituents oil palm and coconut, and `SWPY`'s yam, the one of the three whose group has no catch-all. + +What is worth carrying is where the two stop, because they stop in the same place. Gap-filling only ever bridges a snapshot that is *coarser*; it cannot invent a crop's first appearance. WRI's 42 are the crops SPAM maps in 2005, 2010 *and* 2020, ours are the 32 nameable in 2000 after decomposition, and a crop first mapped in 2020 falls outside both. So rubber has no external anchor rather than merely no model of ours, and the four 2020-only crops need no decision here: `YEAR_TO_UNRECOVERABLE_CROP_NAMES` already excludes them, and E2 could not have offered them an anchor. + +**Sources.** Everything above is WRI's own construction rather than reverse-engineering, with two exceptions flagged as measurements in the text — that the window slides over new loss data, and that the denominator follows FAOSTAT — which are inferred from the published files because no document states them. + +- **Fitts et al., 2025a** — the method. *Geospatial Methods for Corporate GHG Accounting of Deforestation and Land Occupation*, guidebook, version 1, December 2025, World Resources Institute, [doi:10.46830/wrigb.22.00158](https://doi.org/10.46830/wrigb.22.00158). Table 4 is the discounting schedule, Equation 1 the PAF, Table 3 and endnote 22 which loss years each SPAM snapshot allocates, step 5 the production denominator, and the note under Equation 1 the gap-filling of older SPAM releases. +- **Fitts et al., 2025b** — the numbers. *Statistical Land Use Change Emissions from Deforestation and Land Occupation for 42 Agricultural Crop Categories*, technical note, World Resources Institute and Quantis, published as [`wri/GCSC`](https://github.com/wri/GCSC). `pull.WRI_REVISION` pins it by commit and `sources.lock.json` by digest, so every count in this section is reproducible from the lock. +- **The inputs it composes** are cited in the guidebook's own reference list rather than repeated here — tree cover loss (Hansen et al., 2013), the GFW forest carbon flux model (Harris et al., 2021, revised in Gibbs et al., 2025), 1 km loss drivers (Sims et al., 2025), SPAM 2020 v2 (IFPRI, 2024) and Global Pasture Watch (Parente et al., 2024, [doi:10.1038/s41597-024-04139-6](https://doi.org/10.1038/s41597-024-04139-6)) — the last because the pasture term in WRI's allocation denominator has no counterpart in ours. + +## 5. What the report checks, in order + +**The conservation bound comes first, ahead of every anchor.** The sum of forest emissions across crops must not exceed the country's forest-conversion pool — that pool being forest conversion over all 2020 cropland with no crop share applied, so it bounds what any allocation can hand out. It earns first place because it needs no anchor, no baseline and no target selection, and because it is *decisive* where an anchor comparison is merely suggestive: a country over 100% has an attribution bug, and every anchor comparison for it is moot until that is fixed. + +**The pool and the attribution must come from the same run.** A pool from one code version compared against a capture from another compares a numerator and a denominator computed by different code, which is enough to make the result indicative rather than decisive. + +The pipeline emits no pool column, so `validation.capture` derives it: it reads back the cached per-tile emissions layer the capture attributed from — `statistical.get_downscaled_luc_emissions` is keyed on `(skip_glad_crop_filter, tile_id)` alone, so this is the same bands rather than a parallel derivation — clips it to the national geometry, and sums the discounted forest bands with no crop share applied. The discount weights and the pixel areas come from `emit` rather than being restated, so only the assembly lives in two places. + +What that buys is provenance, not merely a number: the pool is computed in the same process, at the same commit, as the capture it is written beside, and carries the same `code_version`. It is therefore freezable. Note this is a *same-run* guarantee rather than a same-function one — the pool and the attribution are computed by different code paths over identical inputs, so a change to how emissions are allocated moves one without the other, which is the point, while a change to the underlying layer moves both, which is also the point. + +The pool is re-derived only for the countries a run recomputed, for the same reason: carrying one forward beside freshly captured emissions would reintroduce the mismatch this is guarding against. + +**Findings come before tables**, sorted by severity then magnitude, because a table invites the reader to draw a conclusion the findings may have already disqualified. **Coverage comes last but is not optional**: a report that silently omits the targets no anchor covered reads as though it checked them, and an empty cell is indistinguishable from agreement unless something says so. + +Severity is deliberately independent of size. `BLOCKING` means a comparison could not be made and figures depending on it are meaningless; `DEFECT` means the run contradicts itself; `ADVISORY` means worth knowing. Magnitude, breadth and confidence are carried separately, because severity gives no ordering — it does not distinguish a gap worth tens of megatonnes across nine countries from one worth a fraction of that in a single country. + +## 6. Controls, and what makes one trustworthy + +Controls are rows where a *stable* relationship is expected, so a change means something broke rather than something was learned. Without them, "biased on palm and peat" and "biased everywhere" look identical. + +**`Orbae / WRI` contains no term of ours.** If it moves, an anchor was revised — distinguishable from a pipeline change, which moves `sLUC/WRI` and `sLUC/Orbae` together while leaving `Orbae/WRI` still. That is what stops a single anchor carrying the whole guard, and it is the only measure that can be frozen before the pipeline has ever run, since both sides are external. + +Four properties of a control are worth stating because each was arrived at by getting it wrong first. + +**A control fires against `baseline`, never against `inherited`.** An inherited figure is a prior number used once, as a sanity check when the baseline is frozen. Tolerance means "how much movement deserves a flag", not a noise band — the pipeline is deterministic, so movement comes only from a code or data change. + +**A rank correlation is not tolerance-checked as though it were a ratio.** Orbae's export is permanently three years off ours, and level cannot cross that offset while rank very nearly can: WRI's own factors span 1.42× at the median across the same three years while its provincial rank holds at +0.961. So those controls hold an ordering, and their tolerance is absolute, in correlation units. A relative band would make the weakest-agreeing control the twitchiest — at ρ +0.443 a relative 10% fires on a move of 0.044 where +0.886 tolerates 0.089 — which is backwards. + +**A baseline is meaningless without the anchors it was frozen against.** Each carries the identity of every external anchor behind it. If they differ from what a run read, the control reports a *stale baseline* rather than a moved one: reading the movement would book an anchor revision as a change in our own pipeline, which is precisely the confusion `Orbae/WRI` exists to prevent. + +**A control cannot rest on a row its own anchor contradicts.** Where WRI attributes more deforestation-linked area to a crop than FAOSTAT reports harvested for it, the pair cannot anchor a comparison whichever side is wrong. This is reported per target rather than only as a release-wide count, because a count of pairs out of thousands reads as a rounding error right up until one of them is the only armed guard on its row. + +**A row whose mechanism is known to be broken cannot guard anything.** Oil palm is a target and a register row, never a control: the annuals-only gap is active in exactly that crop, so its apparent agreement may well be coincidental. + +## 7. Provenance travels in the row + +Timestamps are ambiguous across concurrent branches, and a sidecar goes stale against the artifact it describes. So every produced row carries `code_version` — the commit that produced it, with a digest of any uncommitted result-bearing change — and `source_version`, naming every external anchor behind the figure. + +This is needed because a capture scoped to some countries keeps rows for the ones it did not recompute, so an artifact's modification time says nothing about whether a country's rows are current. A `code_version` column makes such a row self-identifying, and a table spanning more than one value raises a `DEFECT`. The check runs at read time, so it holds however the artifact was assembled. + +**What `code_version` does not cover.** `storage.cache_to_parquet` keys on `(module, qualname, version=, args)` and never on file contents, so it identifies the code that ran rather than the code every cached layer was built from. Clearing our artifact cannot clear jdluc's cache; that needs a `version=` bump on the changed function and on every cached function downstream, or a different `SCRATCH_ROOT`. Read those versions from source rather than from notes — they move, and a module with no cache version of its own is invisible to the key entirely. + +## 8. Limits on what this can conclude + +- **Grassland and peat can be sized but not confirmed.** No external anchor exists. WRI is forest-only, Orbae's grassland carbon is biome-invariant and its peat inverted, and SoilGrids, Harris and Huang *are* our carbon densities. The only non-circular route is transcription work: Spawn et al. (2019) and IPCC for biome grassland carbon, and the IPCC 2013 Wetlands Supplement for drained organic soils. +- **WRI cannot arbitrate allocation.** It shares this leg's MapSPAM expansion-share family, so an agreement may reflect shared method rather than shared answer. It stays independent on the forest *pool* and on carbon density, which is what the controls rest on. Genuinely independent evidence about allocation comes from three places only: Orbae, the US sLUC-versus-jdLUC head-to-head, and the conservation bound, which needs no anchor at all. +- **Where anchors disagree with each other more than they disagree with us, that is an anchor-quality row rather than a pipeline gap.** Brazil is why magnitude is measured against the *nearest* anchor: WRI runs about 5× high there while Orbae sits near our value, so measuring to the mean would book someone else's disagreement as our gap. Orbae and WRI barely agree on shape for the two crops that matter most — cross-country rank +0.018 for oil palm and +0.135 for soy — so for those neither corroborates the other. +- **The target set is partly WRI-shaped.** Materiality is read off WRI's deforestation figure, so a regime WRI is blind to — grassland conversion above all — is under-represented by construction. +- **Never size a share-denominator effect by expansion weighting.** Weighting by expansion predicted −1.4% for Indonesia against an actual −91%, three times running. The pathological cells carry near-zero *net* expansion with large *gross* churn, so they are low-expansion by construction and an expansion-weighted mean suppresses exactly what matters. Weight by the crop's area or by emissions. + +## 9. Anchor quirks that have bitten + +Recorded because each one silently produces a plausible wrong answer rather than an error. + +- **WRI uses two MapSPAM taxonomies within one release.** The emissions-factor files use Crop2005's 42 codes, the yield factors Crop2020's 46. Only two codes differ, and both are pure renames — but joining without resolving them loses 100% of those two crops' rows. +- **WRI's yields are FAOSTAT-derived, not MapSPAM-derived.** They agree with FAOSTAT across all 29 comparable crops, medians 0.947–1.024, including the two product-form risks that would have shown up several-fold: oil palm at 0.993 over 44 countries and seed cotton at 1.017 over 86. MapSPAM runs about 7% under FAOSTAT, so our MapSPAM-based production will show a systematic ~7% gap against WRI's yields that is a **source difference, not a methodology gap**. +- **WRI's five columns are a reporting-year series, not five vintages of one quantity.** The year names where the LSRS 20-year window sits, so each column drops a loss year, adds a real new one, and rescales the denominator to that year's production — §4 has the measurements that rule out a re-discounted fixed series. Factors span 1.42× at the median and 2.90× at the upper decile across the five, and 89% of pairs exceed a ±10% tolerance on that spread alone. That is the ceiling on what an *unpinned* comparison against WRI can resolve, and it is why the reporting year is pinned rather than left implicit. Reading the spread as WRI's own inconsistency overstates it: most of it is real temporal signal. +- **Orbae publishes two commodities on a processed basis** — palm as crude palm oil, sugarcane as cane sugar — so its factors must be divided by the conversion factor the export carries, or they read 3.9× and 8.9× high. +- **Orbae uses different code lengths per grain** — alpha-2 at country level, alpha-3-prefixed at provincial. Reading both as alpha-2 mis-filed 162 rows across 7 countries and dropped 63; there is now a two-way name/ISO injectivity check. +- **Emissions attributed to no crop are discarded.** Shares sum to at most one and the shortfall is charged to nobody — 12.6% of conversion and 13.9% of occupation globally, rising to 23.6% of Indonesia's occupation. Summing the factor table therefore does not give a jurisdiction's total, and a national-totals table without that row misleads. + +## 10. Unverified assumptions + +- **The temporal bases — settled.** WRI's half is answered, and in our favor: a numerator discounted over the 20 loss years ending in the reporting year, over a denominator that is production in the reporting year alone (Fitts et al., 2025a, Equation 4 and step 5). Ours is the same shape, so `SLUC_OVER_WRI` needs no matched-window correction, and comparing deforestation emissions against ours carries no extent term that would need one. The bases agree inside our own artifact too: `crop_hectares` carries the same linear discounting over 2000–2020 that production and emissions do, so every per-hectare figure divides quantities measured over one window. Measured on Ireland and Indonesia, that basis puts yields within ~4% of FAO's for the same country and crop, against ~25% for a 2020-snapshot denominator. +- **Provincial intensity is shape-only, and its shape is contaminated.** `ORBAE_OVER_WRI` compares two anchors province by province, so `LD ÷ deforestation_share` cancels only where the share is constant across a country's provinces, which it is not. A rank correlation survives that better than a level does, which is why those rows are rank rows — but the residual is unmeasured, and it is the reason a strengthening or weakening of an Orbae/WRI correlation cannot yet be read as a method change. +- **WRI's 2000 gap-fill carries three constraints with no stated precedence.** The guidebook sets a jurisdiction-level linear regression per crop, preservation of that data year's total crop area, and proportional division within a disaggregated crop category, all held consistent with FAOSTAT's per-product national physical area (Fitts et al., 2025a, pp. 36–37). Which binds when they conflict is not stated, so §4's reading — that a group's total survives the gap-fill, which is why the two methods agree on a group aggregate and can differ only between siblings — is an inference from the quoted instruction rather than something the guidebook asserts. It is silent too on how a jurisdiction-grain estimate is spatialized to SPAM's 10 km cells, the larger gap of the two: it leaves a sibling-level comparison undefined at the pixel rather than merely uncertain. Both are bounded by the 6.25% of a 2020 factor's numerator the 2000 snapshot carries, and reach only the seven target pairs whose crop is a 2000 group constituent. +- **Orbae's grassland split** — `natgrass` against `pasture` — versus our single derived remainder. Moot while only its forest term is used, and undefined otherwise. +- **The international cropland-extent factor**, which does not exist. CDL is US-only, so the ×1.10 CDL∩GLAD correction has no international counterpart and every international total carries an unquantified factor. diff --git a/jdluc/__tests__/continents_test.py b/jdluc/__tests__/continents_test.py deleted file mode 100644 index fb905be..0000000 --- a/jdluc/__tests__/continents_test.py +++ /dev/null @@ -1,114 +0,0 @@ -import pytest - -from jdluc.continents import ( - ISO_3166_TO_CONTINENT, - Continent, - iter_tile_cluster_to_iso_3166s, -) -from jdluc.tiling import GLOBAL_FOREST_WATCH_TILE_IDS - - -def test_continent_tile_ids_are_valid() -> None: - for continent in Continent: - assert set(GLOBAL_FOREST_WATCH_TILE_IDS).issuperset(continent.value) - - -@pytest.mark.integration -def test_continenets_are_supersets() -> None: - from jdluc.datasets.worldbank_jurisdictions import ( - AdminLevel, - get_ten_degree_tile_ids_for_admin_id, - ) - - for iso_3166, continent in ISO_3166_TO_CONTINENT.items(): - tile_ids = set( - get_ten_degree_tile_ids_for_admin_id( - admin_id=iso_3166, admin_level=AdminLevel.NATIONAL.value - ) - ) & set(GLOBAL_FOREST_WATCH_TILE_IDS) - if iso_3166 == "GRL": - # Drop Greenland because it doesn't contain much agriculture - assert tile_ids - set(continent.value) == { - "80N_070W", - "70N_030W", - "80N_080W", - } - elif iso_3166 == "RUS": - # Drop Kamchatka the because it doesn't contain much agriculture - assert tile_ids - set(continent.value) == {"70N_170W", "70N_180W"} - elif iso_3166 == "USA": - assert tile_ids - set(continent.value) == { - # Drop Hawaii - "20N_160W", - "30N_160W", - "30N_170W", - # Drop Alaska - "60N_150W", - "60N_160W", - "60N_170E", - "60N_170W", - "60N_180W", - "70N_160W", - "70N_170W", - "70N_180W", - "80N_150W", - "80N_160W", - "80N_170W", - } - elif iso_3166 in {"ASM", "FJI", "TON", "WLF", "WSM"}: - # Don't cross the international date line - assert tile_ids - set(continent.value) == {"10S_180W"} - elif continent == Continent.UNCLASSIFIED: - assert not tile_ids - else: - assert set(continent.value).issuperset(tile_ids) - - -def test_iter_tile_cluster_to_iso_3166s() -> None: - it = iter_tile_cluster_to_iso_3166s( - iso_3166s=( - # AFRICA - "AGO", - "COD", - "GHA", - "ZAF", - # ASIA - "KHM", - "IND", - "THA", - # EUROPE - "FRA", - "NOR", - "TUR", - # N AMERICA - "CAN", - "MEX", - "USA", - # OCEANIA - "IDN", - "MYS", - "PNG", - # RUSSIA - "RUS", - # S AMERICA - "ARG", - "BOL", - "COL", - # Unclassified - "COK", - "GUM", - ) - ) - result = {tuple(sorted(tile_ids)): iso_3166s for tile_ids, iso_3166s in it} - assert result == { - Continent.AFRICA.value: {"AGO", "COD", "GHA", "ZAF"}, - Continent.ASIA.value: {"KHM", "IND", "THA"}, - Continent.EUROPE.value: {"FRA", "NOR", "TUR"}, - Continent.NORTH_AMERICA.value: {"CAN", "MEX", "USA"}, - Continent.OCEANIA.value: {"IDN", "MYS", "PNG"}, - Continent.RUSSIA: { - "RUS", - }, - Continent.SOUTH_AMERICA.value: {"ARG", "BOL", "COL"}, - Continent.UNCLASSIFIED.value: {"COK", "GUM"}, - } diff --git a/jdluc/__tests__/datasets_test.py b/jdluc/__tests__/datasets_test.py index 6c189c2..ef2393a 100644 --- a/jdluc/__tests__/datasets_test.py +++ b/jdluc/__tests__/datasets_test.py @@ -1,7 +1,16 @@ +import itertools + import pytest +from jdluc.datasets import ifpri_mapspam from jdluc.datasets.glad_glcluc import flatten_ranges -from jdluc.datasets.worldbank_jurisdictions import get_ten_degree_tile_ids_for_admin_id +from jdluc.datasets.worldbank_jurisdictions import ( + AdminLevel, + get_jurisdiction_for_admin_level, + get_ten_degree_tile_ids_for_admin_id, + iter_jurisdiction_for_iso_3166_tile_id, +) +from jdluc.tiling import get_box_for_tile_id def test_flatten_ranges() -> None: @@ -11,7 +20,7 @@ def test_flatten_ranges() -> None: @pytest.mark.integration def test_get_ten_degree_tile_ids_for_country() -> None: iso_3166 = "BRA" # Brazil - expected = ( + expected = [ "00N_040W", "00N_050W", "00N_060W", @@ -26,12 +35,194 @@ def test_get_ten_degree_tile_ids_for_country() -> None: "10S_060W", "10S_070W", "10S_080W", - "20S_030W", + # This tile is not in the GFW tileset + # "20S_030W", "20S_050W", "20S_060W", "30S_060W", - ) + ] assert ( - tuple(get_ten_degree_tile_ids_for_admin_id(admin_id=iso_3166, admin_level=0)) + sorted( + get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, admin_level=AdminLevel.NATIONAL + ) + ) == expected ) + + +@pytest.mark.integration +def test_iter_jurisdiction_for_iso_3166_tile_id_loses_no_province() -> None: + # The tile prefilter is only safe because every province overlaps at least one of the tiles + # its country covers: that is what keeps attribute.merge_dfs summing over an unchanged + # index, and it is why the prefilter needed no cache version bump. A province is allowed to + # be absent only when it covers no published tile at all. + iso_3166 = "BRA" + seen = { + jurisdiction.id + for tile_id in get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, admin_level=AdminLevel.NATIONAL + ) + for jurisdiction in iter_jurisdiction_for_iso_3166_tile_id( + admin_level=AdminLevel.PROVINCIAL, iso_3166=iso_3166, tile_id=tile_id + ) + } + expected = { + str(admin_id) + for admin_id in get_jurisdiction_for_admin_level( + admin_level=AdminLevel.PROVINCIAL + ).index + if str(admin_id).startswith(iso_3166) + and get_ten_degree_tile_ids_for_admin_id( + admin_id=str(admin_id), admin_level=AdminLevel.PROVINCIAL + ) + } + assert seen == expected + + +@pytest.mark.integration +def test_iter_jurisdiction_for_iso_3166_tile_id_only_yields_overlapping() -> None: + # And the filter must actually filter: a province yielded for a tile has to intersect it, + # otherwise the prefilter is a no-op and the speedup is imaginary. + tile_id = "10S_050W" + box = get_box_for_tile_id(tile_id=tile_id) + yielded = list( + iter_jurisdiction_for_iso_3166_tile_id( + admin_level=AdminLevel.PROVINCIAL, iso_3166="BRA", tile_id=tile_id + ) + ) + assert yielded, f"{tile_id} is a Brazilian tile, so something should overlap it" + assert all(box.intersects(j.geometry) for j in yielded) + + +@pytest.mark.parametrize("year", sorted(ifpri_mapspam.YEARS)) +def test_get_reported_crop_name_always_names_a_band_that_year_has(year: int) -> None: + # The load-bearing property: whatever it returns must be readable from that snapshot. A + # rename or a new group that broke this would otherwise surface as a KeyError deep in a + # pipeline run. + names = {e.name for e in ifpri_mapspam.YEAR_TO_CROP_CLS[year]} + for crop_name in sorted(ifpri_mapspam.RECOVERABLE_CROP_NAMES): + assert ( + ifpri_mapspam.get_reported_crop_name( + canonical_crop_name=crop_name, year=year + ) + in names + ) + + +@pytest.mark.parametrize("year", sorted(ifpri_mapspam.YEARS)) +def test_reported_names_collapse_only_in_the_decompose_year(year: int) -> None: + # 19 constituents share 6 groups in 2000, so 13 names disappear; every other year names all + # 32 separately. This is the whole reason the decomposition exists, asserted directly. + reported = [ + ifpri_mapspam.get_reported_crop_name(canonical_crop_name=name, year=year) + for name in sorted(ifpri_mapspam.RECOVERABLE_CROP_NAMES) + ] + collapsed = len(reported) - len(set(reported)) + assert collapsed == (13 if year == ifpri_mapspam.DECOMPOSITION_YEAR else 0) + + +@pytest.mark.parametrize( + "year", sorted(set(ifpri_mapspam.YEARS) - {ifpri_mapspam.DECOMPOSITION_YEAR}) +) +def test_get_reported_crop_name_defers_to_the_rename_outside_2000( + year: int, +) -> None: + for crop_name in sorted(ifpri_mapspam.RECOVERABLE_CROP_NAMES): + assert ifpri_mapspam.get_reported_crop_name( + canonical_crop_name=crop_name, year=year + ) == ifpri_mapspam.get_renamed_crop_name( + canonical_crop_name=crop_name, year=year + ) + + +def test_is_reported_as_group_is_exactly_the_constituents_in_2000() -> None: + grouped = { + (crop_name, year) + for year in ifpri_mapspam.YEARS + for crop_name in ifpri_mapspam.RECOVERABLE_CROP_NAMES + if ifpri_mapspam.is_reported_as_group(canonical_crop_name=crop_name, year=year) + } + assert grouped == { + (crop_name, ifpri_mapspam.DECOMPOSITION_YEAR) + for crop_name in ifpri_mapspam.CONSTITUENT_TO_GROUP_NAME + } + + +def test_comparable_spans_agree_with_the_vocabularies() -> None: + # Not a restatement of the constant: it checks the constant against how much of the + # unrecoverable vocabulary the two snapshots actually share. Listing a pair as comparable + # when their names barely overlap is what would make name-matching measure the taxonomy + # instead of the land. + for before, after in itertools.combinations(sorted(ifpri_mapspam.YEARS), 2): + one, two = ( + ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[before], + ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[after], + ) + shared = len(one & two) / len(one | two) + if (before, after) in ifpri_mapspam.SPANS_WITH_COMPARABLE_CROP_NAMES: + assert shared > 0.5, ( + f"{before}->{after} called comparable but shares {shared:.0%}" + ) + else: + assert shared < 0.2, ( + f"{before}->{after} called incomparable but shares {shared:.0%}" + ) + + +@pytest.mark.parametrize("quantity", sorted(ifpri_mapspam.Quantity)) +@pytest.mark.parametrize("year", sorted(ifpri_mapspam.YEARS)) +def test_get_band_name_is_positionally_consistent( + year: int, quantity: ifpri_mapspam.Quantity +) -> None: + # `get_band_name` indexes the band list by the crop's position in the taxonomy + # enum. Nothing about a band name reveals a reordering, so a crop silently reading another + # crop's raster is the failure this exists to catch. Checked by matching each band against + # the slugified enum *value*, which is what the band name is built from. + for crop in ifpri_mapspam.YEAR_TO_CROP_CLS[year]: + band = ifpri_mapspam.get_band_name( + quantity=quantity, reported_crop_name=crop.name, year=year + ) + assert crop.value.lower().replace(" ", "-") in band, ( + f"{year} {quantity} {crop.name} resolved to {band}" + ) + + +@pytest.mark.parametrize("quantity", sorted(ifpri_mapspam.Quantity)) +@pytest.mark.parametrize("year", sorted(ifpri_mapspam.YEARS)) +def test_get_band_name_is_injective( + year: int, quantity: ifpri_mapspam.Quantity +) -> None: + # Two crops resolving to one band would silently double-count it + bands = [ + ifpri_mapspam.get_band_name( + quantity=quantity, reported_crop_name=crop.name, year=year + ) + for crop in ifpri_mapspam.YEAR_TO_CROP_CLS[year] + ] + assert len(set(bands)) == len(bands) + + +@pytest.mark.parametrize("year", sorted(ifpri_mapspam.YEARS)) +def test_unrecoverable_crop_names_partition_the_taxonomy(year: int) -> None: + # Every crop a snapshot reports is either reachable from a recoverable crop or unrecoverable, + # never both and never neither -- otherwise the share denominator would drop or double-count + # a band. + reported = {e.name for e in ifpri_mapspam.YEAR_TO_CROP_CLS[year]} + unrecoverable = ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[year] + reachable = { + ifpri_mapspam.get_reported_crop_name(canonical_crop_name=name, year=year) + for name in ifpri_mapspam.RECOVERABLE_CROP_NAMES + } + assert unrecoverable | reachable == reported + assert not (unrecoverable & reachable) + + +def test_unrecoverable_crop_names_grow_with_the_taxonomy() -> None: + # 2000 leaves only its two catch-alls unreachable; later releases name crops the 2000 + # taxonomy has no route to, and 2020 adds four more again. + counts = { + year: len(ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[year]) + for year in ifpri_mapspam.YEARS + } + assert counts == {2000: 2, 2005: 10, 2010: 10, 2020: 14} diff --git a/jdluc/__tests__/emit_test.py b/jdluc/__tests__/emit_test.py index 5fc94cb..ace6743 100644 --- a/jdluc/__tests__/emit_test.py +++ b/jdluc/__tests__/emit_test.py @@ -10,12 +10,13 @@ from jdluc.emit import ( CARBON_PER_BIOMASS_LIVE_WOOD, CO2E_PER_CARBON, + SPAN_TO_LINEAR_DISCOUNT_WEIGHT, get_belowground_carbon, get_dead_organic_matter_carbon, get_grassland_carbon, get_hectares_per_pixel, get_land_class, - get_linear_discounted_emissions, + get_linear_discounted_total, get_mineral_soil_emissions, get_peatland_occupation_emissions, get_soil_emissions, @@ -202,7 +203,7 @@ def test_get_soil_emissions() -> None: def test_get_peatland_occupation_emissions() -> None: - is_peatland = [[1] * 7] + is_peatland = [[1] * 7 + [numpy.nan, 0]] land_class = [ [ LandClass.BUILT_UP.value, @@ -212,28 +213,44 @@ def test_get_peatland_occupation_emissions() -> None: LandClass.OCEAN.value, LandClass.SNOW_ICE.value, LandClass.WATER.value, + # drained, but the peat mask is no-data / not peat + LandClass.CROPLAND.value, + LandClass.CROPLAND.value, ] ] result = get_peatland_occupation_emissions( is_peatland=get_darray_for_data(data=is_peatland), - year_to_land_class={ - 2020: get_darray_for_data(data=land_class), - }, + latest_land_class=get_darray_for_data(data=land_class), ) assert result.name == "tco2e-per-ha" - assert numpy.array_equal(result.data, [[37.3, 37.3, 0, 0, 0, 0, 0]]) + # a no-data peat mask must read as "not peat" rather than propagating NaN into the total + numpy.testing.assert_allclose(result.data, [[37.3, 37.3, 0, 0, 0, 0, 0, 0, 0]]) + + +def test_get_soil_emissions_tolerates_missing_soil_carbon() -> None: + # SoilGrids has genuine gaps over water, rock and ice; a missing stock must not poison + # the pixel, which would silently discard its vegetation emissions too + result = get_soil_emissions( + after=get_darray_for_data(data=[[LandClass.CROPLAND.value] * 2]), + before=get_darray_for_data(data=[[LandClass.FOREST.value] * 2]), + climate_zones=get_darray_for_data(data=[[Zone.TROPICAL_WET.value] * 2]), + is_peatland=get_darray_for_data(data=[[0, 1]]), + soil_organic_carbon=get_darray_for_data(data=[[numpy.nan, numpy.nan]]), + ) + assert not numpy.isnan(result.data).any() + # mineral pixel falls back to zero stock; the peat pulse does not depend on SoilGrids + numpy.testing.assert_allclose(result.data, [[0, 621]]) -def test_get_linear_discounted_emissions() -> None: - result = get_linear_discounted_emissions( - span_to_emissions={ +def test_get_linear_discounted_total() -> None: + result = get_linear_discounted_total( + span_to_value={ (2000, 2005): get_darray_for_data(data=[[1]]), (2005, 2010): get_darray_for_data(data=[[2]]), (2010, 2015): get_darray_for_data(data=[[3]]), (2015, 2020): get_darray_for_data(data=[[4]]), } ) - assert result.name == "tco2e-per-ha" assert numpy.array_equal(result.data, [[0.625]]) @@ -246,3 +263,25 @@ def test_get_hectares_per_pixel() -> None: [[1237126.38106379, 1237126.38106379], [1236937.9607238, 1236937.9607238]] ), ) + + +def test_get_linear_discounted_total_imposes_no_quantity() -> None: + # Area and production run through this too, so a name of its own would be a lie for two of + # the three callers. The caller names the result. + result = get_linear_discounted_total( + span_to_value=dict.fromkeys( + SPAN_TO_LINEAR_DISCOUNT_WEIGHT, get_darray_for_data(data=[[1]]) + ) + ) + assert result.name is None + + +def test_get_linear_discounted_total_over_the_weight_total_is_a_mean() -> None: + # How `statistical` puts area and production on the emissions' window: the same reduction, + # normalised. A quantity that never moves must survive it unchanged. + result = get_linear_discounted_total( + span_to_value=dict.fromkeys( + SPAN_TO_LINEAR_DISCOUNT_WEIGHT, get_darray_for_data(data=[[7]]) + ) + ) / sum(SPAN_TO_LINEAR_DISCOUNT_WEIGHT.values()) + numpy.testing.assert_allclose(result.data, [[7]]) diff --git a/jdluc/__tests__/geo_test.py b/jdluc/__tests__/geo_test.py index 8272fe0..8d5cfbc 100644 --- a/jdluc/__tests__/geo_test.py +++ b/jdluc/__tests__/geo_test.py @@ -1,3 +1,5 @@ +import pathlib + import numpy import pytest import shapely @@ -5,6 +7,7 @@ from jdluc.geo import ( clip_dset, + convert_vector_to_flatgeobuf, get_chunk_size, get_overview_level, ) @@ -92,3 +95,68 @@ def test_clip_dset(geometry: shapely.Polygon, expected: int) -> None: ) result = clip_dset(dset=dset, geometry=geometry) assert int(result["var"].count()) == expected + + +def get_dissolved_names( + records: list[tuple[str, str, float]], tmp_path: pathlib.Path +) -> dict[str, str]: + """`convert_vector_to_flatgeobuf` over one box per (id, name, side length).""" + import geopandas + + path_to_vector, path_to_flatgeobuf = ( + str(tmp_path / "in.gpkg"), + str(tmp_path / "out.fgb"), + ) + geopandas.GeoDataFrame( + crs="EPSG:4326", + data={ + "ISO_A3": [iso for iso, _, _ in records], + "NAM_0": [name for _, name, _ in records], + }, + geometry=[shapely.box(0, 0, side, side) for _, _, side in records], + ).to_file(path_to_vector, driver="GPKG") + convert_vector_to_flatgeobuf( + id_column_names=("ISO_A3",), + name_column_names=("NAM_0",), + path_to_flatgeobuf=path_to_flatgeobuf, + path_to_vector=path_to_vector, + ) + gdf = geopandas.read_file(path_to_flatgeobuf) + return dict(zip(gdf["id"], gdf["name"], strict=True)) + + +def test_convert_vector_to_flatgeobuf_names_a_country_after_its_largest_part( + tmp_path: pathlib.Path, +) -> None: + # `dissolve` keeps the first row of each group, and the admin-0 layer lists Spain's + # exclaves before Spain. Ordering by size is what stops the country taking an exclave's + # name; neither the first row nor the modal name would. + assert get_dissolved_names( + records=[ + ("ESP", "Ceuta (Sp.)", 1.0), + ("ESP", "Melilla (Sp.)", 1.0), + ("ESP", "Spain", 9.0), + ], + tmp_path=tmp_path, + ) == {"ESP": "Spain"} + + +@pytest.mark.parametrize( + ("stored", "expected"), + ( + # The admin-0 layer stores `Türkiye` as the UTF-8 encoding of `Türkiye`, so the file + # is valid UTF-8 and no `encoding=` on the read undoes it + ("Türkiye", "Türkiye"), + ("Curaçao (Neth.)", "Curaçao (Neth.)"), + # ... while a name that was never mangled must not survive the Latin-1 round trip + ("Spain", "Spain"), + ("Andalucía", "Andalucía"), + ("日本", "日本"), + ), +) +def test_convert_vector_to_flatgeobuf_repairs_only_mojibake( + expected: str, stored: str, tmp_path: pathlib.Path +) -> None: + assert get_dissolved_names(records=[("AAA", stored, 1.0)], tmp_path=tmp_path) == { + "AAA": expected + } diff --git a/jdluc/__tests__/harmonize_test.py b/jdluc/__tests__/harmonize_test.py index 3d13d6d..79b586f 100644 --- a/jdluc/__tests__/harmonize_test.py +++ b/jdluc/__tests__/harmonize_test.py @@ -29,78 +29,48 @@ def test_xy_validated(x: int, y: int, fails: bool) -> None: @pytest.mark.parametrize( ( - "tile_ids", + "tile_id", "origin", - "tiles", "transform", - "resolution", ), ( pytest.param( - ("00N_000W",), + "00N_000W", tiling.XY(0, 0), - tiling.XY(1, 1), (0, 1 / 4_000, 0, 0, 0, -1 / 4_000), - tiling.XY(40_000, 40_000), - id="One ten-degree-tile", + id="origin at 0,0", ), pytest.param( - ("00N_000W", "00N_010E", "10S_000W", "10S_010E"), - tiling.XY(0, 0), - tiling.XY(2, 2), - (0, 1 / 4_000, 0, 0, 0, -1 / 4_000), - tiling.XY(2 * 40_000, 2 * 40_000), - id="2x2 ten-degree-tiles with NW=0,0", - ), - pytest.param( - ("60N_060W", "60S_060E"), + "60N_060W", tiling.XY(-60, 60), - tiling.XY(13, 13), (-60, 1 / 4_000, 0, 60, 0, -1 / 4_000), - tiling.XY(13 * 40_000, 13 * 40_000), - id="two tiles spanning a 13x13 area", + id="north-west of the meridian", + ), + pytest.param( + "60S_060E", + tiling.XY(60, -60), + (60, 1 / 4_000, 0, -60, 0, -1 / 4_000), + id="south-east of the meridian", ), ), ) -def test_grid_from_tile_ids_resolution( - tile_ids: tuple[str, ...], +def test_grid_from_tile_id_resolution( + tile_id: str, origin: tiling.XY, - tiles: tiling.XY, transform: tuple[float, float, float, float, float, float], - resolution: tiling.XY, ) -> None: - result = Grid.from_tile_ids_resolution( - tile_ids=tile_ids, tile_resolution=tiling.TileResolution.GLAD + result = Grid.from_tile_id_resolution( + resolution=tiling.TileResolution.GLAD, tile_id=tile_id ) assert result.origin == origin - assert result.tiles == tiles - assert result.tile_resolution == tiling.TileResolution.GLAD + assert result.resolution == tiling.TileResolution.GLAD.value + assert type(result.resolution) is tiling.XY assert result.transform == transform - assert result.resolution == resolution - - -@pytest.mark.parametrize( - ("tile_id", "offset"), - (("00N_000W", tiling.XY(0, 0)), ("90S_180E", tiling.XY(180, 90))), -) -def test_grid_get_offset_for_tile(tile_id: str, offset: tiling.XY) -> None: - grid = Grid.from_tile_ids_resolution( - tile_ids=("00N_000W",), tile_resolution=tiling.XY(10, 10) - ) - assert grid.get_offset_for_tile(tile_id=tile_id) == offset - - -def test_grid_get_offset_for_tile_raises() -> None: - grid = Grid.from_tile_ids_resolution( - tile_ids=("00N_000W",), tile_resolution=tiling.XY(10, 10) - ) - with pytest.raises(AssertionError): - grid.get_offset_for_tile(tile_id="10N_010W") def test_grid_get_offset_for_world() -> None: - grid = Grid.from_tile_ids_resolution( - tile_ids=("60N_060W",), tile_resolution=tiling.XY(10, 10) + grid = Grid.from_tile_id_resolution( + resolution=tiling.XY(10, 10), tile_id="60N_060W" ) assert grid.get_offset_for_world( resolution=tiling.XY(360, 180), span=tiling.XY(360, 180) @@ -108,8 +78,8 @@ def test_grid_get_offset_for_world() -> None: def test_grid_get_resolution_for_world() -> None: - grid = Grid.from_tile_ids_resolution( - tile_ids=("60N_060W",), tile_resolution=tiling.XY(10, 10) + grid = Grid.from_tile_id_resolution( + resolution=tiling.XY(10, 10), tile_id="60N_060W" ) assert grid.get_resolution_for_world( resolution=tiling.XY(360, 180), span=tiling.XY(360, 180) @@ -131,14 +101,12 @@ def test_grid_get_resolution_for_world() -> None: def test_grid_get_resampling_for_band_type( band_type: BandType, resolution: tiling.XY, expected: str ) -> None: - grid = Grid( - origin=tiling.XY(0, 0), tiles=tiling.XY(2, 2), tile_resolution=tiling.XY(2, 2) - ) + grid = Grid(origin=tiling.XY(0, 0), resolution=tiling.XY(2, 2)) assert ( grid.get_resampling_for_band_type( band_type=band_type, src_resolution=resolution, - dest_resolution=grid.tile_resolution, + dest_resolution=grid.resolution, ).name == expected ) @@ -154,14 +122,12 @@ def test_grid_get_resampling_for_band_type( def test_grid_get_resampling_for_band_type_raises( src_resolution: tiling.XY, match: str ) -> None: - grid = Grid( - origin=tiling.XY(0, 0), tiles=tiling.XY(2, 2), tile_resolution=tiling.XY(2, 2) - ) + grid = Grid(origin=tiling.XY(0, 0), resolution=tiling.XY(2, 2)) with pytest.raises(NotImplementedError, match=match): assert grid.get_resampling_for_band_type( band_type=BandType.EXTENSIVE, src_resolution=src_resolution, - dest_resolution=grid.tile_resolution, + dest_resolution=grid.resolution, ) @@ -170,9 +136,7 @@ def test_grid_get_resampling_for_band_type_upsamples_clipped_world() -> None: # degree: comparing its full resolution against the grid would pick # downsampling, but once clipped to the grid extent it is coarser than the # destination and must be upsampled. - grid = Grid( - origin=tiling.XY(0, 0), tiles=tiling.XY(1, 1), tile_resolution=tiling.XY(36, 36) - ) + grid = Grid(origin=tiling.XY(0, 0), resolution=tiling.XY(36, 36)) world_resolution = tiling.XY(360, 180) # 1 px/degree, coarser than the grid src_resolution = grid.get_resolution_for_world( resolution=world_resolution, span=tiling.XY(360, 180) @@ -180,7 +144,7 @@ def test_grid_get_resampling_for_band_type_upsamples_clipped_world() -> None: # Clipped to the grid's 10x10-degree extent the source is only 10x10 px, # versus the 36x36 px destination, even though the full source is 360x180. assert src_resolution == tiling.XY(10, 10) - assert world_resolution.x > grid.tile_resolution.x # the old, buggy comparison + assert world_resolution.x > grid.resolution.x # the old, buggy comparison assert ( grid.get_resampling_for_band_type( band_type=BandType.INTENSIVE, diff --git a/jdluc/__tests__/jurisdictional_direct_test.py b/jdluc/__tests__/jurisdictional_direct_test.py index 8968923..137de0d 100644 --- a/jdluc/__tests__/jurisdictional_direct_test.py +++ b/jdluc/__tests__/jurisdictional_direct_test.py @@ -4,6 +4,8 @@ import pytest import xarray +from jdluc import statistical +from jdluc.datasets import usda_nass_quickstats from jdluc.jurisdictional_direct import Crop, get_crop_name_to_totals @@ -105,3 +107,22 @@ def test_get_crop_name_to_totals_with_no_crop_pixels( "peatland_conversion_emissions_mt": 0, "peatland_occupation_emissions_mt": 0, } + + +def test_every_crop_has_a_nass_series() -> None: + """`jdluc.datasets` cannot import the pipeline, so `CropSeries` repeats this crop list and + nothing else ties the two together. A name in one and not the other is silent rather than loud: + `trace` joins the yield in on the left, so the crop still reaches the artifact, carrying + hectares and emissions against a NaN emissions factor. + """ + assert {crop.name for crop in Crop} == { + crop_series.name for crop_series in usda_nass_quickstats.CropSeries + } + + +def test_every_crop_is_also_a_statistical_crop() -> None: + """The sLUC-against-jdLUC head-to-head is the one comparison with no external arbiter, and it + is per (country, crop). A jdLUC crop the statistical leg does not model has nothing to be + compared against, and drops out of that comparison without failing it. + """ + assert {crop.name for crop in Crop} <= {crop.name for crop in statistical.Crop} diff --git a/jdluc/__tests__/statistical_test.py b/jdluc/__tests__/statistical_test.py index 70030dc..296c8e8 100644 --- a/jdluc/__tests__/statistical_test.py +++ b/jdluc/__tests__/statistical_test.py @@ -3,69 +3,245 @@ import pytest import xarray -from jdluc.datasets import ifpri_mapspam -from jdluc.datasets.base import BandType +from jdluc import emit +from jdluc.datasets import gfw_global_peatlands, ifpri_mapspam from jdluc.statistical import ( + GLAD_TO_MAPSPAM_SPAN, Crop, - get_band_name_for_crop_name, - get_band_type_for_variable_name, + get_crop_name_to_totals, + get_crop_to_area_share, get_crop_to_share, - get_harmonized_quantity, ) AREA = ifpri_mapspam.Quantity.PHYSICAL_AREA PRODUCTION = ifpri_mapspam.Quantity.PRODUCTION -# Two non-shared ("residual") crops present in both the 2005 and 2010 snapshots. +# Crops `Crop` does claim, but which are not in every taxonomy under one name -- read off the +# taxonomy rather than hardcoded so a MapSPAM revision cannot quietly turn them into something +# else. Nothing to do with MapSPAM's "Other ..." crops: OTHER_0 is arabica coffee. OTHER_0, OTHER_1 = sorted( {e.name for e in ifpri_mapspam.YEAR_TO_CROP_CLS[2005]} - ifpri_mapspam.SHARED_CROP_NAMES )[:2] +# The same, narrowed to one the 2020 snapshot also names (ACOF, the first for 2005, is not). +OTHER_2020 = sorted( + {e.name for e in ifpri_mapspam.YEAR_TO_CROP_CLS[2005]} + & {e.name for e in ifpri_mapspam.YEAR_TO_CROP_CLS[2020]} + - ifpri_mapspam.SHARED_CROP_NAMES +)[0] +# Crops no `Crop` claims at all, so nothing can divide by them: they only ever move as a lump. +UNATTRIBUTED_0, UNATTRIBUTED_1 = sorted( + ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[2005] +)[:2] +# A crop no `Crop` claims that MapSPAM only starts reporting in 2020. Compared in 2020's own +# naming, since two attributed crops are renamed there (ACOF -> COFF, SMIL -> MILL) and would +# otherwise look 2020-only themselves. +UNATTRIBUTED_2020_ONLY = sorted( + ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[2020] + - ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[2010] +)[0] def get_dset( - quantity_year_to_values: dict[tuple[ifpri_mapspam.Quantity, int], dict[str, float]], + values: dict[tuple[ifpri_mapspam.Quantity, int], dict[str, float]], ) -> xarray.Dataset: - # A (scalar) band per crop in each (quantity, year)'s enum; unset crops are 0. + # A (scalar) band for every crop of every (quantity, year) -- the real dset always carries + # all four snapshots. Unset crops, and unset years, are 0. return xarray.Dataset( { - get_band_name_for_crop_name( - crop_name=crop.name, quantity=quantity, year=year - ): xarray.DataArray(float(values.get(crop.name, 0.0))) - for (quantity, year), values in quantity_year_to_values.items() + ifpri_mapspam.get_band_name( + quantity=quantity, reported_crop_name=crop.name, year=year + ): xarray.DataArray( + float(values.get((quantity, year), {}).get(crop.name, 0.0)) + ) + for quantity in ifpri_mapspam.Quantity + for year in ifpri_mapspam.YEARS for crop in ifpri_mapspam.YEAR_TO_CROP_CLS[year] } ) def get_dset_for_areas( - year_to_name_to_area: dict[int, dict[str, float]], + areas: dict[int, dict[str, float]], ) -> xarray.Dataset: - return get_dset( - quantity_year_to_values={ - (AREA, year): areas for year, areas in year_to_name_to_area.items() - } - ) + return get_dset(values={(AREA, year): by_crop for year, by_crop in areas.items()}) -def test_get_crop_to_share_composition() -> None: - # MAIZ +100, SOYB +50 (shared); one residual crop +50. total expansion == 200. - dset = get_dset_for_areas( - {2005: {}, 2010: {"MAIZ": 100.0, "SOYB": 50.0, OTHER_0: 50.0}} +@pytest.mark.parametrize( + ("areas", "before", "after", "expected"), + ( + pytest.param( + {2005: {}, 2010: {"MAIZ": 100.0, "SOYB": 50.0, OTHER_0: 50.0}}, + 2005, + 2010, + {Crop.MAIZE: 0.5, Crop.SOYBEAN: 0.25}, + id="each-crop-takes-its-own-gross-expansion-over-the-total", + ), + pytest.param( + {2005: {"MAIZ": 100.0}, 2010: {"MAIZ": 0.0, "SOYB": 50.0}}, + 2005, + 2010, + {Crop.MAIZE: 0.0, Crop.SOYBEAN: 1.0}, + id="a-contracting-crop-clips-to-zero-rather-than-going-negative", + ), + pytest.param( + # The total is masked to NaN rather than dividing 0 by 0; `.fillna(0)` is what turns + # that back into the 0.0 asserted here. + {2005: {}, 2010: {}}, + 2005, + 2010, + {Crop.MAIZE: 0.0}, + id="no-expansion-anywhere-masks-the-total-instead-of-dividing-by-zero", + ), + pytest.param( + # 2000 names only the coarse BANP group, so its 100 ha has to be split before any + # expansion can be measured. 2005 and 2020 pool to BANA 205 / PLNT 35 of 240 (2010 + # is empty), giving BANA 100 * 205/240 == 85.4 ha in 2000 and PLNT 14.6. Note 2020 + # does double duty: it is both a decomposition reference and the `after` snapshot. + # BANA then expands 175 - 85.4 == 89.6 and PLNT 25 - 14.6 == 10.4, of 100 together. + { + 2000: {"BANP": 100.0}, + 2005: {"BANA": 30.0, "PLNT": 10.0}, + 2020: {"BANA": 175.0, "PLNT": 25.0}, + }, + 2000, + 2020, + {Crop.BANANA: 0.8958333, Crop.PLANTAIN: 0.1041667}, + id="a-2000-group-is-decomposed-by-the-pooled-reference-fractions", + ), + pytest.param( + # 2005 names neither constituent, so 2020 carries the pool alone and splits the 2000 + # group 40/60 by area -- not the 50/50 an even split would fabricate. + {2000: {"BANP": 100.0}, 2005: {}, 2020: {"BANA": 80.0, "PLNT": 120.0}}, + 2000, + 2020, + {Crop.BANANA: 0.40, Crop.PLANTAIN: 0.60}, + id="a-later-reference-carries-the-decomposition-when-the-nearest-is-absent", + ), + pytest.param( + # OTHER_0 expands by exactly the 100 ha OTHER_1 gives up. Each crop's expansion is + # clipped at zero before being summed, so the denominator is 200 and MAIZE and + # OTHER_0 take half the cell each. A denominator that instead netted the two + # before clipping would see the pair contribute nothing, leaving 100 -- and MAIZE + # and OTHER_0 would each read 100/100, charging the cell's emissions twice over. + {2005: {OTHER_1: 100.0}, 2010: {"MAIZ": 100.0, OTHER_0: 100.0}}, + 2005, + 2010, + {Crop.MAIZE: 0.5, Crop(OTHER_0): 0.5}, + id="a-contracting-sibling-cannot-push-the-shares-above-one", + ), + pytest.param( + # No `Crop` can claim an unattributed crop's expansion, so the lump is netted whole + # instead: MAIZE keeps the entire cell even though UNATTRIBUTED_1 expanded, because + # UNATTRIBUTED_0 gave up as much. Netting is all we can honestly say about a lump + # nothing divides by. + {2005: {UNATTRIBUTED_0: 50.0}, 2010: {"MAIZ": 100.0, UNATTRIBUTED_1: 50.0}}, + 2005, + 2010, + {Crop.MAIZE: 1.0}, + id="crops-it-cannot-attribute-stay-lumped-and-net-against-each-other", + ), + pytest.param( + # MapSPAM first reports UNATTRIBUTED_2020_ONLY in 2020, so its whole area would read + # as expansion from zero and halve MAIZE's share -- although nothing was necessarily + # planted. It is newly named, not newly grown, so it is dropped from the denominator. + {2010: {}, 2020: {"MAIZ": 100.0, UNATTRIBUTED_2020_ONLY: 100.0}}, + 2010, + 2020, + {Crop.MAIZE: 1.0}, + id="a-crop-the-earlier-snapshot-never-reported-is-ignored", + ), + pytest.param( + # The converse, and the reason the case above cannot just drop every unattributed + # crop: both snapshots report UNATTRIBUTED_0, so its 100 ha really is expansion and + # must dilute MAIZE. Same shape as the case above, one crop different. + {2010: {}, 2020: {"MAIZ": 100.0, UNATTRIBUTED_0: 100.0}}, + 2010, + 2020, + {Crop.MAIZE: 0.5}, + id="a-crop-reported-in-both-snapshots-still-dilutes", + ), + pytest.param( + # 2000 is a different MapSPAM release and shares almost no unattributed names with + # 2005, so the "reported in both" test above would reject nearly the whole lump here + # rather than stabilise it. That is why the span is left out of + # `SPANS_WITH_COMPARABLE_CROP_NAMES` and the 2005 crops still dilute. + {2000: {}, 2005: {"MAIZ": 100.0, UNATTRIBUTED_0: 100.0}}, + 2000, + 2005, + {Crop.MAIZE: 0.5}, + id="the-lump-survives-the-2000-release-boundary", + ), + ), +) +def test_get_crop_to_share( + areas: dict[int, dict[str, float]], + before: int, + after: int, + expected: dict[Crop, float], +) -> None: + dset = get_dset_for_areas(areas=areas) + # Every case asks for all of `Crop`, so each one also checks the invariant that the shares + # never over-attribute the cell + shares = get_crop_to_share(after=after, before=before, crops=tuple(Crop), dset=dset) + assert sum(float(share) for share in shares.values()) <= 1.0 + assert {crop: float(shares[crop]) for crop in expected} == pytest.approx(expected) + # Shares are absolute: asking for fewer crops must not renormalise onto the ones asked for + subset = get_crop_to_share( + after=after, before=before, crops=tuple(expected), dset=dset ) - shares = get_crop_to_share( - after=2010, before=2005, crops=(Crop.MAIZE, Crop.SOYBEAN), dset=dset + assert {crop: float(subset[crop]) for crop in expected} == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("areas", "year", "expected"), + ( + pytest.param( + {2020: {"MAIZ": 100.0, "SOYB": 50.0, OTHER_2020: 50.0}}, + 2020, + {Crop.MAIZE: 0.5, Crop.SOYBEAN: 0.25}, + id="each-crop-takes-its-own-area-over-the-total-occupied", + ), + pytest.param( + {2020: {}}, + 2020, + {Crop.MAIZE: 0.0}, + id="a-cell-no-crop-occupies-is-zero-not-nan", + ), + ), +) +def test_get_crop_to_area_share( + areas: dict[int, dict[str, float]], year: int, expected: dict[Crop, float] +) -> None: + dset = get_dset_for_areas(areas=areas) + shares = get_crop_to_area_share(crops=tuple(Crop), dset=dset, year=year) + assert sum(float(share) for share in shares.values()) <= 1.0 + assert {crop: float(shares[crop]) for crop in expected} == pytest.approx(expected) + # Shares are absolute: asking for fewer crops must not renormalise onto the ones asked for + subset = get_crop_to_area_share(crops=tuple(expected), dset=dset, year=year) + assert {crop: float(subset[crop]) for crop in expected} == pytest.approx(expected) + + +def test_get_crop_to_area_share_survives_zero_expansion() -> None: + # The regression this function exists for: long-established cropland whose area has not + # changed over the window. Expansion share drops it entirely (and with it 100% of the + # cell's peatland occupation emissions); area share still allocates it. + areas = {"MAIZ": 100.0, "SOYB": 100.0} + dset = get_dset_for_areas(areas={2000: areas, 2020: areas}) + expansion_shares = get_crop_to_share( + after=2020, before=2000, crops=(Crop.MAIZE,), dset=dset ) - assert float(shares[Crop.MAIZE]) == 0.5 # 100 / 200 - assert float(shares[Crop.SOYBEAN]) == 0.25 # 50 / 200 - # target shares sum to <= 1; the residual crop's 0.25 stays unattributed - assert float(shares[Crop.MAIZE]) + float(shares[Crop.SOYBEAN]) == 0.75 + assert float(expansion_shares[Crop.MAIZE]) == 0.0 + area_shares = get_crop_to_area_share(crops=(Crop.MAIZE,), dset=dset, year=2020) + assert float(area_shares[Crop.MAIZE]) == 0.5 def test_get_crop_to_share_is_reclassification_invariant() -> None: - # Same total residual expansion (+50), split across the residual bands differently. - lumped = get_dset_for_areas({2005: {}, 2010: {"MAIZ": 100.0, OTHER_0: 50.0}}) + # The same +50 ha of non-MAIZE expansion, spread over one band and then over two. MAIZE's + # share depends on how much other cropland expanded, never on how MapSPAM chose to file it. + # ("Residual" is avoided here: in `ifpri_mapspam` it means a group's catch-all constituent.) + lumped = get_dset_for_areas(areas={2005: {}, 2010: {"MAIZ": 100.0, OTHER_0: 50.0}}) split = get_dset_for_areas( - {2005: {}, 2010: {"MAIZ": 100.0, OTHER_0: 20.0, OTHER_1: 30.0}} + areas={2005: {}, 2010: {"MAIZ": 100.0, OTHER_0: 20.0, OTHER_1: 30.0}} ) shares_lumped = get_crop_to_share( after=2010, before=2005, crops=(Crop.MAIZE,), dset=lumped @@ -76,151 +252,266 @@ def test_get_crop_to_share_is_reclassification_invariant() -> None: assert float(shares_lumped[Crop.MAIZE]) == float(shares_split[Crop.MAIZE]) -def test_get_crop_to_share_clips_contraction() -> None: - # MAIZE contracts (-100); only SOYB expands -> MAIZE share is 0, not negative. - dset = get_dset_for_areas( - {2005: {"MAIZ": 100.0}, 2010: {"MAIZ": 0.0, "SOYB": 50.0}} - ) - shares = get_crop_to_share( - after=2010, before=2005, crops=(Crop.MAIZE, Crop.SOYBEAN), dset=dset - ) - assert float(shares[Crop.MAIZE]) == 0.0 - assert float(shares[Crop.SOYBEAN]) == 1.0 # 50 / 50 - - -def test_get_crop_to_share_keeps_missings() -> None: - # Nothing expands anywhere -> total is masked to NaN, so shares are NaN (not 0/0 errors). - dset = get_dset_for_areas({2005: {}, 2010: {}}) - shares = get_crop_to_share(after=2010, before=2005, crops=(Crop.MAIZE,), dset=dset) - assert shares[Crop.MAIZE] == 0 - - def test_get_crop_to_share_tolerates_nodata_absent_crops() -> None: - dset = get_dset_for_areas({2005: {}, 2010: {"MAIZ": 100.0}}) - dset[get_band_name_for_crop_name(crop_name=OTHER_0, quantity=AREA, year=2010)] = ( - xarray.DataArray(float("nan")) - ) + # Real rasters carry nodata, not zero, where a crop is absent. That must not poison the + # denominator with NaN and take the whole cell down with it. + dset = get_dset_for_areas(areas={2005: {}, 2010: {"MAIZ": 100.0}}) + dset[ + ifpri_mapspam.get_band_name( + quantity=AREA, reported_crop_name=OTHER_0, year=2010 + ) + ] = xarray.DataArray(float("nan")) shares = get_crop_to_share(after=2010, before=2005, crops=(Crop.MAIZE,), dset=dset) assert float(shares[Crop.MAIZE]) == 1.0 # 100 / 100; NaN absent crop counts as 0 ha -def test_get_crop_to_share_decomposes_2000_group_by_reference_fractions() -> None: - # 2000 only has the coarse BANP group (100 ha). 2005 splits it 30/10, so BANA - # gets 75% -> area(BANA, 2000) == 75. BANA grows to 175 (+100), PLNT flat at 25. - dset = get_dset_for_areas( - { - 2000: {"BANP": 100.0}, - 2005: {"BANA": 30.0, "PLNT": 10.0}, - 2020: {"BANA": 175.0, "PLNT": 25.0}, - } - ) - shares = get_crop_to_share( - after=2020, before=2000, crops=(Crop.BANANA, Crop.PLANTAIN), dset=dset - ) - assert float(shares[Crop.BANANA]) == 1.0 # +100 over residual (200 - 100) - assert float(shares[Crop.PLANTAIN]) == 0.0 # decomposed 25 -> 25, no expansion - - -def test_get_crop_to_share_decomposes_2000_group_by_equal_split_when_reference_absent() -> ( - None -): - # 2000 has the coarse BANP group (100 ha) but 2005 has neither constituent, - # so denom == 0 and the split falls back to 1/len(siblings) == 0.5 each: - # area(BANA, 2000) == area(PLNT, 2000) == 50. - dset = get_dset_for_areas( - { - 2000: {"BANP": 100.0}, - 2005: {}, # both BANA and PLNT absent -> reference denom is 0 - 2020: {"BANA": 80.0, "PLNT": 120.0}, - } - ) - shares = get_crop_to_share( - after=2020, before=2000, crops=(Crop.BANANA, Crop.PLANTAIN), dset=dset - ) - # BANA: 80 - 50 = +30 ; PLNT: 120 - 50 = +70 ; residual total expansion == 100 - assert float(shares[Crop.BANANA]) == 0.30 - assert float(shares[Crop.PLANTAIN]) == 0.70 - - def test_get_crop_to_share_decomposition_does_not_warn_on_zero_reference() -> None: - dset = get_dset_for_areas( - {2000: {"BANP": 100.0}, 2005: {}, 2020: {"BANA": 80.0, "PLNT": 120.0}} - ) + # No reference year places BANP, so the within-group split divides by a zero pool. The + # decomposition guards that denominator rather than dividing and discarding the NaN, which + # would work but bury every run in RuntimeWarnings. + dset = get_dset_for_areas(areas={2000: {"BANP": 100.0}}) with warnings.catch_warnings(): - warnings.simplefilter( - "error", RuntimeWarning - ) # any 0/0 divide becomes a failure + # Any 0/0 divide becomes a failure + warnings.simplefilter("error", RuntimeWarning) get_crop_to_share( after=2020, before=2000, crops=(Crop.BANANA, Crop.PLANTAIN), dset=dset ) -def test_get_harmonized_quantity_reads_snapshot() -> None: - # Shared crop, no decomposition -> a straight per-year production lookup. - dset = get_dset({(PRODUCTION, 2020): {"MAIZ": 42.0}}) - got = get_harmonized_quantity( - crop_name="MAIZ", dset=dset, quantity=PRODUCTION, year=2020 +@pytest.mark.parametrize( + ("values", "quantity", "year", "expected"), + ( + pytest.param( + {(PRODUCTION, 2020): {"MAIZ": 42.0}}, + PRODUCTION, + 2020, + {"MAIZ": 42.0}, + id="a-shared-crop-is-a-straight-per-year-lookup", + ), + # The next two share a dset whose 2005 area and production distributions disagree: + # plantain holds a quarter of the area but only a tenth of the production, i.e. it + # yields less. Each quantity must therefore be split by its own distribution. + pytest.param( + { + (AREA, 2005): {"BANA": 30.0, "PLNT": 10.0}, + (PRODUCTION, 2005): {"BANA": 180.0, "PLNT": 20.0}, + (AREA, 2000): {"BANP": 400.0}, + (PRODUCTION, 2000): {"BANP": 200.0}, + }, + AREA, + 2000, + # 400 ha of BANP split 3:1, the 2005 area ratio + {"BANA": 300.0, "PLNT": 100.0}, + id="area-splits-by-the-area-distribution", + ), + pytest.param( + { + (AREA, 2005): {"BANA": 30.0, "PLNT": 10.0}, + (PRODUCTION, 2005): {"BANA": 180.0, "PLNT": 20.0}, + (AREA, 2000): {"BANP": 400.0}, + (PRODUCTION, 2000): {"BANP": 200.0}, + }, + PRODUCTION, + 2000, + # 200 t of BANP split 9:1, the 2005 production ratio. Inheriting the area split + # above would have handed plantain 50 t instead of 20 t. + {"BANA": 180.0, "PLNT": 20.0}, + id="production-splits-by-its-own-distribution-not-the-area-one", + ), + pytest.param( + # All three reference years contribute and they disagree; pooling gives BANA a fifth + # where 2005 alone would have given it 150 t. + { + (PRODUCTION, 2005): {"BANA": 30.0, "PLNT": 10.0}, + (PRODUCTION, 2010): {"BANA": 5.0, "PLNT": 75.0}, + (PRODUCTION, 2020): {"BANA": 5.0, "PLNT": 75.0}, + (PRODUCTION, 2000): {"BANP": 200.0}, + }, + PRODUCTION, + 2000, + {"BANA": 40.0, "PLNT": 160.0}, + id="every-reference-year-pools-into-one-split", + ), + pytest.param( + # 2005 and 2010 are empty, so 2020 alone decides, and it names only bananas: BANP's + # whole 200 t goes to BANA. An even split would have fabricated 100 t of plantain + # in a pixel MapSPAM says grows nothing but bananas. + {(PRODUCTION, 2020): {"BANA": 40.0}, (PRODUCTION, 2000): {"BANP": 200.0}}, + PRODUCTION, + 2000, + {"BANA": 200.0, "PLNT": 0.0}, + id="a-later-reference-carries-the-split-when-2005-is-empty", + ), + pytest.param( + # No reference year places BANP here and it has no catch-all constituent to route + # the remainder to, so an even split is the only option left. This is the fallback + # the two OOIL cases below exist to avoid wherever a catch-all does exist. + {(PRODUCTION, 2000): {"BANP": 200.0}}, + PRODUCTION, + 2000, + {"BANA": 100.0, "PLNT": 100.0}, + id="an-even-split-is-the-last-resort-for-groups-with-no-catch-all", + ), + pytest.param( + # A catch-all changes what happens with NO evidence, not how evidence is used: the + # reference years pool the same way for every group. + { + (PRODUCTION, 2005): {}, + (PRODUCTION, 2020): {"RAPE": 300.0, "SUNF": 100.0}, + (PRODUCTION, 2000): {"OOIL": 400.0}, + }, + PRODUCTION, + 2000, + { + "RAPE": 300.0, + "SUNF": 100.0, + "OOIL": 0.0, + "CNUT": 0.0, + "OILP": 0.0, + "SESA": 0.0, + }, + id="naming-a-crop-in-any-reference-year-beats-the-catch-all", + ), + pytest.param( + # With no area for any constituent in ANY reference year there is no basis for + # dividing the group. Splitting evenly instead put a sixth of Canada's "Other Oil + # Crops" under oil palm and another sixth under coconut, neither of which Canada grows. + {(AREA, 2005): {}, (PRODUCTION, 2000): {"OOIL": 600.0}}, + PRODUCTION, + 2000, + { + "OOIL": 600.0, + "CNUT": 0.0, + "OILP": 0.0, + "RAPE": 0.0, + "SESA": 0.0, + "SUNF": 0.0, + }, + id="the-catch-all-takes-the-whole-group-when-no-reference-places-it", + ), + pytest.param( + # The catch-all is only a fallback for having no evidence at all. Here 2005 does + # place the group, so the proportional split wins outright and OOIL-the-constituent + # gets nothing -- despite OOIL also being the group's catch-all. + { + (PRODUCTION, 2005): {"OILP": 300.0, "CNUT": 100.0}, + (PRODUCTION, 2000): {"OOIL": 400.0}, + }, + PRODUCTION, + 2000, + {"OILP": 300.0, "CNUT": 100.0, "OOIL": 0.0}, + id="reference-shares-win-whenever-a-reference-year-places-the-group", + ), + ), +) +def test_get_canonical_quantity( + values: dict[tuple[ifpri_mapspam.Quantity, int], dict[str, float]], + quantity: ifpri_mapspam.Quantity, + year: int, + expected: dict[str, float], +) -> None: + dset = get_dset(values=values) + assert { + canonical_crop_name: float( + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=canonical_crop_name, + dset=dset, + quantity=quantity, + year=year, + ) + ) + for canonical_crop_name in expected + } == pytest.approx(expected) + + +@pytest.mark.parametrize("group_name", sorted(ifpri_mapspam.GROUP_TO_CONSTITUENT_NAMES)) +def test_get_canonical_quantity_conserves_the_group_total(group_name: str) -> None: + # Whichever fallback applies, the constituents must account for the whole group: an absent + # reference year is a question of WHICH crop receives the production, never how much. + total = 900.0 + dset = get_dset(values={(AREA, 2005): {}, (PRODUCTION, 2000): {group_name: total}}) + got = sum( + float( + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=constituent, + dset=dset, + quantity=PRODUCTION, + year=2000, + ) + ) + for constituent in sorted(ifpri_mapspam.GROUP_TO_CONSTITUENT_NAMES[group_name]) ) - assert float(got) == 42.0 + assert got == pytest.approx(total) -def test_get_harmonized_quantity_splits_group_production_by_area_share() -> None: - # 2000 only has the coarse BANP group (200 t production). 2005 areas split - # BANA/PLNT 30/10, so the split uses the *area* share (0.75 / 0.25), not a - # production share: BANA -> 200 x 0.75 == 150, PLNT -> 200 x 0.25 == 50. +# Each snapshot's share of the reduction is 3.125% for 2000, 12.5% for 2005, 46.875% for 2010 +# and 37.5% for 2020. The expected totals below are written out rather than recomputed from +# those: a test that redoes the weighting can agree with a wrong implementation. +PIXELS = 4 + + +def get_totals( + area_by_year: dict[int, float], production_by_year: dict[int, float] +) -> dict[str, float]: + """`get_crop_name_to_totals` over one crop on a 2x2 grid. + + The emissions and peatland bands it also reads are zero throughout, and so are the shares: + only the area and production columns are under test here. + """ + crop = Crop.WHEAT + zero = xarray.DataArray(0.0) dset = get_dset( - { - (AREA, 2005): {"BANA": 30.0, "PLNT": 10.0}, - (PRODUCTION, 2000): {"BANP": 200.0}, + values={(AREA, year): {crop.value: area} for year, area in area_by_year.items()} + | { + (PRODUCTION, year): {crop.value: production} + for year, production in production_by_year.items() } + ).expand_dims({"y": [0.1, 0.2], "x": [0.1, 0.2]}) + # `get_hectares_per_pixel` reads the peatland band's own coords, so these carry the grid + grid = xarray.zeros_like(other=dset[next(iter(dset.data_vars))]) + return get_crop_name_to_totals( + crop_to_span_to_share={ + crop: dict.fromkeys(GLAD_TO_MAPSPAM_SPAN.values(), zero) + }, + dset=dset.assign( + { + gfw_global_peatlands.DATASET.fully_qualified_band_name: grid, + "peatland-occupation:tco2e-per-ha": grid, + } + | { + f"{source:s}:tco2e-per-ha:{before:d}-{after:d}": grid + for source in ("emissions", "forest", "peatland_conversion") + for before, after in emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT + } + ), + occupation_shares={crop: zero}, + )[crop.name] + + +def test_crop_hectares_is_the_discounted_mean_of_the_snapshots() -> None: + totals = get_totals( + area_by_year={2000: 1.0, 2005: 10.0, 2010: 100.0, 2020: 1000.0}, + production_by_year={}, ) - banana = get_harmonized_quantity( - crop_name="BANA", dset=dset, quantity=PRODUCTION, year=2000 - ) - plantain = get_harmonized_quantity( - crop_name="PLNT", dset=dset, quantity=PRODUCTION, year=2000 + assert totals["crop_hectares"] == pytest.approx(PIXELS * 423.15625) + + +def test_crop_hectares_survives_a_crop_abandoned_before_2020() -> None: + # The defect this replaced: no area in the final snapshot against twenty years of + # production, which made every per-hectare figure for the crop meaningless. + totals = get_totals( + area_by_year={2000: 100.0, 2005: 100.0}, production_by_year={2000: 1.0} ) - assert float(banana) == 150.0 - assert float(plantain) == 50.0 + assert totals["crop_hectares"] == pytest.approx(PIXELS * 15.625) -@pytest.mark.parametrize( - ("variable_name", "expected"), - ( - ("gfw:global-peatlands:is-peatland", BandType.CATEGORICAL), - ("gfw:harris-agb:aboveground-biomass-mg-per-ha", BandType.INTENSIVE), - ("glad:glcluc:year=2000", BandType.CATEGORICAL), - ("glad:glcluc:year=2005", BandType.CATEGORICAL), - ("glad:glcluc:year=2010", BandType.CATEGORICAL), - ("glad:glcluc:year=2015", BandType.CATEGORICAL), - ("glad:glcluc:year=2020", BandType.CATEGORICAL), - ("huang:bgb:belowground-biomass-mg-per-ha", BandType.INTENSIVE), - ("ipcc:climate-zones:climate-zone", BandType.CATEGORICAL), - ( - "soilgrids:organic-carbon-stocks:organic-soil-carbon-mg-per-ha", - BandType.INTENSIVE, - ), - ("land-class:2000", BandType.CATEGORICAL), - ("land-class:2005", BandType.CATEGORICAL), - ("land-class:2010", BandType.CATEGORICAL), - ("land-class:2015", BandType.CATEGORICAL), - ("land-class:2020", BandType.CATEGORICAL), - ("vegetation-emissions:tco2e-per-ha:2000-2005", BandType.INTENSIVE), - ("vegetation-emissions:tco2e-per-ha:2005-2010", BandType.INTENSIVE), - ("vegetation-emissions:tco2e-per-ha:2010-2015", BandType.INTENSIVE), - ("vegetation-emissions:tco2e-per-ha:2015-2020", BandType.INTENSIVE), - ("soil-emissions:tco2e-per-ha:2000-2005", BandType.INTENSIVE), - ("soil-emissions:tco2e-per-ha:2005-2010", BandType.INTENSIVE), - ("soil-emissions:tco2e-per-ha:2010-2015", BandType.INTENSIVE), - ("soil-emissions:tco2e-per-ha:2015-2020", BandType.INTENSIVE), - ("emissions:tco2e-per-ha:2000-2005", BandType.INTENSIVE), - ("emissions:tco2e-per-ha:2005-2010", BandType.INTENSIVE), - ("emissions:tco2e-per-ha:2010-2015", BandType.INTENSIVE), - ("emissions:tco2e-per-ha:2015-2020", BandType.INTENSIVE), - ("peatland-occupation:tco2e-per-ha", BandType.INTENSIVE), - ("emissions-per-hectare:tco2e-per-ha", BandType.INTENSIVE), - ("hectares-per-pixel:ha", BandType.EXTENSIVE), - ), -) -def test_get_downsampling_for_variable_name(variable_name: str, expected: str) -> None: - assert get_band_type_for_variable_name(variable_name=variable_name) == expected +def test_a_constant_yield_survives_the_window() -> None: + # What the shared window buys: area and production reduce over the same spans with the same + # weights, so a crop whose yield never changes reports exactly that yield -- whatever its + # area did in between, and whatever weights the spans carry. + areas = {2000: 1.0, 2005: 50.0, 2010: 7.0, 2020: 0.0} + totals = get_totals( + area_by_year=areas, + production_by_year={year: area * 3.0 for year, area in areas.items()}, + ) + assert totals["production_mt"] / totals["crop_hectares"] == pytest.approx(3.0) diff --git a/jdluc/__tests__/tiling_test.py b/jdluc/__tests__/tiling_test.py index c18d71a..c9310c5 100644 --- a/jdluc/__tests__/tiling_test.py +++ b/jdluc/__tests__/tiling_test.py @@ -2,8 +2,10 @@ import shapely from jdluc.tiling import ( + GLOBAL_FOREST_WATCH_TILE_IDS, PARTITIONING_TO_IS_VALID_TILE_ID, Partitioning, + get_box_for_tile_id, get_lat_lon_for_tile_id, get_tile_id_for_lat_lon, iter_ten_degree_tile_id_for_geometry, @@ -59,3 +61,20 @@ def test_iter_ten_degree_tile_id_for_geometry( geometry: shapely.Polygon | shapely.MultiPolygon, expected: tuple[str, ...] ) -> None: assert tuple(iter_ten_degree_tile_id_for_geometry(geometry=geometry)) == expected + + +def test_get_box_for_tile_id_uses_the_northern_lat() -> None: + assert get_box_for_tile_id(tile_id="50N_080W").equals(BOX_40N_080W) + + +@pytest.mark.parametrize("tile_id", GLOBAL_FOREST_WATCH_TILE_IDS) +def test_get_box_for_tile_id_round_trips(tile_id: str) -> None: + # These two functions must agree on the northern-lat convention, because + # worldbank_jurisdictions.iter_jurisdiction_for_iso_3166_tile_id skips a province whose + # geometry misses this box. Were the conventions to drift by one tile, a province could be + # skipped for every tile it belongs to and disappear from the rollup entirely. + assert tuple( + iter_ten_degree_tile_id_for_geometry( + geometry=get_box_for_tile_id(tile_id=tile_id) + ) + ) == (tile_id,) diff --git a/jdluc/__tests__/trace_test.py b/jdluc/__tests__/trace_test.py index b73c350..50d3760 100644 --- a/jdluc/__tests__/trace_test.py +++ b/jdluc/__tests__/trace_test.py @@ -68,12 +68,12 @@ "forest_emissions_mt", "peatland_conversion_emissions_mt", ], -).set_index(["admin_level", "crop_name", "jurisdiction_name", "methodology"]) +).set_index(["admin_level", "admin_id", "crop_name", "methodology"]) RAW_YIELDS = pandas.DataFrame.from_records( [ - # CORN: 2016 must be excluded; mean(8, 10, 12, 10) over 2017-2020 == 10 + # MAIZE: 2016 must be excluded; mean(8, 10, 12, 10) over 2017-2020 == 10 *( - ("PROVINCIAL", "USA008", "Delaware", "CORN", year, val) + ("PROVINCIAL", "USA008", "Delaware", "MAIZE", year, val) for year, val in ( (2016, 999.0), (2017, 8.0), @@ -82,9 +82,9 @@ (2020, 10.0), ) ), - # SOYBEANS: mean == 30 + # SOYBEAN: mean == 30 *( - ("PROVINCIAL", "USA008", "Delaware", "SOYBEANS", year, 30.0) + ("PROVINCIAL", "USA008", "Delaware", "SOYBEAN", year, 30.0) for year in (2017, 2018, 2019, 2020) ), # (USA016, WHEAT): deliberately absent -> unmatched @@ -110,7 +110,7 @@ def test_merge_jurisdictional_emissions_and_yields() -> None: iter_result = result.iterrows() key, corn = next(iter_result) - assert key == ("PROVINCIAL", "MAIZE", "Delaware", "JURISDICTIONAL_DIRECT") + assert key == ("PROVINCIAL", "USA008", "MAIZE", "JURISDICTIONAL_DIRECT") assert corn["yield_kg_per_ha"] == 10.0 # 4-year mean, 2016 excluded assert corn["production_kg"] == 1000.0 # 100 ha x 10 assert corn["emissions_factor_kgco2e_per_kg"] == 200.0 # 200 t x 1000 / 1000 kg @@ -118,14 +118,14 @@ def test_merge_jurisdictional_emissions_and_yields() -> None: # zero production -> EF guarded to NaN (not inf); fraction still defined (0/80) key, soy = next(iter_result) - assert key == ("PROVINCIAL", "SOYBEAN", "Delaware", "JURISDICTIONAL_DIRECT") + assert key == ("PROVINCIAL", "USA008", "SOYBEAN", "JURISDICTIONAL_DIRECT") assert soy["production_kg"] == 0.0 assert numpy.isnan(soy["emissions_factor_kgco2e_per_kg"]) assert soy["peatland_occupation_fraction"] == 0.0 # unmatched yield -> NaN yield/production/EF; zero emissions_mt -> NaN fraction key, wheat = next(iter_result) - assert key == ("PROVINCIAL", "WHEAT", "Iowa", "JURISDICTIONAL_DIRECT") + assert key == ("PROVINCIAL", "USA016", "WHEAT", "JURISDICTIONAL_DIRECT") assert numpy.isnan(wheat["yield_kg_per_ha"]) assert numpy.isnan(wheat["production_kg"]) assert numpy.isnan(wheat["emissions_factor_kgco2e_per_kg"]) @@ -283,7 +283,7 @@ def test_iter_national_from_provincials_multi_province_crop() -> None: "forest_emissions_mt", "peatland_conversion_emissions_mt", ], -).set_index(["admin_level", "crop_name", "jurisdiction_name", "methodology"]) +).set_index(["admin_level", "admin_id", "crop_name", "methodology"]) def test_derive_statistical_production_kg_stays_indexed() -> None: @@ -291,8 +291,8 @@ def test_derive_statistical_production_kg_stays_indexed() -> None: # regression guard: must return an indexed frame like its jurisdictional twin assert result.index.names == [ "admin_level", + "admin_id", "crop_name", - "jurisdiction_name", "methodology", ] assert "production_mt" not in result.columns diff --git a/jdluc/attribute.py b/jdluc/attribute.py index 467bec0..0842449 100644 --- a/jdluc/attribute.py +++ b/jdluc/attribute.py @@ -1,88 +1,142 @@ """Roll up per-pixel emissions to per-(jurisdiction, crop) totals. -Dispatches each ISO 3166 country to a per-methodology workflow (clustering countries by -continent so they share tile sets): +Dispatches each ten-degree tile to a per-methodology workflow once per overlapping ISO +3166 country and sums the partials, since a jurisdiction may straddle a tile boundary: - STATISTICAL (default): downscales the per-pixel emissions to the MAPSPAM grid and attributes them across crops by MAPSPAM crop-expansion shares. - JURISDICTIONAL_DIRECT: masks the per-pixel emissions to each crop's CDL codes. Both clip to provincial (World Bank admin-1) polygons, restrict to GLAD 2020 cropland unless `--skip-glad-crop-filter` is set, and sum crop area, peatland crop area, peatland-occupation emissions, and total emissions. Returns a pandas.DataFrame indexed by -(admin level, crop, jurisdiction); the per-country sub-workflows are cached. +(admin_level, admin_id, crop_name, methodology); the per-(country, tile) sub-workflows are +cached and `--concurrency` bounds how many tiles are in flight at once. -Example invocation: +The countries come from the positional ISO 3166 alpha-3 codes, or -- with `--backfill` -- +from every country in the World Bank admin-0 layer. + +Example invocations: uv run python jdluc/attribute.py --methodology-name STATISTICAL USA + uv run python jdluc/attribute.py --methodology-name STATISTICAL --backfill """ import argparse import collections.abc +import concurrent.futures import enum import logging import pandas from jdluc import ( - continents, jurisdictional_direct, statistical, ) +from jdluc.datasets import worldbank_jurisdictions logger = logging.getLogger(__name__) -def iso_3166_str(s: str) -> str: - assert len(s) == 3, f"iso_3166 code {s=:s} must be three characters" - return s - - class Methodology(enum.IntEnum): JURISDICTIONAL_DIRECT = enum.auto() STATISTICAL = enum.auto() +def merge_dfs(*dfs: pandas.DataFrame) -> pandas.DataFrame: + combined = pandas.concat(list(dfs)) + jurisdiction_name = ( + combined["jurisdiction_name"].groupby(level=combined.index.names).first() + ) + return ( + combined.select_dtypes("number") + .groupby(level=combined.index.names) + .sum() + .assign(jurisdiction_name=jurisdiction_name) + ) + + def workflow( + concurrency: int, crop_names: tuple[str, ...], - iso_3166s: collections.abc.Sequence[str], + iso_3166s: collections.abc.Iterable[str], methodology: Methodology, skip_glad_crop_filter: bool, ) -> pandas.DataFrame: - workflow_for_tile_ids = ( + workflow_for_tile = ( jurisdictional_direct.workflow if methodology == Methodology.JURISDICTIONAL_DIRECT else statistical.workflow ) - dfs = ( - workflow_for_tile_ids( - crop_names=crop_names, - iso_3166=iso_3166, - skip_glad_crop_filter=skip_glad_crop_filter, - tile_ids=tuple(sorted(tile_cluster)), - ) - for tile_cluster, iso_3166_cluster in continents.iter_tile_cluster_to_iso_3166s( - iso_3166s=iso_3166s - ) - for iso_3166 in sorted(iso_3166_cluster) - ) - return ( - pandas.concat(list(dfs)) + + # NB: have each thread handle a tile, to avoid races in harmonize, emit, or downscale + tile_id_to_iso_3166s: dict[str, list[str]] = collections.defaultdict(list) + for iso_3166 in sorted(iso_3166s): + for tile_id in sorted( + worldbank_jurisdictions.get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL, + ) + ): + tile_id_to_iso_3166s[tile_id].append(iso_3166) + + def workflow_for_tile_id(tile_id: str) -> list[pandas.DataFrame]: + return [ + workflow_for_tile( + crop_names=crop_names, + iso_3166=iso_3166, + skip_glad_crop_filter=skip_glad_crop_filter, + tile_id=tile_id, + ) + for iso_3166 in tile_id_to_iso_3166s[tile_id] + ] + + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [ + pool.submit(workflow_for_tile_id, tile_id=tile_id) + for tile_id in sorted(tile_id_to_iso_3166s) + ] + dfs = [df for future in futures for df in future.result()] + + # NB: ensure empty jurisdictions aren't dropped + all_jurisdiction_crops = pandas.DataFrame.from_records( + data=[ + { + "admin_level": jurisdiction.level, + "admin_id": jurisdiction.id, + "crop_name": crop_name, + "jurisdiction_name": jurisdiction.name, + } + for tile_id, iso_3166s_for_tile in sorted(tile_id_to_iso_3166s.items()) + for iso_3166 in iso_3166s_for_tile + for jurisdiction in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166_tile_id( + admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL, + iso_3166=iso_3166, + tile_id=tile_id, + ) + for crop_name in crop_names + ] + ).set_index(keys=["admin_level", "admin_id", "crop_name"]) + + ret = ( + merge_dfs(all_jurisdiction_crops, *dfs) .assign(methodology=methodology.name) .set_index("methodology", append=True) .sort_index() ) + assert not ret.index.duplicated().any() + return ret -def get_crop_names( - methodology: Methodology, process_few_crops: bool -) -> tuple[str, ...]: - if process_few_crops: - return ("MAIZE", "SOYBEAN", "WHEAT") - elif methodology == Methodology.STATISTICAL: +def get_crop_names(methodology: Methodology) -> tuple[str, ...]: + if methodology == Methodology.STATISTICAL: return tuple(sorted(c.name for c in statistical.Crop)) else: assert methodology == Methodology.JURISDICTIONAL_DIRECT return tuple(sorted(c.name for c in jurisdictional_direct.Crop)) +DEFAULT_CONCURRENCY = 6 + + def main() -> int: logging.basicConfig( level=logging.INFO, @@ -90,23 +144,42 @@ def main() -> int: ) parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("iso_3166s", nargs=argparse.ONE_OR_MORE, type=iso_3166_str) + parser.add_argument( + "iso_3166s", + nargs=argparse.ZERO_OR_MORE, + type=worldbank_jurisdictions.iso_3166_str, + ) + parser.add_argument( + "--backfill", + action="store_true", + help="attribute every country in the World Bank admin-0 layer", + ) + parser.add_argument( + "--concurrency", + default=DEFAULT_CONCURRENCY, + type=int, + ) parser.add_argument( "--methodology-name", choices=sorted(e.name for e in Methodology), default=Methodology.STATISTICAL.name, ) - parser.add_argument("--process-few-crops", action="store_true") parser.add_argument("--skip-display", action="store_true") parser.add_argument("--skip-glad-crop-filter", action="store_true") args = parser.parse_args() + assert bool(args.iso_3166s) ^ bool(args.backfill), ( + "pass either one-or-more iso_3166s or --backfill" + ) methodology = Methodology[str(args.methodology_name)] df = workflow( - crop_names=get_crop_names( - methodology=methodology, process_few_crops=args.process_few_crops + concurrency=int(args.concurrency), + crop_names=get_crop_names(methodology=methodology), + iso_3166s=( + worldbank_jurisdictions.get_all_iso_3166s() + if args.backfill + else args.iso_3166s ), - iso_3166s=args.iso_3166s, methodology=methodology, skip_glad_crop_filter=args.skip_glad_crop_filter, ) diff --git a/jdluc/continents.py b/jdluc/continents.py deleted file mode 100644 index ae3239c..0000000 --- a/jdluc/continents.py +++ /dev/null @@ -1,591 +0,0 @@ -import collections.abc -import enum - - -class Continent(tuple[str, ...], enum.Enum): - AFRICA = ( - "00N_000E", - "00N_010E", - "00N_020E", - "00N_030E", - "00N_040E", - "10N_000E", - "10N_010E", - "10N_010W", - "10N_020E", - "10N_020W", - "10N_030E", - "10N_040E", - "10N_050E", - "10S_010E", - "10S_020E", - "10S_030E", - "10S_040E", - "10S_050E", - "20N_000E", - "20N_010E", - "20N_010W", - "20N_020E", - "20N_020W", - "20N_030E", - "20N_040E", - "20N_050E", - "20S_010E", - "20S_020E", - "20S_030E", - "20S_040E", - "20S_050E", - "30N_000E", - "30N_010E", - "30N_010W", - "30N_020E", - "30N_020W", - "30N_030E", - "30N_040E", - "30N_050E", - "30N_060E", - "30S_010E", - "30S_020E", - "30S_030E", - "40N_000E", - "40N_010E", - "40N_010W", - "40N_020E", - "40N_030E", - "40N_040E", - "40N_050E", - "40N_060E", - ) - ASIA = ( - "00N_070E", - "10N_070E", - "10N_080E", - "10N_090E", - "10N_100E", - "10N_110E", - "10N_120E", - "20N_070E", - "20N_080E", - "20N_090E", - "20N_100E", - "20N_110E", - "20N_120E", - "30N_060E", - "30N_070E", - "30N_080E", - "30N_090E", - "30N_100E", - "30N_110E", - "30N_120E", - "40N_050E", - "40N_060E", - "40N_070E", - "40N_080E", - "40N_090E", - "40N_100E", - "40N_110E", - "40N_120E", - "40N_130E", - "40N_140E", - "50N_040E", - "50N_050E", - "50N_060E", - "50N_070E", - "50N_080E", - "50N_090E", - "50N_100E", - "50N_110E", - "50N_120E", - "50N_130E", - "50N_140E", - "60N_040E", - "60N_050E", - "60N_060E", - "60N_070E", - "60N_080E", - "60N_090E", - "60N_100E", - "60N_110E", - "60N_120E", - ) - RUSSIA = ( - "50N_030E", - "50N_040E", - "50N_080E", - "50N_090E", - "50N_100E", - "50N_110E", - "50N_120E", - "50N_130E", - "50N_140E", - "50N_150E", - "60N_010E", - "60N_020E", - "60N_030E", - "60N_040E", - "60N_050E", - "60N_060E", - "60N_070E", - "60N_080E", - "60N_090E", - "60N_100E", - "60N_110E", - "60N_120E", - "60N_130E", - "60N_140E", - "60N_150E", - "60N_160E", - "60N_170E", - "70N_020E", - "70N_030E", - "70N_040E", - "70N_050E", - "70N_060E", - "70N_070E", - "70N_080E", - "70N_090E", - "70N_100E", - "70N_110E", - "70N_120E", - "70N_130E", - "70N_140E", - "70N_150E", - "70N_160E", - "70N_170E", - "80N_050E", - "80N_060E", - "80N_070E", - "80N_080E", - "80N_090E", - "80N_100E", - "80N_110E", - "80N_120E", - "80N_130E", - "80N_140E", - "80N_150E", - "80N_160E", - "80N_170E", - ) - EUROPE = ( - "20N_110W", - "30N_020W", - "40N_000E", - "40N_010E", - "40N_010W", - "40N_020E", - "40N_020W", - "40N_030E", - "40N_040E", - "50N_000E", - "50N_010E", - "50N_010W", - "50N_020E", - "50N_030E", - "50N_040E", - "50N_050E", - "60N_000E", - "60N_010E", - "60N_010W", - "60N_020E", - "60N_020W", - "60N_030E", - "70N_000E", - "70N_010E", - "70N_020E", - "70N_020W", - "70N_030E", - "70N_030W", - "80N_010E", - "80N_020E", - "80N_030E", - ) - NORTH_AMERICA = ( - "20N_080W", - "20N_090W", - "20N_100W", - "20N_110W", - "20N_120W", - "30N_080W", - "30N_090W", - "30N_100W", - "30N_110W", - "30N_120W", - "40N_070W", - "40N_080W", - "40N_090W", - "40N_100W", - "40N_110W", - "40N_120W", - "40N_130W", - "50N_060W", - "50N_070W", - "50N_080W", - "50N_090W", - "50N_100W", - "50N_110W", - "50N_120W", - "50N_130W", - "60N_060W", - "60N_070W", - "60N_080W", - "60N_090W", - "60N_100W", - "60N_110W", - "60N_120W", - "60N_130W", - "60N_140W", - "70N_070W", - "70N_080W", - "70N_090W", - "70N_100W", - "70N_110W", - "70N_120W", - "70N_130W", - "70N_140W", - "70N_150W", - "80N_070W", - "80N_080W", - "80N_090W", - "80N_100W", - "80N_110W", - "80N_120W", - "80N_130W", - "80N_140W", - ) - OCEANIA = ( - "00N_090E", - "00N_100E", - "00N_110E", - "00N_120E", - "00N_130E", - "00N_140E", - "00N_150E", - "00N_160E", - "10N_090E", - "10N_100E", - "10N_110E", - "10N_120E", - "10N_130E", - "10S_110E", - "10S_120E", - "10S_130E", - "10S_140E", - "10S_150E", - "10S_160E", - "10S_170E", - "20S_110E", - "20S_120E", - "20S_130E", - "20S_140E", - "20S_150E", - "20S_160E", - "30S_110E", - "30S_120E", - "30S_130E", - "30S_140E", - "30S_150E", - "30S_170E", - "40S_140E", - "40S_160E", - "40S_170E", - ) - SOUTH_AMERICA = ( - "00N_040W", - "00N_050W", - "00N_060W", - "00N_070W", - "00N_080W", - "00N_090W", - "00N_100W", - "10N_050W", - "10N_060W", - "10N_070W", - "10N_080W", - "10N_090W", - "10N_100W", - "10S_040W", - "10S_050W", - "10S_060W", - "10S_070W", - "10S_080W", - "20N_060W", - "20N_070W", - "20N_080W", - "20N_090W", - "20S_050W", - "20S_060W", - "20S_070W", - "20S_080W", - "30S_060W", - "30S_070W", - "30S_080W", - "40S_070W", - "40S_080W", - "50S_070W", - "50S_080W", - ) - UNCLASSIFIED = () - - -for continent in Continent: - assert sorted(continent.value) == list(continent.value) -ISO_3166_TO_CONTINENT = { - "ABW": Continent.SOUTH_AMERICA, - "AFG": Continent.ASIA, - "AGO": Continent.AFRICA, - "AIA": Continent.SOUTH_AMERICA, - "ALA": Continent.EUROPE, - "ALB": Continent.EUROPE, - "AND": Continent.EUROPE, - "ARE": Continent.AFRICA, - "ARG": Continent.SOUTH_AMERICA, - "ARM": Continent.EUROPE, - "ASM": Continent.OCEANIA, - "ATF": Continent.AFRICA, - "ATG": Continent.SOUTH_AMERICA, - "AUS": Continent.OCEANIA, - "AUT": Continent.EUROPE, - "AZE": Continent.EUROPE, - "BDI": Continent.AFRICA, - "BEL": Continent.EUROPE, - "BEN": Continent.AFRICA, - "BES": Continent.SOUTH_AMERICA, - "BFA": Continent.AFRICA, - "BGD": Continent.ASIA, - "BGR": Continent.EUROPE, - "BHR": Continent.AFRICA, - "BHS": Continent.NORTH_AMERICA, - "BIH": Continent.EUROPE, - "BLM": Continent.UNCLASSIFIED, - "BLR": Continent.EUROPE, - "BLZ": Continent.NORTH_AMERICA, - "BMU": Continent.NORTH_AMERICA, - "BOL": Continent.SOUTH_AMERICA, - "BRA": Continent.SOUTH_AMERICA, - "BRB": Continent.SOUTH_AMERICA, - "BRN": Continent.OCEANIA, - "BTN": Continent.ASIA, - "BVT": Continent.UNCLASSIFIED, - "BWA": Continent.AFRICA, - "CAF": Continent.AFRICA, - "CAN": Continent.NORTH_AMERICA, - "CCK": Continent.UNCLASSIFIED, - "CHE": Continent.EUROPE, - "CHL": Continent.SOUTH_AMERICA, - "CHN": Continent.ASIA, - "CIV": Continent.AFRICA, - "CMR": Continent.AFRICA, - "COD": Continent.AFRICA, - "COG": Continent.AFRICA, - "COK": Continent.UNCLASSIFIED, - "COL": Continent.SOUTH_AMERICA, - "COM": Continent.AFRICA, - "CPV": Continent.UNCLASSIFIED, - "CRI": Continent.SOUTH_AMERICA, - "CUB": Continent.NORTH_AMERICA, - "CUW": Continent.SOUTH_AMERICA, - "CXR": Continent.UNCLASSIFIED, - "CYM": Continent.SOUTH_AMERICA, - "CYP": Continent.AFRICA, - "CZE": Continent.EUROPE, - "DEU": Continent.EUROPE, - "DJI": Continent.AFRICA, - "DMA": Continent.SOUTH_AMERICA, - "DNK": Continent.EUROPE, - "DOM": Continent.SOUTH_AMERICA, - "DZA": Continent.AFRICA, - "ECU": Continent.SOUTH_AMERICA, - "EGY": Continent.AFRICA, - "ERI": Continent.AFRICA, - "ESP": Continent.EUROPE, - "EST": Continent.EUROPE, - "ETH": Continent.AFRICA, - "FIN": Continent.EUROPE, - "FJI": Continent.OCEANIA, - "FRA": Continent.EUROPE, - "FRO": Continent.UNCLASSIFIED, - "FSM": Continent.OCEANIA, - "GAB": Continent.AFRICA, - "GBR": Continent.EUROPE, - "GEO": Continent.ASIA, - "GGY": Continent.EUROPE, - "GHA": Continent.AFRICA, - "GIB": Continent.UNCLASSIFIED, - "GIN": Continent.AFRICA, - "GLP": Continent.SOUTH_AMERICA, - "GMB": Continent.AFRICA, - "GNB": Continent.AFRICA, - "GNQ": Continent.AFRICA, - "GRC": Continent.EUROPE, - "GRD": Continent.SOUTH_AMERICA, - "GRL": Continent.UNCLASSIFIED, - "GTM": Continent.NORTH_AMERICA, - "GUF": Continent.SOUTH_AMERICA, - "GUM": Continent.UNCLASSIFIED, - "GUY": Continent.SOUTH_AMERICA, - "HKG": Continent.ASIA, - "HMD": Continent.UNCLASSIFIED, - "HND": Continent.NORTH_AMERICA, - "HRV": Continent.EUROPE, - "HTI": Continent.NORTH_AMERICA, - "HUN": Continent.EUROPE, - "IDN": Continent.OCEANIA, - "IMN": Continent.EUROPE, - "IND": Continent.ASIA, - "IRL": Continent.EUROPE, - "IRN": Continent.AFRICA, - "IRQ": Continent.AFRICA, - "ISL": Continent.EUROPE, - "ISR": Continent.AFRICA, - "ITA": Continent.EUROPE, - "JAM": Continent.SOUTH_AMERICA, - "JEY": Continent.EUROPE, - "JOR": Continent.AFRICA, - "JPN": Continent.ASIA, - "KAZ": Continent.ASIA, - "KEN": Continent.AFRICA, - "KGZ": Continent.ASIA, - "KHM": Continent.ASIA, - "KIR": Continent.OCEANIA, - "KNA": Continent.SOUTH_AMERICA, - "KOR": Continent.ASIA, - "KWT": Continent.AFRICA, - "LAO": Continent.ASIA, - "LBN": Continent.AFRICA, - "LBR": Continent.AFRICA, - "LBY": Continent.AFRICA, - "LCA": Continent.SOUTH_AMERICA, - "LIE": Continent.EUROPE, - "LKA": Continent.ASIA, - "LSO": Continent.AFRICA, - "LTU": Continent.EUROPE, - "LUX": Continent.EUROPE, - "LVA": Continent.EUROPE, - "MAC": Continent.ASIA, - "MAF": Continent.UNCLASSIFIED, - "MAR": Continent.AFRICA, - "MCO": Continent.UNCLASSIFIED, - "MDA": Continent.EUROPE, - "MDG": Continent.AFRICA, - "MDV": Continent.ASIA, - "MEX": Continent.NORTH_AMERICA, - "MHL": Continent.UNCLASSIFIED, - "MKD": Continent.EUROPE, - "MLI": Continent.AFRICA, - "MLT": Continent.AFRICA, - "MMR": Continent.ASIA, - "MNE": Continent.EUROPE, - "MNG": Continent.ASIA, - "MNP": Continent.UNCLASSIFIED, - "MOZ": Continent.AFRICA, - "MRT": Continent.AFRICA, - "MSR": Continent.SOUTH_AMERICA, - "MTQ": Continent.SOUTH_AMERICA, - "MUS": Continent.AFRICA, - "MWI": Continent.AFRICA, - "MYS": Continent.OCEANIA, - "MYT": Continent.AFRICA, - "NAM": Continent.AFRICA, - "NCL": Continent.OCEANIA, - "NER": Continent.AFRICA, - "NFK": Continent.OCEANIA, - "NGA": Continent.AFRICA, - "NIC": Continent.NORTH_AMERICA, - "NIU": Continent.UNCLASSIFIED, - "NLD": Continent.EUROPE, - "NOR": Continent.EUROPE, - "NPL": Continent.ASIA, - "NRU": Continent.UNCLASSIFIED, - "NZL": Continent.OCEANIA, - "OMN": Continent.AFRICA, - "PAK": Continent.ASIA, - "PAN": Continent.SOUTH_AMERICA, - "PCN": Continent.UNCLASSIFIED, - "PER": Continent.SOUTH_AMERICA, - "PHL": Continent.ASIA, - "PLW": Continent.OCEANIA, - "PNG": Continent.OCEANIA, - "POL": Continent.EUROPE, - "PRI": Continent.SOUTH_AMERICA, - "PRK": Continent.ASIA, - "PRT": Continent.EUROPE, - "PRY": Continent.SOUTH_AMERICA, - "PSE": Continent.AFRICA, - "PYF": Continent.UNCLASSIFIED, - "QAT": Continent.AFRICA, - "REU": Continent.AFRICA, - "ROU": Continent.EUROPE, - "RUS": Continent.RUSSIA, - "RWA": Continent.AFRICA, - "SAU": Continent.AFRICA, - "SDN": Continent.AFRICA, - "SEN": Continent.AFRICA, - "SGP": Continent.OCEANIA, - "SHN": Continent.UNCLASSIFIED, - "SJM": Continent.EUROPE, - "SLB": Continent.OCEANIA, - "SLE": Continent.AFRICA, - "SLV": Continent.NORTH_AMERICA, - "SMR": Continent.EUROPE, - "SOM": Continent.AFRICA, - "SPM": Continent.NORTH_AMERICA, - "SRB": Continent.EUROPE, - "SSD": Continent.AFRICA, - "STP": Continent.AFRICA, - "SUR": Continent.SOUTH_AMERICA, - "SVK": Continent.EUROPE, - "SVN": Continent.EUROPE, - "SWE": Continent.EUROPE, - "SWZ": Continent.AFRICA, - "SXM": Continent.SOUTH_AMERICA, - "SYC": Continent.AFRICA, - "SYR": Continent.AFRICA, - "TCA": Continent.NORTH_AMERICA, - "TCD": Continent.AFRICA, - "TGO": Continent.AFRICA, - "THA": Continent.ASIA, - "TJK": Continent.ASIA, - "TKL": Continent.UNCLASSIFIED, - "TKM": Continent.ASIA, - "TLS": Continent.OCEANIA, - "TON": Continent.OCEANIA, - "TTO": Continent.SOUTH_AMERICA, - "TUN": Continent.AFRICA, - "TUR": Continent.EUROPE, - "TUV": Continent.UNCLASSIFIED, - "TZA": Continent.AFRICA, - "UGA": Continent.AFRICA, - "UKR": Continent.EUROPE, - "UMI": Continent.UNCLASSIFIED, - "URY": Continent.SOUTH_AMERICA, - "USA": Continent.NORTH_AMERICA, - "UZB": Continent.ASIA, - "VAT": Continent.UNCLASSIFIED, - "VCT": Continent.SOUTH_AMERICA, - "VEN": Continent.SOUTH_AMERICA, - "VGB": Continent.SOUTH_AMERICA, - "VIR": Continent.SOUTH_AMERICA, - "VNM": Continent.ASIA, - "VUT": Continent.OCEANIA, - "WLF": Continent.OCEANIA, - "WSM": Continent.OCEANIA, - "XKX": Continent.EUROPE, - "YEM": Continent.AFRICA, - "ZAF": Continent.AFRICA, - "ZMB": Continent.AFRICA, - "ZWE": Continent.AFRICA, -} -assert len(ISO_3166_TO_CONTINENT) == 244 -assert set(ISO_3166_TO_CONTINENT.values()) == set(Continent) - - -def iter_tile_cluster_to_iso_3166s( - iso_3166s: collections.abc.Sequence[str], -) -> collections.abc.Iterator[tuple[set[str], set[str]]]: - continent_to_iso_3166s: dict[Continent, set[str]] = collections.defaultdict( - set[str] - ) - for iso_3166 in iso_3166s: - continent_to_iso_3166s[ISO_3166_TO_CONTINENT[iso_3166]].add(iso_3166) - for continent, iso_3166_cluster in continent_to_iso_3166s.items(): - yield set(continent.value), iso_3166_cluster diff --git a/jdluc/datasets/__init__.py b/jdluc/datasets/__init__.py index 19c0a7c..0162999 100644 --- a/jdluc/datasets/__init__.py +++ b/jdluc/datasets/__init__.py @@ -2,6 +2,7 @@ from jdluc.datasets import ( base, + faostat_production, gfw_global_peatlands, gfw_harris_agb, gfw_tcl, @@ -24,6 +25,7 @@ def _generate_next_value_( ) -> str: return name + FAOSTAT_PRODUCTION = enum.auto() GFW_GLOBAL_PEATLANDS = enum.auto() GFW_HARRIS_AGB = enum.auto() GFW_TREE_COVER_LOSS = enum.auto() @@ -49,6 +51,7 @@ def _generate_next_value_( NAME_TO_CLS: dict[ DatasetName, base.RasterDataset | base.TabularDataset | base.VectorDataset ] = { + DatasetName.FAOSTAT_PRODUCTION: faostat_production.DATASET, DatasetName.GFW_GLOBAL_PEATLANDS: gfw_global_peatlands.DATASET, DatasetName.GFW_HARRIS_AGB: gfw_harris_agb.DATASET, DatasetName.GFW_TREE_COVER_LOSS: gfw_tcl.DATASET, diff --git a/jdluc/datasets/faostat_production.py b/jdluc/datasets/faostat_production.py new file mode 100644 index 0000000..13ba0ed --- /dev/null +++ b/jdluc/datasets/faostat_production.py @@ -0,0 +1,256 @@ +"""Food and Agriculture Organization of the United Nations | FAOSTAT Production + +license: CC-BY-4.0 + +year: 1961, ..., 2024 + +FAO. 2025. Production: Crops and livestock products. FAOSTAT. Rome. + +https://www.fao.org/faostat/en/#data/QCL +https://bulks-faostat.fao.org/production/ + +# Methodology + +- Tabular national statistics (NOT remote sensing) from annual member-country questionnaires, + with FAO estimating or imputing where a country does not report +- National grain only; provincial production comes from `usda_nass_quickstats` for the US and + from MapSPAM's surfaces elsewhere +- Area harvested counts a field once per harvest, so a doubly-cropped field is counted twice and + the total exceeds physical cropland extent + +Area harvested and production are carried; FAOSTAT's yield column is not. MapSPAM's group crops +each aggregate several FAOSTAT items, and a group's yield is not the sum of its constituents', so a +stored yield would be right for the one-to-one crops and quietly wrong for the rest. Area and +production are additive, so a consumer sums the items it wants and divides. +""" + +import collections.abc +import csv +import dataclasses +import enum +import io +import logging +import os +import tempfile +import zipfile + +import iso3166 +import pandas + +from jdluc import config, storage, utils +from jdluc.datasets import base, ifpri_mapspam, worldbank_jurisdictions + +logger = logging.getLogger(__name__) + +BULK_URL = ( + "https://bulks-faostat.fao.org/production/" + "Production_Crops_Livestock_E_All_Data_(Normalized).zip" +) +BULK_MEMBER_NAME = "Production_Crops_Livestock_E_All_Data_(Normalized).csv" +# Copied rather than taken from `trace`, which the datasets layer may not import. +KG_PER_TONNE = 1000 + +# The two additive elements out of the file's twenty; the rest are livestock, a fifth of its 4.2 +# million rows. +AREA_HARVESTED_ELEMENT_CODE = 5312 +PRODUCTION_ELEMENT_CODE = 5510 +# "Missing value; data cannot exist". Official, estimated, imputed and external figures are all +# kept: dropping the estimated ones would thin the countries that report least. +MISSING_FLAG = "M" + + +class ItemCode(enum.IntEnum): + """The FAOSTAT item each MapSPAM crop corresponds to one-for-one. + + Keyed on item code because the name is not a stable join key: FAOSTAT spells the same item + "Cassava, fresh" in its data file and "Cassava; fresh" in its own code table. + + COTT is seed cotton (328) not ginned lint (767), and OILP is fruit bunches (254) not palm oil + (257), both matching MapSPAM. Either mistake would scale a comparison by the milling yield. + """ + + BANA = 486 # Bananas + BARL = 44 # Barley + BEAN = 176 # Beans, dry + CASS = 125 # Cassava, fresh + CHIC = 191 # Chick peas, dry + CNUT = 249 # Coconuts, in shell + COCO = 661 # Cocoa beans + COTT = 328 # Seed cotton, unginned + COWP = 195 # Cow peas, dry + GROU = 242 # Groundnuts, excluding shelled + LENT = 201 # Lentils, dry + MAIZ = 56 # Maize (corn) + OILP = 254 # Oil palm fruit + PIGE = 197 # Pigeon peas, dry + PLNT = 489 # Plantains and cooking bananas + POTA = 116 # Potatoes + RAPE = 270 # Rape or colza seed + RICE = 27 # Rice + SESA = 289 # Sesame seed + SORG = 83 # Sorghum + SOYB = 236 # Soya beans + SUGB = 157 # Sugar beet + SUGC = 156 # Sugar cane + SUNF = 267 # Sunflower seed + SWPO = 122 # Sweet potatoes + TEAS = 667 # Tea leaves + TOBA = 826 # Unmanufactured tobacco + WHEA = 15 # Wheat + YAMS = 137 # Yams + + +# MapSPAM splits one FAOSTAT item in two, so giving either name the figure invents the split and +# giving both it doubles the total. +SPLIT_CROP_NAMES = { + ifpri_mapspam.CANONICAL_CROP_CLS.ACOF.name, # both are 656, Coffee, green + ifpri_mapspam.CANONICAL_CROP_CLS.RCOF.name, + ifpri_mapspam.CANONICAL_CROP_CLS.PMIL.name, # both are 79, Millet + ifpri_mapspam.CANONICAL_CROP_CLS.SMIL.name, +} +# MapSPAM aggregates many FAOSTAT items under one name, and the member list is MapSPAM's to define. +SPAM_GROUP_CROP_NAMES = { + ifpri_mapspam.CANONICAL_CROP_CLS.OCER.name, + ifpri_mapspam.CANONICAL_CROP_CLS.OFIB.name, + ifpri_mapspam.CANONICAL_CROP_CLS.OOIL.name, + ifpri_mapspam.CANONICAL_CROP_CLS.OPUL.name, + ifpri_mapspam.CANONICAL_CROP_CLS.ORTS.name, + ifpri_mapspam.CANONICAL_CROP_CLS.REST.name, + ifpri_mapspam.CANONICAL_CROP_CLS.TEMF.name, + ifpri_mapspam.CANONICAL_CROP_CLS.TROF.name, + ifpri_mapspam.CANONICAL_CROP_CLS.VEGE.name, +} +# Every canonical crop is mapped, split or grouped, so a MapSPAM rename cannot drop one silently. +assert {item.name for item in ItemCode} | SPLIT_CROP_NAMES | SPAM_GROUP_CROP_NAMES == { + crop.name for crop in ifpri_mapspam.CANONICAL_CROP_CLS +} +assert not {item.name for item in ItemCode} & (SPLIT_CROP_NAMES | SPAM_GROUP_CROP_NAMES) +assert len({item.value for item in ItemCode}) == len(ItemCode) + +ITEM_CODE_TO_CROP_NAME = {int(item.value): item.name for item in ItemCode} + + +def get_iso_3166(m49_code: str) -> str | None: + """The ISO 3166-1 alpha-3 for a FAOSTAT M49 code, or None where it names no country. + + M49 is ISO 3166-1 numeric for countries, so this doubles as the aggregate filter: 202 of + FAOSTAT's 244 areas resolve, and the 42 that do not are exactly its aggregates ("World", + "European Union (27)") and dissolved states ("USSR", "Czechoslovakia"). + + FAOSTAT quotes and zero-pads the code (`'004`). Its own "China" aggregate (159) is among the + codes outside the standard, so mainland arrives as 156 and Taiwan separately as 158. + """ + country = iso3166.countries_by_numeric.get(m49_code.strip("'").zfill(3)) + return None if country is None else country.alpha3 + + +@dataclasses.dataclass +class Production: + admin_id: str + admin_level: str + area_hectares: float + crop_name: str + jurisdiction_name: str + production_kg: float + year: int + + +def iter_rows(path_to_zip: str) -> collections.abc.Iterator[dict[str, str]]: + """Every row of the archive's data member, streamed: it is 520 MiB unzipped.""" + with ( + zipfile.ZipFile(file=path_to_zip) as zf, + zf.open(BULK_MEMBER_NAME) as member, + ): + yield from csv.DictReader( + io.TextIOWrapper(member, encoding="utf8", errors="replace") + ) + + +def get_records_for_path(path_to_zip: str) -> list[dict[str, str | float]]: + """Area harvested and production per (country, MapSPAM crop, year), from a local archive. + + Split from the retrieval so a caller already holding the archive parses it instead of fetching + 32 MiB again -- a test with a fixture, or a validation run reading its own digest-pinned copy. + + Accumulated across one pass because the file is long format: a country-crop-year's area and + production are two rows, arbitrarily far apart. A key missing either is dropped, since a + yield taken from one of them alone would be wrong rather than partial. + """ + element_codes = {AREA_HARVESTED_ELEMENT_CODE, PRODUCTION_ELEMENT_CODE} + accumulated: dict[tuple[str, str, int], dict[str, float]] = {} + names: dict[str, str] = {} + seen = 0 + + for row in iter_rows(path_to_zip=path_to_zip): + seen += 1 + if row["Flag"] == MISSING_FLAG or not row["Value"]: + continue + element_code = int(row["Element Code"]) + if element_code not in element_codes: + continue + crop_name = ITEM_CODE_TO_CROP_NAME.get(int(row["Item Code"])) + if crop_name is None: + continue + iso_3166 = get_iso_3166(m49_code=row["Area Code (M49)"]) + if iso_3166 is None: + continue + names[iso_3166] = row["Area"] + values = accumulated.setdefault((iso_3166, crop_name, int(row["Year"])), {}) + if element_code == AREA_HARVESTED_ELEMENT_CODE: + values["area_hectares"] = float(row["Value"]) + else: + values["production_kg"] = float(row["Value"]) * KG_PER_TONNE + logger.info(f"Read {seen:d} rows; kept {len(accumulated):d} country-crop-years") + + productions = [ + Production( + admin_id=worldbank_jurisdictions.iso_3166_str(iso_3166), + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL.name, + area_hectares=float(values["area_hectares"]), + crop_name=crop_name, + jurisdiction_name=names[iso_3166], + production_kg=float(values["production_kg"]), + year=year, + ) + for (iso_3166, crop_name, year), values in sorted(accumulated.items()) + if "area_hectares" in values + if "production_kg" in values + ] + logger.info(f"After requiring both elements: {len(productions):d} records") + + return list(map(dataclasses.asdict, productions)) + + +def get_records_for_tile(tile_id: str) -> list[dict[str, str | float]]: + """Retrieve the bulk archive and read it.""" + del tile_id # whole-world partitioning, so there is only ever one + with tempfile.TemporaryDirectory() as local_dir: + path_to_zip = os.path.join(local_dir, "data.zip") + utils.save_remote_url_to_local_path( + local_path=path_to_zip, params={}, remote_url=BULK_URL + ) + return get_records_for_path(path_to_zip=path_to_zip) + + +DATASET = base.TabularDataset( + get_records_for_tile_id=get_records_for_tile, + idx_column_names=[ + "admin_level", + "admin_id", + "jurisdiction_name", + "crop_name", + "year", + ], + product_name="production-crops", + source_name="faostat", + version="v0", +) + + +def load() -> pandas.DataFrame: + uri = storage.join_uri( + root=config.Config.from_dot_env().ingest_root, + prefix=DATASET.get_prefix(tile_id="world"), + ) + logger.info(f"Loading production from {uri=:s}") + return pandas.read_parquet(path=uri) diff --git a/jdluc/datasets/gfw_global_peatlands.py b/jdluc/datasets/gfw_global_peatlands.py index b8e23ab..c673999 100644 --- a/jdluc/datasets/gfw_global_peatlands.py +++ b/jdluc/datasets/gfw_global_peatlands.py @@ -7,7 +7,7 @@ - Crezee, B. et al. Mapping peat thickness and carbon stocks of the central Congo Basin using field data. Nature Geoscience 15: 639-644 (2022). https://www.nature.com/articles/s41561-022-00966-7. Data downloaded from https://congopeat.net/maps/, using classes 4 and 5 only (peat classes). - Gumbricht, T. et al. An expert system model for mapping tropical wetlands and peatlands reveals South America as the largest contributor. Global Change Biology 23, 3581-3599 (2017). https://onlinelibrary.wiley.com/doi/full/10.1111/gcb.13689 - Hastie, A. et al. Risks to carbon storage from land-use change revealed by peat thickness maps of Peru. Nature Geoscience 15: 369-374 (2022). https://www.nature.com/articles/s41561-022-00923-4 -- Miettinen, J., Shi, C. & Liew, S. C. Land cover distribution in the peatlands of Peninsular Malaysia, Sumatra and Borneo in 2015 with changes since 1990. Global Ecological Conservation. 6, 67- 78 (2016). https://www.sciencedirect.com/science/article/pii/S2351989415300470 +- Miettinen, J., Shi, C. & Liew, S. C. Land cover distribution in the peatlands of Peninsular Malaysia, Sumatra and Borneo in 2015 with changes since 1990. Global Ecology and Conservation. 6, 67-78 (2016). https://www.sciencedirect.com/science/article/pii/S2351989415300470 - Xu et al. PEATMAP: Refining estimates of global peatland distribution based on a meta-analysis. CATENA 160: 134-140 (2018). https://www.sciencedirect.com/science/article/pii/S0341816217303004 https://data.globalforestwatch.org/datasets/gfw::global-peatlands diff --git a/jdluc/datasets/gfw_harris_agb.py b/jdluc/datasets/gfw_harris_agb.py index 87a148d..7596ef5 100644 --- a/jdluc/datasets/gfw_harris_agb.py +++ b/jdluc/datasets/gfw_harris_agb.py @@ -4,7 +4,7 @@ year: 2000 -Global maps of twenty-first century forest carbon fluxes. Nature Climate Change, 11, 234-240. DOI 10.1038/s41558-020-00976-6. +Harris, N. L. et al. (2021). Global maps of twenty-first century forest carbon fluxes. Nature Climate Change, 11, 234-240. DOI 10.1038/s41558-020-00976-6. https://data.globalforestwatch.org/datasets/gfw::aboveground-live-woody-biomass-density diff --git a/jdluc/datasets/gfw_tcl.py b/jdluc/datasets/gfw_tcl.py index 92bf7ac..27b1a23 100644 --- a/jdluc/datasets/gfw_tcl.py +++ b/jdluc/datasets/gfw_tcl.py @@ -2,13 +2,13 @@ license: CC BY 4.0 -year: 2000 baseline; annual gross loss 2001-2024 +year: 2000 baseline; annual gross loss 2001-2025 -Hansen, M. C. et al. High-Resolution Global Maps of 21st-Century Forest Cover Change. Science 342, 850-853 (2013). Updated through 2023 (v1.11). +Hansen, M. C. et al. High-Resolution Global Maps of 21st-Century Forest Cover Change. Science 342, 850-853 (2013). Updated through 2025 (GFC-2025-v1.13). -https://storage.googleapis.com/earthenginepartners-hansen/GFC-2024-v1.12/download.html +https://storage.googleapis.com/earthenginepartners-hansen/GFC-2025-v1.13/download.html -The `lossyear` band encodes the year of gross tree-cover loss: 0 = no loss, N in 1..24 = loss in calendar year 2000 + N (2001-2024). +The `lossyear` band encodes the year of gross tree-cover loss: 0 = no loss, N in 1..25 = loss in calendar year 2000 + N (2001-2025). """ from jdluc import tiling, utils @@ -22,8 +22,8 @@ def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: local_path=local_path, params={}, remote_url=( - "https://storage.googleapis.com/earthenginepartners-hansen/GFC-2024-v1.12/" - f"Hansen_GFC-2024-v1.12_lossyear_{tile_id:s}.tif" + "https://storage.googleapis.com/earthenginepartners-hansen/GFC-2025-v1.13/" + f"Hansen_GFC-2025-v1.13_lossyear_{tile_id:s}.tif" ), ) @@ -36,5 +36,6 @@ def _save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: product_name="tree-cover-loss", save_tile_id_to_local_path=_save_tile_id_to_local_path, source_name="gfw", - version="v0", + # v1: include 2025 + version="v1", ) diff --git a/jdluc/datasets/huang_bgb.py b/jdluc/datasets/huang_bgb.py index ddb7d72..b5cc949 100644 --- a/jdluc/datasets/huang_bgb.py +++ b/jdluc/datasets/huang_bgb.py @@ -1,12 +1,13 @@ -"""Huang et. al. | A global map of root biomass across the world's forests +"""Huang et al. | A global map of root biomass across the world's forests license: CC BY 4.0 year: ~2010 (the year of the main covariate AGB) +Huang, Y., Ciais, P., Santoro, M., et al. (2021). A global map of root biomass across the world's forests. Earth System Science Data, 13, 4263-4274. https://doi.org/10.5194/essd-13-4263-2021 Huang, Yuanyuan; Ciais, Phillipe; Santoro, Maurizio; Makowski, David; Chave, Jerome; Schepaschenko, Dmitry; et al. (2020). Supporting data and code for A global map of root biomass across the world's forests. figshare. Dataset. https://doi.org/10.6084/m9.figshare.12199637.v1 -https://www.researchgate.net/publication/354263169_A_global_map_of_root_biomass_across_the_world's_forests +https://essd.copernicus.org/articles/13/4263/2021/ # Methodology diff --git a/jdluc/datasets/ifpri_mapspam.py b/jdluc/datasets/ifpri_mapspam.py index 550bdc4..a2366cb 100644 --- a/jdluc/datasets/ifpri_mapspam.py +++ b/jdluc/datasets/ifpri_mapspam.py @@ -28,6 +28,7 @@ import numpy import rasterio +import xarray from jdluc import config, tiling, utils from jdluc.datasets import base @@ -62,7 +63,7 @@ class Crop2000(enum.StrEnum): @enum.unique class Crop2005(enum.StrEnum): - ACOF = "Arabic Coffee" + ACOF = "Arabica Coffee" BANA = "Banana" BARL = "Barley" BEAN = "Bean" @@ -86,7 +87,7 @@ class Crop2005(enum.StrEnum): PMIL = "Pearl Millet" POTA = "Potato" RAPE = "Rapeseed" - RCOF = "Robust Coffee" + RCOF = "Robusta Coffee" REST = "Rest Of Crops" RICE = "Rice" SESA = "Sesame Seed" @@ -109,7 +110,7 @@ class Crop2005(enum.StrEnum): @enum.unique class Crop2010(enum.StrEnum): # NB: the 2005/2010 values are the same but we'd like distinct enums - ACOF = "Arabic Coffee" + ACOF = "Arabica Coffee" BANA = "Banana" BARL = "Barley" BEAN = "Bean" @@ -133,7 +134,7 @@ class Crop2010(enum.StrEnum): PMIL = "Pearl Millet" POTA = "Potato" RAPE = "Rapeseed" - RCOF = "Robust Coffee" + RCOF = "Robusta Coffee" REST = "Rest Of Crops" RICE = "Rice" SESA = "Sesame Seed" @@ -163,7 +164,7 @@ class Crop2020(enum.StrEnum): CITR = "Citrus" CNUT = "Coconut" COCO = "Cocoa" - COFF = "Arabic Coffee" + COFF = "Arabica Coffee" COTT = "Cotton" COWP = "Cowpea" GROU = "Groundnut" @@ -182,7 +183,7 @@ class Crop2020(enum.StrEnum): PMIL = "Pearl Millet" POTA = "Potato" RAPE = "Rapeseed" - RCOF = "Robust Coffee" + RCOF = "Robusta Coffee" REST = "Rest Of Crops" RICE = "Rice" RUBB = "Rubber" @@ -222,17 +223,20 @@ class Crop2020(enum.StrEnum): Crop2000.SUGC.name, Crop2000.WHEA.name, } -for crop_cls in (Crop2000, Crop2005, Crop2010, Crop2020): - assert all(hasattr(crop_cls, name) for name in SHARED_CROP_NAMES) -all_equal = lambda *values: len(set(values)) == 1 -for name in SHARED_CROP_NAMES: - assert all_equal( - crop_cls[name].value for crop_cls in (Crop2000, Crop2005, Crop2010, Crop2020) - ) +assert all( + hasattr(crop_cls, name) + for crop_cls in (Crop2000, Crop2005, Crop2010, Crop2020) + for name in SHARED_CROP_NAMES +) +assert all( + len({crop_cls[name].value for crop_cls in (Crop2000, Crop2005, Crop2010, Crop2020)}) + == 1 + for name in SHARED_CROP_NAMES +) # The 2000 snapshot contains 6 "group" crops which are disaggregated in later snapshots. # Assuming the mix of crops within a group for a given 10km pixel is unchanged from 2000 -# to 2005, we can decompose the 2000 crop group into its constituent crops. +# to a later reference year, we can decompose the 2000 crop group into its constituent crops. GROUP_TO_CONSTITUENT_NAMES: dict[str, set[str]] = { Crop2000.BANP.name: {Crop2005.BANA.name, Crop2005.PLNT.name}, Crop2000.COFF.name: {Crop2005.ACOF.name, Crop2005.RCOF.name}, @@ -254,8 +258,11 @@ class Crop2020(enum.StrEnum): }, Crop2000.SWPY.name: {Crop2005.SWPO.name, Crop2005.YAMS.name}, } -YEAR_TO_DECOMPOSE = 2000 -DECOMPOSITION_REFERENCE_YEAR = 2005 +DECOMPOSITION_YEAR = 2000 +# Reference years for the decomposition, pooled into a single within-group split rather than +# ranked by nearness to 2000: a pixel the 2000 snapshot uses but 2005 does not is an +# inconsistency between MapSPAM's snapshots, not a crop that arrived later. +DECOMPOSITION_REFERENCE_YEARS = (2005, 2010, 2020) CONSTITUENT_TO_GROUP_NAME = { constituent: group_name for group_name, constituents in GROUP_TO_CONSTITUENT_NAMES.items() @@ -263,6 +270,24 @@ class Crop2020(enum.StrEnum): } assert set(SHARED_CROP_NAMES).isdisjoint(GROUP_TO_CONSTITUENT_NAMES) +RECOVERABLE_CROP_NAMES = SHARED_CROP_NAMES | set(CONSTITUENT_TO_GROUP_NAME) +assert len(RECOVERABLE_CROP_NAMES) == len(SHARED_CROP_NAMES) + len( + CONSTITUENT_TO_GROUP_NAME +) + +# When an "other" group contains no area in ANY reference year, there is no basis for +# dividing the crops within the decomposition year and we divide crops evenly across the +# constituents. This can result in tropical crops being attributed to non-tropical geos. +# Instead, we'd like to route the unattributable remainder to the residual. +GROUP_TO_RESIDUAL_NAME: dict[str, str] = { + Crop2000.OOIL.name: Crop2005.OOIL.name, + Crop2000.OPUL.name: Crop2005.OPUL.name, +} +assert all( + residual in GROUP_TO_CONSTITUENT_NAMES[group] + for group, residual in GROUP_TO_RESIDUAL_NAME.items() +) + CropClsType = type[Crop2000] | type[Crop2005] | type[Crop2010] | type[Crop2020] YEAR_TO_CROP_CLS: dict[int, CropClsType] = { @@ -276,19 +301,134 @@ class Crop2020(enum.StrEnum): } YEARS = sorted(YEAR_TO_CROP_CLS) +# Every crop name crossing this module's boundary is one of two things. A *canonical* name is a +# `CANONICAL_CROP_CLS` name -- the spelling `statistical.Crop` and `RECOVERABLE_CROP_NAMES` use, +# stable across years, and what `get_canonical_quantity` takes. A *reported* name is whatever a +# single year's own taxonomy calls that crop, which is what the bands are keyed by and what +# `get_band_name` and `get_reported_quantity` take. 2005 is the canonical one because it is the +# finest taxonomy every later year can be renamed back into, and the one the decomposition splits +# 2000's groups into. `get_reported_crop_name` is the only hop between the two. +CANONICAL_CROP_CLS: type[Crop2005] = Crop2005 -def map_2005_name_to_year(crop_name: str, year: int) -> str: - return YEAR_TO_CROP_CLS[year](Crop2005[crop_name].value).name + +def get_renamed_crop_name(canonical_crop_name: str, year: int) -> str: + """`canonical_crop_name` as `year`'s taxonomy spells it, matched on the shared crop label. + + The rename alone -- it cannot see groups, so it is wrong for a constituent in the decompose + year. `get_reported_crop_name` is the one to call; this stays separate because `datasets_test` + checks that function against it. + """ + return YEAR_TO_CROP_CLS[year](CANONICAL_CROP_CLS[canonical_crop_name].value).name assert all( - map_2005_name_to_year(crop_name=constituent, year=year) + get_renamed_crop_name(canonical_crop_name=constituent, year=year) for constituent in CONSTITUENT_TO_GROUP_NAME for year in YEARS - if year != YEAR_TO_DECOMPOSE + if year != DECOMPOSITION_YEAR ) +def is_reported_as_group(canonical_crop_name: str, year: int) -> bool: + """Whether `year` reports `canonical_crop_name` only as part of a coarser group. + + True for a group constituent in the decompose year and nothing else: that snapshot's taxonomy + carries no band of its own for the crop, which is the reason the decomposition exists. + """ + return ( + year == DECOMPOSITION_YEAR and canonical_crop_name in CONSTITUENT_TO_GROUP_NAME + ) + + +def get_reported_crop_name(canonical_crop_name: str, year: int) -> str: + """The crop `year` reports `canonical_crop_name`'s quantity under. + + Its own name in most years, renamed where that year renames it -- 2020 calls arabica coffee + COFF and small millet MILL -- and the group it belongs to where the year is too coarse to + name it separately. This is the canonical-to-reported hop every band lookup needs: `dset` is + keyed by reported names, so `get_band_name` and `get_reported_quantity` take the result. + """ + if is_reported_as_group(canonical_crop_name=canonical_crop_name, year=year): + return CONSTITUENT_TO_GROUP_NAME[canonical_crop_name] + else: + return get_renamed_crop_name(canonical_crop_name=canonical_crop_name, year=year) + + +# The crops each snapshot reports that no recoverable crop accounts for, in that snapshot's own +# naming: nothing can divide by them, so they can only be carried as a lump. Spelled out rather +# than derived so that a change to MapSPAM's taxonomy breaks the import, instead of silently +# changing which crops the share denominator can reach. +YEAR_TO_UNRECOVERABLE_CROP_NAMES: dict[int, set[str]] = { + 2000: {"OFIB", "OTHE"}, + 2005: { + "COCO", + "OCER", + "OFIB", + "ORTS", + "REST", + "TEAS", + "TEMF", + "TOBA", + "TROF", + "VEGE", + }, + 2010: { + "COCO", + "OCER", + "OFIB", + "ORTS", + "REST", + "TEAS", + "TEMF", + "TOBA", + "TROF", + "VEGE", + }, + 2020: { + "CITR", + "COCO", + "OCER", + "OFIB", + "ONIO", + "ORTS", + "REST", + "RUBB", + "TEAS", + "TEMF", + "TOBA", + "TOMA", + "TROF", + "VEGE", + }, +} +assert { + year: {e.name for e in YEAR_TO_CROP_CLS[year]} + - { + get_reported_crop_name(canonical_crop_name=name, year=year) + for name in RECOVERABLE_CROP_NAMES + } + for year in YEARS +} == YEAR_TO_UNRECOVERABLE_CROP_NAMES + + +# Snapshot pairs whose crop names can be matched directly. Every pair involving the decompose year +# is excluded: it is a different MapSPAM release (v3.0.7 against v3.2) and its unrecoverable +# vocabulary is nearly disjoint from 2005's -- a catch-all OTHE leaves and nine crops appear in its +# place, holding 560 Mha against their 161 and correlating at r = 0.17 per pixel. A name on only +# one side of that boundary says the release did not use it, not that the crop was absent. +SPANS_WITH_COMPARABLE_CROP_NAMES: set[tuple[int, int]] = { + (2005, 2010), + (2005, 2020), + (2010, 2020), +} +assert { + (before, after) + for before in YEARS + for after in YEARS + if before < after and DECOMPOSITION_YEAR not in (before, after) +} == SPANS_WITH_COMPARABLE_CROP_NAMES + + class Quantity(enum.StrEnum): PRODUCTION = "mt" PHYSICAL_AREA = "ha" @@ -455,3 +595,113 @@ def save_tile_id_to_local_path(local_path: str, tile_id: str) -> None: 2020: PHYSICAL_AREA_2020, } assert set(YEAR_TO_PHYSICAL_AREA_DATASET) == set(YEAR_TO_PRODUCTION_DATASET) + + +def get_band_name(quantity: Quantity, reported_crop_name: str, year: int) -> str: + """The fully-qualified band carrying `reported_crop_name`'s `quantity` in `year`. + + Takes a reported name -- it indexes `year`'s own taxonomy, so a canonical name that year + renames or groups is a `KeyError`. Resolve with `get_reported_crop_name` first. + + Positional: the band list is built from the same enum in the same order, so a crop's index in + the taxonomy is its index among the bands. `datasets_test` pins that for every year and + quantity, since nothing in the band names themselves would reveal a reordering. + """ + crop_cls = YEAR_TO_CROP_CLS[year] + name_to_idx = {name: idx for idx, name in enumerate(e.name for e in crop_cls)} + idx = name_to_idx[reported_crop_name] + year_to_dataset = ( + YEAR_TO_PHYSICAL_AREA_DATASET + if quantity == Quantity.PHYSICAL_AREA + else YEAR_TO_PRODUCTION_DATASET + ) + return year_to_dataset[year].fully_qualified_band_names[idx] + + +def get_reported_quantity( + dset: xarray.Dataset, quantity: Quantity, reported_crop_name: str, year: int +) -> xarray.DataArray: + """`reported_crop_name`'s `quantity` exactly as `year` stores it. + + The as-reported twin of `get_canonical_quantity`: same quantity, no decomposition and no + renaming, so it takes a reported name where that one takes a canonical one. + """ + variable_name = get_band_name( + quantity=quantity, reported_crop_name=reported_crop_name, year=year + ) + # NB: a crop absent from a snapshot (or no-data there) counts as zero hectares + return dset[variable_name].fillna(0) + + +def get_canonical_quantity( + canonical_crop_name: str, + dset: xarray.Dataset, + quantity: Quantity, + year: int, +) -> xarray.DataArray: + """`canonical_crop_name`'s `quantity` in `year`. + + Does whatever `year` requires to honor a canonical name -- following that year's renaming, + and decomposing a group where the year is too coarse to name the crop -- so that one name + means one crop across every year. `get_reported_quantity` is the raw counterpart. + """ + if is_reported_as_group(canonical_crop_name=canonical_crop_name, year=year): + # Decompose the grouped crops into their constituents, assuming the within-group + # proportions match those the reference years pool to. Each quantity is split by its + # own distribution -- splitting production by AREA share would assume every constituent + # of the group yields the same in this pixel, and bananas outyield plantains 2-3x. + group_name = CONSTITUENT_TO_GROUP_NAME[canonical_crop_name] + siblings = sorted(GROUP_TO_CONSTITUENT_NAMES[group_name]) + + def reference(canonical_name: str) -> xarray.DataArray: + ret = sum( + get_reported_quantity( + dset=dset, + quantity=quantity, + reported_crop_name=get_reported_crop_name( + canonical_crop_name=canonical_name, year=reference_year + ), + year=reference_year, + ) + for reference_year in DECOMPOSITION_REFERENCE_YEARS + ) + assert isinstance(ret, xarray.DataArray) + return ret + + reference_group = sum(reference(canonical_name=s) for s in siblings) + assert isinstance(reference_group, xarray.DataArray) + constituent_share = xarray.where( + reference_group > 0, + reference(canonical_name=canonical_crop_name) + # NB: this avoids `RuntimeWarning: invalid value encountered in divide` + / reference_group.where(reference_group > 0, other=1), + # No reference year places the group here + ( + # The group has a catch-all constituent, so give it the whole unattributable + # remainder rather than handing a slice to every named sibling + float(canonical_crop_name == GROUP_TO_RESIDUAL_NAME[group_name]) + if group_name in GROUP_TO_RESIDUAL_NAME + # No catch-all to route to, so an even split is the only option among named crops + else 1 / len(siblings) + ), + ) + return ( + get_reported_quantity( + dset=dset, + quantity=quantity, + # NB: the group is already a reported name -- it exists only in the decompose year + reported_crop_name=group_name, + year=DECOMPOSITION_YEAR, + ) + * constituent_share + ) + else: + # Simple lookup + return get_reported_quantity( + dset=dset, + quantity=quantity, + reported_crop_name=get_reported_crop_name( + canonical_crop_name=canonical_crop_name, year=year + ), + year=year, + ) diff --git a/jdluc/datasets/ipcc_climate_zones.py b/jdluc/datasets/ipcc_climate_zones.py index e649c7a..00ca93e 100644 --- a/jdluc/datasets/ipcc_climate_zones.py +++ b/jdluc/datasets/ipcc_climate_zones.py @@ -4,7 +4,8 @@ year: ~static -Calvo Buendia, E et al. (2019). 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories. IPCC, Switzerland. https://doi.org/10.5281/zenodo.7303808 +Lewis, M. (2022). IPCC Climate Zones (from the 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories), version 0.1.0 [Data set]. Zenodo. https://doi.org/10.5281/zenodo.7303808 +Calvo Buendia, E et al. (2019). 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories. IPCC, Switzerland. -- the decision tree the raster implements. https://zenodo.org/records/7303808 diff --git a/jdluc/datasets/usda_nass_cdl.py b/jdluc/datasets/usda_nass_cdl.py index f62e983..eb735c9 100644 --- a/jdluc/datasets/usda_nass_cdl.py +++ b/jdluc/datasets/usda_nass_cdl.py @@ -65,7 +65,11 @@ def _get_dataarray() -> xarray.DataArray: return ( darray.isel(band=0) .drop_vars("band") - .rio.reproject(dst_crs=4326, resampling=rasterio.enums.Resampling.nearest) + .rio.reproject( + dst_crs=4326, + nodata=DATASET.no_data, + resampling=rasterio.enums.Resampling.nearest, + ) ) diff --git a/jdluc/datasets/usda_nass_quickstats.py b/jdluc/datasets/usda_nass_quickstats.py index d4554c5..962eec8 100644 --- a/jdluc/datasets/usda_nass_quickstats.py +++ b/jdluc/datasets/usda_nass_quickstats.py @@ -27,6 +27,7 @@ import csv import dataclasses +import enum import io import logging import typing @@ -41,8 +42,85 @@ HA_PER_ACRE = 0.40468564 KG_PER_LB = 0.45359237 -# 7 CFR 810 (US Grain Standards Act) -CROP_NAME_TO_LB_PER_BUSHEL = {"CORN": 56, "SOYBEANS": 60, "WHEAT": 60} + + +@dataclasses.dataclass(frozen=True, eq=False) +class Series: + """The one NASS series that measures a crop's yield, and what a unit of it weighs. + + NASS publishes yield per acre in four units, and which one applies is a property of the + commodity, so the request cannot pin a single one. `lb_per_unit` is the marketing bushel for + the grains, from table 6 of USDA Agricultural Handbook 697 (ERS, June 1992): barley 48, shelled + corn 56, sorghum grain 56, soybeans 60, wheat 60. Not the grading test weight of 7 CFR 810, + which is a different quantity (barley 47.0, sorghum 57.0, and soybeans carries none at all). + + `commodity_desc` is carried only where NASS spells the commodity differently from the crop; + `CropSeries.commodity_desc` falls back to the member's own name. `reported_fraction` is the + share of the harvested crop the published yield weighs, 1 for everything but cotton, which NASS + reports ginned. + + Compared by identity rather than by field, because two crops can share a unit and a weight -- + soybean and wheat do, as do maize and sorghum -- and under structural equality the second of + each pair would become an alias of the first inside `CropSeries`, silently dropping it from + iteration. + """ + + unit_desc: str + lb_per_unit: float + commodity_desc: str | None = None + class_descs: tuple[str, ...] = ("ALL CLASSES",) + util_practice_desc: str = "ALL UTILIZATION PRACTICES" + reported_fraction: float = 1.0 + + +@enum.unique +class CropSeries(enum.Enum): + """Which NASS series measures each `jurisdictional_direct.Crop`. + + Members are named for the jdLUC crop rather than the NASS commodity, so the ingested table + carries crop names the rest of the pipeline already speaks and needs no crosswalk: NASS + `BEANS` is `BEAN`, `CORN` is `MAIZE`, `POTATOES` is `POTATO`. + """ + + BARLEY = Series(unit_desc="BU / ACRE", lb_per_unit=48) + # NASS renamed the rolled-up dry-bean class in 2019 and publishes no ALL CLASSES to fall back + # on, so spanning a multi-year window takes both names. They are not the same quantity: the + # earlier one includes chickpeas, which the CDL scores separately as CHICK_PEAS. + BEAN = Series( + commodity_desc="BEANS", + unit_desc="LB / ACRE", + lb_per_unit=1, + class_descs=("DRY EDIBLE, INCL CHICKPEAS", "DRY EDIBLE, (EXCL CHICKPEAS)"), + ) + # NASS reports ginned lint, where MapSPAM, FAOSTAT and WRI all carry seed cotton -- see the + # ItemCode docstring in `faostat_production`. 0.36 is what FAOSTAT's US seed cotton implies + # against this series over 2011-2020 (mean 0.361), and agrees with the ~35% gin turnout. + # FAOSTAT implies ~0.40 before 2011, so revisit this if the trace window moves earlier. + COTTON = Series(unit_desc="LB / ACRE", lb_per_unit=1, reported_fraction=0.36) + MAIZE = Series( + commodity_desc="CORN", + unit_desc="BU / ACRE", + lb_per_unit=56, + util_practice_desc="GRAIN", + ) + POTATO = Series(commodity_desc="POTATOES", unit_desc="CWT / ACRE", lb_per_unit=100) + RICE = Series(unit_desc="LB / ACRE", lb_per_unit=1) + SORGHUM = Series(unit_desc="BU / ACRE", lb_per_unit=56, util_practice_desc="GRAIN") + SOYBEAN = Series(commodity_desc="SOYBEANS", unit_desc="BU / ACRE", lb_per_unit=60) + SUGARBEET = Series( + commodity_desc="SUGARBEETS", unit_desc="TONS / ACRE", lb_per_unit=2000 + ) + # Sugarcane publishes no ALL UTILIZATION PRACTICES row at all; SUGAR & SEED is its total. + SUGARCANE = Series( + unit_desc="TONS / ACRE", lb_per_unit=2000, util_practice_desc="SUGAR & SEED" + ) + WHEAT = Series(unit_desc="BU / ACRE", lb_per_unit=60) + + @property + def commodity_desc(self) -> str: + """What NASS calls this crop: its own name, unless its series overrides it.""" + return self.value.commodity_desc or self.name + STATE_FIPS_TO_ADMIN_ID = { 1: "USA001", # Alabama @@ -109,37 +187,44 @@ class Yield: yield_kg_per_ha: float @classmethod - def from_dict(cls, d: dict[str, float | str]) -> typing.Self: - crop_name = str(d["commodity_desc"]) - bu_per_acre = float(str(d["Value"]).replace(",", "")) + def from_dict( + cls, crop_series: CropSeries, d: dict[str, float | str] + ) -> typing.Self: + series = crop_series.value + per_acre = float(str(d["Value"]).replace(",", "")) return cls( admin_id=STATE_FIPS_TO_ADMIN_ID[int(d["state_fips_code"])], admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL.name, - crop_name=crop_name, + crop_name=crop_series.name, jurisdiction_name=str(d["state_name"]), year=int(d["year"]), yield_kg_per_ha=( - bu_per_acre - * CROP_NAME_TO_LB_PER_BUSHEL[crop_name] + per_acre + * series.lb_per_unit * KG_PER_LB / HA_PER_ACRE + / series.reported_fraction ), ) -def get_yield_dicts_from_api(api_key: str) -> list[dict[str, float | str]]: +def get_yield_dicts_from_api( + api_key: str, crop_series: CropSeries +) -> list[dict[str, float | str]]: + series = crop_series.value param_tuples = ( ("key", api_key), ("source_desc", "SURVEY"), ("sector_desc", "CROPS"), ("statisticcat_desc", "YIELD"), ("agg_level_desc", "STATE"), - ("unit_desc", "BU / ACRE"), + ("unit_desc", series.unit_desc), ("freq_desc", "ANNUAL"), ("reference_period_desc", "YEAR"), - ("class_desc", "ALL CLASSES"), + *(("class_desc", c) for c in series.class_descs), ("prodn_practice_desc", "ALL PRODUCTION PRACTICES"), - *[("commodity_desc", c) for c in ("CORN", "SOYBEANS", "WHEAT")], + ("commodity_desc", crop_series.commodity_desc), + ("util_practice_desc", series.util_practice_desc), ("format", "CSV"), ) with utils.get_requests_session().request( @@ -155,16 +240,18 @@ def get_yield_dicts_from_api(api_key: str) -> list[dict[str, float | str]]: def _get_records_for_tile(tile_id: str) -> list[dict[str, str | float]]: api_key = config.Config.from_dot_env().usda_nass_api_key - yield_dicts = get_yield_dicts_from_api(api_key=api_key) - logger.info(f"Received {len(yield_dicts):d} raw yields") - yields = [ - Yield.from_dict(yield_dict) - for yield_dict in yield_dicts + Yield.from_dict(crop_series=crop_series, d=yield_dict) + for crop_series in CropSeries + for yield_dict in get_yield_dicts_from_api( + api_key=api_key, crop_series=crop_series + ) if int(yield_dict["state_fips_code"]) in STATE_FIPS_TO_ADMIN_ID - if yield_dict["Value"] not in {"(D)", "(Z)", "(S)", "(NA)"} + if str(yield_dict["Value"]).strip() not in {"(D)", "(Z)", "(S)", "(NA)"} ] - logger.info(f"After filtering: {len(yields):d} raw yields") + logger.info( + f"After filtering: {len(yields):d} yields over {len(CropSeries):d} crops" + ) return list(map(dataclasses.asdict, yields)) @@ -180,7 +267,8 @@ def _get_records_for_tile(tile_id: str) -> list[dict[str, str | float]]: ], product_name="quickstats", source_name="usda-nass", - version="2025", + # 2025b: expand from three to eleven crops + version="2025b", ) diff --git a/jdluc/datasets/worldbank_jurisdictions.py b/jdluc/datasets/worldbank_jurisdictions.py index 3cdff3a..dd8627c 100644 --- a/jdluc/datasets/worldbank_jurisdictions.py +++ b/jdluc/datasets/worldbank_jurisdictions.py @@ -28,7 +28,7 @@ class AdminLevel(enum.IntEnum): DISTRICT = 2 -def _save_tile_id_to_local_path_for_admin_level( +def save_tile_id_to_local_path_for_admin_level( admin_level: AdminLevel, ) -> base.SaveTileIdToLocalPathType: remote_url = ( @@ -51,7 +51,7 @@ def inner(local_path: str, tile_id: str) -> None: id_column_names=("ISO_A3",), name_column_names=("NAM_0",), product_name="admin-0", - save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + save_tile_id_to_local_path=save_tile_id_to_local_path_for_admin_level( admin_level=AdminLevel.NATIONAL ), source_name="world-bank", @@ -62,7 +62,7 @@ def inner(local_path: str, tile_id: str) -> None: id_column_names=("ADM1CD_c",), name_column_names=("NAM_0", "NAM_1"), product_name="admin-1", - save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + save_tile_id_to_local_path=save_tile_id_to_local_path_for_admin_level( admin_level=AdminLevel.PROVINCIAL ), source_name="world-bank", @@ -73,7 +73,7 @@ def inner(local_path: str, tile_id: str) -> None: id_column_names=("ADM2CD_c",), name_column_names=("NAM_0", "NAM_1", "NAM_2"), product_name="admin-2", - save_tile_id_to_local_path=_save_tile_id_to_local_path_for_admin_level( + save_tile_id_to_local_path=save_tile_id_to_local_path_for_admin_level( admin_level=AdminLevel.DISTRICT ), source_name="world-bank", @@ -90,53 +90,91 @@ def inner(local_path: str, tile_id: str) -> None: @functools.cache -def load_geodataframe_for_admin_level(admin_level: int) -> geopandas.GeoDataFrame: - dataset = ADMIN_LEVEL_TO_DATASET[AdminLevel(admin_level)] +def get_jurisdiction_for_admin_level(admin_level: AdminLevel) -> geopandas.GeoDataFrame: + dataset = ADMIN_LEVEL_TO_DATASET[admin_level] path_to_fgb = storage.join_uri( root=config.Config.from_dot_env().ingest_root, prefix=dataset.get_prefix(tile_id="world"), ) - logger.info(f"Loading {admin_level=:d} geometrys from {path_to_fgb=:s}") + logger.info(f"Loading {admin_level=:} geometrys from {path_to_fgb=:s}") return geopandas.read_file(filename=path_to_fgb).set_index(keys="id") -def get_ten_degree_tile_ids_for_admin_id(admin_id: str, admin_level: int) -> list[str]: - gdf = load_geodataframe_for_admin_level(admin_level=admin_level) +@functools.cache +def get_ten_degree_tile_ids_for_admin_id( + admin_id: str, admin_level: AdminLevel +) -> frozenset[str]: + gdf = get_jurisdiction_for_admin_level(admin_level=admin_level) geometry = gdf.loc[admin_id].geometry assert isinstance(geometry, shapely.Polygon | shapely.MultiPolygon) - return sorted(tiling.iter_ten_degree_tile_id_for_geometry(geometry=geometry)) + return frozenset( + tiling.iter_ten_degree_tile_id_for_geometry(geometry=geometry) + ).intersection(tiling.GLOBAL_FOREST_WATCH_TILE_IDS) + + +def get_ten_degree_tile_ids_for_iso_3166s( + iso_3166s: collections.abc.Iterable[str], +) -> set[str]: + return { + tile_id + for iso_3166 in iso_3166s + for tile_id in get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, + admin_level=AdminLevel.NATIONAL, + ) + } -@functools.cache -def get_jurisdiction_for_admin_level( - admin_level: AdminLevel, -) -> geopandas.GeoDataFrame: - logger.info(f"Loading geometry for {admin_level.name=:s}") - dataset = ADMIN_LEVEL_TO_DATASET[admin_level] - return geopandas.read_file( - filename=storage.join_uri( - root=config.Config.from_dot_env().ingest_root, - prefix=dataset.get_prefix(tile_id="world"), +def iso_3166_str(s: str) -> str: + assert len(s) == 3, f"iso_3166 code {s=:s} must be three characters" + return s + + +# Countries the pipeline produces nothing for. Each does intersect a published tile, so no +# geometric predicate replaces this list; MapSPAM coverage is the real criterion and reading it +# needs the ingested rasters. +UNPRODUCTIVE_ISO_3166S = frozenset( + {"BLM", "GIB", "GRL", "MAF", "MCO", "NRU", "TUV", "VAT"} +) + + +def get_all_iso_3166s() -> set[str]: + """Every country the pipeline can produce an emissions factor for. + + Two halves: the country touches a ten-degree tile the sources publish, and it grows something. + The first is geometry and the second is `UNPRODUCTIVE_ISO_3166S`; see there for why the second + is a list rather than a predicate. + """ + return { + iso_3166_str(s=iso_3166) + for iso_3166 in get_jurisdiction_for_admin_level( + admin_level=AdminLevel.NATIONAL + ).index + if iso_3166 not in UNPRODUCTIVE_ISO_3166S + and get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, admin_level=AdminLevel.NATIONAL ) - ).set_index("id") + } @dataclasses.dataclass class Jurisdiction: - level: AdminLevel + level: str id: str name: str geometry: shapely.Geometry -def iter_jurisdiction_for_iso_3166( - admin_level: AdminLevel, iso_3166: str +def iter_jurisdiction_for_iso_3166_tile_id( + admin_level: AdminLevel, iso_3166: str, tile_id: str ) -> collections.abc.Generator[Jurisdiction]: - provincial = get_jurisdiction_for_admin_level(admin_level=admin_level) - for admin_id, row in sorted(provincial.iterrows()): - if str(admin_id).startswith(iso_3166): + gdf = get_jurisdiction_for_admin_level(admin_level=admin_level) + the_iso_3166 = gdf[gdf.index.str.startswith(iso_3166)].sort_index() + tile = tiling.get_box_for_tile_id(tile_id=tile_id) + for admin_id, row in the_iso_3166.iterrows(): + if row["geometry"].intersects(other=tile): yield Jurisdiction( - level=admin_level, + level=admin_level.name, id=str(admin_id), name=row["name"], geometry=row["geometry"], diff --git a/jdluc/emit.py b/jdluc/emit.py index e5cc9c7..88759ec 100644 --- a/jdluc/emit.py +++ b/jdluc/emit.py @@ -5,8 +5,12 @@ the GHGP 20-year linear discount. Returns an xarray.Dataset — per-hectare emissions plus a hectares-per-pixel band for downstream area-scaling — which is cached. -Example invocation: - uv run python jdluc/emit.py NORTH_AMERICA +The tile set comes from the positional ISO 3166 alpha-3 codes -- or, with `--backfill`, +from `tiling.GLOBAL_FOREST_WATCH_TILE_IDS`. + +Example invocations: + uv run python jdluc/emit.py USA + uv run python jdluc/emit.py --backfill """ import argparse @@ -20,7 +24,7 @@ import numpy import xarray -from jdluc import continents, geo, harmonize, storage, tiling +from jdluc import geo, harmonize, storage, tiling from jdluc.datasets import ( gfw_global_peatlands, gfw_harris_agb, @@ -28,6 +32,7 @@ huang_bgb, ipcc_climate_zones, soilgrids_ocs, + worldbank_jurisdictions, ) logger = logging.getLogger(__name__) @@ -205,7 +210,10 @@ def get_mineral_soil_emissions( dask="parallelized", output_dtypes=[numpy.float32], ) - * soil_organic_carbon + # NB: SoilGrids has genuine gaps (water, rock, ice) that harmonize turns into NaN. + # Treat a missing stock as zero rather than letting NaN reach emissions-per-hectare, + # where it would silently discard the pixel's vegetation emissions too. + * soil_organic_carbon.fillna(0) ).rename("tco2e-per-ha") @@ -248,9 +256,8 @@ def get_soil_emissions( def get_peatland_occupation_emissions( - is_peatland: xarray.DataArray, year_to_land_class: dict[int, xarray.DataArray] + is_peatland: xarray.DataArray, latest_land_class: xarray.DataArray ) -> xarray.DataArray: - latest_land_class = year_to_land_class[max(year_to_land_class)] undrained_classes = ( # still natural glad_glcluc.LandClass.FOREST.value, @@ -263,8 +270,10 @@ def get_peatland_occupation_emissions( is_undrained = functools.reduce( operator.or_, (latest_land_class == value for value in undrained_classes) ) - return PEATLAND_EMISSIONS_ANNUAL_TCO2E_PER_HA * is_peatland.where( - ~is_undrained, other=0 + return ( + PEATLAND_EMISSIONS_ANNUAL_TCO2E_PER_HA + # NB: ensure no nan's are created which would clobber other nonzero emissions when combined + * (is_peatland == 1).astype(numpy.float32).where(~is_undrained, other=0) ).rename("tco2e-per-ha") @@ -279,14 +288,20 @@ def get_peatland_occupation_emissions( assert math.isclose(sum(SPAN_TO_LINEAR_DISCOUNT_WEIGHT.values()), 0.2) -def get_linear_discounted_emissions( - span_to_emissions: dict[SpanType, xarray.DataArray], +def get_linear_discounted_total( + span_to_value: dict[SpanType, xarray.DataArray], ) -> xarray.DataArray: + """`span_to_value` reduced by the linear discount each span carries. + + Quantity-neutral: a flux -- emissions -- takes the weighted sum as it stands, while a level + -- area, production -- divides the result by the weights' total, making the same reduction + a weighted mean over the same windows. + """ ret = sum( - emissions * SPAN_TO_LINEAR_DISCOUNT_WEIGHT[span] - for span, emissions in span_to_emissions.items() + value * SPAN_TO_LINEAR_DISCOUNT_WEIGHT[span] + for span, value in span_to_value.items() ) - return typing.cast(xarray.DataArray, ret).rename("tco2e-per-ha") + return typing.cast(xarray.DataArray, ret) def get_hectares_per_pixel(darray: xarray.DataArray) -> xarray.DataArray: @@ -335,14 +350,14 @@ def merge_name_units(name: str, units: typing.Hashable | None) -> str: @storage.cache_to_zarr(version=0) -def workflow(tile_ids: tuple[str, ...]) -> xarray.Dataset: - logger.info(f"Running the land conversion and emissions worflow for {tile_ids=:}") +def workflow(tile_id: str) -> xarray.Dataset: + logger.info(f"Running the land conversion and emissions worflow for {tile_id=:s}") dset = harmonize.workflow( dataset_names=harmonize.LUC_AND_EMISSIONS_DATASET_NAMES, - ignore_missing_tiles=False, + ignore_missing_tiles=True, skip_ingest=False, - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.GLAD.value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution.GLAD, ) logger.info("Mapping GLCLUC values to land classes") @@ -388,7 +403,7 @@ def workflow(tile_ids: tuple[str, ...]) -> xarray.Dataset: } logger.info("Quantifying soil emissions") - is_peatland = dset[gfw_global_peatlands.DATASET.fully_qualified_band_name] + is_peatland = dset[gfw_global_peatlands.DATASET.fully_qualified_band_name].fillna(0) span_to_soil_emissions = { (before, after): get_soil_emissions( after=year_to_land_class[after], @@ -410,10 +425,11 @@ def workflow(tile_ids: tuple[str, ...]) -> xarray.Dataset: logger.info("Quantifying and adding peatland occupation emissions") peatland_occupation_emissions: xarray.DataArray = get_peatland_occupation_emissions( - is_peatland=is_peatland, year_to_land_class=year_to_land_class + is_peatland=is_peatland, + latest_land_class=year_to_land_class[max(year_to_land_class)], ) emissions_per_hectare: xarray.DataArray = ( - get_linear_discounted_emissions(span_to_emissions=span_to_emissions) + get_linear_discounted_total(span_to_value=span_to_emissions) + peatland_occupation_emissions ) @@ -450,14 +466,25 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "continent_names", - choices=sorted(e.name for e in continents.Continent), - nargs=argparse.ONE_OR_MORE, + "iso_3166s", + help="cover exactly the tiles these countries' boundaries touch", + nargs=argparse.ZERO_OR_MORE, + type=worldbank_jurisdictions.iso_3166_str, ) + parser.add_argument("--backfill", action="store_true", help="cover all GFW tiles") args = parser.parse_args() + assert bool(args.iso_3166s) ^ bool(args.backfill), ( + "pass either one-or-more iso_3166s or --backfill" + ) - for continent_name in map(str, args.continent_names): - workflow(tile_ids=continents.Continent[continent_name].value) + for tile_id in sorted( + tiling.GLOBAL_FOREST_WATCH_TILE_IDS + if args.backfill + else worldbank_jurisdictions.get_ten_degree_tile_ids_for_iso_3166s( + iso_3166s=args.iso_3166s + ) + ): + workflow(tile_id=tile_id) return 0 diff --git a/jdluc/geo.py b/jdluc/geo.py index de34185..d116254 100644 --- a/jdluc/geo.py +++ b/jdluc/geo.py @@ -65,6 +65,10 @@ def convert_geotiff_to_cog( ) +# NB: an equal-area projection, so that parts of one jurisdiction can be ranked by size +EQUAL_AREA_CRS = "EPSG:6933" + + def convert_vector_to_flatgeobuf( id_column_names: tuple[str, ...], name_column_names: tuple[str, ...], @@ -73,6 +77,12 @@ def convert_vector_to_flatgeobuf( ) -> None: import geopandas + def maybe_repair_mojibake(name: str) -> str: + try: + return name.encode("latin-1").decode("utf-8") + except UnicodeError: + return name + logger.info( f"Opening {path_to_vector=:s} for {id_column_names=:} and {name_column_names=:}" ) @@ -89,11 +99,16 @@ def convert_vector_to_flatgeobuf( "id": gdf[list(id_column_names)].astype(str).agg(" | ".join, axis=1), "name": gdf[list(name_column_names)] .astype(str) + .map(maybe_repair_mojibake) .agg(" | ".join, axis=1), + # NB: so that dissolve can inherit values from the largest regions + "square_meters": gdf.geometry.to_crs(crs=EQUAL_AREA_CRS).area, }, geometry=gdf.geometry, ) + .sort_values(by="square_meters", ascending=False) .dissolve(by="id") + .drop(columns="square_meters") .reset_index(drop=False) .to_file(path_to_flatgeobuf, driver="FlatGeobuf", SPATIAL_INDEX=True) ) @@ -184,8 +199,9 @@ def downscale_darray( ).name chunk_size = 1 << 11 # 2k logger.debug(f"Serializing to {path_to_unscaled=:s} with {chunk_size=:d}") - with dask.diagnostics.ProgressBar(dt=5, minimum=1): - darray.chunk(chunks=chunk_size).rio.to_raster( + darray = darray.assign_attrs(long_name=darray.name).chunk(chunks=chunk_size) + with dask.diagnostics.ProgressBar(): + darray.rio.to_raster( path_to_unscaled, blockxsize=chunk_size, blockysize=chunk_size, @@ -209,6 +225,7 @@ def downscale_darray( ) as scaled_vrt, tempfile.NamedTemporaryFile(delete=False, suffix=".warped.vrt") as scaled_fp, ): + assert unscaled_fp.crs.to_epsg() == epsg logger.debug(f"Writing warped VRT to {scaled_fp.name=:s}") rasterio.shutil.copy(scaled_vrt, scaled_fp.name, driver="VRT") ret = rioxarray.open_rasterio( @@ -221,8 +238,5 @@ def downscale_darray( lock=False, ) assert isinstance(ret, xarray.DataArray) - return unify_dtype_and_no_data( - darray=ret.drop_vars("spatial_ref") - .isel(band=0, drop=True) - .rename(ret.attrs.pop("long_name")) - ) + ret = ret.drop_vars("spatial_ref").isel(band=0, drop=True) + return unify_dtype_and_no_data(darray=ret.rename(ret.attrs.pop("long_name"))) diff --git a/jdluc/harmonize.py b/jdluc/harmonize.py index 3fb3692..f142d6d 100644 --- a/jdluc/harmonize.py +++ b/jdluc/harmonize.py @@ -1,10 +1,15 @@ -"""Mosaic ingested source tiles onto a common grid and harmonize them into one dataset. +"""Place one ingested source tile per dataset onto a common grid and harmonize them. -For a tile set, each raster dataset's tiles are stitched into a per-band GDAL VRT, warped to -a shared grid, and returned as an xarray.Dataset with one variable per source band. The result is cached. +For a single ten-degree tile, each raster dataset's tile becomes a per-band GDAL VRT on the +shared grid, returned as an xarray.Dataset with one variable per source band. Whole-world +datasets are windowed onto the same grid. The result is cached. -Example invocation: - uv run python jdluc/harmonize.py NORTH_AMERICA +The tile set comes from the positional ISO 3166 alpha-3 codes -- or, with `--backfill`, +from `tiling.GLOBAL_FOREST_WATCH_TILE_IDS`. + +Example invocations: + uv run python jdluc/harmonize.py USA + uv run python jdluc/harmonize.py --backfill """ import argparse @@ -18,14 +23,11 @@ import rasterio import rasterio.enums import rasterio.errors -import rasterio.shutil -import rasterio.transform -import rasterio.vrt import rioxarray import xarray -from jdluc import config, continents, geo, ingest, storage, tiling -from jdluc.datasets import NAME_TO_CLS, DatasetName, base +from jdluc import config, geo, ingest, storage, tiling +from jdluc.datasets import NAME_TO_CLS, DatasetName, base, worldbank_jurisdictions logger = logging.getLogger(__name__) @@ -74,12 +76,12 @@ def iter_vrt_band_header( ) -> collections.abc.Iterator[str]: yield f' ' yield f" {band_name:s}" - yield f" {no_data}" + if no_data is not None: + yield f" {no_data}" def iter_vrt_band_content( band_idx: int, - dest_offset: tiling.XY, dest_resolution: tiling.XY, path_to_tile: str, resampling: rasterio.enums.Resampling, @@ -90,66 +92,48 @@ def iter_vrt_band_content( yield f' {path_to_tile:s}' yield f" {band_idx:d}" yield f' ' - yield f' ' + # NB: a VRT covers exactly one tile, so the destination rect always starts at the origin + yield f' ' yield " " @dataclasses.dataclass class Grid: origin: tiling.XY - tiles: tiling.XY - tile_resolution: tiling.XY + resolution: tiling.XY @property def epsg(self) -> int: return 4326 @classmethod - def from_tile_ids_resolution( - cls, tile_ids: collections.abc.Sequence[str], tile_resolution: tiling.XY + def from_tile_id_resolution( + cls, tile_id: str, resolution: tiling.XY ) -> typing.Self: - lats, lons = zip(*map(tiling.get_lat_lon_for_tile_id, tile_ids), strict=True) + lat, lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) return cls( - origin=tiling.XY(x=min(lons), y=max(lats)), - tiles=tiling.XY( - x=(max(lons) - min(lons)) // 10 + 1, - y=(max(lats) - min(lats)) // 10 + 1, - ).validated(), - tile_resolution=tile_resolution, + origin=tiling.XY(x=lon, y=lat), + # NB: sanitize the provided resolution to the class we want + resolution=tiling.XY(x=resolution.x, y=resolution.y), ) @property def transform(self) -> tuple[float, float, float, float, float, float]: return ( self.origin.x, - 10 / self.tile_resolution.x, + 10 / self.resolution.x, 0, self.origin.y, 0, - -10 / self.tile_resolution.y, + -10 / self.resolution.y, ) - @property - def resolution(self) -> tiling.XY: - return tiling.XY( - x=self.tiles.x * self.tile_resolution.x, - y=self.tiles.y * self.tile_resolution.y, - ).validated() - @property def iter_preamble(self) -> collections.abc.Iterator[str]: yield f'' yield f" EPSG:{self.epsg:d}" yield f" {', '.join(map(str, self.transform))}" - def get_offset_for_tile(self, tile_id: str) -> tiling.XY: - lat, lon = tiling.get_lat_lon_for_tile_id(tile_id=tile_id) - return tiling.XY( - x=(lon - self.origin.x) // 10 * self.tile_resolution.x, - # Y index increases downward - y=(self.origin.y - lat) // 10 * self.tile_resolution.y, - ).validated() - def get_offset_for_world(self, resolution: tiling.XY, span: tiling.XY) -> tiling.XY: pixels_per_degree = tiling.XY( x=resolution.x // span.x, @@ -164,42 +148,12 @@ def get_resolution_for_world( self, resolution: tiling.XY, span: tiling.XY ) -> tiling.XY: return tiling.XY( - x=resolution.x * 10 // span.x * self.tiles.x, - y=resolution.y * 10 // span.y * self.tiles.y, + x=resolution.x * 10 // span.x, + y=resolution.y * 10 // span.y, ).validated() @staticmethod - def get_downsampling_for_band_type( - band_type: base.BandType, - ) -> rasterio.enums.Resampling: - match band_type: - case base.BandType.CATEGORICAL: - return rasterio.enums.Resampling.mode - case base.BandType.EXTENSIVE: - raise NotImplementedError("GDAL doesn't implement sum resampling") - case base.BandType.INTENSIVE: - return rasterio.enums.Resampling.average - case _: - raise ValueError(band_type) - - @staticmethod - def get_upsampling_for_band_type( - band_type: base.BandType, - ) -> rasterio.enums.Resampling: - match band_type: - case base.BandType.CATEGORICAL: - return rasterio.enums.Resampling.nearest - case base.BandType.EXTENSIVE: - raise NotImplementedError( - "GDAL doesn't implement distribution resampling" - ) - case base.BandType.INTENSIVE: - return rasterio.enums.Resampling.bilinear - case _: - raise ValueError(band_type) - def get_resampling_for_band_type( - self, band_type: base.BandType, dest_resolution: tiling.XY, src_resolution: tiling.XY, @@ -210,65 +164,83 @@ def get_resampling_for_band_type( src_resolution.x > dest_resolution.x and src_resolution.y > dest_resolution.y ): - return self.get_downsampling_for_band_type(band_type=band_type) + match band_type: + case base.BandType.CATEGORICAL: + return rasterio.enums.Resampling.mode + case base.BandType.EXTENSIVE: + raise NotImplementedError("GDAL doesn't implement sum resampling") + case base.BandType.INTENSIVE: + return rasterio.enums.Resampling.average + case _: + raise ValueError(band_type) else: - return self.get_upsampling_for_band_type(band_type=band_type) + match band_type: + case base.BandType.CATEGORICAL: + return rasterio.enums.Resampling.nearest + case base.BandType.EXTENSIVE: + raise NotImplementedError( + "GDAL doesn't implement distribution resampling" + ) + case base.BandType.INTENSIVE: + return rasterio.enums.Resampling.bilinear + case _: + raise ValueError(band_type) -def get_vrt_for_dataset_band_tile_ids( +def get_vrt_for_dataset_band_tile_id( band_idx: int, band_name: str, dataset: base.RasterDataset, grid: Grid, ignore_missing_tiles: bool, root: str, - tile_ids: collections.abc.Sequence[str], + tile_id: str, ) -> str: lines = list(grid.iter_preamble) logger.info(f"Processing {dataset=} and {band_name=:s}") if dataset.partitioning == tiling.Partitioning.TEN_DEGREE_TILE: - includes_band_header = False - for tile_id in tile_ids: - try: - tile = Tile.from_dataset_tile_id( - root=root, - dataset=dataset, - tile_id=tile_id, + try: + tile = Tile.from_dataset_tile_id( + root=root, + dataset=dataset, + tile_id=tile_id, + ) + except rasterio.errors.RasterioIOError: + if ignore_missing_tiles: + logger.warning( + f"{tile_id=:s} is missing for {dataset=} but due to {ignore_missing_tiles=} we are emitting an all-no-data band" ) - except rasterio.errors.RasterioIOError: - if ignore_missing_tiles: - logger.warning( - f"{tile_id=:s} is missing for {dataset=} but due to {ignore_missing_tiles=} we are continuing" + # Yield an empty tile band + lines.extend( + iter_vrt_band_header( + band_name=band_name, dtype="Float32", no_data=dataset.no_data ) - continue - else: - raise + ) else: - if not includes_band_header: - lines.extend( - iter_vrt_band_header( - band_name=band_name, - dtype=tile.dtype, - no_data=tile.no_data, - ) - ) - includes_band_header = True - lines.extend( - iter_vrt_band_content( - band_idx=band_idx, - dest_offset=grid.get_offset_for_tile(tile_id=tile_id), - dest_resolution=grid.tile_resolution, - path_to_tile=tile.gdal_path, - resampling=grid.get_resampling_for_band_type( - band_type=tile.band_type, - dest_resolution=grid.tile_resolution, - src_resolution=tile.resolution, - ), - src_offset=tiling.XY(x=0, y=0), + raise + else: + lines.extend( + iter_vrt_band_header( + band_name=band_name, + dtype=tile.dtype, + no_data=tile.no_data, + ) + ) + lines.extend( + iter_vrt_band_content( + band_idx=band_idx, + dest_resolution=grid.resolution, + path_to_tile=tile.gdal_path, + resampling=grid.get_resampling_for_band_type( + band_type=tile.band_type, + dest_resolution=grid.resolution, src_resolution=tile.resolution, - ) + ), + src_offset=tiling.XY(x=0, y=0), + src_resolution=tile.resolution, ) + ) elif dataset.partitioning == tiling.Partitioning.WHOLE_WORLD: tile = Tile.from_dataset_tile_id( root=root, @@ -292,7 +264,6 @@ def get_vrt_for_dataset_band_tile_ids( lines.extend( iter_vrt_band_content( band_idx=band_idx, - dest_offset=tiling.XY(x=0, y=0), dest_resolution=grid.resolution, path_to_tile=tile.gdal_path, resampling=grid.get_resampling_for_band_type( @@ -309,28 +280,11 @@ def get_vrt_for_dataset_band_tile_ids( lines.append(" ") lines.append("") - with tempfile.NamedTemporaryFile( - delete=False, mode="w", suffix=".mosaic.vrt" - ) as fp: - logger.debug(f"Writing mosaic to {fp.name=:s}") + with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".vrt") as fp: + logger.debug(f"Writing VRT to {fp.name=:s}") fp.writelines(line + "\n" for line in lines) - mosaic_path = fp.name - - with ( - rasterio.open(mosaic_path) as mosaic_fp, - rasterio.vrt.WarpedVRT( - mosaic_fp, - crs="EPSG:4326", - transform=rasterio.transform.Affine.from_gdal(*grid.transform), - width=grid.resolution.x, - height=grid.resolution.y, - resampling=rasterio.enums.Resampling.nearest, - ) as warped_vrt, - tempfile.NamedTemporaryFile(delete=False, suffix=".warped.vrt") as warped_fp, - ): - logger.debug(f"Writing warped VRT to {warped_fp.name=:s}") - rasterio.shutil.copy(warped_vrt, warped_fp.name, driver="VRT") - return warped_fp.name + path_to_vrt = fp.name + return path_to_vrt def get_dset_for_output(path_to_vrts: collections.abc.Sequence[str]) -> xarray.Dataset: @@ -345,11 +299,10 @@ def get_dset_for_output(path_to_vrts: collections.abc.Sequence[str]) -> xarray.D lock=False, ) assert isinstance(darray, xarray.DataArray) + darray = darray.isel(band=0, drop=True) darrays.append( geo.unify_dtype_and_no_data( - darray=darray.isel(band=0, drop=True).rename( - darray.attrs.pop("long_name") - ) + darray=darray.rename(darray.attrs.pop("long_name")) ) ) @@ -361,38 +314,38 @@ def workflow( dataset_names: tuple[DatasetName, ...], ignore_missing_tiles: bool, skip_ingest: bool, - tile_ids: tuple[str, ...], + tile_id: str, tile_resolution: tiling.XY, ) -> xarray.Dataset: logger.info( - f"Running the harmonize workflow for {dataset_names=:} and {tile_ids=:}" + f"Running the harmonize workflow for {dataset_names=:} and {tile_id=:s}" ) datasets = list(map(NAME_TO_CLS.__getitem__, dataset_names)) cfg = config.Config.from_dot_env() if not skip_ingest: for dataset in datasets: + # NB: this doesn't take advantage of ingest's concurrency, so + # consider running ingest over the AOI beforehand ingest.workflow( - concurrency=ingest.DEFAULT_CONCURRENCY, + concurrency=1, dataset=dataset, overwrite=False, root=cfg.ingest_root, - tile_ids=tile_ids, + tile_ids=(tile_id,), ) logger.info(f"Constructing common grid for {tile_resolution=:}") - grid = Grid.from_tile_ids_resolution( - tile_ids=tile_ids, tile_resolution=tile_resolution - ) + grid = Grid.from_tile_id_resolution(tile_id=tile_id, resolution=tile_resolution) path_to_vrts = [ - get_vrt_for_dataset_band_tile_ids( + get_vrt_for_dataset_band_tile_id( band_idx=band_idx, band_name=band_name, root=cfg.ingest_root, dataset=dataset, grid=grid, ignore_missing_tiles=ignore_missing_tiles, - tile_ids=tile_ids, + tile_id=tile_id, ) for dataset in datasets if isinstance(dataset, base.RasterDataset) @@ -421,10 +374,12 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "continent_names", - choices=sorted(e.name for e in continents.Continent), - nargs=argparse.ONE_OR_MORE, + "iso_3166s", + help="cover exactly the tiles these countries' boundaries touch", + nargs=argparse.ZERO_OR_MORE, + type=worldbank_jurisdictions.iso_3166_str, ) + parser.add_argument("--backfill", action="store_true", help="cover all GFW tiles") parser.add_argument( "--grid-name", choices=sorted(e.name for e in tiling.TileResolution), @@ -434,14 +389,23 @@ def main() -> int: parser.add_argument("--ignore-missing-tiles", action="store_true") parser.add_argument("--skip-ingest", action="store_true") args = parser.parse_args() + assert bool(args.iso_3166s) ^ bool(args.backfill), ( + "pass either one-or-more iso_3166s or --backfill" + ) - for continent_name in map(str, args.continent_names): + for tile_id in sorted( + tiling.GLOBAL_FOREST_WATCH_TILE_IDS + if args.backfill + else worldbank_jurisdictions.get_ten_degree_tile_ids_for_iso_3166s( + iso_3166s=args.iso_3166s + ) + ): workflow( dataset_names=LUC_AND_EMISSIONS_DATASET_NAMES, ignore_missing_tiles=args.ignore_missing_tiles, skip_ingest=args.skip_ingest, - tile_ids=continents.Continent[continent_name].value, - tile_resolution=tiling.TileResolution[str(args.grid_name)].value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution[str(args.grid_name)], ) return 0 diff --git a/jdluc/ingest.py b/jdluc/ingest.py index 379497f..845d5f6 100644 --- a/jdluc/ingest.py +++ b/jdluc/ingest.py @@ -1,22 +1,29 @@ -"""Ingest a named dataset over a continent's tile set. +"""Ingest a named dataset over a set of ten-degree tiles. -The parser is positional — `continent_name dataset_name` — plus optional `--concurrency` -and `--overwrite`. Continents come from `continents.Continent`; datasets from -`datasets.DatasetName`. Whole-world datasets are ingested over the "world" tile based on -their partitioning, so the continent only bounds tiled datasets. +The positionals are `dataset_name`, from `datasets.DatasetName`, then the ISO 3166 alpha-3 +codes to cover. The codes resolve to tiles via +`worldbank_jurisdictions.get_ten_degree_tile_ids_for_iso_3166s`, the same derivation +`attribute.workflow` uses, so ingesting ahead of attribution covers precisely what attribution +will ask for. Whole-world datasets are ingested over the "world" tile based on their +partitioning, so the tile set only bounds tiled datasets. + +The tile set comes from the positional ISO 3166 alpha-3 codes -- or, with `--backfill`, +from `tiling.GLOBAL_FOREST_WATCH_TILE_IDS`. Example invocations: - uv run python -m jdluc.ingest NORTH_AMERICA IPCC_CLIMATE_ZONES - uv run python -m jdluc.ingest NORTH_AMERICA GLAD_GLCLUC - uv run python -m jdluc.ingest AFRICA GFW_GLOBAL_PEATLANDS --concurrency=8 --overwrite + uv run python -m jdluc.ingest IPCC_CLIMATE_ZONES USA + uv run python -m jdluc.ingest USDA_NASS_CDL USA MEX --concurrency=8 + uv run python -m jdluc.ingest GLAD_GLCLUC BRA --overwrite + uv run python -m jdluc.ingest GLAD_GLCLUC --backfill """ import argparse +import collections.abc import concurrent.futures import logging -from jdluc import config, continents, datasets, tiling -from jdluc.datasets import base +from jdluc import config, datasets, tiling +from jdluc.datasets import base, worldbank_jurisdictions logger = logging.getLogger(__name__) @@ -26,7 +33,7 @@ def workflow( dataset: base.RasterDataset | base.TabularDataset | base.VectorDataset, overwrite: bool, root: str, - tile_ids: tuple[str, ...], + tile_ids: collections.abc.Iterable[str], ) -> dict[str, str | Exception]: if dataset.partitioning == tiling.Partitioning.WHOLE_WORLD: tile_ids = (tiling.WHOLE_WORLD_TILE_ID,) @@ -39,7 +46,7 @@ def workflow( results: dict[str, str | Exception] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: futures: dict[concurrent.futures.Future[str], str] = {} - for tile_id in tile_ids: + for tile_id in sorted(tile_ids): futures[ executor.submit( dataset.ingest_a_tile, @@ -59,9 +66,6 @@ def workflow( return results -DEFAULT_CONCURRENCY = 4 - - def main() -> int: logging.basicConfig( level=logging.INFO, @@ -70,14 +74,21 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "continent_name", choices=sorted(e.name for e in continents.Continent) + "dataset_name", choices=sorted(e.name for e in datasets.DatasetName) ) parser.add_argument( - "dataset_name", choices=sorted(e.name for e in datasets.DatasetName) + "iso_3166s", + help="cover exactly the tiles these countries' boundaries touch", + nargs=argparse.ZERO_OR_MORE, + type=worldbank_jurisdictions.iso_3166_str, ) - parser.add_argument("--concurrency", default=DEFAULT_CONCURRENCY, type=int) + parser.add_argument("--backfill", action="store_true", help="cover all GFW tiles") + parser.add_argument("--concurrency", default=4, type=int) parser.add_argument("--overwrite", action="store_true") args = parser.parse_args() + assert bool(args.iso_3166s) ^ bool(args.backfill), ( + "pass either one-or-more iso_3166s or --backfill" + ) dataset = datasets.NAME_TO_CLS[datasets.DatasetName[str(args.dataset_name)]] tile_id_to_path_or_exc: dict[str, str | Exception] = workflow( @@ -85,7 +96,13 @@ def main() -> int: dataset=dataset, overwrite=args.overwrite, root=config.Config.from_dot_env().ingest_root, - tile_ids=continents.Continent[str(args.continent_name)].value, + tile_ids=( + tiling.GLOBAL_FOREST_WATCH_TILE_IDS + if args.backfill + else worldbank_jurisdictions.get_ten_degree_tile_ids_for_iso_3166s( + iso_3166s=args.iso_3166s + ) + ), ) for tile_id, path_or_exc in tile_id_to_path_or_exc.items(): print(tile_id, repr(path_or_exc)) diff --git a/jdluc/jurisdictional_direct.py b/jdluc/jurisdictional_direct.py index a078dd7..8e8a314 100644 --- a/jdluc/jurisdictional_direct.py +++ b/jdluc/jurisdictional_direct.py @@ -38,11 +38,6 @@ class Crop(enum.Enum): ) -CDL_TO_COMMON_CROP_NAME = { - crop.value[0].name: crop.name for crop in Crop if len(crop.value) == 1 -} - - DATASET_NAMES = (DatasetName.USDA_NASS_CDL,) @@ -77,11 +72,11 @@ def get_forest_and_peatland_conversion_per_hectare( ) + soil.where((before_class == forest_class) & (is_peat != 1), other=0) span_to_peatland_conversion[(before, after)] = soil.where(is_peat == 1, other=0) return { - "forest-emissions:tco2e-per-ha": emit.get_linear_discounted_emissions( - span_to_emissions=span_to_forest + "forest-emissions:tco2e-per-ha": emit.get_linear_discounted_total( + span_to_value=span_to_forest ), - "peatland-conversion-emissions:tco2e-per-ha": emit.get_linear_discounted_emissions( - span_to_emissions=span_to_peatland_conversion + "peatland-conversion-emissions:tco2e-per-ha": emit.get_linear_discounted_total( + span_to_value=span_to_peatland_conversion ), } @@ -134,43 +129,60 @@ def get_crop_name_to_totals( return utils.get_sum_totals(enum_to_name_to_darray=crop_to_totals) -@storage.cache_to_parquet(version=0) +SCHEMA = { + "admin_id": str, + "admin_level": str, + "crop_hectares": float, + "crop_name": str, + "emissions_mt": float, + "forest_emissions_mt": float, + "jurisdiction_name": str, + "peatland_conversion_emissions_mt": float, + "peatland_crop_hectares": float, + "peatland_occupation_emissions_mt": float, +} + + +@storage.cache_to_parquet(version=1) def workflow( crop_names: tuple[str, ...], iso_3166: str, skip_glad_crop_filter: bool, - tile_ids: tuple[str, ...], + tile_id: str, ) -> pandas.DataFrame: - crops = tuple(map(Crop.__getitem__, crop_names)) + crops = tuple(Crop[crop_name] for crop_name in crop_names) assert iso_3166 == "USA", "JD only supports USA today" - logger.info(f"Computing emissions for {crops=:}") + logger.info(f"Computing emissions for {crops=:} and {tile_id=:s}") merged = geo.exact_merge( harmonize.workflow( dataset_names=harmonize.LUC_AND_EMISSIONS_DATASET_NAMES, - ignore_missing_tiles=False, + ignore_missing_tiles=True, skip_ingest=False, - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.GLAD.value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution.GLAD, ), # NB: this should call the harmonize workflow with identical args and hit # the cache from the preceding call - emit.workflow(tile_ids=tile_ids), + emit.workflow(tile_id=tile_id), # CDL harmonize.workflow( dataset_names=DATASET_NAMES, - ignore_missing_tiles=False, + ignore_missing_tiles=True, skip_ingest=False, - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.GLAD.value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution.GLAD, ), ) merged = merged.assign(get_forest_and_peatland_conversion_per_hectare(dset=merged)) def it() -> collections.abc.Iterator[dict[str, float | str]]: - for jurisdiction in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166( + for ( + jurisdiction + ) in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166_tile_id( admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL, iso_3166=iso_3166, + tile_id=tile_id, ): logger.info(f"Clipping to provincial geometry for {jurisdiction.id=:s}") try: @@ -207,11 +219,15 @@ def it() -> collections.abc.Iterator[dict[str, float | str]]: for crop_name, totals in crop_name_to_totals.items(): yield totals | { "admin_id": jurisdiction.id, - "admin_level": worldbank_jurisdictions.AdminLevel.PROVINCIAL.name, + "admin_level": jurisdiction.level, "crop_name": crop_name, "jurisdiction_name": jurisdiction.name, } - return pandas.DataFrame.from_records(data=it()).set_index( - ["admin_level", "crop_name", "jurisdiction_name"] + if data := list(it()): + assert set(data[0]) == set(SCHEMA) + return ( + pandas.DataFrame.from_records(columns=list(SCHEMA), data=data) + .astype(SCHEMA) + .set_index(["admin_level", "admin_id", "crop_name"]) ) diff --git a/jdluc/statistical.py b/jdluc/statistical.py index 6755bb8..761d10d 100644 --- a/jdluc/statistical.py +++ b/jdluc/statistical.py @@ -10,7 +10,6 @@ from jdluc import emit, geo, harmonize, storage, tiling, utils from jdluc.datasets import ( DatasetName, - base, gfw_global_peatlands, glad_glcluc, ifpri_mapspam, @@ -24,7 +23,7 @@ class Crop(enum.StrEnum): BARLEY = ifpri_mapspam.Crop2000.BARL.name BEAN = ifpri_mapspam.Crop2000.BEAN.name - CASSAVE = ifpri_mapspam.Crop2000.CASS.name + CASSAVA = ifpri_mapspam.Crop2000.CASS.name COTTON = ifpri_mapspam.Crop2000.COTT.name GROUNDNUT = ifpri_mapspam.Crop2000.GROU.name MAIZE = ifpri_mapspam.Crop2000.MAIZ.name @@ -57,11 +56,7 @@ class Crop(enum.StrEnum): YAM = ifpri_mapspam.Crop2005.YAMS.name -assert all( - e.value - in ifpri_mapspam.SHARED_CROP_NAMES | set(ifpri_mapspam.CONSTITUENT_TO_GROUP_NAME) - for e in Crop -) +assert {e.value for e in Crop} == ifpri_mapspam.RECOVERABLE_CROP_NAMES DATASET_NAMES = ( @@ -89,31 +84,22 @@ class Crop(enum.StrEnum): ] -def get_band_type_for_variable_name(variable_name: str) -> base.BandType: - if "-per-ha" in variable_name: - return base.BandType.INTENSIVE - elif variable_name.endswith((":ha", ":tco2e")): - return base.BandType.EXTENSIVE - else: - return base.BandType.CATEGORICAL - - -@storage.cache_to_zarr(version=1) +@storage.cache_to_zarr(version=0) def get_downscaled_luc_emissions( - skip_glad_crop_filter: bool, tile_ids: tuple[str, ...] + skip_glad_crop_filter: bool, tile_id: str ) -> xarray.Dataset: logger.info("Computing emissions on the GLAD grid") glad_emissions = geo.exact_merge( harmonize.workflow( dataset_names=harmonize.LUC_AND_EMISSIONS_DATASET_NAMES, - ignore_missing_tiles=False, + ignore_missing_tiles=True, skip_ingest=False, - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.GLAD.value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution.GLAD, ), # NB: this should call the harmonize workflow with identical args and hit # the cache from the preceding call - emit.workflow(tile_ids=tile_ids), + emit.workflow(tile_id=tile_id), ) logger.info("Splitting forest / peatland-conversion per span") @@ -151,9 +137,8 @@ def get_downscaled_luc_emissions( ) logger.info("Downsampling emissions from the GLAD grid to the MAPSPAM grid") - grid = harmonize.Grid.from_tile_ids_resolution( - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.MAPSPAM.value, # type: ignore + grid = harmonize.Grid.from_tile_id_resolution( + resolution=tiling.TileResolution.MAPSPAM, tile_id=tile_id ) return xarray.Dataset( { @@ -171,131 +156,62 @@ def get_downscaled_luc_emissions( ) -def get_band_name_for_crop_name( - crop_name: str, - quantity: ifpri_mapspam.Quantity, - year: int, -) -> str: - # NB: this works because enum and band orders are the same - crop_cls = ifpri_mapspam.YEAR_TO_CROP_CLS[year] - name_to_idx = {name: idx for idx, name in enumerate(e.name for e in crop_cls)} - idx = name_to_idx[crop_name] - year_to_dataset = ( - ifpri_mapspam.YEAR_TO_PHYSICAL_AREA_DATASET - if quantity == ifpri_mapspam.Quantity.PHYSICAL_AREA - else ifpri_mapspam.YEAR_TO_PRODUCTION_DATASET - ) - dataset = year_to_dataset[year] - return dataset.fully_qualified_band_names[idx] - - -def get_harmonized_quantity( - crop_name: str, - dset: xarray.Dataset, - quantity: ifpri_mapspam.Quantity, - year: int, -) -> xarray.DataArray: - def snapshot( - crop_name: str, quantity: ifpri_mapspam.Quantity, year: int - ) -> xarray.DataArray: - variable_name = get_band_name_for_crop_name( - crop_name=crop_name, quantity=quantity, year=year - ) - return dset[variable_name].fillna(0) - - if ( - year == ifpri_mapspam.YEAR_TO_DECOMPOSE - and crop_name not in ifpri_mapspam.SHARED_CROP_NAMES - ): - # Decompose the grouped crops into their constituents, assuming the within-group - # proportions match that of the reference year - group_name = ifpri_mapspam.CONSTITUENT_TO_GROUP_NAME[crop_name] - siblings = sorted(ifpri_mapspam.GROUP_TO_CONSTITUENT_NAMES[group_name]) - reference_group_area = sum( - snapshot( - crop_name=s, - quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, - year=ifpri_mapspam.DECOMPOSITION_REFERENCE_YEAR, - ) - for s in siblings - ) - assert isinstance(reference_group_area, xarray.DataArray) - constituent_share = xarray.where( - reference_group_area > 0, - snapshot( - crop_name=crop_name, - quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, - year=ifpri_mapspam.DECOMPOSITION_REFERENCE_YEAR, - ) - # NB: this avoids `RuntimeWarning: invalid value encountered in divide` - / reference_group_area.where(reference_group_area > 0, other=1), - # Group absent in the reference year -> split it evenly - 1 / len(siblings), - ) - return ( - snapshot( - crop_name=group_name, - quantity=quantity, - year=ifpri_mapspam.YEAR_TO_DECOMPOSE, - ) - * constituent_share - ) - else: - # Simple lookup - return snapshot( - crop_name=ifpri_mapspam.map_2005_name_to_year( - crop_name=crop_name, year=year - ), - quantity=quantity, - year=year, - ) - - def get_crop_to_share( after: int, before: int, crops: tuple[Crop, ...], dset: xarray.Dataset ) -> dict[Crop, xarray.DataArray]: - def snapshot_area(crop_name: str, year: int) -> xarray.DataArray: - variable_name = get_band_name_for_crop_name( - crop_name=crop_name, - quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, - year=year, - ) - return dset[variable_name].fillna(0) - - def expansion(crop_name: str) -> xarray.DataArray: + def get_expansion(canonical_crop_name: str) -> xarray.DataArray: return ( - get_harmonized_quantity( - crop_name=crop_name, + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=canonical_crop_name, dset=dset, quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, year=after, ) - - get_harmonized_quantity( - crop_name=crop_name, + - ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=canonical_crop_name, dset=dset, quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, year=before, ) ).clip(min=0) - common_expansion = sum(map(expansion, sorted(ifpri_mapspam.SHARED_CROP_NAMES))) + attributed_expansion = sum( + get_expansion(canonical_crop_name=name) + for name in sorted(ifpri_mapspam.RECOVERABLE_CROP_NAMES) + ) - def residual(year: int) -> xarray.DataArray: - # NB: no expansion logic is required so we can do simple snapshot lookups + # Drop any crops which are being newly tracked so they aren't interpreted as an expansion from zero + before_names = ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[before] + after_names = ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES[after] + if (before, after) in ifpri_mapspam.SPANS_WITH_COMPARABLE_CROP_NAMES: + before_names = after_names = before_names & after_names + + def get_unattributed(year: int, names: set[str]) -> xarray.DataArray: + # NB: no canonical lookup is required -- these names are already `year`'s own, and + # nothing divides by them, so the raw band is the whole answer ret = sum( - snapshot_area(crop_name=e.name, year=year) - for e in ifpri_mapspam.YEAR_TO_CROP_CLS[year] - if e.name not in ifpri_mapspam.SHARED_CROP_NAMES + ifpri_mapspam.get_reported_quantity( + dset=dset, + quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, + reported_crop_name=name, + year=year, + ) + for name in sorted(names) ) assert isinstance(ret, xarray.DataArray) return ret - residual_expansion = (residual(year=after) - residual(year=before)).clip(min=0) - total_expansion = common_expansion + residual_expansion + unattributed_expansion = ( + get_unattributed(year=after, names=after_names) + - get_unattributed(year=before, names=before_names) + ).clip(min=0) + total_expansion = attributed_expansion + unattributed_expansion total_expansion = total_expansion.where(total_expansion > 0) return { # Share is zero when there is no expansion at all - crop: (expansion(crop_name=crop.value) / total_expansion).fillna(0) + crop: (get_expansion(canonical_crop_name=crop.value) / total_expansion).fillna( + 0 + ) for crop in crops } @@ -314,7 +230,7 @@ def residual(year: int) -> xarray.DataArray: def get_crop_name_to_totals( dset: xarray.Dataset, crop_to_span_to_share: dict[Crop, dict[emit.SpanType, xarray.DataArray]], - whole_period_shares: dict[Crop, xarray.DataArray], + occupation_shares: dict[Crop, xarray.DataArray], ) -> dict[str, dict[str, float]]: peatland_fraction = dset[gfw_global_peatlands.DATASET.fully_qualified_band_name] hectares = emit.get_hectares_per_pixel(darray=peatland_fraction) @@ -323,9 +239,10 @@ def get_crop_name_to_totals( crop_to_totals: dict[Crop, dict[str, xarray.DataArray]] = collections.defaultdict( dict ) + weight_total = sum(emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT.values()) for crop, span_to_share in crop_to_span_to_share.items(): - conversion = emit.get_linear_discounted_emissions( - span_to_emissions={ + conversion = emit.get_linear_discounted_total( + span_to_value={ (before, after): dset[f"emissions:tco2e-per-ha:{before:d}-{after:d}"] * hectares * span_to_share[GLAD_TO_MAPSPAM_SPAN[(before, after)]] @@ -335,25 +252,35 @@ def get_crop_name_to_totals( ) in emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT } ) - peatland = peatland_occupation * whole_period_shares[crop] - crop_name = ifpri_mapspam.map_2005_name_to_year( - crop_name=crop.value, - year=2020, - ) - crop_hectares = dset[ - get_band_name_for_crop_name( - crop_name=crop_name, - quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, - year=2020, + peatland = peatland_occupation * occupation_shares[crop] + crop_hectares = ( + emit.get_linear_discounted_total( + span_to_value={ + glad_span: ( + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=crop.value, + dset=dset, + quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, + year=before, + ) + + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=crop.value, + dset=dset, + quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, + year=after, + ) + ) + / 2 + for glad_span, (before, after) in GLAD_TO_MAPSPAM_SPAN.items() + } ) - ] - assert isinstance(crop_hectares, xarray.DataArray) - + / weight_total + ) crop_to_totals[crop]["crop_hectares"] = crop_hectares for source in ("forest", "peatland_conversion"): crop_to_totals[crop][f"{source:s}_emissions_mt"] = ( - emit.get_linear_discounted_emissions( - span_to_emissions={ + emit.get_linear_discounted_total( + span_to_value={ (before, after): dset[ f"{source:s}:tco2e-per-ha:{before:d}-{after:d}" ] @@ -368,69 +295,110 @@ def get_crop_name_to_totals( ) crop_to_totals[crop]["peatland_occupation_emissions_mt"] = peatland crop_to_totals[crop]["emissions_mt"] = conversion + peatland - # Reduce production over the windows using the same linear temporal discounting - weight_total = sum(emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT.values()) crop_to_totals[crop]["production_mt"] = ( - sum( - weight - * ( - get_harmonized_quantity( - crop_name=crop.value, - dset=dset, - quantity=ifpri_mapspam.Quantity.PRODUCTION, - year=mapspam_before, + emit.get_linear_discounted_total( + span_to_value={ + glad_span: ( + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=crop.value, + dset=dset, + quantity=ifpri_mapspam.Quantity.PRODUCTION, + year=before, + ) + + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=crop.value, + dset=dset, + quantity=ifpri_mapspam.Quantity.PRODUCTION, + year=after, + ) ) - + get_harmonized_quantity( - crop_name=crop.value, - dset=dset, - quantity=ifpri_mapspam.Quantity.PRODUCTION, - year=mapspam_after, - ) - ) - / 2 - for ( - before, - after, - ), weight in emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT.items() - for (mapspam_before, mapspam_after) in [ - GLAD_TO_MAPSPAM_SPAN[(before, after)] - ] - ) # type: ignore + / 2 + for glad_span, (before, after) in GLAD_TO_MAPSPAM_SPAN.items() + } + ) / weight_total ) return utils.get_sum_totals(enum_to_name_to_darray=crop_to_totals) -@storage.cache_to_parquet(version=0) +def get_crop_to_area_share( + crops: tuple[Crop, ...], dset: xarray.Dataset, year: int +) -> dict[Crop, xarray.DataArray]: + # NB: the denominator walks `year`'s own taxonomy, so every name here is already reported + total_area = sum( + ifpri_mapspam.get_reported_quantity( + dset=dset, + quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, + reported_crop_name=e.name, + year=year, + ) + for e in ifpri_mapspam.YEAR_TO_CROP_CLS[year] + ) + assert isinstance(total_area, xarray.DataArray) + total_area = total_area.where(total_area > 0) + return { + # Share is zero when no crop occupies the cell at all + crop: ( + ifpri_mapspam.get_canonical_quantity( + canonical_crop_name=crop.value, + dset=dset, + quantity=ifpri_mapspam.Quantity.PHYSICAL_AREA, + year=year, + ) + / total_area + ).fillna(0) + for crop in crops + } + + +SCHEMA = { + "admin_id": str, + "admin_level": str, + "crop_hectares": float, + "crop_name": str, + "emissions_mt": float, + "forest_emissions_mt": float, + "jurisdiction_name": str, + "peatland_conversion_emissions_mt": float, + "peatland_crop_hectares": float, + "peatland_occupation_emissions_mt": float, + "production_mt": float, +} + + +@storage.cache_to_parquet(version=1) def workflow( crop_names: tuple[str, ...], iso_3166: str, skip_glad_crop_filter: bool, - tile_ids: tuple[str, ...], + tile_id: str, ) -> pandas.DataFrame: from rioxarray.exceptions import NoDataInBounds crops = tuple(Crop[crop_name] for crop_name in crop_names) - logger.info(f"Computing emissions for {crops=:}") + logger.info(f"Computing emissions for {crops=:} and {tile_id=:s}") merged = geo.exact_merge( # NB: this is deferred because it is expensive and would like to cache it get_downscaled_luc_emissions( - skip_glad_crop_filter=skip_glad_crop_filter, tile_ids=tile_ids + skip_glad_crop_filter=skip_glad_crop_filter, tile_id=tile_id ), # MAPSPAM harmonize.workflow( dataset_names=DATASET_NAMES, - ignore_missing_tiles=False, + ignore_missing_tiles=True, skip_ingest=False, - tile_ids=tile_ids, - tile_resolution=tiling.TileResolution.MAPSPAM.value, + tile_id=tile_id, + tile_resolution=tiling.TileResolution.MAPSPAM, ), ) def it() -> collections.abc.Iterator[dict[str, float | str]]: - for jurisdiction in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166( + for ( + jurisdiction + ) in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166_tile_id( admin_level=worldbank_jurisdictions.AdminLevel.PROVINCIAL, iso_3166=iso_3166, + tile_id=tile_id, ): logger.info(f"Clipping to provincial geometry from {jurisdiction.id=:s}") try: @@ -445,21 +413,15 @@ def it() -> collections.abc.Iterator[dict[str, float | str]]: crops=crops, dset=clipped, ) - for (before, after) in GLAD_TO_MAPSPAM_SPAN.values() + for (before, after) in dict.fromkeys(GLAD_TO_MAPSPAM_SPAN.values()) + } + crop_to_span_to_share = { + crop: { + span: crop_to_share[crop] + for span, crop_to_share in span_to_crop_to_share.items() + } + for crop in crops } - crop_to_span_to_share: dict[ - Crop, dict[emit.SpanType, xarray.DataArray] - ] = collections.defaultdict(dict) - for span, crop_to_share in span_to_crop_to_share.items(): - for crop, share in crop_to_share.items(): - crop_to_span_to_share[crop][span] = share - - whole_period_shares = get_crop_to_share( - after=max(ifpri_mapspam.YEARS), - before=min(ifpri_mapspam.YEARS), - crops=crops, - dset=clipped, - ) logger.info( f"Populating emissions for {jurisdiction.id=!s}/{len(crops)=:d} crops" @@ -467,16 +429,28 @@ def it() -> collections.abc.Iterator[dict[str, float | str]]: crop_name_to_totals = get_crop_name_to_totals( crop_to_span_to_share=crop_to_span_to_share, dset=clipped, - whole_period_shares=whole_period_shares, + # NB: unlike conversion emissions, peatland occupation is a land-management flux on land + # that is drained *now* -- it has no relationship to expansion, and expansion is zero on + # the long-established peat cropland that dominates this pool. So allocate it by each + # crop's share of area occupied -- matching the approach for the jurisdictional-direct leg. + occupation_shares=get_crop_to_area_share( + crops=crops, + dset=clipped, + year=max(ifpri_mapspam.YEARS), + ), ) for crop_name, totals in crop_name_to_totals.items(): yield totals | { "admin_id": jurisdiction.id, - "admin_level": worldbank_jurisdictions.AdminLevel.PROVINCIAL.name, + "admin_level": jurisdiction.level, "crop_name": crop_name, "jurisdiction_name": jurisdiction.name, } - return pandas.DataFrame.from_records(data=it()).set_index( - ["admin_level", "crop_name", "jurisdiction_name"] + if data := list(it()): + assert set(data[0]) == set(SCHEMA) + return ( + pandas.DataFrame.from_records(columns=list(SCHEMA), data=data) + .astype(SCHEMA) + .set_index(["admin_level", "admin_id", "crop_name"]) ) diff --git a/jdluc/storage.py b/jdluc/storage.py index a4643c5..e592497 100644 --- a/jdluc/storage.py +++ b/jdluc/storage.py @@ -4,6 +4,7 @@ import inspect import logging import os +import threading import typing import pandas @@ -11,6 +12,8 @@ logger = logging.getLogger(__name__) +COMPUTE_LOCK = threading.RLock() + def path_exists(uri: str) -> bool: import fsspec # type: ignore[import-untyped] @@ -54,10 +57,8 @@ def write_dask_dataset_to_zarr(dset: xarray.Dataset, path_to_zarr: str) -> None: logging.getLogger("distributed.scheduler").setLevel(logging.CRITICAL) logging.getLogger("gcsfs").setLevel(logging.WARNING) - logger.info( - f"Saving to {path_to_zarr=:s} with {num_workers=:d} and {dset.chunksizes=:}" - ) with ( + COMPUTE_LOCK, rasterio.Env( CPL_VSIL_CURL_ALLOWED_EXTENSIONS=".tif,.vrt", # 256 MiB @@ -77,8 +78,15 @@ def write_dask_dataset_to_zarr(dset: xarray.Dataset, path_to_zarr: str) -> None: n_workers=1, threads_per_worker=num_workers, ) as cluster, - distributed.Client(cluster) as client, + distributed.Client( + cluster, + # NB: avoid race by not sharing across threads + set_as_default=False, + ) as client, ): + logger.info( + f"Saving to {path_to_zarr=:s} with {num_workers=:d} and {dset.chunksizes=:}" + ) logger.info(f"Dask dashboard at {client.dashboard_link:s}") logger.info("Building graph and writing zarr metadata+coords") to_write = dset.drop_vars("spatial_ref", errors="ignore") @@ -88,8 +96,8 @@ def write_dask_dataset_to_zarr(dset: xarray.Dataset, path_to_zarr: str) -> None: compute=False, consolidated=False, group=None, store=path_to_zarr ) logger.info("Submitting task graph to dask scheduler") - future = client.compute(delayed) - distributed.progress(future) + future = client.compute(delayed, retries=5) + distributed.progress(future, scheduler=client.scheduler.address) future.result() logger.info(f"Finished writing to {path_to_zarr=:s}") @@ -161,9 +169,7 @@ def serialize(self, dset: xarray.Dataset) -> None: class CacherDecoratorProtocol(typing.Protocol[R]): - def __call__( - self, func: typing.Callable[P, R] - ) -> functools._lru_cache_wrapper[R]: ... + def __call__(self, func: typing.Callable[P, R]) -> typing.Callable[P, R]: ... def get_cache_decorator( @@ -182,7 +188,7 @@ def get_module_for_func(func: typing.Callable[..., object]) -> str: def decorator( func: typing.Callable[P, R], - ) -> functools._lru_cache_wrapper[R]: + ) -> typing.Callable[P, R]: if ignored_args is not None: assert set(inspect.signature(func).parameters).issuperset(ignored_args) @@ -218,7 +224,7 @@ def inner(*args: P.args, **kwargs: P.kwargs) -> R: # compute graph return cacher.deserialize() - return inner + return typing.cast(typing.Callable[P, R], inner) return decorator diff --git a/jdluc/trace.py b/jdluc/trace.py index 96e8dd2..7a31c65 100644 --- a/jdluc/trace.py +++ b/jdluc/trace.py @@ -9,8 +9,12 @@ peatland-occupation fraction, and rolls the provincials up to national totals. Returns a pandas.DataFrame indexed by (admin level, crop, jurisdiction) which is cached. -Example invocation: +The countries come from the positional ISO 3166 alpha-3 codes, or -- with `--backfill` -- +from every country in the World Bank admin-0 layer. + +Example invocations: uv run python jdluc/trace.py --methodology-name STATISTICAL USA + uv run python jdluc/trace.py --methodology-name STATISTICAL --backfill """ import argparse @@ -19,7 +23,7 @@ import pandas -from jdluc import attribute, jurisdictional_direct, storage +from jdluc import attribute, storage from jdluc.datasets import usda_nass_quickstats, worldbank_jurisdictions logger = logging.getLogger(__name__) @@ -27,6 +31,16 @@ NASS_YIELD_YEARS = (2017, 2018, 2019, 2020) KG_PER_TONNE = 1000 +CANONICAL_KEY = ("admin_level", "admin_id", "crop_name", "methodology") +ADDITIVE_COLUMNS = ( + "crop_hectares", + "forest_emissions_mt", + "peatland_crop_hectares", + "peatland_conversion_emissions_mt", + "peatland_occupation_emissions_mt", + "emissions_mt", + "production_kg", +) def derive_jurisdictional_production_kg( @@ -37,7 +51,6 @@ def derive_jurisdictional_production_kg( .groupby(level=["admin_id", "crop_name"])["yield_kg_per_ha"] .mean() .reset_index() - .replace(jurisdictional_direct.CDL_TO_COMMON_CROP_NAME) ) merged = emissions.reset_index().merge( reduced_yields, how="left", on=["admin_id", "crop_name"] @@ -48,7 +61,7 @@ def derive_jurisdictional_production_kg( merged["production_kg"] = merged["crop_hectares"] * merged["yield_kg_per_ha"] return ( merged.drop(columns="yield_kg_per_ha") - .set_index(["admin_level", "crop_name", "jurisdiction_name", "methodology"]) + .set_index(list(CANONICAL_KEY)) .sort_index() ) @@ -89,15 +102,7 @@ def safe_div(numer: float, denom: float) -> float: ret = { column_name: float(group[column_name].sum()) - for column_name in ( - "crop_hectares", - "forest_emissions_mt", - "peatland_crop_hectares", - "peatland_conversion_emissions_mt", - "peatland_occupation_emissions_mt", - "emissions_mt", - "production_kg", - ) + for column_name in ADDITIVE_COLUMNS } yield ret | { "admin_level": worldbank_jurisdictions.AdminLevel.NATIONAL.name, @@ -117,7 +122,7 @@ def safe_div(numer: float, denom: float) -> float: } # type: ignore -@storage.cache_to_parquet(version=1) +@storage.cache_to_parquet(version=0) def workflow( crop_names: tuple[str, ...], iso_3166s: tuple[str, ...], @@ -125,11 +130,13 @@ def workflow( skip_glad_crop_filter: bool, ) -> pandas.DataFrame: emissions = attribute.workflow( + concurrency=attribute.DEFAULT_CONCURRENCY, crop_names=crop_names, iso_3166s=iso_3166s, methodology=methodology, skip_glad_crop_filter=skip_glad_crop_filter, ) + assert tuple(emissions.index.names) == CANONICAL_KEY emissions_and_yields = ( derive_jurisdictional_production_kg( emissions=emissions, raw_yields=usda_nass_quickstats.load() @@ -155,7 +162,9 @@ def workflow( admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL ).loc[iso_3166]["name"] ), - provincials=provincials[provincials.admin_id.str.startswith(iso_3166)], + provincials=provincials[ + provincials.index.get_level_values("admin_id").str.startswith(iso_3166) + ], ) ) return pandas.concat( @@ -176,24 +185,37 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "iso_3166s", nargs=argparse.ONE_OR_MORE, type=attribute.iso_3166_str + "iso_3166s", + nargs=argparse.ZERO_OR_MORE, + type=worldbank_jurisdictions.iso_3166_str, + ) + parser.add_argument( + "--backfill", + action="store_true", + help="trace every country in the World Bank admin-0 layer", ) parser.add_argument( "--methodology-name", choices=sorted(e.name for e in attribute.Methodology), default=attribute.Methodology.STATISTICAL.name, ) - parser.add_argument("--process-few-crops", action="store_true") parser.add_argument("--skip-display", action="store_true") parser.add_argument("--skip-glad-crop-filter", action="store_true") args = parser.parse_args() + assert bool(args.iso_3166s) ^ bool(args.backfill), ( + "pass either one-or-more iso_3166s or --backfill" + ) methodology = attribute.Methodology[str(args.methodology_name)] df = workflow( - crop_names=attribute.get_crop_names( - methodology=methodology, process_few_crops=args.process_few_crops + crop_names=attribute.get_crop_names(methodology=methodology), + iso_3166s=tuple( + sorted( + worldbank_jurisdictions.get_all_iso_3166s() + if args.backfill + else args.iso_3166s + ) ), - iso_3166s=tuple(sorted(args.iso_3166s)), methodology=methodology, skip_glad_crop_filter=args.skip_glad_crop_filter, ) diff --git a/pyproject.toml b/pyproject.toml index 5ebea3e..6e37fee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "jdluc" version = "0.1.0" -description = "Cornerstone jdLUC: high-resolution jurisdictional direct LUC methodology, and pipeline for US row crops." +description = "Cornerstone LUC: high-resolution jurisdictional direct LUC methodology, and pipeline for US crops." readme = "README.md" requires-python = ">=3.14,<3.15" @@ -12,6 +12,7 @@ dependencies = [ "distributed", "gcsfs", "geopandas", + "iso3166", "netCDF4", "pandas", "pyarrow", @@ -58,7 +59,17 @@ follow_imports = "skip" implicit_reexport = true [tool.importlinter] -root_package = "jdluc" +root_packages = ["jdluc", "validation"] + +[[tool.importlinter.contracts]] +name = "The pipeline never imports the validator" +type = "forbidden" +source_modules = [ + "jdluc", +] +forbidden_modules = [ + "validation", +] [[tool.importlinter.contracts]] name = "ETL" @@ -77,7 +88,6 @@ name = "Utilities are foundational" type = "forbidden" source_modules = [ "jdluc.config", - "jdluc.continents", "jdluc.geo", "jdluc.storage", "jdluc.tiling", diff --git a/tools/backfill-trace.py b/tools/backfill-trace.py deleted file mode 100644 index 1e80780..0000000 --- a/tools/backfill-trace.py +++ /dev/null @@ -1,74 +0,0 @@ -import argparse -import concurrent.futures -import dataclasses -import logging -import random -import subprocess -import sys -import typing - -from jdluc import continents - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass -class Trace: - iso_3166: str - returncode: int - stdout: str - - @classmethod - def from_iso_3166(cls, iso_3166: str) -> typing.Self: - result = subprocess.run( - [sys.executable, "-m", "jdluc.trace", "--skip-display", iso_3166], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - return cls( - iso_3166=iso_3166, returncode=result.returncode, stdout=result.stdout - ) - - -def main() -> int: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--workers", type=int, default=4) - args = parser.parse_args() - - iso_3166s = [ - iso_3166 - for iso_3166, continent in continents.ISO_3166_TO_CONTINENT.items() - if continent != continents.Continent.UNCLASSIFIED - ] - random.shuffle(iso_3166s) - - failed: list[Trace] = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool: - futures: dict[concurrent.futures.Future[Trace], str] = { - pool.submit(Trace.from_iso_3166, iso_3166): iso_3166 - for iso_3166 in iso_3166s - } - for completed, future in enumerate( - concurrent.futures.as_completed(futures), start=1 - ): - trace = future.result() - logger.log( - logging.INFO if trace.returncode == 0 else logging.ERROR, - f"< {trace.iso_3166:s} > exited {trace.returncode:d} ({completed:d}/{len(iso_3166s):d})", - ) - if trace.returncode != 0: - failed.append(trace) - - for trace in failed: - print(trace.iso_3166) - return len(failed) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/build-national-mappings.py b/tools/build-national-mappings.py new file mode 100644 index 0000000..b17e6dd --- /dev/null +++ b/tools/build-national-mappings.py @@ -0,0 +1,587 @@ +"""Materialize the GADM-to-World-Bank admin-1 key map, once, for review and committing. + +WRI publishes provincial factors against GADM's `GID_1` (`USA.10_1`); the capture keys provinces on +the World Bank's `ADM1CD_c` (`USA001`). Nothing published joins the two, so they are matched here on +country and province name -- which is fuzzy, and therefore something to do once, review, and commit, +rather than at the top of every run. + +Runtime then does a strict lookup and fails on a missing key. That is deliberate: silently dropping +a +province would quietly shrink a rollup and make its coverage look better than it is. + +A tool rather than part of `validation/` because its output is the artifact: it runs rarely, reads a +93 MiB GeoPackage, needs geopandas, and nothing on the reporting path imports it. `validation/` is +left as the reporting path, with no geospatial dependency and no large inputs. + +The map is checked at the end of the run rather than by a test suite, and a failed check exits +nonzero. That is deliberate for a generator -- the report's exit code never depends on its results, +but a build tool that emits a broken artifact should fail loudly. + + uv run --with geopandas python tools/build-national-mappings.py + uv run --with geopandas python tools/build-national-mappings.py --overwrite +""" + +import argparse +import collections.abc +import dataclasses +import json +import logging +import pathlib +import unicodedata + +import geopandas +import pandas + +from jdluc import tiling, utils +from jdluc.datasets import worldbank_jurisdictions +from validation import prepare, pull, schema + +logger = logging.getLogger(__name__) + +OUTPUT = pull.DATA / "gadm_to_world_bank_admin_1.json" +# Names GADM and the World Bank spell differently enough that no normalization will join them. Kept +# here rather than in the output so a re-run cannot silently lose a hand-made decision. +OVERRIDES = pull.DATA / "admin_1_overrides.json" +# Orbae's provincial ids are opaque (`USA-20230119-1`), so its provinces join on name like GADM's do. +# Every country in the export is attempted; the countries are read off it rather than listed, so the +# scope cannot drift from the data. +ORBAE_OUTPUT = pull.DATA / "orbae_to_world_bank_admin_1.json" + +# An equal-area projection, so a province's area does not depend on its latitude. +EQUAL_AREA_CRS = "EPSG:6933" +SQUARE_METRES_PER_SQUARE_KILOMETRE = 1e6 + +WRI_REVISION = "559fe23eb752e9df270a1bf93e7f290044026bab" +GADM_KEY_URL = ( + f"https://raw.githubusercontent.com/wri/GCSC/{WRI_REVISION:s}/data/gadm_admin_keys/" + "key_gadm_adm1.csv" +) +# Administrative-type words one side appends and the other does not: the World Bank writes "Anhui +# Sheng" where GADM writes "Anhui". Stripped from both sides, so neither spelling is privileged. +ADMINISTRATIVE_WORDS = frozenset( + { + "administrativeregion", + "autonomousregion", + "canton", + "capital", + "city", + "county", + "department", + "departamento", + "district", + "division", + "emirate", + "governorate", + "krai", + "krong", + "kray", + "municipality", + "oblast", + "oblasti", + "okrug", + "parish", + "prefecture", + "province", + "provincia", + "regiao", + "region", + "republic", + "sheng", + "state", + "territory", + "voivodeship", + "zhou", + } +) +# A floor rather than an expected value: the share moves whenever either source is revised, and +# pinning it exactly would fail on every legitimate change. It is here to catch a collapse -- a +# renamed column matching nothing -- not to police normal drift. 62.8% at the current revisions. +MINIMUM_MATCHED_SHARE = 0.55 +# The Orbae map's own floor. Lower than GADM's because a third of its misses are structural rather +# than spelling -- Cote d'Ivoire's regions against districts, France's post-2016 regions against +# pre-reform ones -- and no map can close those. 77.1% at the current export. +MINIMUM_ORBAE_MATCHED_SHARE = 0.70 + + +def get_keys(name: str) -> set[str]: + """Every comparable form of a name, because one side sometimes gives two. + + The World Bank writes Canadian provinces bilingually -- "British Columbia / Colombie- + Britannique" + -- where GADM gives English only, so each alternative is offered separately as well as the + whole. + """ + parts = [part for part in str(name).split("/") if part.strip()] + return {normalize(name=part) for part in [name, *parts]} - {""} + + +def normalize(name: str) -> str: + """A comparable form of a province name: unaccented, lowercase, alphanumeric only. + + Accents are the largest single source of mismatch -- "Bie" against "Bié" -- and neither side is + consistently accented, so both are folded rather than one being corrected to the other. + """ + decomposed = unicodedata.normalize("NFKD", str(name)) + stripped = "".join( + character + for character in decomposed + if (not unicodedata.combining(character) and character.isalnum()) + or character.isspace() + ) + words = [word for word in stripped.casefold().split() if word] + kept = [word for word in words if word not in ADMINISTRATIVE_WORDS] or words + return "".join(kept) + + +def download_world_bank(path: pathlib.Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + worldbank_jurisdictions.ADMIN_1_DATASET.save_tile_id_to_local_path( + str(path), tiling.WHOLE_WORLD_TILE_ID + ) + + +def download_gadm_keys(path: pathlib.Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + utils.save_remote_url_to_local_path( + local_path=str(path), params={}, remote_url=GADM_KEY_URL + ) + + +def load_gadm(path: pathlib.Path) -> pandas.DataFrame: + """GADM's admin-1 units, minus the 26 of 3,691 rows that cannot identify one. + + Twenty-three are territories with no admin-1 subdivision -- Antarctica, Aruba, the Caspian Sea -- + and three name a real province but record GID_1 as the literal string "NA". Dropping both keeps + them out of the unmatched tally, where they would read as coverage we failed to get. + + Four GID_1s are each assigned to two provinces (GBR.1_1 to England *and* Wales), which is why + `match` requires a name to identify exactly one unit per side rather than taking the first. + """ + frame = pandas.read_csv(path).dropna(subset=["GID_1", "NAME_1"]) + frame["keys"] = frame["NAME_1"].map(get_keys) + return frame.rename(columns={"GID_0": "iso_3166", "GID_1": "gadm_id"}) + + +def load_world_bank(path: pathlib.Path) -> pandas.DataFrame: + """Provinces with their land area, which is what makes coverage meaningful. + + Counting units treats a 12 km2 island as equal to a province the size of France, and it misleads + in both directions: Brazil matches 87% of its units but 100% of its area, while Algeria matches + 96% of its units and only 74% of its area, missing two Saharan wilayas of 607,000 km2 between + them. Area is read here because the GeoPackage already has it, where a cropland weighting would + need a capture. + """ + geometries = geopandas.read_file(path).to_crs(EQUAL_AREA_CRS) + frame = pandas.DataFrame(geometries.drop(columns="geometry")) + frame["square_kilometres"] = ( + geometries.geometry.area / SQUARE_METRES_PER_SQUARE_KILOMETRE + ) + frame["keys"] = frame["NAM_1"].map(get_keys) + return frame.rename(columns={"ISO_A3": "iso_3166", "ADM1CD_c": "world_bank_id"}) + + +def match_orbae( + orbae_names: dict[str, set[str]], world_bank: pandas.DataFrame +) -> dict[str, str]: + """Orbae province name to World Bank admin id, per country. + + Exact after the same normalization GADM gets, and 502 of 651 provinces resolve that way. The 149 + that do not split into two kinds, and only one is fixable: + + Naming, worth overrides. Russia is the bulk at 68 -- "Adygea" against "Adygeya Rep.", "Altai Krai" + against "Altayskiy Kray" -- transliteration stems that no normalization reaches. Czechia, Croatia, + China, Thailand and Poland add a further 36 of the same kind. + + Structural, and no key map can express it. Cote d'Ivoire gives Orbae's 26 regions against the World + Bank's 14 districts, France gives Orbae's 13 post-2016 regions against the World Bank's 22 + pre-reform ones, and the Solomon Islands has no World Bank subdivision at all. These are + many-to-one or absent, the same shape as CAF's 17 prefectures against 7 regions. + """ + matched: dict[str, str] = {} + for iso_3166 in sorted(orbae_names): + provinces = world_bank[world_bank["iso_3166"] == iso_3166] + by_key: dict[str, set[str]] = {} + for world_bank_id, keys in zip( + provinces["world_bank_id"], provinces["keys"], strict=True + ): + for key in keys: + by_key.setdefault(key, set()).add(world_bank_id) + for name in sorted(orbae_names.get(iso_3166, set())): + candidates = { + world_bank_id + for key in get_keys(name=name) + for world_bank_id in by_key.get(key, set()) + } + if len(candidates) == 1: + matched[f"{iso_3166:s}:{name:s}"] = next(iter(candidates)) + return matched + + +def match( + gadm: pandas.DataFrame, overrides: dict[str, str], world_bank: pandas.DataFrame +) -> dict[str, str]: + """GADM id to World Bank id, wherever a name identifies exactly one province on each side. + + A match is a join on (country, normalized name), kept only where it is unambiguous in both + directions. Ambiguity is refused rather than guessed: two provinces normalizing alike would + otherwise file one's emissions under the other's name, permanently and silently. + + Overrides win, and any automatic match colliding with one is dropped, so a hand-made decision + cannot be half-applied. + """ + pairs = gadm.explode("keys")[["iso_3166", "gadm_id", "keys"]].merge( + world_bank.explode("keys")[["iso_3166", "world_bank_id", "keys"]], + on=["iso_3166", "keys"], + ) + unambiguous = pairs[ + pairs["gadm_id"].map(pairs.groupby("gadm_id")["world_bank_id"].nunique()).eq(1) + & pairs["world_bank_id"] + .map(pairs.groupby("world_bank_id")["gadm_id"].nunique()) + .eq(1) + ].drop_duplicates(subset="gadm_id") + overridden = set(overrides.values()) + return { + gadm_id: world_bank_id + for gadm_id, world_bank_id in zip( + unambiguous["gadm_id"], unambiguous["world_bank_id"], strict=True + ) + if gadm_id not in overrides and world_bank_id not in overridden + } | overrides + + +def get_coverage(claimed: set[str], world_bank: pandas.DataFrame) -> dict[str, float]: + """The share of each country's land area whose province the key map can name. + + This is what eligibility filter E3 reads. It is a proxy -- the real weight is cropland, + and Algeria's missing area is mostly desert -- but it is available without a capture, where + cropland is not, and it beats counting units as equal. + """ + world_bank = world_bank.assign(matched=world_bank["world_bank_id"].isin(claimed)) + by_country = world_bank.groupby("iso_3166") + return { + str(iso_3166): float( + group.loc[group["matched"], "square_kilometres"].sum() + / group["square_kilometres"].sum() + ) + for iso_3166, group in by_country + if group["square_kilometres"].sum() + } + + +def get_unmatched( + claimed: set[str], frame: pandas.DataFrame, id_column: str +) -> dict[str, list[str]]: + """What each side has left over, by country, so the gap is a number and not an impression.""" + leftover = frame[~frame[id_column].isin(claimed)] + return { + str(iso_3166): sorted(group[id_column]) + for iso_3166, group in leftover.groupby("iso_3166") + } + + +def read_overrides() -> dict[str, str]: + """Hand-made matches, each carrying the two names it reconciles. + + The names are in the file rather than only here so an override can be reviewed without opening + two other sources: "Kracheh / Kratie" is checkable at a glance, `KHM.11_1 -> KHM011` is not. + """ + if not OVERRIDES.exists(): + return {} + return { + gadm_id: entry["world_bank_id"] + for gadm_id, entry in json.loads(OVERRIDES.read_text()).items() + } + + +@dataclasses.dataclass(frozen=True) +class Check: + """One property the finished map must have, and what was found instead where it does not.""" + + name: str + passed: bool + detail: str + + +def iter_checks( + gadm: pandas.DataFrame, + matched: dict[str, str], + overrides: dict[str, str], + world_bank: pandas.DataFrame, +) -> collections.abc.Iterator[Check]: + """Verify the finished map rather than trusting the code that built it. + + These check the artifact that gets committed, not the functions that produced it, so a bad map + fails whichever route produced it -- including a hand-edited overrides file. A failure means the + map would file one jurisdiction's emissions under another's, silently and permanently. + """ + world_bank_ids = list(matched.values()) + duplicated = sorted( + { + world_bank_id + for world_bank_id in world_bank_ids + if world_bank_ids.count(world_bank_id) > 1 + } + ) + yield Check( + name="injective", + passed=not duplicated, + detail=( + f"{len(duplicated):d} World Bank id(s) claimed by more than one GADM unit: " + f"{', '.join(duplicated[:5])}" + if duplicated + else f"{len(world_bank_ids):d} World Bank ids, each claimed exactly once" + ), + ) + + unknown_gadm = sorted(set(matched) - set(gadm["gadm_id"])) + unknown_world_bank = sorted(set(world_bank_ids) - set(world_bank["world_bank_id"])) + yield Check( + name="ids-exist", + passed=not unknown_gadm and not unknown_world_bank, + detail=( + f"{len(unknown_gadm):d} GADM and {len(unknown_world_bank):d} World Bank id(s) in the " + f"map are absent from their source: " + f"{', '.join(unknown_gadm[:3] + unknown_world_bank[:3])}" + if unknown_gadm or unknown_world_bank + else "every id in the map exists in the source it came from" + ), + ) + + dropped = sorted( + gadm_id + for gadm_id, world_bank_id in overrides.items() + if matched.get(gadm_id) != world_bank_id + ) + yield Check( + name="overrides-applied", + passed=not dropped, + detail=( + f"{len(dropped):d} hand-written override(s) missing from the map or overwritten: " + f"{', '.join(dropped)}" + if dropped + else f"all {len(overrides):d} overrides present with their intended value" + ), + ) + + share = len(matched) / len(gadm) if len(gadm) else 0.0 + yield Check( + name="coverage-floor", + passed=share >= MINIMUM_MATCHED_SHARE, + detail=( + f"{share:.1%} of GADM units matched against a {MINIMUM_MATCHED_SHARE:.0%} floor" + + ( + "; a collapse this size usually means a source renamed a column" + if share < MINIMUM_MATCHED_SHARE + else "" + ) + ), + ) + + +def get_orbae_province_names() -> dict[str, set[str]]: + """Every province name Orbae publishes a factor for, by country. + + Read through `prepare` rather than reparsed here, so the names this matches are exactly the names + the runtime looks up. A second parser would drift. + """ + provincial = prepare.read_orbae() + provincial = provincial[provincial["admin_level"] == schema.PROVINCIAL] + names: dict[str, set[str]] = {} + for row in provincial.to_dict("records"): + names.setdefault(str(row["iso_3166"]), set()).add(str(row["jurisdiction_name"])) + return names + + +def iter_orbae_checks( + matched: dict[str, str], names: dict[str, set[str]], world_bank: pandas.DataFrame +) -> collections.abc.Iterator[Check]: + """The same properties the GADM map is held to, over the Orbae map.""" + claimed = list(matched.values()) + duplicated = sorted({one for one in claimed if claimed.count(one) > 1}) + yield Check( + name="orbae-injective", + passed=not duplicated, + detail=( + f"{len(duplicated):d} World Bank id(s) claimed by more than one Orbae province: " + f"{', '.join(duplicated[:5])}" + if duplicated + else f"{len(claimed):d} World Bank ids, each claimed exactly once" + ), + ) + unknown = sorted(set(claimed) - set(world_bank["world_bank_id"])) + yield Check( + name="orbae-ids-exist", + passed=not unknown, + detail=( + f"{len(unknown):d} id(s) absent from the World Bank frame: {', '.join(unknown[:3])}" + if unknown + else "every id in the map exists in the World Bank frame" + ), + ) + total = sum(map(len, names.values())) + share = len(matched) / total if total else 0.0 + yield Check( + name="orbae-coverage-floor", + passed=share >= MINIMUM_ORBAE_MATCHED_SHARE, + detail=( + f"{share:.1%} of {total:d} Orbae provinces matched against a " + f"{MINIMUM_ORBAE_MATCHED_SHARE:.0%} floor" + ), + ) + + +def write_orbae_map( + names: dict[str, set[str]], world_bank: pandas.DataFrame +) -> dict[str, str]: + """Materialize the Orbae map, recording coverage and every unmatched province. + + The unmatched list is in the output rather than only in a log: a third of it is structural and + will never close, so it is a standing fact about the anchor rather than a to-do. + """ + matched = match_orbae(orbae_names=names, world_bank=world_bank) + ORBAE_OUTPUT.write_text( + json.dumps( + { + "orbae_export": prepare.ORBAE_EXPORT.name, + "world_bank_source": { + "dataset": worldbank_jurisdictions.ADMIN_1_DATASET.product_name, + "version": worldbank_jurisdictions.ADMIN_1_DATASET.version, + }, + "matched": dict(sorted(matched.items())), + "coverage": { + iso_3166: round( + sum(1 for name in these if f"{iso_3166:s}:{name:s}" in matched) + / len(these), + 3, + ) + for iso_3166, these in sorted(names.items()) + }, + "unmatched": { + iso_3166: sorted( + name + for name in these + if f"{iso_3166:s}:{name:s}" not in matched + ) + for iso_3166, these in sorted(names.items()) + if any(f"{iso_3166:s}:{name:s}" not in matched for name in these) + }, + }, + indent=2, + ) + + "\n" + ) + logger.info(f"Wrote {ORBAE_OUTPUT}") + return matched + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cache", + type=pathlib.Path, + default=pull.CACHE, + help="where the downloads land; not committed, and reused unless --overwrite", + ) + parser.add_argument( + "--overwrite", action="store_true", help="re-download both inputs" + ) + args = parser.parse_args() + + gadm_path = args.cache / "key_gadm_adm1.csv" + world_bank_path = args.cache / "world_bank_admin_1.gpkg" + if args.overwrite or not gadm_path.exists(): + download_gadm_keys(path=gadm_path) + if args.overwrite or not world_bank_path.exists(): + download_world_bank(path=world_bank_path) + + overrides = read_overrides() + gadm = load_gadm(path=gadm_path) + world_bank = load_world_bank(path=world_bank_path) + matched = match(gadm=gadm, overrides=overrides, world_bank=world_bank) + unmatched_gadm = get_unmatched( + claimed=set(matched), frame=gadm, id_column="gadm_id" + ) + unmatched_world_bank = get_unmatched( + claimed=set(matched.values()), frame=world_bank, id_column="world_bank_id" + ) + + pull.DATA.mkdir(parents=True, exist_ok=True) + # No timestamp: the inputs' revisions are the identity, and a clock would make two branches + # disagree about an identical map. + OUTPUT.write_text( + json.dumps( + { + "gadm_source": {"url": GADM_KEY_URL, "revision": WRI_REVISION}, + "world_bank_source": { + "dataset": worldbank_jurisdictions.ADMIN_1_DATASET.product_name, + "version": worldbank_jurisdictions.ADMIN_1_DATASET.version, + }, + "matched": dict(sorted(matched.items())), + "unmatched_gadm": { + iso_3166: sorted(ids) + for iso_3166, ids in sorted(unmatched_gadm.items()) + }, + "unmatched_world_bank": dict(sorted(unmatched_world_bank.items())), + "area_coverage": { + iso_3166: round(share, 4) + for iso_3166, share in sorted( + get_coverage( + claimed=set(matched.values()), world_bank=world_bank + ).items() + ) + }, + }, + indent=2, + sort_keys=False, + ) + + "\n" + ) + logger.info(f"Wrote {OUTPUT}") + + total = len(gadm) + print( + f"\n{len(matched):d} of {total:d} GADM units matched ({len(matched) / total:.1%}); " + f"{sum(map(len, unmatched_gadm.values())):d} unmatched over " + f"{len(unmatched_gadm):d} countries" + ) + print(f" overrides applied: {len(overrides):d} -> {OVERRIDES}") + + checks = list( + iter_checks( + gadm=gadm, matched=matched, overrides=overrides, world_bank=world_bank + ) + ) + # Orbae is optional: its export is a supplied file, and the GADM map must not depend on it. + if prepare.ORBAE_EXPORT.exists(): + names = get_orbae_province_names() + orbae_matched = write_orbae_map(names=names, world_bank=world_bank) + checks += list( + iter_orbae_checks(matched=orbae_matched, names=names, world_bank=world_bank) + ) + print( + f" orbae: {len(orbae_matched):d} of {sum(map(len, names.values())):d} provinces " + f"matched -> {ORBAE_OUTPUT}" + ) + else: + logger.warning(f"{prepare.ORBAE_EXPORT} absent; skipping the Orbae map") + print() + for check in checks: + print( + f" {'ok ' if check.passed else 'FAIL'} {check.name:20s} {check.detail:s}" + ) + failed = [check.name for check in checks if not check.passed] + if failed: + # The maps are still on disk, deliberately: a failed check is easier to diagnose against the + # written file than against a run that refused to produce one. + print( + f"\n{len(failed):d} check(s) failed: {', '.join(failed)}. " + f"{OUTPUT} is not safe to commit." + ) + return 1 + else: + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build-tiled-countries.py b/tools/build-tiled-countries.py new file mode 100644 index 0000000..26f668e --- /dev/null +++ b/tools/build-tiled-countries.py @@ -0,0 +1,233 @@ +"""Build the committed set of countries the pipeline can produce a factor for -- E1's second half. + +Eligibility cuts some thirty thousand possible (country, crop) pairs to a few hundred, and all but +one of its filters read anchors `sources.lock.json` already pins by sha256. `prepare.get_eligible` +derives those on every report run, in about a tenth of a second, rather than any of them being +committed a second time. + +This is the exception. Deciding which countries intersect at least one of the 280 ten-degree GFW +tiles needs a spatial join against a 93 MiB GeoPackage and geopandas, neither of which belongs on the +reporting path. So the join happens here and its result is committed -- a few kilobytes, and it moves +only when the tile set or the World Bank layer does, not when the WRI pin does. + +The committed set is that join **minus** `worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S`, and the +subtraction is the part worth reading the source for: the join alone answers a question about +geometry, where E1 needs one about production. The constant is imported rather than restated so that +this set and `get_all_iso_3166s` -- what `--backfill` runs -- cannot drift apart; see it for why the +difference is a list rather than a predicate. + +**The output is the artifact, and a broken one exits nonzero.** A bad committed set is not a finding, +it is a defect: a spatial join that silently returned a handful of countries would fail every pair at +E1, which the reporting path would report as a mis-specified filter rather than as a bad input. + +To see the shortlist this makes possible, and choose from it, use the report rather than this tool: + + uv run --with geopandas python tools/build-tiled-countries.py + uv run python -m validation --section eligible --show 40 +""" + +import argparse +import collections.abc +import dataclasses +import json +import logging +import pathlib + +import geopandas + +from jdluc import tiling +from jdluc.datasets import worldbank_jurisdictions +from validation import prepare, pull, targets + +logger = logging.getLogger(__name__) + +# Below this the join did not merely lose a country, it broke. 220 of the layer's 244 countries +# survive the join and the carve-out below, so the headroom is wide and only a structural failure +# trips it. +MINIMUM_TILED_SHARE = 0.75 + + +@dataclasses.dataclass(frozen=True) +class Check: + """One property the finished set must have, and what was found instead where it does not.""" + + name: str + passed: bool + detail: str + + +def get_tiled_iso_3166s(path_to_geopackage: pathlib.Path) -> tuple[set[str], int]: + """Countries intersecting at least one ten-degree GFW tile, and how many the layer holds. + + The GFW tile set covers 280 ten-degree cells rather than all of them, so this filter genuinely + bites: a country entirely outside it has no tree-cover-loss data and cannot be computed. Read + from the GeoPackage `build-national-mappings` downloads, so this needs no ingest -- run that + tool first on a cold cache, or this one fails on a missing file. + """ + world_bank = geopandas.read_file(path_to_geopackage) + tiles = geopandas.GeoDataFrame( + geometry=[ + tiling.get_box_for_tile_id(tile_id=tile_id) + for tile_id in sorted(tiling.GLOBAL_FOREST_WATCH_TILE_IDS) + ], + crs=world_bank.crs, + ) + joined = geopandas.sjoin(world_bank, tiles, how="inner", predicate="intersects") + return set(joined["ISO_A3"].dropna()), int(world_bank["ISO_A3"].dropna().nunique()) + + +def iter_checks( + countries: int, joined: set[str], tiled: set[str] +) -> collections.abc.Iterator[Check]: + """Verify the finished set rather than trusting the join that produced it. + + These read the artifact that gets committed, so a bad set fails whichever route produced it -- + including a hand-edited file. `joined` is the raw spatial join and `tiled` is what survives the + carve-out, because two of the checks below can only tell a real change from a stale list by + comparing them. + """ + share = len(tiled) / countries if countries else 0.0 + yield Check( + name="coverage-floor", + passed=share >= MINIMUM_TILED_SHARE, + detail=( + f"{len(tiled):d} of {countries:d} countries ({share:.1%}) intersect a tile, against a " + f"{MINIMUM_TILED_SHARE:.0%} floor" + + ( + "; a collapse this size usually means the layer renamed a column or lost its CRS" + if share < MINIMUM_TILED_SHARE + else "" + ) + ), + ) + + malformed = sorted( + iso_3166 + for iso_3166 in tiled + if len(iso_3166) != 3 or not iso_3166.isupper() or not iso_3166.isalpha() + ) + yield Check( + name="well-formed-codes", + passed=not malformed, + detail=( + f"{len(malformed):d} entry(ies) are not ISO 3166 alpha-3: {', '.join(malformed[:6])}" + if malformed + else f"all {len(tiled):d} entries are ISO 3166 alpha-3" + ), + ) + + # A carved-out country the join no longer returns means the list is claiming credit for a + # removal the geometry already made -- harmless today, and a lie the next time someone reads it + # to find out what the carve-out costs. + stale = sorted(worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S - joined) + yield Check( + name="carve-out-current", + passed=not stale, + detail=( + f"{len(stale):d} carved-out country(ies) no longer intersect a tile, so " + f"UNPRODUCTIVE_ISO_3166S is stale: {', '.join(stale)}" + if stale + else ( + f"all {len(worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S):d} carved-out " + f"countries still intersect a " + f"tile, so each is removed by the carve-out rather than by the join" + ) + ), + ) + + # A chosen target that fails E1 cannot be computed at all, so the set silently dropping one + # would empty that pair from the report rather than raising anywhere. + missing = sorted( + { + target.iso_3166 + for target in targets.iter_targets() + if target.iso_3166 not in tiled + } + ) + yield Check( + name="targets-tiled", + passed=not missing, + detail=( + f"{len(missing):d} country(ies) named by {targets.TARGETS.name:s} intersect no tile: " + f"{', '.join(missing)}" + if missing + else f"every country in {targets.TARGETS.name:s} intersects a tile" + ), + ) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--geopackage", + type=pathlib.Path, + default=pull.CACHE / "world_bank_admin_1.gpkg", + help="the World Bank admin-1 GeoPackage build-national-mappings downloads", + ) + args = parser.parse_args() + assert args.geopackage.exists(), ( + f"{args.geopackage} is absent; run `uv run --with geopandas python " + "tools/build-national-mappings.py` first, which is what downloads it" + ) + + joined, countries = get_tiled_iso_3166s(path_to_geopackage=args.geopackage) + tiled = joined - worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S + pull.DATA.mkdir(parents=True, exist_ok=True) + # No timestamp: the inputs' versions are the identity, and a clock would make two branches + # disagree about an identical set. + prepare.TILED_ISO_3166S.write_text( + json.dumps( + { + "note": [ + "DERIVED -- do not hand-edit. Regenerate with", + "`uv run --with geopandas python tools/build-tiled-countries.py`.", + "E1's second half: the countries intersecting at least one GFW tile,", + "less `carved_out_iso_3166s` -- countries that do intersect a tile but", + "carry no crop production to attribute anything to. See", + "worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S for why that is a list.", + "Committed because deriving it needs a spatial join against a 93 MiB", + "GeoPackage, where the rest of eligibility is pandas over anchors the lock", + "already pins -- so the rest is derived on demand rather than committed twice.", + ], + "world_bank_source": { + "dataset": worldbank_jurisdictions.ADMIN_1_DATASET.product_name, + "version": worldbank_jurisdictions.ADMIN_1_DATASET.version, + }, + "tile_ids": len(tiling.GLOBAL_FOREST_WATCH_TILE_IDS), + "carved_out_iso_3166s": sorted( + worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S + ), + "tiled_iso_3166s": sorted(tiled), + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + print( + f"\n {len(tiled):d} tiled country(ies) " + f"({len(joined):d} joined, less " + f"{len(worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S):d} carved out) -> " + f"{prepare.TILED_ISO_3166S}\n" + ) + + checks = tuple(iter_checks(countries=countries, joined=joined, tiled=tiled)) + for check in checks: + print( + f" {'ok ' if check.passed else 'FAIL'} {check.name:20s} {check.detail:s}" + ) + failed = [check.name for check in checks if not check.passed] + if failed: + # The set is still on disk, deliberately: a failed check is easier to diagnose against the + # written file than against a run that refused to produce one. + print( + f"\n{len(failed):d} check(s) failed: {', '.join(failed)}. " + f"{prepare.TILED_ISO_3166S} is not safe to commit." + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/export-graphs.py b/tools/export-graphs.py index c3b7b0a..0e2114a 100644 --- a/tools/export-graphs.py +++ b/tools/export-graphs.py @@ -86,15 +86,54 @@ def csv(self) -> str: attribute.Methodology.JURISDICTIONAL_DIRECT: {"USA": ("MAIZE", "SOYBEAN", "WHEAT")}, attribute.Methodology.STATISTICAL: { "BRA": ("MAIZE", "SOYBEAN", "OILPALM"), - "BOL": ("SOYBEAN",), + "BOL": ("SOYBEAN", "PLANTAIN", "BANANA"), "ARG": ("SOYBEAN",), "CAN": ("SOYBEAN",), "IDN": ("OILPALM",), "PNG": ("OILPALM",), + "GHA": ("RICE", "WHEAT"), + "VNM": ("RICE", "SORGHUM"), + "MYS": ("RICE", "SORGHUM", "OILPALM"), + "CUB": ("SUGARCANE", "RICE", "BANANA"), + "ZAF": ("RICE", "WHEAT", "SORGHUM"), }, } +def subset_to_crop_names( + df: pandas.DataFrame, crop_names: tuple[str, ...] +) -> pandas.DataFrame: + ret = df[df.index.get_level_values("crop_name").isin(crop_names)] + missing = sorted( + set(crop_names) - set(map(str, ret.index.get_level_values("crop_name"))) + ) + if missing: + logger.warning(f"Workflow returned no rows for {missing=}") + return ret + + +def iter_dfs( + iso_3166_to_crop_names: dict[str, tuple[str, ...]], + methodology: attribute.Methodology, +) -> collections.abc.Iterator[pandas.DataFrame]: + # The full crop list, so this reuses the caches an ordinary trace.py run populates + workflow_crop_names = attribute.get_crop_names(methodology=methodology) + for iso_3166, crop_names in iso_3166_to_crop_names.items(): + logger.info( + f"Tracing {methodology.name:s}/{iso_3166:s} with " + f"{len(workflow_crop_names):d} crops, tabulating {len(crop_names):d}" + ) + yield subset_to_crop_names( + df=trace.workflow( + crop_names=workflow_crop_names, + iso_3166s=(iso_3166,), + methodology=methodology, + skip_glad_crop_filter=False, + ), + crop_names=crop_names, + ) + + def iter_path_record( df: pandas.DataFrame, ) -> collections.abc.Generator[tuple[tuple[str, ...], Record]]: @@ -153,13 +192,10 @@ def main() -> int: write_data_to_graph( dfs=[ pandas.concat( - trace.workflow( - crop_names=crop_names, - iso_3166s=(iso_3166,), + iter_dfs( + iso_3166_to_crop_names=iso_3166_to_crop_names, methodology=methodology, - skip_glad_crop_filter=False, ) - for iso_3166, crop_names in iso_3166_to_crop_names.items() ) for methodology, iso_3166_to_crop_names in METHODOLOGY_TO_ISO_3166_TO_CROP_NAMES.items() ] diff --git a/tools/measure-drift.py b/tools/measure-drift.py new file mode 100644 index 0000000..0ba5504 --- /dev/null +++ b/tools/measure-drift.py @@ -0,0 +1,1234 @@ +"""Measure the change in emissions-factor results caused by a code change. + +Capture the emissions-factor table at a baseline commit and again at the working tree, diff the two, +summarize the diff. Each side writes one parquet per (methodology, country) plus a manifest, so +`--compare-only` re-runs every check on artifacts already on disk, and so any question this tool +does not answer can be asked of those parquets directly. + +**The cache is content-blind, so a clean result can be a lie.** `storage.get_cache_decorator` keys +on module path, qualname, a hand-written `version=` int and the call arguments, never on the code +itself, so two commits can resolve to the *same* parquet URI -- and a head capture will then read +the baseline's answer back and report that nothing moved. Diagnosing that is yours: the capture +summary flags byte-identical parquets, and jdluc.storage logs the URI each side loaded, so both +cache keys are in the run log. `--isolated` sidesteps it with a private SCRATCH_ROOT per side, at +the price of the full pipeline rather than a cache read. + +Every crop in the capture gets checked -- Target.crop_names is the same +attribute.get_crop_names call that drives the capture, so the two cannot diverge. The cache key +includes crop_names, so asking for a subset would also cold-miss every layer an ordinary +`uv run python jdluc/trace.py USA` populated; captures are one country at a time over the full +per-methodology list. Scoring the same list means each sum drift is a true all-crop total, and +production_kg must hold to float32 noise under any within-group redistribution. + +That full list carries its own controls. The 13 crops in ifpri_mapspam.SHARED_CROP_NAMES -- MAIZE, +SOYBEAN, WHEAT, RICE and the rest -- take the simple-lookup branch of get_canonical_quantity, and +get_crop_to_share builds its denominator from raw per-year snapshots rather than decomposed ones, +so nothing about the 2000 decomposition can reach them. Any change confined to that decomposition +must leave all 13 at exactly zero on every column; movement there is a wiring break, not drift. + +Whether the drift it reports is welcome is a judgement for the reader, so there is no verdict and +no exit code to read: it prints and exits 0, or fails loudly on an assertion when it cannot compare +at all. stdout is deterministic and free of log noise, so redirecting two runs and diffing them +compares one branch against another. + +Must run from the repo root -- the cache resolves module paths through git, and Config finds .env +by walking up from the working directory. + + uv run python tools/measure-drift.py --baseline 121d40c + uv run python tools/measure-drift.py --baseline 121d40c --iso USA --isolated + uv run python tools/measure-drift.py --baseline 121d40c --compare-only 2>/dev/null > now.txt +""" + +import argparse +import collections.abc +import dataclasses +import datetime +import enum +import hashlib +import itertools +import json +import logging +import math +import os +import pathlib +import shutil +import subprocess +import textwrap +import typing + +import pandas + +from jdluc import attribute, config, storage, trace + +logger = logging.getLogger(__name__) + + +TOOL_NAME = pathlib.Path(__file__).name +# Countries chosen for distinct emissions regimes rather than coverage: temperate row crops, +# a tropical forest frontier, and tropical peat. Every added country multiplies capture cost, so +# widen this deliberately -- and use --iso to work on a subset meanwhile. HND and NIC are the +# cheap exception: both fall entirely inside the one ten-degree tile 20N_090W, and the expensive +# layer -- statistical.get_downscaled_luc_emissions -- is keyed on (skip_glad_crop_filter, +# tile_id) alone, so whichever of the two runs second pays for little beyond its province clips. +# +# Central America earns five slots because it carries every 2000 crop group ifpri_mapspam still +# decomposes, and the weight sits in different groups either side of a border: 44% of Honduran +# OOIL production takes the fallback path against 17% of Nicaraguan, and 77% of Nicaraguan BANP +# against 1% of Honduran. All five also run forest conversion, peatland conversion and peatland +# occupation together over the same hectares. +METHODOLOGY_TO_ISO_3166S: dict[attribute.Methodology, tuple[str, ...]] = { + attribute.Methodology.JURISDICTIONAL_DIRECT: ("USA",), + attribute.Methodology.STATISTICAL: ( + "BLZ", + "BRA", + "GTM", + "HND", + "IDN", + "MEX", + "NIC", + "SLV", + "USA", + ), +} +# Uncommitted changes anywhere else cannot move the numbers, so they do not belong in the +# manifest's dirtiness record. +RESULT_BEARING_PATHS = ("jdluc", "pyproject.toml", "uv.lock") +NATIONAL = "NATIONAL" +PROVINCIAL = "PROVINCIAL" +# The emissions columns are megatonnes while an emissions factor is kg CO2e per kg +KG_PER_TONNE = 1000 +# Two sides can be written with different index levels, so both are re-keyed onto this. admin_id +# rather than jurisdiction_name: a machine identifier, where a display string could be renamed. +CANONICAL_KEY = ("admin_level", "admin_id", "crop_name", "methodology") +# float32 eps is 1.19e-07, so movement at that scale is summation order rather than a real change. +DEFAULT_RTOL = 1.19e-07 +# Reported as counts, because one max|rel| cannot distinguish 3 rows moving 50% from 128 of them. +DRIFT_THRESHOLDS = (1e-07, 1e-03, 1e-01) +# Exactly the columns iter_national_from_provincials sums; everything else is a ratio derived from +# them, and summing a ratio is meaningless. +ADDITIVE_COLUMNS = ( + "crop_hectares", + "forest_emissions_mt", + "peatland_crop_hectares", + "peatland_conversion_emissions_mt", + "peatland_occupation_emissions_mt", + "emissions_mt", + "production_kg", +) +# Invariants are re-derivations within one side, so they hold to float64 epsilon. Anything above +# this is a wiring break rather than arithmetic noise. +INVARIANT_RTOL = 1e-12 +# Summation-order noise is symmetric, so a lopsided up/down split is a systematic shift even where +# every individual magnitude is negligible. +SIGN_ALPHA = 0.05 + + +class Side(enum.StrEnum): + BASELINE = enum.auto() + HEAD = enum.auto() + + +class Severity(enum.StrEnum): + BLOCKING = enum.auto() + DEFECT = enum.auto() + ADVISORY = enum.auto() + + @property + def marker(self) -> str: + match self: + case Severity.BLOCKING: + return "!!" + case Severity.DEFECT: + return "XX" + case _: + return "--" + + +class Invariant(enum.Enum): + """Relationships that must hold inside a single side, whatever the other side says.""" + + ROLLUP = "national totals equal the sum of their provincials" + EMISSIONS_FACTOR = "emissions factor equals emissions x 1000 / production" + YIELD = "yield equals production / hectares" + PEAT_FRACTION = "peatland occupation fraction lies in [0, 1]" + NON_NEGATIVE = "additive columns are non-negative" + + +@dataclasses.dataclass(frozen=True) +class Finding: + severity: Severity + message: str + + +@dataclasses.dataclass(frozen=True) +class Drift: + """How far one column moved between the two sides, over shared rows. + + Not a Finding: drift is the measurement, a Finding means it is not to be trusted. + """ + + slug: str + column: str + rows: int + max_absolute: float + max_relative: float + # Row counts over each of DRIFT_THRESHOLDS, in order + counts_over: tuple[int, ...] + # A value going NaN <-> finite is a categorically different event from one that moved + nan_flips: int + # Direction, which magnitude cannot show: noise splits evenly, a systematic shift does not + moved_down: int + moved_up: int + sign_p_value: float + + def exceeds(self, rtol: float) -> bool: + return ( + self.max_relative > rtol + or bool(self.nan_flips) + or self.sign_p_value < SIGN_ALPHA + ) + + +@dataclasses.dataclass(frozen=True) +class SumDrift: + """A whole-target total, where per-row movement can cancel out or compound.""" + + slug: str + column: str + before: float + after: float + + @property + def relative(self) -> float: + return ( + (self.after - self.before) / abs(self.before) + if self.before + else float("nan") + ) + + def exceeds(self, rtol: float) -> bool: + return abs(self.relative) > rtol if self.before else self.after != 0 + + +@dataclasses.dataclass(frozen=True) +class Target: + methodology: attribute.Methodology + iso_3166: str + + @property + def crop_names(self) -> tuple[str, ...]: + # The same list the capture hands the pipeline, so what gets checked can never be less + # than what ran. A curated subset scores a partial sum, which moves whenever a crop + # trades with a sibling outside it -- indistinguishable from mass being created. + return attribute.get_crop_names(methodology=self.methodology) + + @property + def slug(self) -> str: + return f"{self.methodology.name:s}-{self.iso_3166:s}" + + +def iter_targets( + iso_3166s: tuple[str, ...] | None, +) -> collections.abc.Iterator[Target]: + for methodology, methodology_iso_3166s in sorted(METHODOLOGY_TO_ISO_3166S.items()): + for iso_3166 in sorted(methodology_iso_3166s): + if iso_3166s is None or iso_3166 in iso_3166s: + yield Target(methodology=methodology, iso_3166=iso_3166) + + +@dataclasses.dataclass +class Manifest: + """How one side's parquets were produced, in enough detail to distrust them later.""" + + sha: str + # `git status --porcelain` lines, so "?? path" rather than "path" + dirty_paths: list[str] + captured_at: str + scratch_root: str + row_counts: dict[str, int] + # Differing digests prove little: pyarrow embeds library metadata that moves with the + # environment. Only identical ones are informative. + parquet_sha256: dict[str, str] + + FILENAME: typing.ClassVar[str] = "manifest.json" + + def write(self, directory: pathlib.Path) -> None: + path = directory / self.FILENAME + path.write_text( + json.dumps(dataclasses.asdict(self), indent=2, sort_keys=True) + "\n" + ) + logger.info(f"Wrote {path}") + + @classmethod + def read(cls, directory: pathlib.Path) -> typing.Self: + """Ignores keys it does not recognize, so --compare-only still reads a capture written by + an older version of this tool. A *missing* field still raises, loudly.""" + payload = json.loads((directory / cls.FILENAME).read_text()) + names = {field.name for field in dataclasses.fields(cls)} + return cls(**{key: value for key, value in payload.items() if key in names}) + + +def git(*arguments: str, cwd: pathlib.Path) -> str: + result = subprocess.run( + ["git", *arguments], + check=True, + cwd=cwd, + stdout=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() + + +def get_repo_root() -> pathlib.Path: + cwd = pathlib.Path.cwd() + root = pathlib.Path(git("rev-parse", "--show-toplevel", cwd=cwd)) + assert root == cwd, ( + f"Run from the repo root ({root}), not {cwd}: the cache keys module paths relative to " + "the repo root and Config finds .env by walking up from the working directory" + ) + return root + + +def resolve_commit(sha: str, cwd: pathlib.Path) -> str: + return git("rev-parse", f"{sha:s}^{{commit}}", cwd=cwd) + + +def get_dirty_paths(cwd: pathlib.Path) -> list[str]: + porcelain = git("status", "--porcelain", "--", *RESULT_BEARING_PATHS, cwd=cwd) + return porcelain.splitlines() if porcelain else [] + + +def load_capture( + directory: pathlib.Path, slug: str +) -> tuple[pandas.DataFrame, tuple[str, ...]]: + """Read one side's parquet onto CANONICAL_KEY, whatever index it was written with. + + Whichever of admin_id and jurisdiction_name is not in the index is a column, so resetting and + re-keying realigns two sides written differently. Also returns the index each was *written* + with, since a schema difference and a value disagreement look identical once rows fail to line + up. + """ + raw = pandas.read_parquet(directory / f"{slug:s}.parquet") + written = tuple(str(name) for name in raw.index.names if name is not None) + df = raw.reset_index() + missing = [name for name in CANONICAL_KEY if name not in df.columns] + assert not missing, ( + f"{slug:s} in {directory} lacks {missing}; is it the schema you think it is?" + ) + return df.set_index(list(CANONICAL_KEY)).sort_index(), written + + +def check_alignment( + baseline: pandas.DataFrame, + crop_names: tuple[str, ...], + head: pandas.DataFrame, + slug: str, + written: tuple[tuple[str, ...], tuple[str, ...]], +) -> list[Finding]: + """Do the two sides describe the same rows? Everything downstream assumes they do.""" + findings: list[Finding] = [] + shared = baseline.index.intersection(head.index) + print( + f"\n {slug:s}: {len(baseline):d} baseline row(s), {len(head):d} head row(s), " + f"{len(shared):d} shared" + ) + if not len(shared): + # A column table of zeros would read as agreement when not one value was compared. The + # written index levels are the diagnostic here and nowhere else: a mismatch confined to one + # level is a schema difference that load_capture failed to reconcile, not a numeric one. + findings.append( + Finding( + severity=Severity.BLOCKING, + message=( + f"{slug:s} has no shared rows, so NOTHING was compared -- the keys do not " + f"line up. Written as {' + '.join(written[0])} against " + f"{' + '.join(written[1])}, re-keyed onto {' + '.join(CANONICAL_KEY)}" + ), + ) + ) + for label, df in (("baseline", baseline), ("head", head)): + # Duplicates would make .loc[shared] return more rows than shared, misaligning everything + # downstream rather than failing + if duplicated := int(df.index.duplicated().sum()): + findings.append( + Finding( + severity=Severity.BLOCKING, + message=( + f"{slug:s} has {duplicated:d} duplicate key(s) on the {label:s} side, so " + "no row-to-row comparison is well defined" + ), + ) + ) + # A target that silently stopped existing would otherwise be reported as nothing at all + if absent := sorted( + set(crop_names) - set(df.index.get_level_values("crop_name")) + ): + findings.append( + Finding( + severity=Severity.BLOCKING, + message=( + f"{slug:s} is missing {absent} from the {label:s} capture, so a " + "hardcoded target is going unmeasured" + ), + ) + ) + if len(only := df.index.difference(shared)): + findings.append( + Finding( + severity=Severity.ADVISORY, + message=( + f"{slug:s} has {len(only):d} row(s) only on the {label:s} side, " + f"e.g. {list(only)[:3]}" + ), + ) + ) + return findings + + +def print_worst_rows( + baseline: pandas.DataFrame, columns: list[str], head: pandas.DataFrame, limit: int +) -> None: + """Name the rows that moved most, so a large max|rel| points somewhere specific.""" + records: list[tuple[float, str, tuple[str, ...], float, float]] = [] + for column in columns: + before, after = baseline[column], head[column] + relative = (after - before).abs().div(before.abs().where(before.abs() > 0)) + for key, value in relative.nlargest(limit).items(): + row = typing.cast(tuple[str, ...], key) + records.append( + ( + float(value), + column, + row, + float(before.loc[row]), + float(after.loc[row]), + ) + ) + if not records: + return + print(f"\n worst {limit:d} row(s) by relative movement") + print( + f" {'crop':<16}{'admin':<9}{'column':<34}{'before':>13}{'after':>13}{'rel':>11}" + ) + for value, column, row, before_value, after_value in sorted(records, reverse=True)[ + :limit + ]: + admin_id, crop_name = row[1], row[2] + print( + f" {crop_name!s:<16}{admin_id!s:<9}{column:<34}" + f"{before_value:>13.5g}{after_value:>13.5g}{value:>11.3e}" + ) + + +def check_excluded_crops( + baseline: pandas.DataFrame, crops: list[str], head: pandas.DataFrame, rtol: float +) -> list[Finding]: + """Say what the hardcoded crop filter left unchecked, and whether it moved. + + The filter is deliberate, but a tool that checks a tenth of the rows reads as though it checked + the artifact. Across every numeric column rather than the emissions factor alone, because a + factor is a ratio: it sits still whenever its numerator and denominator move together, so a + column that has not moved says nothing about the emissions and hectares underneath it. + """ + outside = ~baseline.index.get_level_values("crop_name").isin(crops) + left, right = baseline[outside], head[outside] + shared = left.index.intersection(right.index) + excluded = sorted(set(left.index.get_level_values("crop_name"))) + if not len(shared) or not excluded: + return [] + worst, where, flips, blind = 0.0, "", 0, 0 + for column in sorted( + set(left.select_dtypes("number").columns) + & set(right.select_dtypes("number").columns) + ): + before, after = left.loc[shared, column], right.loc[shared, column] + relative = (after - before).abs().div(before.abs().where(before.abs() > 0)) + flips += int((before.isna() != after.isna()).sum()) + # NaN on both sides everywhere: unmeasurable, which is not the same as unmoved + if not relative.notna().any(): + blind += 1 + elif float(relative.max()) > worst: + row = typing.cast(tuple[str, ...], relative.idxmax()) + worst, where = float(relative.max()), f"{column:s} {row[2]:s}/{row[1]:s}" + # Three distinct outcomes, and the first two used to print alike: something moved, nothing was + # comparable, and everything was comparable and sat exactly still + measured = ( + f"worst {worst:.3e} ({where:s})" + if where + else "nothing measurable" + if blind + else "no movement at all" + ) + print( + f" not checked: {len(shared):d} row(s) over {len(excluded):d} crop(s) outside the " + f"hardcoded subset; {measured:s}, {flips:d} NaN flip(s), " + f"{blind:d} column(s) NaN throughout" + ) + if worst <= rtol and not flips: + return [] + return [ + Finding( + severity=Severity.ADVISORY, + message=( + f"{len(excluded):d} crop(s) outside the checked set moved where nothing " + f"checked them: {measured:s}, plus {flips:d} NaN flip(s). Target.crop_names " + "should cover every crop the capture holds, so this means they diverged" + ), + ) + ] + + +def check_value_drift( + baseline: pandas.DataFrame, head: pandas.DataFrame, rtol: float, target: Target +) -> tuple[list[Drift], list[Finding]]: + """Per column, how far did the values move on rows both sides have? + + Filtered to the hardcoded crops, which is what that subset is for; alignment ran over the + whole table. + """ + crops = list(target.crop_names) + left_all = baseline[baseline.index.get_level_values("crop_name").isin(crops)] + right_all = head[head.index.get_level_values("crop_name").isin(crops)] + shared = left_all.index.intersection(right_all.index) + columns = sorted( + set(left_all.select_dtypes("number").columns) + & set(right_all.select_dtypes("number").columns) + ) + print( + f"\n {target.slug:s}: {len(shared):d} shared row(s) over " + f"{len(crops):d} hardcoded crop(s)" + ) + if not len(shared) or not columns: + return [], [] + + left, right = left_all.loc[shared], right_all.loc[shared] + thresholds = "".join(f"{f'>{threshold:g}':>9s}" for threshold in DRIFT_THRESHOLDS) + print( + f" {'column':<34}{'max|abs|':>13}{'max|rel|':>12}{thresholds:s}{'nan':>5}" + f"{'up/dn':>10}{'sign p':>8}" + ) + drifts: list[Drift] = [] + findings: list[Finding] = [] + for column in columns: + before, after = left[column], right[column] + # NaN in the same place on both sides is agreement, not a difference + both_nan = before.isna() & after.isna() + absolute = (after - before).abs().where(~both_nan) + relative = absolute.div(before.abs().where(before.abs() > 0)) + down, up = int((after < before).sum()), int((after > before).sum()) + drift = Drift( + slug=target.slug, + column=column, + rows=len(shared), + max_absolute=float(absolute.max()) if absolute.notna().any() else 0.0, + max_relative=float(relative.max()) if relative.notna().any() else 0.0, + counts_over=tuple( + int((relative > threshold).sum()) for threshold in DRIFT_THRESHOLDS + ), + nan_flips=int((before.isna() != after.isna()).sum()), + moved_down=down, + moved_up=up, + sign_p_value=two_sided_sign_p_value(down=down, up=up), + ) + drifts.append(drift) + counts = "".join(f"{count:>9d}" for count in drift.counts_over) + print( + f" {column:<34}{drift.max_absolute:>13.5g}{drift.max_relative:>12.3e}" + f"{counts:s}{drift.nan_flips:>5d}{f'{up:d}/{down:d}':>10}" + f"{drift.sign_p_value:>8.3f}" + ) + if drift.nan_flips: + findings.append( + Finding( + severity=Severity.ADVISORY, + message=( + f"{target.slug:s} {column:s} changed NaN-ness on {drift.nan_flips:d} " + "row(s) -- a value appearing or disappearing, not merely moving; usually " + "rows gaining or losing production entirely" + ), + ) + ) + if drift.sign_p_value < SIGN_ALPHA: + findings.append( + Finding( + severity=Severity.ADVISORY, + message=( + f"{target.slug:s} {column:s} moved in one direction more than chance " + f"allows: {up:d} up against {down:d} down (p={drift.sign_p_value:.3f}). " + "Summation noise is symmetric, so this is a systematic shift even where " + "the magnitudes are small" + ), + ) + ) + print_worst_rows(baseline=left, columns=columns, head=right, limit=5) + findings += check_excluded_crops( + baseline=baseline, crops=crops, head=head, rtol=rtol + ) + return drifts, findings + + +def check_sum_drift( + baseline: pandas.DataFrame, head: pandas.DataFrame, target: Target +) -> list[SumDrift]: + """Totals for the whole target, which answer a different question from per-row drift. + + Signed, so opposing movements show as cancellation rather than as calm. Summed over NATIONAL + rows only: each national row is already the sum of its provincials, so both levels would double + count. + """ + + def nationals(df: pandas.DataFrame) -> pandas.DataFrame: + return df[ + (df.index.get_level_values("admin_level") == NATIONAL) + & df.index.get_level_values("crop_name").isin(list(target.crop_names)) + ] + + left, right = nationals(baseline), nationals(head) + print( + f"\n {target.slug:s}: {len(left):d} vs {len(right):d} national row(s) over " + f"{len(target.crop_names):d} hardcoded crop(s)" + ) + print(f" {'column':<34}{'baseline':>15}{'head':>15}{'delta':>14}{'rel':>11}") + totals = [ + SumDrift( + slug=target.slug, + column=column, + before=float(left[column].sum()), + after=float(right[column].sum()), + ) + for column in ADDITIVE_COLUMNS + if column in left.columns and column in right.columns + ] + # The aggregate factor these crops carry, which no single row's factor gives + by_column = {total.column: total for total in totals} + if {"emissions_mt", "production_kg"} <= set(by_column): + emissions, production = by_column["emissions_mt"], by_column["production_kg"] + totals.append( + SumDrift( + slug=target.slug, + column="implied_emissions_factor", + before=( + emissions.before * KG_PER_TONNE / production.before + if production.before + else float("nan") + ), + after=( + emissions.after * KG_PER_TONNE / production.after + if production.after + else float("nan") + ), + ) + ) + for total in totals: + # Scientific, not percent: a percentage rounded for readability prints +0.00% for a 1e-06 + # move, which reads as agreement while the verdict counts it as drift + print( + f" {total.column:<34}{total.before:>15.6g}{total.after:>15.6g}" + f"{total.after - total.before:>+14.4g}{total.relative:>+11.2e}" + ) + return totals + + +def two_sided_sign_p_value(down: int, up: int) -> float: + """Exact two-sided binomial p-value for an up/down split, no scipy and no approximation. + + n is at most the provinces in a country, so the exact sum is cheap. Ties are excluded rather + than split. + """ + total = up + down + if not total: + return 1.0 + tail = sum(math.comb(total, k) for k in range(max(up, down), total + 1)) + return min(1.0, 2 * tail / 2**total) + + +def check_invariants(df: pandas.DataFrame) -> dict[Invariant, str]: + """Relationships that must hold within one side, whatever the other side says. + + Returns what broke rather than findings, because severity depends on the other side: see + compare_invariants. + """ + + def worst_relative(left: pandas.Series, right: pandas.Series) -> float: + relative = (left - right).abs().div(left.abs().where(left.abs() > 0)).dropna() + return float(relative.max()) if len(relative) else 0.0 + + broken: dict[Invariant, str] = {} + levels = df.index.get_level_values("admin_level") + national = df[levels == NATIONAL].reset_index().set_index("crop_name") + provincial = df[levels == PROVINCIAL].reset_index().groupby("crop_name") + columns = [column for column in ADDITIVE_COLUMNS if column in df.columns] + + rollup = { + column: worst_relative(national[column], provincial[column].sum()) + for column in columns + } + if rollup and max(rollup.values()) > INVARIANT_RTOL: + column = max(rollup, key=lambda name: rollup[name]) + broken[Invariant.ROLLUP] = ( + f"worst {rollup[column]:.3e} on {column:s}, over {len(national):d} national row(s)" + ) + + # Both admin levels, because attach_ratios derives these for provincials while + # iter_national_from_provincials derives them again for nationals -- two separate code paths + factor = worst_relative( + df["emissions_factor_kgco2e_per_kg"], + (df["emissions_mt"] * KG_PER_TONNE) + .div(df["production_kg"]) + .where(df["production_kg"] > 0), + ) + if factor > INVARIANT_RTOL: + broken[Invariant.EMISSIONS_FACTOR] = f"worst {factor:.3e}" + + per_hectare = worst_relative( + df["yield_kg_per_ha"], + df["production_kg"].div(df["crop_hectares"]).where(df["crop_hectares"] > 0), + ) + if per_hectare > INVARIANT_RTOL: + broken[Invariant.YIELD] = f"worst {per_hectare:.3e}" + + fraction = df["peatland_occupation_fraction"].dropna() + outside = int(((fraction < 0) | (fraction > 1)).sum()) + if outside: + broken[Invariant.PEAT_FRACTION] = ( + f"{outside:d} row(s) outside [0, 1], range " + f"[{fraction.min():.4f}, {fraction.max():.4f}]" + ) + + negative = {column: int((df[column] < 0).sum()) for column in columns} + if any(negative.values()): + broken[Invariant.NON_NEGATIVE] = ", ".join( + f"{column:s}={count:d}" for column, count in negative.items() if count + ) + return broken + + +def compare_invariants( + baseline: dict[Invariant, str], head: dict[Invariant, str], slug: str +) -> list[Finding]: + """Report per-side breaks, as a defect only where the head broke what the baseline did not. + + A break the baseline shares predates this change: worth knowing, but not this change's doing. + """ + findings: list[Finding] = [] + for invariant in Invariant: + if invariant not in baseline and invariant not in head: + continue + broke_here = invariant in head and invariant not in baseline + findings.append( + Finding( + severity=Severity.DEFECT if broke_here else Severity.ADVISORY, + message=( + f"{slug:s} breaks '{invariant.value:s}' -- baseline " + f"{baseline.get(invariant, 'holds'):s}, head " + f"{head.get(invariant, 'holds'):s}." + + ( + " The change broke it, and drift columns can look fine while a table is " + "self-inconsistent" + if broke_here + else " Not introduced by this change" + ) + ), + ) + ) + return findings + + +@dataclasses.dataclass(frozen=True) +class LegAgreement: + """How far apart the two attribution legs price one crop, at one commit.""" + + crop_name: str + statistical: float + jurisdictional_direct: float + + @property + def ratio(self) -> float: + return self.statistical / self.jurisdictional_direct + + +def check_leg_agreement( + statistical: pandas.DataFrame, jurisdictional_direct: pandas.DataFrame +) -> list[LegAgreement]: + """Implied national emission factor per crop on each leg, for the crops both legs price. + + The legs divide the SAME per-pixel emissions layer and differ only in how, so a ratio far from + one is a methodological disagreement rather than noise. The jurisdictional-direct leg has no + yield for most crops, leaving their production at zero and their factor undefined -- not zero + -- so those are dropped rather than counted as perfect agreement. + """ + + def factors(df: pandas.DataFrame) -> pandas.Series: + national = df[df.index.get_level_values("admin_level") == NATIONAL] + priced = national[national["production_kg"] > 0] + factor = priced["emissions_mt"] * KG_PER_TONNE / priced["production_kg"] + return factor.groupby(level="crop_name").first() + + left, right = factors(statistical), factors(jurisdictional_direct) + return [ + LegAgreement( + crop_name=str(crop_name), + statistical=float(left[crop_name]), + jurisdictional_direct=float(right[crop_name]), + ) + for crop_name in sorted(set(left.index) & set(right.index)) + if right[crop_name] + ] + + +def compare_leg_agreement( + baseline: list[LegAgreement], head: list[LegAgreement], iso_3166: str +) -> list[Finding]: + """Print both sides' leg ratios, and report a change that widens the disagreement. + + Only the statistical leg can move: the jurisdictional-direct one reads CDL per pixel and never + touches MapSPAM. So a ratio walking away from parity is the statistical leg drifting from the + one that corroborates it -- which per-column drift cannot show, because every column can move a + defensible amount while the two methods agree less than they did. + """ + print(f"\n=== {iso_3166:s}: agreement between the legs ===") + if not head: + print(" no crop is priced by both legs, so there is nothing to compare") + return [] + before = {agreement.crop_name: agreement for agreement in baseline} + print( + f" {'crop':<16}{'jurisdictional':>16}{'statistical':>14}{'ratio':>8}" + f"{'was':>14}{'ratio':>8}" + ) + for agreement in head: + was = before.get(agreement.crop_name) + print( + f" {agreement.crop_name:<16}{agreement.jurisdictional_direct:>16.4f}" + f"{agreement.statistical:>14.4f}{agreement.ratio:>8.2f}" + + ( + f"{was.statistical:>14.4f}{was.ratio:>8.2f}" + if was is not None + else f"{'--':>14}{'--':>8}" + ) + ) + # Distance from parity in log space, so 0.5x and 2.0x read as equally far apart + distance = lambda agreements: max( + (abs(math.log(one.ratio)) for one in agreements if one.ratio > 0), default=0.0 + ) + worst_before, worst_after = distance(baseline), distance(head) + print( + f" worst disagreement {math.exp(worst_before):.2f}x -> " + f"{math.exp(worst_after):.2f}x (parity is 1.00)" + ) + if worst_after <= worst_before: + return [] + return [ + Finding( + severity=Severity.ADVISORY, + message=( + f"{iso_3166:s} the legs agree less than they did: worst ratio " + f"{math.exp(worst_before):.2f}x to {math.exp(worst_after):.2f}x. Only the " + "statistical leg can move here, so this is that leg drifting from the one that " + "corroborates it. Per-column drift cannot show this: every column can move a " + "defensible amount while the two methods diverge" + ), + ) + ] + + +def print_findings(findings: list[Finding]) -> None: + if not findings: + print("\n no findings") + return + print() + # Severity order, so advisories cannot bury a defect or a blocked comparison + for finding in sorted(findings, key=lambda f: list(Severity).index(f.severity)): + print( + textwrap.fill( + finding.message, + initial_indent=f" {finding.severity.marker:s} ", + subsequent_indent=" ", + width=96, + ) + ) + + +def capture(directory: pathlib.Path, label: Side, targets: tuple[Target, ...]) -> None: + """Trace every target into `directory`, one parquet per (methodology, country). + + Writes a manifest rather than returning one, so the driver only ever trusts what is on disk -- + all it can trust when the capture ran in a clone, in another process. + """ + directory.mkdir(parents=True, exist_ok=True) + cwd = get_repo_root() + row_counts: dict[str, int] = {} + parquet_sha256: dict[str, str] = {} + for index, target in enumerate(targets, start=1): + # The full crop list, so this reuses the caches an ordinary trace.py run populates + crop_names = attribute.get_crop_names(methodology=target.methodology) + logger.info( + f"[{label!s}] tracing {target.slug:s} with {len(crop_names):d} crops " + f"({index:d}/{len(targets):d})" + ) + df = trace.workflow( + crop_names=crop_names, + iso_3166s=(target.iso_3166,), + methodology=target.methodology, + skip_glad_crop_filter=False, + ) + path = directory / f"{target.slug:s}.parquet" + df.to_parquet(path) + row_counts[target.slug] = len(df) + parquet_sha256[target.slug] = hashlib.sha256(path.read_bytes()).hexdigest() + logger.info(f"Wrote {path} ({len(df):d} rows)") + + Manifest( + sha=resolve_commit("HEAD", cwd=cwd), + dirty_paths=get_dirty_paths(cwd=cwd), + captured_at=datetime.datetime.now(tz=datetime.UTC).isoformat(), + scratch_root=config.Config.from_dot_env().scratch_root, + row_counts=row_counts, + parquet_sha256=parquet_sha256, + ).write(directory=directory) + + +def materialize_commit( + repo_root: pathlib.Path, + sha: str, + directory: pathlib.Path, + scratch_root: str | None, + patch: str | None, +) -> pathlib.Path: + """Lay down a checkout of `sha` that can run a capture, reusing it if already present. + + `git clone --shared` rather than `git worktree add`, which writes into the source repo's .git + and so fails when that repo is read-only. Nothing is copied: the clone borrows the source + object store through alternates. + """ + if (directory / ".git").exists(): + logger.info(f"Reusing clone at {directory}") + else: + git( + "clone", + "--shared", + "--no-checkout", + str(repo_root), + str(directory), + cwd=repo_root, + ) + logger.info(f"Cloned {repo_root} to {directory}") + + # Reset to a pristine `sha` on every run, so a reused clone neither carries last run's patch + # nor double-applies this one. Cleaning is scoped to the pipeline paths to spare the clone's + # .venv, which is untracked and expensive to rebuild. + git("checkout", "--force", "--detach", sha, cwd=directory) + git("clean", "--force", "-d", "--", *RESULT_BEARING_PATHS, cwd=directory) + logger.info(f"Checked out {sha[:12]:s} in {directory}") + + if patch: + logger.info(f"Applying {len(patch.splitlines()):d} lines of working-tree diff") + subprocess.run( + ["git", "apply", "-"], check=True, cwd=directory, input=patch, text=True + ) + + # .env is gitignored, so the clone has none and Config.from_dot_env would raise. Config reads + # the file through dotenv_values and never consults the environment, so an isolated cache root + # has to be written into the file rather than exported. + lines = (repo_root / ".env").read_text().splitlines(keepends=True) + if scratch_root is not None: + replaced = [ + line for line in lines if line.split("=")[0].strip() == "SCRATCH_ROOT" + ] + assert len(replaced) == 1, f"Expected one SCRATCH_ROOT in .env, got {replaced=}" + lines = [ + f"SCRATCH_ROOT = '{scratch_root:s}'\n" if line in replaced else line + for line in lines + ] + (directory / ".env").write_text("".join(lines)) + logger.info(f"Wrote {directory / '.env'} ({scratch_root=})") + + # Capture with today's code, so both sides write the same artifacts against the same targets + # even when the baseline predates this tool + (directory / "tools").mkdir(exist_ok=True) + shutil.copy(pathlib.Path(__file__).resolve(), directory / "tools" / TOOL_NAME) + return directory + + +def capture_side( + label: Side, + sha: str, + repo_root: pathlib.Path, + out: pathlib.Path, + targets: tuple[Target, ...], + iso_3166s: tuple[str, ...] | None, + isolated: bool, +) -> Manifest: + directory = out / str(label) + if label == Side.HEAD and not isolated: + # Nothing to isolate and no .env to rewrite, so run the working tree as it stands -- + # which also carries untracked modules that a clone would silently drop + capture(directory=directory, label=label, targets=targets) + return Manifest.read(directory=directory) + + # The baseline runs its own code untouched; only the head side carries the working tree, and + # then only because isolating the cache means it cannot run in place + patch = None + if label == Side.HEAD: + patch = git("diff", "HEAD", "--", *RESULT_BEARING_PATHS, cwd=repo_root) or None + untracked = [ + line for line in get_dirty_paths(cwd=repo_root) if line.startswith("??") + ] + assert not untracked, ( + "A clone cannot carry untracked result-bearing paths, and dropping them would " + f"measure code you are not running; git add them or drop --isolated: {untracked}" + ) + + scratch_root = None + if isolated: + # A cache root private to this exact code, so an isolated run cannot read another's answer. + # The patch digest matters: two runs of a dirty working tree share a SHA, and keying on the + # SHA alone would let the second read the first's -- the very confusion isolation is for. + token = sha[:12] + if patch is not None: + token += "-" + hashlib.sha256(patch.encode()).hexdigest()[:8] + scratch_root = storage.join_uri( + root=config.Config.from_dot_env().scratch_root, + prefix=f"measure-drift/{token:s}", + ) + + clone = materialize_commit( + repo_root=repo_root, + sha=sha, + directory=out / "clones" / sha[:12], + scratch_root=scratch_root, + patch=patch, + ) + command = [ + "uv", + "run", + "python", + f"tools/{TOOL_NAME:s}", + "--capture-to", + str(directory), + "--capture-label", + str(label), + ] + for iso_3166 in iso_3166s or (): + command += ["--iso", iso_3166] + logger.info(f"[{label!s}] running {' '.join(command):s} in {clone}") + subprocess.run(command, check=True, cwd=clone) + return Manifest.read(directory=directory) + + +def compare_target( + out: pathlib.Path, rtol: float, target: Target +) -> tuple[list[Finding], list[Drift], list[SumDrift]]: + """Every check for one (methodology, country), in the order the report prints them. + + Alignment first, because everything after it assumes the rows line up. + """ + pair = { + side: load_capture(directory=out / str(side), slug=target.slug) + for side in (Side.BASELINE, Side.HEAD) + } + left, right = pair[Side.BASELINE][0], pair[Side.HEAD][0] + + print(f"\n=== {target.slug:s}: alignment ===") + findings = check_alignment( + baseline=left, + crop_names=target.crop_names, + head=right, + slug=target.slug, + written=(pair[Side.BASELINE][1], pair[Side.HEAD][1]), + ) + + print(f"\n=== {target.slug:s}: per-value drift ===") + drifts, drift_findings = check_value_drift( + baseline=left, head=right, rtol=rtol, target=target + ) + findings += drift_findings + + print(f"\n=== {target.slug:s}: sum drift ===") + totals = check_sum_drift(baseline=left, head=right, target=target) + + print(f"\n=== {target.slug:s}: within-side invariants ===") + broken = {side: check_invariants(df=pair[side][0]) for side in pair} + for side in (Side.BASELINE, Side.HEAD): + state = f"{len(broken[side]):d} broken" if broken[side] else "all five hold" + print(f" {side!s:<10}{state:s}") + findings += compare_invariants( + baseline=broken[Side.BASELINE], head=broken[Side.HEAD], slug=target.slug + ) + + moved = sum(drift.exceeds(rtol=rtol) for drift in drifts) + sum( + total.exceeds(rtol=rtol) for total in totals + ) + print(f"\n {target.slug:s}: {moved:d} measure(s) moved beyond {rtol:g}") + return findings, drifts, totals + + +def get_parser() -> argparse.ArgumentParser: + """The CLI, kept out of main because it is 45 lines of boilerplate around six decisions.""" + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--baseline", help="commit-ish to compare the working tree against" + ) + parser.add_argument( + "--iso", + action="append", + dest="iso_3166s", + help="restrict the hardcoded targets to these countries; repeatable", + ) + parser.add_argument( + "--out", + type=pathlib.Path, + help="where to write captures; default under $TMPDIR", + ) + parser.add_argument( + "--isolated", + action="store_true", + help="give each side its own SCRATCH_ROOT and recompute from cold; always correct, and " + "priced as the full pipeline rather than a cache read", + ) + parser.add_argument("--rtol", default=DEFAULT_RTOL, type=float) + parser.add_argument( + "--compare-only", + action="store_true", + help="re-run the checks on the captures already in --out, without capturing again", + ) + parser.add_argument( + "--capture-to", + type=pathlib.Path, + help=argparse.SUPPRESS, # set when this tool re-invokes itself inside a clone + ) + parser.add_argument( + "--capture-label", choices=tuple(Side), type=Side, help=argparse.SUPPRESS + ) + return parser + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + args = get_parser().parse_args() + + iso_3166s = tuple(args.iso_3166s) if args.iso_3166s else None + targets = tuple(iter_targets(iso_3166s=iso_3166s)) + assert targets, f"No hardcoded targets match {iso_3166s=}" + + if args.capture_to is not None: + assert args.capture_label is not None, "--capture-to needs --capture-label" + capture(directory=args.capture_to, label=args.capture_label, targets=targets) + return 0 + + assert args.baseline is not None, "--baseline is required" + repo_root = get_repo_root() + baseline_sha = resolve_commit(args.baseline, cwd=repo_root) + head_sha = resolve_commit("HEAD", cwd=repo_root) + assert baseline_sha != head_sha or get_dirty_paths(cwd=repo_root), ( + f"{args.baseline:s} resolves to HEAD and the tree is clean, so both sides would run " + "identical code and any verdict would be vacuous" + ) + out = args.out or ( + pathlib.Path(os.environ.get("TMPDIR", "/tmp")) + / "measure-drift" + / baseline_sha[:12] + ) + logger.info( + f"Capturing {len(targets):d} target(s) at {baseline_sha[:12]:s} and " + f"{head_sha[:12]:s} into {out} ({'isolated' if args.isolated else 'shared':s} cache)" + ) + + manifests = { + label: ( + Manifest.read(directory=out / str(label)) + if args.compare_only + else capture_side( + label=label, + sha=sha, + repo_root=repo_root, + out=out, + targets=targets, + iso_3166s=iso_3166s, + isolated=args.isolated, + ) + ) + for label, sha in ((Side.BASELINE, baseline_sha), (Side.HEAD, head_sha)) + } + + baseline, head = (manifests[label] for label in (Side.BASELINE, Side.HEAD)) + print(f"\n=== captured to {out} ===") + print(f" {'target':<34}{'baseline rows':>14}{'head rows':>11} digest") + identical = [] + for target in targets: + same = baseline.parquet_sha256[target.slug] == head.parquet_sha256[target.slug] + identical += [target.slug] if same else [] + print( + f" {target.slug:<34}{baseline.row_counts[target.slug]:>14d}" + f"{head.row_counts[target.slug]:>11d} {'identical' if same else 'differs':s}" + ) + if identical: + # The likeliest reason for identical bytes is that both sides resolved to one parquet + print( + f"\n {len(identical):d} target(s) wrote byte-identical parquets. If you expected" + "\n movement, check whether both sides read the same cached parquet -- compare the" + "\n 'Loading from' URIs logged above, or rerun with --isolated." + ) + + findings, drifts, totals = ( + list(itertools.chain.from_iterable(kind)) + for kind in zip( + *( + compare_target(out=out, rtol=args.rtol, target=target) + for target in targets + ), + strict=True, + ) + ) + + # Cross-target, so it cannot live in compare_target: agreement is a property of a pair of + # legs, and only shows where one country is covered by both methodologies. + iso_3166_to_methodologies: dict[str, set[attribute.Methodology]] = ( + collections.defaultdict(set) + ) + for target in targets: + iso_3166_to_methodologies[target.iso_3166].add(target.methodology) + for iso_3166, methodologies in sorted(iso_3166_to_methodologies.items()): + if len(methodologies) < 2: + continue + pair = { + side: { + methodology: load_capture( + directory=out / str(side), + slug=Target(methodology=methodology, iso_3166=iso_3166).slug, + )[0] + for methodology in methodologies + } + for side in (Side.BASELINE, Side.HEAD) + } + findings += compare_leg_agreement( + baseline=check_leg_agreement( + statistical=pair[Side.BASELINE][attribute.Methodology.STATISTICAL], + jurisdictional_direct=pair[Side.BASELINE][ + attribute.Methodology.JURISDICTIONAL_DIRECT + ], + ), + head=check_leg_agreement( + statistical=pair[Side.HEAD][attribute.Methodology.STATISTICAL], + jurisdictional_direct=pair[Side.HEAD][ + attribute.Methodology.JURISDICTIONAL_DIRECT + ], + ), + iso_3166=iso_3166, + ) + + print_findings(findings=findings) + print( + f"\n{sum(drift.exceeds(rtol=args.rtol) for drift in drifts):d} of {len(drifts):d} " + f"column(s) and {sum(total.exceeds(rtol=args.rtol) for total in totals):d} of " + f"{len(totals):d} total(s) moved by more than {args.rtol:g}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/show-continents.py b/tools/show-continents.py deleted file mode 100644 index a632d1b..0000000 --- a/tools/show-continents.py +++ /dev/null @@ -1,91 +0,0 @@ -import collections -import logging - -import geopandas -import shapely - -from jdluc.continents import ISO_3166_TO_CONTINENT, Continent -from jdluc.datasets.worldbank_jurisdictions import ( - AdminLevel, - get_jurisdiction_for_admin_level, -) -from jdluc.tiling import get_box_for_tile_id - -logger = logging.getLogger(__name__) - - -def dump_boundaries( - iso_3166_to_continent: dict[str, Continent], path_to_geojson: str -) -> None: - nationals = get_jurisdiction_for_admin_level(admin_level=AdminLevel.NATIONAL) - - continent_to_iso_3166s: dict[Continent, list[str]] = collections.defaultdict(list) - for iso_3166, cluster in iso_3166_to_continent.items(): - continent_to_iso_3166s[cluster].append(iso_3166) - logger.info("Merging national boundaries") - continent_to_geometry = { - continent.name: shapely.unary_union( - [ - nationals.loc[iso_3166].geometry.simplify(tolerance=0.1) - for iso_3166 in iso_3166s - ] - ) - for continent, iso_3166s in continent_to_iso_3166s.items() - } - gdf = geopandas.GeoDataFrame( - crs=4326, - geometry=[geometry for _, geometry in sorted(continent_to_geometry.items())], - index=sorted(continent_to_geometry), - ) - logger.info(f"Writing to {path_to_geojson=:s}") - gdf.to_file(filename=path_to_geojson, driver="GeoJSON") - - -def dump_tiles( - continent_to_tile_ids: dict[str, set[str]], path_to_geojson: str -) -> None: - logger.info("Merging tiles") - continent_to_geometry = { - continent: shapely.unary_union(list(map(get_box_for_tile_id, tile_ids))) - for continent, tile_ids in continent_to_tile_ids.items() - } - gdf = geopandas.GeoDataFrame( - crs=4326, - geometry=[geometry for _, geometry in sorted(continent_to_geometry.items())], - index=sorted(continent_to_tile_ids), - ) - logger.info(f"Writing to {path_to_geojson=:s}") - gdf.to_file(filename=path_to_geojson, driver="GeoJSON") - - -def main() -> int: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - - for continent in Continent: - logger.info(f"{continent.name:s} contains {len(continent.value):d} tiles") - - continent_to_iso_3166s: dict[Continent, set[str]] = collections.defaultdict( - set[str] - ) - for iso_3166, continent in ISO_3166_TO_CONTINENT.items(): - continent_to_iso_3166s[continent].add(iso_3166) - for continent, iso_3166s in continent_to_iso_3166s.items(): - logger.info(f"{continent.name:s} contains {len(iso_3166s):d} countries") - - dump_boundaries( - iso_3166_to_continent=ISO_3166_TO_CONTINENT, - path_to_geojson="boundaries.geojson", - ) - dump_tiles( - continent_to_tile_ids={c.name: c.value for c in Continent}, - path_to_geojson="tiles.geojson", - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/zarr-tile-server.py b/tools/zarr-tile-server.py index b19f3cc..10518db 100644 --- a/tools/zarr-tile-server.py +++ b/tools/zarr-tile-server.py @@ -175,7 +175,7 @@ def main(): parser.add_argument( "--nearest", action="store_true", - help="Use nearest-neighbour resampling instead of bilinear", + help="Use nearest-neighbor resampling instead of bilinear", ) args = parser.parse_args() @@ -199,7 +199,7 @@ def main(): _vmin = args.vmin _vmax = args.vmax - log.info("Colour range: [%g, %g]", _vmin, _vmax) + log.info("Color range: [%g, %g]", _vmin, _vmax) _resampling = Image.NEAREST if args.nearest else Image.BILINEAR log.info("Resampling: %s", "nearest" if args.nearest else "bilinear") diff --git a/uv.lock b/uv.lock index 222140f..d914207 100644 --- a/uv.lock +++ b/uv.lock @@ -820,6 +820,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "iso3166" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/11/b5023c736a185a88ebd0d38646af6f4d1b4c9b91f2ca84e08e5d2bc7ac3c/iso3166-2.1.1.tar.gz", hash = "sha256:fcd551b8dda66b44e9f9e6d6bbbee3a1145a22447c0a556e5d0fb1ad1e491719", size = 12807, upload-time = "2022-07-12T04:07:57.294Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/d0/bf18725b8d47f37858ff801f8e4d40c6982730a899725bdb6ded62199954/iso3166-2.1.1-py3-none-any.whl", hash = "sha256:263660b36f8471c42acd1ff673d28a3715edbce7d24b1550d0cf010f6816c47f", size = 9829, upload-time = "2022-07-12T04:07:55.54Z" }, +] + [[package]] name = "jdluc" version = "0.1.0" @@ -831,6 +840,7 @@ dependencies = [ { name = "gcsfs" }, { name = "geopandas" }, { name = "gitpython" }, + { name = "iso3166" }, { name = "netcdf4" }, { name = "pandas" }, { name = "pyarrow" }, @@ -861,6 +871,7 @@ requires-dist = [ { name = "gcsfs" }, { name = "geopandas" }, { name = "gitpython" }, + { name = "iso3166" }, { name = "netcdf4" }, { name = "pandas" }, { name = "pyarrow" }, diff --git a/validation/__init__.py b/validation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validation/__main__.py b/validation/__main__.py new file mode 100644 index 0000000..e5e1eb1 --- /dev/null +++ b/validation/__main__.py @@ -0,0 +1,222 @@ +"""One entrypoint. Retrieve the anchors, compare against them, print the result. + + uv run python -m validation everything the anchors support + uv run python -m validation --stage pull retrieve and lock, nothing else + uv run python -m validation --section comparisons only that section + uv run python -m validation --year 2015 a different FAOSTAT vintage + +**Sections are individually selectable because the whole document will not always fit.** GitHub caps a +pull-request body at 65,536 characters and the comparison table alone will exceed that once provincial +rows land, so a slice has to be pastable on its own. Every section stands alone for that reason. + +**The exit code does not depend on the findings.** It is 0 whenever the run completed, however bad the +news, because a report that fails on a bad result cannot be used to characterize bad results. It fails +only on unreadable input or a missing join key. `tools/` is the deliberate exception: a generator that +emits a broken committed artifact exits nonzero. + +`capture` is not a stage here. It runs our own pipeline, so it is `validation.capture`, run +by hand; until it has been run, the conservation bound and every measure with a term of ours is +reported as not-yet-run rather than omitted -- a missing section reads as a passing one. +""" + +import argparse +import enum +import logging +import pathlib +import typing + +from validation import prepare, pull, report, targets + +logger = logging.getLogger(__name__) + +# The eligible shortlist runs to hundreds of pairs, and the whole document has to fit a GitHub +# pull-request body. The findings are computed over every pair regardless of what is tabulated. +DEFAULT_ELIGIBLE_ROWS = 20 + + +class Stage(enum.StrEnum): + """What to run. `capture` is not here; see the module docstring.""" + + PULL = enum.auto() + REPORT = enum.auto() + + +class Section(enum.StrEnum): + """The report's sections, in the order they are printed. + + Ordering is deliberate. `ELIGIBLE` comes first because it establishes the universe the rest is + drawn from, and because it carries the one BLOCKING finding measurable without a capture: the + pairs the pipeline cannot produce a comparable number for at all outrank any disagreement about + a number it can. Denominators come next, because a product-form mismatch makes every factor + built on that denominator incomparable. `COMPARISONS` carries the conservation bound and the + findings, which outrank the anchor tables inside it. + """ + + ELIGIBLE = enum.auto() + YIELDS = enum.auto() + ANCHORS = enum.auto() + COMPARISONS = enum.auto() + + +def get_document( + repo_root: pathlib.Path, show: int, wanted: tuple[Section, ...], year: int +) -> str: + """Render the requested sections, in `Section` order, joined into one document. + + Everything is computed once whether or not the section that uses it was asked for. None of these + reads is expensive -- the largest is a 1.4 MiB parquet -- so a compute-only-what-you-need branch per + section bought a little speed at the cost of four conditionals around the code that does the work. + """ + wri_yields = prepare.read_wri_yields(grain_name="national") + comparison = prepare.get_yield_comparison( + faostat_yields=prepare.get_faostat_yields(), wri_yields=wri_yields, year=year + ) + agreements = tuple(prepare.iter_yield_agreements(comparison=comparison)) + emissions = prepare.read_wri_national_emissions() + stability = prepare.get_anchor_stability(emissions=emissions) + deforestation = prepare.get_deforestation_share( + emissions=emissions, + faostat_areas=prepare.read_faostat_areas(year=year), + year=year, + ) + comparisons = prepare.get_comparisons( + repo_root=repo_root, deforestation=deforestation + ) + + eligible = prepare.get_eligible() + rendered = { + Section.ELIGIBLE: report.render_eligible( + eligible=eligible, + findings=prepare.get_perennial_findings(eligible=eligible), + show=show, + ), + Section.YIELDS: report.render_yield_agreement( + agreements=agreements, + findings=prepare.get_yield_findings(agreements=agreements, year=year), + unpaired=prepare.get_unpaired_crop_names( + comparison=comparison, wri_yields=wri_yields + ), + year=year, + ), + Section.ANCHORS: report.render_anchor_emissions( + deforestation=deforestation, + findings=( + prepare.get_stability_findings( + stability=stability, tolerance=targets.DEFAULT_TOLERANCE + ) + + prepare.get_deforestation_share_findings(deforestation=deforestation) + + prepare.get_anchor_disagreement_findings( + agreement=prepare.get_anchor_shape_agreement() + ) + + prepare.get_orbae_findings(frame=prepare.read_orbae()) + ), + scope_ratios=prepare.get_scope_difference()["scope_ratio"], + stability=stability, + ), + Section.COMPARISONS: report.render( + comparisons=comparisons, + # None until a capture has run: the pool needs a raster pass over cached layers, so + # `validation.capture` derives it and writes it beside the emissions it bounds. + forest_pools=prepare.read_forest_pools(), + unanchored=prepare.get_unanchored_targets(comparisons=comparisons), + # Target-keyed rather than comparison-keyed, so a pair no anchor reaches still says its + # anchor contradicts itself; see `get_target_anchor_consistency_findings`. + extra_findings=prepare.get_target_anchor_consistency_findings( + deforestation=deforestation + ), + ), + } + assert set(rendered) == set(Section), "a Section has no rendered text" + document = "\n\n".join( + rendered[section] for section in Section if section in wanted + ) + # Appended here rather than inside a section: only this function sees the assembled length, and + # a section measuring itself would stay silent exactly when the whole set went over. + return document + report.get_size_note(document=document) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--stage", + type=Stage, + choices=tuple(Stage), + default=Stage.REPORT, + help="which stage to run", + ) + parser.add_argument( + "--section", + action="append", + type=Section, + choices=tuple(Section), + default=[], + help="render only this section; repeatable, defaults to all of them", + ) + parser.add_argument( + "--overwrite", + action="append", + default=[], + choices=("wri",), + help="pull stage only: re-retrieve this source even where the digest matches", + ) + parser.add_argument( + "--repo-root", + type=pathlib.Path, + # `validation/` sits at the repo root once landed, so its parent is the checkout. While it + # lives in a scratch tree outside the repo, pass --repo-root explicitly. + default=pathlib.Path(__file__).resolve().parent.parent, + help="the jdluc checkout, read for code_version", + ) + parser.add_argument( + "--show", + type=int, + default=DEFAULT_ELIGIBLE_ROWS, + help="how many of the eligible pairs to tabulate; the findings cover all of them", + ) + parser.add_argument( + "--year", + type=int, + default=prepare.REFERENCE_YEAR, + help="the FAOSTAT year to compare WRI's undated yield against", + ) + args = parser.parse_args() + + # Bound to a typed local so `assert_never` can see the enum: argparse hands back Any, against + # which exhaustiveness cannot be checked. With the annotation, adding a Stage without a case is a + # mypy error naming the member, and at runtime it raises rather than falling through to return 0. + stage: Stage = args.stage + match stage: + case Stage.PULL: + remotes = tuple(pull.iter_remotes()) + counts = pull.workflow(overwrite=tuple(args.overwrite), remotes=remotes) + # Orbae is supplied rather than retrieved, so `pull` cannot fetch it -- but the lock + # should still record every anchor's bytes, and five baselines rest on these. + if prepare.ORBAE_EXPORT.exists(): + prepare.pin_orbae_export() + print( + f"\n{len(remotes):d} file(s): {counts['downloaded']:d} retrieved, " + f"{counts['reused']:d} already current, {counts['changed']:d} changed upstream" + ) + case Stage.REPORT: + document = get_document( + repo_root=args.repo_root, + show=args.show, + wanted=tuple(args.section) or tuple(Section), + year=args.year, + ) + print(document) + if len(document) > report.PULL_REQUEST_BODY_LIMIT: + # To stderr, so piping the document somewhere does not carry the warning into it. + logger.warning( + f"{len(document):,d} characters exceeds GitHub's " + f"{report.PULL_REQUEST_BODY_LIMIT:,d}-character pull-request body limit; " + "render fewer sections with --section" + ) + case _: + typing.assert_never(stage) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/validation/__tests__/__init__.py b/validation/__tests__/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validation/__tests__/fixture.py b/validation/__tests__/fixture.py new file mode 100644 index 0000000..ec3b448 --- /dev/null +++ b/validation/__tests__/fixture.py @@ -0,0 +1,248 @@ +"""Invented rows that exercise the schema, so `report` can be developed before any capture exists. + +Deliberately synthetic and deliberately round. The fixture proves plumbing, never numbers, so +nothing here should be mistakable for a result: the countries are fictional ISO codes in the +user-assigned `XA` range, and every ratio is a value no real comparison would land on exactly. + +It covers what the report has to survive: a control inside tolerance and one outside it, a rank +control inside its absolute band whose movement would exceed a relative one and a rank control +outside it either way, a conservation overrun, a grassland row that exists only as a remainder, a +`PATTERN_ONLY` row that must not be read as a level, a `ROLLED_UP` row with partial coverage, a +`BORROWED` row whose confidence is degraded, a control frozen against anchors this run did not read, +a pair whose own anchor contradicts itself across two of its rows, and a target with no anchor at all +so the coverage section has something to report. +""" + +import pandas + +from validation import schema, targets + +CODE_VERSION = "0000000fixture" +# No row carries this. A table spanning two code versions is a defect state rather than a fixture +# row, so the test that needs one applies this to a copy; it lives here to keep every invented +# identifier in one place. +SUPERSEDED_CODE_VERSION = "0000000stale00" +# The anchors every row here was measured against, and the one a stale control was frozen against. +# Two distinct values are the whole point: a control frozen against anchors the run did not read +# cannot have its movement attributed. +SOURCE_VERSION = "ORBAE=fixture00000,WRI=fixture00000" +SUPERSEDED_SOURCE_VERSION = "ORBAE=fixture00000,WRI=superseded00" + + +def get_comparisons() -> pandas.DataFrame: + """One row per (target, pool, measure) -- the shape `prepare` will emit for real.""" + records = [ + # A control comfortably inside tolerance + { + "iso_3166": "XAA", + "crop_name": "MAIZE", + "anchor_deforestation_share": 1.250, + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.SLUC_OVER_WRI, + "statistic": schema.Statistic.RATIO, + "numerator": 2.0, + "denominator": 5.0, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": True, + "baseline": 0.40, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.PULLED, + "confidence": schema.Confidence.HIGH, + }, + # A control that has moved well outside tolerance -- must surface above the tables + { + "iso_3166": "XAB", + "crop_name": "SOYBEAN", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.SLUC_OVER_WRI, + "statistic": schema.Statistic.RATIO, + "numerator": 1.0, + "denominator": 5.0, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": True, + "baseline": 0.70, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.PULLED, + "confidence": schema.Confidence.HIGH, + }, + # Anchor-versus-anchor, and not a control here only because the two frozen rank controls + # below already cover that path. In targets.json five of these are frozen: both sides are + # external, so the pair can be baselined before any capture exists. + { + "iso_3166": "XAB", + "crop_name": "SOYBEAN", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.ORBAE_OVER_WRI, + "statistic": schema.Statistic.RANK_CORRELATION, + "numerator": 4.0, + "denominator": 5.0, + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": False, + "baseline": None, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.BORROWED, + "confidence": schema.Confidence.LOW, + }, + # Grassland exists only as the remainder, and has no external anchor at all + { + "iso_3166": "XAA", + "crop_name": "MAIZE", + "anchor_deforestation_share": 1.250, + "emission_pool": schema.EmissionPool.GRASSLAND, + "measure": targets.Measure.SLUC_OVER_JDLUC, + "statistic": schema.Statistic.RATIO, + "numerator": 30.0, + "denominator": 12.0, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": False, + "baseline": None, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.MEASURED, + "confidence": schema.Confidence.LOW, + }, + # Provincial per-kg EF: WRI publishes no production at that grain, so shape only + { + "iso_3166": "XAC", + "crop_name": "OILPALM", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.SLUC_OVER_WRI, + "statistic": schema.Statistic.RATIO, + "numerator": 0.5, + "denominator": 25.0, + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": False, + "baseline": None, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.PULLED, + "confidence": schema.Confidence.MEDIUM, + }, + # A rollup on our weights over partial coverage -- never a published figure + { + "iso_3166": "XAC", + "crop_name": "OILPALM", + "emission_pool": schema.EmissionPool.TOTAL, + "measure": targets.Measure.SLUC_OVER_WRI, + "statistic": schema.Statistic.RATIO, + "numerator": 8.0, + "denominator": 10.0, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.ROLLED_UP, + "coverage_fraction": 0.62, + "is_control": False, + "baseline": None, + "tolerance": targets.DEFAULT_TOLERANCE, + "worst_tier": schema.SourceTier.PULLED, + "confidence": schema.Confidence.MEDIUM, + }, + # A rank control inside its band, and the row that pins the arithmetic: +0.443 to +0.500 is + # 0.057 in correlation units, inside the 0.10 band, but 12.9% of the baseline -- so a + # relative test fires on it and an absolute one must not. + { + "iso_3166": "XAF", + "crop_name": "SOYBEAN", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.ORBAE_OVER_WRI, + "ratio": 0.500, + "statistic": schema.Statistic.RANK_CORRELATION, + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": True, + "baseline": 0.443, + "tolerance": targets.DEFAULT_RANK_TOLERANCE, + "worst_tier": schema.SourceTier.SUPPLIED, + "confidence": schema.Confidence.MEDIUM, + }, + # Frozen against a WRI revision this run did not read. Its movement is large enough to fire + # on any tolerance, so what it exercises is that a moved anchor is reported as ADVISORY and + # the movement is not attributed at all. + { + "iso_3166": "XAH", + "crop_name": "SOYBEAN", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.ORBAE_OVER_WRI, + "ratio": 0.900, + "statistic": schema.Statistic.RANK_CORRELATION, + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": True, + "baseline": 0.443, + "baseline_source_version": SUPERSEDED_SOURCE_VERSION, + "tolerance": targets.DEFAULT_RANK_TOLERANCE, + "worst_tier": schema.SourceTier.SUPPLIED, + "confidence": schema.Confidence.MEDIUM, + }, + # The same baseline moved 0.243 -- outside the band under either arithmetic. + { + "iso_3166": "XAG", + "crop_name": "SOYBEAN", + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.ORBAE_OVER_WRI, + "ratio": 0.200, + "statistic": schema.Statistic.RANK_CORRELATION, + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + "is_control": True, + "baseline": 0.443, + "tolerance": targets.DEFAULT_RANK_TOLERANCE, + "worst_tier": schema.SourceTier.SUPPLIED, + "confidence": schema.Confidence.MEDIUM, + }, + ] + frame = pandas.DataFrame.from_records(data=records) + # A rank row has no numerator and denominator -- rho is measured, not divided -- so it carries + # `ratio` directly, and the quotient is derived only where the two parts exist. from_records + # fills each absent key with NaN, which is what makes the two shapes coexist in one frame. + frame["ratio"] = frame["ratio"].fillna(frame["numerator"] / frame["denominator"]) + frame["code_version"] = CODE_VERSION + frame["source_version"] = SOURCE_VERSION + # Every invented row sits on a chosen pair: the fixture exercises rendering, and a row excluded + # from the table would exercise nothing. + frame["is_target"] = True + # Every control but the superseded one was frozen against the anchors this run read. + frame["baseline_source_version"] = frame["baseline_source_version"].fillna( + SOURCE_VERSION + ) + return frame + + +def get_forest_pools() -> pandas.DataFrame: + """Per-country forest-conversion pool against the sum of per-crop forest emissions. + + XAD is over the bound, which is physically impossible and therefore outranks every anchor + disagreement: no allocation can hand out more than the pool holds. + """ + return pandas.DataFrame.from_records( + data=[ + {"iso_3166": "XAA", "attributed_tonnes": 4.0e6, "pool_tonnes": 1.0e7}, + {"iso_3166": "XAB", "attributed_tonnes": 9.0e6, "pool_tonnes": 1.0e7}, + {"iso_3166": "XAD", "attributed_tonnes": 2.5e7, "pool_tonnes": 1.0e7}, + ] + ) + + +def get_unanchored_targets() -> tuple[targets.Target, ...]: + """Targets in scope that no anchor covers, so the coverage section has something to say. + + Silence is not agreement, and a report that omits these reads as though it checked them. + """ + return ( + targets.Target( + iso_3166="XAE", + crop_name="WHEAT", + basis="ranked", + reason="Invented, to exercise the coverage section's unanchored branch", + ), + ) diff --git a/validation/__tests__/prepare_test.py b/validation/__tests__/prepare_test.py new file mode 100644 index 0000000..57275fa --- /dev/null +++ b/validation/__tests__/prepare_test.py @@ -0,0 +1,166 @@ +"""Checks on the yield cross-check that a real run cannot make for us. + +The live run currently finds every crop in agreement, which is the answer we want but leaves the +failure paths unexercised: nothing in it proves a product-form mismatch would actually be caught. A +mismatch is the one outcome this check exists to find, so it is asserted here against invented +ratios rather than waiting for a source to regress. +""" + +import pandas +import pytest + +from validation import prepare, schema + + +def get_comparison(ratios: dict[str, list[float]]) -> pandas.DataFrame: + """A comparison frame carrying only what `iter_yield_agreements` reads.""" + return pandas.DataFrame.from_records( + data=[ + {"crop_name": crop_name, "ratio": ratio} + for crop_name, crop_ratios in ratios.items() + for ratio in crop_ratios + ] + ) + + +def test_a_crop_short_of_the_minimum_is_not_medianed() -> None: + """Two countries agreeing says nothing about a crop, and would read as though it did.""" + agreements = tuple( + prepare.iter_yield_agreements( + comparison=get_comparison( + ratios={ + "XAA": [1.0] * (prepare.MINIMUM_COUNTRIES - 1), + "XAB": [1.0] * prepare.MINIMUM_COUNTRIES, + } + ) + ) + ) + assert [agreement.crop_name for agreement in agreements] == ["XAB"] + + +def test_an_outlier_country_does_not_move_the_median() -> None: + """One near-zero denominator produces a ratio in the hundreds; a mean would follow it.""" + (agreement,) = prepare.iter_yield_agreements( + comparison=get_comparison(ratios={"XAA": [1.0, 1.0, 1.0, 1.0, 1.0, 400.0]}) + ) + assert agreement.median_ratio == 1.0 + assert agreement.highest_ratio == 400.0 + + +@pytest.mark.parametrize( + ("ratio", "is_mismatch", "severity"), + ( + pytest.param(1.00, False, None, id="parity-says-nothing-and-must-say-nothing"), + pytest.param(0.93, False, None, id="mapspam-runs-7-percent-under-faostat"), + pytest.param(0.86, False, None, id="just-inside-the-tolerance"), + pytest.param( + 0.80, + False, + schema.Severity.ADVISORY, + id="a-denominator-difference-is-the-same-quantity-differently-estimated", + ), + pytest.param( + 0.50, + False, + schema.Severity.ADVISORY, + id="the-product-form-bound-is-inclusive", + ), + pytest.param(2.00, False, schema.Severity.ADVISORY, id="at-the-top-as-well"), + pytest.param( + 2.01, + True, + schema.Severity.BLOCKING, + id="just-past-it-is-a-different-quantity", + ), + pytest.param( + 0.20, + True, + schema.Severity.BLOCKING, + id="palm-oil-against-fruit-bunches-is-a-milling-yield", + ), + ), +) +def test_a_yield_disagreement_is_graded_by_how_far_it_is( + ratio: float, is_mismatch: bool, severity: schema.Severity | None +) -> None: + """A mismatch is BLOCKING and a disagreement ADVISORY: the first was never like-for-like. + + 0.86 rather than the 0.85 the tolerance would suggest: 1 - 0.85 is 0.15000000000000002 in + IEEE 754, so the exact bound fires, and pinning that as inclusive would encode a float artefact + as intent. + """ + (agreement,) = prepare.iter_yield_agreements( + comparison=get_comparison(ratios={"XAA": [ratio] * prepare.MINIMUM_COUNTRIES}) + ) + assert agreement.is_product_form_mismatch == is_mismatch + findings = prepare.get_yield_findings(agreements=(agreement,), year=2020) + if severity is None: + assert findings == [] + else: + (finding,) = findings + assert finding.severity == severity + + +def test_every_unpaired_crop_gets_a_reason() -> None: + """An unexplained crop would be dropped from the report, reading as one that agreed.""" + wri_yields = pandas.DataFrame.from_records( + data=[ + {"crop_name": name} + for name in ("MAIZ", "OOIL", "RCOF", "CITR", "SESA", "BARL") + ] + ) + unpaired = prepare.get_unpaired_crop_names( + comparison=get_comparison(ratios={"MAIZ": [1.0], "SESA": [1.0]}), + wri_yields=wri_yields, + ) + assert unpaired == { + schema.UnpairedReason.SPAM_GROUP: ("OOIL",), + schema.UnpairedReason.SPAM_SPLIT: ("RCOF",), + schema.UnpairedReason.TOO_FEW_COUNTRIES: ("BARL",), + schema.UnpairedReason.UNMAPPED: ("CITR",), + } + + +def get_deforestation(shares: dict[tuple[str, str], float]) -> pandas.DataFrame: + """A deforestation frame carrying only what the target consistency check reads.""" + return pandas.DataFrame.from_records( + data=[ + {"iso_3166": iso_3166, "crop_name": crop_name, "deforestation_share": share} + for (iso_3166, crop_name), share in shares.items() + ] + ) + + +def test_a_target_on_a_self_contradictory_anchor_row_is_named() -> None: + """Keyed on the target set, because the pairs this catches are the ones no anchor reaches. + + BOL SOYBEAN and IDN MAIZE are both real targets that no Orbae comparison covers, so a check + hung off the comparison frame would report nothing here and read as agreement. BOL is the pair + whose crop code differs from its name (SOYB), which is where a keying mistake would show. + """ + findings = prepare.get_target_anchor_consistency_findings( + deforestation=get_deforestation( + shares={ + ("BOL", "SOYB"): 1.123, + ("IDN", "MAIZ"): 1.118, + # Below the bound, and a real target, so it must not be reported. + ("PRY", "SOYB"): 0.669, + # Over the bound but not in the target set at all. + ("XAA", "MAIZ"): 9.999, + } + ) + ) + assert [finding.affected_iso_3166s for finding in findings] == [("IDN",), ("BOL",)] + assert all(finding.severity == schema.Severity.ADVISORY for finding in findings) + + +def test_a_target_that_arms_nothing_says_so() -> None: + """Whether the contradictory row carries a control is the difference between the two cases.""" + (armed,) = prepare.get_target_anchor_consistency_findings( + deforestation=get_deforestation(shares={("PRY", "SOYB"): 1.500}) + ) + (unarmed,) = prepare.get_target_anchor_consistency_findings( + deforestation=get_deforestation(shares={("IDN", "MAIZ"): 1.500}) + ) + assert "carries a control" in armed.message + assert "arms nothing" in unarmed.message diff --git a/validation/__tests__/report_test.py b/validation/__tests__/report_test.py new file mode 100644 index 0000000..06a6e55 --- /dev/null +++ b/validation/__tests__/report_test.py @@ -0,0 +1,270 @@ +"""Rendering and provenance checks that a real run cannot make for us. + +What these have in common is that the failure is invisible in the output. The report is Markdown, so +a malformed cell is not an error -- it renders as a broken table nobody notices -- and a body that +overruns GitHub's limit is not an error either, it is silently truncated. So: a pipe that ends a cell +early, an empty table reading as agreement, a shape-only ratio indistinguishable from a level, a +control checked with the wrong arithmetic, and a table drawn from two runs of the pipeline. +""" + +import pandas +import pytest + +from validation import report, schema +from validation.__tests__ import fixture + + +def test_a_pipe_in_a_cell_is_escaped() -> None: + """Every provincial jurisdiction_name looks like "Angola | Bengo", which would end the cell.""" + table = report.format_markdown_table( + headers=("Jurisdiction", "ratio"), rows=[["Angola | Bengo", "0.400"]] + ) + row = table.splitlines()[-1] + assert row == r"| Angola \| Bengo | 0.400 |" + # Three cells, not four: the escape is what keeps the row the width of its header. + assert row.count(" | ") == len(("Jurisdiction", "ratio")) - 1 + + +def test_an_empty_table_says_so() -> None: + """A header with no body reads as "nothing wrong" rather than "nothing measured".""" + assert report.format_markdown_table(headers=("a", "b"), rows=[]) == "_(no rows)_" + + +def test_a_missing_value_renders_as_a_dash() -> None: + assert report.format_number(None) == "—" + assert report.format_number(float("nan")) == "—" + assert report.format_number(0.4) == "0.400" + + +def test_conservation_fires_only_above_the_pool() -> None: + """The bound is the one check that needs no anchor, so it has to be right on its own.""" + pools = pandas.DataFrame.from_records( + data=[ + {"iso_3166": "XAA", "attributed_tonnes": 9.0e6, "pool_tonnes": 1.0e7}, + {"iso_3166": "XAB", "attributed_tonnes": 2.5e7, "pool_tonnes": 1.0e7}, + ] + ) + (finding,) = report.get_conservation_findings(forest_pools=pools) + assert finding.severity == schema.Severity.BLOCKING + assert finding.affected_iso_3166s == ("XAB",) + assert finding.magnitude_tonnes == 1.5e7 + + +@pytest.mark.parametrize( + ("slug", "fires"), + ( + pytest.param( + "control-xaf-soybean-orbae_over_wri", + False, + id="0.057-in-correlation-units-is-inside-the-0.10-band", + ), + pytest.param( + "control-xag-soybean-orbae_over_wri", + True, + id="0.243-is-outside-it-under-either-arithmetic", + ), + pytest.param( + "control-xaa-maize-sluc_over_wri", + False, + id="a-ratio-within-its-relative-band", + ), + pytest.param( + "control-xab-soybean-sluc_over_wri", + True, + id="and-one-well-outside-it", + ), + ), +) +def test_a_control_fires_by_the_arithmetic_its_statistic_implies( + slug: str, fires: bool +) -> None: + """A relative band makes the weakest-agreeing control the twitchiest, which is backwards. + + XAF is the row where the two arithmetics disagree: +0.443 to +0.500 is 0.057 in correlation + units, inside the 0.10 band, but 12.9% of the baseline, so a relative test fires on it and an + absolute one must not. `schema.Statistic` promises a correlation is never "tolerance-checked as + though it were a ratio"; this is the row that holds that promise. + """ + fired = { + finding.slug + for finding in report.get_control_findings( + comparisons=fixture.get_comparisons() + ) + } + assert (slug in fired) is fires + + +def test_a_control_whose_anchors_moved_is_advisory_not_a_defect() -> None: + """A moved yardstick is not a moved pipeline, and calling it a DEFECT books one as the other. + + XAH is 0.900 against a baseline of 0.443 -- far outside any tolerance -- so if the anchor + version were ignored it would be the loudest DEFECT in the fixture. + """ + findings = { + finding.slug: finding + for finding in report.get_control_findings( + comparisons=fixture.get_comparisons() + ) + } + assert "control-xah-soybean-orbae_over_wri" not in findings + stale = findings["stale-baseline-xah-soybean-orbae_over_wri"] + assert stale.severity == schema.Severity.ADVISORY + assert fixture.SUPERSEDED_SOURCE_VERSION in stale.message + + +def test_borrowed_evidence_is_reported_at_all() -> None: + """BORROWED means no recorded provenance, which the figure agreeing does not repair. + + `schema.SourceTier` calls it sufficient for ranking a magnitude and not for a claim, so the row + has to be named even when nothing about its number looks wrong. + """ + findings = { + finding.slug: finding + for finding in report.get_provenance_findings( + comparisons=fixture.get_comparisons() + ) + } + borrowed = findings["borrowed-evidence"] + assert borrowed.severity == schema.Severity.ADVISORY + assert borrowed.confidence == schema.Confidence.LOW + assert borrowed.affected_rows == 1 + assert borrowed.affected_iso_3166s == ("XAB",) + + +def test_a_table_spanning_two_code_versions_is_a_defect() -> None: + """A per-ISO merge keeps rows it did not recompute, so one artifact can hold two versions. + + This is a disagreement between two runs of ours rather than between two sources, so no anchor + can catch it and nothing about the numbers looks wrong. + """ + comparisons = fixture.get_comparisons() + assert "mixed-code-version" not in { + finding.slug + for finding in report.get_provenance_findings(comparisons=comparisons) + } + comparisons.loc[comparisons.index[0], "code_version"] = ( + fixture.SUPERSEDED_CODE_VERSION + ) + findings = { + finding.slug: finding + for finding in report.get_provenance_findings(comparisons=comparisons) + } + mixed = findings["mixed-code-version"] + assert mixed.severity == schema.Severity.DEFECT + # Both versions, since the fix is to recapture whichever of the two is the stale one. + assert fixture.CODE_VERSION in mixed.message + assert fixture.SUPERSEDED_CODE_VERSION in mixed.message + assert mixed.affected_rows == len(comparisons) + + +def test_every_pattern_only_row_is_labelled_as_shape_and_not_level() -> None: + """A shape-only ratio and a level one are the same number; only the column separates them. + + Asserted over the whole table rather than one row, because labelling most of them is the same + failure as labelling none: a reader who finds the column trustworthy reads every unlabelled row + as a level. + """ + comparisons = fixture.get_comparisons() + # Every invented country is XA-something, which is what distinguishes a body row from the + # header and the alignment row without parsing the table. + rows = [ + line + for line in report.render_comparisons(comparisons=comparisons).splitlines() + if line.startswith("| XA") + ] + assert len(rows) == len(comparisons) + assert len([line for line in rows if "| pattern_only |" in line]) == int( + (comparisons["comparability"] == schema.Comparability.PATTERN_ONLY).sum() + ) + # One pair, two pools: WRI publishes no production at the provincial grain, so the per-kg + # factor carries shape while the rollup carries level. The label is the only difference. + (shape,) = [line for line in rows if line.startswith("| XAC OILPALM | forest |")] + (level,) = [line for line in rows if line.startswith("| XAC OILPALM | total |")] + assert "| pattern_only |" in shape + assert "| level |" in level + # And the rollup carries its coverage, because 62% of a country on our own provincial weights + # is not the anchor's published national figure. + assert "| rolled_up (62%) |" in level + + +def test_the_document_leads_with_findings_then_conservation() -> None: + """A table invites a conclusion the findings may already have disqualified, so order matters.""" + document = report.render( + comparisons=fixture.get_comparisons(), + forest_pools=fixture.get_forest_pools(), + unanchored=fixture.get_unanchored_targets(), + ) + for earlier, later in ( + ("### Findings", "### Forest-pool conservation"), + ("### Forest-pool conservation", "### Comparisons"), + ("### Comparisons", "### Coverage"), + ): + assert document.index(earlier) < document.index(later), ( + f"{earlier} after {later}" + ) + + +def test_coverage_names_the_targets_no_anchor_covered() -> None: + """Silence is not agreement; an omitted target reads as a checked one.""" + document = report.render( + comparisons=fixture.get_comparisons(), + forest_pools=fixture.get_forest_pools(), + unanchored=fixture.get_unanchored_targets(), + ) + assert "XAE-WHEAT" in document + + +def test_the_size_note_appears_only_past_the_limit() -> None: + """GitHub truncates an over-long body silently, so the boundary is the whole behaviour.""" + assert report.get_size_note(document="x" * report.PULL_REQUEST_BODY_LIMIT) == "" + over = "x" * (report.PULL_REQUEST_BODY_LIMIT + 1) + note = report.get_size_note(document=over) + # Both numbers: the note is only actionable if it says how far over the document is. + assert f"{len(over):,d}" in note + assert f"{report.PULL_REQUEST_BODY_LIMIT:,d}" in note + + +def test_conservation_says_it_has_not_run_rather_than_going_missing() -> None: + """`forest_pools` is None until a capture exists, so this is the path every run takes today. + + An omitted section reads as a passing one, and this is the check that outranks every anchor: a + country attributing more forest emissions than its pool holds makes its comparisons moot. + """ + document = report.render( + comparisons=fixture.get_comparisons(), + forest_pools=None, + unanchored=fixture.get_unanchored_targets(), + ) + assert "### Forest-pool conservation" in document + assert "Not run" in document + # Still in its place ahead of the tables, so nothing below reads as qualified by it. + assert document.index("### Forest-pool conservation") < document.index( + "### Comparisons" + ) + + +def test_a_blocking_finding_cannot_be_buried_under_a_larger_advisory() -> None: + """Severity first, then magnitude: an advisory worth 90 Mt still sorts below a blocking one.""" + + def get_finding( + slug: str, severity: schema.Severity, magnitude: float | None + ) -> schema.Finding: + return schema.Finding( + slug=slug, + severity=severity, + message="Invented", + confidence=schema.Confidence.HIGH, + magnitude_tonnes=magnitude, + ) + + rendered = report.render_findings( + findings=[ + get_finding("big-advisory", schema.Severity.ADVISORY, 9.0e7), + get_finding("small-blocking", schema.Severity.BLOCKING, 1.0e6), + get_finding("big-blocking", schema.Severity.BLOCKING, 5.0e7), + get_finding("unsized-defect", schema.Severity.DEFECT, None), + ] + ) + assert [ + line.split("**")[1] for line in rendered.splitlines() if line.startswith("- `") + ] == ["big-blocking", "small-blocking", "unsized-defect", "big-advisory"] diff --git a/validation/__tests__/schema_test.py b/validation/__tests__/schema_test.py new file mode 100644 index 0000000..22b5786 --- /dev/null +++ b/validation/__tests__/schema_test.py @@ -0,0 +1,54 @@ +"""Pin schema.py's copied constants to the sources they copy. + +`validation/schema.py` declares the admin-level names, the methodology names and the +tonne-to-kilogram factor itself rather than importing them, so the reporting path needs only pandas: +importing `worldbank_jurisdictions` for an enum member would pull geopandas onto it. A test may +import jdluc freely, not being that path, so these assert the copies still agree. + +The stored strings are the ones that matter. A renamed `AdminLevel` or `Methodology` member would +leave `prepare`'s filters matching nothing, reporting an empty table as agreement, which nothing +downstream would catch. `KG_PER_TONNE` cannot realistically drift and is pinned because it is free. + +`trace.ADDITIVE_COLUMNS` is named there too and so is pinnable, but nothing here copies it yet. +`prepare` will, for the national totals, and it should be pinned at the same time: a column added to +the sum in trace and not here would simply not be summed, with nothing to say so. + +tools/measure-drift.py keeps its own copies too. They are not pinned here: its filename has a hyphen +so it cannot be imported, and policing a sibling tool's internals is not this suite's job. Both +tools copy from the same source, so pinning each to jdluc keeps them consistent with each other. +""" + +from jdluc import attribute, trace +from jdluc.datasets import worldbank_jurisdictions +from validation import schema + + +def test_admin_level_names_match_the_enum() -> None: + """The table stores AdminLevel's member names, and prepare filters on these strings.""" + assert worldbank_jurisdictions.AdminLevel.NATIONAL.name == schema.NATIONAL + assert worldbank_jurisdictions.AdminLevel.PROVINCIAL.name == schema.PROVINCIAL + + +def test_methodology_names_match_the_enum() -> None: + """Both legs share one capture, so these strings are what separates them.""" + assert attribute.Methodology.STATISTICAL.name == schema.STATISTICAL + assert ( + attribute.Methodology.JURISDICTIONAL_DIRECT.name == schema.JURISDICTIONAL_DIRECT + ) + + +def test_every_methodology_is_named_here() -> None: + """A third leg would otherwise be silently absent from every comparison.""" + assert {schema.STATISTICAL, schema.JURISDICTIONAL_DIRECT} == { + methodology.name for methodology in attribute.Methodology + } + + +def test_kg_per_tonne_matches_trace() -> None: + """trace applies this to produce production_kg; schema applies it to check the factor.""" + assert schema.KG_PER_TONNE == trace.KG_PER_TONNE + + +def test_canonical_key_matches_the_index_trace_writes() -> None: + """Re-keying onto the wrong names would drop every row of a comparison.""" + assert schema.CANONICAL_KEY == trace.CANONICAL_KEY diff --git a/validation/__tests__/targets_test.py b/validation/__tests__/targets_test.py new file mode 100644 index 0000000..301de8a --- /dev/null +++ b/validation/__tests__/targets_test.py @@ -0,0 +1,439 @@ +"""Checks on the target set's reader, which a well-formed file cannot make for us. + +`data/targets.json` satisfies every check in `check_document`, which is the difficulty: those +refusals are what stop a control from being dropped silently, taking its guard with it, and a passing +file exercises none of them. They are asserted here against invented documents. + +The invariants over the real file are a different kind of check and are here too: not that the reader +refuses bad input, but that what it hands back preserves what the file said -- order, uniqueness, a +frozen baseline travelling with the yardstick it was frozen against, and a pair naming a crop and a +country that something downstream can resolve. +""" + +import contextlib +import typing + +import iso3166 +import pytest + +from jdluc import statistical +from validation import pull, schema, targets + + +def get_document(**overrides: typing.Any) -> dict[str, typing.Any]: + """One target carrying one control, well-formed, with `overrides` applied to the control. + + Keyed on the control, where all but four of the checks look. A test names only the field it is + breaking, so the diff from a document that passes is what the test is about. + """ + control: dict[str, typing.Any] = { + "emission_pool": schema.EmissionPool.FOREST.name, + "measure": targets.Measure.SLUC_OVER_WRI.name, + "inherited": 0.40, + "baseline": 0.40, + "baseline_source_versions": {"WRI": "fixture00000"}, + "note": "Invented", + } + control.update(overrides) + return { + "provenance": {"wri_revision": pull.WRI_REVISION}, + "targets": [ + { + "iso_3166": "XAA", + "crop_name": "MAIZE", + "basis": "control", + "reason": "Invented, to exercise the reader", + "controls": [control], + } + ], + } + + +def get_context(refusal: str | None) -> typing.Any: + """Refuse with a message matching `refusal`, or accept when there is none to match. + + The checks below are truth tables rather than lists of failures: what a reader needs is the line + between accepted and refused, and a table of refusals alone does not draw it. + """ + if refusal is None: + return contextlib.nullcontext() + return pytest.raises(AssertionError, match=refusal) + + +def test_a_well_formed_document_is_accepted() -> None: + """The anchor for every refusal below, each of which changes one field of this document.""" + targets.check_document(document=get_document()) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + pytest.param("measure", "SLUC_OVER_ATLANTIS", id="an-unknown-measure"), + pytest.param("emission_pool", "MANTLE", id="an-unknown-emission-pool"), + pytest.param("statistic", "VIBES", id="an-unknown-statistic"), + ), +) +def test_a_control_naming_something_that_does_not_exist_is_refused( + field: str, value: str +) -> None: + """An unresolvable name would drop the control, and a dropped control takes its guard with it.""" + with pytest.raises(AssertionError, match=value): + targets.check_document(document=get_document(**{field: value})) + + +@pytest.mark.parametrize( + ("tolerance", "refusal"), + ( + pytest.param(0.10, None, id="the-default-band"), + pytest.param(0.01, None, id="a-tight-band-is-a-choice-not-a-mistake"), + pytest.param( + 1.00, None, id="one-is-the-inclusive-end-useless-but-not-malformed" + ), + pytest.param(0.00, "outside", id="zero-fires-on-everything"), + pytest.param(1.50, "outside", id="above-one-fires-on-nothing"), + pytest.param(-0.10, "outside", id="negative-is-not-a-band-at-all"), + ), +) +def test_a_tolerance_is_read_against_the_unit_interval( + tolerance: float, refusal: str | None +) -> None: + """The bound holds under both readings of the number, which is why only the message differs.""" + with get_context(refusal=refusal): + targets.check_document(document=get_document(tolerance=tolerance)) + + +@pytest.mark.parametrize( + ("measure", "baseline", "versions", "refusal"), + ( + pytest.param( + targets.Measure.SLUC_OVER_WRI, + 0.40, + {"WRI": "fixture00000"}, + None, + id="frozen-against-the-anchors-it-was-measured-with", + ), + pytest.param( + targets.Measure.SLUC_OVER_WRI, + None, + None, + None, + id="not-yet-frozen-so-carried-rather-than-armed", + ), + pytest.param( + targets.Measure.SLUC_OVER_WRI, + 0.40, + None, + "baseline_source_versions", + id="frozen-with-no-yardstick-cannot-attribute-its-own-movement", + ), + pytest.param( + targets.Measure.SLUC_OVER_WRI, + None, + {"WRI": "fixture00000"}, + "baseline_source_versions", + id="a-yardstick-with-nothing-frozen-against-it-describes-nothing", + ), + pytest.param( + targets.Measure.SLUC_OVER_JDLUC, + 0.40, + None, + None, + id="both-terms-ours-so-code-version-is-the-whole-provenance", + ), + pytest.param( + targets.Measure.SLUC_OVER_JDLUC, + 0.40, + {"WRI": "fixture00000"}, + "divides no external anchor", + id="an-anchor-version-on-a-measure-with-no-anchor-is-a-fiction", + ), + ), +) +def test_a_baseline_travels_with_the_anchors_it_was_frozen_against( + measure: targets.Measure, + baseline: float | None, + versions: dict[str, str] | None, + refusal: str | None, +) -> None: + """Either half alone cannot tell an anchor revision from a change in our own pipeline.""" + with get_context(refusal=refusal): + targets.check_document( + document=get_document( + measure=measure.name, + baseline=baseline, + baseline_source_versions=versions, + ) + ) + + +@pytest.mark.parametrize( + ("statistic", "field", "value", "refusal"), + ( + pytest.param( + schema.Statistic.RATIO, + "baseline", + 2.50, + None, + id="a-ratio-above-one-is-ordinary", + ), + pytest.param( + schema.Statistic.RATIO, + "baseline", + 0.008, + None, + id="and-so-is-a-very-small-one", + ), + pytest.param( + schema.Statistic.RATIO, + "baseline", + 0.00, + "baseline", + id="but-zero-is-not-a-ratio", + ), + pytest.param( + schema.Statistic.RATIO, + "baseline", + -0.40, + "baseline", + id="nor-is-a-negative-one", + ), + pytest.param( + schema.Statistic.RATIO, + "inherited", + -0.40, + "inherited", + id="checked-on-both-numbers", + ), + pytest.param( + schema.Statistic.RANK_CORRELATION, + "baseline", + -0.40, + None, + id="an-ordering-may-disagree-so-a-negative-rho-is-fine", + ), + pytest.param( + schema.Statistic.RANK_CORRELATION, + "baseline", + 1.00, + None, + id="perfect-agreement-is-the-inclusive-bound", + ), + pytest.param( + schema.Statistic.RANK_CORRELATION, + "baseline", + 1.40, + "baseline", + id="past-it-is-not-a-correlation", + ), + pytest.param( + schema.Statistic.RANK_CORRELATION, + "inherited", + -1.40, + "inherited", + id="in-either-direction-and-on-both-numbers", + ), + ), +) +def test_a_number_is_read_in_the_units_its_statistic_implies( + statistic: schema.Statistic, field: str, value: float, refusal: str | None +) -> None: + """The same -0.40 is malformed as a ratio and ordinary as a correlation, which is the point.""" + with get_context(refusal=refusal): + targets.check_document( + document=get_document(statistic=statistic.name, **{field: value}) + ) + + +@pytest.mark.parametrize( + ("basis", "has_controls", "refusal"), + ( + pytest.param("control", True, None, id="a-control-pair-carries-one"), + pytest.param("reserved", True, None, id="a-reserved-pair-carries-one-too"), + pytest.param("ranked", False, None, id="a-ranked-pair-carries-none"), + pytest.param( + "gap", False, None, id="a-gap-pair-is-here-to-fail-so-it-arms-nothing" + ), + pytest.param( + "control", + False, + "expected at least one", + id="a-control-pair-with-none-is-silently-disarmed", + ), + pytest.param( + "reserved", False, "expected at least one", id="and-so-is-a-reserved-one" + ), + pytest.param( + "ranked", + True, + "expected none", + id="a-ranked-pair-with-one-arms-a-pair-nobody-chose-to-arm", + ), + pytest.param("gap", True, "expected none", id="as-does-a-gap-pair"), + pytest.param( + "elective", True, "unknown basis", id="a-basis-outside-the-four-is-refused" + ), + ), +) +def test_a_basis_and_its_controls_must_agree( + basis: str, has_controls: bool, refusal: str | None +) -> None: + """Every basis, both ways: the contradiction is silent in both directions, so both are checked.""" + document = get_document() + document["targets"][0]["basis"] = basis + if not has_controls: + document["targets"][0].pop("controls") + with get_context(refusal=refusal): + targets.check_document(document=document) + + +def test_the_document_must_name_a_distinct_and_reasoned_set() -> None: + """A repeated pair makes it ambiguous which row armed it, and a blank reason justifies nothing. + + Not parametrized: the three malformations are three different shapes of document rather than + three values of one field, and a table of mutating callables reads worse than the three cases do. + """ + with pytest.raises(AssertionError, match="names no targets"): + targets.check_document( + document={"provenance": {"wri_revision": pull.WRI_REVISION}, "targets": []} + ) + document = get_document() + document["targets"].append(dict(document["targets"][0])) + with pytest.raises(AssertionError, match="repeats XAA-MAIZE"): + targets.check_document(document=document) + document = get_document() + document["targets"][0]["reason"] = " " + with pytest.raises(AssertionError, match="no reason given"): + targets.check_document(document=document) + + +def test_a_set_chosen_against_another_wri_release_is_refused() -> None: + """Every written reason quotes that release's deforestation figures, so a moved pin leaves them + describing numbers the tool does not read.""" + document = get_document() + document["provenance"]["wri_revision"] = "0" * len(pull.WRI_REVISION) + with pytest.raises(AssertionError, match="re-derive the shortlist"): + targets.check_document(document=document) + + +@pytest.mark.parametrize( + "measure", + tuple(pytest.param(measure, id=measure.name) for measure in targets.Measure), +) +def test_a_measure_names_its_two_sources_in_order(measure: targets.Measure) -> None: + """The name is what the report prints and the value is what the arithmetic reads, so a measure + whose two disagree would label a ratio with its own inverse.""" + assert ( + measure.name == f"{measure.numerator.name:s}_OVER_{measure.denominator.name:s}" + ) + + +def test_only_a_measure_with_both_terms_ours_is_unanchored() -> None: + """`report.get_control_findings` skips the stale-anchor check on exactly these. + + Asserted as a whole set rather than per measure: what matters is that adding one forces a + decision about which side of that check it falls on, which a per-member test cannot ask. + """ + assert {measure for measure in targets.Measure if not measure.is_anchored} == { + targets.Measure.SLUC_OVER_JDLUC + } + + +@pytest.mark.parametrize( + "statistic", + tuple(pytest.param(statistic, id=statistic.name) for statistic in schema.Statistic), +) +def test_every_statistic_has_a_default_band(statistic: schema.Statistic) -> None: + """A new `Statistic` member has to choose a band rather than inheriting the ratio one. + + Only that a band exists and is usable: DEFAULT_TOLERANCE and DEFAULT_RANK_TOLERANCE are both + 0.10 today, so nothing asserted here can tell the two branches apart. What separates them is the + units, and `report_test`'s rank-versus-ratio pair is where that is pinned. + """ + assert 0.0 < targets.get_default_tolerance(statistic=statistic) <= 1.0 + + +def test_a_control_with_no_baseline_is_carried_rather_than_armed() -> None: + """Every control in the file is frozen, so this state exists only here -- and it is the state a + newly added control arrives in, where reading it as armed would fire on the first capture.""" + carried = targets.Control( + target=targets.Target( + iso_3166="XAA", crop_name="MAIZE", basis="control", reason="Invented" + ), + emission_pool=schema.EmissionPool.FOREST, + measure=targets.Measure.SLUC_OVER_WRI, + inherited=0.40, + baseline=None, + note="Invented", + ) + assert not carried.is_frozen + assert carried.tolerance == targets.DEFAULT_TOLERANCE + assert carried.statistic == schema.Statistic.RATIO + + +def test_a_target_slug_is_the_pair_a_report_prints() -> None: + """`render_coverage` prints these and `iter_control_targets` dedups on them, so the separator is + load-bearing in two places that never see each other.""" + target = targets.Target( + iso_3166="XAA", crop_name="MAIZE", basis="ranked", reason="Invented" + ) + assert target.slug == "XAA-MAIZE" + + +def test_the_control_targets_are_the_pairs_carrying_controls_in_file_order() -> None: + """A capture is driven off this list, so a pair missing from it silently loses its guard. + + Deduplicated, because a pair carrying three controls is one target to compute and not three, and + in file order, which is materiality order by construction. + """ + slugs = [target.slug for target in targets.iter_control_targets()] + assert len(slugs) == len(set(slugs)) + assert set(slugs) == {control.target.slug for control in targets.iter_controls()} + assert slugs == [ + target.slug for target in targets.iter_targets() if target.slug in set(slugs) + ] + + +def test_a_frozen_control_carries_the_anchors_it_was_frozen_against() -> None: + """`check_document` asserts this of the file; this asserts `iter_controls` carries it through, + which it builds conditionally and so could drop. + """ + frozen = [control for control in targets.iter_controls() if control.is_frozen] + assert frozen, "no control in the set is frozen, so this proves nothing" + for control in frozen: + assert ( + control.baseline_source_version is not None + ) == control.measure.is_anchored + if control.baseline_source_version is not None: + # Sorted by `schema.get_source_version_key`, which is what lets `report` compare it + # against a row's own `source_version` as one string. + anchors = control.baseline_source_version.split(",") + assert anchors == sorted(anchors) + + +def test_every_target_names_a_crop_and_a_country_that_resolve() -> None: + """A name nothing downstream resolves is malformed input, and this is where it surfaces. + + `prepare.get_target_anchor_consistency_findings` indexes `statistical.Crop` directly, so an + unknown crop raises there rather than dropping the pair's guard and reporting agreement. The + country half has no reader that would raise, and `check_document` catches neither: both fields + are well-formed strings. + + Every offender at once rather than a parametrized case per pair, because parametrizing over the + file's own contents reads it while the module is being collected: a malformed set then fails + collection for the whole file instead of failing the one test that describes it. + """ + unresolvable = sorted( + target.slug + for target in targets.iter_targets() + if target.crop_name not in statistical.Crop.__members__ + or target.iso_3166 not in iso3166.countries_by_alpha3 + ) + assert not unresolvable, ( + f"{', '.join(unresolvable)} name a crop or country nothing resolves" + ) + + +def test_the_target_set_on_disk_is_well_formed() -> None: + """One named test for the real file, so a malformed set says so as a failure rather than as an + error raised out of whichever test happened to read it first.""" + targets.check_document(document=targets.read_document()) diff --git a/validation/capture.py b/validation/capture.py new file mode 100644 index 0000000..32cff60 --- /dev/null +++ b/validation/capture.py @@ -0,0 +1,364 @@ +"""Run the pipeline over the target set and write the artifact every comparison reads. + +The one stage that runs the pipeline itself, so it is run by hand rather than by +`python -m validation`, and its output is cached rather than recomputed per report. + +**One capture, both legs.** `efs.parquet` holds sLUC and jdLUC together, indexed on +`trace.CANONICAL_KEY` -- `(admin_level, admin_id, crop_name, methodology)` -- so the US +sLUC-versus-jdLUC head-to-head is a filter on `methodology` within one table rather than a join +between two files. A join there would invite exactly the alignment failure the comparison exists to +rule out. + +**Every crop, not just the targets.** The conservation bound compares a jurisdiction's forest pool +against the sum of `forest_emissions_mt` over *all* crops, so restricting the run to target crops +would shrink the numerator and let the bound pass by construction. It is also cheaper than it looks: +`statistical.get_downscaled_luc_emissions` is keyed on `(skip_glad_crop_filter, tile_id)` alone, so +the expensive per-tile layer is shared and a crop adds only its share arithmetic. + +**Merging is per (ISO, methodology), and rows keep their own `code_version`.** A capture scoped with +`--isos` leaves other countries in place, so the artifact can span versions and a reader has to be +able to see that -- `report.get_provenance_findings` raises a DEFECT when it does. That is why +provenance travels in the row rather than in a sidecar. + +Run it from the repo root, so `storage`'s cache keying resolves the module path: + + uv run python -m validation.capture --dry-run resolve the plan, touch nothing + uv run python -m validation.capture every target, both legs + uv run python -m validation.capture --isos BRA PRY those countries only, merged in + +`--dry-run` does not run the pipeline: it resolves the target set, the crop set and the +tile count and prints what a real run would compute. Use it to confirm the plan before spending a +capture, since the run itself is hours of compute against a warm cache and rather more against a +cold one. +""" + +import argparse +import collections.abc +import logging +import pathlib + +import pandas + +from jdluc import attribute, emit, geo, statistical, trace +from jdluc.datasets import worldbank_jurisdictions +from validation import pull, schema, targets + +logger = logging.getLogger(__name__) + +# jdLUC asserts `iso_3166 == "USA"` in its own workflow, since it needs the USDA CDL raster and NASS +# yields, both US-only. Naming it here keeps the reason with the filter. +JURISDICTIONAL_DIRECT_ISO_3166 = "USA" + + +def get_iso_3166s(isos: tuple[str, ...]) -> tuple[str, ...]: + """The countries to compute, defaulting to every one the target set names. + + An explicit `--isos` is checked against the target set rather than passed through, because a + typo would otherwise capture a country nothing compares and silently leave a target missing. + """ + wanted = tuple(sorted({target.iso_3166 for target in targets.iter_targets()})) + if not isos: + return wanted + unknown = sorted(set(isos) - set(wanted)) + assert not unknown, ( + f"{', '.join(unknown)} name no target in {targets.TARGETS}; capture computes the target " + f"set, and a country outside it has nothing to compare against" + ) + return tuple(sorted(set(isos))) + + +def iter_methodologies( + iso_3166s: tuple[str, ...], +) -> collections.abc.Iterator[tuple[attribute.Methodology, tuple[str, ...]]]: + """Each leg, with the countries it can run over. + + sLUC is global. jdLUC is US-only and is skipped rather than asserted when the USA is out of + scope, so `--isos BRA` is a valid capture rather than an error. + """ + yield attribute.Methodology.STATISTICAL, iso_3166s + if JURISDICTIONAL_DIRECT_ISO_3166 in iso_3166s: + yield ( + attribute.Methodology.JURISDICTIONAL_DIRECT, + (JURISDICTIONAL_DIRECT_ISO_3166,), + ) + + +def get_tile_ids(iso_3166s: tuple[str, ...]) -> tuple[str, ...] | None: + """Every ten-degree tile the capture touches, deduplicated, or None without the ingested layer. + + Worth reporting because tiles are the unit of cost: two countries sharing a tile pay for it + once, which is why tile overlap is a criterion when the target set is chosen. + + None rather than raising, because this reads the ingested admin-0 layer from `ingest_root` and + `--dry-run` has to work without credentials -- checking the plan before spending a capture is + most useful from a machine that cannot spend one. The rest of the plan is `targets.json` and + resolves offline. + """ + try: + return tuple( + sorted( + { + tile_id + for iso_3166 in iso_3166s + for tile_id in worldbank_jurisdictions.get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL, + ) + } + ) + ) + except Exception as error: + # Broad on purpose: the read goes through GDAL and pyogrio, which surface a missing + # credential as any of several driver errors, and none of them is worth a dependency here. + logger.warning(f"Cannot resolve tiles without the ingested layer: {error!r}") + return None + + +def read_efs() -> pandas.DataFrame | None: + """The existing artifact, or None on the first capture.""" + if not pull.EFS.exists(): + return None + return pandas.read_parquet(pull.EFS) + + +def merge_efs( + captured: pandas.DataFrame, existing: pandas.DataFrame | None +) -> pandas.DataFrame: + """Replace what was recomputed, keep everything else exactly as it was. + + Keyed on (ISO, methodology) rather than on the whole index: a country recomputed for one leg + must lose all of that leg's rows, including any crop that no longer produces one, or a stale row + would survive beside its replacements and be summed with them. + """ + if existing is None: + return captured + recomputed = { + (str(admin_id)[:3], str(methodology)) + for admin_id, methodology in zip( + captured.index.get_level_values("admin_id"), + captured.index.get_level_values("methodology"), + strict=True, + ) + } + keep = [ + (str(admin_id)[:3], str(methodology)) not in recomputed + for admin_id, methodology in zip( + existing.index.get_level_values("admin_id"), + existing.index.get_level_values("methodology"), + strict=True, + ) + ] + logger.info( + f"Keeping {sum(keep):,d} row(s) from the previous capture and replacing " + f"{len(keep) - sum(keep):,d}" + ) + return pandas.concat([existing[keep], captured]).sort_index() + + +def workflow(iso_3166s: tuple[str, ...], repo_root: pathlib.Path) -> pandas.DataFrame: + """Both legs over the given countries, stamped with the code that produced them.""" + frames = [] + for methodology, wanted in iter_methodologies(iso_3166s=iso_3166s): + crop_names = attribute.get_crop_names(methodology=methodology) + logger.info( + f"Capturing {methodology.name:s} over {len(wanted):d} country(ies) and " + f"{len(crop_names):d} crops" + ) + frames.append( + trace.workflow( + crop_names=crop_names, + iso_3166s=wanted, + methodology=methodology, + skip_glad_crop_filter=False, + ) + ) + captured = pandas.concat(frames) + captured["code_version"] = schema.get_code_version(repo_root=repo_root) + return captured + + +def get_forest_pool_tonnes(iso_3166: str) -> float: + """The forest conversion a country's 2020 cropland contains, with no crop share applied. + + The same GHGP-discounted sum each crop's `forest_emissions_mt` is drawn from, minus the + expansion share -- so it bounds what any allocation can hand out, and a country attributing more + than it has an attribution bug rather than a disagreement. + + Computed here rather than emitted by the pipeline, which carries no such column. A second pass + but not a recomputation: `get_downscaled_luc_emissions` is cached on + `(skip_glad_crop_filter, tile_id)` alone, so this reads back the bands the capture attributed + from, and the weights and pixel areas come from `emit` rather than being restated. + + Running in the same process at the same commit as the capture beside it is what makes the bound + freezable rather than indicative. + """ + from rioxarray.exceptions import NoDataInBounds + + tonnes = 0.0 + for tile_id in sorted( + worldbank_jurisdictions.get_ten_degree_tile_ids_for_admin_id( + admin_id=iso_3166, admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL + ) + ): + # Iterated rather than looked up, mirroring `statistical.workflow`: at the national grain + # this yields the one jurisdiction, and yields nothing where the tile meets the country's + # bounding box but not its geometry. + for ( + jurisdiction + ) in worldbank_jurisdictions.iter_jurisdiction_for_iso_3166_tile_id( + admin_level=worldbank_jurisdictions.AdminLevel.NATIONAL, + iso_3166=iso_3166, + tile_id=tile_id, + ): + logger.info(f"Summing the forest pool for {iso_3166=:s} {tile_id=:s}") + try: + clipped = geo.clip_dset( + dset=statistical.get_downscaled_luc_emissions( + skip_glad_crop_filter=False, tile_id=tile_id + ), + geometry=jurisdiction.geometry, + ) + except NoDataInBounds as error: + logger.warning(repr(error)) + continue + bands = { + (before, after): clipped[f"forest:tco2e-per-ha:{before:d}-{after:d}"] + for (before, after) in emit.SPAN_TO_LINEAR_DISCOUNT_WEIGHT + } + hectares = emit.get_hectares_per_pixel(darray=next(iter(bands.values()))) + tonnes += float( + emit.get_linear_discounted_total( + span_to_value={ + span: band * hectares for span, band in bands.items() + } + ) + .sum() + .compute() + ) + return tonnes + + +def get_forest_pools( + captured: pandas.DataFrame, iso_3166s: tuple[str, ...] +) -> pandas.DataFrame: + """Per country, the pool against the sum attributed to crops out of it. + + The shape `report.get_conservation_findings` reads. National rows only, and the statistical leg + only: jdLUC derives no unallocated pool, and summing the two legs would double-count the USA. + + The attributed figure is summed across every crop, which is the quantity the bound constrains -- + restricting it to target crops would shrink the numerator and let the bound pass by + construction. + """ + national = captured[ + (captured.index.get_level_values("admin_level") == schema.NATIONAL) + & (captured.index.get_level_values("methodology") == schema.STATISTICAL) + ] + assert len(national), ( + "the capture holds no national statistical rows, so the conservation bound has nothing to " + "read; a provincial-only artifact means the rollup did not run" + ) + attributed = national.groupby(level="admin_id")["forest_emissions_mt"].sum() + return pandas.DataFrame.from_records( + [ + { + "iso_3166": iso_3166, + "attributed_tonnes": float(attributed.loc[iso_3166]), + "pool_tonnes": get_forest_pool_tonnes(iso_3166=iso_3166), + } + for iso_3166 in iso_3166s + if iso_3166 in attributed.index + ] + ) + + +def render_plan(iso_3166s: tuple[str, ...]) -> str: + """What a real run would compute, resolved without running the pipeline.""" + armed = {target.slug for target in targets.iter_control_targets()} + tile_ids = get_tile_ids(iso_3166s=iso_3166s) + lines = [ + f"{len(iso_3166s):d} country(ies): {', '.join(iso_3166s)}", + f"{len(tile_ids):d} ten-degree tile(s), the unit of cost" + if tile_ids is not None + else "tile count unavailable: the ingested admin-0 layer has not been read", + ] + for methodology, wanted in iter_methodologies(iso_3166s=iso_3166s): + crop_names = attribute.get_crop_names(methodology=methodology) + lines.append( + f"{methodology.name:s}: {len(wanted):d} country(ies) x {len(crop_names):d} crops" + ) + covered = set(iso_3166s) + missing = [ + target.slug + for target in targets.iter_control_targets() + if target.iso_3166 not in covered + ] + lines.append(f"{len(armed):d} target(s) carry a control") + if missing: + # A control whose target was not computed removes its own guard, and does so silently. + lines.append( + f"**{len(missing):d} of them are out of scope and will keep whatever the previous " + f"capture left**: {', '.join(missing)}" + ) + return "\n".join(f" {line:s}" for line in lines) + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--isos", + nargs=argparse.ONE_OR_MORE, + default=(), + type=worldbank_jurisdictions.iso_3166_str, + help="capture only these countries and merge them into the existing artifact; " + "defaults to every country the target set names", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="resolve and print the plan without running the pipeline", + ) + parser.add_argument( + "--repo-root", + type=pathlib.Path, + default=pathlib.Path(__file__).resolve().parent.parent, + help="the jdluc checkout, read for code_version", + ) + args = parser.parse_args() + + iso_3166s = get_iso_3166s(isos=tuple(args.isos)) + print(render_plan(iso_3166s=iso_3166s)) + if args.dry_run: + return 0 + + captured = workflow(iso_3166s=iso_3166s, repo_root=args.repo_root) + merged = merge_efs(captured=captured, existing=read_efs()) + pull.CAPTURE.mkdir(parents=True, exist_ok=True) + merged.to_parquet(pull.EFS) + print(f"\nWrote {len(merged):,d} row(s) to {pull.EFS}") + + # Only for the countries this run recomputed: a pool beside emissions from a different run + # compares a numerator and a denominator built by different code. + pools = get_forest_pools(captured=merged, iso_3166s=iso_3166s) + pools["code_version"] = schema.get_code_version(repo_root=args.repo_root) + pools.to_parquet(pull.FOREST_POOLS) + over = pools[pools["attributed_tonnes"] > pools["pool_tonnes"]] + print( + f"Wrote {len(pools):d} forest pool(s) to {pull.FOREST_POOLS}; {len(over):d} jurisdiction(s) " + f"attribute more than the pool holds{':' if len(over) else '.'}" + ) + for row in over.to_dict("records"): + print( + f" {row['iso_3166']!s} {row['attributed_tonnes'] / row['pool_tonnes']:.1%}" + ) + # The exit code does not depend on the findings; see `__main__`. A conservation overrun is a + # result to be reported, and a capture that ran is a capture that succeeded. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/validation/data/admin_1_overrides.json b/validation/data/admin_1_overrides.json new file mode 100644 index 0000000..2b48476 --- /dev/null +++ b/validation/data/admin_1_overrides.json @@ -0,0 +1,54 @@ +{ + "KHM.11_1": { + "note": "Kracheh / Kratie", + "world_bank_id": "KHM011" + }, + "KHM.14_1": { + "note": "Mondol Kiri / Mondul Kiri", + "world_bank_id": "KHM012" + }, + "KHM.15_1": { + "note": "Otdar Mean Chey / Oddar Meanchey", + "world_bank_id": "KHM013" + }, + "KHM.17_1": { + "note": "Pouthisat / Pursat", + "world_bank_id": "KHM018" + }, + "KHM.20_1": { + "note": "Rotanokiri / Ratanak Kiri", + "world_bank_id": "KHM019" + }, + "KHM.21_1": { + "note": "Siemreab / Siemreap", + "world_bank_id": "KHM020" + }, + "KHM.22_1": { + "note": "Stoeng Treng / Stung Treng", + "world_bank_id": "KHM022" + }, + "KHM.24_1": { + "note": "Takev / Takeo", + "world_bank_id": "KHM024" + }, + "KHM.25_1": { + "note": "Tbong Khmum / Tboung Khmum", + "world_bank_id": "KHM025" + }, + "KHM.2_1": { + "note": "Batdambang / Battambang", + "world_bank_id": "KHM002" + }, + "KHM.5_1": { + "note": "Kampong Spoe / Kampong Speu", + "world_bank_id": "KHM005" + }, + "KHM.6_1": { + "note": "Kampong Thum / Kampong Thom", + "world_bank_id": "KHM006" + }, + "KHM.9_1": { + "note": "Kaoh Kong / Koh Kong", + "world_bank_id": "KHM010" + } +} diff --git a/validation/data/gadm_to_world_bank_admin_1.json b/validation/data/gadm_to_world_bank_admin_1.json new file mode 100644 index 0000000..e659dd2 --- /dev/null +++ b/validation/data/gadm_to_world_bank_admin_1.json @@ -0,0 +1,5987 @@ +{ + "area_coverage": { + "ABW": 0.0, + "AFG": 0.9704, + "AGO": 1.0, + "AIA": 0.0, + "ALA": 0.0, + "ALB": 0.4503, + "AND": 1.0, + "ARE": 0.9902, + "ARG": 0.9999, + "ARM": 0.8107, + "ASM": 0.0, + "ATF": 0.0, + "ATG": 0.3334, + "AUS": 1.0, + "AUT": 1.0, + "AZE": 0.0328, + "BDI": 0.8776, + "BEL": 0.0, + "BEN": 0.7994, + "BES": 1.0, + "BFA": 0.9065, + "BGD": 0.7012, + "BGR": 0.8628, + "BHR": 0.0854, + "BHS": 0.0, + "BIH": 0.0, + "BLM": 0.0, + "BLR": 0.806, + "BLZ": 1.0, + "BMU": 0.0, + "BOL": 1.0, + "BRA": 1.0, + "BRB": 0.1169, + "BRN": 1.0, + "BTN": 1.0, + "BVT": 0.0, + "BWA": 0.8075, + "CAF": 0.0, + "CAN": 1.0, + "CCK": 0.0, + "CHE": 0.9641, + "CHL": 0.7939, + "CHN": 0.5395, + "CIV": 1.0, + "CMR": 1.0, + "COD": 1.0, + "COG": 0.9981, + "COK": 0.0, + "COL": 1.0, + "COM": 0.0, + "CPV": 1.0, + "CRI": 1.0, + "CUB": 0.9635, + "CUW": 0.0, + "CXR": 0.0, + "CYM": 0.0, + "CYP": 0.0, + "CZE": 0.4658, + "DEU": 0.8549, + "DJI": 0.3016, + "DMA": 0.0, + "DNK": 0.0, + "DOM": 0.8805, + "DZA": 0.7371, + "ECU": 1.0, + "EGY": 0.2194, + "ERI": 0.5372, + "ESP": 0.9294, + "EST": 0.0, + "ETH": 0.8314, + "FIN": 1.0, + "FJI": 1.0, + "FRA": 0.2052, + "FRO": 0.0, + "FSM": 0.0, + "GAB": 0.8567, + "GBR": 0.3812, + "GEO": 0.8339, + "GGY": 0.9088, + "GHA": 0.9466, + "GIB": 0.0, + "GIN": 1.0, + "GLP": 0.0, + "GMB": 0.5271, + "GNB": 0.9973, + "GNQ": 1.0, + "GRC": 0.0, + "GRD": 0.0, + "GRL": 0.0, + "GTM": 0.9804, + "GUF": 0.0, + "GUM": 0.0, + "GUY": 0.0, + "HKG": 0.0, + "HMD": 0.0, + "HND": 0.9439, + "HRV": 0.0123, + "HTI": 0.8216, + "HUN": 1.0, + "IDN": 0.848, + "IMN": 0.7638, + "IND": 0.9972, + "IRL": 1.0, + "IRN": 0.606, + "IRQ": 0.4321, + "ISL": 0.0, + "ISR": 0.077, + "ITA": 0.8144, + "JAM": 1.0, + "JEY": 0.8987, + "JOR": 0.9699, + "JPN": 0.4451, + "KAZ": 0.0, + "KEN": 1.0, + "KGZ": 0.8532, + "KHM": 0.9859, + "KIR": 0.0, + "KNA": 1.0, + "KOR": 0.0058, + "KWT": 1.0, + "LAO": 0.9296, + "LBN": 0.6139, + "LBR": 0.8972, + "LBY": 0.5344, + "LCA": 0.0, + "LIE": 0.8859, + "LKA": 0.0, + "LSO": 1.0, + "LTU": 0.8007, + "LUX": 1.0, + "LVA": 0.0045, + "MAC": 0.0, + "MAF": 0.0, + "MAR": 0.0, + "MCO": 0.0, + "MDA": 0.9454, + "MDG": 0.7214, + "MDV": 0.0, + "MEX": 0.8549, + "MHL": 0.0, + "MKD": 0.0, + "MLI": 0.6017, + "MLT": 0.0, + "MMR": 0.9893, + "MNE": 0.9692, + "MNG": 0.4581, + "MNP": 0.0, + "MOZ": 0.8051, + "MRT": 0.5669, + "MSR": 0.0, + "MTQ": 0.0, + "MUS": 0.9297, + "MWI": 0.0, + "MYS": 0.96, + "MYT": 0.0, + "NAM": 0.9825, + "NCL": 0.0, + "NER": 0.9229, + "NFK": 0.0, + "NGA": 0.9631, + "NIC": 0.5371, + "NIU": 0.0, + "NLD": 0.9005, + "NOR": 0.8119, + "NPL": 0.0, + "NRU": 0.0, + "NZL": 0.9969, + "OMN": 0.7569, + "PAK": 0.9989, + "PAN": 0.9065, + "PCN": 0.0, + "PER": 0.9729, + "PHL": 0.0, + "PLW": 0.0, + "PNG": 0.951, + "POL": 0.811, + "PRI": 1.0, + "PRK": 0.854, + "PRT": 1.0, + "PRY": 1.0, + "PSE": 0.0118, + "PYF": 0.0, + "QAT": 0.2086, + "REU": 0.0, + "ROU": 0.9753, + "RUS": 0.0001, + "RWA": 1.0, + "SAU": 0.9026, + "SDN": 0.4125, + "SEN": 1.0, + "SGP": 0.077, + "SHN": 1.0, + "SJM": 0.0, + "SLB": 0.0, + "SLE": 0.8097, + "SLV": 1.0, + "SMR": 0.0, + "SOM": 0.6922, + "SPM": 1.0, + "SRB": 1.0, + "SSD": 0.403, + "STP": 1.0, + "SUR": 1.0, + "SVK": 0.0, + "SVN": 0.8576, + "SWE": 0.0, + "SWZ": 1.0, + "SXM": 0.0, + "SYC": 0.3146, + "SYR": 0.9275, + "TCA": 0.7489, + "TCD": 1.0, + "TGO": 0.7634, + "THA": 0.9454, + "TJK": 0.3542, + "TKL": 0.0, + "TKM": 0.6367, + "TLS": 0.9455, + "TON": 0.0, + "TTO": 1.0, + "TUN": 0.9285, + "TUR": 0.9499, + "TUV": 0.0, + "TZA": 1.0, + "UGA": 0.5145, + "UKR": 0.0433, + "UMI": 0.0, + "URY": 1.0, + "USA": 1.0, + "UZB": 0.2711, + "VAT": 0.0, + "VCT": 1.0, + "VEN": 1.0, + "VGB": 1.0, + "VIR": 0.0, + "VNM": 0.8119, + "VUT": 1.0, + "WLF": 0.0, + "WSM": 0.0, + "XKX": 0.0, + "YEM": 0.9645, + "ZAF": 1.0, + "ZMB": 0.885, + "ZWE": 0.9929 + }, + "gadm_source": { + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "url": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/gadm_admin_keys/key_gadm_adm1.csv" + }, + "matched": { + "AFG.10_1": "AFG010", + "AFG.11_1": "AFG011", + "AFG.12_1": "AFG012", + "AFG.13_1": "AFG013", + "AFG.14_1": "AFG014", + "AFG.15_1": "AFG015", + "AFG.16_1": "AFG016", + "AFG.17_1": "AFG017", + "AFG.18_1": "AFG018", + "AFG.19_1": "AFG019", + "AFG.1_1": "AFG001", + "AFG.20_1": "AFG020", + "AFG.21_1": "AFG021", + "AFG.22_1": "AFG022", + "AFG.23_1": "AFG023", + "AFG.24_1": "AFG024", + "AFG.25_1": "AFG025", + "AFG.26_1": "AFG026", + "AFG.28_1": "AFG028", + "AFG.29_1": "AFG029", + "AFG.2_1": "AFG002", + "AFG.31_1": "AFG031", + "AFG.32_1": "AFG032", + "AFG.33_1": "AFG033", + "AFG.34_1": "AFG034", + "AFG.3_1": "AFG003", + "AFG.4_1": "AFG004", + "AFG.5_1": "AFG005", + "AFG.6_1": "AFG006", + "AFG.7_1": "AFG007", + "AFG.8_1": "AFG008", + "AFG.9_1": "AFG009", + "AGO.10_1": "AGO010", + "AGO.11_1": "AGO011", + "AGO.12_1": "AGO012", + "AGO.13_1": "AGO013", + "AGO.14_1": "AGO014", + "AGO.15_1": "AGO015", + "AGO.16_1": "AGO016", + "AGO.17_1": "AGO017", + "AGO.18_1": "AGO018", + "AGO.1_1": "AGO001", + "AGO.2_1": "AGO002", + "AGO.3_1": "AGO003", + "AGO.4_1": "AGO004", + "AGO.5_1": "AGO005", + "AGO.6_1": "AGO006", + "AGO.7_1": "AGO007", + "AGO.8_1": "AGO008", + "AGO.9_1": "AGO009", + "ALB.10_1": "ALB031", + "ALB.11_1": "ALB034", + "ALB.12_1": "ALB036", + "ALB.1_1": "ALB001", + "ALB.2_1": "ALB005", + "ALB.3_1": "ALB006", + "ALB.4_1": "ALB007", + "ALB.5_1": "ALB008", + "ALB.6_1": "ALB009", + "ALB.7_1": "ALB014", + "ALB.8_1": "ALB017", + "ALB.9_1": "ALB019", + "AND.1_1": "AND001", + "AND.2_1": "AND002", + "AND.3_1": "AND003", + "AND.4_1": "AND004", + "AND.5_1": "AND005", + "AND.6_1": "AND006", + "AND.7_1": "AND007", + "ARE.1_1": "ARE001", + "ARE.2_1": "ARE002", + "ARE.3_1": "ARE003", + "ARE.4_1": "ARE004", + "ARE.5_1": "ARE005", + "ARE.6_1": "ARE006", + "ARG.10_1": "ARG010", + "ARG.11_1": "ARG011", + "ARG.12_1": "ARG012", + "ARG.13_1": "ARG013", + "ARG.14_1": "ARG014", + "ARG.15_1": "ARG015", + "ARG.16_1": "ARG016", + "ARG.17_1": "ARG017", + "ARG.18_1": "ARG018", + "ARG.19_1": "ARG019", + "ARG.1_1": "ARG001", + "ARG.20_1": "ARG020", + "ARG.21_1": "ARG021", + "ARG.22_1": "ARG022", + "ARG.23_1": "ARG023", + "ARG.24_1": "ARG024", + "ARG.2_1": "ARG003", + "ARG.3_1": "ARG004", + "ARG.4_1": "ARG005", + "ARG.6_1": "ARG006", + "ARG.7_1": "ARG007", + "ARG.8_1": "ARG008", + "ARG.9_1": "ARG009", + "ARM.10_1": "ARM009", + "ARM.11_1": "ARM010", + "ARM.1_1": "ARM001", + "ARM.2_1": "ARM002", + "ARM.3_1": "ARM003", + "ARM.6_1": "ARM005", + "ARM.7_1": "ARM006", + "ARM.8_1": "ARM007", + "ARM.9_1": "ARM008", + "ATG.1_1": "ATG001", + "ATG.2_1": "ATG002", + "AUS.10_1": "AUS010", + "AUS.11_1": "AUS011", + "AUS.2_1": "AUS002", + "AUS.3_1": "AUS003", + "AUS.5_1": "AUS004", + "AUS.6_1": "AUS005", + "AUS.7_1": "AUS007", + "AUS.8_1": "AUS008", + "AUS.9_1": "AUS009", + "AUT.1_1": "AUT001", + "AUT.2_1": "AUT002", + "AUT.3_1": "AUT003", + "AUT.4_1": "AUT004", + "AUT.5_1": "AUT005", + "AUT.6_1": "AUT006", + "AUT.7_1": "AUT007", + "AUT.8_1": "AUT008", + "AUT.9_1": "AUT009", + "AZE.1_1": "AZE001", + "AZE.6_1": "AZE043", + "AZE.7_1": "AZE048", + "BDI.10_1": "BDI010", + "BDI.11_1": "BDI011", + "BDI.12_1": "BDI012", + "BDI.13_1": "BDI013", + "BDI.14_1": "BDI014", + "BDI.15_1": "BDI015", + "BDI.16_1": "BDI016", + "BDI.17_1": "BDI017", + "BDI.1_1": "BDI001", + "BDI.4_1": "BDI019", + "BDI.5_1": "BDI005", + "BDI.6_1": "BDI006", + "BDI.7_1": "BDI007", + "BDI.8_1": "BDI008", + "BDI.9_1": "BDI009", + "BEN.10_1": "BEN010", + "BEN.11_1": "BEN011", + "BEN.12_1": "BEN012", + "BEN.1_1": "BEN001", + "BEN.3_1": "BEN003", + "BEN.4_1": "BEN004", + "BEN.5_1": "BEN005", + "BEN.6_1": "BEN007", + "BEN.8_1": "BEN008", + "BEN.9_1": "BEN009", + "BES.1_1": "BES001", + "BES.2_1": "BES002", + "BES.3_1": "BES003", + "BFA.10_1": "BFA010", + "BFA.11_1": "BFA011", + "BFA.12_1": "BFA012", + "BFA.13_1": "BFA013", + "BFA.1_1": "BFA001", + "BFA.2_1": "BFA002", + "BFA.3_1": "BFA004", + "BFA.4_1": "BFA005", + "BFA.5_1": "BFA006", + "BFA.6_1": "BFA007", + "BFA.7_1": "BFA003", + "BFA.8_1": "BFA008", + "BGD.3_1": "BGD003", + "BGD.4_1": "BGD004", + "BGD.5_1": "BGD006", + "BGD.6_1": "BGD007", + "BGD.7_1": "BGD008", + "BGD.8_1": "BGD005", + "BGR.10_1": "BGR010", + "BGR.11_1": "BGR011", + "BGR.12_1": "BGR012", + "BGR.13_1": "BGR013", + "BGR.14_1": "BGR014", + "BGR.15_1": "BGR015", + "BGR.16_1": "BGR016", + "BGR.17_1": "BGR017", + "BGR.18_1": "BGR018", + "BGR.19_1": "BGR019", + "BGR.1_1": "BGR001", + "BGR.21_1": "BGR021", + "BGR.22_1": "BGR023", + "BGR.23_1": "BGR024", + "BGR.24_1": "BGR025", + "BGR.25_1": "BGR026", + "BGR.26_1": "BGR027", + "BGR.2_1": "BGR002", + "BGR.3_1": "BGR003", + "BGR.4_1": "BGR004", + "BGR.6_1": "BGR005", + "BGR.7_1": "BGR007", + "BGR.9_1": "BGR009", + "BHR.3_1": "BHR008", + "BHR.4_1": "BHR009", + "BLR.1_1": "BLR001", + "BLR.2_1": "BLR002", + "BLR.3_1": "BLR003", + "BLR.4_1": "BLR006", + "BLR.6_1": "BLR007", + "BLZ.1_1": "BLZ001", + "BLZ.2_1": "BLZ002", + "BLZ.3_1": "BLZ003", + "BLZ.4_1": "BLZ004", + "BLZ.5_1": "BLZ005", + "BLZ.6_1": "BLZ006", + "BOL.1_1": "BOL002", + "BOL.2_1": "BOL003", + "BOL.3_1": "BOL001", + "BOL.4_1": "BOL004", + "BOL.5_1": "BOL005", + "BOL.6_1": "BOL006", + "BOL.7_1": "BOL007", + "BOL.8_1": "BOL008", + "BOL.9_1": "BOL009", + "BRA.10_1": "BRA010", + "BRA.11_1": "BRA012", + "BRA.12_1": "BRA011", + "BRA.13_1": "BRA013", + "BRA.14_1": "BRA018", + "BRA.15_1": "BRA019", + "BRA.16_1": "BRA020", + "BRA.17_1": "BRA021", + "BRA.18_1": "BRA022", + "BRA.19_1": "BRA023", + "BRA.1_1": "BRA001", + "BRA.20_1": "BRA024", + "BRA.21_1": "BRA025", + "BRA.22_1": "BRA026", + "BRA.23_1": "BRA027", + "BRA.24_1": "BRA028", + "BRA.25_1": "BRA029", + "BRA.26_1": "BRA030", + "BRA.27_1": "BRA031", + "BRA.2_1": "BRA002", + "BRA.3_1": "BRA003", + "BRA.4_1": "BRA004", + "BRA.5_1": "BRA005", + "BRA.6_1": "BRA006", + "BRA.7_1": "BRA007", + "BRA.8_1": "BRA008", + "BRA.9_1": "BRA009", + "BRB.1_1": "BRB001", + "BRN.1_1": "BRN001", + "BRN.2_1": "BRN002", + "BRN.3_1": "BRN003", + "BRN.4_1": "BRN004", + "BTN.10_1": "BTN010", + "BTN.11_1": "BTN011", + "BTN.12_1": "BTN012", + "BTN.13_1": "BTN013", + "BTN.14_1": "BTN014", + "BTN.15_1": "BTN015", + "BTN.16_1": "BTN017", + "BTN.17_1": "BTN018", + "BTN.18_1": "BTN019", + "BTN.19_1": "BTN016", + "BTN.1_1": "BTN001", + "BTN.20_1": "BTN020", + "BTN.2_1": "BTN002", + "BTN.3_1": "BTN003", + "BTN.4_1": "BTN004", + "BTN.5_1": "BTN005", + "BTN.6_1": "BTN006", + "BTN.7_1": "BTN007", + "BTN.8_1": "BTN008", + "BTN.9_1": "BTN009", + "BWA.11_1": "BWA008", + "BWA.14_1": "BWA009", + "BWA.15_1": "BWA010", + "BWA.1_1": "BWA001", + "BWA.2_1": "BWA002", + "BWA.5_1": "BWA003", + "BWA.7_1": "BWA004", + "BWA.8_1": "BWA005", + "BWA.9_1": "BWA006", + "CAN.10_1": "CAN010", + "CAN.11_1": "CAN011", + "CAN.12_1": "CAN012", + "CAN.13_1": "CAN013", + "CAN.1_1": "CAN001", + "CAN.2_1": "CAN002", + "CAN.3_1": "CAN003", + "CAN.4_1": "CAN004", + "CAN.5_1": "CAN005", + "CAN.6_1": "CAN006", + "CAN.7_1": "CAN007", + "CAN.8_1": "CAN008", + "CAN.9_1": "CAN009", + "CHE.10_1": "CHE010", + "CHE.11_1": "CHE011", + "CHE.13_1": "CHE013", + "CHE.14_1": "CHE014", + "CHE.15_1": "CHE015", + "CHE.16_1": "CHE016", + "CHE.17_1": "CHE017", + "CHE.18_1": "CHE018", + "CHE.19_1": "CHE019", + "CHE.1_1": "CHE001", + "CHE.20_1": "CHE020", + "CHE.21_1": "CHE021", + "CHE.22_1": "CHE022", + "CHE.23_1": "CHE023", + "CHE.24_1": "CHE024", + "CHE.25_1": "CHE025", + "CHE.26_1": "CHE026", + "CHE.2_1": "CHE002", + "CHE.3_1": "CHE003", + "CHE.4_1": "CHE004", + "CHE.5_1": "CHE005", + "CHE.6_1": "CHE006", + "CHE.7_1": "CHE007", + "CHE.8_1": "CHE008", + "CHE.9_1": "CHE009", + "CHL.10_1": "CHL010", + "CHL.11_1": "CHL011", + "CHL.12_1": "CHL012", + "CHL.15_1": "CHL014", + "CHL.16_1": "CHL015", + "CHL.2_1": "CHL002", + "CHL.3_1": "CHL003", + "CHL.5_1": "CHL005", + "CHL.6_1": "CHL006", + "CHL.7_1": "CHL007", + "CHL.9_1": "CHL009", + "CHN.10_1": "CHN010", + "CHN.11_1": "CHN011", + "CHN.12_1": "CHN012", + "CHN.13_1": "CHN013", + "CHN.14_1": "CHN014", + "CHN.15_1": "CHN015", + "CHN.16_1": "CHN016", + "CHN.17_1": "CHN017", + "CHN.18_1": "CHN018", + "CHN.1_1": "CHN001", + "CHN.21_1": "CHN021", + "CHN.22_1": "CHN022", + "CHN.23_1": "CHN023", + "CHN.25_1": "CHN025", + "CHN.26_1": "CHN026", + "CHN.30_1": "CHN030", + "CHN.31_1": "CHN031", + "CHN.4_1": "CHN004", + "CHN.5_1": "CHN005", + "CHN.6_1": "CHN006", + "CHN.8_1": "CHN008", + "CHN.9_1": "CHN009", + "CIV.10_1": "CIV033", + "CIV.11_1": "CIV034", + "CIV.12_1": "CIV035", + "CIV.13_1": "CIV036", + "CIV.14_1": "CIV037", + "CIV.1_1": "CIV024", + "CIV.2_1": "CIV025", + "CIV.3_1": "CIV026", + "CIV.4_1": "CIV027", + "CIV.5_1": "CIV028", + "CIV.6_1": "CIV029", + "CIV.7_1": "CIV030", + "CIV.8_1": "CIV031", + "CIV.9_1": "CIV032", + "CMR.10_1": "CMR009", + "CMR.1_1": "CMR001", + "CMR.2_1": "CMR002", + "CMR.3_1": "CMR003", + "CMR.4_1": "CMR004", + "CMR.5_1": "CMR005", + "CMR.6_1": "CMR007", + "CMR.7_1": "CMR006", + "CMR.8_1": "CMR008", + "CMR.9_1": "CMR010", + "COD.10_1": "COD007", + "COD.11_1": "COD002", + "COD.12_1": "COD021", + "COD.13_1": "COD022", + "COD.14_1": "COD023", + "COD.15_1": "COD024", + "COD.16_1": "COD025", + "COD.17_1": "COD008", + "COD.18_1": "COD026", + "COD.19_1": "COD009", + "COD.1_1": "COD012", + "COD.20_1": "COD027", + "COD.21_1": "COD028", + "COD.22_1": "COD011", + "COD.23_1": "COD029", + "COD.24_1": "COD030", + "COD.25_1": "COD031", + "COD.26_1": "COD032", + "COD.2_1": "COD013", + "COD.3_1": "COD014", + "COD.4_1": "COD015", + "COD.5_1": "COD016", + "COD.6_1": "COD017", + "COD.7_1": "COD019", + "COD.8_1": "COD020", + "COD.9_1": "COD018", + "COG.11_1": "COG011", + "COG.12_1": "COG012", + "COG.1_1": "COG001", + "COG.2_1": "COG002", + "COG.3_1": "COG004", + "COG.4_1": "COG003", + "COG.5_1": "COG005", + "COG.6_1": "COG006", + "COG.7_1": "COG007", + "COG.8_1": "COG008", + "COG.9_1": "COG009", + "COL.10_2": "COL011", + "COL.11_2": "COL012", + "COL.12_2": "COL013", + "COL.13_2": "COL014", + "COL.14_2": "COL015", + "COL.15_2": "COL016", + "COL.16_2": "COL017", + "COL.17_2": "COL018", + "COL.18_2": "COL019", + "COL.19_2": "COL020", + "COL.1_2": "COL001", + "COL.20_2": "COL021", + "COL.21_2": "COL022", + "COL.22_2": "COL023", + "COL.23_2": "COL024", + "COL.24_2": "COL025", + "COL.25_2": "COL026", + "COL.26_2": "COL027", + "COL.28_2": "COL028", + "COL.29_2": "COL029", + "COL.2_2": "COL002", + "COL.30_2": "COL030", + "COL.31_2": "COL031", + "COL.32_2": "COL032", + "COL.33_2": "COL033", + "COL.3_2": "COL003", + "COL.4_2": "COL005", + "COL.5_2": "COL006", + "COL.6_2": "COL007", + "COL.7_2": "COL008", + "COL.8_2": "COL009", + "COL.9_2": "COL010", + "CPV.10_1": "CPV020", + "CPV.11_1": "CPV009", + "CPV.12_1": "CPV021", + "CPV.13_1": "CPV022", + "CPV.14_1": "CPV023", + "CPV.15_1": "CPV012", + "CPV.16_1": "CPV024", + "CPV.17_1": "CPV025", + "CPV.18_1": "CPV014", + "CPV.19_1": "CPV026", + "CPV.1_1": "CPV001", + "CPV.20_1": "CPV016", + "CPV.21_1": "CPV017", + "CPV.22_1": "CPV027", + "CPV.2_1": "CPV002", + "CPV.3_1": "CPV003", + "CPV.4_1": "CPV004", + "CPV.5_1": "CPV005", + "CPV.6_1": "CPV006", + "CPV.7_1": "CPV018", + "CPV.8_1": "CPV019", + "CPV.9_1": "CPV008", + "CRI.1_1": "CRI001", + "CRI.2_1": "CRI002", + "CRI.3_1": "CRI003", + "CRI.4_1": "CRI004", + "CRI.5_1": "CRI005", + "CRI.6_1": "CRI006", + "CRI.7_1": "CRI007", + "CUB.10_1": "CUB009", + "CUB.11_1": "CUB010", + "CUB.12_1": "CUB011", + "CUB.13_1": "CUB012", + "CUB.14_1": "CUB013", + "CUB.15_1": "CUB014", + "CUB.16_1": "CUB015", + "CUB.1_1": "CUB002", + "CUB.2_1": "CUB003", + "CUB.3_1": "CUB004", + "CUB.5_1": "CUB005", + "CUB.6_1": "CUB006", + "CUB.7_1": "CUB007", + "CUB.8_1": "CUB000", + "CUB.9_1": "CUB008", + "CZE.12_1": "CZE006", + "CZE.1_1": "CZE001", + "CZE.2_1": "CZE002", + "DEU.10_1": "DEU010", + "DEU.11_1": "DEU011", + "DEU.12_1": "DEU012", + "DEU.13_1": "DEU014", + "DEU.14_1": "DEU013", + "DEU.15_1": "DEU015", + "DEU.2_1": "DEU002", + "DEU.3_1": "DEU003", + "DEU.4_1": "DEU004", + "DEU.5_1": "DEU005", + "DEU.6_1": "DEU006", + "DEU.7_1": "DEU007", + "DEU.8_1": "DEU008", + "DEU.9_1": "DEU009", + "DJI.4_1": "DJI004", + "DJI.6_1": "DJI006", + "DOM.10_1": "DOM011", + "DOM.11_1": "DOM012", + "DOM.13_1": "DOM013", + "DOM.14_1": "DOM014", + "DOM.16_1": "DOM016", + "DOM.17_1": "DOM017", + "DOM.18_1": "DOM018", + "DOM.19_1": "DOM019", + "DOM.1_1": "DOM001", + "DOM.20_1": "DOM020", + "DOM.21_1": "DOM021", + "DOM.22_1": "DOM022", + "DOM.23_1": "DOM023", + "DOM.24_1": "DOM024", + "DOM.25_1": "DOM025", + "DOM.26_1": "DOM026", + "DOM.27_1": "DOM027", + "DOM.28_1": "DOM028", + "DOM.29_1": "DOM030", + "DOM.30_1": "DOM029", + "DOM.31_1": "DOM031", + "DOM.32_1": "DOM032", + "DOM.3_1": "DOM003", + "DOM.4_1": "DOM004", + "DOM.5_1": "DOM005", + "DOM.6_1": "DOM006", + "DOM.8_1": "DOM009", + "DOM.9_1": "DOM010", + "DZA.10_1": "DZA010", + "DZA.12_1": "DZA012", + "DZA.13_1": "DZA013", + "DZA.14_1": "DZA014", + "DZA.15_1": "DZA015", + "DZA.16_1": "DZA016", + "DZA.17_1": "DZA017", + "DZA.18_1": "DZA018", + "DZA.19_1": "DZA019", + "DZA.1_1": "DZA001", + "DZA.20_1": "DZA020", + "DZA.21_1": "DZA021", + "DZA.22_1": "DZA022", + "DZA.23_1": "DZA023", + "DZA.24_1": "DZA024", + "DZA.25_1": "DZA025", + "DZA.26_1": "DZA026", + "DZA.27_1": "DZA027", + "DZA.28_1": "DZA028", + "DZA.29_1": "DZA029", + "DZA.2_1": "DZA002", + "DZA.30_1": "DZA030", + "DZA.31_1": "DZA031", + "DZA.32_1": "DZA032", + "DZA.33_1": "DZA033", + "DZA.34_1": "DZA034", + "DZA.35_1": "DZA035", + "DZA.36_1": "DZA036", + "DZA.37_1": "DZA037", + "DZA.38_1": "DZA038", + "DZA.39_1": "DZA039", + "DZA.3_1": "DZA003", + "DZA.40_1": "DZA040", + "DZA.42_1": "DZA042", + "DZA.43_1": "DZA043", + "DZA.44_1": "DZA044", + "DZA.45_1": "DZA045", + "DZA.46_1": "DZA046", + "DZA.47_1": "DZA047", + "DZA.48_1": "DZA048", + "DZA.4_1": "DZA004", + "DZA.5_1": "DZA005", + "DZA.6_1": "DZA006", + "DZA.7_1": "DZA007", + "DZA.8_1": "DZA008", + "DZA.9_1": "DZA009", + "ECU.10_1": "ECU009", + "ECU.11_1": "ECU010", + "ECU.12_1": "ECU011", + "ECU.13_1": "ECU012", + "ECU.14_1": "ECU013", + "ECU.15_1": "ECU014", + "ECU.16_1": "ECU015", + "ECU.17_1": "ECU022", + "ECU.18_1": "ECU016", + "ECU.19_1": "ECU017", + "ECU.1_1": "ECU001", + "ECU.20_1": "ECU024", + "ECU.21_1": "ECU023", + "ECU.22_1": "ECU021", + "ECU.23_1": "ECU018", + "ECU.24_1": "ECU019", + "ECU.2_1": "ECU002", + "ECU.3_1": "ECU003", + "ECU.4_1": "ECU004", + "ECU.5_1": "ECU006", + "ECU.6_1": "ECU005", + "ECU.7_1": "ECU007", + "ECU.8_1": "ECU008", + "ECU.9_1": "ECU020", + "EGY.17_1": "EGY003", + "EGY.24_1": "EGY016", + "ERI.1_1": "ERI006", + "ERI.4_1": "ERI005", + "ERI.5_1": "ERI003", + "ERI.6_1": "ERI001", + "ESP.11_1": "ESP014", + "ESP.12_1": "ESP015", + "ESP.15_1": "ESP017", + "ESP.16_1": "ESP018", + "ESP.17_1": "ESP019", + "ESP.18_1": "ESP020", + "ESP.1_1": "ESP002", + "ESP.2_1": "ESP003", + "ESP.3_1": "ESP005", + "ESP.4_1": "ESP007", + "ESP.5_1": "ESP006", + "ESP.6_1": "ESP008", + "ESP.8_1": "ESP011", + "ESP.9_1": "ESP012", + "ETH.11_1": "ETH013", + "ETH.2_1": "ETH002", + "ETH.3_1": "ETH003", + "ETH.5_1": "ETH005", + "ETH.8_1": "ETH008", + "ETH.9_1": "ETH011", + "FIN.1_1": "FIN001", + "FIN.2_1": "FIN002", + "FIN.3_1": "FIN003", + "FIN.4_1": "FIN004", + "FIN.5_1": "FIN005", + "FJI.1_1": "FJI001", + "FJI.2_1": "FJI002", + "FJI.3_1": "FJI003", + "FJI.5_1": "FJI004", + "FRA.12_1": "FRA019", + "FRA.13_1": "FRA022", + "FRA.3_1": "FRA007", + "FRA.5_1": "FRA010", + "FRA.8_1": "FRA013", + "GAB.1_1": "GAB001", + "GAB.2_1": "GAB002", + "GAB.3_1": "GAB003", + "GAB.4_1": "GAB004", + "GAB.5_1": "GAB005", + "GAB.6_1": "GAB006", + "GAB.7_1": "GAB007", + "GAB.8_1": "GAB008", + "GBR.2_1": "GBR003", + "GBR.3_1": "GBR004", + "GEO.10_1": "GEO010", + "GEO.11_1": "GEO011", + "GEO.12_1": "GEO012", + "GEO.3_1": "GEO003", + "GEO.4_1": "GEO004", + "GEO.5_1": "GEO005", + "GEO.6_1": "GEO006", + "GEO.7_1": "GEO007", + "GEO.8_1": "GEO008", + "GEO.9_1": "GEO009", + "GGY.11_1": "GGY009", + "GGY.12_1": "GGY010", + "GGY.13_1": "GGY011", + "GGY.14_1": "GGY012", + "GGY.15_1": "GGY013", + "GGY.1_1": "GGY001", + "GGY.3_1": "GGY002", + "GGY.4_1": "GGY003", + "GGY.7_1": "GGY005", + "GGY.8_1": "GGY006", + "GGY.9_1": "GGY008", + "GHA10_2": "GHA016", + "GHA11_2": "GHA017", + "GHA12_2": "GHA007", + "GHA13_2": "GHA008", + "GHA14_2": "GHA009", + "GHA15_2": "GHA010", + "GHA16_2": "GHA018", + "GHA1_2": "GHA011", + "GHA2_2": "GHA001", + "GHA3_2": "GHA012", + "GHA4_2": "GHA013", + "GHA5_2": "GHA003", + "GHA6_2": "GHA004", + "GHA9_2": "GHA006", + "GIN.1_1": "GIN001", + "GIN.2_1": "GIN002", + "GIN.3_1": "GIN003", + "GIN.4_1": "GIN004", + "GIN.5_1": "GIN005", + "GIN.6_1": "GIN006", + "GIN.7_1": "GIN007", + "GIN.8_1": "GIN008", + "GMB.2_1": "GMB005", + "GMB.4_1": "GMB006", + "GMB.5_1": "GMB007", + "GNB.1_1": "GNB001", + "GNB.2_1": "GNB002", + "GNB.4_1": "GNB003", + "GNB.5_1": "GNB004", + "GNB.6_1": "GNB005", + "GNB.7_1": "GNB006", + "GNB.8_1": "GNB007", + "GNB.9_1": "GNB009", + "GNQ.1_1": "GNQ001", + "GNQ.2_1": "GNQ002", + "GNQ.3_1": "GNQ003", + "GNQ.4_1": "GNQ004", + "GNQ.5_1": "GNQ005", + "GNQ.6_1": "GNQ006", + "GNQ.7_1": "GNQ007", + "GTM.10_1": "GTM010", + "GTM.11_1": "GTM011", + "GTM.12_1": "GTM012", + "GTM.14_1": "GTM014", + "GTM.15_1": "GTM015", + "GTM.16_1": "GTM016", + "GTM.17_1": "GTM017", + "GTM.18_1": "GTM018", + "GTM.19_1": "GTM019", + "GTM.1_1": "GTM001", + "GTM.20_1": "GTM020", + "GTM.21_1": "GTM021", + "GTM.22_1": "GTM022", + "GTM.2_1": "GTM002", + "GTM.3_1": "GTM003", + "GTM.4_1": "GTM004", + "GTM.5_1": "GTM005", + "GTM.6_1": "GTM006", + "GTM.7_1": "GTM007", + "GTM.8_1": "GTM008", + "GTM.9_1": "GTM009", + "HND.10_1": "HND009", + "HND.12_1": "HND011", + "HND.13_1": "HND012", + "HND.14_1": "HND014", + "HND.15_1": "HND015", + "HND.16_1": "HND017", + "HND.17_1": "HND018", + "HND.18_1": "HND019", + "HND.1_1": "HND001", + "HND.2_1": "HND002", + "HND.3_1": "HND003", + "HND.4_1": "HND004", + "HND.5_1": "HND005", + "HND.6_1": "HND006", + "HND.8_1": "HND007", + "HND.9_1": "HND008", + "HRV.4_1": "HRV003", + "HTI.10_1": "HTI009", + "HTI.1_1": "HTI002", + "HTI.2_1": "HTI003", + "HTI.4_1": "HTI004", + "HTI.5_1": "HTI006", + "HTI.6_1": "HTI007", + "HTI.7_1": "HTI005", + "HTI.8_1": "HTI008", + "HTI.9_1": "HTI010", + "HUN.10_1": "HUN010", + "HUN.11_1": "HUN011", + "HUN.12_1": "HUN012", + "HUN.13_1": "HUN013", + "HUN.14_1": "HUN014", + "HUN.15_1": "HUN015", + "HUN.16_1": "HUN016", + "HUN.17_1": "HUN017", + "HUN.18_1": "HUN018", + "HUN.19_1": "HUN019", + "HUN.1_1": "HUN001", + "HUN.20_1": "HUN020", + "HUN.2_1": "HUN002", + "HUN.3_1": "HUN003", + "HUN.4_1": "HUN004", + "HUN.5_1": "HUN005", + "HUN.6_1": "HUN006", + "HUN.7_1": "HUN007", + "HUN.8_1": "HUN008", + "HUN.9_1": "HUN009", + "IDN.10_1": "IDN013", + "IDN.11_1": "IDN015", + "IDN.12_1": "IDN020", + "IDN.13_1": "IDN022", + "IDN.14_1": "IDN021", + "IDN.16_1": "IDN010", + "IDN.17_1": "IDN008", + "IDN.18_1": "IDN032", + "IDN.19_1": "IDN031", + "IDN.1_1": "IDN001", + "IDN.20_1": "IDN018", + "IDN.21_1": "IDN019", + "IDN.22_1": "IDN035", + "IDN.23_1": "IDN033", + "IDN.24_1": "IDN004", + "IDN.25_1": "IDN030", + "IDN.26_1": "IDN027", + "IDN.27_1": "IDN026", + "IDN.28_1": "IDN028", + "IDN.29_1": "IDN025", + "IDN.2_1": "IDN017", + "IDN.30_1": "IDN003", + "IDN.31_1": "IDN006", + "IDN.32_1": "IDN002", + "IDN.34_1": "IDN023", + "IDN.35_1": "IDN024", + "IDN.4_1": "IDN016", + "IDN.5_1": "IDN007", + "IDN.6_1": "IDN029", + "IDN.8_1": "IDN005", + "IDN.9_1": "IDN012", + "IMN.10_2": "IMN009", + "IMN.11_2": "IMN011", + "IMN.12_2": "IMN013", + "IMN.13_2": "IMN014", + "IMN.14_2": "IMN016", + "IMN.15_2": "IMN017", + "IMN.16_2": "IMN018", + "IMN.17_2": "IMN019", + "IMN.18_2": "IMN020", + "IMN.1_2": "IMN001", + "IMN.20_2": "IMN022", + "IMN.3_2": "IMN003", + "IMN.4_2": "IMN004", + "IMN.5_2": "IMN005", + "IMN.6_2": "IMN006", + "IMN.7_2": "IMN007", + "IMN.9_2": "IMN008", + "IND.10_1": "IND011", + "IND.11_1": "IND012", + "IND.12_1": "IND013", + "IND.13_1": "IND014", + "IND.15_1": "IND016", + "IND.16_1": "IND017", + "IND.17_1": "IND018", + "IND.18_1": "IND019", + "IND.19_1": "IND020", + "IND.20_1": "IND021", + "IND.21_1": "IND022", + "IND.22_1": "IND023", + "IND.23_1": "IND024", + "IND.24_1": "IND025", + "IND.26_1": "IND026", + "IND.27_1": "IND027", + "IND.28_1": "IND028", + "IND.29_1": "IND029", + "IND.2_1": "IND002", + "IND.30_1": "IND030", + "IND.31_1": "IND031", + "IND.32_1": "IND032", + "IND.33_1": "IND033", + "IND.34_1": "IND034", + "IND.35_1": "IND035", + "IND.36_1": "IND036", + "IND.3_1": "IND003", + "IND.4_1": "IND004", + "IND.5_1": "IND005", + "IND.6_1": "IND006", + "IND.7_1": "IND007", + "IND.8_1": "IND008", + "IND.9_1": "IND009", + "IRL.10_1": "IRL010", + "IRL.11_1": "IRL011", + "IRL.12_1": "IRL012", + "IRL.13_1": "IRL013", + "IRL.14_1": "IRL014", + "IRL.15_1": "IRL015", + "IRL.16_1": "IRL016", + "IRL.17_1": "IRL017", + "IRL.18_1": "IRL018", + "IRL.19_1": "IRL019", + "IRL.1_1": "IRL001", + "IRL.20_1": "IRL020", + "IRL.21_1": "IRL021", + "IRL.22_1": "IRL022", + "IRL.23_1": "IRL023", + "IRL.24_1": "IRL024", + "IRL.25_1": "IRL025", + "IRL.26_1": "IRL026", + "IRL.2_1": "IRL002", + "IRL.3_1": "IRL003", + "IRL.4_1": "IRL004", + "IRL.5_1": "IRL005", + "IRL.6_1": "IRL006", + "IRL.7_1": "IRL007", + "IRL.8_1": "IRL008", + "IRL.9_1": "IRL009", + "IRN.11_1": "IRN012", + "IRN.12_1": "IRN013", + "IRN.13_1": "IRN014", + "IRN.14_1": "IRN015", + "IRN.15_1": "IRN017", + "IRN.17_1": "IRN019", + "IRN.18_1": "IRN020", + "IRN.19_1": "IRN021", + "IRN.20_1": "IRN022", + "IRN.25_1": "IRN023", + "IRN.28_1": "IRN025", + "IRN.2_1": "IRN001", + "IRN.30_1": "IRN027", + "IRN.31_1": "IRN028", + "IRN.3_1": "IRN002", + "IRN.6_1": "IRN005", + "IRN.7_1": "IRN006", + "IRN.8_1": "IRN009", + "IRN.9_1": "IRN010", + "IRQ.10_1": "IRQ008", + "IRQ.13_1": "IRQ009", + "IRQ.15_1": "IRQ014", + "IRQ.1_1": "IRQ001", + "IRQ.2_1": "IRQ002", + "IRQ.9_1": "IRQ007", + "ISR.3_1": "ISR002", + "ISR.6_1": "ISR003", + "ISR.7_1": "ISR006", + "ITA.10_1": "ITA009", + "ITA.11_1": "ITA010", + "ITA.12_1": "ITA011", + "ITA.13_1": "ITA012", + "ITA.14_1": "ITA014", + "ITA.16_1": "ITA016", + "ITA.17_1": "ITA017", + "ITA.18_1": "ITA018", + "ITA.19_1": "ITA019", + "ITA.20_1": "ITA020", + "ITA.3_1": "ITA002", + "ITA.4_1": "ITA003", + "ITA.5_1": "ITA004", + "ITA.6_1": "ITA005", + "ITA.7_1": "ITA006", + "ITA.8_1": "ITA007", + "ITA.9_1": "ITA008", + "JAM.10_1": "JAM010", + "JAM.11_1": "JAM011", + "JAM.12_1": "JAM012", + "JAM.13_1": "JAM013", + "JAM.14_1": "JAM014", + "JAM.1_1": "JAM001", + "JAM.2_1": "JAM002", + "JAM.3_1": "JAM003", + "JAM.4_1": "JAM004", + "JAM.5_1": "JAM005", + "JAM.6_1": "JAM006", + "JAM.7_1": "JAM007", + "JAM.8_1": "JAM008", + "JAM.9_1": "JAM009", + "JEY.10_1": "JEY010", + "JEY.11_1": "JEY011", + "JEY.12_1": "JEY012", + "JEY.1_1": "JEY002", + "JEY.3_1": "JEY003", + "JEY.4_1": "JEY004", + "JEY.5_1": "JEY005", + "JEY.6_1": "JEY006", + "JEY.7_1": "JEY007", + "JEY.8_1": "JEY008", + "JEY.9_1": "JEY009", + "JOR.10_1": "JOR010", + "JOR.12_1": "JOR012", + "JOR.2_1": "JOR002", + "JOR.3_1": "JOR003", + "JOR.4_1": "JOR004", + "JOR.5_1": "JOR005", + "JOR.6_1": "JOR006", + "JOR.7_1": "JOR007", + "JOR.8_1": "JOR008", + "JOR.9_1": "JOR009", + "JPN.10_1": "JPN006", + "JPN.14_1": "JPN013", + "JPN.16_1": "JPN015", + "JPN.17_1": "JPN016", + "JPN.19_1": "JPN018", + "JPN.21_1": "JPN020", + "JPN.23_1": "JPN022", + "JPN.24_1": "JPN023", + "JPN.25_1": "JPN024", + "JPN.26_1": "JPN025", + "JPN.28_1": "JPN027", + "JPN.29_1": "JPN028", + "JPN.2_1": "JPN002", + "JPN.31_1": "JPN029", + "JPN.32_1": "JPN030", + "JPN.34_1": "JPN033", + "JPN.35_1": "JPN034", + "JPN.3_1": "JPN003", + "JPN.42_1": "JPN042", + "JPN.43_1": "JPN043", + "JPN.44_1": "JPN044", + "JPN.45_1": "JPN045", + "JPN.5_1": "JPN004", + "JPN.9_1": "JPN005", + "KEN.10_1": "KEN010", + "KEN.11_1": "KEN011", + "KEN.12_1": "KEN012", + "KEN.13_1": "KEN013", + "KEN.14_1": "KEN014", + "KEN.15_1": "KEN015", + "KEN.16_1": "KEN016", + "KEN.17_1": "KEN017", + "KEN.18_1": "KEN018", + "KEN.19_1": "KEN019", + "KEN.1_1": "KEN001", + "KEN.20_1": "KEN020", + "KEN.21_1": "KEN021", + "KEN.22_1": "KEN022", + "KEN.23_1": "KEN023", + "KEN.24_1": "KEN024", + "KEN.25_1": "KEN025", + "KEN.26_1": "KEN026", + "KEN.27_1": "KEN027", + "KEN.28_1": "KEN028", + "KEN.29_1": "KEN029", + "KEN.2_1": "KEN002", + "KEN.30_1": "KEN030", + "KEN.31_1": "KEN031", + "KEN.32_1": "KEN032", + "KEN.33_1": "KEN033", + "KEN.34_1": "KEN034", + "KEN.35_1": "KEN035", + "KEN.36_1": "KEN036", + "KEN.37_1": "KEN037", + "KEN.38_1": "KEN038", + "KEN.39_1": "KEN039", + "KEN.3_1": "KEN003", + "KEN.40_1": "KEN040", + "KEN.41_1": "KEN041", + "KEN.42_1": "KEN042", + "KEN.43_1": "KEN043", + "KEN.44_1": "KEN044", + "KEN.45_1": "KEN045", + "KEN.46_1": "KEN046", + "KEN.47_1": "KEN047", + "KEN.4_1": "KEN004", + "KEN.5_1": "KEN005", + "KEN.6_1": "KEN006", + "KEN.7_1": "KEN007", + "KEN.8_1": "KEN008", + "KEN.9_1": "KEN009", + "KGZ.1_1": "KGZ001", + "KGZ.3_1": "KGZ002", + "KGZ.4_1": "KGZ003", + "KGZ.5_1": "KGZ004", + "KGZ.8_1": "KGZ006", + "KGZ.9_1": "KGZ007", + "KHM.10_1": "KHM009", + "KHM.11_1": "KHM011", + "KHM.12_1": "KHM014", + "KHM.13_1": "KHM021", + "KHM.14_1": "KHM012", + "KHM.15_1": "KHM013", + "KHM.16_1": "KHM015", + "KHM.17_1": "KHM018", + "KHM.18_1": "KHM016", + "KHM.19_1": "KHM017", + "KHM.1_1": "KHM001", + "KHM.20_1": "KHM019", + "KHM.21_1": "KHM020", + "KHM.22_1": "KHM022", + "KHM.23_1": "KHM023", + "KHM.24_1": "KHM024", + "KHM.25_1": "KHM025", + "KHM.2_1": "KHM002", + "KHM.3_1": "KHM003", + "KHM.4_1": "KHM004", + "KHM.5_1": "KHM005", + "KHM.6_1": "KHM006", + "KHM.7_1": "KHM007", + "KHM.8_1": "KHM008", + "KHM.9_1": "KHM010", + "KNA.10_1": "KNA010", + "KNA.11_1": "KNA011", + "KNA.12_1": "KNA012", + "KNA.13_1": "KNA013", + "KNA.14_1": "KNA014", + "KNA.1_1": "KNA001", + "KNA.2_1": "KNA002", + "KNA.3_1": "KNA003", + "KNA.4_1": "KNA004", + "KNA.5_1": "KNA005", + "KNA.6_1": "KNA006", + "KNA.7_1": "KNA007", + "KNA.8_1": "KNA008", + "KNA.9_1": "KNA009", + "KOR.16_1": "KOR013", + "KWT.1_1": "KWT001", + "KWT.2_1": "KWT002", + "KWT.3_1": "KWT003", + "KWT.4_1": "KWT004", + "KWT.5_1": "KWT005", + "LAO.10_1": "LAO010", + "LAO.11_1": "LAO011", + "LAO.12_1": "LAO012", + "LAO.15_1": "LAO015", + "LAO.16_1": "LAO016", + "LAO.17_1": "LAO017", + "LAO.18_1": "LAO018", + "LAO.1_1": "LAO001", + "LAO.2_1": "LAO002", + "LAO.3_1": "LAO003", + "LAO.4_1": "LAO004", + "LAO.5_1": "LAO005", + "LAO.6_1": "LAO006", + "LAO.7_1": "LAO007", + "LAO.8_1": "LAO008", + "LAO.9_1": "LAO009", + "LBN.1_1": "LBN001", + "LBN.3_1": "LBN003", + "LBN.4_1": "LBN004", + "LBN.5_1": "LBN005", + "LBN.7_1": "LBN007", + "LBN.8_1": "LBN008", + "LBR.10_1": "LBR010", + "LBR.11_1": "LBR011", + "LBR.12_1": "LBR012", + "LBR.13_1": "LBR014", + "LBR.14_1": "LBR013", + "LBR.15_1": "LBR015", + "LBR.1_1": "LBR001", + "LBR.2_1": "LBR002", + "LBR.4_1": "LBR005", + "LBR.5_1": "LBR004", + "LBR.6_1": "LBR006", + "LBR.7_1": "LBR007", + "LBR.8_1": "LBR008", + "LBR.9_1": "LBR009", + "LBY.13_1": "LBY012", + "LBY.16_1": "LBY016", + "LBY.18_1": "LBY018", + "LBY.2_1": "LBY004", + "LBY.5_1": "LBY005", + "LBY.6_1": "LBY007", + "LIE.10_1": "LIE010", + "LIE.1_1": "LIE001", + "LIE.2_1": "LIE002", + "LIE.3_1": "LIE003", + "LIE.4_1": "LIE004", + "LIE.5_1": "LIE005", + "LIE.6_1": "LIE006", + "LIE.7_1": "LIE007", + "LIE.8_1": "LIE008", + "LIE.9_1": "LIE009", + "LSO.10_1": "LSO010", + "LSO.1_1": "LSO001", + "LSO.2_1": "LSO002", + "LSO.3_1": "LSO003", + "LSO.4_1": "LSO004", + "LSO.5_1": "LSO005", + "LSO.6_1": "LSO006", + "LSO.7_1": "LSO007", + "LSO.8_1": "LSO008", + "LSO.9_1": "LSO009", + "LTU.10_1": "LTU010", + "LTU.1_1": "LTU001", + "LTU.2_1": "LTU002", + "LTU.3_1": "LTU003", + "LTU.4_1": "LTU004", + "LTU.5_1": "LTU005", + "LTU.7_1": "LTU007", + "LTU.9_1": "LTU009", + "LUX.1_1": "LUX001", + "LUX.2_1": "LUX002", + "LUX.3_1": "LUX003", + "LVA.3_1": "LVA085", + "MDA.10_1": "MDA010", + "MDA.11_1": "MDA011", + "MDA.12_1": "MDA012", + "MDA.13_1": "MDA013", + "MDA.14_1": "MDA014", + "MDA.15_1": "MDA015", + "MDA.16_1": "MDA016", + "MDA.17_1": "MDA017", + "MDA.18_1": "MDA018", + "MDA.1_1": "MDA001", + "MDA.20_1": "MDA019", + "MDA.21_1": "MDA020", + "MDA.22_1": "MDA021", + "MDA.23_1": "MDA022", + "MDA.24_1": "MDA023", + "MDA.25_1": "MDA024", + "MDA.26_1": "MDA025", + "MDA.27_1": "MDA026", + "MDA.28_1": "MDA027", + "MDA.29_1": "MDA028", + "MDA.2_1": "MDA002", + "MDA.30_1": "MDA029", + "MDA.31_1": "MDA030", + "MDA.32_1": "MDA031", + "MDA.33_1": "MDA032", + "MDA.34_1": "MDA033", + "MDA.35_1": "MDA034", + "MDA.36_1": "MDA035", + "MDA.37_1": "MDA036", + "MDA.3_1": "MDA003", + "MDA.4_1": "MDA004", + "MDA.5_1": "MDA005", + "MDA.6_1": "MDA006", + "MDA.7_1": "MDA007", + "MDA.8_1": "MDA008", + "MDA.9_1": "MDA009", + "MDG.1_1": "MDG029", + "MDG.2_1": "MDG030", + "MDG.3_1": "MDG031", + "MDG.4_1": "MDG032", + "MDG.5_1": "MDG033", + "MEX.10_1": "MEX010", + "MEX.11_1": "MEX011", + "MEX.12_1": "MEX012", + "MEX.13_1": "MEX013", + "MEX.14_1": "MEX014", + "MEX.15_1": "MEX015", + "MEX.17_1": "MEX017", + "MEX.18_1": "MEX018", + "MEX.19_1": "MEX019", + "MEX.1_1": "MEX001", + "MEX.20_1": "MEX020", + "MEX.21_1": "MEX021", + "MEX.22_1": "MEX022", + "MEX.23_1": "MEX023", + "MEX.24_1": "MEX024", + "MEX.25_1": "MEX025", + "MEX.26_1": "MEX026", + "MEX.27_1": "MEX027", + "MEX.28_1": "MEX028", + "MEX.29_1": "MEX029", + "MEX.2_1": "MEX003", + "MEX.31_1": "MEX031", + "MEX.32_1": "MEX032", + "MEX.3_1": "MEX002", + "MEX.4_1": "MEX004", + "MEX.5_1": "MEX007", + "MEX.6_1": "MEX008", + "MEX.8_1": "MEX006", + "MLI.2_1": "MLI002", + "MLI.3_1": "MLI003", + "MLI.4_1": "MLI004", + "MLI.5_1": "MLI005", + "MLI.6_1": "MLI006", + "MLI.7_1": "MLI007", + "MLI.8_1": "MLI008", + "MMR.11_1": "MMR011", + "MMR.12_1": "MMR012", + "MMR.13_1": "MMR013", + "MMR.14_1": "MMR014", + "MMR.15_1": "MMR015", + "MMR.1_1": "MMR001", + "MMR.2_1": "MMR002", + "MMR.3_1": "MMR003", + "MMR.4_1": "MMR004", + "MMR.5_1": "MMR005", + "MMR.6_1": "MMR006", + "MMR.7_1": "MMR007", + "MMR.8_1": "MMR008", + "MMR.9_1": "MMR009", + "MNE.10_1": "MNE010", + "MNE.11_1": "MNE011", + "MNE.12_1": "MNE012", + "MNE.13_1": "MNE013", + "MNE.14_1": "MNE014", + "MNE.15_1": "MNE015", + "MNE.16_1": "MNE016", + "MNE.18_1": "MNE018", + "MNE.19_1": "MNE019", + "MNE.1_1": "MNE001", + "MNE.20_1": "MNE020", + "MNE.21_1": "MNE021", + "MNE.2_1": "MNE002", + "MNE.3_1": "MNE003", + "MNE.4_1": "MNE004", + "MNE.5_1": "MNE005", + "MNE.6_1": "MNE006", + "MNE.7_1": "MNE007", + "MNE.8_1": "MNE008", + "MNE.9_1": "MNE009", + "MNG.11_1": "MNG042", + "MNG.15_1": "MNG046", + "MNG.18_1": "MNG043", + "MNG.20_1": "MNG041", + "MNG.21_1": "MNG011", + "MNG.22_1": "MNG085", + "MNG.4_1": "MNG063", + "MNG.6_1": "MNG021", + "MNG.7_1": "MNG044", + "MNG.8_1": "MNG048", + "MOZ.10_1": "MOZ010", + "MOZ.11_1": "MOZ011", + "MOZ.1_1": "MOZ001", + "MOZ.2_1": "MOZ003", + "MOZ.3_1": "MOZ004", + "MOZ.4_1": "MOZ005", + "MOZ.7_1": "MOZ007", + "MOZ.9_1": "MOZ009", + "MRT.11_1": "MRT011", + "MRT.13_1": "MRT013", + "MRT.1_1": "MRT001", + "MRT.2_1": "MRT002", + "MRT.3_1": "MRT003", + "MRT.4_1": "MRT004", + "MRT.5_1": "MRT005", + "MRT.8_1": "MRT008", + "MRT.9_1": "MRT009", + "MUS.12_1": "MUS010", + "MUS.2_1": "MUS002", + "MUS.3_1": "MUS003", + "MUS.4_1": "MUS004", + "MUS.5_1": "MUS005", + "MUS.6_1": "MUS006", + "MUS.7_1": "MUS007", + "MUS.8_1": "MUS008", + "MUS.9_1": "MUS009", + "MYS.10_1": "MYS010", + "MYS.11_1": "MYS011", + "MYS.13_1": "MYS012", + "MYS.14_1": "MYS013", + "MYS.15_1": "MYS014", + "MYS.1_1": "MYS001", + "MYS.2_1": "MYS002", + "MYS.3_1": "MYS003", + "MYS.4_1": "MYS004", + "MYS.5_1": "MYS005", + "MYS.6_1": "MYS006", + "MYS.7_1": "MYS007", + "MYS.8_1": "MYS008", + "MYS.9_1": "MYS009", + "NAM.10_1": "NAM011", + "NAM.11_1": "NAM012", + "NAM.12_1": "NAM013", + "NAM.1_1": "NAM004", + "NAM.2_1": "NAM002", + "NAM.3_1": "NAM003", + "NAM.4_1": "NAM005", + "NAM.5_1": "NAM006", + "NAM.6_1": "NAM007", + "NAM.7_1": "NAM008", + "NAM.8_1": "NAM009", + "NAM.9_1": "NAM010", + "NER.1_1": "NER001", + "NER.2_1": "NER002", + "NER.3_1": "NER003", + "NER.4_1": "NER004", + "NER.6_1": "NER005", + "NER.8_1": "NER007", + "NGA.10_1": "NGA010", + "NGA.11_1": "NGA011", + "NGA.12_1": "NGA012", + "NGA.13_1": "NGA013", + "NGA.14_1": "NGA014", + "NGA.15_1": "NGA015", + "NGA.16_1": "NGA016", + "NGA.17_1": "NGA017", + "NGA.18_1": "NGA018", + "NGA.19_1": "NGA019", + "NGA.1_1": "NGA001", + "NGA.20_1": "NGA020", + "NGA.21_1": "NGA021", + "NGA.22_1": "NGA022", + "NGA.23_1": "NGA023", + "NGA.24_1": "NGA024", + "NGA.25_1": "NGA025", + "NGA.27_1": "NGA027", + "NGA.28_1": "NGA028", + "NGA.29_1": "NGA029", + "NGA.2_1": "NGA002", + "NGA.30_1": "NGA030", + "NGA.31_1": "NGA031", + "NGA.32_1": "NGA032", + "NGA.33_1": "NGA033", + "NGA.34_1": "NGA034", + "NGA.35_1": "NGA035", + "NGA.36_1": "NGA036", + "NGA.37_1": "NGA037", + "NGA.4_1": "NGA004", + "NGA.5_1": "NGA005", + "NGA.6_1": "NGA006", + "NGA.7_1": "NGA007", + "NGA.8_1": "NGA008", + "NGA.9_1": "NGA009", + "NIC.11_1": "NIC008", + "NIC.12_1": "NIC009", + "NIC.13_1": "NIC010", + "NIC.14_1": "NIC011", + "NIC.15_1": "NIC012", + "NIC.16_1": "NIC013", + "NIC.17_1": "NIC016", + "NIC.18_1": "NIC017", + "NIC.3_1": "NIC001", + "NIC.4_1": "NIC002", + "NIC.5_1": "NIC003", + "NIC.6_1": "NIC004", + "NIC.7_1": "NIC005", + "NIC.8_1": "NIC006", + "NIC.9_1": "NIC007", + "NLD.10_1": "NLD009", + "NLD.11_1": "NLD010", + "NLD.12_1": "NLD011", + "NLD.14_1": "NLD012", + "NLD.1_1": "NLD001", + "NLD.2_1": "NLD002", + "NLD.4_1": "NLD004", + "NLD.5_1": "NLD005", + "NLD.7_1": "NLD006", + "NLD.8_1": "NLD007", + "NLD.9_1": "NLD008", + "NOR.10_1": "NOR009", + "NOR.11_1": "NOR011", + "NOR.12_1": "NOR012", + "NOR.13_1": "NOR013", + "NOR.14_1": "NOR015", + "NOR.16_1": "NOR016", + "NOR.17_1": "NOR017", + "NOR.18_1": "NOR018", + "NOR.19_1": "NOR019", + "NOR.1_1": "NOR001", + "NOR.3_1": "NOR002", + "NOR.4_1": "NOR003", + "NOR.5_1": "NOR004", + "NOR.6_1": "NOR005", + "NOR.7_1": "NOR006", + "NZL.11_1": "NZL01", + "NZL.12_1": "NZL14", + "NZL.14_1": "NZL15", + "NZL.15_1": "NZL07", + "NZL.16_1": "NZL16", + "NZL.17_1": "NZL03", + "NZL.18_1": "NZL09", + "NZL.19_1": "NZL12", + "NZL.1_1": "NZL02", + "NZL.2_1": "NZL04", + "NZL.3_1": "NZL13", + "NZL.5_1": "NZL05", + "NZL.6_1": "NZL06", + "NZL.7_1": "NZL08", + "NZL.8_1": "NZL18", + "NZL.9_1": "NZL17", + "OMN.10_1": "OMN007", + "OMN.11_1": "OMN008", + "OMN.5_1": "OMN004", + "OMN.6_1": "OMN005", + "OMN.9_1": "OMN006", + "PAK.2_1": "PAK002", + "PAK.5_1": "PAK005", + "PAK.7_1": "PAK006", + "PAK.8_1": "PAK007", + "PAN.11_1": "PAN012", + "PAN.12_1": "PAN011", + "PAN.13_1": "PAN013", + "PAN.1_1": "PAN001", + "PAN.2_1": "PAN002", + "PAN.3_1": "PAN003", + "PAN.4_1": "PAN004", + "PAN.5_1": "PAN005", + "PAN.6_1": "PAN006", + "PAN.7_1": "PAN007", + "PAN.8_1": "PAN008", + "PAN.9_1": "PAN009", + "PER.10_1": "PER010", + "PER.11_1": "PER011", + "PER.12_1": "PER012", + "PER.13_1": "PER013", + "PER.14_1": "PER014", + "PER.17_1": "PER016", + "PER.18_1": "PER017", + "PER.19_1": "PER018", + "PER.1_1": "PER001", + "PER.20_1": "PER019", + "PER.21_1": "PER020", + "PER.22_1": "PER021", + "PER.23_1": "PER022", + "PER.24_1": "PER023", + "PER.25_1": "PER024", + "PER.26_1": "PER025", + "PER.2_1": "PER002", + "PER.3_1": "PER003", + "PER.4_1": "PER004", + "PER.5_1": "PER005", + "PER.6_1": "PER006", + "PER.7_1": "PER007", + "PER.8_1": "PER008", + "PER.9_1": "PER009", + "PNG.10_1": "PNG010", + "PNG.11_1": "PNG011", + "PNG.12_1": "PNG012", + "PNG.13_1": "PNG013", + "PNG.14_1": "PNG014", + "PNG.15_1": "PNG015", + "PNG.16_1": "PNG016", + "PNG.18_1": "PNG018", + "PNG.19_1": "PNG019", + "PNG.1_1": "PNG001", + "PNG.20_1": "PNG020", + "PNG.21_1": "PNG022", + "PNG.22_1": "PNG021", + "PNG.2_1": "PNG002", + "PNG.3_1": "PNG003", + "PNG.4_1": "PNG004", + "PNG.5_1": "PNG005", + "PNG.6_1": "PNG006", + "PNG.7_1": "PNG007", + "PNG.8_1": "PNG008", + "PNG.9_1": "PNG009", + "POL.10_1": "POL010", + "POL.11_1": "POL011", + "POL.12_1": "POL012", + "POL.13_1": "POL013", + "POL.14_1": "POL014", + "POL.15_1": "POL015", + "POL.16_1": "POL016", + "POL.1_1": "POL001", + "POL.2_1": "POL002", + "POL.5_1": "POL005", + "POL.7_1": "POL007", + "POL.8_1": "POL008", + "POL.9_1": "POL009", + "PRI.11_1": "PRI003", + "PRI.30_1": "PRI004", + "PRI.36_1": "PRI005", + "PRI.3_1": "PRI001", + "PRI.50_1": "PRI006", + "PRI.58_1": "PRI007", + "PRI.65_1": "PRI008", + "PRI.7_1": "PRI002", + "PRK.10_1": "PRK009", + "PRK.1_1": "PRK001", + "PRK.2_1": "PRK002", + "PRK.3_1": "PRK003", + "PRK.4_1": "PRK004", + "PRK.5_1": "PRK005", + "PRK.7_1": "PRK007", + "PRK.9_1": "PRK008", + "PRT.10_1": "PRT010", + "PRT.11_1": "PRT011", + "PRT.12_1": "PRT012", + "PRT.13_1": "PRT013", + "PRT.14_1": "PRT014", + "PRT.15_1": "PRT015", + "PRT.16_1": "PRT016", + "PRT.17_1": "PRT017", + "PRT.18_1": "PRT018", + "PRT.19_1": "PRT019", + "PRT.1_1": "PRT001", + "PRT.20_1": "PRT020", + "PRT.2_1": "PRT002", + "PRT.3_1": "PRT003", + "PRT.4_1": "PRT004", + "PRT.5_1": "PRT005", + "PRT.6_1": "PRT006", + "PRT.7_1": "PRT007", + "PRT.8_1": "PRT008", + "PRT.9_1": "PRT009", + "PRY.10_1": "PRY009", + "PRY.11_1": "PRY010", + "PRY.12_1": "PRY011", + "PRY.13_1": "PRY012", + "PRY.14_1": "PRY013", + "PRY.15_1": "PRY014", + "PRY.16_1": "PRY015", + "PRY.17_1": "PRY016", + "PRY.18_1": "PRY017", + "PRY.1_1": "PRY001", + "PRY.2_1": "PRY002", + "PRY.3_1": "PRY003", + "PRY.5_1": "PRY004", + "PRY.6_1": "PRY005", + "PRY.7_1": "PRY006", + "PRY.8_1": "PRY007", + "PRY.9_1": "PRY008", + "PSE.1_1": "PSE006", + "QAT.1_1": "QAT001", + "QAT.4_1": "QAT005", + "QAT.5_1": "QAT006", + "QAT.7_1": "QAT009", + "ROU.11_1": "ROU011", + "ROU.12_1": "ROU012", + "ROU.13_1": "ROU013", + "ROU.14_1": "ROU014", + "ROU.15_1": "ROU015", + "ROU.16_1": "ROU016", + "ROU.17_1": "ROU017", + "ROU.18_1": "ROU018", + "ROU.19_1": "ROU019", + "ROU.1_1": "ROU001", + "ROU.20_1": "ROU020", + "ROU.22_1": "ROU022", + "ROU.23_1": "ROU023", + "ROU.24_1": "ROU025", + "ROU.25_1": "ROU024", + "ROU.26_1": "ROU026", + "ROU.27_1": "ROU027", + "ROU.28_1": "ROU028", + "ROU.29_1": "ROU029", + "ROU.2_1": "ROU002", + "ROU.30_1": "ROU030", + "ROU.31_1": "ROU031", + "ROU.32_1": "ROU032", + "ROU.33_1": "ROU033", + "ROU.34_1": "ROU034", + "ROU.35_1": "ROU035", + "ROU.36_1": "ROU036", + "ROU.37_1": "ROU037", + "ROU.38_1": "ROU038", + "ROU.39_1": "ROU039", + "ROU.3_1": "ROU003", + "ROU.40_1": "ROU040", + "ROU.41_1": "ROU041", + "ROU.42_1": "ROU042", + "ROU.4_1": "ROU004", + "ROU.5_1": "ROU005", + "ROU.6_1": "ROU006", + "ROU.7_1": "ROU007", + "ROU.8_1": "ROU009", + "ROU.9_1": "ROU008", + "RUS.44_1": "RUS048", + "RWA.1_1": "RWA003", + "RWA.2_1": "RWA004", + "RWA.3_1": "RWA001", + "RWA.4_1": "RWA005", + "RWA.5_1": "RWA002", + "SAU.10_1": "SAU010", + "SAU.11_1": "SAU011", + "SAU.12_1": "SAU012", + "SAU.13_1": "SAU013", + "SAU.1_1": "SAU001", + "SAU.2_1": "SAU002", + "SAU.3_1": "SAU003", + "SAU.4_1": "SAU004", + "SAU.5_1": "SAU005", + "SAU.7_1": "SAU007", + "SAU.8_1": "SAU008", + "SDN.10_1": "SDN007", + "SDN.11_1": "SDN010", + "SDN.13_1": "SDN011", + "SDN.18_1": "SDN015", + "SDN.3_1": "SDN002", + "SDN.6_1": "SDN004", + "SDN.7_1": "SDN005", + "SEN.10_1": "SEN015", + "SEN.11_1": "SEN021", + "SEN.12_1": "SEN022", + "SEN.13_1": "SEN009", + "SEN.14_1": "SEN010", + "SEN.1_1": "SEN001", + "SEN.2_1": "SEN011", + "SEN.3_1": "SEN016", + "SEN.4_1": "SEN017", + "SEN.5_1": "SEN018", + "SEN.6_1": "SEN019", + "SEN.7_1": "SEN020", + "SEN.8_1": "SEN013", + "SEN.9_1": "SEN014", + "SGP.4_1": "SGP006", + "SHN.1_1": "SHN001", + "SHN.2_1": "SHN002", + "SHN.3_1": "SHN003", + "SLE.1_1": "SLE001", + "SLE.2_1": "SLE003", + "SLE.3_1": "SLE004", + "SLV.10_1": "SLV010", + "SLV.11_1": "SLV011", + "SLV.12_1": "SLV012", + "SLV.13_1": "SLV013", + "SLV.14_1": "SLV014", + "SLV.1_1": "SLV001", + "SLV.2_1": "SLV002", + "SLV.3_1": "SLV003", + "SLV.4_1": "SLV004", + "SLV.5_1": "SLV005", + "SLV.6_1": "SLV006", + "SLV.7_1": "SLV007", + "SLV.8_1": "SLV008", + "SLV.9_1": "SLV009", + "SOM.11_1": "SOM011", + "SOM.12_1": "SOM012", + "SOM.13_1": "SOM013", + "SOM.16_1": "SOM016", + "SOM.17_1": "SOM017", + "SOM.18_1": "SOM018", + "SOM.1_1": "SOM001", + "SOM.2_1": "SOM002", + "SOM.4_1": "SOM004", + "SOM.5_1": "SOM005", + "SOM.7_1": "SOM007", + "SPM.1_1": "SPM001", + "SPM.2_1": "SPM002", + "SRB.10_1": "SRB010", + "SRB.11_1": "SRB011", + "SRB.12_1": "SRB012", + "SRB.13_1": "SRB013", + "SRB.14_1": "SRB014", + "SRB.15_1": "SRB015", + "SRB.16_1": "SRB016", + "SRB.17_1": "SRB017", + "SRB.18_1": "SRB018", + "SRB.19_1": "SRB019", + "SRB.1_1": "SRB001", + "SRB.20_1": "SRB020", + "SRB.21_1": "SRB021", + "SRB.22_1": "SRB022", + "SRB.23_1": "SRB023", + "SRB.24_1": "SRB024", + "SRB.25_1": "SRB025", + "SRB.2_1": "SRB002", + "SRB.3_1": "SRB003", + "SRB.4_1": "SRB004", + "SRB.5_1": "SRB005", + "SRB.6_1": "SRB006", + "SRB.7_1": "SRB007", + "SRB.8_1": "SRB008", + "SRB.9_1": "SRB009", + "SSD.1_1": "SSD001", + "SSD.2_1": "SSD002", + "SSD.4_1": "SSD004", + "SSD.6_1": "SSD008", + "SSD.7_1": "SSD009", + "STP.1_1": "STP001", + "STP.2_1": "STP002", + "SUR.10_1": "SUR010", + "SUR.1_1": "SUR001", + "SUR.2_1": "SUR002", + "SUR.3_1": "SUR003", + "SUR.4_1": "SUR004", + "SUR.5_1": "SUR005", + "SUR.6_1": "SUR006", + "SUR.7_1": "SUR007", + "SUR.8_1": "SUR008", + "SUR.9_1": "SUR009", + "SVN.10_1": "SVN010", + "SVN.11_1": "SVN011", + "SVN.12_1": "SVN012", + "SVN.1_1": "SVN001", + "SVN.2_1": "SVN002", + "SVN.4_1": "SVN004", + "SVN.5_1": "SVN005", + "SVN.6_1": "SVN006", + "SVN.7_1": "SVN007", + "SVN.8_1": "SVN008", + "SVN.9_1": "SVN009", + "SWZ.1_1": "SWZ001", + "SWZ.2_1": "SWZ002", + "SWZ.3_1": "SWZ003", + "SWZ.4_1": "SWZ004", + "SYC.10_1": "SYC014", + "SYC.11_1": "SYC017", + "SYC.12_1": "SYC029", + "SYC.13_1": "SYC033", + "SYC.17_1": "SYC038", + "SYC.18_1": "SYC040", + "SYC.19_1": "SYC041", + "SYC.1_1": "SYC003", + "SYC.21_1": "SYC044", + "SYC.22_1": "SYC046", + "SYC.23_1": "SYC048", + "SYC.24_1": "SYC052", + "SYC.26_1": "SYC059", + "SYC.2_1": "SYC004", + "SYC.3_1": "SYC005", + "SYC.4_1": "SYC006", + "SYC.5_1": "SYC010", + "SYC.6_1": "SYC011", + "SYC.8_1": "SYC012", + "SYC.9_1": "SYC013", + "SYR.10_1": "SYR010", + "SYR.12_1": "SYR012", + "SYR.13_1": "SYR013", + "SYR.14_1": "SYR014", + "SYR.1_1": "SYR001", + "SYR.2_1": "SYR002", + "SYR.3_1": "SYR003", + "SYR.4_1": "SYR004", + "SYR.5_1": "SYR005", + "SYR.6_1": "SYR006", + "SYR.7_1": "SYR007", + "SYR.9_1": "SYR009", + "TCA.1_1": "TCA001", + "TCA.2_1": "TCA002", + "TCA.3_1": "TCA003", + "TCA.4_1": "TCA004", + "TCD.10_1": "TCD010", + "TCD.11_1": "TCD011", + "TCD.12_1": "TCD012", + "TCD.13_1": "TCD013", + "TCD.14_1": "TCD014", + "TCD.15_1": "TCD015", + "TCD.16_1": "TCD016", + "TCD.17_1": "TCD017", + "TCD.18_1": "TCD018", + "TCD.19_1": "TCD019", + "TCD.1_1": "TCD001", + "TCD.20_1": "TCD020", + "TCD.21_1": "TCD021", + "TCD.22_1": "TCD022", + "TCD.23_1": "TCD023", + "TCD.2_1": "TCD002", + "TCD.3_1": "TCD003", + "TCD.4_1": "TCD004", + "TCD.5_1": "TCD005", + "TCD.6_1": "TCD006", + "TCD.7_1": "TCD007", + "TCD.8_1": "TCD008", + "TCD.9_1": "TCD009", + "TGO.2_1": "TGO002", + "TGO.3_1": "TGO003", + "TGO.4_1": "TGO004", + "TGO.5_1": "TGO005", + "THA.10_1": "THA009", + "THA.11_1": "THA010", + "THA.12_1": "THA011", + "THA.13_1": "THA012", + "THA.14_1": "THA013", + "THA.16_1": "THA015", + "THA.17_1": "THA016", + "THA.18_1": "THA017", + "THA.19_1": "THA018", + "THA.1_1": "THA001", + "THA.20_1": "THA019", + "THA.21_1": "THA020", + "THA.22_1": "THA021", + "THA.23_1": "THA022", + "THA.24_1": "THA023", + "THA.25_1": "THA024", + "THA.26_1": "THA025", + "THA.27_1": "THA026", + "THA.28_1": "THA027", + "THA.29_1": "THA028", + "THA.2_1": "THA002", + "THA.30_1": "THA029", + "THA.31_1": "THA030", + "THA.32_1": "THA031", + "THA.33_1": "THA032", + "THA.34_1": "THA033", + "THA.35_1": "THA034", + "THA.36_1": "THA035", + "THA.37_1": "THA036", + "THA.38_1": "THA037", + "THA.39_1": "THA039", + "THA.40_1": "THA040", + "THA.41_1": "THA041", + "THA.42_1": "THA042", + "THA.43_1": "THA043", + "THA.44_1": "THA044", + "THA.45_1": "THA045", + "THA.47_1": "THA047", + "THA.48_1": "THA048", + "THA.51_1": "THA050", + "THA.52_1": "THA051", + "THA.53_1": "THA052", + "THA.54_1": "THA053", + "THA.55_1": "THA054", + "THA.56_1": "THA055", + "THA.58_1": "THA057", + "THA.5_1": "THA004", + "THA.60_1": "THA059", + "THA.61_1": "THA060", + "THA.62_1": "THA061", + "THA.63_1": "THA062", + "THA.64_1": "THA063", + "THA.65_1": "THA064", + "THA.66_1": "THA065", + "THA.67_1": "THA066", + "THA.68_1": "THA067", + "THA.69_1": "THA068", + "THA.6_1": "THA005", + "THA.70_1": "THA070", + "THA.72_1": "THA071", + "THA.73_1": "THA072", + "THA.74_1": "THA073", + "THA.75_1": "THA074", + "THA.76_1": "THA075", + "THA.77_1": "THA076", + "THA.7_1": "THA006", + "THA.8_1": "THA007", + "THA.9_1": "THA008", + "TJK.1_1": "TJK003", + "TJK.3_1": "TJK004", + "TJK.4_1": "TJK005", + "TKM.1_1": "TKM001", + "TKM.3_1": "TKM002", + "TKM.5_1": "TKM004", + "TLS.10_1": "TLS009", + "TLS.11_1": "TLS010", + "TLS.12_1": "TLS011", + "TLS.13_1": "TLS013", + "TLS.1_1": "TLS001", + "TLS.2_1": "TLS002", + "TLS.4_1": "TLS003", + "TLS.5_1": "TLS004", + "TLS.6_1": "TLS005", + "TLS.7_1": "TLS006", + "TLS.8_1": "TLS007", + "TLS.9_1": "TLS008", + "TTO.10_1": "TTO010", + "TTO.11_1": "TTO011", + "TTO.12_1": "TTO012", + "TTO.13_1": "TTO013", + "TTO.14_1": "TTO014", + "TTO.15_1": "TTO015", + "TTO.1_1": "TTO001", + "TTO.2_1": "TTO002", + "TTO.3_1": "TTO003", + "TTO.4_1": "TTO004", + "TTO.5_1": "TTO009", + "TTO.6_1": "TTO005", + "TTO.7_1": "TTO006", + "TTO.8_1": "TTO007", + "TTO.9_1": "TTO008", + "TUN.10_1": "TUN010", + "TUN.11_1": "TUN011", + "TUN.12_1": "TUN012", + "TUN.15_1": "TUN014", + "TUN.16_1": "TUN015", + "TUN.17_1": "TUN016", + "TUN.18_1": "TUN017", + "TUN.19_1": "TUN018", + "TUN.1_1": "TUN024", + "TUN.20_1": "TUN019", + "TUN.21_1": "TUN020", + "TUN.22_1": "TUN021", + "TUN.23_1": "TUN022", + "TUN.24_1": "TUN023", + "TUN.2_1": "TUN002", + "TUN.4_1": "TUN004", + "TUN.5_1": "TUN005", + "TUN.6_1": "TUN006", + "TUN.7_1": "TUN007", + "TUN.8_1": "TUN008", + "TUN.9_1": "TUN009", + "TUR.10_1": "TUR010", + "TUR.11_1": "TUR011", + "TUR.12_1": "TUR012", + "TUR.14_1": "TUR014", + "TUR.15_1": "TUR015", + "TUR.16_1": "TUR016", + "TUR.17_1": "TUR017", + "TUR.18_1": "TUR018", + "TUR.19_1": "TUR019", + "TUR.1_1": "TUR001", + "TUR.20_1": "TUR020", + "TUR.21_1": "TUR021", + "TUR.22_1": "TUR022", + "TUR.23_1": "TUR023", + "TUR.24_1": "TUR024", + "TUR.25_1": "TUR025", + "TUR.26_1": "TUR026", + "TUR.28_1": "TUR027", + "TUR.2_1": "TUR002", + "TUR.30_1": "TUR029", + "TUR.31_1": "TUR030", + "TUR.32_1": "TUR031", + "TUR.33_1": "TUR032", + "TUR.34_1": "TUR033", + "TUR.35_1": "TUR034", + "TUR.36_1": "TUR035", + "TUR.37_1": "TUR036", + "TUR.39_1": "TUR039", + "TUR.3_1": "TUR003", + "TUR.40_1": "TUR040", + "TUR.41_1": "TUR041", + "TUR.42_1": "TUR042", + "TUR.43_1": "TUR043", + "TUR.44_1": "TUR044", + "TUR.45_1": "TUR045", + "TUR.46_1": "TUR046", + "TUR.47_1": "TUR047", + "TUR.48_1": "TUR048", + "TUR.4_1": "TUR004", + "TUR.50_1": "TUR050", + "TUR.51_1": "TUR051", + "TUR.52_1": "TUR052", + "TUR.53_1": "TUR053", + "TUR.54_1": "TUR054", + "TUR.55_1": "TUR055", + "TUR.56_1": "TUR056", + "TUR.57_1": "TUR057", + "TUR.59_1": "TUR058", + "TUR.5_1": "TUR005", + "TUR.60_1": "TUR059", + "TUR.61_1": "TUR060", + "TUR.62_1": "TUR061", + "TUR.63_1": "TUR062", + "TUR.64_1": "TUR063", + "TUR.65_1": "TUR064", + "TUR.66_1": "TUR065", + "TUR.67_1": "TUR066", + "TUR.68_1": "TUR067", + "TUR.69_1": "TUR068", + "TUR.6_1": "TUR006", + "TUR.70_1": "TUR069", + "TUR.71_1": "TUR070", + "TUR.72_1": "TUR071", + "TUR.73_1": "TUR072", + "TUR.74_1": "TUR073", + "TUR.75_1": "TUR074", + "TUR.76_1": "TUR075", + "TUR.77_1": "TUR076", + "TUR.78_1": "TUR077", + "TUR.79_1": "TUR078", + "TUR.7_1": "TUR007", + "TUR.80_1": "TUR079", + "TUR.8_1": "TUR008", + "TUR.9_1": "TUR009", + "TZA.10_1": "TZA014", + "TZA.11_1": "TZA015", + "TZA.12_1": "TZA016", + "TZA.13_1": "TZA017", + "TZA.14_1": "TZA019", + "TZA.15_1": "TZA020", + "TZA.16_1": "TZA021", + "TZA.17_1": "TZA022", + "TZA.18_1": "TZA007", + "TZA.19_1": "TZA012", + "TZA.1_1": "TZA001", + "TZA.20_1": "TZA023", + "TZA.21_1": "TZA024", + "TZA.22_1": "TZA025", + "TZA.23_1": "TZA026", + "TZA.24_1": "TZA027", + "TZA.25_1": "TZA028", + "TZA.26_1": "TZA029", + "TZA.27_1": "TZA030", + "TZA.28_1": "TZA008", + "TZA.29_1": "TZA013", + "TZA.2_1": "TZA002", + "TZA.30_1": "TZA018", + "TZA.3_1": "TZA003", + "TZA.4_1": "TZA004", + "TZA.5_1": "TZA005", + "TZA.6_1": "TZA006", + "TZA.7_1": "TZA009", + "TZA.8_1": "TZA010", + "TZA.9_1": "TZA011", + "UGA.10_1": "UGA033", + "UGA.11_1": "UGA035", + "UGA.12_1": "UGA037", + "UGA.13_1": "UGA038", + "UGA.14_1": "UGA039", + "UGA.15_1": "UGA040", + "UGA.16_1": "UGA043", + "UGA.17_1": "UGA044", + "UGA.18_1": "UGA045", + "UGA.19_1": "UGA046", + "UGA.1_1": "UGA002", + "UGA.20_1": "UGA047", + "UGA.21_1": "UGA048", + "UGA.22_1": "UGA049", + "UGA.23_1": "UGA050", + "UGA.26_1": "UGA055", + "UGA.27_1": "UGA056", + "UGA.28_1": "UGA059", + "UGA.29_1": "UGA060", + "UGA.2_1": "UGA009", + "UGA.30_1": "UGA064", + "UGA.33_1": "UGA066", + "UGA.34_1": "UGA068", + "UGA.35_1": "UGA073", + "UGA.36_1": "UGA074", + "UGA.37_1": "UGA075", + "UGA.38_1": "UGA076", + "UGA.39_1": "UGA077", + "UGA.3_1": "UGA010", + "UGA.40_1": "UGA080", + "UGA.41_1": "UGA081", + "UGA.42_1": "UGA082", + "UGA.43_1": "UGA083", + "UGA.44_1": "UGA084", + "UGA.45_1": "UGA085", + "UGA.46_1": "UGA087", + "UGA.47_1": "UGA091", + "UGA.48_1": "UGA094", + "UGA.49_1": "UGA098", + "UGA.4_1": "UGA013", + "UGA.50_1": "UGA099", + "UGA.51_1": "UGA100", + "UGA.52_1": "UGA102", + "UGA.54_1": "UGA105", + "UGA.55_1": "UGA106", + "UGA.56_1": "UGA108", + "UGA.57_1": "UGA109", + "UGA.58_1": "UGA110", + "UGA.5_1": "UGA021", + "UGA.6_1": "UGA022", + "UGA.7_1": "UGA023", + "UGA.8_1": "UGA030", + "UGA.9_1": "UGA031", + "UKR.4_1": "UKR004", + "URY.10_2": "URY010", + "URY.11_2": "URY011", + "URY.12_2": "URY012", + "URY.13_2": "URY013", + "URY.14_2": "URY014", + "URY.15_2": "URY015", + "URY.16_2": "URY016", + "URY.17_2": "URY017", + "URY.18_2": "URY018", + "URY.19_2": "URY019", + "URY.1_2": "URY001", + "URY.2_2": "URY002", + "URY.3_2": "URY003", + "URY.4_2": "URY004", + "URY.5_2": "URY005", + "URY.6_2": "URY006", + "URY.7_2": "URY007", + "URY.8_2": "URY008", + "URY.9_2": "URY009", + "USA.10_1": "USA010", + "USA.11_1": "USA011", + "USA.12_1": "USA012", + "USA.13_1": "USA013", + "USA.14_1": "USA014", + "USA.15_1": "USA015", + "USA.16_1": "USA016", + "USA.17_1": "USA017", + "USA.18_1": "USA018", + "USA.19_1": "USA019", + "USA.1_1": "USA001", + "USA.20_1": "USA020", + "USA.21_1": "USA021", + "USA.22_1": "USA022", + "USA.23_1": "USA023", + "USA.24_1": "USA024", + "USA.25_1": "USA025", + "USA.26_1": "USA026", + "USA.27_1": "USA027", + "USA.28_1": "USA028", + "USA.29_1": "USA029", + "USA.2_1": "USA002", + "USA.30_1": "USA030", + "USA.31_1": "USA031", + "USA.32_1": "USA032", + "USA.33_1": "USA033", + "USA.34_1": "USA034", + "USA.35_1": "USA035", + "USA.36_1": "USA036", + "USA.37_1": "USA037", + "USA.38_1": "USA038", + "USA.39_1": "USA039", + "USA.3_1": "USA003", + "USA.40_1": "USA040", + "USA.41_1": "USA041", + "USA.42_1": "USA042", + "USA.43_1": "USA043", + "USA.44_1": "USA044", + "USA.45_1": "USA045", + "USA.46_1": "USA046", + "USA.47_1": "USA047", + "USA.48_1": "USA048", + "USA.49_1": "USA049", + "USA.4_1": "USA004", + "USA.50_1": "USA050", + "USA.51_1": "USA051", + "USA.5_1": "USA005", + "USA.6_1": "USA006", + "USA.7_1": "USA007", + "USA.8_1": "USA008", + "USA.9_1": "USA009", + "UZB.8_1": "UZB008", + "UZB.9_1": "UZB009", + "VCT.1_1": "VCT001", + "VCT.2_1": "VCT002", + "VCT.3_1": "VCT003", + "VCT.4_1": "VCT004", + "VCT.5_1": "VCT005", + "VCT.6_1": "VCT006", + "VEN.10_1": "VEN010", + "VEN.11_1": "VEN011", + "VEN.12_1": "VEN012", + "VEN.13_1": "VEN013", + "VEN.14_1": "VEN014", + "VEN.15_1": "VEN015", + "VEN.16_1": "VEN016", + "VEN.17_1": "VEN017", + "VEN.18_1": "VEN018", + "VEN.19_1": "VEN019", + "VEN.1_1": "VEN001", + "VEN.20_1": "VEN020", + "VEN.21_1": "VEN021", + "VEN.22_1": "VEN022", + "VEN.23_1": "VEN023", + "VEN.24_1": "VEN024", + "VEN.25_1": "VEN025", + "VEN.2_1": "VEN002", + "VEN.3_1": "VEN003", + "VEN.4_1": "VEN004", + "VEN.5_1": "VEN005", + "VEN.6_1": "VEN006", + "VEN.7_1": "VEN007", + "VEN.8_1": "VEN008", + "VEN.9_1": "VEN009", + "VGB.1_1": "VGB001", + "VGB.2_1": "VGB002", + "VGB.3_1": "VGB003", + "VGB.4_1": "VGB004", + "VGB.5_1": "VGB005", + "VNM.10_1": "VNM010", + "VNM.11_1": "VNM011", + "VNM.12_1": "VNM013", + "VNM.13_1": "VNM012", + "VNM.14_1": "VNM014", + "VNM.1_1": "VNM001", + "VNM.21_1": "VNM021", + "VNM.22_1": "VNM025", + "VNM.23_1": "VNM026", + "VNM.24_1": "VNM028", + "VNM.25_1": "VNM029", + "VNM.26_1": "VNM022", + "VNM.27_1": "VNM027", + "VNM.28_1": "VNM023", + "VNM.29_1": "VNM024", + "VNM.2_1": "VNM005", + "VNM.30_1": "VNM030", + "VNM.31_1": "VNM031", + "VNM.32_1": "VNM032", + "VNM.33_1": "VNM033", + "VNM.34_1": "VNM034", + "VNM.35_1": "VNM037", + "VNM.36_1": "VNM035", + "VNM.38_1": "VNM038", + "VNM.39_1": "VNM039", + "VNM.3_1": "VNM004", + "VNM.41_1": "VNM041", + "VNM.42_1": "VNM042", + "VNM.43_1": "VNM043", + "VNM.44_1": "VNM044", + "VNM.45_1": "VNM045", + "VNM.46_1": "VNM046", + "VNM.47_1": "VNM047", + "VNM.48_1": "VNM048", + "VNM.49_1": "VNM049", + "VNM.50_1": "VNM050", + "VNM.51_1": "VNM051", + "VNM.52_1": "VNM052", + "VNM.53_1": "VNM053", + "VNM.54_1": "VNM057", + "VNM.55_1": "VNM054", + "VNM.56_1": "VNM055", + "VNM.57_1": "VNM056", + "VNM.58_1": "VNM058", + "VNM.59_1": "VNM059", + "VNM.5_1": "VNM006", + "VNM.60_1": "VNM060", + "VNM.61_1": "VNM061", + "VNM.62_1": "VNM062", + "VNM.63_1": "VNM063", + "VNM.6_1": "VNM007", + "VNM.7_1": "VNM002", + "VNM.9_1": "VNM009", + "VUT.1_1": "VUT001", + "VUT.2_1": "VUT002", + "VUT.3_1": "VUT003", + "VUT.4_1": "VUT004", + "VUT.5_1": "VUT005", + "VUT.6_1": "VUT006", + "YEM.10_1": "YEM009", + "YEM.11_1": "YEM010", + "YEM.12_1": "YEM011", + "YEM.13_1": "YEM012", + "YEM.14_1": "YEM013", + "YEM.15_1": "YEM014", + "YEM.16_1": "YEM015", + "YEM.17_1": "YEM016", + "YEM.18_1": "YEM017", + "YEM.1_1": "YEM001", + "YEM.20_1": "YEM020", + "YEM.21_1": "YEM022", + "YEM.2_1": "YEM002", + "YEM.3_1": "YEM003", + "YEM.4_1": "YEM004", + "YEM.5_1": "YEM005", + "YEM.6_1": "YEM006", + "YEM.7_1": "YEM007", + "YEM.8_1": "YEM008", + "ZAF.1_1": "ZAF001", + "ZAF.2_1": "ZAF002", + "ZAF.3_1": "ZAF003", + "ZAF.4_1": "ZAF004", + "ZAF.5_1": "ZAF005", + "ZAF.6_1": "ZAF006", + "ZAF.7_1": "ZAF007", + "ZAF.8_1": "ZAF008", + "ZAF.9_1": "ZAF009", + "ZMB.10_1": "ZMB110", + "ZMB.1_1": "ZMB101", + "ZMB.2_1": "ZMB102", + "ZMB.3_1": "ZMB103", + "ZMB.4_1": "ZMB104", + "ZMB.5_1": "ZMB105", + "ZMB.7_1": "ZMB108", + "ZMB.8_1": "ZMB107", + "ZMB.9_1": "ZMB109", + "ZWE.10_1": "ZWE010", + "ZWE.1_1": "ZWE001", + "ZWE.2_1": "ZWE002", + "ZWE.3_1": "ZWE003", + "ZWE.4_1": "ZWE004", + "ZWE.5_1": "ZWE005", + "ZWE.6_1": "ZWE006", + "ZWE.7_1": "ZWE007", + "ZWE.8_1": "ZWE008", + "ZWE.9_1": "ZWE009" + }, + "unmatched_gadm": { + "AFG": [ + "AFG.27_1", + "AFG.30_1" + ], + "AIA": [ + "AIA.10_1", + "AIA.11_1", + "AIA.12_1", + "AIA.13_1", + "AIA.14_1", + "AIA.15_1", + "AIA.16_1", + "AIA.17_1", + "AIA.18_1", + "AIA.1_1", + "AIA.2_1", + "AIA.3_1", + "AIA.4_1", + "AIA.5_1", + "AIA.6_1", + "AIA.7_1", + "AIA.8_1", + "AIA.9_1" + ], + "ALA": [ + "ALA.1_2", + "ALA.2_2", + "ALA.3_2" + ], + "ARE": [ + "ARE.7_1" + ], + "ARG": [ + "ARG.5_1" + ], + "ARM": [ + "ARM.4_1", + "ARM.5_1" + ], + "ASM": [ + "ASM.1_1", + "ASM.2_1", + "ASM.3_1", + "ASM.4_1" + ], + "ATF": [ + "ATF.1_1", + "ATF.2_1", + "ATF.3_1", + "ATF.4_1" + ], + "ATG": [ + "ATG.3_1", + "ATG.4_1", + "ATG.5_1", + "ATG.6_1", + "ATG.7_1", + "ATG.8_1" + ], + "AUS": [ + "AUS.1_1", + "AUS.4_1" + ], + "AZE": [ + "AZE.10_1", + "AZE.2_1", + "AZE.3_1", + "AZE.4_1", + "AZE.5_1", + "AZE.8_1", + "AZE.9_1" + ], + "BDI": [ + "BDI.2_1", + "BDI.3_1" + ], + "BEL": [ + "BEL.1_1", + "BEL.2_1", + "BEL.3_1" + ], + "BEN": [ + "BEN.2_1", + "BEN.7_1" + ], + "BFA": [ + "BFA.9_1" + ], + "BGD": [ + "BGD.1_1", + "BGD.2_1" + ], + "BGR": [ + "BGR.20_1", + "BGR.27_1", + "BGR.28_1", + "BGR.5_1", + "BGR.8_1" + ], + "BHR": [ + "BHR.1_1", + "BHR.5_1" + ], + "BHS": [ + "BHS.10_1", + "BHS.11_1", + "BHS.12_1", + "BHS.13_1", + "BHS.14_1", + "BHS.15_1", + "BHS.16_1", + "BHS.17_1", + "BHS.18_1", + "BHS.19_1", + "BHS.1_1", + "BHS.20_1", + "BHS.21_1", + "BHS.22_1", + "BHS.23_1", + "BHS.24_1", + "BHS.25_1", + "BHS.26_1", + "BHS.27_1", + "BHS.28_1", + "BHS.29_1", + "BHS.2_1", + "BHS.30_1", + "BHS.31_1", + "BHS.32_1", + "BHS.3_1", + "BHS.4_1", + "BHS.5_1", + "BHS.6_1", + "BHS.7_1", + "BHS.8_1", + "BHS.9_1" + ], + "BIH": [ + "BIH.1_1", + "BIH.2_1", + "BIH.3_1" + ], + "BLM": [ + "BLM.1_1", + "BLM.2_1" + ], + "BLR": [ + "BLR.5_1", + "BLR.7_1" + ], + "BMU": [ + "BMU.10_1", + "BMU.11_1", + "BMU.1_1", + "BMU.2_1", + "BMU.3_1", + "BMU.4_1", + "BMU.5_1", + "BMU.6_1", + "BMU.7_1", + "BMU.8_1", + "BMU.9_1" + ], + "BRB": [ + "BRB.10_1", + "BRB.11_1", + "BRB.2_1", + "BRB.3_1", + "BRB.4_1", + "BRB.5_1", + "BRB.6_1", + "BRB.7_1", + "BRB.8_1", + "BRB.9_1" + ], + "BWA": [ + "BWA.10_1", + "BWA.12_1", + "BWA.13_1", + "BWA.16_1", + "BWA.3_1", + "BWA.4_1", + "BWA.6_1" + ], + "CAF": [ + "CAF.10_1", + "CAF.11_1", + "CAF.12_1", + "CAF.13_1", + "CAF.14_1", + "CAF.15_1", + "CAF.16_1", + "CAF.17_1", + "CAF.1_1", + "CAF.2_1", + "CAF.3_1", + "CAF.4_1", + "CAF.5_1", + "CAF.6_1", + "CAF.7_1", + "CAF.8_1", + "CAF.9_1" + ], + "CHE": [ + "CHE.12_1" + ], + "CHL": [ + "CHL.13_1", + "CHL.14_1", + "CHL.1_1", + "CHL.4_1", + "CHL.8_1" + ], + "CHN": [ + "CHN.19_1", + "CHN.20_1", + "CHN.24_1", + "CHN.27_1", + "CHN.28_1", + "CHN.29_1", + "CHN.2_1", + "CHN.3_1", + "CHN.7_1", + "CHN.HKG", + "CHN.MAC" + ], + "COG": [ + "COG.10_1" + ], + "COK": [ + "COK.10_1", + "COK.11_1", + "COK.1_1", + "COK.2_1", + "COK.3_1", + "COK.4_1", + "COK.5_1", + "COK.6_1", + "COK.7_1", + "COK.8_1", + "COK.9_1" + ], + "COL": [ + "COL.27_1" + ], + "COM": [ + "COM.1_1", + "COM.2_1", + "COM.3_1" + ], + "CUB": [ + "CUB.4_1" + ], + "CYM": [ + "CYM.1_1", + "CYM.3_1", + "CYM.4_1", + "CYM.6_1", + "CYM.7_1", + "CYM.8_1" + ], + "CYP": [ + "CYP.1_1", + "CYP.2_1", + "CYP.3_1", + "CYP.4_1", + "CYP.5_1" + ], + "CZE": [ + "CZE.10_1", + "CZE.11_1", + "CZE.13_1", + "CZE.14_1", + "CZE.3_1", + "CZE.4_1", + "CZE.5_1", + "CZE.6_1", + "CZE.7_1", + "CZE.8_1", + "CZE.9_1" + ], + "DEU": [ + "DEU.16_1", + "DEU.1_1" + ], + "DJI": [ + "DJI.1_2", + "DJI.2_1", + "DJI.3_1", + "DJI.5_1" + ], + "DMA": [ + "DMA.10_1", + "DMA.1_1", + "DMA.2_1", + "DMA.3_1", + "DMA.4_1", + "DMA.5_1", + "DMA.6_1", + "DMA.7_1", + "DMA.8_1", + "DMA.9_1" + ], + "DNK": [ + "DNK.1_1", + "DNK.2_1", + "DNK.3_1", + "DNK.4_1", + "DNK.5_1" + ], + "DOM": [ + "DOM.12_1", + "DOM.15_1", + "DOM.2_1", + "DOM.7_1" + ], + "DZA": [ + "DZA.11_1", + "DZA.41_1" + ], + "EGY": [ + "EGY.10_1", + "EGY.11_1", + "EGY.12_1", + "EGY.13_1", + "EGY.14_1", + "EGY.15_1", + "EGY.16_1", + "EGY.18_1", + "EGY.19_1", + "EGY.1_1", + "EGY.20_1", + "EGY.21_1", + "EGY.22_1", + "EGY.23_1", + "EGY.25_1", + "EGY.26_1", + "EGY.27_1", + "EGY.2_1", + "EGY.3_1", + "EGY.4_1", + "EGY.5_1", + "EGY.6_1", + "EGY.7_1", + "EGY.8_1", + "EGY.9_1" + ], + "ERI": [ + "ERI.2_1", + "ERI.3_1" + ], + "ESH": [ + "ESH.1_1", + "ESH.2_1", + "ESH.3_1", + "ESH.4_1" + ], + "ESP": [ + "ESP.10_1", + "ESP.13_1", + "ESP.14_1", + "ESP.7_1" + ], + "EST": [ + "EST.10_1", + "EST.11_1", + "EST.12_1", + "EST.13_1", + "EST.14_1", + "EST.15_1", + "EST.16_1", + "EST.1_1", + "EST.2_1", + "EST.3_1", + "EST.4_1", + "EST.5_1", + "EST.6_1", + "EST.7_1", + "EST.8_1", + "EST.9_1" + ], + "ETH": [ + "ETH.10_1", + "ETH.1_1", + "ETH.4_1", + "ETH.6_1", + "ETH.7_1" + ], + "FJI": [ + "FJI.4_1" + ], + "FRA": [ + "FRA.10_1", + "FRA.11_1", + "FRA.1_1", + "FRA.2_1", + "FRA.4_1", + "FRA.6_1", + "FRA.7_1", + "FRA.9_1" + ], + "FRO": [ + "FRO.1_1", + "FRO.2_1", + "FRO.3_1", + "FRO.4_1", + "FRO.5_1", + "FRO.6_1" + ], + "FSM": [ + "FSM.1_1", + "FSM.2_1", + "FSM.3_1", + "FSM.4_1" + ], + "GAB": [ + "GAB.9_1" + ], + "GBR": [ + "GBR.1_1", + "GBR.1_1", + "GBR.4_1" + ], + "GEO": [ + "GEO.1_1", + "GEO.2_1" + ], + "GGY": [ + "GGY.10_1" + ], + "GHA": [ + "GHA7_2", + "GHA8_2" + ], + "GLP": [ + "GLP.1_1", + "GLP.2_1" + ], + "GMB": [ + "GMB.1_1", + "GMB.3_1", + "GMB.6_1" + ], + "GNB": [ + "GNB.3_1" + ], + "GRC": [ + "GRC.1_1", + "GRC.2_1", + "GRC.3_1", + "GRC.4_1", + "GRC.5_1", + "GRC.6_1", + "GRC.7_1", + "GRC.8_1" + ], + "GRD": [ + "GRD.1_1", + "GRD.2_1", + "GRD.3_1", + "GRD.4_1", + "GRD.5_1", + "GRD.6_1", + "GRD.7_1" + ], + "GRL": [ + "GRL.1_1", + "GRL.2_1", + "GRL.3_1", + "GRL.4_1", + "GRL.5_1" + ], + "GTM": [ + "GTM.13_1" + ], + "GUF": [ + "GUF.1_1", + "GUF.2_1" + ], + "GUM": [ + "GUM.10_1", + "GUM.11_1", + "GUM.12_1", + "GUM.13_1", + "GUM.14_1", + "GUM.15_1", + "GUM.16_1", + "GUM.17_1", + "GUM.18_1", + "GUM.19_1", + "GUM.1_1", + "GUM.2_1", + "GUM.3_1", + "GUM.4_1", + "GUM.5_1", + "GUM.6_1", + "GUM.7_1", + "GUM.8_1", + "GUM.9_1" + ], + "GUY": [ + "GUY.10_1", + "GUY.1_1", + "GUY.2_1", + "GUY.3_1", + "GUY.4_1", + "GUY.5_1", + "GUY.6_1", + "GUY.7_1", + "GUY.8_1", + "GUY.9_1" + ], + "HND": [ + "HND.11_1", + "HND.7_1" + ], + "HRV": [ + "HRV.10_1", + "HRV.11_1", + "HRV.12_1", + "HRV.13_1", + "HRV.14_1", + "HRV.15_1", + "HRV.16_1", + "HRV.17_1", + "HRV.18_1", + "HRV.19_1", + "HRV.1_1", + "HRV.20_1", + "HRV.21_1", + "HRV.2_1", + "HRV.3_1", + "HRV.5_1", + "HRV.6_1", + "HRV.7_1", + "HRV.8_1", + "HRV.9_1" + ], + "HTI": [ + "HTI.3_1" + ], + "IDN": [ + "IDN.33_1", + "IDN.3_1", + "IDN.7_1" + ], + "IMN": [ + "IMN.19_2", + "IMN.21_2", + "IMN.2_2", + "IMN.8_2" + ], + "IND": [ + "IND.1_1", + "IND.25_1" + ], + "IRN": [ + "IRN.10_1", + "IRN.16_1", + "IRN.1_1", + "IRN.21_1", + "IRN.22_1", + "IRN.23_1", + "IRN.24_1", + "IRN.26_1", + "IRN.27_1", + "IRN.29_1", + "IRN.4_1", + "IRN.5_1" + ], + "IRQ": [ + "IRQ.11_1", + "IRQ.12_1", + "IRQ.14_1", + "IRQ.16_1", + "IRQ.17_1", + "IRQ.18_1", + "IRQ.3_1", + "IRQ.4_1", + "IRQ.5_1", + "IRQ.6_1", + "IRQ.7_1", + "IRQ.8_1" + ], + "ISL": [ + "ISL.1_1", + "ISL.2_1", + "ISL.3_1", + "ISL.4_1", + "ISL.5_1", + "ISL.6_1", + "ISL.7_1", + "ISL.8_1" + ], + "ISR": [ + "ISR.1_1", + "ISR.2_1", + "ISR.4_1", + "ISR.5_1" + ], + "ITA": [ + "ITA.15_1", + "ITA.1_1", + "ITA.2_1" + ], + "JEY": [ + "JEY.2_1" + ], + "JOR": [ + "JOR.11_1", + "JOR.1_1" + ], + "JPN": [ + "JPN.11_1", + "JPN.12_1", + "JPN.13_1", + "JPN.15_1", + "JPN.18_1", + "JPN.1_1", + "JPN.20_1", + "JPN.22_1", + "JPN.27_1", + "JPN.30_1", + "JPN.33_1", + "JPN.36_1", + "JPN.37_1", + "JPN.38_1", + "JPN.39_1", + "JPN.40_1", + "JPN.41_1", + "JPN.46_1", + "JPN.47_1", + "JPN.4_1", + "JPN.6_1", + "JPN.7_1", + "JPN.8_1" + ], + "KAZ": [ + "KAZ.10_1", + "KAZ.11_1", + "KAZ.12_1", + "KAZ.13_1", + "KAZ.14_1", + "KAZ.1_1", + "KAZ.2_1", + "KAZ.3_1", + "KAZ.4_1", + "KAZ.5_1", + "KAZ.6_1", + "KAZ.7_1", + "KAZ.8_1", + "KAZ.9_1" + ], + "KGZ": [ + "KGZ.2_1", + "KGZ.6_1", + "KGZ.7_1" + ], + "KOR": [ + "KOR.10_1", + "KOR.11_1", + "KOR.12_1", + "KOR.13_1", + "KOR.14_1", + "KOR.15_1", + "KOR.17_1", + "KOR.1_1", + "KOR.2_1", + "KOR.3_1", + "KOR.4_1", + "KOR.5_1", + "KOR.6_1", + "KOR.7_1", + "KOR.8_1", + "KOR.9_1" + ], + "KWT": [ + "KWT.6_1" + ], + "LAO": [ + "LAO.13_1", + "LAO.14_1" + ], + "LBN": [ + "LBN.2_1", + "LBN.6_1" + ], + "LBR": [ + "LBR.3_1" + ], + "LBY": [ + "LBY.10_1", + "LBY.11_1", + "LBY.12_1", + "LBY.14_1", + "LBY.15_1", + "LBY.17_1", + "LBY.19_1", + "LBY.1_1", + "LBY.20_1", + "LBY.21_1", + "LBY.22_1", + "LBY.3_1", + "LBY.4_1", + "LBY.7_1", + "LBY.8_1", + "LBY.9_1" + ], + "LCA": [ + "LCA.10_1", + "LCA.1_1", + "LCA.2_1", + "LCA.3_1", + "LCA.4_1", + "LCA.5_1", + "LCA.6_1", + "LCA.7_1", + "LCA.8_1", + "LCA.9_1" + ], + "LIE": [ + "LIE.11_1" + ], + "LKA": [ + "LKA.10_1", + "LKA.11_1", + "LKA.12_1", + "LKA.13_1", + "LKA.14_1", + "LKA.15_1", + "LKA.16_1", + "LKA.17_1", + "LKA.18_1", + "LKA.19_1", + "LKA.1_1", + "LKA.20_1", + "LKA.21_1", + "LKA.22_1", + "LKA.23_1", + "LKA.24_1", + "LKA.25_1", + "LKA.2_1", + "LKA.3_1", + "LKA.4_1", + "LKA.5_1", + "LKA.6_1", + "LKA.7_1", + "LKA.8_1", + "LKA.9_1" + ], + "LTU": [ + "LTU.6_1", + "LTU.8_1" + ], + "LVA": [ + "LVA.1_1", + "LVA.2_1", + "LVA.4_1", + "LVA.5_1" + ], + "MAR": [ + "MAR.10_1", + "MAR.11_1", + "MAR.12_1", + "MAR.13_1", + "MAR.14_1", + "MAR.15_1", + "MAR.1_1", + "MAR.2_1", + "MAR.3_1", + "MAR.4_1", + "MAR.5_1", + "MAR.6_1", + "MAR.7_1", + "MAR.8_1", + "MAR.9_1" + ], + "MDA": [ + "MDA.19_1" + ], + "MDG": [ + "MDG.6_1" + ], + "MEX": [ + "MEX.16_1", + "MEX.30_1", + "MEX.7_1", + "MEX.9_1" + ], + "MHL": [ + "MHL.10_1", + "MHL.11_1", + "MHL.12_1", + "MHL.13_1", + "MHL.14_1", + "MHL.15_1", + "MHL.16_1", + "MHL.17_1", + "MHL.18_1", + "MHL.19_1", + "MHL.19_1", + "MHL.1_1", + "MHL.20_1", + "MHL.21_1", + "MHL.22_1", + "MHL.23_1", + "MHL.2_1", + "MHL.3_1", + "MHL.4_1", + "MHL.5_1", + "MHL.6_1", + "MHL.7_1", + "MHL.8_1", + "MHL.9_1" + ], + "MKD": [ + "MKD.10_1", + "MKD.11_1", + "MKD.12_1", + "MKD.13_1", + "MKD.14_1", + "MKD.15_1", + "MKD.16_1", + "MKD.17_1", + "MKD.18_1", + "MKD.19_1", + "MKD.1_1", + "MKD.20_1", + "MKD.21_1", + "MKD.23_1", + "MKD.24_1", + "MKD.25_1", + "MKD.26_1", + "MKD.27_1", + "MKD.28_1", + "MKD.29_1", + "MKD.2_1", + "MKD.30_1", + "MKD.31_1", + "MKD.32_1", + "MKD.33_1", + "MKD.34_1", + "MKD.35_1", + "MKD.36_1", + "MKD.37_1", + "MKD.38_1", + "MKD.39_1", + "MKD.3_1", + "MKD.40_1", + "MKD.41_1", + "MKD.42_1", + "MKD.43_1", + "MKD.44_1", + "MKD.45_1", + "MKD.46_1", + "MKD.47_1", + "MKD.48_1", + "MKD.49_1", + "MKD.4_1", + "MKD.50_1", + "MKD.51_1", + "MKD.52_1", + "MKD.54_1", + "MKD.55_1", + "MKD.56_1", + "MKD.57_1", + "MKD.58_1", + "MKD.59_1", + "MKD.5_1", + "MKD.60_1", + "MKD.61_1", + "MKD.62_1", + "MKD.63_1", + "MKD.64_1", + "MKD.65_1", + "MKD.66_1", + "MKD.67_1", + "MKD.68_1", + "MKD.69_1", + "MKD.6_1", + "MKD.70_1", + "MKD.71_1", + "MKD.72_1", + "MKD.73_1", + "MKD.74_1", + "MKD.75_1", + "MKD.76_1", + "MKD.77_1", + "MKD.78_1", + "MKD.79_1", + "MKD.7_1", + "MKD.81_1", + "MKD.83_1", + "MKD.84_1", + "MKD.85_1", + "MKD.8_1", + "MKD.9_1" + ], + "MLI": [ + "MLI.1_1", + "MLI.9_1" + ], + "MLT": [ + "MLT.1_1", + "MLT.2_1", + "MLT.3_1", + "MLT.4_1", + "MLT.5_1" + ], + "MMR": [ + "MMR.10_1" + ], + "MNE": [ + "MNE.17_1" + ], + "MNG": [ + "MNG.10_1", + "MNG.12_1", + "MNG.13_1", + "MNG.14_1", + "MNG.16_1", + "MNG.17_1", + "MNG.19_1", + "MNG.1_1", + "MNG.2_1", + "MNG.3_1", + "MNG.5_1", + "MNG.9_1" + ], + "MNP": [ + "MNP.1_1", + "MNP.2_1", + "MNP.3_1", + "MNP.4_1" + ], + "MOZ": [ + "MOZ.5_1", + "MOZ.6_1", + "MOZ.8_1" + ], + "MRT": [ + "MRT.10_1", + "MRT.12_1", + "MRT.6_1", + "MRT.7_1" + ], + "MSR": [ + "MSR.1_1", + "MSR.2_1", + "MSR.3_1" + ], + "MTQ": [ + "MTQ.1_1", + "MTQ.2_1", + "MTQ.3_1", + "MTQ.4_1" + ], + "MUS": [ + "MUS.10_1", + "MUS.11_1", + "MUS.1_1" + ], + "MWI": [ + "MWI.10_1", + "MWI.11_1", + "MWI.12_1", + "MWI.13_1", + "MWI.14_1", + "MWI.15_1", + "MWI.16_1", + "MWI.17_1", + "MWI.18_1", + "MWI.19_1", + "MWI.1_1", + "MWI.20_1", + "MWI.21_1", + "MWI.22_1", + "MWI.23_1", + "MWI.24_1", + "MWI.25_1", + "MWI.26_1", + "MWI.27_1", + "MWI.28_1", + "MWI.2_1", + "MWI.3_1", + "MWI.4_1", + "MWI.5_1", + "MWI.6_1", + "MWI.7_1", + "MWI.8_1", + "MWI.9_1" + ], + "MYS": [ + "MYS.12_1", + "MYS.16_1" + ], + "MYT": [ + "MYT.10_1", + "MYT.11_1", + "MYT.12_1", + "MYT.13_1", + "MYT.14_1", + "MYT.15_1", + "MYT.16_1", + "MYT.17_1", + "MYT.1_1", + "MYT.2_1", + "MYT.3_1", + "MYT.4_1", + "MYT.5_1", + "MYT.6_1", + "MYT.7_1", + "MYT.8_1", + "MYT.9_1" + ], + "NAM": [ + "NAM.13_1" + ], + "NCL": [ + "NCL.1_1", + "NCL.2_1", + "NCL.3_1" + ], + "NER": [ + "NER.5_1", + "NER.7_1" + ], + "NGA": [ + "NGA.26_1", + "NGA.3_1" + ], + "NIC": [ + "NIC.10_1", + "NIC.1_1", + "NIC.2_1" + ], + "NLD": [ + "NLD.13_1", + "NLD.3_1", + "NLD.6_1" + ], + "NOR": [ + "NOR.15_1", + "NOR.2_1", + "NOR.8_1", + "NOR.9_1" + ], + "NPL": [ + "NPL.1_1", + "NPL.2_1", + "NPL.3_1", + "NPL.4_1", + "NPL.5_1" + ], + "NRU": [ + "NRU.10_1", + "NRU.11_1", + "NRU.12_1", + "NRU.13_1", + "NRU.14_1", + "NRU.1_1", + "NRU.2_1", + "NRU.3_1", + "NRU.4_1", + "NRU.5_1", + "NRU.6_1", + "NRU.7_1", + "NRU.8_1", + "NRU.9_1" + ], + "NZL": [ + "NZL.10_1", + "NZL.13_1", + "NZL.4_1" + ], + "OMN": [ + "OMN.1_1", + "OMN.2_1", + "OMN.3_1", + "OMN.4_1", + "OMN.7_1", + "OMN.8_1" + ], + "PAK": [ + "PAK.3_1", + "PAK.4_1" + ], + "PAN": [ + "PAN.10_1" + ], + "PER": [ + "PER.15_1", + "PER.16_1" + ], + "PHL": [ + "PHL.10_1", + "PHL.11_1", + "PHL.12_1", + "PHL.13_1", + "PHL.14_1", + "PHL.15_1", + "PHL.16_1", + "PHL.17_1", + "PHL.18_1", + "PHL.19_1", + "PHL.1_1", + "PHL.20_1", + "PHL.21_1", + "PHL.22_1", + "PHL.23_1", + "PHL.24_1", + "PHL.25_1", + "PHL.26_1", + "PHL.27_1", + "PHL.28_1", + "PHL.29_1", + "PHL.2_1", + "PHL.30_1", + "PHL.31_1", + "PHL.32_1", + "PHL.33_1", + "PHL.34_1", + "PHL.35_1", + "PHL.36_1", + "PHL.37_1", + "PHL.38_1", + "PHL.39_1", + "PHL.3_1", + "PHL.40_1", + "PHL.41_1", + "PHL.42_1", + "PHL.43_1", + "PHL.44_1", + "PHL.45_1", + "PHL.46_1", + "PHL.47_1", + "PHL.48_1", + "PHL.49_1", + "PHL.4_1", + "PHL.50_1", + "PHL.51_1", + "PHL.52_1", + "PHL.53_1", + "PHL.54_1", + "PHL.55_1", + "PHL.56_1", + "PHL.57_1", + "PHL.58_1", + "PHL.59_1", + "PHL.5_1", + "PHL.60_1", + "PHL.61_1", + "PHL.62_1", + "PHL.63_1", + "PHL.64_1", + "PHL.65_1", + "PHL.66_1", + "PHL.67_1", + "PHL.68_1", + "PHL.69_1", + "PHL.6_1", + "PHL.70_1", + "PHL.71_1", + "PHL.72_1", + "PHL.73_1", + "PHL.74_1", + "PHL.75_1", + "PHL.76_1", + "PHL.77_1", + "PHL.78_1", + "PHL.79_1", + "PHL.7_1", + "PHL.80_1", + "PHL.81_1", + "PHL.8_1", + "PHL.9_1" + ], + "PLW": [ + "PLW.10_1", + "PLW.11_1", + "PLW.12_1", + "PLW.13_1", + "PLW.14_1", + "PLW.15_1", + "PLW.16_1", + "PLW.1_1", + "PLW.2_1", + "PLW.3_1", + "PLW.4_1", + "PLW.5_1", + "PLW.6_1", + "PLW.7_1", + "PLW.8_1", + "PLW.9_1" + ], + "PNG": [ + "PNG.17_1" + ], + "POL": [ + "POL.3_1", + "POL.4_1", + "POL.6_1" + ], + "PRI": [ + "PRI.10_1", + "PRI.12_1", + "PRI.13_1", + "PRI.14_1", + "PRI.15_1", + "PRI.16_1", + "PRI.17_1", + "PRI.18_1", + "PRI.19_1", + "PRI.1_1", + "PRI.20_1", + "PRI.21_1", + "PRI.22_1", + "PRI.23_1", + "PRI.24_1", + "PRI.25_1", + "PRI.26_1", + "PRI.27_1", + "PRI.28_1", + "PRI.29_1", + "PRI.2_1", + "PRI.31_1", + "PRI.32_1", + "PRI.33_1", + "PRI.34_1", + "PRI.35_1", + "PRI.37_1", + "PRI.38_1", + "PRI.39_1", + "PRI.40_1", + "PRI.41_1", + "PRI.42_1", + "PRI.43_1", + "PRI.44_1", + "PRI.45_1", + "PRI.46_1", + "PRI.47_1", + "PRI.48_1", + "PRI.49_1", + "PRI.4_1", + "PRI.51_1", + "PRI.52_1", + "PRI.53_1", + "PRI.54_1", + "PRI.55_1", + "PRI.56_1", + "PRI.57_1", + "PRI.59_1", + "PRI.5_1", + "PRI.60_1", + "PRI.61_1", + "PRI.62_1", + "PRI.63_1", + "PRI.64_1", + "PRI.66_1", + "PRI.67_1", + "PRI.68_1", + "PRI.69_1", + "PRI.6_1", + "PRI.70_1", + "PRI.71_1", + "PRI.72_1", + "PRI.73_1", + "PRI.74_1", + "PRI.75_1", + "PRI.76_1", + "PRI.77_1", + "PRI.78_1", + "PRI.8_1", + "PRI.9_1" + ], + "PRK": [ + "PRK.11_1", + "PRK.12_1", + "PRK.13_1", + "PRK.14_1", + "PRK.6_1", + "PRK.8_1" + ], + "PRY": [ + "PRY.4_1" + ], + "PSE": [ + "PSE.2_1" + ], + "PYF": [ + "PYF.1_1", + "PYF.2_1", + "PYF.3_1", + "PYF.4_1", + "PYF.5_1" + ], + "QAT": [ + "QAT.2_1", + "QAT.3_1", + "QAT.6_1" + ], + "REU": [ + "REU.1_1", + "REU.2_1", + "REU.3_1", + "REU.4_1" + ], + "ROU": [ + "ROU.10_1", + "ROU.21_1" + ], + "RUS": [ + "RUS.10_1", + "RUS.11_1", + "RUS.12_1", + "RUS.13_1", + "RUS.14_1", + "RUS.15_1", + "RUS.16_1", + "RUS.17_1", + "RUS.18_1", + "RUS.19_1", + "RUS.1_1", + "RUS.20_1", + "RUS.21_1", + "RUS.22_1", + "RUS.23_1", + "RUS.24_1", + "RUS.25_1", + "RUS.26_1", + "RUS.27_1", + "RUS.28_1", + "RUS.29_1", + "RUS.2_1", + "RUS.30_1", + "RUS.31_1", + "RUS.32_1", + "RUS.33_1", + "RUS.34_1", + "RUS.35_1", + "RUS.36_1", + "RUS.37_1", + "RUS.38_1", + "RUS.39_1", + "RUS.3_1", + "RUS.40_1", + "RUS.41_1", + "RUS.42_1", + "RUS.43_1", + "RUS.45_1", + "RUS.46_1", + "RUS.47_1", + "RUS.48_1", + "RUS.49_1", + "RUS.4_1", + "RUS.50_1", + "RUS.51_1", + "RUS.52_1", + "RUS.53_1", + "RUS.54_1", + "RUS.55_1", + "RUS.56_1", + "RUS.57_1", + "RUS.58_1", + "RUS.59_1", + "RUS.5_1", + "RUS.60_1", + "RUS.61_1", + "RUS.62_1", + "RUS.63_1", + "RUS.64_1", + "RUS.65_1", + "RUS.66_1", + "RUS.67_1", + "RUS.68_1", + "RUS.69_1", + "RUS.6_1", + "RUS.70_1", + "RUS.71_1", + "RUS.72_1", + "RUS.73_1", + "RUS.74_1", + "RUS.75_1", + "RUS.76_1", + "RUS.77_1", + "RUS.78_1", + "RUS.79_1", + "RUS.7_1", + "RUS.80_1", + "RUS.81_1", + "RUS.82_1", + "RUS.83_1", + "RUS.8_1", + "RUS.9_1" + ], + "SAU": [ + "SAU.6_1", + "SAU.9_1" + ], + "SDN": [ + "SDN.12_1", + "SDN.14_1", + "SDN.15_1", + "SDN.16_1", + "SDN.17_1", + "SDN.1_1", + "SDN.2_1", + "SDN.4_1", + "SDN.5_1", + "SDN.8_1", + "SDN.9_1" + ], + "SGP": [ + "SGP.1_1", + "SGP.2_1", + "SGP.3_1", + "SGP.5_1" + ], + "SJM": [ + "SJM.1_1", + "SJM.2_1" + ], + "SLB": [ + "SLB.10_1", + "SLB.1_1", + "SLB.2_1", + "SLB.3_1", + "SLB.4_1", + "SLB.5_1", + "SLB.6_1", + "SLB.7_1", + "SLB.8_1", + "SLB.9_1" + ], + "SLE": [ + "SLE.4_1" + ], + "SMR": [ + "SMR.1_1", + "SMR.2_1", + "SMR.3_1", + "SMR.4_1", + "SMR.5_1", + "SMR.6_1", + "SMR.7_1", + "SMR.8_1", + "SMR.9_1" + ], + "SOM": [ + "SOM.10_1", + "SOM.14_1", + "SOM.15_1", + "SOM.3_1", + "SOM.6_1", + "SOM.8_1", + "SOM.9_1" + ], + "SSD": [ + "SSD.10_1", + "SSD.3_1", + "SSD.5_1", + "SSD.8_1", + "SSD.9_1" + ], + "SVK": [ + "SVK.1_1", + "SVK.2_1", + "SVK.3_1", + "SVK.4_1", + "SVK.5_1", + "SVK.6_1", + "SVK.7_1", + "SVK.8_1" + ], + "SVN": [ + "SVN.3_1" + ], + "SWE": [ + "SWE.10_1", + "SWE.11_1", + "SWE.12_1", + "SWE.13_1", + "SWE.14_1", + "SWE.15_1", + "SWE.16_1", + "SWE.17_1", + "SWE.18_1", + "SWE.19_1", + "SWE.1_1", + "SWE.20_1", + "SWE.21_1", + "SWE.2_1", + "SWE.3_1", + "SWE.4_1", + "SWE.5_1", + "SWE.6_1", + "SWE.7_1", + "SWE.8_1", + "SWE.9_1" + ], + "SYC": [ + "SYC.14_1", + "SYC.15_1", + "SYC.16_1", + "SYC.20_1", + "SYC.25_1", + "SYC.7_1" + ], + "SYR": [ + "SYR.11_1", + "SYR.8_1" + ], + "TCA": [ + "TCA.5_1", + "TCA.6_1" + ], + "TGO": [ + "TGO.1_1" + ], + "THA": [ + "THA.15_1", + "THA.3_1", + "THA.46_1", + "THA.49_1", + "THA.4_1", + "THA.50_1", + "THA.57_1", + "THA.59_1", + "THA.71_1" + ], + "TJK": [ + "TJK.2_1", + "TJK.5_1" + ], + "TKL": [ + "TKL.1_1", + "TKL.2_1", + "TKL.3_1" + ], + "TKM": [ + "TKM.2_1", + "TKM.4_1", + "TKM.6_1" + ], + "TLS": [ + "TLS.3_1" + ], + "TON": [ + "TON.1_1", + "TON.2_1", + "TON.3_1", + "TON.4_1", + "TON.5_1" + ], + "TUN": [ + "TUN.13_1", + "TUN.14_1", + "TUN.3_1" + ], + "TUR": [ + "TUR.13_1", + "TUR.27_1", + "TUR.29_1", + "TUR.38_1", + "TUR.49_1", + "TUR.58_1", + "TUR.81_1" + ], + "TUV": [ + "TUV.1_1", + "TUV.2_1", + "TUV.3_1", + "TUV.4_1", + "TUV.5_1", + "TUV.6_1", + "TUV.7_1", + "TUV.8_1", + "TUV.9_1" + ], + "TWN": [ + "TWN.1_1", + "TWN.2_1", + "TWN.3_1", + "TWN.4_1", + "TWN.5_1", + "TWN.6_1", + "TWN.7_1" + ], + "TZA": [ + "TZA.31_1" + ], + "UGA": [ + "UGA.24_1", + "UGA.25_1", + "UGA.31_1", + "UGA.32_1", + "UGA.53_1" + ], + "UKR": [ + "?", + "UKR.10_1", + "UKR.11_1", + "UKR.12_1", + "UKR.13_1", + "UKR.14_1", + "UKR.15_1", + "UKR.16_1", + "UKR.17_1", + "UKR.18_1", + "UKR.19_1", + "UKR.1_1", + "UKR.20_1", + "UKR.21_1", + "UKR.22_1", + "UKR.23_1", + "UKR.24_1", + "UKR.25_1", + "UKR.26_1", + "UKR.27_1", + "UKR.2_1", + "UKR.3_1", + "UKR.5_1", + "UKR.6_1", + "UKR.7_1", + "UKR.8_1", + "UKR.9_1" + ], + "UMI": [ + "UMI.1_1", + "UMI.2_1", + "UMI.3_1", + "UMI.4_1", + "UMI.5_1", + "UMI.6_1", + "UMI.7_1", + "UMI.8_1", + "UMI.9_1" + ], + "UZB": [ + "UZB.10_1", + "UZB.11_1", + "UZB.12_1", + "UZB.13_1", + "UZB.14_1", + "UZB.1_1", + "UZB.2_1", + "UZB.3_1", + "UZB.4_1", + "UZB.5_1", + "UZB.6_1", + "UZB.7_1" + ], + "VIR": [ + "VIR.1_1", + "VIR.2_1", + "VIR.3_1" + ], + "VNM": [ + "VNM.15_1", + "VNM.16_1", + "VNM.17_1", + "VNM.18_1", + "VNM.19_1", + "VNM.20_1", + "VNM.37_1", + "VNM.40_1", + "VNM.4_1", + "VNM.8_1" + ], + "WLF": [ + "WLF.1_1", + "WLF.2_1", + "WLF.3_1" + ], + "WSM": [ + "WSM.10_1", + "WSM.11_1", + "WSM.1_1", + "WSM.2_1", + "WSM.3_1", + "WSM.4_1", + "WSM.5_1", + "WSM.6_1", + "WSM.7_1", + "WSM.8_1", + "WSM.9_1" + ], + "XAD": [ + "XAD.1_1", + "XAD.2_1" + ], + "XKO": [ + "XKO.1_1", + "XKO.2_1", + "XKO.3_1", + "XKO.4_1", + "XKO.5_1", + "XKO.6_1", + "XKO.7_1" + ], + "YEM": [ + "YEM.19_1", + "YEM.9_1" + ], + "Z01": [ + "Z01.14_1" + ], + "Z02": [ + "Z02.28_1" + ], + "Z03": [ + "Z03.28_1", + "Z03.29_1" + ], + "Z04": [ + "Z04.13_1" + ], + "Z05": [ + "Z05.35_1" + ], + "Z06": [ + "Z06.1_1", + "Z06.6_1" + ], + "Z07": [ + "Z07.3_1" + ], + "Z08": [ + "Z08.29_1" + ], + "Z09": [ + "Z09.13_1", + "Z09.35_1" + ], + "ZMB": [ + "ZMB.6_1" + ], + "ZNC": [ + "ZNC.1_1", + "ZNC.2_1", + "ZNC.3_1", + "ZNC.4_1", + "ZNC.5_1" + ] + }, + "unmatched_world_bank": { + "ABW": [ + "ABW001" + ], + "AFG": [ + "AFG027", + "AFG030" + ], + "AIA": [ + "AIA001", + "AIA002", + "AIA003", + "AIA004", + "AIA005", + "AIA006", + "AIA007", + "AIA008", + "AIA009", + "AIA010" + ], + "ALA": [ + "ALA001" + ], + "ALB": [ + "ALB002", + "ALB003", + "ALB004", + "ALB010", + "ALB011", + "ALB012", + "ALB013", + "ALB015", + "ALB016", + "ALB018", + "ALB020", + "ALB021", + "ALB022", + "ALB023", + "ALB024", + "ALB025", + "ALB026", + "ALB027", + "ALB028", + "ALB029", + "ALB030", + "ALB032", + "ALB033", + "ALB035" + ], + "ARE": [ + "ARE007" + ], + "ARG": [ + "ARG002" + ], + "ARM": [ + "ARM004", + "ARM011" + ], + "ASM": [ + "ASM001" + ], + "ATF": [ + "ATF001", + "ATF002", + "ATF003", + "ATF004", + "ATF005", + "ATF006" + ], + "ATG": [ + "ATG003", + "ATG004", + "ATG005", + "ATG006", + "ATG007", + "ATG008" + ], + "AUS": [ + "AUS001", + "AUS006" + ], + "AZE": [ + "AZE002", + "AZE003", + "AZE004", + "AZE005", + "AZE006", + "AZE007", + "AZE008", + "AZE009", + "AZE010", + "AZE011", + "AZE012", + "AZE013", + "AZE014", + "AZE015", + "AZE016", + "AZE017", + "AZE018", + "AZE019", + "AZE020", + "AZE021", + "AZE022", + "AZE023", + "AZE024", + "AZE025", + "AZE026", + "AZE027", + "AZE028", + "AZE029", + "AZE030", + "AZE031", + "AZE032", + "AZE033", + "AZE034", + "AZE035", + "AZE036", + "AZE037", + "AZE038", + "AZE039", + "AZE040", + "AZE041", + "AZE042", + "AZE044", + "AZE045", + "AZE046", + "AZE047", + "AZE049", + "AZE050", + "AZE051", + "AZE052", + "AZE053", + "AZE054", + "AZE055", + "AZE056", + "AZE057", + "AZE058", + "AZE059", + "AZE060", + "AZE061", + "AZE062", + "AZE063", + "AZE064", + "AZE065", + "AZE066", + "AZE067", + "AZE068", + "AZE069", + "AZE070", + "AZE071", + "AZE072", + "AZE073", + "AZE074" + ], + "BDI": [ + "BDI002", + "BDI018", + "BDI020" + ], + "BEL": [ + "BEL001", + "BEL002", + "BEL003" + ], + "BEN": [ + "BEN002", + "BEN006" + ], + "BFA": [ + "BFA009" + ], + "BGD": [ + "BGD001", + "BGD002" + ], + "BGR": [ + "BGR006", + "BGR008", + "BGR020", + "BGR022", + "BGR028" + ], + "BHR": [ + "BHR001", + "BHR002", + "BHR003", + "BHR004", + "BHR005", + "BHR006", + "BHR007", + "BHR010", + "BHR011", + "BHR012" + ], + "BHS": [ + "BHS001" + ], + "BIH": [ + "BIH001", + "BIH002" + ], + "BLM": [ + "BLM001" + ], + "BLR": [ + "BLR004", + "BLR005" + ], + "BMU": [ + "BMU001" + ], + "BRA": [ + "BRA014", + "BRA015", + "BRA016", + "BRA017" + ], + "BRB": [ + "BRB002", + "BRB003", + "BRB004", + "BRB005", + "BRB006", + "BRB007", + "BRB008", + "BRB009", + "BRB010", + "BRB011" + ], + "BVT": [ + "BVT001" + ], + "BWA": [ + "BWA007" + ], + "CAF": [ + "CAF001", + "CAF002", + "CAF003", + "CAF004", + "CAF005", + "CAF006", + "CAF007" + ], + "CCK": [ + "CCK001" + ], + "CHE": [ + "CHE012" + ], + "CHL": [ + "CHL001", + "CHL004", + "CHL008", + "CHL013" + ], + "CHN": [ + "CHN002", + "CHN003", + "CHN007", + "CHN019", + "CHN020", + "CHN024", + "CHN027", + "CHN028", + "CHN029", + "TWN001" + ], + "COG": [ + "COG010" + ], + "COK": [ + "COK001" + ], + "COL": [ + "COL004" + ], + "COM": [ + "COM001", + "COM002", + "COM003" + ], + "CUB": [ + "CUB001" + ], + "CUW": [ + "CUW001" + ], + "CXR": [ + "CXR001" + ], + "CYM": [ + "CYM001", + "CYM002", + "CYM003" + ], + "CYP": [ + "CYP001", + "CYP002", + "CYP003", + "CYP004", + "CYP005", + "CYP006" + ], + "CZE": [ + "CZE003", + "CZE004", + "CZE005", + "CZE007", + "CZE008" + ], + "DEU": [ + "DEU001", + "DEU016" + ], + "DJI": [ + "DJI001", + "DJI002", + "DJI003", + "DJI005" + ], + "DMA": [ + "DMA001", + "DMA002", + "DMA003", + "DMA004", + "DMA005", + "DMA006", + "DMA007", + "DMA008", + "DMA009", + "DMA010" + ], + "DNK": [ + "DNK001", + "DNK002", + "DNK003", + "DNK004", + "DNK005" + ], + "DOM": [ + "DOM002", + "DOM007", + "DOM008", + "DOM015" + ], + "DZA": [ + "DZA011", + "DZA041" + ], + "EGY": [ + "EGY001", + "EGY002", + "EGY004", + "EGY005", + "EGY006", + "EGY007", + "EGY008", + "EGY009", + "EGY010", + "EGY011", + "EGY012", + "EGY013", + "EGY014", + "EGY015", + "EGY017", + "EGY018", + "EGY019", + "EGY020", + "EGY021", + "EGY022", + "EGY023", + "EGY024", + "EGY025", + "EGY026", + "EGY027" + ], + "ERI": [ + "ERI002", + "ERI004", + "ERI007" + ], + "ESP": [ + "ESP004", + "ESP013", + "ESP016", + "ESP021", + "ESP022", + "ESP023", + "ESP024", + "ESP025" + ], + "EST": [ + "EST002", + "EST003", + "EST004", + "EST005", + "EST006", + "EST007", + "EST008", + "EST009", + "EST010", + "EST011", + "EST012", + "EST013", + "EST014", + "EST015", + "EST016" + ], + "ETH": [ + "ETH001", + "ETH004", + "ETH006", + "ETH007", + "ETH009", + "ETH010", + "ETH012" + ], + "FRA": [ + "FRA001", + "FRA002", + "FRA003", + "FRA004", + "FRA005", + "FRA006", + "FRA008", + "FRA009", + "FRA011", + "FRA012", + "FRA014", + "FRA015", + "FRA016", + "FRA017", + "FRA018", + "FRA020", + "FRA021", + "FRA023" + ], + "FRO": [ + "FRO001" + ], + "FSM": [ + "FSM001" + ], + "GAB": [ + "GAB009" + ], + "GBR": [ + "GBR001", + "GBR002", + "GBR005" + ], + "GEO": [ + "GEO001", + "GEO002" + ], + "GGY": [ + "GGY004", + "GGY007" + ], + "GHA": [ + "GHA005", + "GHA014", + "GHA015" + ], + "GIB": [ + "GIB001" + ], + "GLP": [ + "GLP001" + ], + "GMB": [ + "GMB001", + "GMB002", + "GMB003", + "GMB004", + "GMB008" + ], + "GNB": [ + "GNB008" + ], + "GRC": [ + "GRC001", + "GRC002", + "GRC003", + "GRC004", + "GRC005", + "GRC006", + "GRC007", + "GRC008", + "GRC009", + "GRC010", + "GRC011", + "GRC012", + "GRC013" + ], + "GRD": [ + "GRD001", + "GRD002", + "GRD003", + "GRD004", + "GRD005", + "GRD006", + "GRD007" + ], + "GRL": [ + "GRL001", + "GRL002", + "GRL003" + ], + "GTM": [ + "GTM013" + ], + "GUF": [ + "GUF001" + ], + "GUM": [ + "GUM001" + ], + "GUY": [ + "GUY001", + "GUY002", + "GUY003", + "GUY004", + "GUY005", + "GUY006", + "GUY007", + "GUY008", + "GUY009", + "GUY010" + ], + "HKG": [ + "HKG001" + ], + "HMD": [ + "HMD001" + ], + "HND": [ + "HND010", + "HND013", + "HND016" + ], + "HRV": [ + "HRV001", + "HRV002", + "HRV004", + "HRV005", + "HRV006", + "HRV007", + "HRV008", + "HRV009", + "HRV010", + "HRV011", + "HRV012", + "HRV013", + "HRV014", + "HRV015", + "HRV016", + "HRV017", + "HRV018", + "HRV019", + "HRV020", + "HRV021" + ], + "HTI": [ + "HTI001" + ], + "IDN": [ + "IDN009", + "IDN011", + "IDN014", + "IDN034", + "IDN036", + "IDN037", + "IDN038" + ], + "IMN": [ + "IMN002", + "IMN010", + "IMN012", + "IMN015", + "IMN021", + "IMN023", + "IMN024" + ], + "IND": [ + "IND001", + "IND010" + ], + "IRN": [ + "IRN003", + "IRN004", + "IRN007", + "IRN008", + "IRN011", + "IRN016", + "IRN018", + "IRN024", + "IRN026" + ], + "IRQ": [ + "IRQ003", + "IRQ004", + "IRQ005", + "IRQ006", + "IRQ010", + "IRQ011", + "IRQ012", + "IRQ013", + "IRQ015", + "IRQ016", + "IRQ017", + "IRQ018" + ], + "ISL": [ + "ISL001", + "ISL002", + "ISL003", + "ISL004", + "ISL005", + "ISL006", + "ISL007", + "ISL008", + "ISL009", + "ISL010", + "ISL011", + "ISL012", + "ISL013", + "ISL014", + "ISL015", + "ISL016", + "ISL017", + "ISL018", + "ISL019", + "ISL020", + "ISL021", + "ISL022", + "ISL023" + ], + "ISR": [ + "ISR001", + "ISR004", + "ISR005" + ], + "ITA": [ + "ITA001", + "ITA013", + "ITA015" + ], + "JEY": [ + "JEY001" + ], + "JOR": [ + "JOR001", + "JOR011" + ], + "JPN": [ + "JPN001", + "JPN007", + "JPN008", + "JPN009", + "JPN010", + "JPN011", + "JPN012", + "JPN014", + "JPN017", + "JPN019", + "JPN021", + "JPN026", + "JPN031", + "JPN032", + "JPN035", + "JPN036", + "JPN037", + "JPN038", + "JPN039", + "JPN040", + "JPN041", + "JPN046", + "JPN047" + ], + "KAZ": [ + "KAZ001", + "KAZ002", + "KAZ003", + "KAZ004", + "KAZ005", + "KAZ006", + "KAZ007", + "KAZ008", + "KAZ009", + "KAZ010", + "KAZ011", + "KAZ012", + "KAZ013", + "KAZ014", + "KAZ015" + ], + "KGZ": [ + "KGZ005" + ], + "KHM": [ + "KHM000" + ], + "KIR": [ + "KIR001" + ], + "KOR": [ + "KOR001", + "KOR002", + "KOR003", + "KOR004", + "KOR005", + "KOR006", + "KOR007", + "KOR008", + "KOR009", + "KOR010", + "KOR011", + "KOR012", + "KOR014", + "KOR015" + ], + "LAO": [ + "LAO013", + "LAO014" + ], + "LBN": [ + "LBN002", + "LBN006" + ], + "LBR": [ + "LBR003" + ], + "LBY": [ + "LBY001", + "LBY002", + "LBY003", + "LBY006", + "LBY008", + "LBY009", + "LBY010", + "LBY011", + "LBY013", + "LBY014", + "LBY015", + "LBY017", + "LBY019", + "LBY020", + "LBY021", + "LBY022", + "LBY023", + "LBY024", + "LBY025" + ], + "LCA": [ + "LCA001", + "LCA002", + "LCA003", + "LCA004", + "LCA005", + "LCA006", + "LCA007", + "LCA008", + "LCA009" + ], + "LIE": [ + "LIE011" + ], + "LKA": [ + "LKA001", + "LKA003", + "LKA004", + "LKA005", + "LKA006", + "LKA007", + "LKA008", + "LKA009", + "LKA010" + ], + "LTU": [ + "LTU006", + "LTU008" + ], + "LVA": [ + "LVA001", + "LVA002", + "LVA003", + "LVA004", + "LVA005", + "LVA006", + "LVA007", + "LVA008", + "LVA009", + "LVA010", + "LVA011", + "LVA012", + "LVA013", + "LVA014", + "LVA015", + "LVA016", + "LVA017", + "LVA018", + "LVA019", + "LVA020", + "LVA021", + "LVA022", + "LVA023", + "LVA024", + "LVA025", + "LVA026", + "LVA027", + "LVA028", + "LVA029", + "LVA030", + "LVA031", + "LVA032", + "LVA033", + "LVA034", + "LVA035", + "LVA036", + "LVA037", + "LVA038", + "LVA039", + "LVA040", + "LVA041", + "LVA042", + "LVA043", + "LVA044", + "LVA045", + "LVA046", + "LVA047", + "LVA048", + "LVA049", + "LVA050", + "LVA051", + "LVA052", + "LVA053", + "LVA054", + "LVA055", + "LVA056", + "LVA057", + "LVA058", + "LVA059", + "LVA060", + "LVA061", + "LVA062", + "LVA063", + "LVA064", + "LVA065", + "LVA066", + "LVA067", + "LVA068", + "LVA069", + "LVA070", + "LVA071", + "LVA072", + "LVA073", + "LVA074", + "LVA075", + "LVA076", + "LVA077", + "LVA078", + "LVA079", + "LVA080", + "LVA081", + "LVA082", + "LVA083", + "LVA084", + "LVA086", + "LVA087", + "LVA088", + "LVA089", + "LVA090", + "LVA091", + "LVA092", + "LVA093", + "LVA094", + "LVA095", + "LVA096", + "LVA097", + "LVA098", + "LVA099", + "LVA100", + "LVA101", + "LVA102", + "LVA103", + "LVA104", + "LVA105", + "LVA106", + "LVA107", + "LVA108", + "LVA109", + "LVA110", + "LVA111", + "LVA112", + "LVA113", + "LVA114", + "LVA115", + "LVA116", + "LVA117", + "LVA118", + "LVA119" + ], + "MAC": [ + "MAC001" + ], + "MAF": [ + "MAF001" + ], + "MAR": [ + "MAR001", + "MAR002", + "MAR003", + "MAR004", + "MAR005", + "MAR006", + "MAR007", + "MAR008", + "MAR009", + "MAR010", + "MAR011" + ], + "MCO": [ + "MCO001" + ], + "MDA": [ + "MDA037" + ], + "MDG": [ + "MDG034" + ], + "MDV": [ + "MDV001", + "MDV002", + "MDV003", + "MDV004", + "MDV005", + "MDV006", + "MDV007", + "MDV008", + "MDV009", + "MDV010", + "MDV011", + "MDV012", + "MDV013", + "MDV014", + "MDV015", + "MDV016", + "MDV017", + "MDV018", + "MDV019", + "MDV020" + ], + "MEX": [ + "MEX005", + "MEX009", + "MEX016", + "MEX030" + ], + "MHL": [ + "MHL001" + ], + "MKD": [ + "MKD001", + "MKD002", + "MKD003", + "MKD004", + "MKD005", + "MKD006", + "MKD007", + "MKD008" + ], + "MLI": [ + "MLI001", + "MLI009" + ], + "MLT": [ + "MLT001", + "MLT002", + "MLT003", + "MLT004", + "MLT005", + "MLT006", + "MLT007", + "MLT008", + "MLT009", + "MLT010", + "MLT011", + "MLT012", + "MLT013", + "MLT014", + "MLT015", + "MLT016", + "MLT017", + "MLT018", + "MLT019", + "MLT020", + "MLT021", + "MLT022", + "MLT023", + "MLT024", + "MLT025", + "MLT026", + "MLT027", + "MLT028", + "MLT029", + "MLT030", + "MLT031", + "MLT032", + "MLT033", + "MLT034", + "MLT035", + "MLT036", + "MLT037", + "MLT038", + "MLT039", + "MLT040", + "MLT041", + "MLT042", + "MLT043", + "MLT044", + "MLT045", + "MLT046", + "MLT047", + "MLT048", + "MLT049", + "MLT050", + "MLT051", + "MLT052", + "MLT053", + "MLT054", + "MLT055", + "MLT056", + "MLT057", + "MLT058", + "MLT059", + "MLT060", + "MLT061", + "MLT062", + "MLT063", + "MLT064", + "MLT065", + "MLT066", + "MLT067" + ], + "MMR": [ + "MMR010" + ], + "MNE": [ + "MNE017" + ], + "MNG": [ + "MNG022", + "MNG023", + "MNG045", + "MNG061", + "MNG062", + "MNG064", + "MNG065", + "MNG067", + "MNG081", + "MNG082", + "MNG083", + "MNG084" + ], + "MNP": [ + "MNP001" + ], + "MOZ": [ + "MOZ002", + "MOZ006", + "MOZ008", + "MOZXXX" + ], + "MRT": [ + "MRT006", + "MRT007", + "MRT012", + "MRT014", + "MRT015", + "MRT016" + ], + "MSR": [ + "MSR001", + "MSR002", + "MSR003", + "MSR004", + "MSR005", + "MSR006", + "MSR007" + ], + "MTQ": [ + "MTQ001" + ], + "MUS": [ + "MUS001" + ], + "MWI": [ + "MWI001", + "MWI002", + "MWI003", + "MWI004" + ], + "MYS": [ + "MYS015" + ], + "MYT": [ + "MYT001" + ], + "NAM": [ + "NAM001" + ], + "NCL": [ + "NCL001" + ], + "NER": [ + "NER006", + "NER008" + ], + "NFK": [ + "NFK001" + ], + "NGA": [ + "NGA003", + "NGA026" + ], + "NIC": [ + "NIC014", + "NIC015" + ], + "NIU": [ + "NIU001" + ], + "NLD": [ + "NLD003" + ], + "NOR": [ + "NOR007", + "NOR008", + "NOR010", + "NOR014" + ], + "NPL": [ + "NPL001", + "NPL002", + "NPL003", + "NPL004", + "NPL005", + "NPL006", + "NPL007" + ], + "NRU": [ + "NRU001" + ], + "NZL": [ + "NZL99" + ], + "OMN": [ + "OMN001", + "OMN002", + "OMN003", + "OMN009" + ], + "PAK": [ + "PAK003" + ], + "PAN": [ + "PAN010" + ], + "PCN": [ + "PCN001" + ], + "PER": [ + "PER015" + ], + "PHL": [ + "PHL001", + "PHL002", + "PHL003", + "PHL004", + "PHL005", + "PHL006", + "PHL007", + "PHL008", + "PHL009", + "PHL010", + "PHL011", + "PHL012", + "PHL013", + "PHL014", + "PHL015", + "PHL016", + "PHL017" + ], + "PLW": [ + "PLW001" + ], + "PNG": [ + "PNG017" + ], + "POL": [ + "POL003", + "POL004", + "POL006" + ], + "PRK": [ + "PRK006", + "PRK010", + "PRK011" + ], + "PSE": [ + "PSE001", + "PSE002", + "PSE003", + "PSE004", + "PSE005", + "PSE007", + "PSE008", + "PSE009", + "PSE010", + "PSE011", + "PSE012", + "PSE013", + "PSE014", + "PSE015", + "PSE016" + ], + "PYF": [ + "PYF001" + ], + "QAT": [ + "QAT002", + "QAT003", + "QAT004", + "QAT007", + "QAT008" + ], + "REU": [ + "REU001" + ], + "ROU": [ + "ROU010", + "ROU021" + ], + "RUS": [ + "RUS001", + "RUS002", + "RUS003", + "RUS004", + "RUS005", + "RUS006", + "RUS007", + "RUS008", + "RUS009", + "RUS010", + "RUS011", + "RUS012", + "RUS013", + "RUS014", + "RUS015", + "RUS016", + "RUS017", + "RUS018", + "RUS019", + "RUS020", + "RUS021", + "RUS022", + "RUS023", + "RUS024", + "RUS025", + "RUS026", + "RUS027", + "RUS028", + "RUS029", + "RUS030", + "RUS031", + "RUS032", + "RUS033", + "RUS034", + "RUS035", + "RUS036", + "RUS037", + "RUS038", + "RUS039", + "RUS040", + "RUS041", + "RUS042", + "RUS043", + "RUS044", + "RUS045", + "RUS046", + "RUS047", + "RUS049", + "RUS050", + "RUS051", + "RUS052", + "RUS053", + "RUS054", + "RUS055", + "RUS056", + "RUS057", + "RUS058", + "RUS059", + "RUS060", + "RUS061", + "RUS062", + "RUS063", + "RUS064", + "RUS065", + "RUS066", + "RUS067", + "RUS068", + "RUS069", + "RUS070", + "RUS071", + "RUS072", + "RUS073", + "RUS074", + "RUS075", + "RUS076", + "RUS077", + "RUS078", + "RUS079", + "RUS080", + "RUS081", + "RUS082", + "RUS083", + "RUS084", + "RUS085", + "RUS086", + "RUS087", + "RUS088", + "RUS089", + "RUS090" + ], + "SAU": [ + "SAU006", + "SAU009" + ], + "SDN": [ + "SDN001", + "SDN003", + "SDN006", + "SDN008", + "SDN009", + "SDN012", + "SDN013", + "SDN014" + ], + "SGP": [ + "SGP001", + "SGP002", + "SGP003", + "SGP004", + "SGP005", + "SGP007", + "SGP008", + "SGP009" + ], + "SJM": [ + "SJM001" + ], + "SLB": [ + "SLB001" + ], + "SLE": [ + "SLE002", + "SLE005" + ], + "SMR": [ + "SMR001" + ], + "SOM": [ + "SOM003", + "SOM006", + "SOM008", + "SOM009", + "SOM010", + "SOM014", + "SOM015" + ], + "SSD": [ + "SSD003", + "SSD005", + "SSD006", + "SSD007", + "SSD010", + "SSD011", + "SSD012" + ], + "SVK": [ + "SVK001", + "SVK002", + "SVK003", + "SVK004", + "SVK005", + "SVK006", + "SVK007", + "SVK008" + ], + "SVN": [ + "SVN003" + ], + "SWE": [ + "SWE001", + "SWE002", + "SWE003", + "SWE004", + "SWE005", + "SWE006", + "SWE007", + "SWE008", + "SWE009", + "SWE010", + "SWE011", + "SWE012", + "SWE013", + "SWE014", + "SWE015", + "SWE016", + "SWE017", + "SWE018", + "SWE019", + "SWE020", + "SWE021" + ], + "SXM": [ + "SXM001" + ], + "SYC": [ + "SYC001", + "SYC002", + "SYC007", + "SYC008", + "SYC009", + "SYC016", + "SYC018", + "SYC019", + "SYC020", + "SYC021", + "SYC022", + "SYC023", + "SYC024", + "SYC025", + "SYC026", + "SYC027", + "SYC028", + "SYC030", + "SYC031", + "SYC032", + "SYC034", + "SYC035", + "SYC036", + "SYC037", + "SYC039", + "SYC042", + "SYC043", + "SYC045", + "SYC047", + "SYC049", + "SYC050", + "SYC051", + "SYC053", + "SYC054", + "SYC055", + "SYC056", + "SYC057", + "SYC058", + "SYC060" + ], + "SYR": [ + "SYR008", + "SYR011" + ], + "TCA": [ + "TCA005", + "TCA006" + ], + "TGO": [ + "TGO001" + ], + "THA": [ + "THA003", + "THA014", + "THA038", + "THA046", + "THA049", + "THA056", + "THA058", + "THA069" + ], + "TJK": [ + "TJK001", + "TJK002" + ], + "TKL": [ + "TKL001" + ], + "TKM": [ + "TKM003", + "TKM005" + ], + "TLS": [ + "TLS012" + ], + "TON": [ + "TON001" + ], + "TUN": [ + "TUN003", + "TUN013", + "TUN025" + ], + "TUR": [ + "TUR013", + "TUR028", + "TUR037", + "TUR038", + "TUR049", + "TUR080" + ], + "TUV": [ + "TUV001" + ], + "UGA": [ + "UGA001", + "UGA003", + "UGA004", + "UGA005", + "UGA006", + "UGA007", + "UGA008", + "UGA011", + "UGA012", + "UGA014", + "UGA015", + "UGA016", + "UGA017", + "UGA018", + "UGA019", + "UGA020", + "UGA024", + "UGA025", + "UGA026", + "UGA027", + "UGA028", + "UGA029", + "UGA032", + "UGA034", + "UGA036", + "UGA041", + "UGA042", + "UGA051", + "UGA052", + "UGA053", + "UGA054", + "UGA057", + "UGA058", + "UGA061", + "UGA062", + "UGA063", + "UGA065", + "UGA067", + "UGA069", + "UGA070", + "UGA071", + "UGA072", + "UGA078", + "UGA079", + "UGA086", + "UGA088", + "UGA089", + "UGA090", + "UGA092", + "UGA093", + "UGA095", + "UGA096", + "UGA097", + "UGA101", + "UGA103", + "UGA104", + "UGA107", + "UGA111" + ], + "UKR": [ + "UKR001", + "UKR002", + "UKR003", + "UKR005", + "UKR006", + "UKR007", + "UKR008", + "UKR009", + "UKR010", + "UKR011", + "UKR012", + "UKR013", + "UKR014", + "UKR015", + "UKR016", + "UKR017", + "UKR018", + "UKR019", + "UKR020", + "UKR021", + "UKR022", + "UKR023", + "UKR024", + "UKR025" + ], + "UMI": [ + "UMI001", + "UMI002", + "UMI003", + "UMI004", + "UMI006", + "UMI007", + "UMI008" + ], + "UZB": [ + "UZB001", + "UZB002", + "UZB003", + "UZB004", + "UZB005", + "UZB006", + "UZB007", + "UZB010", + "UZB011", + "UZB012", + "UZB013", + "UZB014" + ], + "VAT": [ + "VAT001" + ], + "VIR": [ + "VIR001", + "VIR002", + "VIR003" + ], + "VNM": [ + "VNM003", + "VNM008", + "VNM015", + "VNM016", + "VNM017", + "VNM018", + "VNM019", + "VNM020", + "VNM036", + "VNM040" + ], + "WLF": [ + "WLF001" + ], + "WSM": [ + "WSM001" + ], + "XKX": [ + "XKX001", + "XKX002", + "XKX003", + "XKX004", + "XKX005", + "XKX006", + "XKX007" + ], + "YEM": [ + "YEM018", + "YEM019", + "YEM021" + ], + "ZMB": [ + "ZMB106" + ], + "ZWE": [ + "ZWE011" + ] + }, + "world_bank_source": { + "dataset": "admin-1", + "version": "v0" + } +} diff --git a/validation/data/orbae_to_world_bank_admin_1.json b/validation/data/orbae_to_world_bank_admin_1.json new file mode 100644 index 0000000..a7671eb --- /dev/null +++ b/validation/data/orbae_to_world_bank_admin_1.json @@ -0,0 +1,731 @@ +{ + "coverage": { + "ARG": 1.0, + "AUS": 1.0, + "AUT": 1.0, + "BOL": 1.0, + "BRA": 1.0, + "CAN": 1.0, + "CHN": 0.71, + "CIV": 0.0, + "COL": 1.0, + "CZE": 0.214, + "DEU": 0.875, + "ESP": 1.0, + "FRA": 0.385, + "GBR": 1.0, + "GHA": 0.909, + "GTM": 1.0, + "HND": 1.0, + "HRV": 0.6, + "HUN": 0.95, + "IDN": 1.0, + "MEX": 0.933, + "MYS": 0.929, + "PNG": 0.9, + "POL": 0.812, + "PRY": 1.0, + "ROU": 0.976, + "RUS": 0.0, + "SLB": 0.0, + "SVK": 0.875, + "THA": 0.907, + "URY": 1.0, + "USA": 1.0, + "ZAF": 1.0 + }, + "matched": { + "ARG:Buenos Aires": "ARG001", + "ARG:Catamarca": "ARG003", + "ARG:Chaco": "ARG004", + "ARG:Corrientes": "ARG007", + "ARG:C\u00f3rdoba": "ARG006", + "ARG:Entre R\u00edos": "ARG008", + "ARG:Formosa": "ARG009", + "ARG:Jujuy": "ARG010", + "ARG:La Pampa": "ARG011", + "ARG:Misiones": "ARG014", + "ARG:R\u00edo negro": "ARG016", + "ARG:Salta": "ARG017", + "ARG:San Luis": "ARG019", + "ARG:Santa Fe": "ARG021", + "ARG:Santiago del Estero": "ARG022", + "ARG:Tucum\u00e1n": "ARG024", + "AUS:New South Wales": "AUS004", + "AUS:Queensland": "AUS007", + "AUS:South Australia": "AUS008", + "AUS:Tasmania": "AUS009", + "AUS:Victoria": "AUS010", + "AUS:Western Australia": "AUS011", + "AUT:Burgenland": "AUT001", + "AUT:K\u00e4rnten": "AUT002", + "AUT:Nieder\u00f6sterreich": "AUT003", + "AUT:Ober\u00f6sterreich": "AUT004", + "AUT:Salzburg": "AUT005", + "AUT:Steiermark": "AUT006", + "AUT:Tirol": "AUT007", + "AUT:Vorarlberg": "AUT008", + "AUT:Wien": "AUT009", + "BOL:Beni": "BOL001", + "BOL:Santa Cruz": "BOL008", + "BOL:Tarija": "BOL009", + "BRA:Acre": "BRA001", + "BRA:Alagoas": "BRA002", + "BRA:Amap\u00e1": "BRA003", + "BRA:Amazonas": "BRA004", + "BRA:Bahia": "BRA005", + "BRA:Cear\u00e1": "BRA006", + "BRA:Distrito Federal": "BRA007", + "BRA:Esp\u00edrito Santo": "BRA008", + "BRA:Goi\u00e1s": "BRA009", + "BRA:Maranh\u00e3o": "BRA010", + "BRA:Mato Grosso": "BRA011", + "BRA:Mato Grosso do Sul": "BRA012", + "BRA:Minas Gerais": "BRA013", + "BRA:Paran\u00e1": "BRA020", + "BRA:Para\u00edba": "BRA019", + "BRA:Par\u00e1": "BRA018", + "BRA:Pernambuco": "BRA021", + "BRA:Piau\u00ed": "BRA022", + "BRA:Rio Grande do Norte": "BRA024", + "BRA:Rio Grande do Sul": "BRA025", + "BRA:Rio de Janeiro": "BRA023", + "BRA:Rond\u00f4nia": "BRA026", + "BRA:Roraima": "BRA027", + "BRA:Santa Catarina": "BRA028", + "BRA:Sergipe": "BRA030", + "BRA:S\u00e3o Paulo": "BRA029", + "BRA:Tocantins": "BRA031", + "CAN:Alberta": "CAN001", + "CAN:British Columbia": "CAN002", + "CAN:Manitoba": "CAN003", + "CAN:New Brunswick": "CAN004", + "CAN:Nova Scotia": "CAN007", + "CAN:Ontario": "CAN009", + "CAN:Prince Edward Island": "CAN010", + "CAN:Qu\u00e9bec": "CAN011", + "CAN:Saskatchewan": "CAN012", + "CHN:Anhui": "CHN001", + "CHN:Fujian": "CHN004", + "CHN:Gansu": "CHN005", + "CHN:Guangdong": "CHN006", + "CHN:Guizhou": "CHN008", + "CHN:Hainan": "CHN009", + "CHN:Hebei": "CHN010", + "CHN:Heilongjiang": "CHN011", + "CHN:Henan": "CHN012", + "CHN:Hubei": "CHN013", + "CHN:Hunan": "CHN014", + "CHN:Jiangsu": "CHN015", + "CHN:Jiangxi": "CHN016", + "CHN:Jilin": "CHN017", + "CHN:Liaoning": "CHN018", + "CHN:Qinghai": "CHN021", + "CHN:Shaanxi": "CHN022", + "CHN:Shandong": "CHN023", + "CHN:Shanxi": "CHN025", + "CHN:Sichuan": "CHN026", + "CHN:Yunnan": "CHN030", + "CHN:Zhejiang": "CHN031", + "COL:Antioquia": "COL002", + "COL:Atl\u00e1ntico": "COL005", + "COL:Bol\u00edvar": "COL007", + "COL:Boyac\u00e1": "COL008", + "COL:Caldas": "COL009", + "COL:Casanare": "COL011", + "COL:Cesar": "COL013", + "COL:Choc\u00f3": "COL014", + "COL:Cundinamarca": "COL016", + "COL:C\u00f3rdoba": "COL015", + "COL:La Guajira": "COL020", + "COL:Magdalena": "COL021", + "COL:Meta": "COL022", + "COL:Nari\u00f1o": "COL023", + "COL:Norte de Santander": "COL024", + "COL:Putumayo": "COL025", + "COL:Santander": "COL028", + "COL:Sucre": "COL029", + "COL:Vichada": "COL033", + "CZE:Jihomoravsk\u00fd": "CZE002", + "CZE:Jiho\u010desk\u00fd": "CZE001", + "CZE:St\u0159edo\u010desk\u00fd": "CZE006", + "DEU:Bayern": "DEU002", + "DEU:Berlin": "DEU003", + "DEU:Brandenburg": "DEU004", + "DEU:Bremen": "DEU005", + "DEU:Hamburg": "DEU006", + "DEU:Hessen": "DEU007", + "DEU:Mecklenburg-Vorpommern": "DEU008", + "DEU:Niedersachsen": "DEU009", + "DEU:Nordrhein-Westfalen": "DEU010", + "DEU:Rheinland-Pfalz": "DEU011", + "DEU:Saarland": "DEU012", + "DEU:Sachsen": "DEU013", + "DEU:Sachsen-Anhalt": "DEU014", + "DEU:Schleswig-Holstein": "DEU015", + "ESP:Andaluc\u00eda": "ESP002", + "ESP:Arag\u00f3n": "ESP003", + "ESP:Cantabria": "ESP005", + "ESP:Castilla y Le\u00f3n": "ESP006", + "ESP:Castilla-La Mancha": "ESP007", + "ESP:Catalu\u00f1a / Catalunya": "ESP008", + "ESP:Comunidad Foral de Navarra": "ESP012", + "ESP:Comunidad Valenciana / Comunitat Valenciana": "ESP013", + "ESP:Comunidad de Madrid": "ESP011", + "ESP:Extremadura": "ESP014", + "ESP:Galicia": "ESP015", + "ESP:Islas Baleares / Illes Balears": "ESP016", + "ESP:La Rioja": "ESP017", + "ESP:Pa\u00eds Vasco / Euskadi": "ESP018", + "ESP:Principado de Asturias": "ESP019", + "ESP:Regi\u00f3n de Murcia": "ESP020", + "FRA:Bretagne": "FRA007", + "FRA:Corse": "FRA010", + "FRA:Pays de la Loire": "FRA019", + "FRA:Provence-Alpes-C\u00f4te d'Azur": "FRA022", + "FRA:\u00cele-de-France": "FRA013", + "GBR:England": "GBR002", + "GBR:Northern Ireland": "GBR003", + "GBR:Scotland": "GBR004", + "GBR:Wales": "GBR005", + "GHA:Ahafo": "GHA011", + "GHA:Ashanti": "GHA001", + "GHA:Bono": "GHA012", + "GHA:Bono East": "GHA013", + "GHA:Central": "GHA003", + "GHA:Eastern": "GHA004", + "GHA:Oti": "GHA016", + "GHA:Volta": "GHA009", + "GHA:Western": "GHA010", + "GHA:Western North": "GHA018", + "GTM:Alta Verapaz": "GTM001", + "GTM:Escuintla": "GTM006", + "GTM:Izabal": "GTM009", + "GTM:Pet\u00e9n": "GTM012", + "GTM:Quetzaltenango": "GTM013", + "GTM:Quich\u00e9": "GTM014", + "GTM:Retalhuleu": "GTM015", + "GTM:San Marcos": "GTM017", + "GTM:Suchitep\u00e9quez": "GTM020", + "HND:Atl\u00e1ntida": "HND001", + "HND:Col\u00f3n": "HND003", + "HND:Comayagua": "HND004", + "HND:Cort\u00e9s": "HND006", + "HND:Gracias a Dios": "HND008", + "HND:Olancho": "HND015", + "HND:Santa B\u00e1rbara": "HND017", + "HND:Yoro": "HND019", + "HRV:Bjelovar-Bilogora": "HRV001", + "HRV:Karlovac": "HRV005", + "HRV:Koprivnica-Kri\u017eevci": "HRV006", + "HRV:Krapina-Zagorje": "HRV007", + "HRV:Osijek-Baranja": "HRV010", + "HRV:Sisak-Moslavina": "HRV014", + "HRV:Vara\u017edin": "HRV017", + "HRV:Virovitica-Podravina": "HRV018", + "HRV:Zagreb": "HRV021", + "HUN:Bacs Kiskun": "HUN001", + "HUN:Baranya": "HUN002", + "HUN:Bekes": "HUN003", + "HUN:Borsod Abauj Zemplen": "HUN004", + "HUN:Budapest": "HUN005", + "HUN:Csongrad": "HUN006", + "HUN:Fejer": "HUN007", + "HUN:Gyor Moson Sopron": "HUN008", + "HUN:Hajdu Bihar": "HUN009", + "HUN:Heves": "HUN010", + "HUN:Jasz Nagykun Szolnok": "HUN011", + "HUN:Komarom Esztergom": "HUN012", + "HUN:Nograd": "HUN013", + "HUN:Somogy": "HUN015", + "HUN:Szabolcs Szatmar Bereg": "HUN016", + "HUN:Tolna": "HUN017", + "HUN:Vas": "HUN018", + "HUN:Veszprem": "HUN019", + "HUN:Zala": "HUN020", + "IDN:Aceh": "IDN001", + "IDN:Banten": "IDN016", + "IDN:Bengkulu": "IDN007", + "IDN:Gorontalo": "IDN029", + "IDN:Jambi": "IDN005", + "IDN:Jawa Barat": "IDN012", + "IDN:Kalimantan Barat": "IDN020", + "IDN:Kalimantan Selatan": "IDN022", + "IDN:Kalimantan Tengah": "IDN021", + "IDN:Kalimantan Timur": "IDN023", + "IDN:Kalimantan Utara": "IDN024", + "IDN:Kepulauan Bangka Belitung": "IDN009", + "IDN:Kepulauan Riau": "IDN010", + "IDN:Lampung": "IDN008", + "IDN:Maluku": "IDN031", + "IDN:Maluku Utara": "IDN032", + "IDN:Papua": "IDN033", + "IDN:Papua Barat": "IDN035", + "IDN:Riau": "IDN004", + "IDN:Sulawesi Barat": "IDN030", + "IDN:Sulawesi Selatan": "IDN027", + "IDN:Sulawesi Tengah": "IDN026", + "IDN:Sulawesi Tenggara": "IDN028", + "IDN:Sumatera Barat": "IDN003", + "IDN:Sumatera Selatan": "IDN006", + "IDN:Sumatera Utara": "IDN002", + "MEX:Aguascalientes": "MEX001", + "MEX:Campeche": "MEX004", + "MEX:Chiapas": "MEX007", + "MEX:Chihuahua": "MEX008", + "MEX:Ciudad de M\u00e9xico": "MEX009", + "MEX:Coahuila de Zaragoza": "MEX005", + "MEX:Colima": "MEX006", + "MEX:Durango": "MEX010", + "MEX:Guanajuato": "MEX011", + "MEX:Guerrero": "MEX012", + "MEX:Hidalgo": "MEX013", + "MEX:Jalisco": "MEX014", + "MEX:Michoac\u00e1n de Ocampo": "MEX016", + "MEX:Morelos": "MEX017", + "MEX:Nayarit": "MEX018", + "MEX:Nuevo Le\u00f3n": "MEX019", + "MEX:Oaxaca": "MEX020", + "MEX:Puebla": "MEX021", + "MEX:Quintana Roo": "MEX023", + "MEX:San Luis Potos\u00ed": "MEX024", + "MEX:Sinaloa": "MEX025", + "MEX:Sonora": "MEX026", + "MEX:Tabasco": "MEX027", + "MEX:Tamaulipas": "MEX028", + "MEX:Tlaxcala": "MEX029", + "MEX:Veracruz de Ignacio de la Llave": "MEX030", + "MEX:Yucat\u00e1n": "MEX031", + "MEX:Zacatecas": "MEX032", + "MYS:Johor": "MYS001", + "MYS:Kedah": "MYS002", + "MYS:Kelantan": "MYS003", + "MYS:Melaka": "MYS006", + "MYS:Negeri Sembilan": "MYS007", + "MYS:Pahang": "MYS008", + "MYS:Perak": "MYS009", + "MYS:Perlis": "MYS010", + "MYS:Pulau Pinang": "MYS011", + "MYS:Sabah": "MYS012", + "MYS:Sarawak": "MYS013", + "MYS:Selangor": "MYS014", + "MYS:Terengganu": "MYS015", + "PNG:Central": "PNG002", + "PNG:East New Britain": "PNG004", + "PNG:Eastern Highlands": "PNG006", + "PNG:Madang": "PNG011", + "PNG:Milne Bay": "PNG013", + "PNG:Morobe": "PNG014", + "PNG:New Ireland": "PNG016", + "PNG:Sandaun": "PNG018", + "PNG:West New Britain": "PNG020", + "POL:Dolno\u015bl\u0105skie": "POL001", + "POL:Kujawsko-Pomorskie": "POL002", + "POL:Lubuskie": "POL005", + "POL:Mazowieckie": "POL007", + "POL:Opolskie": "POL008", + "POL:Podkarpackie": "POL009", + "POL:Podlaskie": "POL010", + "POL:Pomorskie": "POL011", + "POL:Warmi\u0144sko-Mazurskie": "POL014", + "POL:Wielkopolskie": "POL015", + "POL:Zachodniopomorskie": "POL016", + "POL:\u015al\u0105skie": "POL012", + "POL:\u015awi\u0119tokrzyskie": "POL013", + "PRY:Alto Paraguay": "PRY001", + "PRY:Alto Paran\u00e1": "PRY002", + "PRY:Amambay": "PRY003", + "PRY:Boquer\u00f3n": "PRY004", + "PRY:Caaguaz\u00fa": "PRY005", + "PRY:Caazap\u00e1": "PRY006", + "PRY:Canindey\u00fa": "PRY007", + "PRY:Central": "PRY008", + "PRY:Concepci\u00f3n": "PRY009", + "PRY:Guair\u00e1": "PRY011", + "PRY:Itap\u00faa": "PRY012", + "PRY:Misiones": "PRY013", + "PRY:Paraguar\u00ed": "PRY015", + "PRY:Presidente Hayes": "PRY016", + "PRY:San Pedro": "PRY017", + "PRY:\u00d1eembuc\u00fa": "PRY014", + "ROU:Alba": "ROU001", + "ROU:Arad": "ROU002", + "ROU:Arges": "ROU003", + "ROU:Bacau": "ROU004", + "ROU:Bihor": "ROU005", + "ROU:Bistrita Nasaud": "ROU006", + "ROU:Botosani": "ROU007", + "ROU:Braila": "ROU008", + "ROU:Brasov": "ROU009", + "ROU:Bucuresti": "ROU010", + "ROU:Buzau": "ROU011", + "ROU:Calarasi": "ROU012", + "ROU:Caras Severin": "ROU013", + "ROU:Cluj": "ROU014", + "ROU:Constanta": "ROU015", + "ROU:Covasna": "ROU016", + "ROU:Dambovita": "ROU017", + "ROU:Dolj": "ROU018", + "ROU:Galati": "ROU019", + "ROU:Giurgiu": "ROU020", + "ROU:Harghita": "ROU022", + "ROU:Hunedoara": "ROU023", + "ROU:Ialomita": "ROU024", + "ROU:Iasi": "ROU025", + "ROU:Ilfov": "ROU026", + "ROU:Maramures": "ROU027", + "ROU:Mehedinti": "ROU028", + "ROU:Mures": "ROU029", + "ROU:Neamt": "ROU030", + "ROU:Olt": "ROU031", + "ROU:Prahova": "ROU032", + "ROU:Salaj": "ROU033", + "ROU:Satu Mare": "ROU034", + "ROU:Sibiu": "ROU035", + "ROU:Suceava": "ROU036", + "ROU:Teleorman": "ROU037", + "ROU:Timis": "ROU038", + "ROU:Tulcea": "ROU039", + "ROU:Valcea": "ROU040", + "ROU:Vaslui": "ROU041", + "ROU:Vrancea": "ROU042", + "SVK:Bansk\u00e1 Bystrica": "SVK001", + "SVK:Bratislava": "SVK002", + "SVK:Ko\u0161ice": "SVK003", + "SVK:Nitra": "SVK004", + "SVK:Pre\u0161ov": "SVK005", + "SVK:Tren\u010d\u00edn": "SVK006", + "SVK:Trnava": "SVK007", + "THA:Amnat Charoen": "THA001", + "THA:Ang Thong": "THA002", + "THA:Bangkok": "THA003", + "THA:Buri Ram": "THA004", + "THA:Chachoengsao": "THA005", + "THA:Chai Nat": "THA006", + "THA:Chaiyaphum": "THA007", + "THA:Chanthaburi": "THA008", + "THA:Chiang Mai": "THA009", + "THA:Chiang Rai": "THA010", + "THA:Chon Buri": "THA011", + "THA:Chumphon": "THA012", + "THA:Kalasin": "THA013", + "THA:Kanchanaburi": "THA015", + "THA:Khon Kaen": "THA016", + "THA:Krabi": "THA017", + "THA:Lampang": "THA018", + "THA:Lamphun": "THA019", + "THA:Loei": "THA020", + "THA:Lop Buri": "THA021", + "THA:Mae Hong Son": "THA022", + "THA:Maha Sarakham": "THA023", + "THA:Mukdahan": "THA024", + "THA:Nakhon Nayok": "THA025", + "THA:Nakhon Pathom": "THA026", + "THA:Nakhon Phanom": "THA027", + "THA:Nakhon Ratchasima": "THA028", + "THA:Nakhon Sawan": "THA029", + "THA:Nakhon Si Thammarat": "THA030", + "THA:Nan": "THA031", + "THA:Narathiwat": "THA032", + "THA:Nong Bua Lam Phu": "THA033", + "THA:Nong Khai": "THA034", + "THA:Nonthaburi": "THA035", + "THA:Pathum Thani": "THA036", + "THA:Pattani": "THA037", + "THA:Phangnga": "THA039", + "THA:Phatthalung": "THA040", + "THA:Phayao": "THA041", + "THA:Phetchabun": "THA042", + "THA:Phetchaburi": "THA043", + "THA:Phichit": "THA044", + "THA:Phitsanulok": "THA045", + "THA:Phrae": "THA047", + "THA:Phuket": "THA048", + "THA:Ranong": "THA050", + "THA:Ratchaburi": "THA051", + "THA:Rayong": "THA052", + "THA:Roi Et": "THA053", + "THA:Sa Kaeo": "THA054", + "THA:Sakon Nakhon": "THA055", + "THA:Saraburi": "THA059", + "THA:Satun": "THA060", + "THA:Si Sa Ket": "THA061", + "THA:Sing Buri": "THA062", + "THA:Songkhla": "THA063", + "THA:Sukhothai": "THA064", + "THA:Suphan Buri": "THA065", + "THA:Surat Thani": "THA066", + "THA:Surin": "THA067", + "THA:Tak": "THA068", + "THA:Trang": "THA070", + "THA:Ubon Ratchathani": "THA071", + "THA:Udon Thani": "THA072", + "THA:Uthai Thani": "THA073", + "THA:Uttaradit": "THA074", + "THA:Yala": "THA075", + "THA:Yasothon": "THA076", + "URY:Artigas": "URY001", + "URY:Canelones": "URY002", + "URY:Cerro Largo": "URY003", + "URY:Colonia": "URY004", + "URY:Durazno": "URY005", + "URY:Flores": "URY006", + "URY:Florida": "URY007", + "URY:Lavalleja": "URY008", + "URY:Maldonado": "URY009", + "URY:Montevideo": "URY010", + "URY:Paysand\u00fa": "URY011", + "URY:Rivera": "URY013", + "URY:Rocha": "URY014", + "URY:R\u00edo Negro": "URY012", + "URY:Salto": "URY015", + "URY:San Jos\u00e9": "URY016", + "URY:Soriano": "URY017", + "URY:Tacuaremb\u00f3": "URY018", + "URY:Treinta y Tres": "URY019", + "USA:Alabama": "USA001", + "USA:Arizona": "USA003", + "USA:Arkansas": "USA004", + "USA:California": "USA005", + "USA:Colorado": "USA006", + "USA:Connecticut": "USA007", + "USA:Delaware": "USA008", + "USA:Florida": "USA010", + "USA:Georgia": "USA011", + "USA:Idaho": "USA013", + "USA:Illinois": "USA014", + "USA:Indiana": "USA015", + "USA:Iowa": "USA016", + "USA:Kansas": "USA017", + "USA:Kentucky": "USA018", + "USA:Louisiana": "USA019", + "USA:Maine": "USA020", + "USA:Maryland": "USA021", + "USA:Massachusetts": "USA022", + "USA:Michigan": "USA023", + "USA:Minnesota": "USA024", + "USA:Mississippi": "USA025", + "USA:Missouri": "USA026", + "USA:Montana": "USA027", + "USA:Nebraska": "USA028", + "USA:Nevada": "USA029", + "USA:New Hampshire": "USA030", + "USA:New Jersey": "USA031", + "USA:New Mexico": "USA032", + "USA:New York": "USA033", + "USA:North Carolina": "USA034", + "USA:North Dakota": "USA035", + "USA:Ohio": "USA036", + "USA:Oklahoma": "USA037", + "USA:Oregon": "USA038", + "USA:Pennsylvania": "USA039", + "USA:Rhode Island": "USA040", + "USA:South Carolina": "USA041", + "USA:South Dakota": "USA042", + "USA:Tennessee": "USA043", + "USA:Texas": "USA044", + "USA:Utah": "USA045", + "USA:Vermont": "USA046", + "USA:Virginia": "USA047", + "USA:Washington": "USA048", + "USA:West Virginia": "USA049", + "USA:Wisconsin": "USA050", + "USA:Wyoming": "USA051", + "ZAF:Eastern Cape": "ZAF001", + "ZAF:Free State": "ZAF002", + "ZAF:Gauteng": "ZAF003", + "ZAF:KwaZulu-Natal": "ZAF004", + "ZAF:Limpopo": "ZAF005", + "ZAF:Mpumalanga": "ZAF006" + }, + "orbae_export": "20260807_orbae_export.zip", + "unmatched": { + "CHN": [ + "Beijing", + "Chongqing", + "Guangxi", + "Inner Mongolia", + "Ningxia", + "Shanghai", + "Tianjin", + "Tibet", + "Xinjiang" + ], + "CIV": [ + "Agneby-Tiassa", + "Bafing", + "Belier", + "Bere", + "Cavally", + "District Autonome D'Abidjan", + "District Autonome De Yamoussoukro", + "Gbeke", + "Gbokle", + "Goh", + "Gontougo", + "Grands Ponts", + "Guemon", + "Haut-Sassandra", + "Iffou", + "Indenie-Djuablin", + "Loh-Djiboua", + "Marahoue", + "Me", + "Moronou", + "N'Zi", + "Nawa", + "San Pedro", + "Sud-Comoe", + "Tonkpi", + "Worodougou" + ], + "CZE": [ + "Hlavn\u00ed m\u011bsto Praha", + "Karlovarsk\u00fd", + "Kraj Vyso\u010dina", + "Kr\u00e1lov\u00e9hradeck\u00fd", + "Libereck\u00fd", + "Moravskoslezsk\u00fd", + "Olomouck\u00fd", + "Pardubick\u00fd", + "Plze\u0148sk\u00fd", + "Zl\u00ednsk\u00fd", + "\u00dasteck\u00fd" + ], + "DEU": [ + "Baden-W\u00fcrttemberg", + "Th\u00fcringen" + ], + "FRA": [ + "Auvergne-Rh\u00f4ne-Alpes", + "Bourgogne-Franche-Comt\u00e9", + "Centre-Val de Loire", + "Grand Est", + "Hauts-de-France", + "Normandie", + "Nouvelle-Aquitaine", + "Occitanie" + ], + "GHA": [ + "Greater Accra" + ], + "HRV": [ + "Brod-Posavina", + "City of Zagreb", + "Istria", + "Me\u0111imurje", + "Po\u017eega-Slavonia", + "Vukovar-Syrmia" + ], + "HUN": [ + "Pest megye" + ], + "MEX": [ + "M\u00e9xico (Est.)", + "Quer\u00e9taro de Arteaga" + ], + "MYS": [ + "W.P. Putrajaya" + ], + "PNG": [ + "Oro" + ], + "POL": [ + "Lubelskie", + "Ma\u0142opolskie", + "\u0141\u00f3dzkie" + ], + "ROU": [ + "Gorj" + ], + "RUS": [ + "Adygea", + "Altai", + "Altai Krai", + "Amur Oblast", + "Astrakhan Oblast", + "Bashkortostan", + "Belgorod Oblast", + "Bryansk Oblast", + "Buryatia", + "Chechnya", + "Chelyabinsk Oblast", + "Chuvashia", + "Dagestan", + "Ingushetia", + "Irkutsk Oblast", + "Ivanovo Oblast", + "Jewish Autonomous Oblast", + "Kabardino-Balkaria", + "Kaliningrad", + "Kalmykia", + "Kaluga Oblast", + "Karachay-Cherkessia", + "Kemerovo Oblast", + "Khakassia", + "Kirov Oblast", + "Kostroma Oblast", + "Krasnodar Krai", + "Krasnoyarsk Krai", + "Kurgan Oblast", + "Kursk Oblast", + "Leningrad Oblast", + "Lipetsk Oblast", + "Mari El Republic", + "Mordovia", + "Nizhny Novgorod Oblast", + "North Ossetia\u2013Alania", + "Novgorod Oblast", + "Novosibirsk Oblast", + "Omsk Oblast", + "Orenburg Oblast", + "Oryol Oblast", + "Penza Oblast", + "Perm Krai", + "Podmoskovye", + "Primorsky Krai", + "Pskov Oblast", + "Rostov Oblast", + "Ryazan Oblast", + "Samara Oblast", + "Saratov Oblast", + "Smolensk Oblast", + "Stavropol Krai", + "Sverdlovsk Oblast", + "Tambov Oblast", + "Tatarstan", + "Tomsk Oblast", + "Tula Oblast", + "Tuva", + "Tver Oblast", + "Tyumen Oblast", + "Udmurtia", + "Ulyanovsk Oblast", + "Vladimir Oblast", + "Volgograd Oblast", + "Vologda Oblast", + "Voronezh Oblast", + "Yaroslavl Oblast", + "Zabaykalsky Krai" + ], + "SLB": [ + "Guadalcanal" + ], + "SVK": [ + "\u017dilina" + ], + "THA": [ + "Bueng Kan", + "Kamphaeng Phet", + "Phra Nakhon Si Ayutthaya", + "Prachin Buri", + "Prachuap Khiri Khan", + "Samut Prakan", + "Trat" + ] + }, + "world_bank_source": { + "dataset": "admin-1", + "version": "v0" + } +} diff --git a/validation/data/sources.lock.json b/validation/data/sources.lock.json new file mode 100644 index 0000000..5b79b8a --- /dev/null +++ b/validation/data/sources.lock.json @@ -0,0 +1,915 @@ +{ + "faostat": { + "faostat_production.parquet": { + "bytes": 1377837, + "origin": "gs://cornerstone-ingest-us-central1/tabular/faostat/production-crops/v0/whole-world/world.parquet", + "path": "faostat_production.parquet", + "sha256": "db9b69d42a77c1ca1fe9281ba64d0b36949b5afa10316e908e1e7f45251f9b9d" + } + }, + "orbae": { + "20260807_orbae_export.zip": { + "bytes": 31832479, + "path": "raw/orbae/20260807_orbae_export.zip", + "sha256": "22830b2b1f1f33488ea65dadb644a443ecd900e4dab70e10cdda1945854851ed" + } + }, + "wri": { + "wri/559fe23eb752/national/CO2/EF_ADM0_ACOF_CO2.csv": { + "bytes": 13244, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_ACOF_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_ACOF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "e11e824e9efbe729fc71353023bf055da1e76e88a62a36efc478f9ec8e60ba34" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_BANA_CO2.csv": { + "bytes": 29504, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_BANA_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_BANA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "f25101e9ebc671361be773cf63aba246bc53fa60b4731a930af51161983fd679" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_BARL_CO2.csv": { + "bytes": 27870, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_BARL_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_BARL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "62dc08ce3b55cda79e148a96cca4b9d09df9d0a930069c04b797d7503fb235b2" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_BEAN_CO2.csv": { + "bytes": 29529, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_BEAN_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_BEAN_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5c36a7585d5673cf61eccff4844c3510b299fd3851bbc34ff343e71824c821ff" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_CASS_CO2.csv": { + "bytes": 21865, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_CASS_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_CASS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "dc54e340f58ed43c01aa290fa8536f697816a320c49c483e2d3522514a261f97" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_CHIC_CO2.csv": { + "bytes": 19905, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_CHIC_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_CHIC_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "86fe16d220dff940d842bc6003c937d9c4d92404b8f83c4a52bd8e9210383be0" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_CNUT_CO2.csv": { + "bytes": 19005, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_CNUT_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_CNUT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "142e0b2c7dc136535bc37dfbb2b563cff1c27dd73452e48e6edcec6f75584907" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_COCO_CO2.csv": { + "bytes": 15376, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_COCO_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_COCO_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "8008a786897589d9a6d7385e16f3753669ce96237cc8dfca9e182dea6272b959" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_COTT_CO2.csv": { + "bytes": 23784, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_COTT_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_COTT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "e5e1d55697220233c62104b0dec0c64d65c09e18a3181c6c4a899108641fcd3b" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_COWP_CO2.csv": { + "bytes": 17785, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_COWP_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_COWP_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b3c963d977d0ce4f885f103110b2f5a7878c25e69fdbbea723d153bd31d29ec4" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_GROU_CO2.csv": { + "bytes": 27584, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_GROU_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_GROU_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1966087728d6a2bd382ca2599b3c47f805df8649cfdc880a2692523bc4fd4a1d" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_LENT_CO2.csv": { + "bytes": 15673, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_LENT_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_LENT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b72c2808dc1b6d2aa82f1f96d2a686817c41677213ace6335bc3431033c46c4a" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_MAIZ_CO2.csv": { + "bytes": 38225, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_MAIZ_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_MAIZ_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9f18beba88f6ffb6e1007cd6d1e5743a280bec15099ff3e8bdfd539e722edae6" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_OCER_CO2.csv": { + "bytes": 29059, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_OCER_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_OCER_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5a9e29f144ac7eaad6956bd1d09e4cca8459905d8458c014d0e8cdf62f58d3b0" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_OFIB_CO2.csv": { + "bytes": 26441, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_OFIB_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_OFIB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "08f01fc2fd3f5190a453b7662a793b21114c4afa1ed0e692b8648ab606d7a3e7" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_OILP_CO2.csv": { + "bytes": 12746, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_OILP_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_OILP_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "bfb569d3b524216482f20638418a3cc771782e0f1361bf7f912ff0f3119f7eb4" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_OOIL_CO2.csv": { + "bytes": 32150, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_OOIL_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_OOIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "893a307e067cd01b19228d83496df25d04d6810ba9fe4aeaeb91483a8e73f8d5" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_OPUL_CO2.csv": { + "bytes": 37902, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_OPUL_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_OPUL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b8d282e61bbb4c81ef5d6ddbc2e1de01803c9699c4c42afb9fb26aabc8ffcac8" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_ORTS_CO2.csv": { + "bytes": 24118, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_ORTS_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_ORTS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "66dee62605616b5bd548aecd9fcee26e7a2d67a494e9af87fc334a707d69cd75" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_PIGE_CO2.csv": { + "bytes": 8288, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_PIGE_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_PIGE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "fa1f3aafbb4415a9dffc330e334085a229b200d7fc40eb81c76a29a642103da2" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_PLNT_CO2.csv": { + "bytes": 15917, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_PLNT_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_PLNT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "7c73861e90df448b4a8044827b67dbe403815a5414b84892cd5f05367a372726" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_PMIL_CO2.csv": { + "bytes": 12915, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_PMIL_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_PMIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "d128c92d61f35f81f51be70cca499cf1865a046a5ffea211b10b1ebb0ade0e15" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_POTA_CO2.csv": { + "bytes": 38582, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_POTA_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_POTA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "39b773a176aad48da9caf5390bb9351356692844f7326c4ebf0d6166ee1391b2" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_RAPE_CO2.csv": { + "bytes": 19423, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_RAPE_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_RAPE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "0d2b973db841b0ef053ac982e544d7de59b5c238d1c937b3a78ede7b6cc98da7" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_RCOF_CO2.csv": { + "bytes": 13688, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_RCOF_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_RCOF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1f41b658ed30fe73318762de7f1d3f04ff4521bbaf232f03e6ed1c3c88a8e461" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_REST_CO2.csv": { + "bytes": 40286, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_REST_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_REST_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "a363056720c7fec748a768c1004a7fdfd958c84051edc76cc765f09941a9376c" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_RICE_CO2.csv": { + "bytes": 31327, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_RICE_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_RICE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3debbb0989465fdd2df0c0470bae93f1de1b031ee95c07856c2ef8c8ed65d638" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SESA_CO2.csv": { + "bytes": 23665, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SESA_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SESA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b60b163dfffa21cc7eeaad397ef0b50973db8a8cdf9f009c6ac4774b19acc262" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SMIL_CO2.csv": { + "bytes": 14077, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SMIL_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SMIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "09e0656b16536fd65dc3739f2221f43d53df58869439b7377f12c534c8d7aa80" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SORG_CO2.csv": { + "bytes": 30033, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SORG_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SORG_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5a9b6ed074d009f63f51828dfd583b22e80ff2e03c6d5dff0beb4ea22bdbf700" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SOYB_CO2.csv": { + "bytes": 28777, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SOYB_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SOYB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "42693d466ad8b24c8de0771f89583ec30531bd30be9809cbe5e28f80f86e0af8" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SUGB_CO2.csv": { + "bytes": 17094, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SUGB_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SUGB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "ef600bbd434ec005cef55a4e4ebaf97e4e793c0f8caf998f8ee1015febc2019d" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SUGC_CO2.csv": { + "bytes": 23603, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SUGC_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SUGC_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "67ee18ecd4b4d90e510ca09fdcbff230a4c3772419b6ba96f4d335abcb58dd1b" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SUNF_CO2.csv": { + "bytes": 23382, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SUNF_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SUNF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "867da7457ccf0f2f9faeb6e0de15e9cad19b5c838f0c73984ec71118ec4972de" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_SWPO_CO2.csv": { + "bytes": 26335, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_SWPO_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_SWPO_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "867b77446d7cef7d913fb96c5928a8f9cea7e7fa97f7db84d50a2fda37522b23" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_TEAS_CO2.csv": { + "bytes": 14097, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_TEAS_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_TEAS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "03e284e1a1f0834b34b2020d635dd586ddff501c683603d93b6c67b701b1961c" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_TEMF_CO2.csv": { + "bytes": 40756, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_TEMF_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_TEMF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1d91b7adca05c92de010b9eb0c86ad9f764e805b3ce882ed40822ef0e229049b" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_TOBA_CO2.csv": { + "bytes": 31484, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_TOBA_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_TOBA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "39db890d917b39dcac5ce0bd6b0125184be1bf529326e4e753f7f9391161d9e2" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_TROF_CO2.csv": { + "bytes": 38837, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_TROF_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_TROF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "71688101ed75a6a36192faaa17aa88eeaf76088c3e42f445358fdb4524352be8" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_VEGE_CO2.csv": { + "bytes": 41647, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_VEGE_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_VEGE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9d54a47e4c2bfa6a8dfd343e67efea9361e890bc31a1a3711e3a1dc103dc85d3" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_WHEA_CO2.csv": { + "bytes": 32078, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_WHEA_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_WHEA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "783f334b43e44c6bbbbd481950e1ff199460f2491f61e5dc87dc7ea4716b601f" + }, + "wri/559fe23eb752/national/CO2/EF_ADM0_YAMS_CO2.csv": { + "bytes": 16132, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2/EF_ADM0_YAMS_CO2.csv", + "path": "raw/wri/559fe23eb752/national/CO2/EF_ADM0_YAMS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "0c817751fa5b073a3f8fd1b1446b166ed62c99c369f0d2863defacf429fbdd22" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_ACOF_CO2e.csv": { + "bytes": 13245, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_ACOF_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_ACOF_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "46e4e9a181181958be79a7e07b933728b51df2157c7ba2cb13daf4fbd66308b7" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_BANA_CO2e.csv": { + "bytes": 29462, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_BANA_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_BANA_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1b8df56cce1802eaae1f3834b3ddbb7e84bf49c166744851066c2bc29a9fbf86" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_BARL_CO2e.csv": { + "bytes": 27812, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_BARL_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_BARL_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "fa80976f1b9ac1b9e01657ac78853455b4ff4034981ca3e9b7c603c8d4195535" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_BEAN_CO2e.csv": { + "bytes": 29507, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_BEAN_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_BEAN_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "39bd2bdf703f17496d25e0dec694990ec9d1f221bced72dab7e13db9e809fc49" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_CASS_CO2e.csv": { + "bytes": 21879, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_CASS_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_CASS_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "99426b947fba79454e64845e1ef00279722938c7bfcf2c3d746124075388b7e8" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_CHIC_CO2e.csv": { + "bytes": 19892, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_CHIC_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_CHIC_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "cd67b62e7c94d0c6751ee5233ad4ea38461899165e106c334073cbd0ef631716" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_CNUT_CO2e.csv": { + "bytes": 19032, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_CNUT_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_CNUT_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "09178253e79f604f3c9be82877473d7622715c0be0bc33eb755f6cee4a014e74" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_COCO_CO2e.csv": { + "bytes": 15391, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_COCO_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_COCO_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5b97d492d9f0a806518399f61d6c28290c3f51fcac253974d8413aea1e851733" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_COTT_CO2e.csv": { + "bytes": 23809, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_COTT_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_COTT_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3d08a8d020da5ab48747809db1f4cf2bf7a61a59ccee085d822690349b405035" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_COWP_CO2e.csv": { + "bytes": 17829, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_COWP_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_COWP_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "cabd377225e5840d8418a27ab27cda8fd67e850e974d9136fdf2b2fdd0bcaa9a" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_GROU_CO2e.csv": { + "bytes": 27578, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_GROU_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_GROU_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "a45131bf1d8a59a3c733da75fd9f78baf2033a9f4034b347629136309cdf351f" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_LENT_CO2e.csv": { + "bytes": 15702, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_LENT_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_LENT_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "6c822cdcbc6a115bfaed5440be5c594e9d3888b89c75ca93278fe3a255d35c47" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_MAIZ_CO2e.csv": { + "bytes": 38181, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_MAIZ_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_MAIZ_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "892969153a3a6f7f0b83ccf6e6125ba44a24227b6500e46d6f1c63aef12fe7ce" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_OCER_CO2e.csv": { + "bytes": 29042, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_OCER_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_OCER_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "cbcfde5ddef654b0b817b71592c7b41f58163212fc3a91820fbd4d85e5a8e472" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_OFIB_CO2e.csv": { + "bytes": 26426, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_OFIB_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_OFIB_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "8265c11f383949423e8f1ca02d19c832efdcb23ece55875388272ee6591a9608" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_OILP_CO2e.csv": { + "bytes": 12726, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_OILP_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_OILP_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5841a958b66d4281cc7b2f38549510721b06fe14a51d183df3a61a8a8e3fb8da" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_OOIL_CO2e.csv": { + "bytes": 32143, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_OOIL_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_OOIL_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "36f64008ecde30c0225d2939447c76af6bb9d376a82eb7057290db045d68019f" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_OPUL_CO2e.csv": { + "bytes": 37909, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_OPUL_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_OPUL_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "d18456c43adc93579f02441629415a7e1bacbc877ec0daf041c1adc01739c0f3" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_ORTS_CO2e.csv": { + "bytes": 24107, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_ORTS_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_ORTS_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "4cdc3bc26486eb98789084e52a1a3b431cfb4e65aa1ec082e01661aed09748ed" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_PIGE_CO2e.csv": { + "bytes": 8292, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_PIGE_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_PIGE_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9c7b3e4675a9210bd1e281b49816fd02257142d1b340c623263076111d3159af" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_PLNT_CO2e.csv": { + "bytes": 15902, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_PLNT_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_PLNT_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9d5d75a6472abf3e5e6410fb166da61de0227b8b1cd6702409e1ae6b1ff7b2f2" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_PMIL_CO2e.csv": { + "bytes": 12916, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_PMIL_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_PMIL_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "f5fde80f2f7a08dceab00a42137b3958cd4923f9adcedfb0345f7583c16f6695" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_POTA_CO2e.csv": { + "bytes": 38678, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_POTA_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_POTA_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1b43c7f5161d968e4b7d361497bebe2e574e785543edfb601e6c55f2c474c0df" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_RAPE_CO2e.csv": { + "bytes": 19411, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_RAPE_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_RAPE_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "894f613411fc9f34cfbf2c10a46a93624bc06173b60f54a888c7b253ce7f14c3" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_RCOF_CO2e.csv": { + "bytes": 13702, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_RCOF_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_RCOF_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "a9fe4853430208db9c481e7404e1011be964dbb1b5accebad3dffc0601fa47a0" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_REST_CO2e.csv": { + "bytes": 40330, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_REST_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_REST_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "c695d0260527e0c8a1dee95953267dba3d8f2d4a814a9e0d6303148079263fa2" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_RICE_CO2e.csv": { + "bytes": 31335, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_RICE_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_RICE_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9a653d3daca1e12f2dddb9baec655cb03b5f7d02be5ed2cb0609de018bfe1d13" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SESA_CO2e.csv": { + "bytes": 23666, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SESA_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SESA_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "709ef806b0cfcb53bd5dd3f80f6507557f70f748afde1516a91191cc3c268e6b" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SMIL_CO2e.csv": { + "bytes": 14111, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SMIL_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SMIL_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "2e9bb03a95b7624926e77dbbc75c13cdffd719b7a5bcae769592095d552b5645" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SORG_CO2e.csv": { + "bytes": 30045, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SORG_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SORG_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "28e2c9befbdb8a4951ced7100128511f8e27e2d757302ac3b8cdfcebbb3a700c" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SOYB_CO2e.csv": { + "bytes": 28719, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SOYB_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SOYB_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "cf5e280284721a1392f0c52be0237f0150b99c448a3b4ae9a010a9402088750b" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SUGB_CO2e.csv": { + "bytes": 17095, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SUGB_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SUGB_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "908f36ec94bdf08f7902b2ac52b177573e7d3f8004f4709cfa7889c80473a4c0" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SUGC_CO2e.csv": { + "bytes": 23627, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SUGC_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SUGC_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1db808f8864e6632b7193b5664d431f462be991a2d7185a615899783bc36ea7d" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SUNF_CO2e.csv": { + "bytes": 23377, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SUNF_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SUNF_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "7dcac3031ec33a7f317168d71433bc4e903e25d98a3b92cde2bb15118966e56e" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_SWPO_CO2e.csv": { + "bytes": 26328, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_SWPO_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_SWPO_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "153bfe4336bdfb3b94c849c07671f9f3006804db0ccc3cb2a23e82201a44b5c0" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_TEAS_CO2e.csv": { + "bytes": 14082, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_TEAS_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_TEAS_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "7c0eb426364a477eba94b9b3372363ed168aab1c618baa63e47f85d79161689e" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_TEMF_CO2e.csv": { + "bytes": 40818, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_TEMF_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_TEMF_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b65460cf496373de3e19d2cba493c6dc447f891b5c1c94606ffecf6a94d4a812" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_TOBA_CO2e.csv": { + "bytes": 31439, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_TOBA_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_TOBA_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "2117909291f1c79c4d5ce4f3258e90169b3d4f7ec28d42f96a96bd27ddea85b0" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_TROF_CO2e.csv": { + "bytes": 38828, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_TROF_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_TROF_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "ed56da6de938c8fb4c7057a24128f927e8a4b92a56ced5fa672c6d90ba7957c2" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_VEGE_CO2e.csv": { + "bytes": 41676, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_VEGE_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_VEGE_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "6372bbd0dea90f84b4b1e225ae4731bcd015a31217dad553822bb71c1ba70ded" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_WHEA_CO2e.csv": { + "bytes": 32074, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_WHEA_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_WHEA_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "98ac8d30ac0368a896870a479fbfca74f0338d48e5a92f6396aa17ab62422d2a" + }, + "wri/559fe23eb752/national/CO2e/EF_ADM0_YAMS_CO2e.csv": { + "bytes": 16110, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_admin0/individual_commodities_CO2e/EF_ADM0_YAMS_CO2e.csv", + "path": "raw/wri/559fe23eb752/national/CO2e/EF_ADM0_YAMS_CO2e.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "cfe2b57df41e0126b51e2bf35f774dc6ec2e670dd687bc71e4b228486c258ead" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_ACOF_CO2.csv": { + "bytes": 39072, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_ACOF_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_ACOF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3e8c68d821d06c0ff4b9da42c83402643a7ce2dfa921ecc69d16136bf74e0ba2" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_BANA_CO2.csv": { + "bytes": 75693, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_BANA_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_BANA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "c26106f94f204b62c111d4b20f5a15f961eaa0caf549c2b4d8b9fbf1d91be6f7" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_BARL_CO2.csv": { + "bytes": 79752, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_BARL_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_BARL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "82b59523da4e641c097346b156412999caf745b34f4ccf4a3f82aad5c0af065b" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_BEAN_CO2.csv": { + "bytes": 94381, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_BEAN_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_BEAN_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "775310dda6fa1dcd22cfef321492d8d45b585b36f1fe37e6ceb03ce455fd869e" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_CASS_CO2.csv": { + "bytes": 71847, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_CASS_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_CASS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "8c603f73ee3ad468e8b393daecf19d4958e323a4eed23b5eb381bca8e4d93c76" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_CHIC_CO2.csv": { + "bytes": 40300, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_CHIC_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_CHIC_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "7feaf7c1a7c4f42884e50e8bf484ce573fb47d314cfcb571013405d1fd0e0765" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_CNUT_CO2.csv": { + "bytes": 47215, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_CNUT_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_CNUT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "f795a422b155c6b2e194c5bbe5165411fa702044cfa4e6764ed752b55552363f" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_COCO_CO2.csv": { + "bytes": 40797, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_COCO_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_COCO_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1368bc62db11d66d04dffdf744b647c157e0ab04123c20cd7231a8ec6dac503c" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_COTT_CO2.csv": { + "bytes": 58552, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_COTT_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_COTT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3ba2836a212e0fcdfa597279cda5649ebbdf85b87182db09d11a8a8b1b9a5381" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_COWP_CO2.csv": { + "bytes": 32741, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_COWP_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_COWP_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "139df92715186d955c76fa82c7e607b15ee9a9cd4ec82321ade36056fba27401" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_GROU_CO2.csv": { + "bytes": 82692, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_GROU_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_GROU_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "4c4d7db1633647067d2469049562756ad4659eee78d984ff73ed76939a4b8d35" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_LENT_CO2.csv": { + "bytes": 33752, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_LENT_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_LENT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "e14c5f94a32bed5b0a0d7aaca9e6e343baa12da3ec040137dd8955cb8889b793" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_MAIZ_CO2.csv": { + "bytes": 126662, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_MAIZ_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_MAIZ_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5e60782900368171f75ba8ef22718bc710811f783e903c4b033c9c8e251c3dce" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_OCER_CO2.csv": { + "bytes": 81106, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_OCER_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_OCER_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "df8f3953032f41d883291cfd3668bbddcc4b75ffc1e0a276f2d7571d31f4622f" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_OFIB_CO2.csv": { + "bytes": 61157, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_OFIB_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_OFIB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "bbe66c6d093e3377d9a0f928094ab0154a7a7950401fe52cc318c55019108a7b" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_OILP_CO2.csv": { + "bytes": 31572, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_OILP_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_OILP_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "5147e557c7c5e4e966b62c1437b537c2f1676b1c1a382ae0c8ed02da211e36ce" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_OOIL_CO2.csv": { + "bytes": 83325, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_OOIL_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_OOIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "42885fb629105f283467b924d8ade3c805303e78b82dbe6097da24da4e2e3255" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_OPUL_CO2.csv": { + "bytes": 102846, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_OPUL_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_OPUL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "11e096c962faf0e7e17d2df4dfba352523632bb70d16f95488fa3086fa0a9def" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_ORTS_CO2.csv": { + "bytes": 61734, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_ORTS_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_ORTS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "96be8ec39767d35cfa79079bca8ae8e13dcccaa3bec28c82a09157ae14949d0a" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_PIGE_CO2.csv": { + "bytes": 20102, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_PIGE_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_PIGE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "91aea980915bf3d8cc8eeb117b15f26427f45d150b8789eed0f88e28d7d01557" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_PLNT_CO2.csv": { + "bytes": 41153, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_PLNT_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_PLNT_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "d954f1641b0e5e7b50a635d23b4ae38a5c476bda80402cabcc30cd9fcd53052b" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_PMIL_CO2.csv": { + "bytes": 24044, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_PMIL_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_PMIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "64a954a4114e6bbc27da91f3ff3250f1d169bead3961c674d52a1b4ad813fb5e" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_POTA_CO2.csv": { + "bytes": 118765, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_POTA_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_POTA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "b5caf2ae082f844017063aa7cbc44bd6b299b530849b0f2e112d0457431ef739" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_RAPE_CO2.csv": { + "bytes": 54361, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_RAPE_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_RAPE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "130d49a9e13de53d89275c426511af7d0f3e69866bde7c063790342af8f41098" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_RCOF_CO2.csv": { + "bytes": 34528, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_RCOF_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_RCOF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "ba86a322444c7c276121587c2c232a8f42dc78bee5286046ef149c63707a36eb" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_REST_CO2.csv": { + "bytes": 133552, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_REST_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_REST_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "315fc12fe9290ac81d3e1f84d47c3902b8cc5c39ba8b929c36f03f67076effcc" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_RICE_CO2.csv": { + "bytes": 95600, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_RICE_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_RICE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1f8bb1a2984ac9ebc5e6fa76bd3d127818acb33a4bef904960563846d11f37ad" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SESA_CO2.csv": { + "bytes": 52416, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SESA_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SESA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "15b61b1c2366e2eca8333600f65cb194f8b954c9df54fb5fe073d54f1a1b59f7" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SMIL_CO2.csv": { + "bytes": 34073, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SMIL_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SMIL_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "07c34b548232d494472743ccee1c10a911537ea26687f2f1f1dd0cfcd8bfc34c" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SORG_CO2.csv": { + "bytes": 83305, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SORG_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SORG_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "1068922edbf17680e8bdc408b7a5cb5e21dc38ff2fbd04e712f2cd636a0e7c83" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SOYB_CO2.csv": { + "bytes": 88493, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SOYB_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SOYB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "0903220471752c3430d329196004d1a3ca931ab29c6f676f9b918500ca267fab" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SUGB_CO2.csv": { + "bytes": 49602, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SUGB_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SUGB_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "0caa20dc2ac64d05892e828e1219b79d6afd177765d13ffd3a35ee95607b18ff" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SUGC_CO2.csv": { + "bytes": 71526, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SUGC_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SUGC_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "9fb596742bbfa87bc1f79c1fc29d8c6636b7dd538b3284d07a16a692e4f2afc7" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SUNF_CO2.csv": { + "bytes": 66889, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SUNF_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SUNF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "aa86b52d3ec581b86f35b9c889f97478c2e66edbee8bdcd8aaa27e6f9374ea71" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_SWPO_CO2.csv": { + "bytes": 78582, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_SWPO_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_SWPO_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3309a4e3c2e17392d4586d44d6464d8b0d46841405239c4f55eceedf437616b3" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_TEAS_CO2.csv": { + "bytes": 33209, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_TEAS_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_TEAS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "3526aae5df9830d0c27a03d43923382cbd38418d6694cbaf2c2e59db93be5d3d" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_TEMF_CO2.csv": { + "bytes": 131161, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_TEMF_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_TEMF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "ba09be47c549024b06b4d95a9031ce036847f56926ebed40647fb373eed9ac91" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_TOBA_CO2.csv": { + "bytes": 84270, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_TOBA_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_TOBA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "e21e82bb8e05d6a6a551954eff39f6a9b8ec241e8abd3712ec2dac1d2b8bcbce" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_TROF_CO2.csv": { + "bytes": 123710, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_TROF_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_TROF_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "bcd0385298068ed67955d540de960048295e10422242aa528667fe6034ebd2ea" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_VEGE_CO2.csv": { + "bytes": 141659, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_VEGE_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_VEGE_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "070c4cfad28ca34f784ba22d302c9d7326cbe1964646326990be0ec6f4464d0b" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_WHEA_CO2.csv": { + "bytes": 93099, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_WHEA_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_WHEA_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "e09fb75d435ae724fa99067fecca14f2d136fdfa09ccc82b5abd8ae7a16d1a96" + }, + "wri/559fe23eb752/provincial/CO2/EF_ADM1_YAMS_CO2.csv": { + "bytes": 40082, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/sLUC_emission_factors/deforestation_emission_factors_adm1/individual_commodities_CO2/EF_ADM1_YAMS_CO2.csv", + "path": "raw/wri/559fe23eb752/provincial/CO2/EF_ADM1_YAMS_CO2.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "c111916843928c9a7106b343bbe5fc1ce45392ac9afb8c7f3576f17720925f1e" + }, + "wri/559fe23eb752/yield_factors/national.csv": { + "bytes": 363533, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/yield_factors/yield_factor_gadm0.csv", + "path": "raw/wri/559fe23eb752/yield_factors/national.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "dabe9722c6a6d1e2729873969e71280b55709ccfd95fc6ce7c1192d87783c5be" + }, + "wri/559fe23eb752/yield_factors/provincial.csv": { + "bytes": 5449791, + "origin": "https://raw.githubusercontent.com/wri/GCSC/559fe23eb752e9df270a1bf93e7f290044026bab/data/yield_factors/yield_factor_gadm1.csv", + "path": "raw/wri/559fe23eb752/yield_factors/provincial.csv", + "revision": "559fe23eb752e9df270a1bf93e7f290044026bab", + "sha256": "bf9018d37df8738ab33da4d76bed3fd3daad414fa3ab33d2f43bf79d583d7b1e" + } + } +} diff --git a/validation/data/targets.json b/validation/data/targets.json new file mode 100644 index 0000000..c8b2282 --- /dev/null +++ b/validation/data/targets.json @@ -0,0 +1,609 @@ +{ + "note": [ + "Chosen by hand from the shortlist `--section eligible` prints,", + "not by a score; docs/validation.md section 3 says why. Each pair carries the reason it is here.", + "Controls live here too, one `controls` list per pair, so the pair identity has a single source.", + "`inherited` is a prior figure used once as a sanity check when the first capture sets `baseline`;", + "a control fires against `baseline` alone, and a null baseline means not yet frozen.", + "Orbae measures are rank controls, not level ones: its export is assessment year 2023 with no 2020", + "export coming, so `statistic` is RANK_CORRELATION and their tolerance is in correlation units.", + "ORBAE_OVER_WRI holds no term of ours, so movement in it means an anchor was revised rather than", + "the pipeline moving, and it is the one measure freezable before any capture. Its cross-country", + "rank agreement is weakest for soy (+0.135) and oil palm (+0.018), which the per-country baselines", + "below measure every run rather than noting once.", + "Deforestation figures in the reasons are WRI's for 2020, in Mt of CO2, at the pinned commit.", + "They are linearly discounted over 20 years -- the same GHGP basis as ours, which is what makes", + "them comparable -- and they are why a pair is here, not a result.", + "All thirty controls are frozen. The Orbae/WRI ranks were baselined from the anchors alone,", + "needing no capture since both sides are external; the rest come from the first capture. Six of", + "the seven sLUC/WRI controls carrying an inherited figure reproduce it within its quoted", + "precision, CAF MAIZE sitting 11% under. The three sLUC/jdLUC controls carrying one do not:", + "peatland occupation is weighted by each crop's hectares rather than by its share of the", + "expansion, which moved them after those prior figures were set.", + "The sLUC/jdLUC controls span 0.667 for rice to 3.259 for cotton, and over all eleven jdLUC crops", + "the ratio reaches 53.6 for sugarbeet. Production agrees within 0.78-1.26x throughout and hectares", + "within 0.82-1.07x, so the spread is conversion emissions rather than a denominator, and it widens", + "as the crop gets smaller. The three Midwestern pairs at 1.155-1.748 are its narrow end, not its", + "typical value, which is why holding the internal head-to-head to those three understated it." + ], + "provenance": { + "eligible_pairs": 864, + "faostat_release": "2025", + "reference_year": 2020, + "wri_revision": "559fe23eb752e9df270a1bf93e7f290044026bab" + }, + "targets": [ + { + "basis": "gap", + "crop_name": "OILPALM", + "iso_3166": "IDN", + "reason": "The largest deforestation figure anywhere at 218 Mt, and the measured case of the woody-perennial detection gap: sLUC/WRI is 0.008, decomposing as 0.020 detection x 0.38 allocation. Expected to fail badly; that failure is the point." + }, + { + "basis": "gap", + "crop_name": "OILPALM", + "iso_3166": "MYS", + "reason": "68 Mt, second largest. Confirms whether the Indonesian woody-perennial result is a country artifact or the mechanism. 96% provincial area coverage, so the provincial grain is available." + }, + { + "basis": "gap", + "crop_name": "COCONUT", + "iso_3166": "IDN", + "reason": "31 Mt on a woody perennial that is not oil palm, so it separates 'any woody perennial' from 'oil palm specifically' in the GLAD cropland-class finding." + }, + { + "basis": "gap", + "crop_name": "OILPALM", + "iso_3166": "COD", + "reason": "14 Mt, and African rather than South-East Asian: tests the woody-perennial gap away from peat, where the peatland term cannot be the explanation." + }, + { + "basis": "gap", + "crop_name": "COCONUT", + "iso_3166": "PHL", + "reason": "6.2 Mt, and NOT eligible at the provincial grain: the World Bank layer gives PHL 17 regions where GADM gives 81 provinces, so the relation is many-to-one and no key map can express it -- 0 of the 81 names match even fuzzily, because the two sources are at different administrative levels. Carried as a national ratio only, like CAF MAIZE. PHL is the largest single country E3 excludes, holding 24.8 Mt across all crops against 45 Mt for every excluded country combined, and it is a tropical archipelago where coconut dominates -- the woody-perennial regime the detection gap is largest in, so excluding it under-samples the gap most worth measuring. Pairs against IDN COCONUT, holding the crop fixed across two archipelagos." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.19, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.19, + "measure": "SLUC_OVER_WRI", + "note": "WRI is the outlier in Brazil: sLUC sits at 0.16-0.22x WRI across all three BRA crops, so this ratio is expected to sit low. Reproduces the inherited 0.19.", + "tolerance": 0.1 + }, + { + "baseline": 0.918, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "The closest sLUC/Orbae rank agreement in the set.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.868, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 16 shared provinces.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "SOYBEAN", + "iso_3166": "BRA", + "reason": "111 Mt, the largest annual-crop pair in the world, and the largest figure in the eligible set." + }, + { + "basis": "ranked", + "crop_name": "MAIZE", + "iso_3166": "BRA", + "reason": "49 Mt. Same jurisdictions as BRA SOYBEAN, so it shares tiles and costs little beyond province clips, and it separates crop allocation from country-level forest attribution." + }, + { + "basis": "ranked", + "crop_name": "CASSAVA", + "iso_3166": "COD", + "reason": "48 Mt on a root crop, which is neither of the two commodities the US leg covers. 26 provinces at 100% area coverage." + }, + { + "basis": "ranked", + "crop_name": "MAIZE", + "iso_3166": "IDN", + "reason": "39 Mt. An annual in the same country as the largest woody-perennial gap, so it isolates whether Indonesia's disagreement is the crop or the country." + }, + { + "basis": "ranked", + "crop_name": "RICE", + "iso_3166": "IDN", + "reason": "25 Mt, and the largest flooded-crop pair. Rice carries a methane term the LUC model does not touch, so it checks that the LUC comparison is unaffected by it." + }, + { + "basis": "ranked", + "crop_name": "CASSAVA", + "iso_3166": "BRA", + "reason": "15 Mt. Pairs against COD CASSAVA on one crop across two continents, holding the crop fixed while the forest regime changes." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.067, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.07, + "measure": "SLUC_OVER_WRI", + "note": "The dry-forest/soy guard. Reproduces the inherited 0.07 at the precision it was published.", + "tolerance": 0.1 + }, + { + "baseline": 0.206, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "The weakest sLUC/Orbae rank in the set.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.441, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 16 shared provinces at 100% area coverage. The value is low because soy is the crop the two anchors agree least on, and freezing it is how a change in that disagreement surfaces.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "SOYBEAN", + "iso_3166": "PRY", + "reason": "15 Mt on 18 provinces at 100% coverage. Chaco dry forest rather than humid tropical, and soy holds constant against BRA, ARG and BOL. Holds the dry-forest/soy control slot, on two qualifications the other Chaco soy pairs lack: WRI's row here is self-consistent, at a deforestation-linked share of 0.669 of harvested area, and Orbae shares 16 of its provinces, enough to freeze the Orbae/WRI rank from the anchors alone." + }, + { + "basis": "ranked", + "crop_name": "MAIZE", + "iso_3166": "MEX", + "reason": "14 Mt. Dry and temperate-mixed rather than tropical, and the only North American entry besides the USA." + }, + { + "basis": "ranked", + "crop_name": "SUGARCANE", + "iso_3166": "BRA", + "reason": "11 Mt on 756 Mt of production, the largest denominator in the set by far. A ratooned perennial that is herbaceous, so GLAD's cropland class admits it: the control case for the woody-perennial finding." + }, + { + "basis": "ranked", + "crop_name": "SUGARCANE", + "iso_3166": "THA", + "reason": "9 Mt across 75 provinces, the finest provincial grain available at this materiality. Pairs against BRA SUGARCANE." + }, + { + "basis": "ranked", + "crop_name": "COTTON", + "iso_3166": "BRA", + "reason": "10 Mt on a fibre crop, so the product form is seed cotton rather than a food weight. The yield cross-check put WRI and FAOSTAT at 1.017 here, so the denominator is known good." + }, + { + "basis": "ranked", + "crop_name": "RICE", + "iso_3166": "KHM", + "reason": "9 Mt, and the country whose 13 hand-written province overrides the key map depends on: a wrong override would surface here first." + }, + { + "basis": "ranked", + "crop_name": "MAIZE", + "iso_3166": "ZMB", + "reason": "9 Mt on miombo woodland, a dry-forest regime absent from every other entry. Only 10 provinces, so provincial coverage is thin." + }, + { + "basis": "ranked", + "crop_name": "BEAN", + "iso_3166": "MOZ", + "reason": "8 Mt on a pulse, a crop family otherwise unrepresented, in a second miombo country." + }, + { + "basis": "ranked", + "crop_name": "RICE", + "iso_3166": "LAO", + "reason": "8 Mt. Shares ten-degree tiles with THA and KHM, so it is nearly free once those are computed." + }, + { + "basis": "ranked", + "crop_name": "YAM", + "iso_3166": "NGA", + "reason": "7 Mt, 56 Mt of production, and a MapSPAM group crop decomposed rather than observed (SWPY). Tests the decomposition path at materiality." + }, + { + "basis": "ranked", + "crop_name": "RICE", + "iso_3166": "PHL", + "reason": "6.7 Mt on 19 Mt of production, the largest PHL pair. National ratio only, for the same structural reason as PHL COCONUT: 17 World Bank regions against 81 GADM provinces, so E3 fails and the pair is off the eligible shortlist entirely. Carried as the annual counterpart to PHL COCONUT -- rice is herbaceous, so GLAD's cropland class does see it, and holding the country fixed across the two separates 'PHL is under-detected because its crops are perennial' from 'PHL is under-detected because it is PHL'. Pairs against IDN RICE on one crop across two archipelagos. Together the two PHL pairs cover 12.9 of the country's 24.8 Mt." + }, + { + "basis": "gap", + "crop_name": "OILPALM", + "iso_3166": "PNG", + "reason": "5 Mt, carried for the forest-pool conservation bound rather than for its materiality: PNG's forest allocation is the most sensitive in the set to the expansion-share denominator, so it is where a regression there surfaces first." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 1.634, + "emission_pool": "TOTAL", + "inherited": 1.05, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. 56% above the inherited 1.05: peatland occupation is weighted by each crop's hectares rather than by its share of the expansion, which favors maize's large area. The jdLUC leg is unmoved.", + "tolerance": 0.1 + }, + { + "baseline": 0.813, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "JDLUC_OVER_ORBAE", + "note": "The only method-family-matched comparison in the design: every Orbae row is Method=jdLUC, and the USA is the one jurisdiction where we run a jdLUC leg.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.648, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "sLUC against the same anchor the jdLUC pair uses, so the two together separate method family from leg.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.738, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 44 shared provinces, the widest join in the set.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "MAIZE", + "iso_3166": "USA", + "reason": "The only jdLUC leg, so the only internal head-to-head between the two methodologies. 44 of 51 provinces in WRI's file." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 1.748, + "emission_pool": "TOTAL", + "inherited": 1.62, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. 8% above the inherited 1.62, from the same hectare-weighting of peatland occupation that moved the maize pair.", + "tolerance": 0.1 + }, + { + "baseline": 0.835, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "JDLUC_OVER_ORBAE", + "note": "The only method-family-matched comparison in the design: every Orbae row is Method=jdLUC, and the USA is the one jurisdiction where we run a jdLUC leg.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.613, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "sLUC against the same anchor the jdLUC pair uses, so the two together separate method family from leg.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.443, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 32 shared provinces, and the weakest anchor-against-anchor rank frozen here. The disagreement is a handful of states rather than a broad method difference: dropping the five worst-ranked puts it above the USA MAIZE pair.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "SOYBEAN", + "iso_3166": "USA", + "reason": "Largest US pair at 3.45 Mt, and the US soy comparison against BRA and ARG soy on one crop across three regimes." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 1.155, + "emission_pool": "TOTAL", + "inherited": 1.19, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. 3% below the inherited 1.19: hectare-weighting of peatland occupation barely touches wheat, so this pair carries only the drift common to every sLUC pool.", + "tolerance": 0.1 + } + ], + "crop_name": "WHEAT", + "iso_3166": "USA", + "reason": "The only wheat pair, at 1.35 Mt. Orbae publishes no US wheat, so this pair carries the internal head-to-head alone." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 3.259, + "emission_pool": "TOTAL", + "inherited": null, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. Frozen at 3.259, outside the 1.155-1.748 the three Midwestern pairs span. The disagreement is in the numerator rather than the denominator this pair was chosen for: production agrees to 0.78x and hectares to 0.82x, so sLUC charges cotton 2.5x the conversion emissions jdLUC does. The two Orbae rows below place that on the statistical leg.", + "tolerance": 0.1 + }, + { + "baseline": 0.924, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "JDLUC_OVER_ORBAE", + "note": "The only method-family-matched comparison in the design: every Orbae row is Method=jdLUC, and the USA is the one jurisdiction where we run a jdLUC leg. Frozen at +0.924 over the 17 provinces Orbae covers, the closest any US pair comes to it -- ahead of maize at +0.813 -- which is what makes the sLUC row beside it worth reading. Rank rather than ratio: Orbae's export is assessment year 2023 with no 2020 export coming, and level cannot cross that offset (WRI's own spans 1.42x at the median) where provincial rank very nearly can (+0.961).", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.162, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "sLUC against the same anchor the jdLUC pair uses, so the two together separate method family from leg. Frozen at +0.162 where jdLUC reaches +0.924 over the same provinces against the same anchor: a third party built the jdLUC way ranks cotton the way our jdLUC leg does and not the way our statistical leg does, which is what makes the head-to-head above a finding about that leg rather than about the comparison. Rank rather than ratio, for the reason the jdLUC row gives.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.718, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Baselined at +0.718 over 17 shared provinces at full coverage, from the anchors alone with no capture, the same way the maize and soy rows are. It contains no term of ours, so movement means an anchor was revised rather than the pipeline moving. Cotton is the second-strongest US rank agreement the two anchors reach, behind maize at +0.738 and ahead of soy at +0.443, which is what makes it worth freezing: a weak baseline would report noise. Tolerance is absolute, in correlation units.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "COTTON", + "iso_3166": "USA", + "reason": "Largest US pair after maize and soy at 1.18 Mt, and southern rather than Midwestern. The one US pair whose denominator carries a product-form conversion: NASS reports ginned lint where MapSPAM and FAOSTAT carry seed cotton, and `usda_nass_quickstats.Series.reported_fraction` divides by 0.36 to cross between them. The first capture clears that constant rather than implicating it -- the two legs' production agrees to 0.78x, and FAOSTAT's own US seed-cotton yield agrees with this denominator to 0.95x -- which leaves the head-to-head on conversion emissions. Orbae covers US cotton at 17 provinces with area, enough for the rank controls." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 0.667, + "emission_pool": "TOTAL", + "inherited": null, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. Frozen at 0.667, the one pair where sLUC sits below jdLUC rather than above: the flooded regime this pair was chosen to reach disagrees in the other direction, which is a different result from disagreeing by a different amount. Production agrees to 0.97x, so the 0.65x on conversion emissions is the whole of it.", + "tolerance": 0.1 + } + ], + "crop_name": "RICE", + "iso_3166": "USA", + "reason": "The flooded regime, which no other US pair reaches. Maize, soybean and wheat are all Midwestern row crops on former grassland, so a spread between the legs across them cannot be told apart from a property of that one regime; rice is the cheapest pair that separates the two readings. 0.092 Mt, small on the anchor's own figure and here for what it distinguishes rather than its size. Orbae publishes no US rice, so this pair carries the internal head-to-head alone." + }, + { + "basis": "reserved", + "controls": [ + { + "baseline": 3.005, + "emission_pool": "TOTAL", + "inherited": null, + "measure": "SLUC_OVER_JDLUC", + "note": "The only internal head-to-head, independent of every external anchor. Frozen at 3.005. Hectares agree within 0.2% and production to 0.91x, so conversion emissions carry all of it, on the one US pair that lands on peat. Orbae publishes US sugarcane but only four provinces carry area, too few for a rank control, so nothing external adjudicates this pair.", + "tolerance": 0.1 + } + ], + "crop_name": "SUGARCANE", + "iso_3166": "USA", + "reason": "Subtropical, and the only US pair that lands on peat -- the Everglades Agricultural Area that `further_research.md` measures under repeated peatland drainage pulses -- so it exercises the peatland term the grassland pairs never touch. 0.151 Mt. Orbae does publish US sugarcane, but only four provinces carry any area, too few for a rank correlation to mean anything, so this pair carries the internal head-to-head alone." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.423, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.43, + "measure": "SLUC_OVER_WRI", + "note": "Temperate belt, two anchors. Reproduces the inherited 0.43.", + "tolerance": 0.1 + }, + { + "baseline": 0.875, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "Second anchor on the Argentine belt.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.886, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 15 shared provinces.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "SOYBEAN", + "iso_3166": "ARG", + "reason": "6 Mt, temperate grassland-cropland, where the grassland pool has no external anchor at all." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.441, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.46, + "measure": "SLUC_OVER_WRI", + "note": "Temperate belt, two anchors. Reproduces the inherited 0.46.", + "tolerance": 0.1 + }, + { + "baseline": 0.882, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f" + }, + "emission_pool": "LUC_ONLY", + "inherited": null, + "measure": "SLUC_OVER_ORBAE", + "note": "Second anchor on the Argentine belt, maize beside soy.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + }, + { + "baseline": 0.8, + "baseline_source_versions": { + "ORBAE": "22830b2b1f1f", + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": null, + "measure": "ORBAE_OVER_WRI", + "note": "Over 15 shared provinces.", + "statistic": "RANK_CORRELATION", + "tolerance": 0.1 + } + ], + "crop_name": "MAIZE", + "iso_3166": "ARG", + "reason": "Holds the country fixed against ARG SOYBEAN so a crop-allocation change is separable from a country-level one." + }, + { + "basis": "ranked", + "crop_name": "SOYBEAN", + "iso_3166": "BOL", + "reason": "9 Mt on only 7 provinces, the thinnest provincial coverage carried, which is itself worth watching. Not a control, on three counts: WRI's row here contradicts itself, giving more deforestation-linked area than FAOSTAT reports harvested (share 1.123); only 3 provinces are shared with Orbae, below the five-province floor, so the Orbae/WRI rank cannot be frozen; and over those 3 the rank is -0.500. A control cannot rest on a row its own anchor disagrees with itself about, so PRY SOYBEAN holds the dry-forest/soy slot instead. This pair remains a target: agreement is a result, and thin coverage is worth watching." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.319, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.32, + "measure": "SLUC_OVER_WRI", + "note": "Forest is 68% of the total EF; a second Congo Basin row guards against a single-country fluke. Reproduces the inherited 0.32.", + "tolerance": 0.1 + } + ], + "crop_name": "MAIZE", + "iso_3166": "COD", + "reason": "7 Mt, and the implied intensity of ~1,270 tCO2/ha is far above any biome carbon density, so it is the standing example of emissions allocated onto a small production footprint." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.571, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.58, + "measure": "SLUC_OVER_WRI", + "note": "Miombo and mixed woodland: a different biome from the Congo Basin at a similar ratio. Reproduces the inherited 0.58.", + "tolerance": 0.1 + } + ], + "crop_name": "MAIZE", + "iso_3166": "AGO", + "reason": "2 Mt, the smallest carried, kept because it guards the low-materiality end where a rounding or coverage bug would surface." + }, + { + "basis": "control", + "controls": [ + { + "baseline": 0.357, + "baseline_source_versions": { + "WRI": "559fe23eb752" + }, + "emission_pool": "FOREST", + "inherited": 0.4, + "measure": "SLUC_OVER_WRI", + "note": "Forest-dominated frontier where the method should be strongest; forest is 83% of the total EF. Sits 11% under the inherited 0.40, the loosest of the seven.", + "tolerance": 0.1 + } + ], + "crop_name": "MAIZE", + "iso_3166": "CAF", + "reason": "NOT eligible at the provincial grain: E3 fails structurally, since GADM gives 17 prefectures where the World Bank gives 7 regions and the relation is many-to-one. Carried as a national ratio only." + } + ] +} diff --git a/validation/data/tiled_iso_3166s.json b/validation/data/tiled_iso_3166s.json new file mode 100644 index 0000000..268ca74 --- /dev/null +++ b/validation/data/tiled_iso_3166s.json @@ -0,0 +1,250 @@ +{ + "carved_out_iso_3166s": [ + "BLM", + "GIB", + "GRL", + "MAF", + "MCO", + "NRU", + "TUV", + "VAT" + ], + "note": [ + "DERIVED -- do not hand-edit. Regenerate with", + "`uv run --with geopandas python tools/build-tiled-countries.py`.", + "E1's second half: the countries intersecting at least one GFW tile,", + "less `carved_out_iso_3166s` -- countries that do intersect a tile but", + "carry no crop production to attribute anything to. See", + "worldbank_jurisdictions.UNPRODUCTIVE_ISO_3166S for why that is a list.", + "Committed because deriving it needs a spatial join against a 93 MiB", + "GeoPackage, where the rest of eligibility is pandas over anchors the lock", + "already pins -- so the rest is derived on demand rather than committed twice." + ], + "tile_ids": 280, + "tiled_iso_3166s": [ + "ABW", + "AFG", + "AGO", + "AIA", + "ALA", + "ALB", + "AND", + "ARE", + "ARG", + "ARM", + "ASM", + "ATF", + "ATG", + "AUS", + "AUT", + "AZE", + "BDI", + "BEL", + "BEN", + "BES", + "BFA", + "BGD", + "BGR", + "BHR", + "BHS", + "BIH", + "BLR", + "BLZ", + "BMU", + "BOL", + "BRA", + "BRB", + "BRN", + "BTN", + "BWA", + "CAF", + "CAN", + "CHE", + "CHL", + "CHN", + "CIV", + "CMR", + "COD", + "COG", + "COL", + "COM", + "CRI", + "CUB", + "CUW", + "CYM", + "CYP", + "CZE", + "DEU", + "DJI", + "DMA", + "DNK", + "DOM", + "DZA", + "ECU", + "EGY", + "ERI", + "ESP", + "EST", + "ETH", + "FIN", + "FJI", + "FRA", + "FSM", + "GAB", + "GBR", + "GEO", + "GGY", + "GHA", + "GIN", + "GLP", + "GMB", + "GNB", + "GNQ", + "GRC", + "GRD", + "GTM", + "GUF", + "GUY", + "HKG", + "HND", + "HRV", + "HTI", + "HUN", + "IDN", + "IMN", + "IND", + "IRL", + "IRN", + "IRQ", + "ISL", + "ISR", + "ITA", + "JAM", + "JEY", + "JOR", + "JPN", + "KAZ", + "KEN", + "KGZ", + "KHM", + "KIR", + "KNA", + "KOR", + "KWT", + "LAO", + "LBN", + "LBR", + "LBY", + "LCA", + "LIE", + "LKA", + "LSO", + "LTU", + "LUX", + "LVA", + "MAC", + "MAR", + "MDA", + "MDG", + "MDV", + "MEX", + "MKD", + "MLI", + "MLT", + "MMR", + "MNE", + "MNG", + "MOZ", + "MRT", + "MSR", + "MTQ", + "MUS", + "MWI", + "MYS", + "MYT", + "NAM", + "NCL", + "NER", + "NFK", + "NGA", + "NIC", + "NLD", + "NOR", + "NPL", + "NZL", + "OMN", + "PAK", + "PAN", + "PER", + "PHL", + "PLW", + "PNG", + "POL", + "PRI", + "PRK", + "PRT", + "PRY", + "PSE", + "QAT", + "REU", + "ROU", + "RUS", + "RWA", + "SAU", + "SDN", + "SEN", + "SGP", + "SJM", + "SLB", + "SLE", + "SLV", + "SMR", + "SOM", + "SPM", + "SRB", + "SSD", + "STP", + "SUR", + "SVK", + "SVN", + "SWE", + "SWZ", + "SXM", + "SYC", + "SYR", + "TCA", + "TCD", + "TGO", + "THA", + "TJK", + "TKM", + "TLS", + "TON", + "TTO", + "TUN", + "TUR", + "TZA", + "UGA", + "UKR", + "URY", + "USA", + "UZB", + "VCT", + "VEN", + "VGB", + "VIR", + "VNM", + "VUT", + "WLF", + "WSM", + "XKX", + "YEM", + "ZAF", + "ZMB", + "ZWE" + ], + "world_bank_source": { + "dataset": "admin-1", + "version": "v0" + } +} diff --git a/validation/prepare.py b/validation/prepare.py new file mode 100644 index 0000000..34cbf8c --- /dev/null +++ b/validation/prepare.py @@ -0,0 +1,1655 @@ +"""Read the pulled anchors into the frames `report` renders, and check them against each other. + +Anchors only, for now: WRI's factors and yield factors, and FAOSTAT's production. The capture side +-- our own `efs.parquet` -- joins here once it exists, which runs the pipeline and so is separate. + +The yield cross-check is the first thing this does, and it runs before any emissions comparison for +a reason. WRI's factor is an intensity over a yield: + + EF [kgCO2/kg] = intensity [kgCO2/ha] x yield_factor_kg [ha/kg] + +so a yield disagreement moves every factor built on it. If WRI and FAOSTAT are on different product +forms for a crop -- fruit bunches against palm oil, seed cotton against ginned lint -- the ratio is +off by the milling yield, several-fold, and every emissions comparison for that crop inherits it +while looking like a methodology gap. This finds that before a capture is spent on it. + + Transforms only; `python -m validation` runs them and renders the result. +""" + +import collections.abc +import csv +import dataclasses +import enum +import io +import json +import logging +import pathlib +import statistics +import zipfile + +import iso3166 +import pandas + +from jdluc import statistical +from jdluc.datasets import faostat_production, ifpri_mapspam +from validation import pull, schema, targets + +logger = logging.getLogger(__name__) + +# WRI publishes one yield per (country, crop) with no year on it, so which year it is drawn from has +# to be inferred rather than read. 2020 is the comparison year for everything else here, and +# `--year` sweeps the neighbourhood to see which fits best. +# +# It is also the WRI *reporting* year to read, and that is a match of windows rather than a +# convention: a reporting year names where the LSRS 20-year window sits, so 2020 covers loss years +# 2001-2020, which is the span our own GLAD epochs cover. See WRI_REPORTING_YEARS. +REFERENCE_YEAR = 2020 +# Below this a median is one or two countries and says more about them than about the crop. +MINIMUM_COUNTRIES = 5 +# Below this two sources are not describing the same ordering, applied to the NATIONAL, +# cross-country agreement per crop that `get_anchor_shape_agreement` measures. Set well under WRI's +# +0.961 self-agreement across the same offset, so only a real method difference trips it. +# +# The provincial, within-country correlations `iter_orbae_wri_comparisons` yields are a different +# quantity and are deliberately NOT gated on this -- they are floored on MINIMUM_PROVINCES alone. A +# control measures movement, not agreement, so a weakly-agreeing pair still guards: USA soybean is +# frozen at +0.443 precisely so that a change in how much the two anchors disagree surfaces on every +# run. +MINIMUM_CROSS_COUNTRY_RANK_AGREEMENT = 0.5 +# Below this a rank correlation says more about which provinces happen to be shared than about the two +# sources. BOL soybean has three and is deliberately left uncompared. +MINIMUM_PROVINCES = 5 +# WRI's two file families use different MapSPAM taxonomies within one release: the emissions-factor +# files use Crop2005's 42 codes, the yield factors Crop2020's 46. Only two codes differ, and losing +# them loses every row for those crops, so the join resolves the rename rather than dropping them. +YIELD_TAXONOMY_YEAR = 2020 +# WRI's five columns are one reporting-year series, not five estimates of one quantity. The reporting +# year names the position of the LSRS 20-year window, so 2024 covers loss years 2005-2024: each column +# drops the oldest loss year, adds a real new one, and reweights the rest by the linear discount -- +# 9.75% for the reporting year itself, falling 0.5pt/yr to 0.25% nineteen years back. The denominator +# moves with it: SPAM 2020 production rescaled by the FAOSTAT national ratio for that year, and for +# 2024 by a random forest, FAOSTAT not having published 2024 when WRI built it. +# +# That the window slides over new loss data rather than re-discounting a fixed 2001-2020 series is +# measurable in the published files: re-discounting bounds LD_2024/LD_2020 at 31/39 = 0.795 for any +# non-negative series, and 62% of pairs exceed it, 864 of them rising monotonically. So the spread +# across the five is the price of leaving the reporting year unpinned, not evidence that WRI disagrees +# with itself -- and REFERENCE_YEAR selects a column rather than averaging them. +# +# The crop side does not move with the reporting year at all: allocation comes from crop-area +# expansion between SPAM snapshots, and the 2010-2020 snapshot allocates every loss year from 2011 to +# 2024. That is why all 42 crops appear in all five columns; see docs/validation.md. +WRI_REPORTING_YEARS = (2020, 2021, 2022, 2023, 2024) +# Where the ingested FAOSTAT parquet is kept once fetched. 1.3 MiB against the 33 MiB archive it +# replaces, and cached so the report keeps running offline after the first read. +FAOSTAT_CACHE = pull.CACHE / "faostat_production.parquet" +# How much of a revision or digest identifies an anchor in a `source_version`. Long enough not to +# collide, short enough that a stale-baseline message is readable. +SOURCE_VERSION_LENGTH = 12 +# The one part of eligibility that cannot be computed from the pinned anchors: which countries +# intersect a GFW tile, which needs a spatial join against a 93 MiB GeoPackage. 228 ISO codes, so +# `tools/build-tiled-countries.py` commits this and `get_eligible` derives the rest here. +# +# The shortlist itself is deliberately NOT committed. Every other column in it -- WRI's +# deforestation emissions, FAOSTAT's +# production, WRI's provincial unit counts -- is already pinned by sha256 in sources.lock.json, and +# committing a second copy would break the rule the lock rests on: the lock is committed, the bytes are +# not. Recomputing all 864 pairs from those anchors takes about a tenth of a second, and it cannot go +# stale against the pin because it is read from whatever the pin currently names. +TILED_ISO_3166S = pull.DATA / "tiled_iso_3166s.json" +# E4. An emissions factor that cannot move a companywide number is not worth capture time. +MINIMUM_PRODUCTION_KG = 100 * 1_000 * 1_000 +# E3. Land area, a proxy for the cropland weighting that would need a capture -- and the capture needs +# the target set, which needs E3, which is why the proxy stands. +MINIMUM_AREA_COVERAGE = 0.5 +# GLAD GLCLUC's cropland class excludes perennial *woody* crops by construction -- it covers "annual +# and perennial herbaceous crops", and "perennial woody crops, permanent pastures and shifting +# cultivation are excluded" (Potapov et al. 2022, Nature Food 3, 19-28, +# doi:10.1038/s43016-021-00429-z). Those appear as tree cover instead, which +# `emit.get_land_class` maps to FOREST. Both legs then restrict the emissions numerator to 2020 GLAD +# cropland while MapSPAM keeps the production in the denominator, so the pipeline attributes close to +# zero to exactly these crops. They stay eligible on purpose: a pair we would rank highly and cannot +# compute is the finding, not a filtering mistake. +PERENNIAL_CROP_NAMES = frozenset( + { + statistical.Crop.ARABICA_COFFEE.value, + statistical.Crop.ROBUSTA_COFFEE.value, + statistical.Crop.COCONUT.value, + statistical.Crop.OILPALM.value, + } +) +# Three herbaceous perennials are deliberately absent, and they are the test of whether this list +# tracks the definition rather than the word "perennial". Sugarcane is ratooned; banana and plantain +# are giant herbs with a pseudostem rather than wood. GLAD's definition admits all three, so the +# pipeline should see them and they are not part of the detection gap. +for _herbaceous in ( + statistical.Crop.SUGARCANE, + statistical.Crop.BANANA, + statistical.Crop.PLANTAIN, +): + assert _herbaceous.value not in PERENNIAL_CROP_NAMES +# docs/further_research.md headings. The register measures that document's entries rather than +# keeping a second list, so a slug here is a heading there, verbatim. +PERENNIAL_RESEARCH_SLUG = "Woody perennial crops fall outside GLAD's cropland class" +OIL_PALM_RESEARCH_SLUG = "The statistical leg under-allocates oil palm relative to WRI" + + +def read_wri_yields(grain_name: str) -> pandas.DataFrame: + """WRI's yield per country and crop, as published. + + `yield_kg` is kg/ha and `yield_factor_kg` its reciprocal; only the former is read, since the + reciprocal carries no information the comparison needs. + + The crop set is asserted rather than trusted, because this file is the only place a change in what + WRI publishes would show up cheaply. It carries 46 crops against the factor files' 42 -- the + Crop2020 taxonomy against Crop2005's -- and the difference has to be exactly the two renames plus + `pull.WRI_CROPS_WITHOUT_FACTORS`. A release that gave rubber a factor, or dropped one we pull, + would otherwise leave `WRI_CROP_CODES` quietly describing the wrong release: every count in the + report would still reconcile, over a crop list no longer matching the anchor. + """ + path = pull.get_pulled_path(remote=pull.get_wri_yield_remote(grain_name=grain_name)) + frame = pandas.read_csv(path) + assert len(frame), f"{path} is empty" + expected = { + ifpri_mapspam.get_reported_crop_name( + canonical_crop_name=crop_code, year=YIELD_TAXONOMY_YEAR + ) + for crop_code in pull.WRI_CROP_CODES + } | set(pull.WRI_CROPS_WITHOUT_FACTORS) + published = set(frame["crop"]) + assert published == expected, ( + f"{path} carries a different crop set than {len(pull.WRI_CROP_CODES):d} factor families " + f"plus {len(pull.WRI_CROPS_WITHOUT_FACTORS):d} factorless crops describe: " + f"{sorted(published - expected)} unexpected, {sorted(expected - published)} missing. WRI has " + f"changed what it publishes, so WRI_CROP_CODES and WRI_CROPS_WITHOUT_FACTORS need updating " + f"against the new revision rather than this assertion relaxing" + ) + renamed = frame.rename( + columns={"GID_0": "iso_3166", "GID_1": "gadm_id", "crop": "crop_name"} + ) + # gadm_id only exists at the provincial grain; callers select what they need. + return renamed[ + [ + name + for name in ("iso_3166", "gadm_id", "crop_name", "yield_kg") + if name in renamed + ] + ] + + +def read_wri_national_emissions(gas_scope: str = "CO2") -> pandas.DataFrame: + """WRI's national figures, long over its five reporting years. + + Wide-to-long because WRI publishes `{LD,production,EF}_{2020..2024}` as columns, `LD` being its + name for linearly discounted deforestation emissions. All five are populated, and they are a + reporting-year series rather than five estimates of one quantity: each column slides the LSRS + 20-year window forward a year over both the loss series and the production denominator. + """ + frames = [] + for mapspam_code in pull.WRI_CROP_CODES: + path = pull.get_pulled_path( + remote=pull.get_wri_factor_remote( + crop_code=mapspam_code, + gas_scope=gas_scope, + grain=pull.get_grain(grain_name="national"), + ) + ) + wide = pandas.read_csv(path).rename(columns={"GID_0": "iso_3166"}) + for year in WRI_REPORTING_YEARS: + frames.append( + pandas.DataFrame( + { + "iso_3166": wide["iso_3166"], + "crop_name": mapspam_code, + "year": year, + "gas_scope": gas_scope, + "deforestation_tonnes": wide[f"LD_{year:d}"], + "production_tonnes": wide[f"production_{year:d}"], + "ef_kg_per_kg": wide[f"EF_{year:d}"], + } + ) + ) + return pandas.concat(frames, ignore_index=True) + + +def get_anchor_stability(emissions: pandas.DataFrame) -> pandas.DataFrame: + """How much WRI's factor moves across its five reporting years, per country and crop. + + This bounds what a comparison against WRI can resolve *while the reporting year is unpinned*. If + WRI's 2020 and 2024 factors differ by more than a control's tolerance, then a control firing on + that pair tells us nothing about our pipeline until the year is pinned -- the movement could be + entirely the window sliding. + + It is not a measure of WRI's self-consistency, and must not be reported as one: the five columns + are a series, so most of this spread is real temporal signal -- loss years entering and leaving + the window, and a denominator rescaled to each year's production. + + Spread is max over min rather than a standard deviation: five points is too few for a moment, and + the ratio is what compares against a tolerance expressed as a fraction. + """ + usable = emissions[emissions["ef_kg_per_kg"] > 0] + grouped = usable.groupby(["iso_3166", "crop_name"])["ef_kg_per_kg"] + stability = grouped.agg( + reporting_years="size", lowest="min", highest="max", median="median" + ) + stability = stability[stability["reporting_years"] == len(WRI_REPORTING_YEARS)] + stability["spread"] = stability["highest"] / stability["lowest"] + return stability.sort_values("spread", ascending=False) + + +def get_stability_findings( + stability: pandas.DataFrame, tolerance: float +) -> list[schema.Finding]: + """One finding, sized by how much of the anchor outruns a control's tolerance across the years.""" + beyond = stability[stability["spread"] > 1.0 + tolerance] + if not len(beyond): + return [] + return [ + schema.Finding( + slug="wri-reporting-year-sensitivity", + severity=schema.Severity.ADVISORY, + message=( + f"{len(beyond):,d} of {len(stability):,d} (country, crop) pairs " + f"({len(beyond) / len(stability):.0%}) have a WRI factor that moves more than the " + f"{tolerance:.0%} control tolerance across its five reporting years; the median " + f"spread is {stability['spread'].median():.2f}x and the upper decile " + f"{stability['spread'].quantile(0.9):.2f}x. Most of that is the LSRS window sliding " + "rather than anchor noise, so a control firing on such a pair cannot distinguish a " + "pipeline change from a change of reporting year: the year has to be pinned with the " + "baseline rather than left implicit" + ), + confidence=schema.Confidence.HIGH, + affected_rows=len(beyond), + ) + ] + + +def get_deforestation_share_findings( + deforestation: pandas.DataFrame, +) -> list[schema.Finding]: + """Pairs where WRI's deforestation-linked area exceeds the crop's whole harvested area. + + Internally inconsistent in WRI's own release rather than a disagreement with us: a crop cannot be + grown on more land than it is harvested from. Advisory because it does not invalidate the pairs + that are consistent, but a pair over 1.0 cannot anchor a share comparison. + """ + over = deforestation[deforestation["deforestation_share"] > 1.0] + if not len(over): + return [] + return [ + schema.Finding( + slug="wri-deforestation-area-exceeds-harvested-area", + severity=schema.Severity.ADVISORY, + message=( + f"{len(over):,d} of {len(deforestation):,d} (country, crop) pairs " + f"({len(over) / len(deforestation):.1%}) give WRI more deforestation-linked area " + f"than FAOSTAT reports harvested, up to {over['deforestation_share'].max():.1f}x. " + "A crop cannot be grown on more land than it is harvested from, so these pairs " + "cannot anchor a share comparison whichever side is wrong" + ), + confidence=schema.Confidence.MEDIUM, + affected_rows=len(over), + affected_iso_3166s=tuple(sorted(set(over["iso_3166"]))[:12]), + ) + ] + + +@dataclasses.dataclass(frozen=True) +class Candidate: + """One eligible (country, crop) pair, with the figures a person needs to judge it.""" + + iso_3166: str + crop_name: str + mapspam_code: str + deforestation_tonnes: float + production_kg: float + is_perennial: bool + is_decomposed_group_crop: bool + provincial_units: int + area_coverage: float + + +def read_wri_deforestation(mapspam_code: str) -> pandas.DataFrame: + """WRI's deforestation-linked emissions for one crop, all countries. + + The CO2 scope rather than CO2e: the two are the same product differing by about half a percent, + and this figure is read to compare pairs against each other, not as a level. + """ + path = pull.get_pulled_path( + remote=pull.get_wri_factor_remote( + crop_code=mapspam_code, + gas_scope="CO2", + grain=pull.get_grain(grain_name="national"), + ) + ) + frame = pandas.read_csv(path).rename(columns={"GID_0": "iso_3166"}) + return frame[["iso_3166", f"LD_{REFERENCE_YEAR:d}"]].rename( + columns={f"LD_{REFERENCE_YEAR:d}": "deforestation_tonnes"} + ) + + +def get_provincial_unit_counts(mapspam_code: str) -> dict[str, int]: + """How many GADM provinces WRI publishes a factor for, per country. E3's first half. + + A country with no provincial row is not eligible at the provincial grain, which is the default + grain, however large its national figure is. + """ + path = pull.get_pulled_path( + remote=pull.get_wri_factor_remote( + crop_code=mapspam_code, + gas_scope="CO2", + grain=pull.get_grain(grain_name="provincial"), + ) + ) + counts = pandas.read_csv(path).groupby("GID_0")["GID_1"].nunique() + return {str(iso_3166): int(units) for iso_3166, units in counts.items()} + + +def iter_candidates( + area_coverage: dict[str, float], + production_by_pair: pandas.DataFrame, + tiled_iso_3166s: set[str], +) -> collections.abc.Iterator[Candidate]: + """Every pair passing E1 through E4, with its figures attached. + + The filters are applied in E1-to-E4 order, so a rejection is attributable to exactly one of them. + + `statistical.Crop` is a StrEnum whose name is the pipeline vocabulary and whose value is the + MapSPAM code, so there is no cross-walk to build: WRI and FAOSTAT both key on the code. + """ + production = production_by_pair.set_index(["iso_3166", "crop_name"])[ + "production_kg" + ] + codes = {crop.value: crop.name for crop in statistical.Crop} + for mapspam_code, crop_name in sorted(codes.items()): + national = read_wri_deforestation(mapspam_code=mapspam_code) + provincial_units = get_provincial_unit_counts(mapspam_code=mapspam_code) + for row in national.to_dict("records"): + iso_3166 = str(row["iso_3166"]) + if iso_3166 not in tiled_iso_3166s: # E1 + continue + units = provincial_units.get(iso_3166, 0) + if ( + not units or area_coverage.get(iso_3166, 0.0) < MINIMUM_AREA_COVERAGE + ): # E3 + continue + production_kg = float(production.get((iso_3166, mapspam_code), 0.0)) + if production_kg < MINIMUM_PRODUCTION_KG: # E4 + continue + yield Candidate( + iso_3166=iso_3166, + crop_name=crop_name, + mapspam_code=mapspam_code, + deforestation_tonnes=float(row["deforestation_tonnes"]), + production_kg=production_kg, + is_perennial=mapspam_code in PERENNIAL_CROP_NAMES, + is_decomposed_group_crop=( + mapspam_code in ifpri_mapspam.CONSTITUENT_TO_GROUP_NAME + ), + provincial_units=units, + area_coverage=area_coverage.get(iso_3166, 0.0), + ) + + +def get_eligible() -> pandas.DataFrame: + """The eligible shortlist, derived from the pinned anchors on every run. + + Derived rather than committed. Every figure here already exists in an anchor the lock pins by + sha256, so a committed copy would be a second, un-pinned record of the same numbers -- and one + that goes stale silently the moment the WRI pin moves. Reading it from the pin instead costs + about a tenth of a second and cannot disagree with what it was derived from. + + The single input that is not derivable this way is `tiled_iso_3166s`, which needs a spatial join + against a GeoPackage; that is committed, and `tools/build-tiled-countries.py` regenerates it. + """ + tiled = set(json.loads(TILED_ISO_3166S.read_text())["tiled_iso_3166s"]) + assert tiled, f"{TILED_ISO_3166S} names no countries, so every pair would fail E1" + key_map = json.loads((pull.DATA / "gadm_to_world_bank_admin_1.json").read_text()) + faostat = read_faostat_production() + candidates = tuple( + iter_candidates( + area_coverage=key_map["area_coverage"], + production_by_pair=faostat[faostat["year"] == REFERENCE_YEAR][ + ["iso_3166", "crop_name", "production_kg"] + ], + tiled_iso_3166s=tiled, + ) + ) + assert candidates, ( + "no pair passed E1-E4, which would mean a filter is mis-specified" + ) + frame = pandas.DataFrame.from_records( + [dataclasses.asdict(candidate) for candidate in candidates] + ) + counted = targets.read_document()["provenance"]["eligible_pairs"] + assert len(frame) == counted, ( + f"{len(frame):,d} pairs are eligible but {targets.TARGETS} was chosen against " + f"{counted:,d}; the target set was picked from a different shortlist than this one, so " + "re-derive it before reading either" + ) + # Sorted on the anchor's own figure, the one quantity here that is neither ours nor a judgement. + # Ordering is for reading; the choosing happens in targets.json. + return frame.sort_values("deforestation_tonnes", ascending=False).reset_index( + drop=True + ) + + +def get_perennial_findings(eligible: pandas.DataFrame) -> list[schema.Finding]: + """What it means that some of the largest eligible pairs are crops the pipeline cannot see. + + Sized from the anchors alone, so it needs no capture: these pairs carry WRI's own deforestation + attribution while both legs attribute close to nothing to them. This survived dropping the + scorer unchanged, which is the clearest evidence the ranking was never what produced it. + + BLOCKING rather than advisory. For these pairs the pipeline is not disagreeing with an anchor, it + is structurally unable to produce a comparable number at all. + """ + perennial = eligible[eligible["is_perennial"]] + if not len(perennial): + return [] + at_stake = float(perennial["deforestation_tonnes"].sum()) + largest = perennial.nlargest(1, "deforestation_tonnes").to_dict("records")[0] + return [ + schema.Finding( + slug=PERENNIAL_RESEARCH_SLUG, + severity=schema.Severity.BLOCKING, + message=( + f"{len(perennial):d} eligible pairs are woody perennials, carrying " + f"{at_stake / schema.TONNES_PER_MEGATONNE:,.0f} Mt of WRI-attributed deforestation " + f"between them and led by {largest['iso_3166']!s} {largest['crop_name']!s} at " + f"{float(largest['deforestation_tonnes']) / schema.TONNES_PER_MEGATONNE:,.0f} Mt. " + "GLAD's cropland class excludes perennial woody crops, so these appear as tree cover and " + "both legs exclude them from the emissions numerator while MapSPAM keeps their " + "production in the denominator. The pipeline attributes close to zero where the " + "anchor does not" + ), + confidence=schema.Confidence.HIGH, + magnitude_tonnes=at_stake, + affected_rows=len(perennial), + affected_iso_3166s=tuple(sorted(set(perennial["iso_3166"]))), + ), + schema.Finding( + slug=OIL_PALM_RESEARCH_SLUG, + severity=schema.Severity.ADVISORY, + message=( + "Oil palm is the measured case of the entry above: the sLUC-to-WRI ratio for " + "Indonesian oil palm is 0.008, decomposing as a 0.020 detection term and a 0.38 " + "allocation term. The detection term belongs to the perennial entry" + ), + confidence=schema.Confidence.HIGH, + affected_rows=int( + (eligible["mapspam_code"] == statistical.Crop.OILPALM.value).sum() + ), + ), + ] + + +def get_target_anchor_consistency_findings( + deforestation: pandas.DataFrame, +) -> list[schema.Finding]: + """Targets whose own WRI row gives more deforested area than the crop is harvested on. + + Keyed on the target set rather than on the comparisons, which is the whole point. Orbae reaches + 50 pairs and neither affected target is among them, so this check hung off the comparison frame + would find nothing and its silence would read as agreement -- the failure mode `render_coverage` + exists to prevent everywhere else. + + `get_deforestation_share_findings` counts the same contradiction across WRI's whole release. + That count is what decides whether the anchor is broadly usable; this names the rows we chose to + stand on, and says which of them carries a control. 71 pairs out of 2,349 reads as a rounding + error right up until one of them is the only armed guard on its row. + """ + shares = { + (row["iso_3166"], row["crop_name"]): row["deforestation_share"] + for row in deforestation.to_dict("records") + } + armed = {target.slug for target in targets.iter_control_targets()} + findings = [] + for target in targets.iter_targets(): + # The deforestation frame is keyed on MapSPAM codes, where a target names the crop. An + # unknown name is malformed input rather than a pair to pass over: passing over it would + # drop the guard and report agreement. + crop = statistical.Crop[target.crop_name] + share = shares.get((target.iso_3166, crop.value)) + if share is None or share <= 1.0: + continue + findings.append( + schema.Finding( + # The same slug as the release-wide count: one mechanism, so the register groups + # them rather than ranking the same thing twice. + slug="wri-deforestation-area-exceeds-harvested-area", + severity=schema.Severity.ADVISORY, + message=( + f"{target.slug:s} stands on a WRI row that contradicts itself, giving " + f"{float(share):.3f}x more deforestation-linked area than FAOSTAT reports " + f"harvested for the crop" + + ( + ", and it carries a control" + if target.slug in armed + else f", though it is only `{target.basis:s}` and arms nothing" + ) + + ". A crop cannot be grown on more land than it is harvested from, so a " + "disagreement on this pair is the anchor's before it is ours" + ), + confidence=schema.Confidence.MEDIUM, + affected_rows=1, + affected_iso_3166s=(target.iso_3166,), + ) + ) + return findings + + +def get_anchor_shape_agreement() -> pandas.DataFrame: + """Per crop, whether Orbae and WRI agree on which countries have the highest factor. + + Rank rather than ratio, because Orbae's export is three years off ours and no matching export is + coming. That offset is affordable for rank and not for level: WRI's own provincial rank holds at + +0.961 across the same three years while its level spans 1.42x, so a rank disagreement here is + method rather than vintage. + + National grain only. Both sources key countries on ISO 3166, so this needs no key map, where a + provincial comparison would need a third fuzzy-matched artifact. + """ + national = read_orbae() + national = national[national["admin_level"] == schema.NATIONAL] + rows = [] + for crop_name, group in national.groupby("crop_name"): + wri = read_wri_national_emissions() + wri = wri[(wri["crop_name"] == crop_name) & (wri["year"] == REFERENCE_YEAR)] + merged = group[["iso_3166", "ef_kg_per_kg"]].merge( + wri[wri["ef_kg_per_kg"] > 0][["iso_3166", "ef_kg_per_kg"]], + on="iso_3166", + suffixes=("_orbae", "_wri"), + ) + if len(merged) < MINIMUM_COUNTRIES: + continue + rows.append( + { + "crop_name": crop_name, + "countries": len(merged), + "rank_correlation": merged["ef_kg_per_kg_orbae"] + .rank() + .corr(merged["ef_kg_per_kg_wri"].rank()), + "median_level_ratio": ( + merged["ef_kg_per_kg_orbae"] / merged["ef_kg_per_kg_wri"] + ).median(), + } + ) + return pandas.DataFrame(rows).sort_values("rank_correlation") + + +def get_anchor_disagreement_findings( + agreement: pandas.DataFrame, +) -> list[schema.Finding]: + """Crops where the two external anchors disagree with each other more than usefully. + + This is an anchor-quality row, not a pipeline gap: it says the two sources cannot corroborate each + other for these crops, so neither can be treated as independent confirmation of the other. It is + measurable with no capture, which is why it is seeded rather than waiting. + """ + weak = agreement[ + agreement["rank_correlation"] < MINIMUM_CROSS_COUNTRY_RANK_AGREEMENT + ] + if not len(weak): + return [] + named = ", ".join( + f"{row['crop_name']!s} {float(row['rank_correlation']):+.2f} over " + f"{int(row['countries']):d} countries" + for row in weak.to_dict("records") + ) + return [ + schema.Finding( + slug="external-anchors-disagree-on-shape", + severity=schema.Severity.ADVISORY, + message=( + f"Orbae and WRI barely agree on which countries carry the highest factor for " + f"{len(weak):d} of {len(agreement):d} shared crops: {named:s}. WRI's own provincial " + "rank holds at +0.961 across the same three-year offset, so this is method rather " + "than vintage. For these crops the two anchors cannot corroborate each other, so an " + "agreement with either is not independent confirmation, and the Orbae control " + "measures are rank controls rather than level ones, and `ORBAE_OVER_WRI` is where " + "this disagreement gets measured on every run" + ), + confidence=schema.Confidence.MEDIUM, + affected_rows=int(weak["countries"].sum()), + ) + ] + + +def get_orbae_admin_ids() -> dict[tuple[str, str], str]: + """(iso_3166, Orbae province name) to the World Bank `admin_id` our capture is keyed on. + + Orbae's provincial ids are opaque strings, so its provinces reach anything else by name through a + committed, reviewed map -- a strict lookup, with no matching at runtime. + """ + matched = json.loads((pull.DATA / "orbae_to_world_bank_admin_1.json").read_text())[ + "matched" + ] + return { + (key.split(":", 1)[0], key.split(":", 1)[1]): admin_id + for key, admin_id in matched.items() + } + + +def get_orbae_gadm_ids() -> dict[tuple[str, str], str]: + """The same provinces carried one hop further, to GADM `GID_1`, for joining against WRI. + + WRI keys on GADM where our own rows never do, so this is the World Bank map above composed with + the GADM one. A province the second map cannot reach is dropped rather than guessed at; the map's + own `unmatched` block records which, so the gap is a number and not a surprise. + """ + keys = json.loads((pull.DATA / "gadm_to_world_bank_admin_1.json").read_text())[ + "matched" + ] + world_bank_to_gadm = { + world_bank_id: gadm_id for gadm_id, world_bank_id in keys.items() + } + return { + key: world_bank_to_gadm[admin_id] + for key, admin_id in get_orbae_admin_ids().items() + if admin_id in world_bank_to_gadm + } + + +def iter_orbae_wri_comparisons( + minimum_provinces: int = MINIMUM_PROVINCES, +) -> collections.abc.Iterator[dict[str, object]]: + """Orbae against WRI, one row per (country, crop), as a provincial rank correlation. + + The only comparison that can be made before a capture, because both sides are external. Rank + rather than ratio: Orbae's export is permanently three years off ours and WRI's own level spans + 1.42x over that gap while its provincial rank holds at +0.961. + + FOREST pool throughout, since WRI is forest-only -- comparing its factor against Orbae's total + would be the category mismatch that makes the URY 35x and CAN 109x rows meaningless. + """ + gadm_ids = get_orbae_gadm_ids() + provincial = read_orbae() + provincial = provincial[provincial["admin_level"] == schema.PROVINCIAL] + for (iso_3166, crop_name), group in provincial.groupby(["iso_3166", "crop_name"]): + # A province with no committed key is dropped here rather than joined loosely; the map's + # own `unmatched` block records which, so the gap is a number and not a surprise. + resolved = pandas.Series( + [ + gadm_ids.get((str(iso_3166), name)) + for name in group["jurisdiction_name"] + ], + dtype="object", + index=group.index, + ) + left = group.assign(gadm_id=resolved) + left = left[resolved.notna() & (left["forest_kg_per_kg"] > 0)] + path = pull.get_pulled_path( + remote=pull.get_wri_factor_remote( + crop_code=str(crop_name), + gas_scope="CO2", + grain=pull.get_grain(grain_name="provincial"), + ) + ) + wri = pandas.read_csv(path).rename(columns={"GID_1": "gadm_id"}) + merged = left[["gadm_id", "forest_kg_per_kg"]].merge( + wri[wri[f"EF_{REFERENCE_YEAR:d}"] > 0][ + ["gadm_id", f"EF_{REFERENCE_YEAR:d}"] + ], + on="gadm_id", + ) + if len(merged) < minimum_provinces: + continue + yield { + "iso_3166": str(iso_3166), + "crop_name": str(crop_name), + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.ORBAE_OVER_WRI, + "statistic": schema.Statistic.RANK_CORRELATION, + "ratio": float( + merged["forest_kg_per_kg"] + .rank() + .corr(merged[f"EF_{REFERENCE_YEAR:d}"].rank()) + ), + "provinces": len(merged), + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": len(merged) / len(group), + # Both sides external, so no term of ours is in it; MEDIUM rather than HIGH because rank + # is a weaker claim than level and Orbae is a supplied file rather than a pinned pull. + "confidence": schema.Confidence.MEDIUM, + "worst_tier": schema.SourceTier.SUPPLIED, + } + + +def get_orbae_export_version() -> str: + """The digest of the export the Orbae figures came from, read from the committed lock. + + The bytes rather than the filename: Orbae is `SUPPLIED`, so a new export can arrive under the + same name, and the digest is the only thing tying a frozen baseline to the data it was set + against. + """ + recorded = pull.read_lock().get("orbae", {}).get(ORBAE_EXPORT.name, {}) + digest = recorded.get("sha256") + assert digest, ( + f"{ORBAE_EXPORT.name} has no digest in {pull.LOCK}; run `--stage pull` to pin it. The " + "frozen Orbae baselines are identified by these bytes, so a comparison against an " + "unpinned export cannot say which release it agreed with" + ) + return str(digest)[:SOURCE_VERSION_LENGTH] + + +def get_anchor_versions(measure: targets.Measure) -> dict[str, str]: + """The identity of every external anchor a measure divides, keyed by `schema.Source` name. + + Our own legs are absent rather than null, since `code_version` already identifies them. That + makes ORBAE_OVER_WRI the case this exists for: it has no term of ours, so these versions are its + whole identity, and a change in them is the only thing that can move it. + """ + versions = {} + for source in (measure.numerator, measure.denominator): + match source: + case schema.Source.WRI: + versions[source.name] = pull.WRI_REVISION[:SOURCE_VERSION_LENGTH] + case schema.Source.ORBAE: + versions[source.name] = get_orbae_export_version() + case schema.Source.SLUC | schema.Source.JDLUC: + continue + case _: + raise NotImplementedError( + f"{measure.name:s} divides {source.name:s}, which records no version here; " + "add one before a control can be frozen against it" + ) + return versions + + +def read_efs() -> pandas.DataFrame | None: + """The captured emissions factors, or None before a capture has run. + + Read here rather than through `validation.capture` so the reporting path never imports the + module that runs the pipeline -- the same division `read_forest_pools` uses. + """ + if not pull.EFS.exists(): + return None + return pandas.read_parquet(pull.EFS) + + +def iter_sluc_jdluc_comparisons() -> collections.abc.Iterator[dict[str, object]]: + """Our two legs against each other, nationally, per crop both of them model. + + The only comparison in the design with no external anchor in it, and so the only one where a + disagreement proves one of our own legs wrong rather than raising a question about a yardstick. + It is a filter on `methodology` within one artifact rather than a join between two, which is what + `trace.CANONICAL_KEY` carrying `methodology` buys. + + The published emissions factor is the quantity compared, because that is what a control on this + pair guards. It is not a pure emissions ratio: the legs use different production denominators -- + NASS for jdLUC, MapSPAM for sLUC -- so a movement here is either leg's numerator or either leg's + denominator. Localizing it further would need the two legs' production compared directly, which + nothing here does. + """ + efs = read_efs() + if efs is None: + return + national = efs[ + efs.index.get_level_values("admin_level") == schema.NATIONAL + ].reset_index() + by_methodology = national.pivot_table( + index=["admin_id", "crop_name"], + columns="methodology", + values="emissions_factor_kgco2e_per_kg", + ) + both = by_methodology.dropna( + subset=[schema.STATISTICAL, schema.JURISDICTIONAL_DIRECT] + ).reset_index() + for row in both.to_dict("records"): + jdluc = float(row[schema.JURISDICTIONAL_DIRECT]) + if jdluc <= 0: + # A zero denominator is not a disagreement, it is a leg with nothing to say here. + # jdLUC carries a NASS yield for every crop it models, so what reaches this is an + # admin unit where NASS suppressed or never surveyed one, leaving hectares and + # emissions with no production behind them. + continue + yield { + "iso_3166": str(row["admin_id"]), + # The MapSPAM code, because `get_comparisons` renames back to the canonical name once. + "crop_name": statistical.Crop[str(row["crop_name"])].value, + "emission_pool": schema.EmissionPool.TOTAL, + "measure": targets.Measure.SLUC_OVER_JDLUC, + "statistic": schema.Statistic.RATIO, + "ratio": float(row[schema.STATISTICAL]) / jdluc, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + # Both sides are ours, measured from ingested rasters, and the comparison carries a level + # rather than a shape -- the strongest evidence in the design. + "confidence": schema.Confidence.HIGH, + "worst_tier": schema.SourceTier.MEASURED, + } + + +def iter_sluc_wri_comparisons() -> collections.abc.Iterator[dict[str, object]]: + """Our forest emissions against WRI's published deforestation emissions, tonnes against tonnes. + + Nationally, with no yield, area or production term on either side. A provincial rollup is ruled + out by arithmetic rather than by preference. WRI's provincial + factor is a ratio over deforestation-linked production while its yield factor is a + whole-jurisdiction yield, so the intensity built from the two is emissions over WRI's + deforestation-linked *area* rather than over the crop's: + + intensity x area == (LD / production_defor) x (production_total / area_total) x area + == LD / deforestation_share + + Carrying that on our crop area therefore compares against WRI's own figure multiplied by + 1 / deforestation_share -- 6.55x at the median, 1.02x for CIV COCO but 18.96x for USA MAIZ -- and + the factor does not cancel in the ratio, because our denominator is all of our production where + WRI's is the deforested part. A control tolerance frozen on that would be frozen mostly on the + share. `LD` carries no such basis: it is published in tonnes, and our forest total is the same + quantity in the same units. + + **The cost is the provincial grain, and WRI's release cannot avoid it.** Provincially it publishes + a factor and no production, so no provincial `LD` can be recovered -- the intensity was the only + route to one, and the intensity is what carries the share. The provincial factors keep the one job + where that does not bite: `iter_orbae_wri_comparisons`, which is anchor against anchor, so the + share sits on both sides of the ratio and cancels. + + CO2e on both sides, which the national grain is what buys: WRI publishes CO2e nationally and CO2 + only provincially, so only a national comparison matches our own CO2e. + + FOREST throughout, because WRI is forest-only. Against our total this would be the category + mismatch that makes a grassland-dominated country read as a 35x disagreement. + """ + efs = read_efs() + if efs is None: + return + deforestation = { + (row["iso_3166"], row["crop_name"]): float(row["deforestation_tonnes"]) + for row in read_wri_national_emissions(gas_scope="CO2e").to_dict("records") + if row["year"] == REFERENCE_YEAR and row["deforestation_tonnes"] > 0 + } + levels = efs.index.get_level_values + national = efs[ + (levels("admin_level") == schema.NATIONAL) + & (levels("methodology") == schema.STATISTICAL) + ].reset_index() + national["crop_code"] = [ + statistical.Crop[name].value for name in national["crop_name"] + ] + for row in sorted( + national.to_dict("records"), + key=lambda record: (record["admin_id"], record["crop_code"]), + ): + anchor_tonnes = deforestation.get((row["admin_id"], row["crop_code"])) + if not anchor_tonnes: + continue + if row["production_kg"] <= 0: + # MapSPAM puts no production of this crop in this country, so we attribute nothing and + # the ratio is a hard zero. That is an absence rather than a disagreement, and seven of + # them would drag any statistic over these rows. A zero ratio where production *is* + # present stays: attributing nothing to a crop we do grow is a finding. + continue + yield { + "iso_3166": row["admin_id"], + "crop_name": row["crop_code"], + "emission_pool": schema.EmissionPool.FOREST, + "measure": targets.Measure.SLUC_OVER_WRI, + "statistic": schema.Statistic.RATIO, + "ratio": float(row["forest_emissions_mt"]) / anchor_tonnes, + # Nothing is joined provincially, so nothing can be dropped: both sides are the whole + # national figure, and a zero here means "no provinces used" rather than "a thin join". + "provinces": 0, + "comparability": schema.Comparability.LEVEL, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": 1.0, + # One anchor, and it shares this leg's MapSPAM expansion-share family, so it + # corroborates the forest pool and carbon density rather than the allocation. + "confidence": schema.Confidence.MEDIUM, + "worst_tier": schema.SourceTier.PULLED, + } + + +def iter_orbae_capture_comparisons() -> collections.abc.Iterator[dict[str, object]]: + """Each of our legs against Orbae, province by province, as a rank correlation. + + Rank rather than ratio, permanently. Every Orbae row is assessment year 2023 against our 2020, + and no 2020 export is coming, so level cannot cross the offset while rank very nearly can -- + WRI's own factors span 1.42x at the median over the same three years while its provincial rank + holds at +0.961. + + LUC_ONLY on both sides, which is what makes the two comparable at all: Orbae's published factor + excludes peatland occupation, so ours has to as well. Orbae's own `ef_kg_per_kg` is taken as that + figure rather than the sum of its pool columns. The pools do not reconcile against the total for + a large minority of rows -- 47% of the schema 2.2.0 rows against 14% of the 2.0.0 ones -- which is + the mixed-schema defect `get_orbae_findings` already reports, showing up in the decomposition. The + total is the quantity Orbae publishes; the breakdown is the part that disagrees with itself. + + JDLUC_OVER_ORBAE is the only method-family-matched comparison in the design: every Orbae row is + `Method = jdLUC`, and the USA is the one jurisdiction where we run a jdLUC leg too. + """ + efs = read_efs() + if efs is None: + return + admin_ids = get_orbae_admin_ids() + orbae = read_orbae() + orbae = orbae[ + (orbae["admin_level"] == schema.PROVINCIAL) & (orbae["ef_kg_per_kg"] > 0) + ].copy() + orbae["admin_id"] = [ + admin_ids.get((str(iso_3166), str(name))) + for iso_3166, name in zip( + orbae["iso_3166"], orbae["jurisdiction_name"], strict=True + ) + ] + orbae = orbae.dropna(subset=["admin_id"]) + + levels = efs.index.get_level_values + provincial = efs[levels("admin_level") == schema.PROVINCIAL].reset_index() + provincial["crop_code"] = [ + statistical.Crop[name].value for name in provincial["crop_name"] + ] + # `admin_id` leads with the ISO, which is the only place the country appears on our rows. + provincial["iso_3166"] = provincial["admin_id"].str[:3] + provincial["luc_only_kg_per_kg"] = ( + (provincial["emissions_mt"] - provincial["peatland_occupation_emissions_mt"]) + * schema.KG_PER_TONNE + / provincial["production_kg"] + ) + provincial = provincial[ + (provincial["production_kg"] > 0) & (provincial["luc_only_kg_per_kg"] > 0) + ] + + for methodology, measure in ( + (schema.STATISTICAL, targets.Measure.SLUC_OVER_ORBAE), + (schema.JURISDICTIONAL_DIRECT, targets.Measure.JDLUC_OVER_ORBAE), + ): + ours = provincial[provincial["methodology"] == methodology] + merged = ours.merge( + orbae[["admin_id", "crop_name", "ef_kg_per_kg"]], + left_on=["admin_id", "crop_code"], + right_on=["admin_id", "crop_name"], + suffixes=("", "_orbae"), + ) + for (iso_3166, crop_code), group in merged.groupby(["iso_3166", "crop_code"]): + if len(group) < MINIMUM_PROVINCES: + continue + available = len( + orbae[ + (orbae["iso_3166"] == iso_3166) & (orbae["crop_name"] == crop_code) + ] + ) + yield { + "iso_3166": str(iso_3166), + "crop_name": str(crop_code), + "emission_pool": schema.EmissionPool.LUC_ONLY, + "measure": measure, + "statistic": schema.Statistic.RANK_CORRELATION, + "ratio": float( + group["luc_only_kg_per_kg"] + .rank() + .corr(group["ef_kg_per_kg"].rank()) + ), + "provinces": len(group), + "comparability": schema.Comparability.PATTERN_ONLY, + "aggregation": schema.Aggregation.AS_PUBLISHED, + "coverage_fraction": len(group) / available if available else 0.0, + # Rank is a weaker claim than level, and Orbae is a supplied file rather than a + # pinned pull, so this cannot carry what a LEVEL comparison would. + "confidence": schema.Confidence.MEDIUM, + "worst_tier": schema.SourceTier.SUPPLIED, + } + + +def read_forest_pools() -> pandas.DataFrame | None: + """The conservation bound's inputs from the last capture, or None before one has run. + + None rather than an empty frame, because `report.render` distinguishes them: a missing frame + renders as "not run", where an empty one would render as a table with no country over its bound + and read as a pass. This is the check that outranks every anchor, so its silence must not. + + Read here rather than through `validation.capture` so the reporting path never imports the + module that runs the pipeline. + """ + if not pull.FOREST_POOLS.exists(): + return None + return pandas.read_parquet(pull.FOREST_POOLS) + + +def get_comparisons( + repo_root: pathlib.Path, deforestation: pandas.DataFrame +) -> pandas.DataFrame: + """Every comparison this run can make, with its control expectation attached. + + One row per (target, pool, measure). Orbae-against-WRI needs no capture, both sides being + external; sLUC-against-jdLUC needs one and yields nothing until it has run. A measure with no row + is absent rather than null, so `render_coverage` reports the targets no comparison reached. + + `deforestation` rides along so each row carries whether its own anchor is self-consistent. That + fact belongs on the row for the same reason `comparability` does: a comparison against a WRI row + claiming more deforested area than the crop is harvested from looks identical to a sound one once + it is a number, and it is the row's own anchor that is wrong rather than us. + """ + frame = pandas.DataFrame.from_records( + list(iter_orbae_wri_comparisons()) + + list(iter_sluc_jdluc_comparisons()) + + list(iter_sluc_wri_comparisons()) + + list(iter_orbae_capture_comparisons()) + ) + assert len(frame), "no comparison could be made, which means a join broke" + expectations = { + (control.target.iso_3166, control.target.crop_name, control.measure): control + for control in targets.iter_controls() + } + crop_to_name = { + statistical.Crop[name].value: name for name in statistical.Crop.__members__ + } + controls = [ + expectations.get( + (row["iso_3166"], crop_to_name.get(row["crop_name"], ""), row["measure"]) + ) + for row in frame.to_dict("records") + ] + # Joined on the MapSPAM code, before the rename below puts both sides in different vocabularies. + shares = { + (row["iso_3166"], row["crop_name"]): row["deforestation_share"] + for row in deforestation.to_dict("records") + } + frame["anchor_deforestation_share"] = [ + shares.get((row["iso_3166"], row["crop_name"])) + for row in frame.to_dict("records") + ] + frame["crop_name"] = [crop_to_name.get(code, code) for code in frame["crop_name"]] + frame["is_control"] = [control is not None for control in controls] + # A target pair is what the report is about; everything else is context the comparison happens + # to reach. `is_control` is narrower -- a target can be compared without carrying an expectation. + chosen = {(target.iso_3166, target.crop_name) for target in targets.iter_targets()} + frame["is_target"] = [ + (row["iso_3166"], row["crop_name"]) in chosen + for row in frame.to_dict("records") + ] + frame["baseline"] = [control.baseline if control else None for control in controls] + frame["tolerance"] = [ + control.tolerance if control else targets.DEFAULT_TOLERANCE + for control in controls + ] + frame["baseline_source_version"] = [ + control.baseline_source_version if control else None for control in controls + ] + frame["code_version"] = schema.get_code_version(repo_root=repo_root) + # Every anchor the row divides, not just WRI: an ORBAE_OVER_WRI ratio moves when either side is + # revised, and recording one of the two would let an Orbae re-export pass as a pipeline change. + frame["source_version"] = [ + schema.get_source_version_key(versions=get_anchor_versions(measure=measure)) + for measure in frame["measure"] + ] + return frame + + +def get_unanchored_targets( + comparisons: pandas.DataFrame, +) -> tuple[targets.Target, ...]: + """Targets no comparison reached, so their silence is not read as agreement. + + Today that is most of them: only Orbae-against-WRI can be computed before a capture, and Orbae + covers 12 of the 30 targets. + """ + compared = set(zip(comparisons["iso_3166"], comparisons["crop_name"], strict=True)) + return tuple( + target + for target in targets.iter_targets() + if (target.iso_3166, target.crop_name) not in compared + ) + + +def get_scope_difference() -> pandas.DataFrame: + """CO2e against CO2 for the same country, crop and year. + + WRI publishes both as separate directories of the same product, differing by CH4 and N2O. Having + the ratio means a comparison never has to guess which scope an outside figure came from. + """ + merged = read_wri_national_emissions(gas_scope="CO2").merge( + read_wri_national_emissions(gas_scope="CO2e"), + on=["iso_3166", "crop_name", "year"], + suffixes=("_co2", "_co2e"), + ) + merged = merged[merged["ef_kg_per_kg_co2"] > 0] + merged["scope_ratio"] = merged["ef_kg_per_kg_co2e"] / merged["ef_kg_per_kg_co2"] + return merged[["iso_3166", "crop_name", "year", "scope_ratio"]] + + +def get_deforestation_share( + emissions: pandas.DataFrame, faostat_areas: pandas.DataFrame, year: int +) -> pandas.DataFrame: + """The share of a crop's harvested area WRI treats as deforestation-linked. + + WRI's `production` column is deforestation-linked production, not national, so dividing it by the + yield gives the deforestation-linked area. Against FAOSTAT's total harvested area that becomes a + share, which is the quantity our own expansion share should be compared against -- a share + against a share, with no emissions term in either. + """ + wri_yields = read_wri_yields(grain_name="national") + for_year = emissions[ + (emissions["year"] == year) & (emissions["production_tonnes"] > 0) + ] + merged = for_year.merge(wri_yields, on=["iso_3166", "crop_name"]).merge( + faostat_areas, on=["iso_3166", "crop_name"] + ) + merged = merged[(merged["yield_kg"] > 0) & (merged["area_hectares"] > 0)] + merged["deforestation_hectares"] = ( + merged["production_tonnes"] * schema.KG_PER_TONNE / merged["yield_kg"] + ) + merged["deforestation_share"] = ( + merged["deforestation_hectares"] / merged["area_hectares"] + ) + return merged[ + [ + "iso_3166", + "crop_name", + "deforestation_hectares", + "area_hectares", + "deforestation_share", + ] + ] + + +# --- Orbae ----------------------------------------------------------------------------- +# Three source-specific decisions stand between the export and any join: a commodity crosswalk, a +# product-form rebasing, and a country code whose length depends on the grain. +# **The rebasing is the one that would silently ruin a comparison.** Orbae publishes two of its fifteen +# commodities against a *processed* product -- palm as crude palm oil, sugarcane as cane sugar -- and its +# per-kg factors are per kg of that product. WRI and FAOSTAT are both on the raw commodity, verified at +# 0.993 for oil palm fruit. So Orbae's palm factor reads 3.94x high and its sugarcane factor 8.91x high +# against them until divided by the conversion factor the export helpfully carries. Five of the thirty +# targets are affected. +# **Everything here is PATTERN_ONLY, permanently.** Every row is `Assessment year = 2023` against our +# 2020 comparison year, and no 2020 export is coming, so the offset is a property of the anchor rather +# than a temporary gap. +# That is a limit, not a disqualification, and the difference is measured rather than assumed. WRI +# publishes all five of its own reporting years, so it can be asked what a three-year offset costs. +# Level does not survive it: its own 2020-to-2024 factors span 1.42x at the median and 2.90x at the +# upper decile. +# Provincial *shape* very nearly does: rank correlation between its 2020 and 2023 provincial factors is +# +0.961 at the median over 3,186 (country, crop) pairs, 95% of them above +0.7 and none below +0.3, and +# every Orbae-covered control pair sits between +0.964 and +0.995. So a rank comparison against Orbae +# survives the offset and a ratio does not. +# Two further hazards are recorded rather than fixed. The export mixes schema versions -- 2.0.0, 2.1.0 +# and 2.2.0 in one file -- which is the same defect `code_version` guards against in our own +# artifact. And provincial ids are opaque Orbae strings (`AUS-20230119-1`), so a provincial join needs +# name matching like the GADM-to-World-Bank map, which is not built. + + +ORBAE_EXPORT = pull.RAW / "orbae" / "20260807_orbae_export.zip" +ORBAE_MEMBER_NAME = "20260807_orbae_export.csv" +# The year every row carries. Named so the PATTERN_ONLY reason is checkable rather than remembered, +# and read against the module's REFERENCE_YEAR above -- the offset between the two is what makes +# every Orbae row PATTERN_ONLY. +ORBAE_ASSESSMENT_YEAR = 2023 + + +class OrbaeCommodity(enum.Enum): + """Orbae's commodity name to the MapSPAM code it corresponds to one-for-one. + + Member name is the MapSPAM code; value is Orbae's spelling. Twelve of fifteen map; the three that + do not are in `ORBAE_UNMAPPED_COMMODITIES` with the reason. + """ + + BARL = "Barley" + COCO = "Cocoa" + MAIZ = "Corn" + COTT = "Cotton" + OILP = "Palm" + GROU = "Peanut" + POTA = "Potato" + RAPE = "Rapeseed" + SOYB = "Soy" + SUGC = "Sugarcane" + SUNF = "Sunflower" + WHEA = "Wheat" + + +# Why the other three are absent, so a reader is not left wondering whether they were forgotten. +ORBAE_UNMAPPED_COMMODITIES = { + "Beef cattle": "livestock, and this pipeline models crops", + "Coffee": ( + "MapSPAM splits arabica and robusta where Orbae publishes one green-coffee figure, so " + "assigning it to either would invent the split" + ), + "Oats": "a member of MapSPAM's OCER group rather than a code of its own", +} +ORBAE_COMMODITY_TO_CROP_NAME = {member.value: member.name for member in OrbaeCommodity} +assert not set(ORBAE_COMMODITY_TO_CROP_NAME) & set(ORBAE_UNMAPPED_COMMODITIES) +assert all( + crop_name in ifpri_mapspam.CANONICAL_CROP_CLS.__members__ + for crop_name in ORBAE_COMMODITY_TO_CROP_NAME.values() +) + +ORBAE_TRACEABILITY_TO_ADMIN_LEVEL = { + "0. Jurisdiction - Country": schema.NATIONAL, + "1. State (ADM1)-level": schema.PROVINCIAL, +} +ORBAE_LUC_FACTOR_COLUMN = "LUC emission factor [kg CO2e / kg product]" +ORBAE_LUC_INTENSITY_COLUMN = "LUC emission factor [kg CO2e / hectare]" +ORBAE_CONVERSION_FACTOR_COLUMN = ( + "Product conversion factor [kg input product per kg of output product]" +) +ORBAE_POOL_TO_COLUMN = { + schema.EmissionPool.FOREST: "Forest conversion emission factor [kg CO2e / kg product]", + schema.EmissionPool.PEATLAND_CONVERSION: ( + "Peatland conversion emission factor [kg CO2e / kg product]" + ), + schema.EmissionPool.PEATLAND_OCCUPATION: ( + "Peatland occupation emission factor [kg CO2e / kg product]" + ), +} +# Orbae splits grassland where we derive a single remainder, so both parts are read and summed rather +# than one being chosen. Ours is a single derived remainder with no such split, so the comparison is +# undefined and is not attempted. +ORBAE_GRASSLAND_COLUMNS = ( + "Natural grassland conversion emission factor [kg CO2e / kg product]", + "Pastureland conversion emission factor [kg CO2e / kg product]", +) + + +@dataclasses.dataclass(frozen=True) +class OrbaeRow: + """One Orbae figure, rebased and keyed the way the rest of the tool keys things.""" + + iso_3166: str + admin_level: str + country_name: str + jurisdiction_name: str + crop_name: str + schema_version: str + ef_kg_per_kg: float + intensity_tonnes_per_ha: float + forest_kg_per_kg: float + grassland_kg_per_kg: float + peatland_conversion_kg_per_kg: float + peatland_occupation_kg_per_kg: float + + +def get_orbae_country(admin_level: str, jurisdiction_id: str) -> iso3166.Country | None: + """The country an Orbae jurisdiction id names, or None where it names none. + + The grains use different code lengths: a country row's id is ISO alpha-2 (`AR`), a provincial row's + is prefixed with alpha-3 (`AUS-20230119-1`). Reading both as alpha-2 mis-files seven countries, + because one alpha-3's first two letters are another country's alpha-2 -- CHN reads as Switzerland, + MEX as Montenegro, PRY as Puerto Rico -- and drops POL and URY entirely. + """ + if admin_level == schema.NATIONAL: + return iso3166.countries_by_alpha2.get(jurisdiction_id[:2]) + return iso3166.countries_by_alpha3.get(jurisdiction_id.split("-")[0]) + + +def get_float(row: dict[str, str], column: str) -> float: + value = row.get(column, "") + return float(value) if value else 0.0 + + +def iter_orbae_rows(path_to_zip: pathlib.Path) -> collections.abc.Iterator[OrbaeRow]: + """Every national or provincial row carrying a factor, rebased onto the commodity. + + Rows without a factor are skipped: 41% of the export carries one, the rest being jurisdictions + listed for completeness. ADM2 and ADM3 rows are skipped too -- nothing here joins below the + provincial grain. + """ + with ( + zipfile.ZipFile(file=path_to_zip) as archive, + archive.open(ORBAE_MEMBER_NAME) as member, + ): + for row in csv.DictReader( + io.TextIOWrapper(member, encoding="utf8", errors="replace") + ): + if not row[ORBAE_LUC_FACTOR_COLUMN]: + continue + admin_level = ORBAE_TRACEABILITY_TO_ADMIN_LEVEL.get( + row["Traceability level"] + ) + if admin_level is None: + continue + crop_name = ORBAE_COMMODITY_TO_CROP_NAME.get(row["Commodity"]) + if crop_name is None: + continue + country = get_orbae_country( + admin_level=admin_level, jurisdiction_id=row["Jurisdiction ID"] + ) + if country is None: + continue + # kg of input commodity per kg of published product, so dividing returns a per-kg factor + # on the commodity basis. 1 for thirteen commodities, 3.9405 for palm, 8.9108 for cane. + conversion = ( + get_float(row=row, column=ORBAE_CONVERSION_FACTOR_COLUMN) or 1.0 + ) + yield OrbaeRow( + iso_3166=country.alpha3, + admin_level=admin_level, + country_name=row["Administrative level 0"], + jurisdiction_name=row["Administrative level 1"] + or row["Administrative level 0"], + crop_name=crop_name, + schema_version=row["Version"], + ef_kg_per_kg=get_float(row=row, column=ORBAE_LUC_FACTOR_COLUMN) + / conversion, + intensity_tonnes_per_ha=( + get_float(row=row, column=ORBAE_LUC_INTENSITY_COLUMN) + / schema.KG_PER_TONNE + ), + forest_kg_per_kg=( + get_float( + row=row, column=ORBAE_POOL_TO_COLUMN[schema.EmissionPool.FOREST] + ) + / conversion + ), + grassland_kg_per_kg=sum( + get_float(row=row, column=column) + for column in ORBAE_GRASSLAND_COLUMNS + ) + / conversion, + peatland_conversion_kg_per_kg=( + get_float( + row=row, + column=ORBAE_POOL_TO_COLUMN[ + schema.EmissionPool.PEATLAND_CONVERSION + ], + ) + / conversion + ), + peatland_occupation_kg_per_kg=( + get_float( + row=row, + column=ORBAE_POOL_TO_COLUMN[ + schema.EmissionPool.PEATLAND_OCCUPATION + ], + ) + / conversion + ), + ) + + +def pin_orbae_export(path_to_zip: pathlib.Path = ORBAE_EXPORT) -> None: + """Record the export's digest, because five frozen baselines derive from these bytes. + + Orbae is `SUPPLIED` rather than `PULLED` -- a file placed on the operator's machine, with no URL to + re-retrieve it from -- so the digest is the only thing tying a baseline to the data it was set + against. Swap the export and the Orbae/WRI baselines silently describe a different release. + """ + # No `origin`: a supplied file was never retrieved from anywhere, and its absence says so. + pull.record_digest(key=path_to_zip.name, source="orbae", path=path_to_zip) + + +def read_orbae(path_to_zip: pathlib.Path = ORBAE_EXPORT) -> pandas.DataFrame: + """The export as a frame, with the comparability it is limited to attached to every row. + + `comparability` is a column rather than a caveat because a `PATTERN_ONLY` ratio and a `LEVEL` one + look identical once they are numbers, and averaging the two together is the mistake this prevents. + """ + frame = pandas.DataFrame.from_records( + [dataclasses.asdict(row) for row in iter_orbae_rows(path_to_zip=path_to_zip)] + ) + assert len(frame), f"{path_to_zip} yielded no usable rows" + # One ISO per Orbae country name and one name per ISO. This is the check that would have caught + # the alpha-2 truncation: the mis-mapped rows kept Orbae's own country name while their resolved + # ISO said something else, so the pairing was two-to-one. + pairs = frame.groupby("country_name")["iso_3166"].nunique() + ambiguous = sorted(pairs[pairs > 1].index) + assert not ambiguous, ( + f"country names resolving to several ISOs: {', '.join(ambiguous)}" + ) + reverse = frame.groupby("iso_3166")["country_name"].nunique() + shared = sorted(reverse[reverse > 1].index) + assert not shared, f"ISOs claimed by several country names: {', '.join(shared)}" + + frame["source"] = schema.Source.ORBAE + frame["comparability"] = schema.Comparability.PATTERN_ONLY + frame["discount_basis"] = schema.DiscountBasis.GHGP_LINEAR + frame["gas_scope"] = schema.GasScope.CO2E + frame["reporting_year"] = ORBAE_ASSESSMENT_YEAR + return frame + + +def get_orbae_findings(frame: pandas.DataFrame) -> list[schema.Finding]: + """The two things about this export that limit every comparison drawn from it.""" + versions = sorted(set(frame["schema_version"])) + return [ + schema.Finding( + slug="orbae-vintage-offset", + severity=schema.Severity.BLOCKING, + message=( + f"Every row is assessment year {ORBAE_ASSESSMENT_YEAR:d} against our " + f"{REFERENCE_YEAR:d} comparison year, and no {REFERENCE_YEAR:d} export is coming, " + "so every Orbae measure is permanently PATTERN_ONLY. WRI's own reporting years " + "span 1.42x at the median across the same offset, against a provincial rank " + "correlation of " + "+0.961, so rank survives it and ratio does not. Every sLUC/Orbae and Orbae/WRI " + "control therefore holds a rank expectation, with a tolerance in correlation units" + ), + confidence=schema.Confidence.HIGH, + affected_rows=len(frame), + affected_iso_3166s=tuple(sorted(set(frame["iso_3166"]))), + ), + schema.Finding( + slug="orbae-mixed-schema-version", + severity=schema.Severity.ADVISORY, + message=( + f"The export mixes {len(versions):d} schema versions " + f"({', '.join(versions)}) in one file, so a cross-country comparison drawn from it " + "spans anchor versions. This is the defect a `code_version` column exists to catch " + "in our own artifact, here on the anchor side" + ), + confidence=schema.Confidence.MEDIUM, + affected_rows=len(frame), + ), + ] + + +# The one EPA figure this tool uses, hardcoded rather than parsed: a transcription pipeline for a single +# number is not worth the dependency, and the number is checkable by hand in a minute. +# +# EPA GHG Inventory 1990-2022, published April 2024, Chapter 6, Table 6-40 on page 6-75, "Net CO2 Flux +# from Soil, Dead Organic Matter and Biomass Carbon Stock Changes in Land Converted to Cropland by +# Land-Use Change Category (MMT CO2 Eq.)", row "Grassland Converted to Cropland". +# +# To reproduce, from a machine with network access: +# +# uv run --with pdfplumber python -c " +# import pdfplumber, urllib.request, pathlib +# u = ('https://www.epa.gov/system/files/documents/2024-04/' +# 'us-ghg-inventory-2024-chapter-6-land-use-land-use-change-and-forestry_0.pdf') +# r = urllib.request.Request(u, headers={'User-Agent': 'Mozilla/5.0'}) +# pathlib.Path('ch6.pdf').write_bytes(urllib.request.urlopen(r).read()) +# print([l for p in pdfplumber.open('ch6.pdf').pages if 'Table 6-40:' in (p.extract_text() or '') +# for l in p.extract_text().split(chr(10)) if l.startswith('Grassland Converted')])" +# +# That PDF is sha256 a9914b673a61b3bba926253f9d68168dde2ef1487c388455fd78922a69263d5d, 5.5 MiB, 194 +# pages. It is not in `sources.lock.json` because nothing here retrieves it, so there is nothing to pin. +# +# The comparison year is 2020, where EPA reports 10.6, putting sLUC's 30.9 Mt US grassland figure at +# about 2.9x -- the same comparison `docs/further_research.md` draws. The 2022 column is the easy +# mistake to make here: at 16.3 it is the row's most recent number and it is not our year. +# +# Only one mismatch remains, and it is not the temporal basis. The two sides are both annual: the +# GHGP per-year weights integrate to exactly 1.0 over the 20-year window, so `SPAN_TO_LINEAR_DISCOUNT_WEIGHT` +# allocates a conversion's emissions to one sourcing year rather than accumulating twenty, and the +# production denominator is a weighted mean over the same spans. What does differ is scope -- EPA's row +# covers all cropland where sLUC's 30.9 Mt is the corn-soy-wheat subset -- and closing that gap can only +# raise our side, so the disagreement is a floor rather than an artifact. +EPA_RELEASE = "1990-2022, published 2024" +EPA_GRASSLAND_TO_CROPLAND_MMT_BY_YEAR = { + 1990: 27.3, + 2005: 17.2, + 2018: 13.7, + 2019: 13.0, + 2020: 10.6, + 2021: 16.1, + 2022: 16.3, +} + + +def read_faostat_production() -> pandas.DataFrame: + """FAOSTAT's ingested production and area, cached locally and pinned by digest. + + Read from the ingest rather than from a copy of the upstream archive: the dataset is the single + record of what this data is, and duplicating it here duplicated its provenance too. The first read + fetches from `ingest_root`; every read after it is local, which is what keeps the report fast. + """ + if not FAOSTAT_CACHE.exists(): + logger.info(f"Fetching the ingested FAOSTAT parquet to {FAOSTAT_CACHE}") + FAOSTAT_CACHE.parent.mkdir(parents=True, exist_ok=True) + faostat_production.load().to_parquet(FAOSTAT_CACHE) + pull.record_digest( + key="faostat_production.parquet", + source="faostat", + path=FAOSTAT_CACHE, + origin=faostat_production.DATASET.get_prefix(tile_id="world"), + ) + frame = pandas.read_parquet(FAOSTAT_CACHE) + if frame.index.names != [None]: + frame = frame.reset_index() + return frame.rename(columns={"admin_id": "iso_3166"}) + + +def get_faostat_yields() -> pandas.DataFrame: + """FAOSTAT's yield per country, crop and year, derived rather than read. + + `jdluc.datasets.faostat_production` carries area and production and deliberately not yield, + because a MapSPAM group crop's yield is not the sum of its constituents'. Dividing here + reproduces FAOSTAT's own published yield exactly for the one-to-one crops, which are all this + compares. + + Read through `read_faostat_production`, so this comes from the ingested parquet rather than + from a second copy of the upstream archive. Its digest is recorded in sources.lock.json, so + the bytes are pinned either way. + """ + frame = read_faostat_production() + frame["yield_kg"] = frame["production_kg"] / frame["area_hectares"] + return frame[frame["area_hectares"] > 0][ + ["iso_3166", "crop_name", "year", "yield_kg"] + ] + + +def read_faostat_areas(year: int) -> pandas.DataFrame: + """FAOSTAT's harvested area per country and crop for one year.""" + frame = read_faostat_production() + return frame[frame["year"] == year][["iso_3166", "crop_name", "area_hectares"]] + + +def get_yield_comparison( + faostat_yields: pandas.DataFrame, wri_yields: pandas.DataFrame, year: int +) -> pandas.DataFrame: + """One row per country and crop the two sources both report, with their ratio. + + An inner join, so a crop WRI publishes and FAOSTAT does not simply does not appear. That is the + intended behavior and `get_unpaired_crop_names` reports what it dropped, since a crop silently + absent from a comparison reads as a crop that agreed. + """ + for_year = faostat_yields[faostat_yields["year"] == year] + assert len(for_year), f"FAOSTAT has no rows for {year:d}" + merged = wri_yields.merge( + for_year, on=["iso_3166", "crop_name"], suffixes=("_wri", "_faostat") + ) + merged = merged[(merged["yield_kg_wri"] > 0) & (merged["yield_kg_faostat"] > 0)] + merged["ratio"] = merged["yield_kg_wri"] / merged["yield_kg_faostat"] + return merged + + +def iter_yield_agreements( + comparison: pandas.DataFrame, +) -> collections.abc.Iterator[schema.YieldAgreement]: + """Per crop, the median ratio over countries and the spread around it. + + Median rather than mean, and spread rather than a standard deviation: a handful of countries + with a near-zero denominator produce ratios in the hundreds, which would move a mean and tell + us nothing about the product form. + """ + for crop_name, group in comparison.groupby("crop_name"): + ratios = sorted(group["ratio"]) + if len(ratios) < MINIMUM_COUNTRIES: + continue + yield schema.YieldAgreement( + crop_name=str(crop_name), + countries=len(ratios), + median_ratio=statistics.median(ratios), + lowest_ratio=ratios[0], + highest_ratio=ratios[-1], + ) + + +def get_unpaired_crop_names( + comparison: pandas.DataFrame, wri_yields: pandas.DataFrame +) -> dict[schema.UnpairedReason, tuple[str, ...]]: + """WRI's crops the comparison could not reach, grouped by why. + + Grouped rather than listed, because the reasons are not interchangeable and a single list + invites one explanation to be read over all of them. A group crop cannot be compared until + someone writes down its member items; a crop outside the canonical taxonomy is simply not + mapped yet; and one short of MINIMUM_COUNTRIES has a median that would say more about its + reporting countries than about the crop. + """ + unpaired = set(wri_yields["crop_name"]) - set(comparison["crop_name"]) + compared = set(comparison["crop_name"]) + mapped = {item.name for item in faostat_production.ItemCode} + reasons: dict[schema.UnpairedReason, tuple[str, ...]] = {} + for reason, names in ( + ( + schema.UnpairedReason.SPAM_GROUP, + unpaired & faostat_production.SPAM_GROUP_CROP_NAMES, + ), + ( + schema.UnpairedReason.SPAM_SPLIT, + unpaired & faostat_production.SPLIT_CROP_NAMES, + ), + (schema.UnpairedReason.TOO_FEW_COUNTRIES, (unpaired & mapped) - compared), + ( + schema.UnpairedReason.UNMAPPED, + unpaired + - mapped + - faostat_production.SPAM_GROUP_CROP_NAMES + - faostat_production.SPLIT_CROP_NAMES, + ), + ): + if names: + reasons[reason] = tuple(sorted(names)) + assert sum(map(len, reasons.values())) == len(unpaired), ( + f"{len(unpaired):d} unpaired crops but {sum(map(len, reasons.values())):d} explained" + ) + return reasons + + +def get_yield_findings( + agreements: tuple[schema.YieldAgreement, ...], year: int +) -> list[schema.Finding]: + """A finding per crop whose two yields do not agree, product-form mismatches first. + + A mismatch is BLOCKING rather than a disagreement: it means the comparison was never + like-for-like, so the emissions factors built on that yield cannot be read at all until the + product form is settled. + """ + findings = [] + for agreement in agreements: + if agreement.is_product_form_mismatch: + findings.append( + schema.Finding( + slug=f"yield-product-form-{agreement.crop_name.lower():s}", + severity=schema.Severity.BLOCKING, + message=( + f"{agreement.crop_name:s}: WRI's yield is {agreement.median_ratio:.2f}x " + f"FAOSTAT's across {agreement.countries:d} countries in {year:d}. That is " + "a different product form, not a disagreement -- a milling or ginning " + "yield -- so every emissions factor built on this yield is off by the same " + "factor and cannot be compared until the form is settled" + ), + confidence=schema.Confidence.HIGH, + affected_rows=agreement.countries, + ) + ) + elif agreement.is_beyond_tolerance: + findings.append( + schema.Finding( + slug=f"yield-disagreement-{agreement.crop_name.lower():s}", + severity=schema.Severity.ADVISORY, + message=( + f"{agreement.crop_name:s}: WRI's yield is {agreement.median_ratio:.3f}x " + f"FAOSTAT's across {agreement.countries:d} countries in {year:d}, beyond " + f"the {schema.YIELD_RATIO_TOLERANCE:.0%} tolerance. The two are on the same " + "product form, so this is a denominator difference and it moves every " + "factor for this crop proportionally" + ), + confidence=schema.Confidence.MEDIUM, + affected_rows=agreement.countries, + ) + ) + return findings diff --git a/validation/pull.py b/validation/pull.py new file mode 100644 index 0000000..523f8c8 --- /dev/null +++ b/validation/pull.py @@ -0,0 +1,331 @@ +"""Retrieve the external anchors, exactly as published, and record what was retrieved. + +Not `jdluc.ingest`, which mirrors a registered dataset into managed storage, normalized and +tile-partitioned, so the pipeline can compute over it. This keeps an anchor's published bytes as they +are, under `data/.cache/raw///`, and records each in `data/sources.lock.json`. +Reading an anchor is then a local file read, which is what lets the report run offline. + +Every lock entry has one shape, whichever writer produced it: `path` is where the bytes are, relative +to the cache root, `sha256` and `bytes` describe them, `origin` says where they came from and is +absent for a supplied file that was never retrieved, and `revision` appears where one is pinned. Both +are strings a reader cannot tell apart by inspection, so keeping them in named fields is what stops a +consumer reading one as the other. + +FAOSTAT is absent from the retrieval loop: it arrives as an ingested `jdluc` dataset, and +`record_digest` pins the ingested parquet instead of an upstream URL. + +**The lock is committed; the bytes are not.** The anchors run to 11 MiB across 128 files, and none of +it belongs in a repository when the lock reproduces all of it: a committed record keyed on +content makes a stale or swapped file visible in a diff, where inherited CSVs with no provenance +could not be traced to a source at all. Every source pins a revision, so a re-run reproduces rather +than fetching whatever is published today -- WRI to a commit, FAOSTAT to its digest, since it +publishes no revision. A fresh checkout runs `pull` once and the digests confirm it got what the lock +describes. + + Retrieval only; run it with `python -m validation --stage pull`. +""" + +import collections.abc +import dataclasses +import hashlib +import json +import logging +import pathlib +import shutil +import urllib.request + +logger = logging.getLogger(__name__) + +# The data directory's layout, held here because `prepare` already imports this module and the +# tools/ generators need the same paths. Everything in DATA is committed except CACHE. +DATA = pathlib.Path(__file__).resolve().parent / "data" +LOCK = DATA / "sources.lock.json" +CACHE = DATA / ".cache" +RAW = CACHE / "raw" +# What `validation.capture` writes. Not committed: these are re-keyed extracts of the pipeline's own +# parquet outputs, so a copy here would be a second one. Unlike everything else under CACHE they are +# not re-fetchable either -- producing them takes hours of pipeline -- so a report carrying +# capture-dependent numbers reproduces only alongside the run that made them. `prepare` reads both +# back, so the report can render a capture's results without importing the module that produced them. +CAPTURE = CACHE / "capture" +EFS = CAPTURE / "efs.parquet" +FOREST_POOLS = CAPTURE / "forest_pools.parquet" + +# The commit these files were read at. Bumping it is a deliberate act: every anchor number moves. +WRI_REVISION = "559fe23eb752e9df270a1bf93e7f290044026bab" +WRI_ROOT = f"https://raw.githubusercontent.com/wri/GCSC/{WRI_REVISION:s}/data" +# All 42, because the national rollup needs every commodity, not just the ones we model. The 42 are +# not a choice of ours: they are the crops SPAM maps in 2005, 2010 *and* 2020, which is what the +# allocation needs, since a crop with only one snapshot has no expansion to allocate on (Fitts et al. +# 2025a; docs/validation.md). +WRI_CROP_CODES = ( + "ACOF", + "BANA", + "BARL", + "BEAN", + "CASS", + "CHIC", + "CNUT", + "COCO", + "COTT", + "COWP", + "GROU", + "LENT", + "MAIZ", + "OCER", + "OFIB", + "OILP", + "OOIL", + "OPUL", + "ORTS", + "PIGE", + "PLNT", + "PMIL", + "POTA", + "RAPE", + "RCOF", + "REST", + "RICE", + "SESA", + "SMIL", + "SORG", + "SOYB", + "SUGB", + "SUGC", + "SUNF", + "SWPO", + "TEAS", + "TEMF", + "TOBA", + "TROF", + "VEGE", + "WHEA", + "YAMS", +) +# The four crops SPAM 2020 v2 adds, which therefore have a yield factor and no emission factor -- +# `EF_ADM0_RUBB_CO2.csv` and its three siblings 404 at WRI_REVISION. Spelled out rather than derived +# from the taxonomy, for the same reason `ifpri_mapspam.YEAR_TO_UNRECOVERABLE_CROP_NAMES` is: a +# derived list would absorb a new crop silently, where the point is to notice. Rubber is the one that +# matters -- the largest crop our own leg cannot model, so a factor appearing here would be the first +# external anchor it has ever had. +WRI_CROPS_WITHOUT_FACTORS = ("CITR", "ONIO", "RUBB", "TOMA") +CHUNK_BYTES = 1 << 22 + + +@dataclasses.dataclass(frozen=True) +class Grain: + """A grain WRI publishes factors at, and what differs between the two.""" + + directory: str + gas_scopes: tuple[str, ...] + name: str + token: str + + +# Both gas scopes nationally: one product differing by CH4 and N2O, and having both means a +# comparison +# never has to guess which an anchor figure came from. Provincially the 0.5% cannot matter to a +# shape. +WRI_GRAINS = ( + Grain( + directory="sLUC_emission_factors/deforestation_emission_factors_admin0", + gas_scopes=("CO2", "CO2e"), + name="national", + token="ADM0", + ), + Grain( + directory="sLUC_emission_factors/deforestation_emission_factors_adm1", + gas_scopes=("CO2",), + name="provincial", + token="ADM1", + ), +) +# The yield factors are what make a provincial comparison possible at all: WRI does not publish +# deforestation-linked production provincially, but factor over yield factor is an emissions +# intensity +# per hectare, with no production term left in it. +WRI_YIELD_FACTORS = (("national", "gadm0"), ("provincial", "gadm1")) + + +@dataclasses.dataclass(frozen=True) +class Remote: + """One file to retrieve, and where it lands. + + `relative_path` leads with the source and carries the revision, so the source is read back off + it + rather than repeated, and a pin bump is additive: the previous pin stays readable. + """ + + revision: str + url: str + relative_path: pathlib.PurePosixPath + + @property + def path(self) -> pathlib.Path: + return RAW / self.relative_path + + @property + def source(self) -> str: + return self.relative_path.parts[0] + + +def get_wri_factor_remote(crop_code: str, gas_scope: str, grain: Grain) -> Remote: + name = f"EF_{grain.token:s}_{crop_code:s}_{gas_scope:s}.csv" + return Remote( + revision=WRI_REVISION, + url=f"{WRI_ROOT:s}/{grain.directory:s}/individual_commodities_{gas_scope:s}/{name:s}", + relative_path=pathlib.PurePosixPath( + f"wri/{WRI_REVISION[:12]:s}/{grain.name:s}/{gas_scope:s}/{name:s}" + ), + ) + + +def get_wri_yield_remote(grain_name: str) -> Remote: + suffix = dict(WRI_YIELD_FACTORS)[grain_name] + return Remote( + revision=WRI_REVISION, + url=f"{WRI_ROOT:s}/yield_factors/yield_factor_{suffix:s}.csv", + relative_path=pathlib.PurePosixPath( + f"wri/{WRI_REVISION[:12]:s}/yield_factors/{grain_name:s}.csv" + ), + ) + + +def get_grain(grain_name: str) -> Grain: + """The grain WRI publishes at, by the name the rest of the tool uses for it.""" + by_name = {grain.name: grain for grain in WRI_GRAINS} + assert grain_name in by_name, ( + f"unknown grain {grain_name!r}; WRI publishes at {', '.join(sorted(by_name))}" + ) + return by_name[grain_name] + + +def get_pulled_path(remote: Remote) -> pathlib.Path: + """Where `pull` put a file, refusing to hand back one that is not there. + + Readers resolve a path through the same `Remote` that fetched it, rather than rebuilding the + pinned layout at each call site, so the writer and its readers agree by construction instead of + by convention. + + Absence raises, and that is the point. The lock names a file for every crop at every grain, so + a missing one is a half-run `pull` rather than a fact about what WRI publishes -- and a reader + that shrugged at it would report thinner eligibility in the voice of a result. + """ + assert remote.path.exists(), ( + f"{remote.path} is absent, though {LOCK.name} pins it. That is a broken or half-run pull " + f"rather than a gap in the anchor: run `python -m validation --stage pull`" + ) + return remote.path + + +def iter_remotes() -> collections.abc.Iterator[Remote]: + """Every anchor file the validation reads.""" + for grain in WRI_GRAINS: + for gas_scope in grain.gas_scopes: + for crop_code in WRI_CROP_CODES: + yield get_wri_factor_remote( + crop_code=crop_code, gas_scope=gas_scope, grain=grain + ) + for grain_name, _ in WRI_YIELD_FACTORS: + yield get_wri_yield_remote(grain_name=grain_name) + + +def get_sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +def download(remote: Remote) -> None: + remote.path.parent.mkdir(parents=True, exist_ok=True) + logger.info(f"GET {remote.url:s}") + # Streamed, not read whole: the FAOSTAT archive is 32 MiB and there is no reason to hold it. + with ( + urllib.request.urlopen(remote.url, timeout=300) as response, + remote.path.open("wb") as handle, + ): + shutil.copyfileobj(response, handle, CHUNK_BYTES) + + +def read_lock() -> dict[str, dict[str, dict[str, object]]]: + return json.loads(LOCK.read_text()) if LOCK.exists() else {} + + +def write_lock(lock: dict[str, dict[str, dict[str, object]]]) -> None: + LOCK.parent.mkdir(parents=True, exist_ok=True) + # Sorted and indented so a bump is a readable diff, and with no timestamp: the content is the + # identity, and a clock would make two branches disagree about identical files. + LOCK.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n") + logger.info(f"Wrote {LOCK}") + + +def get_record( + path: pathlib.Path, origin: str | None = None, revision: str | None = None +) -> dict[str, object]: + """One lock entry, in the single shape every writer produces and every reader can rely on. + + `path` is always where the bytes are, relative to the cache root; `origin` is always where they + came from, and is absent for a supplied file that was never retrieved. Keeping the two apart is + the point: their values are indistinguishable by inspection, so a reader that conflated them + would produce a confident, wrong answer. + """ + record: dict[str, object] = { + "path": str(path.relative_to(CACHE)), + "sha256": get_sha256(path=path), + "bytes": path.stat().st_size, + } + if origin is not None: + record["origin"] = origin + if revision is not None: + record["revision"] = revision + return record + + +def record_digest( + key: str, source: str, path: pathlib.Path, origin: str | None = None +) -> None: + """Pin an artifact this tool depends on but does not retrieve. + + FAOSTAT arrives through `jdluc.ingest` rather than through `pull`, so there is no upstream URL to + lock. What can still be pinned is the ingested parquet we actually compute from, which is better + provenance than the bytes it came from: it identifies the artifact the numbers were derived off. + `TabularDataset` records only a hand-maintained `version`, so without this the version would be an + assertion with nothing checking it. + """ + lock = read_lock() + lock.setdefault(source, {})[key] = get_record(path=path, origin=origin) + write_lock(lock=lock) + + +def workflow(overwrite: tuple[str, ...], remotes: tuple[Remote, ...]) -> dict[str, int]: + """Retrieve what is missing or stale, and record every file's digest. + + A file whose digest matches the lock is left alone, so a re-run is cheap and offline. One that + no + longer matches is reported rather than accepted: either it was edited here, or the upstream + moved + under a pin saying it could not have. + """ + lock = read_lock() + counts = {"downloaded": 0, "reused": 0, "changed": 0} + for remote in remotes: + key = str(remote.relative_path) + recorded = lock.get(remote.source, {}).get(key, {}) + wanted = remote.source in overwrite + if not wanted and remote.path.exists() and recorded.get("sha256"): + if get_sha256(path=remote.path) == recorded["sha256"]: + counts["reused"] += 1 + continue + logger.warning( + f"{key:s} no longer matches its recorded digest; re-retrieving" + ) + counts["changed"] += 1 + download(remote=remote) + counts["downloaded"] += 1 + lock.setdefault(remote.source, {})[key] = get_record( + path=remote.path, origin=remote.url, revision=remote.revision + ) + write_lock(lock=lock) + return counts diff --git a/validation/report.py b/validation/report.py new file mode 100644 index 0000000..d2b6387 --- /dev/null +++ b/validation/report.py @@ -0,0 +1,633 @@ +"""Render the comparisons into Markdown: what broke, what disagrees, and what went unchecked. + +Every filter, rescale, factorization and rollup has already happened in `prepare`, so a section here +is a pivot plus a caption. Nothing in this module reads a file or computes a quantity; if a renderer +needs a number that is not in its arguments, the number belongs in `prepare`. + +Three ordering choices matter more than they look. + +The conservation bound goes first, ahead of every anchor. A country whose per-crop forest emissions +exceed the pool available to allocate has an attribution bug, and every anchor comparison for it is +moot until that is fixed -- so it is not a caveat on the tables below, it is the reason not to read +them yet. + +Findings come before tables, sorted by severity then magnitude, because a table invites the reader +to draw a conclusion the findings may have already disqualified. + +Coverage comes last but is not optional. A report that silently omits the targets no anchor covered +reads as though it checked them, and an empty cell is indistinguishable from agreement unless +something says otherwise. + +Sections are individually renderable so a slice can be pasted into a PR description: GitHub caps a +body at 65,536 characters, and the full set over provincial rows will exceed it. No HTML and no +footnote syntax, both unreliable there. +""" + +import typing + +import pandas + +from validation import schema, targets + +# GitHub's pull-request body limit. Exceeding it truncates silently, which is worse than refusing. +PULL_REQUEST_BODY_LIMIT = 65_536 + + +def format_number(value: float | None, precision: int = 3) -> str: + if value is None or pandas.isna(value): + return "—" + return f"{value:.{precision}f}" + + +def escape_cell(value: str) -> str: + """Escape a pipe, which would otherwise end the cell and break the row. + + Not hypothetical: every provincial `jurisdiction_name` is of the form "Angola | Bengo", so any + table naming a province emits broken markdown without this. `tabulate` does not escape them + either, so reaching for a library would not have covered it. + """ + return value.replace("|", "\\|") + + +def format_markdown_table(headers: tuple[str, ...], rows: list[list[str]]) -> str: + """A table, or an honest note that there was nothing to put in one. + + An empty table renders as a header with no body, which reads as "nothing wrong" rather than + "nothing measured", so the two are distinguished here. + """ + if not rows: + return "_(no rows)_" + alignments = ["---" if index == 0 else "--:" for index in range(len(headers))] + return "\n".join( + [ + "| " + " | ".join(escape_cell(value=header) for header in headers) + " |", + "| " + " | ".join(alignments) + " |", + *[ + "| " + " | ".join(escape_cell(value=value) for value in row) + " |" + for row in rows + ], + ] + ) + + +def get_conservation_findings(forest_pools: pandas.DataFrame) -> list[schema.Finding]: + """A finding per jurisdiction attributing more forest emissions than its pool holds. + + The pool is forest conversion over all 2020 cropland with no crop share applied, so it bounds + what + any allocation can distribute. Exceeding it is an attribution error rather than a disagreement, + which is why these are BLOCKING and are reported before any comparison. + + The pool and the attributed total must come from the same run: mixing a pool from one code + version + with a capture from another compares a numerator and denominator computed by different code. + """ + findings = [] + for row in forest_pools.to_dict("records"): + attributed, pool = float(row["attributed_tonnes"]), float(row["pool_tonnes"]) + share = attributed / pool + if share <= 1.0: + continue + findings.append( + schema.Finding( + slug="forest-pool-conservation", + severity=schema.Severity.BLOCKING, + message=( + f"{row['iso_3166']!s} attributes {share:.1%} of its forest pool " + f"({attributed:,.0f} t against {pool:,.0f} t). No " + "allocation can distribute more than the pool holds, so this is an error and " + "every comparison for this jurisdiction is moot until it is fixed" + ), + confidence=schema.Confidence.HIGH, + magnitude_tonnes=attributed - pool, + affected_rows=1, + affected_iso_3166s=(str(row["iso_3166"]),), + ) + ) + return findings + + +def get_control_findings(comparisons: pandas.DataFrame) -> list[schema.Finding]: + """A finding per control that has moved beyond tolerance from its frozen baseline. + + Controls fire against `baseline` only. An inherited figure is a sanity check applied when the + baseline is set, not a threshold; see `targets.Control`. + + Three outcomes, not two. A control with no baseline is carried rather than armed and is filtered + out here. One whose anchors have moved since it was frozen is ADVISORY: its baseline describes a + different yardstick, so the movement cannot be attributed, and calling that a DEFECT would book + an anchor revision as a change in our own pipeline. Only a control measured against the anchors + it was frozen against can produce a DEFECT -- and a measure with no external term skips that + check entirely, having no yardstick that could have moved. + + How far a control has moved depends on what its baseline is. A RATIO is compared relatively, + having no natural scale. A RANK_CORRELATION is compared absolutely, because rho lives on + [-1, 1]: dividing by the baseline would make the weakest-agreeing control the twitchiest -- at + +0.443 a relative 10% fires on a move of 0.044 where +0.886 tolerates 0.089 -- and a baseline + near zero would fire on noise. `schema.Statistic` exists to keep the two apart, and it is read + strictly so that a row which cannot say which it is fails rather than defaulting to the ratio + arithmetic. + """ + controls = comparisons[comparisons["is_control"] & comparisons["baseline"].notna()] + findings = [] + for row in controls.to_dict("records"): + value, baseline = float(row["ratio"]), float(row["baseline"]) + tolerance = float(row["tolerance"]) + frozen_against, ran_against = ( + str(row["baseline_source_version"]), + str(row["source_version"]), + ) + if row["measure"].is_anchored and frozen_against != ran_against: + # Not a DEFECT: nothing is wrong, the yardstick is simply a different one, and reading + # the movement would book an anchor revision as a change in our pipeline. + findings.append( + schema.Finding( + slug=( + f"stale-baseline-{str(row['iso_3166']).lower():s}" + f"-{str(row['crop_name']).lower():s}" + f"-{row['measure'].name.lower():s}" + ), + severity=schema.Severity.ADVISORY, + message=( + f"{row['iso_3166']!s} {row['crop_name']!s} {row['measure'].name:s} is " + f"frozen against {frozen_against:s} but this run read {ran_against:s}, so " + f"its baseline of {baseline:.3f} describes a different yardstick and the " + f"movement to {value:.3f} cannot be attributed. Re-freeze it against the " + "current anchors rather than reading it" + ), + confidence=schema.Confidence(row["confidence"]), + affected_rows=1, + affected_iso_3166s=(str(row["iso_3166"]),), + ) + ) + continue + # pandas does not preserve enum identity through a frame, so restore it from the value. + statistic = schema.Statistic(row["statistic"]) + match statistic: + case schema.Statistic.RATIO: + movement = abs(value - baseline) / abs(baseline) + moved = f"{movement:.1%} away with a tolerance of {tolerance:.0%}" + addendum = "" + case schema.Statistic.RANK_CORRELATION: + movement = abs(value - baseline) + moved = ( + f"{movement:.3f} away in correlation units with a tolerance of " + f"{tolerance:.3f}" + ) + addendum = ( + ". This one holds a rank correlation rather than a ratio, so the movement is " + "in the ordering across jurisdictions, not in a magnitude" + ) + case _: + typing.assert_never(statistic) + if movement <= tolerance: + continue + findings.append( + schema.Finding( + slug=( + f"control-{str(row['iso_3166']).lower():s}-{str(row['crop_name']).lower():s}" + f"-{row['measure'].name.lower():s}" + ), + severity=schema.Severity.DEFECT, + message=( + f"{row['iso_3166']!s} {row['crop_name']!s} {row['measure'].name:s} is " + f"{value:.3f} against a baseline of {baseline:.3f}, {moved:s}. A control " + "moved, so something changed that was not supposed to" + addendum + ), + # pandas stores an IntEnum column as int64, so the enum identity does not survive + # a round trip through the frame and has to be restored here + confidence=schema.Confidence(row["confidence"]), + affected_rows=1, + affected_iso_3166s=(str(row["iso_3166"]),), + ) + ) + return findings + + +def get_provenance_findings(comparisons: pandas.DataFrame) -> list[schema.Finding]: + """Borrowed evidence, and tables built from more than one `code_version`. + + A mixed-version table is a defect: a cross-country comparison drawn from two pipeline versions + is not a comparison. Checked at read time, so it holds regardless of how the artifact was + assembled. + """ + findings = [] + versions = sorted(set(comparisons["code_version"])) + if len(versions) > 1: + findings.append( + schema.Finding( + slug="mixed-code-version", + severity=schema.Severity.DEFECT, + message=( + f"These rows span {len(versions):d} code versions " + f"({', '.join(versions)}). Rows carried forward by a per-ISO merge keep " + "their own version, so a cross-country comparison here would mix " + "pipeline versions" + ), + confidence=schema.Confidence.HIGH, + affected_rows=len(comparisons), + ) + ) + borrowed = comparisons[comparisons["worst_tier"] == schema.SourceTier.BORROWED] + if len(borrowed): + findings.append( + schema.Finding( + slug="borrowed-evidence", + severity=schema.Severity.ADVISORY, + message=( + f"{len(borrowed):d} comparison(s) rest on borrowed inputs with no " + "provenance. Usable for ranking a magnitude, never for a claim" + ), + confidence=schema.Confidence.LOW, + affected_rows=len(borrowed), + affected_iso_3166s=tuple(sorted(set(borrowed["iso_3166"]))), + ) + ) + return findings + + +def render_findings(findings: list[schema.Finding]) -> str: + """Severity first, then magnitude, so a blocking result cannot be buried under advisories.""" + if not findings: + return "### Findings\n\n_None._" + ordered = sorted( + findings, + key=lambda finding: ( + list(schema.Severity).index(finding.severity), + -(finding.magnitude_tonnes or 0.0), + ), + ) + lines = [] + for finding in ordered: + magnitude = ( + f" [{finding.magnitude_tonnes / schema.TONNES_PER_MEGATONNE:,.1f} Mt]" + if finding.magnitude_tonnes + else "" + ) + lines.append( + f"- `{finding.severity.marker:s}` **{finding.slug:s}**{magnitude:s} " + f"(confidence {finding.confidence.name.lower():s}) — {finding.message:s}" + ) + return "### Findings\n\n" + "\n".join(lines) + + +def render_conservation(forest_pools: pandas.DataFrame) -> str: + rows = [ + [ + str(row["iso_3166"]), + format_number( + float(row["attributed_tonnes"]) / schema.TONNES_PER_MEGATONNE, 2 + ), + format_number(float(row["pool_tonnes"]) / schema.TONNES_PER_MEGATONNE, 2), + f"{float(row['attributed_tonnes']) / float(row['pool_tonnes']):.1%}", + "over" + if float(row["attributed_tonnes"]) > float(row["pool_tonnes"]) + else "ok", + ] + for row in forest_pools.sort_values( + "attributed_tonnes", ascending=False + ).to_dict("records") + ] + return ( + "### Forest-pool conservation\n\n" + + format_markdown_table( + headers=("Country", "attributed (Mt)", "pool (Mt)", "share", ""), rows=rows + ) + + "\n\n_The pool is forest conversion over all 2020 cropland with no crop share " + "applied, so it bounds what any allocation can hand out. Needs no anchor and no " + "baseline, which is why it is reported first: a country over 100% has an attribution " + "bug, and its anchor comparisons below are not yet worth reading._" + ) + + +def render_comparisons(comparisons: pandas.DataFrame) -> str: + """The target rows in full, and everything else summarised by measure. + + `comparisons` reaches well beyond the chosen set -- every country where an anchor and our capture + happen to overlap -- and tabulating all of it costs more characters than a pull-request body + holds. The chosen pairs are what the report is about, so those are listed; the rest is a + distribution per measure rather than nothing, because a row that is silently dropped reads as a + row that was never computed. + + `comparability` and `aggregation` are columns rather than footnotes because a `PATTERN_ONLY` + ratio and a `LEVEL` one look identical otherwise, and a rollup on our own weights is not the + anchor's published figure. + """ + on_target = comparisons[comparisons["is_target"]] + rows = [ + [ + f"{row['iso_3166']!s} {row['crop_name']!s}", + str(row["emission_pool"]), + str(row["measure"].name), + str(row["statistic"]), + format_number(float(row["ratio"]), 3), + format_number( + None if pandas.isna(row["baseline"]) else float(row["baseline"]), 3 + ), + str(row["comparability"]), + str(row["aggregation"]) + + ( + f" ({float(row['coverage_fraction']):.0%})" + if row["aggregation"] == schema.Aggregation.ROLLED_UP + else "" + ), + format_number(row["anchor_deforestation_share"], 3), + schema.Confidence(row["confidence"]).name.lower(), + ] + for row in on_target.sort_values( + ["iso_3166", "crop_name", "emission_pool"] + ).to_dict("records") + ] + rest = comparisons[~comparisons["is_target"]] + # Iterated over the distinct measures rather than grouped, because a groupby key types as + # Hashable and a measure carries the name this table is sorted and labeled by. + summary = [] + for measure in sorted(set(rest["measure"]), key=lambda value: value.name): + group = rest[rest["measure"] == measure] + summary.append( + [ + f"{measure.name:s} ({group['statistic'].iloc[0]!s})", + f"{len(group):d}", + format_number(float(group["ratio"].median()), 3), + format_number(float(group["ratio"].quantile(0.1)), 3), + format_number(float(group["ratio"].quantile(0.9)), 3), + ] + ) + return "\n\n".join( + [ + f"### Comparisons — {len(on_target):d} rows over " + f"{len(set(zip(on_target['iso_3166'], on_target['crop_name'], strict=True))):d} " + "chosen pairs", + format_markdown_table( + headers=( + "Target", + "Pool", + "Measure", + "statistic", + "value", + "baseline", + "comparability", + "aggregation", + "deforestation share", + "confidence", + ), + rows=rows, + ), + "_`pattern_only` rows have no production at the provincial grain, so they carry shape " + "and not level. A `rank_correlation` row's value is a correlation rather than a ratio, " + "and its baseline moves in correlation units. `rolled_up` rows use our provincial " + "weights over the coverage shown, so they are not the anchor's own national figure — " + "the difference between the two is itself a measurement. `deforestation share` is how much of " + "the crop's harvested area WRI treats as deforestation-linked; above 1.0 the anchor " + "contradicts itself, since a crop cannot be grown on more land than it is harvested " + "from._", + f"#### The other {len(rest):d} comparisons, which reach beyond the chosen set", + format_markdown_table( + headers=("Measure", "rows", "median", "p10", "p90"), rows=summary + ), + "_These are every pair where an anchor and the capture happen to overlap. They are not " + "targets and carry no expectation, but they are what a chosen pair's figure should be " + "read against — a control at the median of its own measure is a different claim from " + "one in the tail._", + ] + ) + + +def render_coverage( + comparisons: pandas.DataFrame, unanchored: tuple[targets.Target, ...] +) -> str: + """What was not checked, which an empty cell cannot say for itself.""" + lines = [ + f"- {len(comparisons):d} comparison(s) over " + f"{comparisons['iso_3166'].nunique():d} countries.", + f"- {int(comparisons['is_target'].sum()):d} of them are targets.", + ] + if unanchored: + lines.append( + f"- **{len(unanchored):d} target(s) had no anchor at all**: " + + ", ".join(target.slug for target in unanchored) + + ". Their silence is not agreement." + ) + pattern_only = int( + (comparisons["comparability"] == schema.Comparability.PATTERN_ONLY).sum() + ) + if pattern_only: + lines.append( + f"- {pattern_only:d} comparison(s) are shape-only and must not be read as levels." + ) + lines.append( + "- Grassland and peat have no external anchor in any run: WRI is forest-only, Orbae " + "is defective on both, and our carbon densities are the datasets we would check " + "against. Those pools can be sized, never confirmed." + ) + return "### Coverage\n\n" + "\n".join(lines) + + +def render_anchor_emissions( + deforestation: pandas.DataFrame, + findings: list[schema.Finding], + scope_ratios: pandas.Series, + stability: pandas.DataFrame, +) -> str: + """WRI measured against itself, which is all an emissions section can do before a capture. + + Three facts, none needing a number of ours. The reporting-year spread bounds what a comparison + against WRI can resolve while the year is unpinned; the scope ratio removes any guessing about + which gas basis an outside figure used; and the deforestation share is a share against a share, + which is what our expansion share should be compared against once it exists. + """ + quantiles = [0.5, 0.9, 0.99] + rows = [ + [ + "WRI reporting-year spread, max/min over 2020-2024", + *[ + format_number(float(stability["spread"].quantile(q)), 2) + for q in quantiles + ], + ], + [ + "CO2e / CO2, same country, crop and year", + *[format_number(float(scope_ratios.quantile(q)), 4) for q in quantiles], + ], + [ + "deforestation-linked share of harvested area", + *[ + format_number( + float(deforestation["deforestation_share"].quantile(q)), 3 + ) + for q in quantiles + ], + ], + ] + return "\n\n".join( + [ + "### WRI against itself", + format_markdown_table(headers=("Measure", "p50", "p90", "p99"), rows=rows), + render_findings(findings=findings), + "_The spread is WRI's own five reporting years for one country and crop. Those are a " + "series rather than five estimates of one quantity -- each column slides the LSRS " + "20-year window forward over both the loss series and the production denominator -- so " + "the spread is the price of leaving the reporting year unpinned, and the ceiling on " + "what any sLUC/WRI comparison can resolve without pinning it._", + ] + ) + + +def render_eligible( + eligible: pandas.DataFrame, findings: list[schema.Finding], show: int +) -> str: + """The eligible shortlist and the figures to choose from -- not a chosen set. + + Ordered by the anchor's own deforestation figure, so the table reads top-down. The choosing + happens in `data/targets.json`, where each pair carries a written reason. + """ + rows = [ + [ + f"{row['iso_3166']!s} {row['crop_name']!s}", + f"{float(row['deforestation_tonnes']) / schema.TONNES_PER_MEGATONNE:,.2f}", + f"{float(row['production_kg']) / schema.KG_PER_TONNE / 1e6:,.2f}", + "".join( + marker + for marker, present in ( + ("P", row["is_perennial"]), + ("D", row["is_decomposed_group_crop"]), + ) + if present + ) + or "\u2014", + f"{int(row['provincial_units']):d} / {float(row['area_coverage']):.0%}", + ] + for row in eligible.head(show).to_dict("records") + ] + return "\n\n".join( + [ + f"### Eligible pairs \u2014 {len(eligible):d} passed E1\u2013E4, " + f"largest {min(show, len(eligible)):d} by WRI deforestation", + format_markdown_table( + headers=( + "Pair", + "WRI deforestation (Mt)", + "production (Mt)", + "flags", + "provinces / area", + ), + rows=rows, + ), + render_findings(findings=findings), + "_`P` a woody perennial, which GLAD\u2019s cropland class excludes by construction; " + "`D` a crop " + "MapSPAM decomposes rather than observes. This is a shortlist to choose from, not a " + "selected set \u2014 see `data/targets.json` for what was chosen and why._", + ] + ) + + +def render_yield_agreement( + agreements: tuple[schema.YieldAgreement, ...], + findings: list[schema.Finding], + unpaired: dict[schema.UnpairedReason, tuple[str, ...]], + year: int, +) -> str: + """WRI's yields against FAOSTAT's, per crop. + + A denominator section rather than an emissions one, and it belongs ahead of the comparisons for + the same reason conservation does: a crop whose two yields are on different product forms has no + comparable emissions factor, so its rows below are not worth reading. + """ + rows = [ + [ + agreement.crop_name, + str(agreement.countries), + format_number(agreement.median_ratio, 3), + format_number(agreement.lowest_ratio, 2), + format_number(agreement.highest_ratio, 2), + ( + "product form" + if agreement.is_product_form_mismatch + else "off" + if agreement.is_beyond_tolerance + else "ok" + ), + ] + for agreement in sorted(agreements, key=lambda a: a.median_ratio) + ] + sections = [ + f"### Yield agreement, WRI against FAOSTAT {year:d}", + format_markdown_table( + headers=("Crop", "countries", "median", "min", "max", ""), rows=rows + ), + render_findings(findings=findings), + ] + if unpaired: + total = sum(len(names) for names in unpaired.values()) + sections.append( + "\n".join( + [ + f"**{total:d} of WRI's crops are not compared.** Their absence is not " + "agreement.", + "", + *[ + f"- `{', '.join(names)}` — {reason.value:s}" + for reason, names in unpaired.items() + ], + ] + ) + ) + return "\n\n".join(sections) + + +def render( + comparisons: pandas.DataFrame, + forest_pools: pandas.DataFrame | None, + unanchored: tuple[targets.Target, ...], + extra_findings: list[schema.Finding] | None = None, +) -> str: + """The whole document, or as much of it as the available data supports. + + `forest_pools` is None until a capture exists, because the pool needs a raster pass over cached + layers. The conservation section then says it has not run rather than being omitted: a missing + section reads as a passing one, and this is the check that outranks every anchor. + """ + findings = ( + ( + get_conservation_findings(forest_pools=forest_pools) + if forest_pools is not None + else [] + ) + + get_control_findings(comparisons=comparisons) + + get_provenance_findings(comparisons=comparisons) + + (extra_findings or []) + ) + document = "\n\n".join( + [ + "## LUC validation", + render_findings(findings=findings), + render_conservation(forest_pools=forest_pools) + if forest_pools is not None + else ( + "### Forest-pool conservation\n\n_Not run: the pool needs a raster pass over " + "cached layers, so it arrives with the first capture. **This is the check that " + "outranks every anchor**, so nothing below has been qualified by it — a country " + "attributing more forest emissions than its pool holds would make its comparisons " + "moot, and that has not yet been tested._" + ), + render_comparisons(comparisons=comparisons), + render_coverage(comparisons=comparisons, unanchored=unanchored), + ] + ) + return document + + +def get_size_note(document: str) -> str: + """A note appended where the *whole* document will not fit a pull-request body. + + It belongs to whatever assembles the sections, not to one of them: a section that measured only + itself and then described "this document" would stay silent exactly when the assembled set went + over, which is the case that matters. + """ + if len(document) <= PULL_REQUEST_BODY_LIMIT: + return "" + return ( + f"\n\n_This document is {len(document):,d} characters and will not fit in a GitHub " + f"pull-request body ({PULL_REQUEST_BODY_LIMIT:,d}). Render a subset of sections._" + ) diff --git a/validation/schema.py b/validation/schema.py new file mode 100644 index 0000000..e1d855f --- /dev/null +++ b/validation/schema.py @@ -0,0 +1,294 @@ +"""Shared vocabulary: what a row means, and where it came from. + +Nothing here computes anything. It exists so `prepare` and `report` cannot disagree about what a +column holds. + +Two conventions are load-bearing. + +Provenance travels in the row, not in a sidecar. Every produced row carries `code_version` and +`source_version`. A merged artifact can hold rows from several runs, since `capture`'s per-ISO merge +keeps rows it did not recompute, so a table spanning more than one `code_version` is a defect and is +reported as one. + +`Comparability` records whether a ratio can be read as a level. WRI publishes emissions factors at +the provincial grain but no production there, so a provincial per-kg factor cannot be rebased onto +our denominator and carries shape only. Intensity needs no denominator and carries level. Marking +this per row keeps the two from being averaged together. +""" + +import dataclasses +import enum +import hashlib +import pathlib +import subprocess +import typing + +# The emissions columns are metric tonnes despite the `_mt` suffix reading as megatonnes: +# `production_kg = production_mt * KG_PER_TONNE` in jdluc/trace.py settles it. Presentation divides +# by TONNES_PER_MEGATONNE once, in the formatter, so no intermediate carries mixed units. +KG_PER_TONNE = 1000 +TONNES_PER_MEGATONNE = 1e6 +# Uncommitted changes outside these cannot move a number, so they do not belong in `code_version`. +# Same set tools/measure-drift.py uses. +RESULT_BEARING_PATHS = ("jdluc", "pyproject.toml", "uv.lock") +# efs.parquet's own index, and what any other capture is re-keyed onto. admin_id rather than +# jurisdiction_name: a machine identifier, where a display string could be renamed. `methodology` +# is a key level because both legs share one artifact, so it separates sLUC from jdLUC. +CANONICAL_KEY = ("admin_level", "admin_id", "crop_name", "methodology") +NATIONAL = "NATIONAL" +PROVINCIAL = "PROVINCIAL" +# The two legs, as `methodology` records them in efs.parquet. +STATISTICAL = "STATISTICAL" +JURISDICTIONAL_DIRECT = "JURISDICTIONAL_DIRECT" +# A yield ratio outside this is not a disagreement about one quantity, it is two different +# quantities: milling and ginning yields all fall well outside it, and no genuine national yield +# estimate differs from another by half. +PRODUCT_FORM_BOUNDS = (0.5, 2.0) +# Within those bounds, how far a median may sit from parity before it is worth reporting. MapSPAM +# runs about 7% under FAOSTAT, so a tolerance below that would fire on every crop. +YIELD_RATIO_TOLERANCE = 0.15 + + +class Source(enum.StrEnum): + """Who reports a figure. + + SLUC and JDLUC are the legs under test; the rest are what they are tested against. + """ + + SLUC = enum.auto() + JDLUC = enum.auto() + WRI = enum.auto() + ORBAE = enum.auto() + EPA = enum.auto() + FAOSTAT = enum.auto() + GFW_TCL = enum.auto() + SPAWN = enum.auto() + + +class SourceTier(enum.StrEnum): + """How reproducible a figure is. `Confidence` is derived from this.""" + + # Re-downloadable from the pinned revision recorded in sources.lock.json + PULLED = enum.auto() + # Transcribed from a publication, with table and page cited where it is transcribed + TRANSCRIBED = enum.auto() + # Requires a file on the operator's machine; recorded with a sha256 so the bytes are pinned + SUPPLIED = enum.auto() + # Computed here from ingested rasters + MEASURED = enum.auto() + # Copied from a working directory with no recorded provenance. Sufficient for ranking a + # magnitude, not for a claim + BORROWED = enum.auto() + + +class EmissionPool(enum.StrEnum): + """Which pool an emission came from. + + GRASSLAND has no column of its own: it is the remainder after the three named pools are + subtracted from the total. + + LUC_ONLY is the total excluding peatland occupation. Orbae's LUC figure excludes occupation, so + a like-for-like comparison against Orbae uses LUC_ONLY rather than TOTAL. + """ + + FOREST = enum.auto() + GRASSLAND = enum.auto() + PEATLAND_CONVERSION = enum.auto() + PEATLAND_OCCUPATION = enum.auto() + TOTAL = enum.auto() + LUC_ONLY = enum.auto() + # Hectares rather than emissions, for the extent probes + CROPLAND_EXTENT = enum.auto() + + +class GasScope(enum.StrEnum): + """Which gases a figure covers. + + WRI publishes both scopes of one product; they differ by CH4 and N2O, about 0.5% for USA maize. + """ + + CO2 = enum.auto() + CO2E = enum.auto() + + +class Statistic(enum.StrEnum): + """What a control's expected number actually is. + + RANK_CORRELATION exists because Orbae's export is permanently three years off ours: level cannot + cross that offset while rank very nearly can, so those controls hold an ordering rather than a + magnitude. Declared per control so a correlation is never read, averaged or tolerance-checked as + though it were a ratio. + """ + + RATIO = enum.auto() + RANK_CORRELATION = enum.auto() + + +class Comparability(enum.StrEnum): + """Whether a ratio can be read as a level, or as a shape only. See the module docstring.""" + + LEVEL = enum.auto() + PATTERN_ONLY = enum.auto() + + +class Aggregation(enum.StrEnum): + """Whether a figure is as its source published it, or rolled up here. + + A rollup applies our provincial weights, so it is not the source's own national figure. The + difference between a rollup and the published figure isolates the activity-data term. + """ + + AS_PUBLISHED = enum.auto() + ROLLED_UP = enum.auto() + + +class DiscountBasis(enum.StrEnum): + """Whether emissions are GHGP linearly discounted or undiscounted. + + sLUC, WRI and Orbae are all GHGP-LSRS discounted and therefore matched. A committed figure is + not comparable to a discounted one. + """ + + GHGP_LINEAR = enum.auto() + COMMITTED = enum.auto() + + +class Confidence(enum.IntEnum): + """How much weight a finding's magnitude can carry. Ordered, because the register sorts on it. + + Reduced by BORROWED evidence, by having one anchor where a comparison elsewhere has two, and by + PATTERN_ONLY comparability. Carbon density is permanently LOW: SoilGrids, Harris and Huang are + the datasets our carbon densities come from, so there is no external anchor for them. + """ + + LOW = 1 + MEDIUM = 2 + HIGH = 3 + + +class UnpairedReason(enum.Enum): + """Why a crop one source publishes has no counterpart in the other. + + The value is the sentence the report prints, so a reason cannot be shown without its + explanation. None of these is a disagreement: they are all reasons a comparison was never + attempted, which is what makes them worth naming rather than omitting. + """ + + SPAM_GROUP = ( + "MapSPAM aggregates several FAOSTAT items under one name, and the member list is " + "MapSPAM's to define, so no FAOSTAT figure can be assigned yet" + ) + SPAM_SPLIT = ( + "MapSPAM splits one FAOSTAT item in two, so giving either name the figure would invent " + "the split and giving both would double the total" + ) + UNMAPPED = ( + "outside the canonical MapSPAM taxonomy the dataset maps, so it has no FAOSTAT item code " + "at all" + ) + TOO_FEW_COUNTRIES = "reported by too few countries for a median to describe the crop rather than its reporters" + + +class Severity(enum.StrEnum): + """Why a finding is worth reading, independent of its size. + + BLOCKING: a comparison could not be made, and figures depending on it are meaningless. + DEFECT: the run contradicts itself. + ADVISORY: worth knowing. + """ + + BLOCKING = enum.auto() + DEFECT = enum.auto() + ADVISORY = enum.auto() + + @property + def marker(self) -> str: + match self: + case Severity.BLOCKING: + return "!!" + case Severity.DEFECT: + return "XX" + case Severity.ADVISORY: + return "--" + case _: + typing.assert_never(self) + + +@dataclasses.dataclass(frozen=True) +class YieldAgreement: + """How one source's yield for a crop compares against another's, over the countries with both. + + Here rather than in `prepare` because the two thresholds below are what "mismatch" means, and + `report` labels rows by them. A definition both modules read cannot drift between them. + """ + + crop_name: str + countries: int + median_ratio: float + lowest_ratio: float + highest_ratio: float + + @property + def is_product_form_mismatch(self) -> bool: + low, high = PRODUCT_FORM_BOUNDS + return not low <= self.median_ratio <= high + + @property + def is_beyond_tolerance(self) -> bool: + return abs(self.median_ratio - 1.0) > YIELD_RATIO_TOLERANCE + + +@dataclasses.dataclass(frozen=True) +class Finding: + """One thing wrong with one target, sized so findings can be ranked against each other. + + Magnitude, breadth and confidence are carried because severity gives no ordering: it does not + distinguish a gap worth tens of megatonnes across nine countries from one worth a fraction of + that in a single country. + + `slug` is stable across runs, so the same gap stays recognizable as its magnitude moves. Where a + gap corresponds to an entry in docs/further_research.md, `slug` is that entry's heading; this + tool measures the entries in that document rather than maintaining a second list. + """ + + slug: str + severity: Severity + message: str + confidence: Confidence + magnitude_tonnes: float | None = None + affected_rows: int = 0 + affected_iso_3166s: tuple[str, ...] = () + + +def git(*arguments: str, cwd: pathlib.Path) -> str: + result = subprocess.run( + ["git", *arguments], check=True, cwd=cwd, stdout=subprocess.PIPE, text=True + ) + return result.stdout.strip() + + +def get_source_version_key(versions: dict[str, str]) -> str: + """Every external anchor behind a figure, as one comparable string. + + Sorted, so two runs naming the same anchors in either order compare equal. A string rather than + a mapping because it occupies one column and travels into a parquet. + """ + return ",".join(f"{name:s}={versions[name]:s}" for name in sorted(versions)) + + +def get_code_version(repo_root: pathlib.Path) -> str: + """The commit that produced a row, with a digest of any uncommitted result-bearing change. + + A bare SHA does not identify a dirty tree, and two dirty runs at one SHA would be + indistinguishable, so the diff is hashed in. Returned as a single string because it occupies a + single column. + + This identifies the code that ran, not the code whose results were used: jdluc's cache keys on + (module, qualname, version=, args) and never on file contents, so a stale cached layer can + contradict the version recorded here. + """ + sha = git("rev-parse", "HEAD", cwd=repo_root)[:12] + diff = git("diff", "HEAD", "--", *RESULT_BEARING_PATHS, cwd=repo_root) + if not diff: + return sha + return f"{sha:s}+{hashlib.sha256(diff.encode()).hexdigest()[:8]:s}" diff --git a/validation/targets.py b/validation/targets.py new file mode 100644 index 0000000..97d3ffc --- /dev/null +++ b/validation/targets.py @@ -0,0 +1,285 @@ +"""Read the target set, and the relationships within it expected to hold still. + +`data/targets.json` is the single source of truth: which pairs are validated, why each is there, and +which carry an expectation. This module reads it and refuses malformed input -- it does not hold the +set. Selection is a person's judgement over the shortlist `prepare.get_eligible` derives; see +`docs/validation.md` section 3. + +Each control holds two numbers. `inherited` is a figure from earlier work, used once as a sanity check +when the first capture sets `baseline`; a large difference between the two is itself reported. +Controls fire against `baseline` alone, and a null baseline means not yet frozen. Tolerance is a +flagging threshold, not a noise band: the pipeline is deterministic, so movement comes only from a +code or data change. + +What these controls do not cover: WRI shares this leg's MapSPAM expansion-share allocation, so an +sLUC/WRI agreement does not independently confirm allocation. WRI remains independent on the forest +pool and on carbon density, which is what these controls guard. Independent evidence about allocation +comes from Orbae (per-H3, jdLUC-family), the US sLUC-versus-jdLUC comparison, and the forest +conservation bound. +""" + +import collections.abc +import dataclasses +import enum +import functools +import json +import typing + +from validation import pull, schema + +TARGETS = pull.DATA / "targets.json" +# A fraction of the baseline, for RATIO controls. +DEFAULT_TOLERANCE = 0.10 +# Absolute correlation units, for RANK_CORRELATION controls. Rho lives on [-1, 1], so a fraction of +# the baseline makes the weakest-agreeing control the twitchiest: at +0.443 a relative 10% fires on a +# move of 0.044 where +0.886 tolerates 0.089, and a baseline near zero would fire on noise. +DEFAULT_RANK_TOLERANCE = 0.10 +# `gap` marks a pair included *because* it is expected +# to fail -- IDN oil palm is in the set to fail at 0.008 -- which `ranked` would misrepresent. +BASES = frozenset({"ranked", "control", "reserved", "gap"}) +# The two bases that carry an expectation. A pair outside them with a `controls` list, or inside them +# without one, is a contradiction rather than an omission, so reading asserts both directions. +BASES_WITH_CONTROLS = frozenset({"control", "reserved"}) + + +class Measure(enum.Enum): + """A ratio between two sources. The value holds them, so a measure carries its own arithmetic. + + ORBAE_OVER_WRI contains no term of ours. A change in our pipeline moves SLUC_OVER_WRI and + SLUC_OVER_ORBAE together while leaving ORBAE_OVER_WRI unchanged, so an anchor revision is + distinguishable from a change in our own figures. + """ + + SLUC_OVER_WRI = (schema.Source.SLUC, schema.Source.WRI) + SLUC_OVER_ORBAE = (schema.Source.SLUC, schema.Source.ORBAE) + ORBAE_OVER_WRI = (schema.Source.ORBAE, schema.Source.WRI) + SLUC_OVER_JDLUC = (schema.Source.SLUC, schema.Source.JDLUC) + # The only method-family-matched comparison in the design: Orbae's every row is Method=jdLUC, and + # the USA is the one jurisdiction where we run a jdLUC leg too. Our leg is the numerator, matching + # every measure but ORBAE_OVER_WRI, which is anchor-against-anchor and has no term of ours. + JDLUC_OVER_ORBAE = (schema.Source.JDLUC, schema.Source.ORBAE) + + @property + def numerator(self) -> schema.Source: + numerator, _ = self.value + return numerator + + @property + def denominator(self) -> schema.Source: + _, denominator = self.value + return denominator + + @property + def is_anchored(self) -> bool: + """Whether either side of the ratio is an external anchor. + + SLUC_OVER_JDLUC is the one measure where neither is: both terms are ours, so nothing outside + this repository can move it. It therefore has no anchor identity to freeze a baseline + against, and needs none -- `code_version` already identifies everything that can move it, + which is exactly why a disagreement here implicates one of our own legs. + """ + ours = {schema.Source.SLUC, schema.Source.JDLUC} + return not {self.numerator, self.denominator} <= ours + + +@dataclasses.dataclass(frozen=True) +class Target: + iso_3166: str + crop_name: str + basis: str + reason: str + + @property + def slug(self) -> str: + return f"{self.iso_3166:s}-{self.crop_name:s}" + + +@dataclasses.dataclass(frozen=True) +class Control: + """A relationship expected to hold still, with the source of the expectation. + + `tolerance` is in the units its `statistic` implies: a fraction of the baseline for a RATIO, + absolute correlation units for a RANK_CORRELATION. `iter_controls` supplies the appropriate + default when the file names none; the field default here is the RATIO one. + """ + + target: Target + emission_pool: schema.EmissionPool + measure: Measure + inherited: float | None + baseline: float | None + note: str + statistic: schema.Statistic = schema.Statistic.RATIO + tolerance: float = DEFAULT_TOLERANCE + # The anchors the baseline was frozen against, as `schema.get_source_version_key` renders them. + # Null exactly when `baseline` is: a frozen number without the yardstick it was measured with + # cannot distinguish a pipeline change from an anchor revision, so the two travel together. + baseline_source_version: str | None = None + + @property + def is_frozen(self) -> bool: + """Whether a baseline exists to fire against. Until then a control is carried, not armed.""" + return self.baseline is not None + + +def check_document(document: dict[str, typing.Any]) -> None: + """Refuse malformed input. + + Every failure here is malformed input rather than a finding, so it raises: a control naming a + measure that does not exist would otherwise be dropped silently, taking its guard with it. + + Takes a document rather than reading one, so a refusal can be exercised against input that was + never on disk. `data/targets.json` satisfies every check below, which is exactly why: a check + nothing can reach is one that stops holding without saying so. + """ + # The set was chosen against a particular WRI release, and its deforestation figures and counts + # are that release's. If the pin moved underneath it, every written reason would describe + # numbers the tool does not read. + pinned = document["provenance"]["wri_revision"] + assert pinned == pull.WRI_REVISION, ( + f"{TARGETS} was chosen against WRI {pinned[:12]:s} but pull is pinned to " + f"{pull.WRI_REVISION[:12]:s}; re-derive the shortlist and re-freeze the baselines rather " + "than reading the set against a release it was not chosen from" + ) + targets = document["targets"] + assert targets, f"{TARGETS} names no targets" + slugs = [f"{target['iso_3166']:s}-{target['crop_name']:s}" for target in targets] + duplicated = sorted({slug for slug in slugs if slugs.count(slug) > 1}) + assert not duplicated, f"{TARGETS} repeats {', '.join(duplicated)}" + for target, slug in zip(targets, slugs, strict=True): + assert target["basis"] in BASES, f"{slug}: unknown basis {target['basis']!r}" + assert target["reason"].strip(), f"{slug}: no reason given" + rows = target.get("controls", []) + expects = target["basis"] in BASES_WITH_CONTROLS + assert bool(rows) == expects, ( + f"{slug}: basis {target['basis']!r} with {len(rows):d} control(s); " + f"{'expected at least one' if expects else 'expected none'}" + ) + for row in rows: + assert row["measure"] in Measure.__members__, ( + f"{slug}: unknown measure {row['measure']!r}" + ) + assert row["emission_pool"] in schema.EmissionPool.__members__, ( + f"{slug}: unknown emission pool {row['emission_pool']!r}" + ) + # (0, 1] holds under both readings; what differs is what the number means, so the + # message says so rather than the bound changing. + assert 0.0 < row.get("tolerance", DEFAULT_TOLERANCE) <= 1.0, ( + f"{slug}: tolerance {row.get('tolerance')!r} outside (0, 1] -- a fraction of the " + f"baseline for a RATIO, correlation units for a RANK_CORRELATION" + ) + statistic = row.get("statistic", schema.Statistic.RATIO.name) + assert statistic in schema.Statistic.__members__, ( + f"{slug}: unknown statistic {statistic!r}" + ) + # A baseline and the anchors it was frozen against travel together -- a frozen number + # with no yardstick recorded would report an anchor revision as a pipeline change, and a + # yardstick with nothing frozen against it describes nothing. A measure with no external + # term is the exception: there is no yardstick, and recording one would be a fiction. + frozen_against = row.get("baseline_source_versions") + if Measure[row["measure"]].is_anchored: + assert (row["baseline"] is None) == (not frozen_against), ( + f"{slug}: baseline {row['baseline']!r} with " + f"{'no' if not frozen_against else 'a'} baseline_source_versions; a frozen " + "baseline needs the anchor versions it was measured against, and vice versa" + ) + else: + assert not frozen_against, ( + f"{slug}: {row['measure']} divides no external anchor, so it cannot be frozen " + "against one; what can move it is our own code, which `code_version` records" + ) + # A correlation outside [-1, 1] is not a correlation, and a ratio is never negative. + for field in ("inherited", "baseline"): + value = row[field] + if value is None: + continue + if statistic == schema.Statistic.RANK_CORRELATION.name: + assert -1.0 <= value <= 1.0, ( + f"{slug}: {field} {value!r} outside [-1, 1]" + ) + else: + assert value > 0.0, ( + f"{slug}: {field} {value!r} is not a positive ratio" + ) + + +@functools.cache +def read_document() -> dict[str, typing.Any]: + """The parsed file, checked on the way through.""" + document = json.loads(TARGETS.read_text()) + check_document(document=document) + return document + + +def get_default_tolerance(statistic: schema.Statistic) -> float: + """The band a control gets when `targets.json` names none. + + A `match` closed with `assert_never`, so a new `Statistic` member is a mypy error naming it + rather than a rank control silently inheriting the ratio band. + """ + match statistic: + case schema.Statistic.RATIO: + return DEFAULT_TOLERANCE + case schema.Statistic.RANK_CORRELATION: + return DEFAULT_RANK_TOLERANCE + case _: + typing.assert_never(statistic) + + +def iter_targets() -> collections.abc.Iterator[Target]: + """Every validated target, in file order, which is materiality order by construction.""" + for target in read_document()["targets"]: + yield Target( + iso_3166=target["iso_3166"], + crop_name=target["crop_name"], + basis=target["basis"], + reason=target["reason"], + ) + + +def iter_controls() -> collections.abc.Iterator[Control]: + """Every control, flattened across targets, so a caller need not know which pair holds which.""" + for target in read_document()["targets"]: + for row in target.get("controls", []): + # Bound before the yield: the default tolerance depends on it, and the two are in + # different units. + statistic = schema.Statistic[ + row.get("statistic", schema.Statistic.RATIO.name) + ] + yield Control( + target=Target( + iso_3166=target["iso_3166"], + crop_name=target["crop_name"], + basis=target["basis"], + reason=target["reason"], + ), + emission_pool=schema.EmissionPool[row["emission_pool"]], + measure=Measure[row["measure"]], + inherited=row["inherited"], + baseline=row["baseline"], + note=row["note"], + statistic=statistic, + tolerance=row.get( + "tolerance", get_default_tolerance(statistic=statistic) + ), + baseline_source_version=( + schema.get_source_version_key( + versions=row["baseline_source_versions"] + ) + if row.get("baseline_source_versions") + else None + ), + ) + + +def iter_control_targets() -> collections.abc.Iterator[Target]: + """The distinct targets carrying a control, in file order. + + A capture must include all of these: a control whose target was not computed removes its own + guard, and does so silently. + """ + seen: set[str] = set() + for control in iter_controls(): + if control.target.slug not in seen: + seen.add(control.target.slug) + yield control.target