From a3f4ddd8895d7c94fc8649c918802e6e66944981 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 6 Aug 2026 13:53:14 -0400 Subject: [PATCH 1/2] Add GeoPandas and DuckDB usage instructions Added instructions for using GeoPandas and DuckDB with the Data Proxy. Took me some trial and error, so figured it may be helpful to others. --- docs/using-source/data-proxy.md | 51 ++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/using-source/data-proxy.md b/docs/using-source/data-proxy.md index 574a630..da794d2 100644 --- a/docs/using-source/data-proxy.md +++ b/docs/using-source/data-proxy.md @@ -62,10 +62,59 @@ See [Upload Your Data](/data-upload#get-credentials-with-the-source-cli-recommen If you don't have the AWS CLI installed, follow the [AWS CLI Getting Started Guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html). +## Using GeoPandas + +For working with spatial data using Python, you can use [GeoPandas](https://geopandas.org/). You'll need Python and the `geopandas` and `s3fs` packages. + +```sh +pip install geopandas s3fs +``` + +Then, in your code: + +```python +import geopandas + +# Use the Data Proxy +storage_options = { + "endpoint_url": "https://data.source.coop", + "anon": True +} + +gdf = geopandas.read_parquet( + "s3://kerner-lab/fields-of-the-world/vietnam/chips_vietnam.parquet", + storage_options=storage_options, +) +``` + +The code for using pandas (for non-geospatial data) is similar. + +## Using [DuckDB](https://duckdb.org/) + +```sql +-- https://duckdb.org/docs/current/core_extensions/httpfs/overview#installation-and-loading +INSTALL httpfs; +LOAD httpfs; + +-- https://duckdb.org/docs/current/core_extensions/spatial/overview#installing-and-loading +INSTALL spatial; +LOAD spatial; + +CREATE OR REPLACE SECRET secret ( + TYPE s3, + PROVIDER config, + ENDPOINT 'data.source.coop', + URL_STYLE 'path' +); + +SELECT * +FROM 's3://kerner-lab/fields-of-the-world/vietnam/chips_vietnam.parquet'; +``` + ## Current Status The Source Data Proxy is currently in beta. We're working to make performance indistinguishable from accessing objects directly via AWS S3. ## Uploading Data Through the Source Data Proxy -Uploading data through the Source Data Proxy is currently disabled while we work on new, easier, and more secure ways to upload data. If you need to upload data, please contact hello@source.coop. \ No newline at end of file +Uploading data through the Source Data Proxy is currently disabled while we work on new, easier, and more secure ways to upload data. If you need to upload data, please contact hello@source.coop. From d5edbdf075980b7a97821a9b1a5be4a49fed4406 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 6 Aug 2026 14:01:04 -0400 Subject: [PATCH 2/2] add scoping information for DuckDB S3 config --- docs/using-source/data-proxy.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/using-source/data-proxy.md b/docs/using-source/data-proxy.md index da794d2..2d620c4 100644 --- a/docs/using-source/data-proxy.md +++ b/docs/using-source/data-proxy.md @@ -100,11 +100,14 @@ LOAD httpfs; INSTALL spatial; LOAD spatial; -CREATE OR REPLACE SECRET secret ( +CREATE OR REPLACE SECRET source_coop ( TYPE s3, PROVIDER config, ENDPOINT 'data.source.coop', URL_STYLE 'path' + -- If you're working with multiple S3-compatible sources (beyond the Data Proxy), you'll want to only apply this SECRET to the Source Cooperative account name(s). + -- https://duckdb.org/docs/current/configuration/secrets_manager#creating-multiple-secrets-for-the-same-service-type + -- SCOPE 's3://kerner-lab' ); SELECT *