Push hugging face storage resolution into vortex-cloud rather than vortex-python

Cargo.toml

@@ -170,6 +170,7 @@ glob = "0.3.2"
 goldenfile = "1"
 half = { version = "2.7.1", features = ["std", "num-traits"] }
 hashbrown = "0.17.1"
+http = "1.5.0"
 humansize = "2.1.3"
 indicatif = "0.18.0"
 insta = "1.43"
@@ -207,6 +208,7 @@ parquet-variant = "58.3"
 parquet-variant-compute = "58.3"
 paste = "1.0.15"
 pco = "1.0.1"
+percent-encoding = "2.3.2"
 pin-project-lite = "0.2.15"
 primitive-types = { version = "0.14.0" }
 proc-macro2 = "1.0.95"

docs/api/python/datasets.rst

@@ -19,8 +19,9 @@ transforms.
 
 Hub repositories are streamed in place: files are read with HTTP range requests, so only the
 projected columns and matching rows are ever transferred. Private and gated repositories
-authenticate with the ``token`` argument or the locally saved login. Files are downloaded (with
-the usual Hub caching) only when ``streaming=False`` or ``local_files_only=True``.
+authenticate with the ``token`` argument, ``HF_TOKEN``, or the locally saved login — see
+:doc:`store/huggingface` for the full precedence. Files are downloaded (with the usual Hub
+caching) only when ``streaming=False`` or ``local_files_only=True``.
 
 .. code-block:: python
 

docs/api/python/store.rst

@@ -11,6 +11,7 @@ Vortex arrays support reading and writing to many object storage systems:
    store/gcs
    store/azure
    store/http
+   store/huggingface
    store/local
    store/memory
    store/opendal

docs/api/python/store/huggingface.rst

@@ -0,0 +1,97 @@
+==================
+Hugging Face Hub
+==================
+
+Vortex reads Hugging Face Hub repositories over ``hf://`` URLs. A Hub repository is a set of files
+behind an HTTP endpoint that honours range requests, so no cloud SDK is involved and no extra build
+feature is needed.
+
+.. list-table::
+   :header-rows: 1
+
+   * - URL
+     - Repository kind
+   * - ``hf://datasets/<owner>/<name>[@<revision>][/<path>]``
+     - Dataset
+   * - ``hf://spaces/<owner>/<name>[@<revision>][/<path>]``
+     - Space
+   * - ``hf://<owner>/<name>[@<revision>][/<path>]``
+     - Model
+
+``<revision>`` is a branch, tag or commit, defaulting to ``main``. A revision containing ``/`` must
+be percent-encoded, e.g. ``hf://datasets/org/name@refs%2Fconvert%2Fparquet/data/train.vortex``.
+
+Configuration comes from the same environment variables ``huggingface_hub`` reads:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Variable
+     - Meaning
+   * - ``HF_TOKEN``
+     - API token for private and gated repositories. Falls back to the token file at
+       ``HF_TOKEN_PATH``, then ``$HF_HOME/token``, then ``$HOME/.cache/huggingface/token``.
+   * - ``HF_ENDPOINT``
+     - Hub endpoint, defaulting to ``https://huggingface.co``.
+
+Reading from the Hub
+====================
+
+Pass an ``hf://`` URL directly. Public repositories need no credentials; private and gated ones
+authenticate from ``HF_TOKEN`` or the saved login:
+
+.. code-block:: python
+
+   import vortex as vx
+
+   vxf = vx.open("hf://datasets/org/name/data/train.vortex")
+   for batch in vxf.to_arrow():
+       ...
+
+:class:`vortex.store.HfStore`
+=============================
+
+.. py:class:: vortex.store.HfStore(repo_id, *, repo_type="dataset", revision=None, token=None, endpoint=None)
+
+   A Hugging Face Hub object store, rooted at one repository and revision.
+
+   A URL is enough for most reads, so reach for this class only for the two things a URL cannot
+   express: a token held in a variable rather than the environment, and a read that must stay
+   anonymous even though the environment offers credentials.
+
+   Because the store is rooted at the repository and revision, the path passed alongside it is a
+   path *within* the repository.
+
+   :param repo_id: The repository, as ``"<owner>/<name>"``.
+   :param repo_type: ``"dataset"``, ``"model"`` or ``"space"``. Defaults to ``"dataset"``.
+   :param revision: A branch, tag or commit. Defaults to ``main``. Unlike in a URL, a revision
+       containing ``/`` is passed literally — the store percent-encodes it.
+   :param token: ``None`` (the default) or ``True`` authenticates from ``HF_TOKEN`` or the saved
+       login; ``False`` forces an anonymous read even when credentials are available; a string is
+       used as the token directly.
+   :param endpoint: Hub endpoint. Defaults to ``HF_ENDPOINT``, then ``https://huggingface.co``.
+
+.. code-block:: python
+
+   import vortex as vx
+   from vortex.store import HfStore
+
+   store = HfStore("org/name", revision="refs/convert/parquet", token="hf_...")
+
+   # With `store=`, the path is a path within the repository.
+   vxf = vx.open("data/train.vortex", store=store)
+
+Listing
+=======
+
+The Hub does not implement WebDAV ``PROPFIND``, which is how object-store HTTP listing works, so a
+Hub store cannot list a prefix. Opening a known path works, since that is a ``HEAD`` plus ranged
+``GET``. To expand a glob, list the repository through the Hub's own API first — which is what
+``vortex.datasets.load_dataset`` does — and then open each path it returns.
+
+Hugging Face Datasets
+=====================
+
+``vortex.datasets.load_dataset`` builds on this to load Vortex files from the Hub as Hugging Face
+``Datasets`` objects, expanding globs and pushing projections, filters and row limits into each
+scan. See :doc:`../datasets`.

docs/conf.py

@@ -56,6 +56,10 @@
     # and the `ObjectStore` type alias resolves to the private path. The public class is
     # fully documented in `opendal.rst`; the private path is intentionally not.
     ("py:class", "vortex.store._cos.CosStore"),
+    # `vortex.store.HfStore` is the native class re-exported through `vortex.store._hf`, so
+    # annotations resolve to its `vortex._lib` module path. The public class is fully documented
+    # in `huggingface.rst`; the native path is intentionally not.
+    ("py:class", "vortex._lib.HfStore"),
 ]
 
 doctest_global_setup = "import pyarrow; import vortex; import vortex as vx; import random; random.seed(a=0)"

vortex-cloud/Cargo.toml

@@ -17,14 +17,20 @@ categories = { workspace = true }
 all-features = true
 
 [dependencies]
+http = { workspace = true, optional = true }
 object_store = { workspace = true, features = ["fs"] }
 object_store_opendal = { workspace = true, optional = true }
 opendal = { workspace = true, optional = true }
 parking_lot = { workspace = true, optional = true }
+percent-encoding = { workspace = true, optional = true }
 tracing = { workspace = true, optional = true }
 url = { workspace = true }
 vortex-utils = { workspace = true }
 
+[dev-dependencies]
+rstest = { workspace = true }
+tempfile = { workspace = true }
+
 [features]
 default = []
 # The URL -> ObjectStore registry, plus the cloud backends it resolves URLs to. Kept optional so
@@ -36,6 +42,9 @@ registry = [
     "object_store/gcp",
     "object_store/http",
 ]
+# The Hugging Face Hub, the `hf://` scheme. Served over `object_store`'s HTTP store, so it adds no
+# cloud SDK of its own.
+hf = ["dep:http", "dep:percent-encoding", "object_store/http"]
 # Tencent Cloud COS, the `cos://` scheme.
 cos = [
     "dep:opendal",