Ship as an installable package, fix first-launch data, add an MCP server

.env.example

@@ -9,10 +9,6 @@ LLM_BASE_URL=https://api.openai.com/v1
 LLM_MODEL=gpt-4o
 LLM_API_KEY=sk-...
 
-# ── Exchange APIs (optional — enhances data quality) ──────
-BINANCE_API_KEY=
-BINANCE_API_SECRET=
-
 # ── Alert Notifications (optional) ────────────────────────
 # Telegram: message @BotFather, send /newbot, copy the token
 TELEGRAM_BOT_TOKEN=

.github/workflows/ci.yml

@@ -13,11 +13,11 @@ jobs:
       matrix:
         python-version: ["3.12", "3.13"]
         # "lock": the pinned set from requirements.lock (reproducible).
-        # "pyproject": plain `pip install -e .` — exactly what the README
-        # tells a user to run. Both must pass. The lock leg used to install
-        # the package with dependency resolution disabled, which hid that the
-        # lock contradicted pyproject.toml; it now resolves, and `pip check`
-        # below fails on any conflict.
+        # "pyproject": plain `pip install -e ".[mcp]"`, i.e. what a user gets
+        # from PyPI with the MCP extra. Both must pass. The lock leg used to
+        # install the package with dependency resolution disabled, which hid
+        # that the lock contradicted pyproject.toml; it now resolves, and
+        # `pip check` below fails on any conflict.
         install: ["lock"]
         include:
           - python-version: "3.13"
@@ -38,8 +38,10 @@ jobs:
         run: |
           if [ "${{ matrix.install }}" = "lock" ]; then
             pip install -r requirements.lock
+            pip install -e .
+          else
+            pip install -e ".[mcp]"
           fi
-          pip install -e .
           pip install pytest==9.1.1 pytest-asyncio==1.4.0 ruff==0.15.20 pip-audit==2.10.1
 
       - name: Verify the installed set is consistent
@@ -48,7 +50,7 @@ jobs:
         run: pip check
 
       - name: Lint
-        run: ruff check src/ tests/
+        run: ruff check hyperdata_terminal/ tests/ run_api.py run_dashboard.py
 
       - name: Dependency vulnerability audit
         # Blocking: a known CVE in a pinned runtime dep fails CI. Dependabot
@@ -57,13 +59,45 @@ jobs:
 
       - name: Test imports
         run: |
-          python -c "from src.data_layer.hub import HyperDataHub; print('hub: OK')"
-          python -c "from src.strategies import Strategy, Signal, PaperTrader; print('strategies: OK')"
-          python -c "from src.strategies.llm_agent import LLMAgent; print('llm: OK')"
-          python -c "from src.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard; print('heatmap: OK')"
+          python -c "from hyperdata_terminal.data_layer.hub import HyperDataHub; print('hub: OK')"
+          python -c "from hyperdata_terminal.strategies import Strategy, Signal, PaperTrader; print('strategies: OK')"
+          python -c "from hyperdata_terminal.strategies.llm_agent import LLMAgent; print('llm: OK')"
+          python -c "from hyperdata_terminal.dashboards.liquidation_heatmap import LiquidationHeatmapDashboard; print('heatmap: OK')"
 
       - name: Run tests
         # The full suite, position scanner included — its network calls are
         # mocked; only tests marked `live` need real exchange connections.
         # tests/test_dependency_lock.py asserts the lock satisfies pyproject.
         run: python -m pytest tests/ -q
+
+  package:
+    # Build the wheel and install it the way pipx / uvx / PyPI users do: a
+    # regular (non-editable) install, run from outside the checkout. This is
+    # the path that used to crash with ModuleNotFoundError: run_dashboard.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+
+      - name: Build sdist and wheel
+        run: |
+          pip install build==1.3.0
+          python -m build
+
+      - name: Install the wheel into a clean venv and run it
+        run: |
+          python -m venv /tmp/clean
+          /tmp/clean/bin/pip install dist/*.whl
+          cd /tmp
+          /tmp/clean/bin/hyperdata --version
+          /tmp/clean/bin/hyperdata --help > /dev/null
+          /tmp/clean/bin/hyperdata paper --help > /dev/null
+          /tmp/clean/bin/python -m hyperdata_terminal --version
+          /tmp/clean/bin/python -c "import hyperdata_terminal.terminal, hyperdata_terminal.cli, hyperdata_terminal.mcp_server"
+          site=$(/tmp/clean/bin/python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")
+          for stray in src config; do
+            if [ -e "$site/$stray" ]; then echo "wheel installs a top-level '$stray' package"; exit 1; fi
+          done

.github/workflows/release.yml

@@ -0,0 +1,61 @@
+name: Release
+
+# Publishes to PyPI when a version tag is pushed (git tag v1.0.0 && git push --tags).
+# Uses PyPI Trusted Publishing: no API token is stored anywhere. One-time setup
+# on pypi.org > Account > Publishing > "Add a new pending publisher":
+#   PyPI project name: hyperdata-terminal
+#   Owner: Co-Messi   Repository: HyperData-Terminal
+#   Workflow: release.yml   Environment: pypi
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+
+      - name: Tag must match the package version
+        run: |
+          version=$(python -c "import re,pathlib; print(re.search(r'__version__ = \"(.+?)\"', pathlib.Path('hyperdata_terminal/__init__.py').read_text()).group(1))")
+          if [ "v$version" != "$GITHUB_REF_NAME" ]; then
+            echo "tag $GITHUB_REF_NAME does not match hyperdata_terminal.__version__ ($version)"; exit 1
+          fi
+
+      - name: Build sdist and wheel
+        run: |
+          pip install build==1.3.0
+          python -m build
+
+      - name: Smoke test the wheel
+        run: |
+          python -m venv /tmp/clean
+          /tmp/clean/bin/pip install dist/*.whl
+          cd /tmp && /tmp/clean/bin/hyperdata --version
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/hyperdata-terminal/
+    permissions:
+      id-token: write  # OIDC token for Trusted Publishing; nothing else
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

.gitignore

@@ -24,3 +24,4 @@ venv/
 # Editor/tool backup files (a hook in this environment drops these next to edited files)
 *.bak
 .venv-verify/
+.gstack/

CLAUDE.md

@@ -5,70 +5,92 @@ This file provides guidance to Claude Code when working with this repository.
 ## Commands
 
 ```bash
-# Run tests
-python -m pytest tests/ -v --ignore=tests/test_position_scanner.py
+# Dev install (editable, with the MCP extra)
+pip install -e ".[mcp]" pytest pytest-asyncio ruff
 
-# Run with live exchange connections
+# Run tests (network calls are mocked; tests marked `live` need --live)
+python -m pytest tests/ -q
 python -m pytest tests/ -v --live
 
 # Lint
-ruff check src/
+ruff check hyperdata_terminal tests
 
-# Start terminal dashboard (live data from 5 exchanges)
-python3 run_dashboard.py
+# Terminal dashboards (live data from 5 exchanges)
+hyperdata                      # menu
+hyperdata all --no-boot        # one dashboard directly: liq, stream, heatmap, cvd, market, whales, all
 
-# Start headless API server
-python3 run_api.py --port 8420
+# Headless API server
+hyperdata api --port 8420
 
-# Run the data-integrity verification (cross-checks live data vs Binance/Deribit)
-python3 src/verify_data.py --wait 30
+# Paper trading, MCP server, data-integrity report
+hyperdata paper -s cvd_momentum -s ./my_strategy.py
+hyperdata mcp
+hyperdata verify --wait 30
+
+# Regenerate the README demo GIF (needs vhs)
+vhs assets/demo.tape
 ```
 
-See `docs/DATA_INTEGRITY.md` for coverage caveats (sampled vs confirmed vs
-heuristic liquidations), the staleness watchdog, and the health checks.
+`run_dashboard.py` and `run_api.py` at the repo root are thin shims over the CLI for old clones.
+
+See `docs/DATA_INTEGRITY.md` for coverage caveats (confirmed vs sampled
+liquidations, Hyperliquid large prints, regional fallbacks, CVD warmup), the
+staleness watchdog, and the health checks.
 
 ## Architecture
 
-**HyperDataHub** (`src/data_layer/hub.py`) is the central orchestrator. It owns all data components and manages their async lifecycles via `start()`/`stop()`.
+The package is `hyperdata_terminal/` (PyPI: `hyperdata-terminal`; the PyPI name
+`hyperdata` belongs to someone else). Entry point: `hyperdata_terminal/cli.py`.
+
+**HyperDataHub** (`hyperdata_terminal/data_layer/hub.py`) is the central orchestrator. It owns all data components and manages their async lifecycles via `start()`/`stop()`.
 
 ```
-Exchanges (Hyperliquid, Binance, Bybit, OKX, Deribit)
+Exchanges (Hyperliquid, Binance, Bybit, OKX, Coinbase, Deribit)
     |  WebSocket + REST
     v
 HyperDataHub (14 data components)
     |
-    +---> Terminal Dashboards (Rich TUI)
-    +---> REST API + WebSocket (/v1/*)
-    +---> Paper Trading Engine (pluggable strategies)
+    +---> Terminal dashboards (Rich TUI)          terminal.py, dashboards/
+    +---> REST API + WebSocket (/v1/*)            api_server.py
+    +---> MCP server (stdio)                      mcp_server.py
+    +---> Paper trading (pluggable strategies)    strategies/
 ```
 
-**Data components** (all in `src/data_layer/`): Each follows the same pattern — a dataclass for the data model, a collector/engine class with `start()`/`stop()`, and the hub wires them together.
+**Data components** (all in `hyperdata_terminal/data_layer/`): each is a dataclass for the data model plus a collector/engine class with `start()`/`stop()`; the hub wires them together.
 
 Components: liquidation_feed (4 exchanges), orderflow_engine (CVD), position_scanner, market_data, funding_rates, long_short_ratio, orderbook, spot_prices, deribit (DVOL), smart_money, hlp_tracker, alerts, persistence (SQLite), address_store.
 
-**API Server** (`src/api_server.py`): aiohttp.web embedded in the hub's event loop. All endpoints under `/v1/`. WebSocket at `/v1/ws` streams events. CORS enabled.
+**API Server** (`api_server.py`): aiohttp.web embedded in the hub's event loop. All endpoints under `/v1/`. WebSocket at `/v1/ws` streams events. Loopback by default; no cross-origin access unless `HYPERDATA_CORS_ORIGINS` lists the origin.
+
+**MCP Server** (`mcp_server.py`): `HubTools` holds the tool logic as plain methods over a hub (unit-tested with a fake hub); `build_server()` wires them into `mcp.server.mcpserver.MCPServer` (mcp 2.x) with the hub started in the lifespan. stdout is the protocol: nothing in the data layer may print. Tool wrappers must stay `async def`: the SDK runs plain `def` tools on a worker thread, which races the hub's event loop.
+
+**Paper Trading** (`strategies/`): subclass `Strategy`, implement `evaluate(hub)`, return a `Signal`. `strategies/loader.py` resolves `--strategy` specs (built-in name, `.py` path, `module:Class`).
 
-**Paper Trading** (`src/strategies/`): Pluggable strategy engine. Subclass `Strategy`, implement `evaluate(hub)`, return a `Signal`. The `PaperTrader` runs strategies on real market data with fake money.
+**Persistence** (`data_layer/persistence.py`): SQLite in WAL mode at `<data dir>/hyperdata.db`, written by a dedicated writer thread.
 
-**Persistence** (`src/data_layer/persistence.py`): SQLite in WAL mode at `data/hyperdata.db`. Batch commits every 50 events. Thread-safe via `threading.Lock`.
+**Data dir** (`paths.py`): `HYPERDATA_DATA_DIR`, else an existing `<checkout>/data`, else the per-user data dir. Never site-packages. Resolved once at import; tests set the env var in `conftest.py` before importing the package.
 
 ## Import Conventions
 
-- Hub imports: `from src.data_layer.X import Y`
-- Test imports: `from data_layer.X import Y` (conftest.py adds `src/` to sys.path)
-- API server: `from src.X import Y`
+- Everything, including tests: `from hyperdata_terminal.data_layer.X import Y`
+- No `sys.path` manipulation anywhere.
 
 ## Key Patterns
 
 - **Persistent aiohttp sessions**: Components create `aiohttp.ClientSession()` in `start()`, close in `stop()`. Never create sessions per-request.
-- **WebSocket broadcast**: `_broadcast()` iterates client list copy, uses `_safe_send()` with 2-second timeout. Dead clients removed immediately.
-- **Live by default**: Both product entry points (`run_dashboard.py`, `run_api.py`) run `HyperDataHub(demo=False)` — all data comes from real exchange feeds. A `demo=True` path (synthetic generators in `hub.py`) and the standalone dashboard scripts' `--live`-off mock mode exist only as offline dev/preview tools; they are not part of the shipped product and the health monitor is disabled under demo.
+- **WebSocket broadcast**: `_broadcast()` enqueues per client; each client has a bounded queue drained by its own writer task.
+- **Live by default**: every CLI entry point runs `HyperDataHub(demo=False)`. A `demo=True` path (synthetic generators in `hub_demo.py`) exists only as an offline dev tool; the health monitor is disabled under demo.
+- **Confirmed vs estimated**: Hyperliquid large prints are `LiquidationEvent(confirmed=False)`. Anything that totals liquidations for display, alerts or strategies uses `get_stats(include_estimated=False)`.
+- **HLP**: AUM and session PnL from the parent's `vaultDetails` (never PnL from the AUM change), positions netted per coin, gross exposure per vault, fills via `userFillsByTime` watermarks with forward paging, absorptions grouped by transaction hash and upserted by hash.
+- **Estimated liquidations live in `LiquidationFeed.estimated_events`**, apart from confirmed `events`, so they can never evict confirmed ones.
+- **Source fallbacks**: spot (Binance, Coinbase, OKX) and L/S (Binance, Bybit, OKX) record `source` on every snapshot and skip a failing source for 10 minutes if it is geoblocked (401/403/451), 30 seconds otherwise (`utils.helpers.source_cooldown_seconds`).
 - **Symbol normalization**: `normalize_symbol()` strips USDT/USD/PERP suffixes.
 
 ## External Data Sources
 
-- **Hyperliquid**: Positions, liquidations, funding. WebSocket + REST. No key.
-- **Binance**: Trades, liquidations, orderbook. WebSocket. No key.
-- **Bybit**: Liquidations. WebSocket. No key.
-- **OKX**: Liquidations. WebSocket. No key.
+- **Hyperliquid**: Positions, trades, funding, OI, HLP vault (parent + child vaults). WebSocket + REST. No key.
+- **Binance**: Futures trades, liquidations, orderbook, spot, L/S. WebSocket + REST. No key. Returns 451 in some regions.
+- **Bybit**: Liquidations; L/S fallback. WebSocket + REST. No key. Returns 403 in some regions.
+- **OKX**: Liquidations; spot, L/S and price cross-check fallback. WebSocket + REST. No key.
+- **Coinbase**: Spot fallback for basis. REST. No key.
 - **Deribit**: DVOL implied volatility. REST. No key.