Migrate the dashboard to spaday and serve it outside the cluster

.github/dependabot.yaml

@@ -14,12 +14,3 @@ updates:
     labels:
       - "lang: python"
       - "part: dependencies"
-
-  - package-ecosystem: "npm"
-    directory: "/js"
-    schedule:
-      interval: "monthly"
-    labels:
-      - "lang: javascript"
-      - "part: dependencies"
-

.github/workflows/build.yaml

@@ -39,10 +39,6 @@ jobs:
       with:
         version: ${{ matrix.python-version }}
 
-    - uses: actions-ext/node/setup@main
-      with:
-        version: 22.x
-
     - name: Install dependencies
       run: make develop
 
@@ -64,7 +60,7 @@ jobs:
     - name: Upload test results
       uses: actions/upload-artifact@v7
       with:
-        name: test-results-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}
+        name: test-results-${{ matrix.os }}-${{ matrix.python-version }}
         path: '**/junit.xml'
       if: ${{ always() }}
 

.gitignore

@@ -120,17 +120,6 @@ docs/src/_build/
 docs/superpowers
 index.md
 
-# JS
-js/coverage
-js/dist
-js/lib
-js/node_modules
-js/test-results
-js/playwright-report
-js/*.tgz
-raydar/dashboard/static/*
-!raydar/dashboard/static/index.psp2.js
-
 # Jupyter
 .ipynb_checkpoints
 .autoversion

.vscode/settings.json

@@ -1,3 +1 @@
-{
-    "eslint.workingDirectories": ["./js"]
-}
+{}

Makefile

@@ -1,71 +1,51 @@
 #########
 # BUILD #
 #########
-.PHONY: develop-py develop-js develop
-develop-py:
+.PHONY: develop
+develop:  ## setup project for development
 	uv pip install -e .[develop]
 
-develop-js: requirements-js
-
-develop: develop-js develop-py  ## setup project for development
-
-.PHONY: requirements-py requirements-js requirements
-requirements-py:  ## install prerequisite python build requirements
+.PHONY: requirements
+requirements:  ## install prerequisite python build requirements
 	python -m pip install --upgrade pip toml
 	python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["build-system"]["requires"]))'`
 	python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print(" ".join(c["project"]["optional-dependencies"]["develop"]))'`
 
-requirements-js:  ## install prerequisite javascript build requirements
-	cd js; pnpm install && npx playwright install
-
-requirements: requirements-js requirements-py  ## setup project for development
-
-.PHONY: build-py build-js build
-build-py:
+.PHONY: build
+build:  ## build the project
 	python -m build -w -n
 
-build-js:
-	cd js; pnpm build
-
-build: build-js build-py  ## build the project
-
 .PHONY: install
 install:  ## install python library
 	uv pip install .
 
 #########
 # LINTS #
 #########
-.PHONY: lint-py lint-js lint lints
+.PHONY: lint-py lint-docs lint lints
 lint-py:  ## run python linter with ruff
 	python -m ruff check raydar
 	python -m ruff format --check raydar
 
-lint-js:  ## run js linter
-	cd js; pnpm lint
-
 lint-docs:  ## lint docs with mdformat and codespell
 	python -m mdformat --check README.md docs/wiki/
 	python -m codespell_lib README.md docs/wiki/
 
-lint: lint-js lint-py lint-docs  ## run project linters
+lint: lint-py lint-docs  ## run project linters
 
 # alias
 lints: lint
 
-.PHONY: fix-py fix-js fix-docs fix format
+.PHONY: fix-py fix-docs fix format
 fix-py:  ## fix python formatting with ruff
 	python -m ruff check --fix raydar
 	python -m ruff format raydar
 
-fix-js:  ## fix js formatting
-	cd js; pnpm fix
-
 fix-docs:  ## autoformat docs with mdformat and codespell
 	python -m mdformat README.md docs/wiki/
 	python -m codespell_lib --write README.md docs/wiki/
 
-fix: fix-js fix-py fix-docs  ## run project autoformatters
+fix: fix-py fix-docs  ## run project autoformatters
 
 # alias
 format: fix
@@ -99,18 +79,9 @@ tests-py: test-py
 coverage-py:  ## run python tests and collect test coverage
 	python -m pytest -v raydar/tests --cov=raydar --cov-report term-missing --cov-report xml
 
-.PHONY: test-js tests-js coverage-js
-test-js:  ## run js tests
-	cd js; pnpm test
-
-# alias
-tests-js: test-js
-
-coverage-js: test-js  ## run js tests and collect test coverage
-
 .PHONY: test coverage tests
-test: test-py test-js  ## run all tests
-coverage: coverage-py coverage-js  ## run all tests and collect test coverage
+test: test-py  ## run all tests
+coverage: coverage-py  ## run all tests and collect test coverage
 
 # alias
 tests: test
@@ -135,18 +106,15 @@ major:  ## bump a major version
 ########
 # DIST #
 ########
-.PHONY: dist dist-py dist-js dist-check publish
+.PHONY: dist dist-py dist-check publish
 
 dist-py:  ## build python dists
 	python -m build -w -s
 
-dist-js:  # build js dists
-	cd js; pnpm pack
-
 dist-check:  ## run python dist checker with twine
 	python -m twine check dist/*
 
-dist: clean build dist-js dist-py dist-check  ## build all dists
+dist: clean build dist-py dist-check  ## build all dists
 
 publish: dist  ## publish python assets
 

README.md

@@ -10,18 +10,21 @@
 [![License](https://img.shields.io/github/license/Point72/raydar)](https://github.com/Point72/raydar)
 [![PyPI](https://img.shields.io/pypi/v/raydar.svg)](https://pypi.python.org/pypi/raydar)
 
-A [perspective](https://perspective.finos.org/) powered, user editable ray dashboard via ray serve.
+A [perspective](https://perspective.finos.org/) powered, user editable ray dashboard.
 
 Ray offers powerful metrics visualizations powered by graphana and prometheus. Although useful, the setup can take time - and customizations can be challenging.
 
 Raydar, enables out-of-the-box live cluster metrics and user visualizations for Ray workflows with just a simple pip install. It helps unlock distributed machine learning visualizations on Anyscale clusters, runs live and at scale, is easily customizable, and enables all the in-browser aggregations that [perspective](https://perspective.finos.org/) has to offer.
 
+By default the dashboard runs in **your** process and pulls data over the Ray connection you already have, so the cluster never needs an inbound port.
+
 ![Example](https://media.githubusercontent.com/media/Point72/raydar/refs/heads/main/docs/img/ml_example.gif)
 
 ## Features
 
 - Convenience wrappers for the tracking and persistence of ray GCS task metadata. Can scale beyond the existing ray dashboard / GCS task tracking limitations.
-- Serves a UI through [ray serve](https://docs.ray.io/en/latest/serve/index.html) for the visualization of [perspective](https://github.com/finos/perspective) tables.
+- A UI built with [spaday](https://github.com/1kbgz/spaday), [spaday-perspective](https://github.com/1kbgz/spaday-perspective) and [spaday-webawesome](https://github.com/1kbgz/spaday-webawesome) — authored in Python, with UI state synced over [transports](https://github.com/1kbgz/transports).
+- Serve it locally (no open port on the cluster) or from [ray serve](https://docs.ray.io/en/latest/serve/index.html).
 - A python interface to create and update perspective tables from within ray tasks.
 
 [More information is available in our wiki](https://github.com/Point72/raydar/wiki)
@@ -36,7 +39,8 @@ The raydar module provides an actor which can process collections of ray object
 
 ```python
 from raydar import RayTaskTracker
-task_tracker = RayTaskTracker(enable_perspective_dashboard=True)
+task_tracker = RayTaskTracker(dashboard="local")
+print(task_tracker.dashboard_url)
 ```
 
 Passing collections of object references to this actor's process method causes those references to be tracked in an internal polars dataframe, as they finish running.
@@ -55,7 +59,9 @@ refs = [example_remote_function.remote() for _ in range(100)]
 task_tracker.process(refs)
 ```
 
-The perspective UI is served on port 8000 by default.
+The UI is served from this process on a free local port, printed by `task_tracker.dashboard_url`. Pass `dashboard_port=` to pin it. Data reaches the dashboard over Ray's existing connection, so nothing needs to listen on the cluster.
+
+If your cluster already exposes Ray Serve's HTTP ingress, `dashboard="cluster"` serves the same UI from a Ray Serve deployment instead.
 
 ![Example](https://media.githubusercontent.com/media/Point72/raydar/refs/heads/main/docs/img/example_perspective_dashboard.gif)
 
@@ -65,7 +71,7 @@ Passing a `name` and `namespace` arguments allows the RayTaskTracker to skip con
 from raydar import RayTaskTracker
 
 task_tracker = RayTaskTracker(
-    enable_perspective_dashboard=True,
+    dashboard="local",
     name="my_actor_name",
     namespace="my_actor_namespace"
 )
@@ -109,13 +115,15 @@ for i in range(100):
 
 - _Where is the perspective data stored?_
 
-Currently, in memory. There are plans to integrate alternatives to this configuration, but currently the data is stored in machine memory on the ray head.
+Currently, in memory. With `dashboard="local"` that is the memory of the process that created the `RayTaskTracker`; with `dashboard="cluster"` it is the Ray Serve replica on the ray head.
 
-- _How can I save and restore my perspective layouts?_
+- _Does the cluster need an open port?_
 
-The `Save Layout` button saves a json file containing layout information. Dragging and dropping this file into the UI browser window restores that layout.
+Not with `dashboard="local"`, the default topology. The dashboard binds a port on your own machine and pulls table updates from the tracker actor over Ray, so the browser only ever talks to localhost. `dashboard="cluster"` does need Ray Serve's HTTP ingress to be reachable.
+
+- _How can I save and restore my perspective layouts?_
 
-![Example](https://media.githubusercontent.com/media/Point72/raydar/refs/heads/main/docs/img/layout_restoration.gif)
+Layouts are Python-side. Pass a perspective-workspace layout to the dashboard and it is restored in every connected tab.
 
 ## License
 

docs/wiki/Installation.md

@@ -1,6 +1,6 @@
 ## Pre-requisites
 
-You need Python >=3.10 on your machine to install `raydar`.
+You need Python >=3.11 on your machine to install `raydar`.
 
 ## Install with `pip`
 

docs/wiki/Key-Features.md

@@ -37,42 +37,41 @@ This internal dataframe can be accessed via the `.get_df()` method.
 | e0dc174c83... | `null`                | 0              | `example_remote_function` | ... | 2024-01-29 07:17:09.343 EST     | 2024-01-29 07:17:12.115 EST     | `{"/tmp/ray/session_2024-01-29_07...` | `null`        |
 | f4402ec78d... | `null`                | 0              | `example_remote_function` | ... | 2024-01-29 07:17:09.343 EST     | 2024-01-29 07:17:12.115 EST     | `{"/tmp/ray/session_2024-01-29_07...` | `null`        |
 
-Additionally, setting the `enable_perspective_dashboard` flag to `True` in the `RayTaskTracker`'s construction serves a perspective dashboard with live views of your completed references.
+Additionally, passing `dashboard="local"` to the `RayTaskTracker`'s construction serves a perspective dashboard with live views of your completed references.
 
 ```python
-task_tracker = RayTaskTracker(enable_perspective_dashboard=True)
+task_tracker = RayTaskTracker(dashboard="local")
+print(task_tracker.dashboard_url)
 ```
 
+The dashboard runs in this process and pulls updates from the tracker actor over Ray, so the cluster needs no inbound port. Use `dashboard="cluster"` to serve it from Ray Serve instead, when the cluster's HTTP ingress is reachable.
+
 ![Example](images/example_perspective_dashboard.gif)
 
 ## Create/Store Custom Views
 
-From the developer console, save your workspace layout locally.
-
-```javascript
-let workspace = document.getElementById("perspective-workspace");
-
-// Save the current layout
-workspace.save().then((config) => {
-  // Convert the configuration object to a JSON string
-  let json = JSON.stringify(config);
-
-  // Create a Blob object from the JSON string
-  let blob = new Blob([json], { type: "application/json" });
+Layouts live in Python. Pass a [perspective-workspace](https://perspective.finos.org/) layout and it is restored in every connected tab:
 
-  // Create a download link
-  let link = document.createElement("a");
-  link.href = URL.createObjectURL(blob);
-  link.download = "workspace.json";
+```python
+layout = {
+    "sizes": [1],
+    "detail": {"main": {"type": "tab-area", "widgets": ["task_tracker_data"], "currentIndex": 0}},
+    "master": {"sizes": [], "widgets": []},
+    "mode": "globalFilters",
+    "viewers": {
+        "task_tracker_data": {
+            "table": "task_tracker_data",
+            "plugin": "Datagrid",
+            "group_by": ["func_or_class_name"],
+            "columns": ["state"],
+        }
+    },
+}
 
-  // Append the link to the document body and click it to start the download
-  document.body.appendChild(link);
-  link.click();
-  document.body.removeChild(link);
-});
+task_tracker = RayTaskTracker(dashboard="local", dashboard_options={"layout": layout})
 ```
 
-Then, move this json file to `js/src/layouts/default.json`.
+`dashboard_options` also accepts `title` and `limit` (a per-table row cap). Without a layout override, raydar generates one datagrid tab per table.
 
 ![Example](images/example_perspective_dashboard_layouts.gif)
 
@@ -111,13 +110,11 @@ Specifically, tracked fields include:
 
 ## Custom Sources / Update Logic
 
-The proxy server helpd by the `RayTaskTracker` is exposed via the `.proxy_server()` property, meaning we can create new tables as follows:
+The `RayTaskTracker` can create and update arbitrary tables:
 
 ```python
-task_tracker = RayTaskTracker(enable_perspective_dashboard=True)
-proxy_server = task_tracker.proxy_server()
-proxy_server.remote(
-    "new",
+task_tracker = RayTaskTracker(dashboard="local")
+task_tracker.create_table(
     "metrics_table",
     {
         "node_id": "string",
@@ -133,18 +130,17 @@ proxy_server.remote(
 If a user were to then update this table with data coming from, for example, a pytorch model training loop with metrics:
 
 ```python
-def my_model_training_loop()
-
-	for epoch in range(num_epochs):
+def my_model_training_loop():
+    for epoch in range(num_epochs):
         # ... my training code here ...
 
-		data = dict(
-			node_id=ray.get_runtime_context().get_node_id(),
-			metric_name="loss",
-			value=loss.item(),
-			timestamp=time.time(),
-		)
-		proxy_server.remote("update", "metrics_table", [data])
+        data = dict(
+            node_id=ray.get_runtime_context().get_node_id(),
+            metric_name="loss",
+            value=loss.item(),
+            timestamp=datetime.datetime.now(),
+        )
+        task_tracker.update_table("metrics_table", [data])
 ```
 
 Then they can expose a live view at per-node loss metrics across our model training process: