Adding qemu vm driver support with GPU pass-through

architecture/podman-rootless-networking.md

@@ -35,7 +35,7 @@ For rootful bridge networking:
 6. Netavark configures iptables/nftables rules -- masquerade for outbound, DNAT for port mappings
 7. Netavark starts aardvark-dns if DNS is enabled, listening on the bridge gateway address
 
-```
+```text
 Host Kernel
   |
   +-- Bridge interface (e.g., "podman0")  <-- created by Netavark
@@ -60,7 +60,7 @@ Unprivileged users cannot create network interfaces on the host. They cannot cre
 
 Pasta (part of the `passt` project -- same binary, different command name) operates entirely in userspace, translating between the container's L2 TAP interface and the host's L4 sockets. It requires no capabilities or privileges.
 
-```
+```text
 Container Network Namespace
   |
   +-- TAP device (e.g., "eth0")
@@ -131,7 +131,7 @@ Unlike bridge networking, pasta containers are isolated from each other by defau
 
 The Podman compute driver creates three layers of network isolation:
 
-```
+```text
 Namespace 1: Host
   |
   pasta manages port forwarding (127.0.0.1:<ephemeral>)
@@ -164,7 +164,7 @@ client.ensure_network(&config.network_name).await?;
 
 This creates a bridge network named `"openshell"` (default from `DEFAULT_NETWORK_NAME` in `openshell-core/src/config.rs`) with `dns_enabled: true`. In rootless mode, this bridge exists inside a user namespace managed by pasta. The bridge IP range (e.g., `10.89.x.x`) is not routable from the host.
 
-```
+```text
 Host (your machine)
   |
   127.0.0.1:<ephemeral>  <--- pasta binds this on the host
@@ -212,7 +212,7 @@ The bridge gateway IP does NOT work for this purpose in rootless mode because it
 
 Inside the container, the supervisor creates another network namespace (`netns.rs:53-178`, setup at lines 53-63, `ip netns add` at line 77) for the user workload:
 
-```
+```text
 Container (10.89.1.2 on the Podman bridge)
   |
   [Supervisor process - runs in container's default netns]
@@ -247,7 +247,7 @@ A tmpfs is mounted at `/run/netns` in the container spec (`container.rs:458-463`
 
 ### SSH Session: Client to Sandbox Shell
 
-```
+```text
 Client (CLI on user's machine)
   |
   1. gRPC: CreateSshSession -> gateway (returns token, connect_path)
@@ -281,7 +281,7 @@ The SSH daemon listens on a Unix socket (not a TCP port) with 0600 permissions.
 
 ### Outbound HTTP Request from Sandbox Process
 
-```
+```text
 User's code (inner netns, 10.200.0.2)
   |
   1. curl https://api.example.com
@@ -306,7 +306,7 @@ Supervisor proxy (10.200.0.1:3128 in container netns)
 
 ### Supervisor gRPC Callback to Gateway
 
-```
+```text
 Supervisor (container netns, 10.89.x.2)
   |
   1. gRPC connect to http://host.containers.internal:8080

crates/openshell-cli/src/main.rs

@@ -1138,6 +1138,11 @@ enum SandboxCommands {
         #[arg(long)]
         gpu: bool,
 
+        /// Target a specific GPU by PCI address (e.g. "0000:2d:00.0") or index (e.g. "0", "1").
+        /// Only valid with --gpu. When omitted with --gpu, the first available GPU is assigned.
+        #[arg(long, requires = "gpu")]
+        gpu_device: Option<String>,
+
         /// SSH destination for remote bootstrap (e.g., user@hostname).
         /// Only used when no cluster exists yet; ignored if a cluster is
         /// already active.
@@ -2307,6 +2312,7 @@ async fn main() -> Result<()> {
                     no_keep,
                     editor,
                     gpu,
+                    gpu_device,
                     remote,
                     ssh_key,
                     providers,
@@ -2402,6 +2408,7 @@ async fn main() -> Result<()> {
                                 upload_spec.as_ref(),
                                 keep,
                                 gpu,
+                                gpu_device.as_deref(),
                                 editor,
                                 remote.as_deref(),
                                 ssh_key.as_deref(),
@@ -2425,6 +2432,7 @@ async fn main() -> Result<()> {
                                 upload_spec.as_ref(),
                                 keep,
                                 gpu,
+                                gpu_device.as_deref(),
                                 editor,
                                 remote.as_deref(),
                                 ssh_key.as_deref(),

crates/openshell-cli/src/run.rs

@@ -1923,6 +1923,7 @@ pub async fn sandbox_create_with_bootstrap(
     upload: Option<&(String, Option<String>, bool)>,
     keep: bool,
     gpu: bool,
+    gpu_device: Option<&str>,
     editor: Option<Editor>,
     remote: Option<&str>,
     ssh_key: Option<&str>,
@@ -1954,6 +1955,7 @@ pub async fn sandbox_create_with_bootstrap(
         upload,
         keep,
         gpu,
+        gpu_device,
         editor,
         remote,
         ssh_key,
@@ -2010,6 +2012,7 @@ pub async fn sandbox_create(
     upload: Option<&(String, Option<String>, bool)>,
     keep: bool,
     gpu: bool,
+    gpu_device: Option<&str>,
     editor: Option<Editor>,
     remote: Option<&str>,
     ssh_key: Option<&str>,
@@ -2117,6 +2120,7 @@ pub async fn sandbox_create(
     let request = CreateSandboxRequest {
         spec: Some(SandboxSpec {
             gpu: requested_gpu,
+            gpu_device: gpu_device.unwrap_or_default().to_string(),
             policy,
             providers: configured_providers,
             template,

crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs

@@ -574,6 +574,7 @@ async fn sandbox_create_keeps_command_sessions_by_default() {
         None,
         None,
         None,
+        None,
         &[],
         None,
         None,
@@ -615,6 +616,7 @@ async fn sandbox_create_deletes_command_sessions_with_no_keep() {
         None,
         None,
         None,
+        None,
         &[],
         None,
         None,
@@ -659,6 +661,7 @@ async fn sandbox_create_deletes_shell_sessions_with_no_keep() {
         None,
         None,
         None,
+        None,
         &[],
         None,
         None,
@@ -703,6 +706,7 @@ async fn sandbox_create_keeps_sandbox_with_hidden_keep_flag() {
         None,
         None,
         None,
+        None,
         &[],
         None,
         None,
@@ -744,6 +748,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() {
         None,
         None,
         None,
+        None,
         &[],
         None,
         Some(openshell_core::forward::ForwardSpec::new(8080)),

crates/openshell-driver-docker/src/lib.rs

@@ -231,6 +231,7 @@ impl DockerComputeDriver {
             driver_version: self.config.daemon_version.clone(),
             default_image: self.config.default_image.clone(),
             supports_gpu: false,
+            gpu_count: 0,
         }
     }
 

crates/openshell-driver-docker/src/tests.rs

@@ -30,6 +30,7 @@ fn test_sandbox() -> DriverSandbox {
                 platform_config: None,
             }),
             gpu: false,
+            gpu_device: String::new(),
         }),
         status: None,
     }

crates/openshell-driver-kubernetes/src/driver.rs

@@ -150,6 +150,7 @@ impl KubernetesComputeDriver {
             driver_version: openshell_core::VERSION.to_string(),
             default_image: self.config.default_image.clone(),
             supports_gpu: self.has_gpu_capacity().await.unwrap_or(false),
+            gpu_count: 0,
         })
     }
 

crates/openshell-driver-podman/src/driver.rs

@@ -161,6 +161,7 @@ impl PodmanComputeDriver {
             driver_version: openshell_core::VERSION.to_string(),
             default_image: self.config.default_image.clone(),
             supports_gpu,
+            gpu_count: 0,
         })
     }
 

crates/openshell-driver-vm/Cargo.toml

@@ -20,6 +20,7 @@ path = "src/main.rs"
 
 [dependencies]
 openshell-core = { path = "../openshell-core" }
+openshell-vfio = { path = "../openshell-vfio" }
 
 tokio = { workspace = true }
 tonic = { workspace = true, features = ["transport"] }
@@ -32,6 +33,8 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 miette = { workspace = true }
 url = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 libc = "0.2"
 libloading = "0.8"
 tar = "0.4"

crates/openshell-driver-vm/README.md

@@ -36,9 +36,17 @@ mise run gateway:vm
 ```
 
 First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-port-$PORT/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set.
-The wrapper also prints the recommended gateway name (`vm-driver-port-$PORT` by default) plus the exact repo-local `scripts/bin/openshell gateway add` and `scripts/bin/openshell gateway select` commands to use from another terminal. This avoids accidentally hitting an older `openshell` binary elsewhere on your `PATH`.
+The wrapper auto-registers the gateway with the CLI (`gateway destroy` + `gateway add`) so no manual registration step is needed. When running under `sudo`, it uses `sudo -u $SUDO_USER` for the registration so the config is written under the invoking user's home directory. Re-runs are idempotent.
 It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`.
 
+For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges:
+
+```shell
+sudo -E env "PATH=$PATH" mise run gateway:vm -- --gpu
+```
+
+See [`architecture/vm-gpu-sandbox-guide.md`](../../architecture/vm-gpu-sandbox-guide.md) for full GPU prerequisites and usage.
+
 Override via environment:
 
 ```shell
@@ -129,13 +137,11 @@ See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the full fl
 
 ## Verifying the gateway
 
-In another terminal:
+The gateway is auto-registered by `start.sh`. In another terminal:
 
 ```shell
-export OPENSHELL_GATEWAY_URL=http://127.0.0.1:8080
-cargo run -p openshell-cli -- gateway register local --url $OPENSHELL_GATEWAY_URL --no-tls
-cargo run -p openshell-cli -- sandbox create --name demo
-cargo run -p openshell-cli -- sandbox connect demo
+scripts/bin/openshell sandbox create --name demo
+scripts/bin/openshell sandbox connect demo
 ```
 
 First sandbox takes 10–30 seconds to boot (rootfs extraction + libkrun + guest init). Subsequent creates reuse the prepared sandbox rootfs.

crates/openshell-driver-vm/build.rs

@@ -22,6 +22,7 @@ fn main() {
             "libkrunfw.5.dylib.zst",
             "gvproxy.zst",
             "rootfs.tar.zst",
+            "rootfs-gpu.tar.zst",
         ] {
             println!("cargo:rerun-if-changed={dir}/{name}");
         }
@@ -36,7 +37,15 @@ fn main() {
         "linux" => ("libkrun.so", "libkrunfw.so.5"),
         _ => {
             println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}");
-            generate_stub_resources(&out_dir, &["libkrun", "libkrunfw", "rootfs.tar.zst"]);
+            generate_stub_resources(
+                &out_dir,
+                &[
+                    "libkrun",
+                    "libkrunfw",
+                    "rootfs.tar.zst",
+                    "rootfs-gpu.tar.zst",
+                ],
+            );
             return;
         }
     };
@@ -53,6 +62,7 @@ fn main() {
                 &format!("{libkrunfw_name}.zst"),
                 "gvproxy.zst",
                 "rootfs.tar.zst",
+                "rootfs-gpu.tar.zst",
             ],
         );
         return;
@@ -71,6 +81,7 @@ fn main() {
                 &format!("{libkrunfw_name}.zst"),
                 "gvproxy.zst",
                 "rootfs.tar.zst",
+                "rootfs-gpu.tar.zst",
             ],
         );
         return;
@@ -84,6 +95,10 @@ fn main() {
         ),
         ("gvproxy.zst".to_string(), "gvproxy.zst".to_string()),
         ("rootfs.tar.zst".to_string(), "rootfs.tar.zst".to_string()),
+        (
+            "rootfs-gpu.tar.zst".to_string(),
+            "rootfs-gpu.tar.zst".to_string(),
+        ),
     ];
 
     let mut all_found = true;
@@ -124,12 +139,13 @@ fn main() {
                 &format!("{libkrunfw_name}.zst"),
                 "gvproxy.zst",
                 "rootfs.tar.zst",
+                "rootfs-gpu.tar.zst",
             ],
         );
     }
 }
 
-fn generate_stub_resources(out_dir: &PathBuf, names: &[&str]) {
+fn generate_stub_resources(out_dir: &std::path::Path, names: &[&str]) {
     for name in names {
         let path = out_dir.join(name);
         if !path.exists() {