From db0b125ead9f99b36442232a7d727c0799ae6852 Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Wed, 21 Oct 2026 13:08:11 -0700 Subject: [PATCH 1/4] CI: stop display container before switching driver modes Closes #3183 Stop NVDisplay.ContainerLocalSystem before invoking nvidia-smi -fdm and wait for every NVDisplay.Container process to exit. After a driver install, the display container can crash-loop while the GPU is cycled. If its teardown overlaps PnP removal, nvlddmkm can remain loaded and nvidia-smi may never recover. Bound the process drain at 60 seconds and report remaining PIDs on failure. Preserve the existing mode-specific logging and device-cycle stabilization behavior. --- ci/tools/configure_driver_mode.ps1 | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/ci/tools/configure_driver_mode.ps1 b/ci/tools/configure_driver_mode.ps1 index b055ca39102..c3fde86396e 100644 --- a/ci/tools/configure_driver_mode.ps1 +++ b/ci/tools/configure_driver_mode.ps1 @@ -11,6 +11,27 @@ # Inputs (env): # DRIVER_MODE One of WDDM, TCC, MCDM. +function Stop-DisplayContainer { + + # A crash-looping display-container process can race the subsequent device + # cycle and prevent the NVIDIA kernel driver from unloading. + Write-Output "Stopping NVDisplay.ContainerLocalSystem..." + Stop-Service -Name "NVDisplay.ContainerLocalSystem" -Force -ErrorAction Stop + + Write-Output "Waiting for NVDisplay.Container processes to exit..." + $deadline = (Get-Date).AddSeconds(60) + $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) + while ($container_processes.Count -gt 0 -and (Get-Date) -lt $deadline) { + Start-Sleep -Milliseconds 500 + $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) + } + if ($container_processes.Count -gt 0) { + $process_ids = $container_processes.Id -join ", " + Write-Error "NVDisplay.Container processes ($process_ids) did not exit within 60s of stopping NVDisplay.ContainerLocalSystem" + exit 1 + } +} + function Set-DriverMode { # Map matrix DRIVER_MODE to nvidia-smi -fdm code. @@ -18,18 +39,22 @@ function Set-DriverMode { $driver_mode = $env:DRIVER_MODE if ($driver_mode -eq "WDDM") { Write-Output "Setting driver mode to WDDM..." - nvidia-smi -fdm 0 + $driver_mode_code = 0 } elseif ($driver_mode -eq "TCC") { Write-Output "Setting driver mode to TCC..." - nvidia-smi -fdm 1 + $driver_mode_code = 1 } elseif ($driver_mode -eq "MCDM") { Write-Output "Setting driver mode to MCDM..." - nvidia-smi -fdm 2 + $driver_mode_code = 2 } else { Write-Output "Unknown driver mode: $driver_mode" exit 1 } + Stop-DisplayContainer + + nvidia-smi -fdm $driver_mode_code + # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") $gpu_count = $nvidia_devices.Count From bb9970e1bd3afed9b560e97d109d77bb9d2b1fa1 Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Wed, 21 Oct 2026 13:18:52 -0700 Subject: [PATCH 2/4] Put the service stop under 60 second timer as well. --- ci/tools/configure_driver_mode.ps1 | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ci/tools/configure_driver_mode.ps1 b/ci/tools/configure_driver_mode.ps1 index c3fde86396e..81903181711 100644 --- a/ci/tools/configure_driver_mode.ps1 +++ b/ci/tools/configure_driver_mode.ps1 @@ -16,19 +16,26 @@ function Stop-DisplayContainer { # A crash-looping display-container process can race the subsequent device # cycle and prevent the NVIDIA kernel driver from unloading. Write-Output "Stopping NVDisplay.ContainerLocalSystem..." - Stop-Service -Name "NVDisplay.ContainerLocalSystem" -Force -ErrorAction Stop - - Write-Output "Waiting for NVDisplay.Container processes to exit..." + $service = Get-Service -Name "NVDisplay.ContainerLocalSystem" -ErrorAction Stop $deadline = (Get-Date).AddSeconds(60) - $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) - while ($container_processes.Count -gt 0 -and (Get-Date) -lt $deadline) { - Start-Sleep -Milliseconds 500 - $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) + if ($service.Status -ne "Stopped" -and $service.Status -ne "StopPending") { + Stop-Service -InputObject $service -Force -NoWait -ErrorAction Stop } - if ($container_processes.Count -gt 0) { - $process_ids = $container_processes.Id -join ", " - Write-Error "NVDisplay.Container processes ($process_ids) did not exit within 60s of stopping NVDisplay.ContainerLocalSystem" - exit 1 + + Write-Output "Waiting for NVDisplay.ContainerLocalSystem to stop and NVDisplay.Container processes to exit..." + while ($true) { + $service.Refresh() + $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) + if ($service.Status -eq "Stopped" -and $container_processes.Count -eq 0) { + break + } + if ((Get-Date) -ge $deadline) { + $process_ids = $container_processes.Id -join ", " + if (-not $process_ids) { $process_ids = "none" } + Write-Error "Timed out after 60s waiting for the NVIDIA display container (service status: $($service.Status); remaining NVDisplay.Container PIDs: $process_ids)" + exit 1 + } + Start-Sleep -Milliseconds 500 } } From 55cbcbb032554da8b472b1f8434f541738ae2aa1 Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Wed, 21 Oct 2026 14:30:15 -0700 Subject: [PATCH 3/4] CI: prevent display container restart during mode switching Temporarily disable NVDisplay.ContainerLocalSystem while changing driver modes so queued recovery actions cannot restart it during the device cycle. Handle in-flight starts, restore the original startup type in a finally block, and report the requested driver mode by name. --- ci/tools/configure_driver_mode.ps1 | 84 +++++++++++++++++------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/ci/tools/configure_driver_mode.ps1 b/ci/tools/configure_driver_mode.ps1 index 81903181711..592eb7dc39a 100644 --- a/ci/tools/configure_driver_mode.ps1 +++ b/ci/tools/configure_driver_mode.ps1 @@ -12,28 +12,32 @@ # DRIVER_MODE One of WDDM, TCC, MCDM. function Stop-DisplayContainer { + param ( + [System.ServiceProcess.ServiceController]$Service + ) # A crash-looping display-container process can race the subsequent device # cycle and prevent the NVIDIA kernel driver from unloading. Write-Output "Stopping NVDisplay.ContainerLocalSystem..." - $service = Get-Service -Name "NVDisplay.ContainerLocalSystem" -ErrorAction Stop $deadline = (Get-Date).AddSeconds(60) - if ($service.Status -ne "Stopped" -and $service.Status -ne "StopPending") { - Stop-Service -InputObject $service -Force -NoWait -ErrorAction Stop - } Write-Output "Waiting for NVDisplay.ContainerLocalSystem to stop and NVDisplay.Container processes to exit..." while ($true) { - $service.Refresh() + $Service.Refresh() + # A recovery start may already be in flight when the service is + # disabled. Stop it as soon as it reaches a stoppable state. + if ($Service.Status -eq "Running" -or $Service.Status -eq "Paused") { + Stop-Service -InputObject $Service -Force -NoWait -ErrorAction Stop + $Service.Refresh() + } $container_processes = @(Get-Process -Name "NVDisplay.Container" -ErrorAction SilentlyContinue) - if ($service.Status -eq "Stopped" -and $container_processes.Count -eq 0) { + if ($Service.Status -eq "Stopped" -and $container_processes.Count -eq 0) { break } if ((Get-Date) -ge $deadline) { $process_ids = $container_processes.Id -join ", " if (-not $process_ids) { $process_ids = "none" } - Write-Error "Timed out after 60s waiting for the NVIDIA display container (service status: $($service.Status); remaining NVDisplay.Container PIDs: $process_ids)" - exit 1 + throw "Timed out after 60s waiting for the NVIDIA display container (service status: $($Service.Status); remaining NVDisplay.Container PIDs: $process_ids)" } Start-Sleep -Milliseconds 500 } @@ -45,48 +49,56 @@ function Set-DriverMode { # This assumes we have the prior knowledge on which GPU can use which mode. $driver_mode = $env:DRIVER_MODE if ($driver_mode -eq "WDDM") { - Write-Output "Setting driver mode to WDDM..." $driver_mode_code = 0 } elseif ($driver_mode -eq "TCC") { - Write-Output "Setting driver mode to TCC..." $driver_mode_code = 1 } elseif ($driver_mode -eq "MCDM") { - Write-Output "Setting driver mode to MCDM..." $driver_mode_code = 2 } else { Write-Output "Unknown driver mode: $driver_mode" exit 1 } - Stop-DisplayContainer + $display_container_service = Get-Service -Name "NVDisplay.ContainerLocalSystem" -ErrorAction Stop + $display_container_start_type = $display_container_service.StartType + try { + # A queued recovery restart cannot be canceled, so keep the service + # disabled until the device cycle and NVML recovery are complete. + Write-Output "Temporarily disabling NVDisplay.ContainerLocalSystem..." + Set-Service -InputObject $display_container_service -StartupType Disabled -ErrorAction Stop + Stop-DisplayContainer -Service $display_container_service - nvidia-smi -fdm $driver_mode_code + Write-Output "Setting driver mode to $driver_mode..." + nvidia-smi -fdm $driver_mode_code - # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). - $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") - $gpu_count = $nvidia_devices.Count - foreach ($device in $nvidia_devices) { - Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))" - pnputil /disable-device "$($device.InstanceId)" - pnputil /enable-device "$($device.InstanceId)" - } + # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). + $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") + $gpu_count = $nvidia_devices.Count + foreach ($device in $nvidia_devices) { + Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))" + pnputil /disable-device "$($device.InstanceId)" + pnputil /enable-device "$($device.InstanceId)" + } - # Initial settle after the device cycle. - Start-Sleep -Seconds 5 + # Initial settle after the device cycle. + Start-Sleep -Seconds 5 - # Poll nvidia-smi for N consecutive successes (N == cycled GPUs) - # so a mid-init "ok" flap doesn't fool the loop; bail after ~60s. - Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..." - $deadline = (Get-Date).AddSeconds(60) - $consecutive_ok = 0 - do { - Start-Sleep -Seconds 3 - & nvidia-smi.exe 2>&1 | Out-Null - if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 } - } while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline) - if ($consecutive_ok -lt $gpu_count) { - Write-Error "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle" - exit 1 + # Poll nvidia-smi for N consecutive successes (N == cycled GPUs) + # so a mid-init "ok" flap doesn't fool the loop; bail after ~60s. + Write-Output "Waiting for nvidia-smi/NVML to come back up after device cycle..." + $deadline = (Get-Date).AddSeconds(60) + $consecutive_ok = 0 + do { + Start-Sleep -Seconds 3 + & nvidia-smi.exe 2>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { $consecutive_ok++ } else { $consecutive_ok = 0 } + } while ($consecutive_ok -lt $gpu_count -and (Get-Date) -lt $deadline) + if ($consecutive_ok -lt $gpu_count) { + throw "nvidia-smi did not return cleanly $gpu_count times in a row within 60s of the device cycle" + } + } finally { + Write-Output "Restoring NVDisplay.ContainerLocalSystem startup type to $display_container_start_type..." + Set-Service -Name $display_container_service.Name -StartupType $display_container_start_type -ErrorAction Stop } } From 12a6d7524e969bf5d1dcd80d3df37282e2f1a4bf Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Wed, 21 Oct 2026 14:41:56 -0700 Subject: [PATCH 4/4] Implement coderabbit suggestions --- ci/tools/configure_driver_mode.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/tools/configure_driver_mode.ps1 b/ci/tools/configure_driver_mode.ps1 index 592eb7dc39a..ab196b6effe 100644 --- a/ci/tools/configure_driver_mode.ps1 +++ b/ci/tools/configure_driver_mode.ps1 @@ -70,10 +70,16 @@ function Set-DriverMode { Write-Output "Setting driver mode to $driver_mode..." nvidia-smi -fdm $driver_mode_code + if ($LASTEXITCODE -ne 0) { + throw "nvidia-smi -fdm $driver_mode_code ($driver_mode) failed with exit code $LASTEXITCODE" + } # Only restart NVIDIA display adapters, not other display devices (e.g. QEMU VGA). $nvidia_devices = @(Get-PnpDevice -Class Display -FriendlyName "NVIDIA*") $gpu_count = $nvidia_devices.Count + if ($gpu_count -eq 0) { + throw "No NVIDIA display devices were found." + } foreach ($device in $nvidia_devices) { Write-Output "Restarting device: $($device.FriendlyName) ($($device.InstanceId))" pnputil /disable-device "$($device.InstanceId)"