diff --git a/cmd/nvidia-cdi-hook/commands/commands.go b/cmd/nvidia-cdi-hook/commands/commands.go index a9f6c05ce..a15b36cc2 100644 --- a/cmd/nvidia-cdi-hook/commands/commands.go +++ b/cmd/nvidia-cdi-hook/commands/commands.go @@ -26,6 +26,7 @@ import ( symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks" "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat" disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification" + updateapplicationprofile "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-application-profile" ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache" "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" ) @@ -90,6 +91,7 @@ func ConfigureCDIHookCommand(logger logger.Interface, base *cli.Command) *cli.Co chmod.NewCommand(logger), cudacompat.NewCommand(logger), disabledevicenodemodification.NewCommand(logger), + updateapplicationprofile.NewCommand(logger), { Name: "noop", Usage: "The noop hook performs no actions and is only added to facilitate basic testing of the CLI", diff --git a/cmd/nvidia-cdi-hook/update-application-profile/minors_linux.go b/cmd/nvidia-cdi-hook/update-application-profile/minors_linux.go new file mode 100644 index 000000000..8ac837788 --- /dev/null +++ b/cmd/nvidia-cdi-hook/update-application-profile/minors_linux.go @@ -0,0 +1,88 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package updateapplicationprofile + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + + "golang.org/x/sys/unix" +) + +// readDirBatchSize is the number of directory entries read from the +// container's /dev directory per ReadDir call. +const readDirBatchSize = 100 + +// listDeviceNodes enumerates the entries under the container's /dev directory. +// The char-device check uses the cheap readdir type; the minor number is +// resolved for char-device entries. +func listDeviceNodes(containerRoot *os.Root) ([]deviceNode, error) { + devDir, err := containerRoot.Open("dev") + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to open /dev in container: %w", err) + } + defer devDir.Close() + + var nodes []deviceNode + for { + entries, readErr := devDir.ReadDir(readDirBatchSize) + for _, entry := range entries { + node := deviceNode{ + name: entry.Name(), + isCharDevice: entry.Type()&os.ModeCharDevice != 0, + } + if node.isCharDevice { + minor, err := deviceNodeMinor(containerRoot, entry.Name()) + if err != nil { + return nil, err + } + node.minor = minor + } + nodes = append(nodes, node) + } + if readErr == io.EOF { + break + } + if readErr != nil { + return nil, fmt.Errorf("failed to read /dev in container: %w", readErr) + } + } + + return nodes, nil +} + +// deviceNodeMinor returns the device minor number for the named entry under the +// container's /dev directory. +func deviceNodeMinor(containerRoot *os.Root, name string) (int, error) { + info, err := containerRoot.Lstat(filepath.Join("dev", name)) + if err != nil { + return 0, fmt.Errorf("failed to stat %s: %w", name, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("unexpected stat type for %s", name) + } + return int(unix.Minor(stat.Rdev)), nil +} diff --git a/cmd/nvidia-cdi-hook/update-application-profile/minors_other.go b/cmd/nvidia-cdi-hook/update-application-profile/minors_other.go new file mode 100644 index 000000000..0909f377a --- /dev/null +++ b/cmd/nvidia-cdi-hook/update-application-profile/minors_other.go @@ -0,0 +1,27 @@ +//go:build !linux + +/** +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package updateapplicationprofile + +import "os" + +// listDeviceNodes is not supported on non-Linux platforms. +func listDeviceNodes(_ *os.Root) ([]deviceNode, error) { + return nil, nil +} diff --git a/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile.go b/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile.go new file mode 100644 index 000000000..2c03c82ae --- /dev/null +++ b/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile.go @@ -0,0 +1,145 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package updateapplicationprofile + +import ( + "context" + "fmt" + "os" + "regexp" + + "github.com/urfave/cli/v3" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" +) + +const ( + applicationProfileDir = "etc/nvidia/nvidia-application-profiles-rc.d" + applicationProfileFile = applicationProfileDir + "/10-container.conf" +) + +// nvidiaGPUDeviceNode matches the device node names for physical NVIDIA GPUs, +// e.g. nvidia0, nvidia12. +var nvidiaGPUDeviceNode = regexp.MustCompile(`^nvidia[0-9]+$`) + +// A deviceNode describes a single entry discovered under the container's /dev +// directory. minor is only meaningful when isCharDevice is true. +type deviceNode struct { + name string + isCharDevice bool + minor int +} + +// selectGPUMinors returns the minor numbers of the entries that are physical +// NVIDIA GPU device nodes (/dev/nvidiaN char devices). +func selectGPUMinors(nodes []deviceNode) []int { + var minors []int + for _, n := range nodes { + if n.isCharDevice && nvidiaGPUDeviceNode.MatchString(n.name) { + minors = append(minors, n.minor) + } + } + return minors +} + +type options struct { + containerSpec string +} + +// NewCommand constructs an update-application-profile subcommand with the specified logger +func NewCommand(logger logger.Interface) *cli.Command { + cfg := options{} + + c := cli.Command{ + Name: "update-application-profile", + Usage: `Updates driver settings through "application profiles". Currently, this hook sets EGLVisibleDGPUDevices to restrict EGL/Vulkan GPU visibility inside the container`, + Action: func(ctx context.Context, cmd *cli.Command) error { + return run(ctx, cmd, &cfg, logger) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "container-spec", + Hidden: true, + Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN", + Destination: &cfg.containerSpec, + }, + }, + } + + return &c +} + +func run(_ context.Context, _ *cli.Command, cfg *options, logger logger.Interface) error { + s, err := oci.LoadContainerState(cfg.containerSpec) + if err != nil { + return fmt.Errorf("failed to load container state: %w", err) + } + + containerRootDirPath, err := s.GetContainerRoot() + if err != nil { + return fmt.Errorf("failed to determine container root: %w", err) + } + + containerRoot, err := os.OpenRoot(containerRootDirPath) + if err != nil { + return fmt.Errorf("failed to open root: %w", err) + } + defer containerRoot.Close() + + nodes, err := listDeviceNodes(containerRoot) + if err != nil { + return fmt.Errorf("failed to enumerate device nodes in container: %w", err) + } + + minors := selectGPUMinors(nodes) + if len(minors) == 0 { + return nil + } + + var mask uint64 + for _, minor := range minors { + if minor < 0 || minor >= 64 { + logger.Warningf("dropping invalid minor number %d: must be in range [0, 63]", minor) + continue + } + mask |= uint64(1) << uint(minor) + } + + if mask == 0 { + return nil + } + + if err := containerRoot.MkdirAll(applicationProfileDir, 0555); err != nil { + return fmt.Errorf("failed to create application profiles directory: %w", err) + } + + if err := containerRoot.WriteFile(applicationProfileFile, buildApplicationProfileConfig(mask), 0444); err != nil { + return fmt.Errorf("failed to write application profile: %w", err) + } + + return nil +} + +// buildApplicationProfileConfig returns the contents of the NVIDIA driver application profile. +func buildApplicationProfileConfig(mask uint64) []byte { + return fmt.Appendf([]byte{}, + `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x%x]}],"rules":[{"pattern":[],"profile":"_container_"}]}`, + mask, + ) +} diff --git a/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile_test.go b/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile_test.go new file mode 100644 index 000000000..f2aaea6d5 --- /dev/null +++ b/cmd/nvidia-cdi-hook/update-application-profile/update-application-profile_test.go @@ -0,0 +1,118 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package updateapplicationprofile + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSelectGPUMinors(t *testing.T) { + testCases := map[string]struct { + nodes []deviceNode + expected []int + }{ + "single GPU": { + nodes: []deviceNode{{name: "nvidia0", isCharDevice: true, minor: 0}}, + expected: []int{0}, + }, + "multiple GPUs, minor != filename order": { + nodes: []deviceNode{ + {name: "nvidia0", isCharDevice: true, minor: 1}, + {name: "nvidia1", isCharDevice: true, minor: 0}, + }, + expected: []int{1, 0}, + }, + "multi-digit minor": { + nodes: []deviceNode{{name: "nvidia10", isCharDevice: true, minor: 10}}, + expected: []int{10}, + }, + "control and modeset nodes are excluded": { + nodes: []deviceNode{ + {name: "nvidia0", isCharDevice: true, minor: 0}, + {name: "nvidiactl", isCharDevice: true, minor: 255}, + {name: "nvidia-modeset", isCharDevice: true, minor: 254}, + {name: "nvidia-uvm", isCharDevice: true, minor: 511}, + {name: "nvidia-uvm-tools", isCharDevice: true, minor: 510}, + }, + expected: []int{0}, + }, + "caps and imex nodes are excluded": { + nodes: []deviceNode{ + {name: "nvidia-caps", isCharDevice: false, minor: 0}, + {name: "nvidia-caps-imex-channels", isCharDevice: false, minor: 0}, + {name: "nvidia1", isCharDevice: true, minor: 1}, + }, + expected: []int{1}, + }, + "non-char-device with matching name is excluded": { + nodes: []deviceNode{ + {name: "nvidia0", isCharDevice: false, minor: 0}, + }, + expected: nil, + }, + "unrelated device nodes are excluded": { + nodes: []deviceNode{ + {name: "null", isCharDevice: true, minor: 3}, + {name: "zero", isCharDevice: true, minor: 5}, + }, + expected: nil, + }, + "no nodes": { + nodes: nil, + expected: nil, + }, + } + + for description, tc := range testCases { + t.Run(description, func(t *testing.T) { + require.Equal(t, tc.expected, selectGPUMinors(tc.nodes)) + }) + } +} + +func TestBuildApplicationProfileConfig(t *testing.T) { + testCases := map[string]struct { + mask uint64 + expected string + }{ + "single GPU, minor 0": { + mask: 1 << 0, + expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x1]}],"rules":[{"pattern":[],"profile":"_container_"}]}`, + }, + "minor 8": { + mask: 1 << 8, + expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x100]}],"rules":[{"pattern":[],"profile":"_container_"}]}`, + }, + "minors 0 and 15 combined": { + mask: (1 << 0) | (1 << 15), + expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x8001]}],"rules":[{"pattern":[],"profile":"_container_"}]}`, + }, + "no GPUs": { + mask: 0, + expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x0]}],"rules":[{"pattern":[],"profile":"_container_"}]}`, + }, + } + + for description, tc := range testCases { + t.Run(description, func(t *testing.T) { + require.EqualValues(t, tc.expected, string(buildApplicationProfileConfig(tc.mask))) + }) + } +} diff --git a/cmd/nvidia-ctk/cdi/generate/generate_test.go b/cmd/nvidia-ctk/cdi/generate/generate_test.go index 998d8bd0e..a954d2ba7 100644 --- a/cmd/nvidia-ctk/cdi/generate/generate_test.go +++ b/cmd/nvidia-ctk/cdi/generate/generate_test.go @@ -146,6 +146,13 @@ containerEdits: - disable-device-node-modification env: - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-application-profile + env: + - NVIDIA_CTK_DEBUG=false mounts: - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77 containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77 @@ -233,6 +240,13 @@ containerEdits: - disable-device-node-modification env: - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-application-profile + env: + - NVIDIA_CTK_DEBUG=false mounts: - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77 containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77 @@ -309,6 +323,13 @@ containerEdits: - disable-device-node-modification env: - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-application-profile + env: + - NVIDIA_CTK_DEBUG=false mounts: - hostPath: {{ .driverRoot }}/lib/x86_64-linux-gnu/libcuda.so.999.88.77 containerPath: /lib/x86_64-linux-gnu/libcuda.so.999.88.77 diff --git a/internal/discover/application_profile.go b/internal/discover/application_profile.go new file mode 100644 index 000000000..93619ff81 --- /dev/null +++ b/internal/discover/application_profile.go @@ -0,0 +1,22 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +// NewApplicationProfileHookDiscoverer creates a discoverer for the update-application-profile hook. +func NewApplicationProfileHookDiscoverer(hookCreator HookCreator) Discover { + return hookCreator.Create(ApplicationProfileHook) +} diff --git a/internal/discover/application_profile_test.go b/internal/discover/application_profile_test.go new file mode 100644 index 000000000..cbf701dc9 --- /dev/null +++ b/internal/discover/application_profile_test.go @@ -0,0 +1,66 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package discover + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewApplicationProfileHookDiscoverer(t *testing.T) { + t.Run("enabled hook creator returns a single hook", func(t *testing.T) { + hookCreator := NewHookCreator(WithNVIDIACDIHookPath(defaultNvidiaCDIHookPath)) + + d := NewApplicationProfileHookDiscoverer(hookCreator) + + hooks, err := d.Hooks() + require.NoError(t, err) + require.Len(t, hooks, 1) + require.Equal(t, "createContainer", hooks[0].Lifecycle) + require.Equal(t, defaultNvidiaCDIHookPath, hooks[0].Path) + require.Equal(t, []string{"nvidia-cdi-hook", "update-application-profile"}, hooks[0].Args) + + devices, err := d.Devices() + require.NoError(t, err) + require.Empty(t, devices) + + mounts, err := d.Mounts() + require.NoError(t, err) + require.Empty(t, mounts) + }) + + t.Run("all-disabled hook creator returns no hook", func(t *testing.T) { + hookCreator := &allDisabledHookCreator{} + + d := NewApplicationProfileHookDiscoverer(hookCreator) + + hooks, err := d.Hooks() + require.NoError(t, err) + require.Empty(t, hooks) + }) + + t.Run("explicitly disabled hook returns no hook", func(t *testing.T) { + hookCreator := NewHookCreator(WithDisabledHooks(ApplicationProfileHook)) + + d := NewApplicationProfileHookDiscoverer(hookCreator) + + hooks, err := d.Hooks() + require.NoError(t, err) + require.Empty(t, hooks) + }) +} diff --git a/internal/discover/hooks.go b/internal/discover/hooks.go index a31e097fc..95cc35e61 100644 --- a/internal/discover/hooks.go +++ b/internal/discover/hooks.go @@ -42,6 +42,10 @@ const ( // AllHooks is a special hook name that allows all hooks to be matched. AllHooks = HookName("all") + // An ApplicationProfileHook updates driver settings through "application + // profiles". It currently restricts EGL/Vulkan GPU visibility inside the + // container to the GPUs actually mounted. + ApplicationProfileHook = HookName("update-application-profile") // A ChmodHook is used to set the file mode of the specified paths. // // Deprecated: The chmod hook is deprecated and will be removed in a future release. @@ -216,7 +220,7 @@ func (c cdiHookCreator) Create(name HookName, args ...string) *Hook { func (c cdiHookCreator) getOCIHookType(name HookName) OCIHookType { switch name { - case CreateSymlinksHook, ChmodHook, DisableDeviceNodeModificationHook, EnableCudaCompatHook, UpdateLDCacheHook: + case CreateSymlinksHook, ChmodHook, DisableDeviceNodeModificationHook, EnableCudaCompatHook, UpdateLDCacheHook, ApplicationProfileHook: return OCIHookTypeCreateContainer default: return OCIHookTypeCreateContainer diff --git a/internal/discover/hooks_test.go b/internal/discover/hooks_test.go index 666a3f933..1bac3190a 100644 --- a/internal/discover/hooks_test.go +++ b/internal/discover/hooks_test.go @@ -257,6 +257,18 @@ func TestCDIHookCreator_Create(t *testing.T) { Env: []string{"NVIDIA_CTK_DEBUG=false"}, }, }, + { + name: "ApplicationProfileHook", + hookCreator: NewHookCreator(WithNVIDIACDIHookPath(defaultNvidiaCDIHookPath)), + hookName: ApplicationProfileHook, + args: []string{}, + expectedHook: &Hook{ + Lifecycle: "createContainer", + Path: defaultNvidiaCDIHookPath, + Args: []string{"nvidia-cdi-hook", "update-application-profile"}, + Env: []string{"NVIDIA_CTK_DEBUG=false"}, + }, + }, { name: "nvidia-ctk binary uses different args format", hookCreator: NewHookCreator(WithNVIDIACDIHookPath("/usr/bin/nvidia-ctk")), diff --git a/pkg/nvcdi/common-nvml.go b/pkg/nvcdi/common-nvml.go index 54a3a1e4c..244c8535b 100644 --- a/pkg/nvcdi/common-nvml.go +++ b/pkg/nvcdi/common-nvml.go @@ -37,10 +37,13 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) { return nil, fmt.Errorf("failed to create discoverer for driver files: %v", err) } + applicationProfileHook := discover.NewApplicationProfileHookDiscoverer(l.hookCreator) + d := discover.Merge( metaDevices, graphicsMounts, driverFiles, + applicationProfileHook, ) return d, nil