Generate app profile for EGL/Vulkan use case

cmd/nvidia-cdi-hook/commands/commands.go

@@ -26,6 +26,7 @@ import (
 	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
 	disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification"
+	updateapplicationprofile "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-application-profile"
 	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 )
@@ -90,6 +91,7 @@ func ConfigureCDIHookCommand(logger logger.Interface, base *cli.Command) *cli.Co
 		chmod.NewCommand(logger),
 		cudacompat.NewCommand(logger),
 		disabledevicenodemodification.NewCommand(logger),
+		updateapplicationprofile.NewCommand(logger),
 		{
 			Name:   "noop",
 			Usage:  "The noop hook performs no actions and is only added to facilitate basic testing of the CLI",

cmd/nvidia-cdi-hook/update-application-profile/minors_linux.go

@@ -0,0 +1,88 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package updateapplicationprofile
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// readDirBatchSize is the number of directory entries read from the
+// container's /dev directory per ReadDir call.
+const readDirBatchSize = 100
+
+// listDeviceNodes enumerates the entries under the container's /dev directory.
+// The char-device check uses the cheap readdir type; the minor number is
+// resolved for char-device entries.
+func listDeviceNodes(containerRoot *os.Root) ([]deviceNode, error) {
+	devDir, err := containerRoot.Open("dev")
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to open /dev in container: %w", err)
+	}
+	defer devDir.Close()
+
+	var nodes []deviceNode
+	for {
+		entries, readErr := devDir.ReadDir(readDirBatchSize)
+		for _, entry := range entries {
+			node := deviceNode{
+				name:         entry.Name(),
+				isCharDevice: entry.Type()&os.ModeCharDevice != 0,
+			}
+			if node.isCharDevice {
+				minor, err := deviceNodeMinor(containerRoot, entry.Name())
+				if err != nil {
+					return nil, err
+				}
+				node.minor = minor
+			}
+			nodes = append(nodes, node)
+		}
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return nil, fmt.Errorf("failed to read /dev in container: %w", readErr)
+		}
+	}
+
+	return nodes, nil
+}
+
+// deviceNodeMinor returns the device minor number for the named entry under the
+// container's /dev directory.
+func deviceNodeMinor(containerRoot *os.Root, name string) (int, error) {
+	info, err := containerRoot.Lstat(filepath.Join("dev", name))
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat %s: %w", name, err)
+	}
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("unexpected stat type for %s", name)
+	}
+	return int(unix.Minor(stat.Rdev)), nil
+}

cmd/nvidia-cdi-hook/update-application-profile/minors_other.go

@@ -0,0 +1,27 @@
+//go:build !linux
+
+/**
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package updateapplicationprofile
+
+import "os"
+
+// listDeviceNodes is not supported on non-Linux platforms.
+func listDeviceNodes(_ *os.Root) ([]deviceNode, error) {
+	return nil, nil
+}

cmd/nvidia-cdi-hook/update-application-profile/update-application-profile.go

@@ -0,0 +1,145 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package updateapplicationprofile
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/urfave/cli/v3"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	applicationProfileDir  = "etc/nvidia/nvidia-application-profiles-rc.d"
+	applicationProfileFile = applicationProfileDir + "/10-container.conf"
+)
+
+// nvidiaGPUDeviceNode matches the device node names for physical NVIDIA GPUs,
+// e.g. nvidia0, nvidia12.
+var nvidiaGPUDeviceNode = regexp.MustCompile(`^nvidia[0-9]+$`)
+
+// A deviceNode describes a single entry discovered under the container's /dev
+// directory. minor is only meaningful when isCharDevice is true.
+type deviceNode struct {
+	name         string
+	isCharDevice bool
+	minor        int
+}
+
+// selectGPUMinors returns the minor numbers of the entries that are physical
+// NVIDIA GPU device nodes (/dev/nvidiaN char devices).
+func selectGPUMinors(nodes []deviceNode) []int {
+	var minors []int
+	for _, n := range nodes {
+		if n.isCharDevice && nvidiaGPUDeviceNode.MatchString(n.name) {
+			minors = append(minors, n.minor)
+		}
+	}
+	return minors
+}
+
+type options struct {
+	containerSpec string
+}
+
+// NewCommand constructs an update-application-profile subcommand with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	cfg := options{}
+
+	c := cli.Command{
+		Name:  "update-application-profile",
+		Usage: `Updates driver settings through "application profiles". Currently, this hook sets EGLVisibleDGPUDevices to restrict EGL/Vulkan GPU visibility inside the container`,
+		Action: func(ctx context.Context, cmd *cli.Command) error {
+			return run(ctx, cmd, &cfg, logger)
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:        "container-spec",
+				Hidden:      true,
+				Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+				Destination: &cfg.containerSpec,
+			},
+		},
+	}
+
+	return &c
+}
+
+func run(_ context.Context, _ *cli.Command, cfg *options, logger logger.Interface) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %w", err)
+	}
+
+	containerRootDirPath, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determine container root: %w", err)
+	}
+
+	containerRoot, err := os.OpenRoot(containerRootDirPath)
+	if err != nil {
+		return fmt.Errorf("failed to open root: %w", err)
+	}
+	defer containerRoot.Close()
+
+	nodes, err := listDeviceNodes(containerRoot)
+	if err != nil {
+		return fmt.Errorf("failed to enumerate device nodes in container: %w", err)
+	}
+
+	minors := selectGPUMinors(nodes)
+	if len(minors) == 0 {
+		return nil
+	}
+
+	var mask uint64
+	for _, minor := range minors {
+		if minor < 0 || minor >= 64 {
+			logger.Warningf("dropping invalid minor number %d: must be in range [0, 63]", minor)
+			continue
+		}
+		mask |= uint64(1) << uint(minor)
+	}
+
+	if mask == 0 {
+		return nil
+	}
+
+	if err := containerRoot.MkdirAll(applicationProfileDir, 0555); err != nil {
+		return fmt.Errorf("failed to create application profiles directory: %w", err)
+	}
+
+	if err := containerRoot.WriteFile(applicationProfileFile, buildApplicationProfileConfig(mask), 0444); err != nil {
+		return fmt.Errorf("failed to write application profile: %w", err)
+	}
+
+	return nil
+}
+
+// buildApplicationProfileConfig returns the contents of the NVIDIA driver application profile.
+func buildApplicationProfileConfig(mask uint64) []byte {
+	return fmt.Appendf([]byte{},
+		`{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x%x]}],"rules":[{"pattern":[],"profile":"_container_"}]}`,
+		mask,
+	)
+}

cmd/nvidia-cdi-hook/update-application-profile/update-application-profile_test.go

@@ -0,0 +1,118 @@
+/**
+# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package updateapplicationprofile
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSelectGPUMinors(t *testing.T) {
+	testCases := map[string]struct {
+		nodes    []deviceNode
+		expected []int
+	}{
+		"single GPU": {
+			nodes:    []deviceNode{{name: "nvidia0", isCharDevice: true, minor: 0}},
+			expected: []int{0},
+		},
+		"multiple GPUs, minor != filename order": {
+			nodes: []deviceNode{
+				{name: "nvidia0", isCharDevice: true, minor: 1},
+				{name: "nvidia1", isCharDevice: true, minor: 0},
+			},
+			expected: []int{1, 0},
+		},
+		"multi-digit minor": {
+			nodes:    []deviceNode{{name: "nvidia10", isCharDevice: true, minor: 10}},
+			expected: []int{10},
+		},
+		"control and modeset nodes are excluded": {
+			nodes: []deviceNode{
+				{name: "nvidia0", isCharDevice: true, minor: 0},
+				{name: "nvidiactl", isCharDevice: true, minor: 255},
+				{name: "nvidia-modeset", isCharDevice: true, minor: 254},
+				{name: "nvidia-uvm", isCharDevice: true, minor: 511},
+				{name: "nvidia-uvm-tools", isCharDevice: true, minor: 510},
+			},
+			expected: []int{0},
+		},
+		"caps and imex nodes are excluded": {
+			nodes: []deviceNode{
+				{name: "nvidia-caps", isCharDevice: false, minor: 0},
+				{name: "nvidia-caps-imex-channels", isCharDevice: false, minor: 0},
+				{name: "nvidia1", isCharDevice: true, minor: 1},
+			},
+			expected: []int{1},
+		},
+		"non-char-device with matching name is excluded": {
+			nodes: []deviceNode{
+				{name: "nvidia0", isCharDevice: false, minor: 0},
+			},
+			expected: nil,
+		},
+		"unrelated device nodes are excluded": {
+			nodes: []deviceNode{
+				{name: "null", isCharDevice: true, minor: 3},
+				{name: "zero", isCharDevice: true, minor: 5},
+			},
+			expected: nil,
+		},
+		"no nodes": {
+			nodes:    nil,
+			expected: nil,
+		},
+	}
+
+	for description, tc := range testCases {
+		t.Run(description, func(t *testing.T) {
+			require.Equal(t, tc.expected, selectGPUMinors(tc.nodes))
+		})
+	}
+}
+
+func TestBuildApplicationProfileConfig(t *testing.T) {
+	testCases := map[string]struct {
+		mask     uint64
+		expected string
+	}{
+		"single GPU, minor 0": {
+			mask:     1 << 0,
+			expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x1]}],"rules":[{"pattern":[],"profile":"_container_"}]}`,
+		},
+		"minor 8": {
+			mask:     1 << 8,
+			expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x100]}],"rules":[{"pattern":[],"profile":"_container_"}]}`,
+		},
+		"minors 0 and 15 combined": {
+			mask:     (1 << 0) | (1 << 15),
+			expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x8001]}],"rules":[{"pattern":[],"profile":"_container_"}]}`,
+		},
+		"no GPUs": {
+			mask:     0,
+			expected: `{"profiles":[{"name":"_container_","settings":["EGLVisibleDGPUDevices",0x0]}],"rules":[{"pattern":[],"profile":"_container_"}]}`,
+		},
+	}
+
+	for description, tc := range testCases {
+		t.Run(description, func(t *testing.T) {
+			require.EqualValues(t, tc.expected, string(buildApplicationProfileConfig(tc.mask)))
+		})
+	}
+}