fix: NVIDIA CUDA flat repos don't follow Debian repo spec

Although the Debian repo spec for 'Filename' (see
https://wiki.debian.org/DebianRepository/Format#Filename) clearly says
that 'Filename' should be relative to the base directory of the repo and
should be in canonical form (i.e. without '.' or '..') there are cases
where this is not honored.

In those cases we try to work around this by assuming 'Filename' is
relative to the sources.list directory/ so we combine them and normalize
the new 'Filename' path.

Note that, so far, only the NVIDIA CUDA repos needed this workaround so
maybe this heuristic will break for other repos that don't conform to
the Debian repo spec.

MODULE.bazel

@@ -6,7 +6,7 @@ module(
     compatibility_level = 1,
 )
 
-bazel_dep(name = "bazel_skylib", version = "1.5.0")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "aspect_bazel_lib", version = "2.7.9")
 
 bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
@@ -21,7 +21,7 @@ use_repo(bazel_lib_toolchains, "yq_linux_s390x")
 use_repo(bazel_lib_toolchains, "yq_windows_amd64")
 
 bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle")
-bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True)
+bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True)
 bazel_dep(name = "buildifier_prebuilt", version = "6.1.2", dev_dependency = True)
 bazel_dep(name = "platforms", version = "0.0.10", dev_dependency = True)
 bazel_dep(name = "rules_oci", version = "2.0.0-rc0", dev_dependency = True)

WORKSPACE.bazel

@@ -47,6 +47,17 @@ load("@bullseye_rproject//:packages.bzl", "bullseye_rproject_packages")
 
 bullseye_rproject_packages()
 
+# bazel run @nvidia_ubuntu2404_cuda//:lock
+deb_index(
+    name = "nvidia_ubuntu2404_cuda",
+    lock = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.lock.json",
+    manifest = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.yaml",
+)
+
+load("@nvidia_ubuntu2404_cuda//:packages.bzl", "nvidia_ubuntu2404_cuda_packages")
+
+nvidia_ubuntu2404_cuda_packages()
+
 # bazel run @apt_security//:lock
 deb_index(
     name = "apt_security",

apt/private/BUILD.bazel

@@ -42,7 +42,10 @@ bzl_library(
     name = "package_index",
     srcs = ["package_index.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = [":util"],
+    deps = [
+        ":util",
+        "@bazel_skylib//lib:paths",
+    ],
 )
 
 bzl_library(

apt/private/lockfile.bzl

@@ -20,7 +20,7 @@ def _add_package(lock, package, arch):
         "key": k,
         "name": package["Package"],
         "version": package["Version"],
-        "url": "%s/%s" % (package["Root"], package["Filename"]),
+        "url": package["FileUrl"],
         "sha256": package["SHA256"],
         "arch": arch,
         "dependencies": [],

apt/private/package_index.bzl

@@ -1,5 +1,6 @@
 "package index"
 
+load("@bazel_skylib//lib:paths.bzl", "paths")
 load(":util.bzl", "util")
 
 def _fetch_package_index(rctx, url, arch, dist = None, comp = None, directory = None):
@@ -83,6 +84,59 @@ def _fetch_package_index(rctx, url, arch, dist = None, comp = None, directory =
 
     return (output, integrity)
 
+def _parse_url(url):
+    scheme = ""
+    host = ""
+    path = "/"
+
+    if "://" not in url:
+        fail("Invalid URL: %s" % url)
+
+    scheme, url_ = url.split("://", 1)
+
+    if "/" in url_:
+        host, path_ = url_.split("/", 1)
+        path += path_
+    else:
+        host = url
+
+    return struct(scheme = scheme, host = host, path = path)
+
+def _make_file_url(pkg, root_url_, directory = None):
+    root_url = _parse_url(root_url_)
+
+    filename = pkg["Filename"]
+
+    invalid_filename = not paths.is_normalized(
+        filename,
+        look_for_same_level_references = True,
+    )
+
+    if invalid_filename:
+        # NOTE:
+        # Although the Debian repo spec for 'Filename' (see
+        # https://wiki.debian.org/DebianRepository/Format#Filename) clearly
+        # says that 'Filename' should be relative to the base directory of the
+        # repo and should be in canonical form (i.e. without '.' or '..') there
+        # are cases where this is not honored.
+        #
+        # In those cases we try to work around this by assuming 'Filename' is
+        # relative to the sources.list directory/ so we combine them and
+        # normalize the new 'Filename' path.
+        #
+        # Note that, so far, only the NVIDIA CUDA repos needed this workaround
+        # so maybe this heuristic will break for other repos that don't conform
+        # to the Debian repo spec.
+        filename = paths.normalize(paths.join(directory, filename))
+
+    file_url = "{}://{}{}".format(
+        root_url.scheme,
+        root_url.host,
+        paths.join(root_url.path, filename),
+    )
+
+    return file_url, invalid_filename
+
 def _parse_package_index(state, contents, arch, root_url, directory = None):
     last_key = ""
     pkg = {}
@@ -115,14 +169,20 @@ def _parse_package_index(state, contents, arch, root_url, directory = None):
             pkg[key] = value
 
         if len(pkg.keys()) != 0:
-            pkg["Root"] = root_url
+            pkg["FileUrl"], invalid_filename = _make_file_url(pkg, root_url, directory)
+
+            if invalid_filename:
+                out_of_spec.append(pkg["Package"])
 
             # NOTE: this fixes the arch for multi-arch flat repos
             arch_ = arch if pkg["Architecture"] == "all" else pkg["Architecture"]
 
             util.set_dict(state.packages, value = pkg, keys = (arch_, pkg["Package"], pkg["Version"]))
             last_key = ""
             pkg = {}
+            total_pkgs += 1
+
+    return out_of_spec, total_pkgs
 
 def _package_versions(state, name, arch):
     if name not in state.packages[arch]:
@@ -172,7 +232,19 @@ def _create(rctx, sources, archs):
             rctx.report_progress("Parsing %s package index: %s" % (arch, index))
 
             # TODO: this is expensive to perform.
-            _parse_package_index(state, rctx.read(output), arch, url, directory)
+            out_of_spec, total_pkgs = _parse_package_index(
+                state,
+                rctx.read(output),
+                arch,
+                url,
+                directory,
+            )
+
+            if out_of_spec:
+                count = len(out_of_spec)
+                pct = int(100.0 * count / total_pkgs)
+                msg = "Warning: {} index {} has {} packages ({}%) with invalid 'Filename' fields"
+                print(msg.format(arch, index, count, pct))
 
     return struct(
         package_versions = lambda **kwargs: _package_versions(state, **kwargs),

examples/debian_flat_repo/BUILD.bazel

@@ -5,28 +5,51 @@ load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load")
 PACKAGES = [
     "@bullseye//dpkg",
     "@bullseye//apt",
+]
+
+PACKAGES_AMD64 = PACKAGES + [
     "@bullseye_rproject//r-mathlib",
+    "@nvidia_ubuntu2404_cuda//nvidia-container-toolkit-base",
+]
+
+PACKAGES_ARM64 = PACKAGES + [
+    "@nvidia_ubuntu2404_cuda//nvidia-container-toolkit-base",
 ]
 
 # Creates /var/lib/dpkg/status with installed package information.
 dpkg_status(
     name = "dpkg_status",
-    controls = [
-        "%s/amd64:control" % package
-        for package in PACKAGES
-    ],
+    controls = select({
+        "@platforms//cpu:x86_64": [
+            "%s/amd64:control" % package
+            for package in PACKAGES_AMD64
+        ],
+        "@platforms//cpu:arm64": [
+            "%s/arm64:control" % package
+            for package in PACKAGES_ARM64
+        ],
+    }),
 )
 
 oci_image(
     name = "apt",
-    architecture = "amd64",
+    architecture = select({
+        "@platforms//cpu:x86_64": "amd64",
+        "@platforms//cpu:arm64": "arm64",
+    }),
     os = "linux",
     tars = [
         ":dpkg_status",
-    ] + [
-        "%s/amd64" % package
-        for package in PACKAGES
-    ],
+    ] + select({
+        "@platforms//cpu:x86_64": [
+            "%s/amd64" % package
+            for package in PACKAGES_AMD64
+        ],
+        "@platforms//cpu:arm64": [
+            "%s/arm64" % package
+            for package in PACKAGES_ARM64
+        ],
+    }),
 )
 
 oci_load(
@@ -39,10 +62,15 @@ oci_load(
 
 container_structure_test(
     name = "test",
-    configs = ["test_linux_amd64.yaml"],
+    configs = select({
+        "@platforms//cpu:x86_64": ["test_linux_amd64.yaml"],
+        "@platforms//cpu:arm64": ["test_linux_arm64.yaml"],
+    }),
     image = ":apt",
-    target_compatible_with = [
-        "@platforms//cpu:x86_64",
+    target_compatible_with = select({
+        "@platforms//cpu:x86_64": ["@platforms//cpu:x86_64"],
+        "@platforms//cpu:arm64": ["@platforms//cpu:arm64"],
+    }) + [
         "@platforms//os:linux",
     ],
 )

examples/debian_flat_repo/nvidia_ubuntu2404_cuda.lock.json

@@ -0,0 +1,23 @@
+{
+	"packages": [
+		{
+			"arch": "amd64",
+			"dependencies": [],
+			"key": "nvidia-container-toolkit-base_1.16.1-1_amd64",
+			"name": "nvidia-container-toolkit-base",
+			"sha256": "8184d04f88215de4f630e4f5ba24d9bf7e64a7a597ba2e3c6fbd94f86bea0599",
+			"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/nvidia-container-toolkit-base_1.16.1-1_amd64.deb",
+			"version": "1.16.1-1"
+		},
+		{
+			"arch": "arm64",
+			"dependencies": [],
+			"key": "nvidia-container-toolkit-base_1.16.1-1_arm64",
+			"name": "nvidia-container-toolkit-base",
+			"sha256": "dfc068e5ff69274351e59376078d9bda6a6c95423c7de1619b6a54aa9ba0f494",
+			"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/arm64/nvidia-container-toolkit-base_1.16.1-1_arm64.deb",
+			"version": "1.16.1-1"
+		}
+	],
+	"version": 1
+}

examples/debian_flat_repo/nvidia_ubuntu2404_cuda.yaml

@@ -0,0 +1,23 @@
+# Packages for examples/debian_flat_repo.
+#
+#  Anytime this file is changed, the lockfile needs to be regenerated.
+#
+#  To generate the nvidia_cuda.lock.json run the following command
+#
+#     bazel run @nvidia_ubuntu2404_cuda//:lock
+#
+# See debian_package_index at WORKSPACE.bazel
+version: 1
+
+sources:
+  - channel: ubuntu2404/x86_64/
+    url: https://developer.download.nvidia.com/compute/cuda/repos
+  - channel: ubuntu2404/arm64/
+    url: https://developer.download.nvidia.com/compute/cuda/repos
+
+archs:
+  - amd64
+  - arm64
+
+packages:
+  - nvidia-container-toolkit-base

examples/debian_flat_repo/test_linux_amd64.yaml

@@ -7,3 +7,4 @@ commandTests:
     expectedOutput:
       - Listing\.\.\.
       - r-mathlib/now 4.4.1-1~bullseyecran.0 amd64 \[installed,local\]
+      - nvidia-container-toolkit-base/now 1.16.1-1 amd64 \[installed,local\]

examples/debian_flat_repo/test_linux_arm64.yaml

@@ -0,0 +1,9 @@
+schemaVersion: "2.0.0"
+
+commandTests:
+  - name: "apt list --installed"
+    command: "apt"
+    args: ["list", "--installed"]
+    expectedOutput:
+      - Listing\.\.\.
+      - nvidia-container-toolkit-base/now 1.16.1-1 arm64 \[installed,local\]