Add flat repository support for debian images

WORKSPACE.bazel

@@ -58,6 +58,17 @@ load("@shared_dependencies//:packages.bzl", "shared_dependencies_packages")
 
 shared_dependencies_packages()
 
+# bazel run @flat_repository//:lock
+deb_index(
+    name = "flat_repository",
+    lock = "//examples/debian_flat_repository:bullseye.lock.json",
+    manifest = "//examples/debian_flat_repository:bullseye.yaml",
+)
+
+load("@flat_repository//:packages.bzl", "flat_repository_packages")
+
+flat_repository_packages()
+
 # bazel run @noble//:lock
 deb_index(
     name = "noble",

apt/private/package_index.bzl

@@ -2,17 +2,21 @@
 
 load(":util.bzl", "util")
 
-def _fetch_package_index(rctx, url, dist, comp, arch, integrity):
+def _fetch_package_index(rctx, url, dist, comp, arch, integrity, flat_repository):
     target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch)
 
     file_types = {"xz": ["xz", "--decompress"], "gz": ["gzip", "-d"]}
     r = {"success": False, "integrity": None}
 
+    repository_url = "{}/dists/{}/{}/binary-{}".format(url, dist, comp, arch)
+    if flat_repository:
+        repository_url = "{}/{}".format(url, comp)
+
     decompression_successful = False
     for file_type, tool in file_types.items():
         output = "{}/Packages.{}".format(target_triple, file_type)
         r = rctx.download(
-            url = "{}/dists/{}/{}/binary-{}/Packages.{}".format(url, dist, comp, arch, file_type),
+            url = "{}/Packages.{}".format(repository_url, file_type),
             output = output,
             integrity = integrity,
             allow_fail = True,
@@ -61,6 +65,12 @@ def _parse_package_index(state, contents, arch, root):
 
         if len(pkg.keys()) != 0:
             pkg["Root"] = root
+            if "Filename" in pkg:
+                pkg["Filename"] = pkg["Filename"].strip("./")
+            pkg_arch = pkg.get("Architecture")
+            if pkg_arch and pkg_arch != arch and pkg_arch != "all":
+                pkg = {}
+                continue
             util.set_dict(state.packages, value = pkg, keys = (arch, pkg["Package"], pkg["Version"]))
             last_key = ""
             pkg = {}
@@ -83,16 +93,18 @@ def _create(rctx, sources, archs):
     )
 
     for arch in archs:
-        for (url, dist, comp) in sources:
+        for (url, dist, comp, flat_repository, repository_arch) in sources:
             # We assume that `url` does not contain a trailing forward slash when passing to
             # functions below. If one is present, remove it. Some HTTP servers do not handle
             # redirects properly when a path contains "//"
             # (ie. https://mymirror.com/ubuntu//dists/noble/stable/... may return a 404
             # on misconfigured HTTP servers)
+            if repository_arch and repository_arch != arch:
+                continue
             url = url.rstrip("/")
 
             rctx.report_progress("Fetching package index: {}/{}".format(dist, arch))
-            (output, _) = _fetch_package_index(rctx, url, dist, comp, arch, "")
+            (output, _) = _fetch_package_index(rctx, url, dist, comp, arch, "", flat_repository)
 
             # TODO: this is expensive to perform.
             rctx.report_progress("Parsing package index: {}/{}".format(dist, arch))

apt/private/resolve.bzl

@@ -55,7 +55,7 @@ def _deb_resolve_impl(rctx):
     if type(manifest["archs"]) != "list":
         fail("`archs` should be an array")
 
-    if type(manifest["packages"]) != "list":
+    if type(manifest.get("packages", [])) != "list":
         fail("`packages` should be an array")
 
     sources = []
@@ -69,6 +69,8 @@ def _deb_resolve_impl(rctx):
             src["url"],
             distr,
             comp,
+            src.get("flat_repository", False),
+            src.get("arch", None),
         ))
 
     pkgindex = package_index.new(rctx, sources = sources, archs = manifest["archs"])
@@ -77,11 +79,15 @@ def _deb_resolve_impl(rctx):
 
     for arch in manifest["archs"]:
         dep_constraint_set = {}
-        for dep_constraint in manifest["packages"]:
+        packages = manifest.get("packages", [])
+        arch_specific_packages = manifest.get("arch_specific_packages", {})
+
+        if arch in arch_specific_packages:
+            packages = packages + arch_specific_packages[arch]
+        for dep_constraint in packages:
             if dep_constraint in dep_constraint_set:
                 fail("Duplicate package, {}. Please remove it from your manifest".format(dep_constraint))
             dep_constraint_set[dep_constraint] = True
-
             constraint = package_resolution.parse_depends(dep_constraint).pop()
 
             rctx.report_progress("Resolving %s" % dep_constraint)

examples/debian_flat_repository/BUILD.bazel

@@ -0,0 +1,103 @@
+load("@aspect_bazel_lib//lib:tar.bzl", "tar")
+load("@container_structure_test//:defs.bzl", "container_structure_test")
+load("@rules_distroless//apt:defs.bzl", "dpkg_status")
+load("@rules_distroless//distroless:defs.bzl", "cacerts", "group", "passwd")
+load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball")
+
+passwd(
+    name = "passwd",
+    entries = [
+        {
+            "uid": 0,
+            "gid": 0,
+            "home": "/root",
+            "shell": "/bin/bash",
+            "username": "r00t",
+        },
+        {
+            "uid": 100,
+            "gid": 65534,
+            "home": "/home/_apt",
+            "shell": "/usr/sbin/nologin",
+            "username": "_apt",
+        },
+    ],
+)
+
+group(
+    name = "group",
+    entries = [
+        {
+            "name": "root",
+            "gid": 0,
+        },
+        {
+            "name": "_apt",
+            "gid": 65534,
+        },
+    ],
+)
+
+tar(
+    name = "sh",
+    mtree = [
+        # needed as dpkg assumes sh is installed in a typical debian installation.
+        "./bin/sh type=link link=/bin/bash",
+    ],
+)
+
+cacerts(
+    name = "cacerts",
+    package = "@flat_repository//ca-certificates/amd64:data",
+)
+
+PACKAGES = [
+    "@flat_repository//ncurses-base",
+    "@flat_repository//libncurses6",
+    "@flat_repository//tzdata",
+    "@flat_repository//bash",
+    "@flat_repository//coreutils",
+    "@flat_repository//dpkg",
+    "@flat_repository//apt",
+    "@flat_repository//perl",
+    "@flat_repository//r-doc-html",
+]
+
+# Creates /var/lib/dpkg/status with installed package information.
+dpkg_status(
+    name = "dpkg_status",
+    controls = [
+        "%s/amd64:control" % package
+        for package in PACKAGES
+    ],
+)
+
+oci_image(
+    name = "apt",
+    architecture = "amd64",
+    os = "linux",
+    tars = [
+        ":sh",
+        ":passwd",
+        ":group",
+        ":dpkg_status",
+        ":cacerts",
+    ] + [
+        "%s/amd64" % package
+        for package in PACKAGES
+    ],
+)
+
+oci_tarball(
+    name = "tarball",
+    image = ":apt",
+    repo_tags = [
+        "distroless/test:latest",
+    ],
+)
+
+container_structure_test(
+    name = "test",
+    configs = ["test_linux_amd64.yaml"],
+    image = ":apt",
+)