refactor: lockfile

apt/apt.bzl

@@ -126,8 +126,10 @@ def _apt_install(
     )
 
     if not lock and not nolock:
-        # buildifier: disable=print
-        print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % name)
+        print(
+            "\nNo lockfile was given. To create one please run " +
+            "`bazel run @{}//:lock`".format(name),
+        )
 
     _deb_translate_lock(
         name = name,

apt/extensions.bzl

@@ -1,6 +1,6 @@
 "apt extensions"
 
-load("//apt/private:deb_import.bzl", "deb_import")
+load("//apt/private:deb_import.bzl", "deb_import", "make_deb_import_key")
 load("//apt/private:deb_resolve.bzl", "deb_resolve", "internal_resolve")
 load("//apt/private:deb_translate_lock.bzl", "deb_translate_lock")
 load("//apt/private:lockfile.bzl", "lockfile")
@@ -21,22 +21,20 @@ def _distroless_extension(module_ctx):
                 )
 
                 if not install.nolock:
-                    # buildifier: disable=print
-                    print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % install.name)
+                    print(
+                        "\nNo lockfile was given. To create one please run " +
+                        "`bazel run @{}//:lock`".format(install.name),
+                    )
             else:
                 lockf = lockfile.from_json(module_ctx, module_ctx.read(install.lock))
 
-            for (package) in lockf.packages():
-                package_key = lockfile.make_package_key(
-                    package["name"],
-                    package["version"],
-                    package["arch"],
-                )
+            for package in lockf.packages():
+                deb_import_key = make_deb_import_key(install.name, package)
 
                 deb_import(
-                    name = "%s_%s" % (install.name, package_key),
-                    urls = [package["url"]],
-                    sha256 = package["sha256"],
+                    name = deb_import_key,
+                    url = package.url,
+                    sha256 = package.sha256,
                 )
 
             deb_resolve(

apt/private/BUILD.bazel

@@ -26,6 +26,7 @@ bzl_library(
     srcs = ["deb_translate_lock.bzl"],
     visibility = ["//apt:__subpackages__"],
     deps = [
+        ":deb_import",
         ":lockfile",
         ":starlark_codegen_utils",
         "@bazel_skylib//lib:new_sets",
@@ -39,15 +40,24 @@ bzl_library(
     name = "lockfile",
     srcs = ["lockfile.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = [":util"],
+    deps = [":pkg"],
+)
+
+bzl_library(
+    name = "pkg",
+    srcs = ["pkg.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [
+        ":util",
+    ],
 )
 
 bzl_library(
     name = "apt_deb_repository",
     srcs = ["apt_deb_repository.bzl"],
     visibility = ["//apt:__subpackages__"],
     deps = [
-        ":util",
+        ":nested_dict",
         ":version_constraint",
     ],
 )
@@ -70,6 +80,7 @@ bzl_library(
         ":apt_deb_repository",
         ":apt_dep_resolver",
         ":lockfile",
+        ":util",
         "@aspect_bazel_lib//lib:repo_utils",
     ],
 )
@@ -85,7 +96,10 @@ bzl_library(
     name = "deb_import",
     srcs = ["deb_import.bzl"],
     visibility = ["//apt:__subpackages__"],
-    deps = ["@bazel_tools//tools/build_defs/repo:http.bzl"],
+    deps = [
+        ":pkg",
+        "@bazel_tools//tools/build_defs/repo:http.bzl",
+    ],
 )
 
 bzl_library(
@@ -106,3 +120,9 @@ bzl_library(
     srcs = ["util.bzl"],
     visibility = ["//apt:__subpackages__"],
 )
+
+bzl_library(
+    name = "nested_dict",
+    srcs = ["nested_dict.bzl"],
+    visibility = ["//apt:__subpackages__"],
+)

apt/private/apt_deb_repository.bzl

@@ -1,11 +1,9 @@
 "https://wiki.debian.org/DebianRepository"
 
-load(":util.bzl", "util")
+load(":nested_dict.bzl", "nested_dict")
 load(":version_constraint.bzl", "version_constraint")
 
-def _fetch_package_index(rctx, url, dist, comp, arch, integrity):
-    target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch)
-
+def _fetch_package_index(rctx, url, dist, comp, arch):
     # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2
     #  --keep       -> keep the original file (Bazel might be still committing the output to the cache)
     #  --force      -> overwrite the output if it exists
@@ -15,49 +13,74 @@ def _fetch_package_index(rctx, url, dist, comp, arch, integrity):
         ".xz": ["xz", "--decompress", "--keep", "--force"],
         ".gz": ["gzip", "--decompress", "--keep", "--force"],
         ".bz2": ["bzip2", "--decompress", "--keep", "--force"],
-        "": ["true"],
+        "": None,
     }
 
     failed_attempts = []
 
-    for (ext, cmd) in supported_extensions.items():
-        output = "{}/Packages{}".format(target_triple, ext)
-        dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext)
+    for ext, cmd in supported_extensions.items():
+        index = "Packages"
+        index_full = "{}{}".format(index, ext) if ext else index
+
+        output = "{dist}/{comp}/{arch}/{index}".format(
+            dist = dist,
+            comp = comp,
+            arch = arch,
+            index = index,
+        )
+        output_full = "{}{}".format(output, ext) if ext else output
+
+        index_url = "{url}/dists/{dist}/{comp}/binary-{arch}/{index_full}".format(
+            url = url,
+            dist = dist,
+            comp = comp,
+            arch = arch,
+            index_full = index_full,
+        )
+
         download = rctx.download(
-            url = dist_url,
-            output = output,
-            integrity = integrity,
+            url = index_url,
+            output = output_full,
             allow_fail = True,
         )
-        decompress_r = None
-        if download.success:
-            decompress_r = rctx.execute(cmd + [output])
-            if decompress_r.return_code == 0:
-                integrity = download.integrity
-                break
 
-        failed_attempts.append((dist_url, download, decompress_r))
+        if not download.success:
+            reason = "Download failed. See warning above for details."
+            failed_attempts.append((index_url, reason))
+            continue
 
-    if len(failed_attempts) == len(supported_extensions):
-        attempt_messages = []
-        for (url, download, decompress) in failed_attempts:
-            reason = "unknown"
-            if not download.success:
-                reason = "Download failed. See warning above for details."
-            elif decompress.return_code != 0:
-                reason = "Decompression failed with non-zero exit code.\n\n{}\n{}".format(decompress.stderr, decompress.stdout)
+        if cmd == None:
+            # index is already decompressed
+            break
 
-            attempt_messages.append("""\n*) Failed '{}'\n\n{}""".format(url, reason))
+        decompress_cmd = cmd + [output_full]
+        decompress_res = rctx.execute(decompress_cmd)
 
-        fail("""
-** Tried to download {} different package indices and all failed. 
+        if decompress_res.return_code == 0:
+            break
 
-{}
-        """.format(len(failed_attempts), "\n".join(attempt_messages)))
+        reason = "'{cmd}' returned a non-zero exit code: {return_code}"
+        reason += "\n\n{stderr}\n{stdout}"
+        reason = reason.format(
+            cmd = decompress_cmd,
+            return_code = decompress_res.return_code,
+            stderr = decompress_res.stderr,
+            stdout = decompress_res.stdout,
+        )
+
+        failed_attempts.append((index_url, reason))
+
+    if len(failed_attempts) == len(supported_extensions):
+        attempt_messages = [
+            "\n  * '{}' FAILED:\n\n  {}".format(url, reason)
+            for url, reason in failed_attempts
+        ]
+
+        fail("Failed to fetch packages index:\n" + "\n".join(attempt_messages))
 
-    return ("{}/Packages".format(target_triple), integrity)
+    return rctx.read(output)
 
-def _parse_repository(state, contents, root):
+def _parse_package_index(state, contents, root):
     last_key = ""
     pkg = {}
     for group in contents.split("\n\n"):
@@ -92,28 +115,24 @@ def _parse_repository(state, contents, root):
             pkg = {}
 
 def _add_package(state, package):
-    util.set_dict(state.packages, value = package, keys = (package["Architecture"], package["Package"], package["Version"]))
+    state.packages.set(
+        keys = (package["Architecture"], package["Package"], package["Version"]),
+        value = package,
+    )
 
     # https://www.debian.org/doc/debian-policy/ch-relationships.html#virtual-packages-provides
     if "Provides" in package:
-        provides = version_constraint.parse_dep(package["Provides"])
-        vp = util.get_dict(state.virtual_packages, (package["Architecture"], provides["name"]), [])
-        vp.append((provides, package))
-        util.set_dict(state.virtual_packages, vp, (package["Architecture"], provides["name"]))
+        provides = version_constraint.parse_provides(package["Provides"])
 
-def _virtual_packages(state, name, arch):
-    return util.get_dict(state.virtual_packages, [arch, name], [])
-
-def _package_versions(state, name, arch):
-    return util.get_dict(state.packages, [arch, name], {}).keys()
-
-def _package(state, name, version, arch):
-    return util.get_dict(state.packages, keys = (arch, name, version))
+        state.virtual_packages.add(
+            keys = (package["Architecture"], provides["name"]),
+            value = (provides, package),
+        )
 
-def _create(rctx, sources, archs):
+def _new(rctx, sources, archs):
     state = struct(
-        packages = dict(),
-        virtual_packages = dict(),
+        packages = nested_dict.new(),
+        virtual_packages = nested_dict.new(),
     )
 
     for arch in archs:
@@ -125,39 +144,24 @@ def _create(rctx, sources, archs):
             # on misconfigured HTTP servers)
             url = url.rstrip("/")
 
-            rctx.report_progress("Fetching package index: {}/{} for {}".format(dist, comp, arch))
-            (output, _) = _fetch_package_index(rctx, url, dist, comp, arch, "")
-
-            # TODO: this is expensive to perform.
-            rctx.report_progress("Parsing package index: {}/{} for {}".format(dist, comp, arch))
-            _parse_repository(state, rctx.read(output), url)
-
-    return struct(
-        package_versions = lambda **kwargs: _package_versions(state, **kwargs),
-        virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs),
-        package = lambda **kwargs: _package(state, **kwargs),
-    )
+            index = "{}/{} for {}".format(dist, comp, arch)
 
-deb_repository = struct(
-    new = _create,
-)
+            rctx.report_progress("Fetching package index: %s" % index)
+            output = _fetch_package_index(rctx, url, dist, comp, arch)
 
-# TESTONLY: DO NOT DEPEND ON THIS
-def _create_test_only():
-    state = struct(
-        packages = dict(),
-        virtual_packages = dict(),
-    )
+            rctx.report_progress("Parsing package index: %s" % index)
+            _parse_package_index(state, output, url)
 
     return struct(
-        package_versions = lambda **kwargs: _package_versions(state, **kwargs),
-        virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs),
-        package = lambda **kwargs: _package(state, **kwargs),
-        parse_repository = lambda contents: _parse_repository(state, contents, "http://nowhere"),
-        packages = state.packages,
-        reset = lambda: state.packages.clear(),
+        package_versions = lambda arch, name: state.packages.get((arch, name), {}).keys(),
+        virtual_packages = lambda arch, name: state.virtual_packages.get((arch, name), []),
+        package = lambda arch, name, version: state.packages.get((arch, name, version)),
     )
 
-DO_NOT_DEPEND_ON_THIS_TEST_ONLY = struct(
-    new = _create_test_only,
+deb_repository = struct(
+    new = _new,
+    __test__ = struct(
+        _fetch_package_index = _fetch_package_index,
+        _parse_package_index = _parse_package_index,
+    ),
 )