Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support flat repos #97

Open
wants to merge 23 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
82f07c4
fix: apt_deb_repository bzl_library missing dep
jjmaestro Nov 22, 2024
02d14c1
refactor: apt_dep_resolver
jjmaestro Nov 21, 2024
410db46
refactor: _fetch_package_index
jjmaestro Sep 13, 2024
a209d8c
feat: support uncompressed Package indexes
jjmaestro Nov 18, 2024
d3d6a92
refactor: _version_relop
jjmaestro Nov 18, 2024
f9d2c4c
chore: move version_constraint tests to their own test file
jjmaestro Nov 18, 2024
a5596ab
chore: mv resolution_test to apt_dep_resolver_test
jjmaestro Nov 18, 2024
4f504f4
refactor: _parse_version_constraint
jjmaestro Nov 18, 2024
8a531cd
refactor: _resolve_all
jjmaestro Nov 18, 2024
c1887fa
refactor: nested_dict struct
jjmaestro Nov 21, 2024
8c899f0
refactor: parse_provides
jjmaestro Nov 21, 2024
7fb01cf
refactor: remove is_satisfied_by
jjmaestro Nov 21, 2024
8988628
refactor: cleanup _resolve_package virtual_package logic
jjmaestro Nov 21, 2024
b0cf234
test: apt_deb_repository
jjmaestro Nov 18, 2024
9def5ff
refactor: apt_dep_resolver_test
jjmaestro Nov 22, 2024
9ecc82e
feat: util_test
jjmaestro Nov 26, 2024
cfbaf3e
refactor: lockfile, pkg, and lock v1-to-v2 auto-migration
jjmaestro Nov 22, 2024
4c76750
chore: migrate repo locks to v2
jjmaestro Nov 22, 2024
5d7e37a
refactor: move all manifest parsing and validation to manifest.bzl
jjmaestro Nov 22, 2024
53606b0
feat: add Starlark code generation
jjmaestro Nov 25, 2024
acf8b64
refactor: centralize all of the deb_import functionality in deb_impor…
jjmaestro Nov 22, 2024
f2cc792
feat: support Debian flat repos
jjmaestro Nov 22, 2024
ea90090
fix: NVIDIA CUDA flat repos don't follow Debian repo spec
jjmaestro Nov 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ apt.install(
manifest = "//examples/debian_snapshot:bullseye_nolock.yaml",
nolock = True,
)
apt.install(
name = "bullseye_rproject",
lock = "//examples/debian_flat_repo:bullseye_rproject.lock.json",
manifest = "//examples/debian_flat_repo:bullseye_rproject.yaml",
)
apt.install(
name = "apt_security",
manifest = "//examples/debian_snapshot_security:security.yaml",
Expand All @@ -85,4 +90,9 @@ apt.install(
lock = "//examples/ubuntu_snapshot:noble.lock.json",
manifest = "//examples/ubuntu_snapshot:noble.yaml",
)
use_repo(apt, "apt_security", "apt_security_resolve", "bullseye", "bullseye_nolock", "noble", "shared_dependencies")
apt.install(
name = "nvidia_ubuntu2404_cuda",
lock = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.lock.json",
manifest = "//examples/debian_flat_repo:nvidia_ubuntu2404_cuda.yaml",
)
use_repo(apt, "apt_security", "apt_security_resolve", "bullseye", "bullseye_nolock", "bullseye_rproject", "noble", "nvidia_ubuntu2404_cuda", "shared_dependencies")
1 change: 1 addition & 0 deletions apt/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,6 @@ bzl_library(
"//apt/private:deb_resolve",
"//apt/private:deb_translate_lock",
"//apt/private:lockfile",
"//apt/private:manifest",
],
)
10 changes: 6 additions & 4 deletions apt/apt.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ def _apt_install(
and avoid the DEBUG messages.
package_template: (EXPERIMENTAL!) a template file for generated BUILD
files. Available template replacement keys are:
`{target_name}`, `{deps}`, `{urls}`, `{name}`,
`{arch}`, `{sha256}`, `{repo_name}`
`{target_name}`, `{src}`, `{deps}`, `{urls}`,
`{name}`, `{arch}`, `{sha256}`, `{repo_name}`
resolve_transitive: whether dependencies of dependencies should be
resolved and added to the lockfile.
"""
Expand All @@ -126,8 +126,10 @@ def _apt_install(
)

if not lock and not nolock:
# buildifier: disable=print
print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % name)
print(
"\nNo lockfile was given. To create one please run " +
"`bazel run @{}//:lock`".format(name),
)

_deb_translate_lock(
name = name,
Expand Down
28 changes: 13 additions & 15 deletions apt/extensions.bzl
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"apt extensions"

load("//apt/private:deb_import.bzl", "deb_import")
load("//apt/private:deb_resolve.bzl", "deb_resolve", "internal_resolve")
load("//apt/private:deb_import.bzl", "deb_import", "make_deb_import_key")
load("//apt/private:deb_resolve.bzl", "deb_resolve")
load("//apt/private:deb_translate_lock.bzl", "deb_translate_lock")
load("//apt/private:lockfile.bzl", "lockfile")
load("//apt/private:manifest.bzl", "manifest")

def _distroless_extension(module_ctx):
root_direct_deps = []
Expand All @@ -13,30 +14,27 @@ def _distroless_extension(module_ctx):
for install in mod.tags.install:
lockf = None
if not install.lock:
lockf = internal_resolve(
lockf = manifest.lock(
module_ctx,
"yq",
install.manifest,
install.resolve_transitive,
)

if not install.nolock:
# buildifier: disable=print
print("\nNo lockfile was given, please run `bazel run @%s//:lock` to create the lockfile." % install.name)
print(
"\nNo lockfile was given. To create one please run " +
"`bazel run @{}//:lock`".format(install.name),
)
else:
lockf = lockfile.from_json(module_ctx, module_ctx.read(install.lock))

for (package) in lockf.packages():
package_key = lockfile.make_package_key(
package["name"],
package["version"],
package["arch"],
)
for package in lockf.packages():
deb_import_key = make_deb_import_key(install.name, package)

deb_import(
name = "%s_%s" % (install.name, package_key),
urls = [package["url"]],
sha256 = package["sha256"],
name = deb_import_key,
url = package.url,
sha256 = package.sha256,
)

deb_resolve(
Expand Down
55 changes: 52 additions & 3 deletions apt/private/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ bzl_library(
srcs = ["deb_translate_lock.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [
":deb_import",
":lockfile",
"@bazel_tools//tools/build_defs/repo:cache.bzl",
"@bazel_tools//tools/build_defs/repo:http.bzl",
Expand All @@ -37,21 +38,46 @@ bzl_library(
name = "lockfile",
srcs = ["lockfile.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [":util"],
deps = [":pkg"],
)

bzl_library(
name = "pkg",
srcs = ["pkg.bzl"],
visibility = ["//apt:__subpackages__"],
)

bzl_library(
name = "manifest",
srcs = ["manifest.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [
":apt_deb_repository",
":apt_dep_resolver",
":lockfile",
":util",
":version_constraint",
"@aspect_bazel_lib//lib:repo_utils",
],
)

bzl_library(
name = "apt_deb_repository",
srcs = ["apt_deb_repository.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [":util"],
deps = [
":nested_dict",
":version_constraint",
"@bazel_skylib//lib:paths",
],
)

bzl_library(
name = "apt_dep_resolver",
srcs = ["apt_dep_resolver.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [
":util",
":version",
":version_constraint",
],
Expand All @@ -65,6 +91,7 @@ bzl_library(
":apt_deb_repository",
":apt_dep_resolver",
":lockfile",
":manifest",
"@aspect_bazel_lib//lib:repo_utils",
],
)
Expand All @@ -80,13 +107,20 @@ bzl_library(
name = "deb_import",
srcs = ["deb_import.bzl"],
visibility = ["//apt:__subpackages__"],
deps = ["@bazel_tools//tools/build_defs/repo:http.bzl"],
deps = [
":starlark_codegen",
":util",
"@bazel_tools//tools/build_defs/repo:http.bzl",
],
)

bzl_library(
name = "util",
srcs = ["util.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [
"@bazel_skylib//lib:sets",
],
)

bzl_library(
Expand All @@ -95,3 +129,18 @@ bzl_library(
visibility = ["//apt:__subpackages__"],
deps = [":version"],
)

bzl_library(
name = "nested_dict",
srcs = ["nested_dict.bzl"],
visibility = ["//apt:__subpackages__"],
)

bzl_library(
name = "starlark_codegen",
srcs = ["starlark_codegen.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [
":util",
],
)
Loading