From 7156956d1c47e830ec207487e53903dd80d8910a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 24 Oct 2024 13:19:45 +0100 Subject: [PATCH 01/10] Add @JimMadge as a contributor --- .all-contributorsrc | 31 +++++++++++++++++++++++++++++++ README.md | 25 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..9a31f3b --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,31 @@ +{ + "projectName": "nexus-allowlist", + "projectOwner": "The contributors", + "repoType": "github", + "repoHost": "https://github.com", + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": true, + "commitConvention": "none", + "contributors": [ + { + "login": "JimMadge", + "name": "Jim Madge", + "avatar_url": "https://avatars.githubusercontent.com/u/23616154?v=4", + "profile": "https://github.com/JimMadge", + "contributions": [ + "bug", + "code", + "doc", + "ideas", + "infra", + "review", + "test" + ] + } + ], + "contributorsPerLine": 7, + "linkToUsage": false +} diff --git a/README.md b/README.md index f49157b..b1a9c08 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # Nexus Allowlist + +[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-) + A package for configuring [Sonatype Nexus Repository Manager OSS](https://github.com/sonatype/nexus-public) to only allow selected packages to be installed from proxy repositories. @@ -92,3 +95,25 @@ For example, - `install.packages("data.table")` should succeed - `install.packages("ggplot2")` should fail + +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + + + +
Jim Madge
Jim Madge

🐛 💻 📖 🤔 🚇 👀 ⚠️
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file From 0611642d694ff8c787ecdda463c693b40c913608 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 24 Oct 2024 13:23:05 +0100 Subject: [PATCH 02/10] Add @craddm as a contributor --- .all-contributorsrc | 11 +++++++++++ README.md | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 9a31f3b..d1732ae 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -24,6 +24,17 @@ "review", "test" ] + }, + { + "login": "craddm", + "name": "Matt Craddock", + "avatar_url": "https://avatars.githubusercontent.com/u/5796417?v=4", + "profile": "https://github.com/craddm", + "contributions": [ + "bug", + "code", + "infra" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index b1a9c08..fdb8f4b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Nexus Allowlist -[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-) +[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-) A package for configuring [Sonatype Nexus Repository Manager OSS](https://github.com/sonatype/nexus-public) to only allow selected packages to be installed from proxy repositories. @@ -107,6 +107,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Jim Madge
Jim Madge

🐛 💻 📖 🤔 🚇 👀 ⚠️ + Matt Craddock
Matt Craddock

🐛 💻 🚇 From 5c0a79cbff2b1b168334c616d48d24a84ee57188 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 24 Oct 2024 13:23:45 +0100 Subject: [PATCH 03/10] Add @jemrobinson as a contributor --- .all-contributorsrc | 11 +++++++++++ README.md | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index d1732ae..73825dc 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -35,6 +35,17 @@ "code", "infra" ] + }, + { + "login": "jemrobinson", + "name": "James Robinson", + "avatar_url": "https://avatars.githubusercontent.com/u/3502751?v=4", + "profile": "https://github.com/jemrobinson", + "contributions": [ + "bug", + "code", + "review" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index fdb8f4b..1f3f6d8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Nexus Allowlist -[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-) +[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-) A package for configuring [Sonatype Nexus Repository Manager OSS](https://github.com/sonatype/nexus-public) to only allow selected packages to be installed from proxy repositories. @@ -108,6 +108,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Jim Madge
Jim Madge

🐛 💻 📖 🤔 🚇 👀 ⚠️ Matt Craddock
Matt Craddock

🐛 💻 🚇 + James Robinson
James Robinson

🐛 💻 👀 From 97acf685f3d38bf0d8a732e1cbca867d899da375 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 24 Oct 2024 13:24:54 +0100 Subject: [PATCH 04/10] Add @Jbaudon as a contributor --- .all-contributorsrc | 11 +++++++++++ README.md | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 73825dc..e59c9c6 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -46,6 +46,17 @@ "code", "review" ] + }, + { + "login": "Jbaudon", + "name": "Jbaudon", + "avatar_url": "https://avatars.githubusercontent.com/u/81579455?v=4", + "profile": "https://github.com/Jbaudon", + "contributions": [ + "code", + "doc", + "ideas" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 1f3f6d8..3e1c0ff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Nexus Allowlist -[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-) +[![All Contributors](https://img.shields.io/badge/all_contributors-4-orange.svg?style=flat-square)](#contributors-) A package for configuring [Sonatype Nexus Repository Manager OSS](https://github.com/sonatype/nexus-public) to only allow selected packages to be installed from proxy repositories. @@ -109,6 +109,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Jim Madge
Jim Madge

🐛 💻 📖 🤔 🚇 👀 ⚠️ Matt Craddock
Matt Craddock

🐛 💻 🚇 James Robinson
James Robinson

🐛 💻 👀 + Jbaudon
Jbaudon

💻 📖 🤔 From 50940ebe6dd39351d8995af4128127d19ab6c319 Mon Sep 17 00:00:00 2001 From: Julien Baudon Date: Wed, 13 Nov 2024 11:34:18 +0100 Subject: [PATCH 05/10] #69 add apt proxy repo support --- README.md | 18 +++++- allowlists/apt.allowlist | 13 ++++ entrypoint.sh | 9 +-- integration_tests/Dockerfile | 1 + integration_tests/sources.list | 1 + nexus_allowlist/__about__.py | 2 +- nexus_allowlist/actions.py | 111 +++++++++++++++++++++++++++++---- nexus_allowlist/cli.py | 13 ++-- nexus_allowlist/nexus.py | 7 ++- nexus_allowlist/settings.py | 6 ++ 10 files changed, 157 insertions(+), 24 deletions(-) create mode 100644 allowlists/apt.allowlist create mode 100644 integration_tests/sources.list create mode 100644 nexus_allowlist/settings.py diff --git a/README.md b/README.md index 3e1c0ff..c60a2b4 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Check and, if you would like, change the following environment variables for the | NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | | ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | -Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist) and [CRAN](allowlists/cran.allowlist). +Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist). The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies. The CRAN allowlist includes cli and data.table You can add more packages by writing the package names, one per line, in the allowlist files. @@ -96,6 +96,22 @@ For example, - `install.packages("data.table")` should succeed - `install.packages("ggplot2")` should fail +#### APT + +You can edit '/etc/apt/sources.list' to use the Nexus APT proxy. + +For example + +``` +deb http://localhost:8080/repository/apt-proxy bookworm main +``` + +You should now only be able to install packages from the allowlist. +For example, + +- `sudo apt install libcurl4-openssl-dev` should succeed +- `sudo apt install tcpdump` should fail + ## Contributors ✨ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): diff --git a/allowlists/apt.allowlist b/allowlists/apt.allowlist new file mode 100644 index 0000000..76a0965 --- /dev/null +++ b/allowlists/apt.allowlist @@ -0,0 +1,13 @@ +r-recommended +r-cran-matrixmodels +libcurl4-openssl-dev +libv8-dev +libxml2-dev +cmake +libfontconfig1-dev +libharfbuzz-dev +libfribidi-dev +libfreetype6-dev +libpng-dev +libtiff5-dev +libjpeg-dev \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 3cea716..fe06c87 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,6 +4,7 @@ export NEXUS_DATA_DIR=/nexus-data export ALLOWLIST_DIR=/allowlists export PYPI_ALLOWLIST="$ALLOWLIST_DIR"/pypi.allowlist export CRAN_ALLOWLIST="$ALLOWLIST_DIR"/cran.allowlist +export APT_ALLOWLIST="$ALLOWLIST_DIR"/apt.allowlist timestamp() { date -Is @@ -37,7 +38,7 @@ nexus-allowlist --version if [ -f "$NEXUS_DATA_DIR/admin.password" ]; then echo "$(timestamp) Initial password file present, running initial configuration" nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" change-initial-password --path "$NEXUS_DATA_DIR" - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$ALLOWLIST_DIR/pypi.allowlist" --cran-package-file "$ALLOWLIST_DIR/cran.allowlist" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" else echo "$(timestamp) No initial password file found, skipping initial configuration" fi @@ -51,13 +52,13 @@ fi if [ -n "$ENTR_FALLBACK" ]; then echo "$(timestamp) Using fallback file monitoring" # Run allowlist configuration now - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" # Periodically check for modification of allowlist files and run configuration again when they are hash=$(hashes) while true; do new_hash=$(hashes) if [ "$hash" != "$new_hash" ]; then - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" hash=$new_hash fi sleep 5 @@ -65,5 +66,5 @@ if [ -n "$ENTR_FALLBACK" ]; then else echo "$(timestamp) Using entr for file monitoring" # Run allowlist configuration now, and again whenever allowlist files are modified - find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" fi diff --git a/integration_tests/Dockerfile b/integration_tests/Dockerfile index 69190bd..1502f89 100644 --- a/integration_tests/Dockerfile +++ b/integration_tests/Dockerfile @@ -4,3 +4,4 @@ RUN apk add --no-cache --update python3 py3-pip R RUN mkdir -p /root/.config/pip COPY pip.conf /root/.config/pip/pip.conf COPY Rprofile /root/.Rprofile +COPY sources.list /etc/apt/sources.list \ No newline at end of file diff --git a/integration_tests/sources.list b/integration_tests/sources.list new file mode 100644 index 0000000..3a61f46 --- /dev/null +++ b/integration_tests/sources.list @@ -0,0 +1 @@ +deb http://localhost:8080/repository/apt-proxy bookworm main \ No newline at end of file diff --git a/nexus_allowlist/__about__.py b/nexus_allowlist/__about__.py index 2d81ab7..fb6e1af 100644 --- a/nexus_allowlist/__about__.py +++ b/nexus_allowlist/__about__.py @@ -1 +1 @@ -__version__ = "v0.11.0" +__version__ = "v0.12.0" diff --git a/nexus_allowlist/actions.py b/nexus_allowlist/actions.py index 40cd349..bbf595d 100644 --- a/nexus_allowlist/actions.py +++ b/nexus_allowlist/actions.py @@ -4,6 +4,12 @@ from pathlib import Path from nexus_allowlist.nexus import NexusAPI, RepositoryType +from nexus_allowlist.settings import ( + APT_DISTRO, + APT_REMOTE_URL, + CRAN_REMOTE_URL, + PYPI_REMOTE_URL, +) @dataclass @@ -17,12 +23,17 @@ class Repository: "pypi_proxy": Repository( repo_type=RepositoryType.PYPI, name="pypi-proxy", - remote_url="https://pypi.org/", + remote_url=PYPI_REMOTE_URL, ), "cran_proxy": Repository( repo_type=RepositoryType.CRAN, name="cran-proxy", - remote_url="https://cran.r-project.org/", + remote_url=CRAN_REMOTE_URL, + ), + "apt_proxy": Repository( + repo_type=RepositoryType.APT, + name="apt-proxy", + remote_url=APT_REMOTE_URL, ), } @@ -37,28 +48,33 @@ def check_package_files(args: argparse.Namespace) -> None: raise: Exception: if any declared allowlist file does not exist """ - for package_file in [args.pypi_package_file, args.cran_package_file]: + for package_file in [ + args.pypi_package_file, + args.cran_package_file, + args.apt_package_file, + ]: if package_file and not package_file.is_file(): msg = f"Package allowlist file {package_file} does not exist" raise Exception(msg) def get_allowlists( - pypi_package_file: Path, cran_package_file: Path -) -> tuple[list[str], list[str]]: + pypi_package_file: Path, cran_package_file: Path, apt_package_file: Path +) -> tuple[list[str], list[str], list[str]]: """ - Create allowlists for PyPI and CRAN packages + Create allowlists for PyPI, CRAN and APT packages Args: pypi_package_file: Path to the PyPI allowlist file or None cran_package_file: Path to the CRAN allowlist file or None Returns: - A tuple of the PyPI and CRAN allowlists (in that order). The lists are + A tuple of the PyPI, CRAN and APT allowlists (in that order). The lists are [] if the corresponding package file argument was None """ pypi_allowlist = [] cran_allowlist = [] + apt_allowlist = [] if pypi_package_file: pypi_allowlist = get_allowlist(pypi_package_file, repo_type=RepositoryType.PYPI) @@ -66,7 +82,10 @@ def get_allowlists( if cran_package_file: cran_allowlist = get_allowlist(cran_package_file, repo_type=RepositoryType.CRAN) - return (pypi_allowlist, cran_allowlist) + if apt_package_file: + apt_allowlist = get_allowlist(apt_package_file, repo_type=RepositoryType.APT) + + return (pypi_allowlist, cran_allowlist, apt_allowlist) def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: @@ -83,9 +102,9 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: allowlist = [] with open(allowlist_path) as allowlist_file: # Sanitise package names - # - convert to lower case if the package is on PyPI. Leave alone on CRAN to - # prevent issues with case-sensitivity - for PyPI replace strings of '.', '_' - # or '-' with '-' + # - convert to lower case if the package is on PyPI or APT. Leave alone on CRAN + # to prevent issues with case-sensitivity - for PyPI replace strings of '.', + # '_' or '-' with '-' # https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#name # - remove any blank entries, which act as a wildcard that would allow any # package @@ -94,6 +113,8 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: match repo_type: case RepositoryType.CRAN: package_name_parsed = package_name.strip() + case RepositoryType.APT: + package_name_parsed = package_name.lower().strip() case RepositoryType.PYPI: package_name_parsed = pypi_replace_characters.sub( "-", package_name.lower().strip() @@ -104,7 +125,7 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: def recreate_repositories(nexus_api: NexusAPI) -> None: """ - Create PyPI and CRAN proxy repositories in an idempotent manner + Create PyPI, CRAN and APT proxy repositories in an idempotent manner Args: nexus_api: NexusAPI object @@ -125,6 +146,7 @@ def recreate_privileges( nexus_api: NexusAPI, pypi_allowlist: list[str], cran_allowlist: list[str], + apt_allowlist: list[str], ) -> list[str]: """ Create content selectors and content selector privileges based on the @@ -134,6 +156,7 @@ def recreate_privileges( nexus_api: NexusAPI object pypi_allowlist: List of allowed PyPI packages cran_allowlist: List of allowed CRAN packages + apt_allowlist: List of allowed APT packages Returns: List of the names of all content selector privileges @@ -148,6 +171,7 @@ def recreate_privileges( pypi_privilege_names = [] cran_privilege_names = [] + apt_privilege_names = [] # Content selector and privilege for PyPI 'simple' path, used to search for # packages @@ -185,6 +209,44 @@ def recreate_privileges( ) cran_privilege_names.append(privilege_name) + # Content selector and privilege for APT 'Packages.gz' file which contains an + # metadata for all archived packages + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-packages", + description="Allow access to 'Packages.gz' file in APT repository", + expression=f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Packages.gz"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + # Content selector and privilege for APT 'InRelease' file which contains an + # metadata about the APT distribution + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="inrelease", + description="Allow access to 'InRelease' file in APT repository", + expression=f'format == "apt" and path=="/dists/{APT_DISTRO}/InRelease"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + # Content selector and privilege for APT 'Translation-*' files which contains an + # metadata about the APT distribution + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-translation", + description="Allow access to 'Translation-*' file in APT repository", + expression=( + f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Translation-.*"' + ), + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + # Create content selectors and privileges for packages according to the # package setting if packages == "all": @@ -209,6 +271,17 @@ def recreate_privileges( repo=_NEXUS_REPOSITORIES["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) + + # Allow all APT packages + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-all", + description="Allow access to all APT packages", + expression='format == "apt" and path=^"/pool/"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) elif packages == "selected": # Allow selected PyPI packages for package in pypi_allowlist: @@ -238,7 +311,19 @@ def recreate_privileges( ) cran_privilege_names.append(privilege_name) - return pypi_privilege_names + cran_privilege_names + # Allow selected APT packages + for package in apt_allowlist: + privilege_name = create_content_selector_and_privilege( + nexus_api, + name=f"apt-{package}", + description=f"Allow access to {packages} APT package", + expression=f'format == "apt" and path=~"^/pool/.*/{package}.*"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + return pypi_privilege_names + cran_privilege_names + apt_privilege_names def create_content_selector_and_privilege( diff --git a/nexus_allowlist/cli.py b/nexus_allowlist/cli.py index bad5cfe..abab00c 100644 --- a/nexus_allowlist/cli.py +++ b/nexus_allowlist/cli.py @@ -76,6 +76,11 @@ def main() -> None: "Path of the file of allowed CRAN packages, ignored when PACKAGES is all" ), ) + packages_parser.add_argument( + "--apt-package-file", + type=Path, + help="Path of the file of allowed APT packages, ignored when PACKAGES is all", + ) subparsers = parser.add_subparsers(title="subcommands", required=True) @@ -168,7 +173,7 @@ def initial_configuration(args: argparse.Namespace) -> None: This includes: - Deleting all respositories - - Creating CRAN and PyPI proxies + - Creating CRAN, APT and PyPI proxies - Deleting all content selectors and content selector privileges - Deleting all non-default roles - Creating a role @@ -234,14 +239,14 @@ def update_allow_lists(args: argparse.Namespace) -> None: ) # Parse allowlists - pypi_allowlist, cran_allowlist = actions.get_allowlists( - args.pypi_package_file, args.cran_package_file + pypi_allowlist, cran_allowlist, apt_allowlist = actions.get_allowlists( + args.pypi_package_file, args.cran_package_file, args.apt_package_file ) # Recreate all content selectors and associated privileges according to the # allowlists privileges = actions.recreate_privileges( - args.packages, nexus_api, pypi_allowlist, cran_allowlist + args.packages, nexus_api, pypi_allowlist, cran_allowlist, apt_allowlist ) # Grant privileges to the nexus allowlist role diff --git a/nexus_allowlist/nexus.py b/nexus_allowlist/nexus.py index a32fb2f..9045052 100644 --- a/nexus_allowlist/nexus.py +++ b/nexus_allowlist/nexus.py @@ -4,6 +4,8 @@ import requests +from nexus_allowlist.settings import APT_DISTRO + _REQUEST_TIMEOUT = 10 @@ -21,6 +23,7 @@ class ResponseCode(Enum): class RepositoryType(Enum): PYPI = "pypi" CRAN = "r" + APT = "apt" class NexusAPI: @@ -96,7 +99,7 @@ def create_proxy_repository( self, repo_type: RepositoryType, name: str, remote_url: str ) -> None: """ - Create a proxy repository. Currently supports PyPI and R formats + Create a proxy repository. Currently supports PyPI, R and APT formats Args: repo_type: Type of repository @@ -123,6 +126,8 @@ def create_proxy_repository( } payload["name"] = name payload["proxy"]["remoteUrl"] = remote_url + if repo_type == RepositoryType.APT: + payload["apt"] = {"distribution": APT_DISTRO, "flat": False} logging.info(f"Creating {repo_type.value} repository: {name}") response = requests.post( diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py new file mode 100644 index 0000000..de6940f --- /dev/null +++ b/nexus_allowlist/settings.py @@ -0,0 +1,6 @@ +import os + +PYPI_REMOTE_URL = os.getenv("PYPI_REMOTE_URL", "https://pypi.org/") +CRAN_REMOTE_URL = os.getenv("CRAN_REMOTE_URL", "https://cran.r-project.org/") +APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") +APT_DISTRO = os.getenv("APT_DISTRO", "bookworm") From 8df6dc0d82814dc0920434758c0a35e07524fbab Mon Sep 17 00:00:00 2001 From: Julien Baudon Date: Mon, 25 Nov 2024 11:16:47 +0100 Subject: [PATCH 06/10] document new env vars --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index c60a2b4..fee81ce 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ Check and, if you would like, change the following environment variables for the | NEXUS_PORT | Port of Nexus OSS | | NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | | ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | +| PYPI_REMOTE_URL | URL of the PYPI Remote repository (`https://pypi.org/` by default) | +| CRAN_REMOTE_URL | URL of the CRAN Remote repository (`https://cran.r-project.org/` by default) | +| APT_REMOTE_URL | URL of the APT Remote repository (`http://deb.debian.org/debian` by default) | +| APT_DISTRO | Name of the APT distribution (`bookworm` by default) | Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist). The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies. From c615901c98f71173d42461d2b6fbf7726c29ea22 Mon Sep 17 00:00:00 2001 From: Julien Baudon Date: Mon, 25 Nov 2024 15:41:39 +0100 Subject: [PATCH 07/10] support apt archive restriction --- README.md | 1 + entrypoint.sh | 2 +- nexus_allowlist/actions.py | 11 +++++++++-- nexus_allowlist/settings.py | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fee81ce..b7981a9 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Check and, if you would like, change the following environment variables for the | CRAN_REMOTE_URL | URL of the CRAN Remote repository (`https://cran.r-project.org/` by default) | | APT_REMOTE_URL | URL of the APT Remote repository (`http://deb.debian.org/debian` by default) | | APT_DISTRO | Name of the APT distribution (`bookworm` by default) | +| APT_ALLOWED_ARCHIVES | Comma-separated list of the authorized APT archives (`main,contrib,non-free-firmware,non-free` by default) | Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist). The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies. diff --git a/entrypoint.sh b/entrypoint.sh index fe06c87..bb70416 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,7 +11,7 @@ timestamp() { } hashes() { - md5sum $PYPI_ALLOWLIST $CRAN_ALLOWLIST + md5sum $PYPI_ALLOWLIST $CRAN_ALLOWLIST $APT_ALLOWLIST } # Ensure allowlist files exist diff --git a/nexus_allowlist/actions.py b/nexus_allowlist/actions.py index bbf595d..c89c6a5 100644 --- a/nexus_allowlist/actions.py +++ b/nexus_allowlist/actions.py @@ -5,6 +5,7 @@ from nexus_allowlist.nexus import NexusAPI, RepositoryType from nexus_allowlist.settings import ( + ALLOWED_ARCHIVES, APT_DISTRO, APT_REMOTE_URL, CRAN_REMOTE_URL, @@ -277,7 +278,10 @@ def recreate_privileges( nexus_api, name="apt-all", description="Allow access to all APT packages", - expression='format == "apt" and path=^"/pool/"', + expression=( + 'format == "apt" and ' + f'path=~"^/pool/({'|'.join(ALLOWED_ARCHIVES)})/.*"' + ), repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, repo=_NEXUS_REPOSITORIES["apt_proxy"].name, ) @@ -317,7 +321,10 @@ def recreate_privileges( nexus_api, name=f"apt-{package}", description=f"Allow access to {packages} APT package", - expression=f'format == "apt" and path=~"^/pool/.*/{package}.*"', + expression=( + 'format == "apt" and ' + f'path=~"^/pool/({'|'.join(ALLOWED_ARCHIVES)})/.*/{package}.*"' + ), repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, repo=_NEXUS_REPOSITORIES["apt_proxy"].name, ) diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py index de6940f..9fd560e 100644 --- a/nexus_allowlist/settings.py +++ b/nexus_allowlist/settings.py @@ -4,3 +4,7 @@ CRAN_REMOTE_URL = os.getenv("CRAN_REMOTE_URL", "https://cran.r-project.org/") APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") APT_DISTRO = os.getenv("APT_DISTRO", "bookworm") +ALLOWED_ARCHIVES = os.getenv( + "APT_ALLOWED_ARCHIVES", + "main,contrib,non-free-firmware,non-free" +).split(",") From ca487510797a7adc7ad4d0c653c0bfbec7c77917 Mon Sep 17 00:00:00 2001 From: Julien Baudon Date: Mon, 25 Nov 2024 15:47:36 +0100 Subject: [PATCH 08/10] fmt --- nexus_allowlist/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py index 9fd560e..2ee1791 100644 --- a/nexus_allowlist/settings.py +++ b/nexus_allowlist/settings.py @@ -5,6 +5,5 @@ APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") APT_DISTRO = os.getenv("APT_DISTRO", "bookworm") ALLOWED_ARCHIVES = os.getenv( - "APT_ALLOWED_ARCHIVES", - "main,contrib,non-free-firmware,non-free" + "APT_ALLOWED_ARCHIVES", "main,contrib,non-free-firmware,non-free" ).split(",") From 91c81912835011bfcee7d357f0ea52a1d1510e5a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 6 Dec 2024 10:19:56 +0000 Subject: [PATCH 09/10] Remove CRAN/PyPI env vars --- README.md | 4 +--- nexus_allowlist/actions.py | 6 ++---- nexus_allowlist/settings.py | 2 -- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b7981a9..9209e64 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,6 @@ Check and, if you would like, change the following environment variables for the | NEXUS_PORT | Port of Nexus OSS | | NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | | ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | -| PYPI_REMOTE_URL | URL of the PYPI Remote repository (`https://pypi.org/` by default) | -| CRAN_REMOTE_URL | URL of the CRAN Remote repository (`https://cran.r-project.org/` by default) | | APT_REMOTE_URL | URL of the APT Remote repository (`http://deb.debian.org/debian` by default) | | APT_DISTRO | Name of the APT distribution (`bookworm` by default) | | APT_ALLOWED_ARCHIVES | Comma-separated list of the authorized APT archives (`main,contrib,non-free-firmware,non-free` by default) | @@ -140,4 +138,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/nexus_allowlist/actions.py b/nexus_allowlist/actions.py index c89c6a5..6da674a 100644 --- a/nexus_allowlist/actions.py +++ b/nexus_allowlist/actions.py @@ -8,8 +8,6 @@ ALLOWED_ARCHIVES, APT_DISTRO, APT_REMOTE_URL, - CRAN_REMOTE_URL, - PYPI_REMOTE_URL, ) @@ -24,12 +22,12 @@ class Repository: "pypi_proxy": Repository( repo_type=RepositoryType.PYPI, name="pypi-proxy", - remote_url=PYPI_REMOTE_URL, + remote_url="https://pypi.org", ), "cran_proxy": Repository( repo_type=RepositoryType.CRAN, name="cran-proxy", - remote_url=CRAN_REMOTE_URL, + remote_url="https://cran.r-project.org", ), "apt_proxy": Repository( repo_type=RepositoryType.APT, diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py index 2ee1791..e7d7958 100644 --- a/nexus_allowlist/settings.py +++ b/nexus_allowlist/settings.py @@ -1,7 +1,5 @@ import os -PYPI_REMOTE_URL = os.getenv("PYPI_REMOTE_URL", "https://pypi.org/") -CRAN_REMOTE_URL = os.getenv("CRAN_REMOTE_URL", "https://cran.r-project.org/") APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") APT_DISTRO = os.getenv("APT_DISTRO", "bookworm") ALLOWED_ARCHIVES = os.getenv( From 6c7fffe1be4ef60e6f7bc0b77eedba339aec3995 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 6 Dec 2024 11:51:09 +0000 Subject: [PATCH 10/10] WIP: Move apt parameters to arguments --- README.md | 22 +++---- entrypoint.sh | 20 +++++-- nexus_allowlist/actions.py | 114 +++++++++++++++++++----------------- nexus_allowlist/cli.py | 30 +++++++++- nexus_allowlist/nexus.py | 2 - nexus_allowlist/settings.py | 7 --- 6 files changed, 115 insertions(+), 80 deletions(-) delete mode 100644 nexus_allowlist/settings.py diff --git a/README.md b/README.md index 9209e64..c1c6e94 100644 --- a/README.md +++ b/README.md @@ -17,17 +17,17 @@ A [Dockerfile](Dockerfile) and example [docker compose](docker-compose.yaml) con Check and, if you would like, change the following environment variables for the Nexus Allowlist container in [`docker-compose.yaml`](./docker-compose.yaml). -| Environment variable | meaning | -| ---------------------- | ------------------------------------------------------------------------------------------------------------- | -| NEXUS_ADMIN_PASSWORD | Password for the Nexus OSS admin user (changes from the default on first rune then used for authentication) | -| NEXUS_PACKAGES | Whether to allow all packages or only selected packages [`all`, `selected`] | -| NEXUS_HOST | Hostname of Nexus OSS host | -| NEXUS_PORT | Port of Nexus OSS | -| NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | -| ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | -| APT_REMOTE_URL | URL of the APT Remote repository (`http://deb.debian.org/debian` by default) | -| APT_DISTRO | Name of the APT distribution (`bookworm` by default) | -| APT_ALLOWED_ARCHIVES | Comma-separated list of the authorized APT archives (`main,contrib,non-free-firmware,non-free` by default) | +| Environment variable | meaning | +| ---------------------- | ------------------------------------------------------------------------------------------------------------- | +| NEXUS_ADMIN_PASSWORD | Password for the Nexus OSS admin user (changes from the default on first rune then used for authentication) | +| NEXUS_PACKAGES | Whether to allow all packages or only selected packages [`all`, `selected`] | +| NEXUS_HOST | Hostname of Nexus OSS host | +| NEXUS_PORT | Port of Nexus OSS | +| NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | +| ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | +| APT_URL | URL of the APT Remote repository (`http://deb.debian.org/debian` by default) | +| APT_RELEASE | Name of the APT distribution (`bookworm` by default) | +| APT_ARCHIVES | Allowed APT archives (`main contrib non-free-firmware non-free` by default) | Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist). The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies. diff --git a/entrypoint.sh b/entrypoint.sh index bb70416..f8f07ca 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,6 +6,18 @@ export PYPI_ALLOWLIST="$ALLOWLIST_DIR"/pypi.allowlist export CRAN_ALLOWLIST="$ALLOWLIST_DIR"/cran.allowlist export APT_ALLOWLIST="$ALLOWLIST_DIR"/apt.allowlist +if [ -z "$APT_URL" ]; then + export APT_URL="https://deb.debian.org/debian" +fi + +if [ -z "$APT_RELEASE" ]; then + export APT_RELEASE="bookworm" +fi + +if [ -z "$APT_ARCHIVES" ]; then + export APT_ARCHIVES="main contrib non-free-firmware non-free" +fi + timestamp() { date -Is } @@ -38,7 +50,7 @@ nexus-allowlist --version if [ -f "$NEXUS_DATA_DIR/admin.password" ]; then echo "$(timestamp) Initial password file present, running initial configuration" nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" change-initial-password --path "$NEXUS_DATA_DIR" - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" --apt-repository-url "$APT_URL" --apt-repository-release "$APT_RELEASE" --apt-repository-archives "$APT_ARCHIVES" else echo "$(timestamp) No initial password file found, skipping initial configuration" fi @@ -52,13 +64,13 @@ fi if [ -n "$ENTR_FALLBACK" ]; then echo "$(timestamp) Using fallback file monitoring" # Run allowlist configuration now - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" --apt-repository-url "$APT_URL" --apt-repository-release "$APT_RELEASE" --apt-repository-archives "$APT_ARCHIVES" # Periodically check for modification of allowlist files and run configuration again when they are hash=$(hashes) while true; do new_hash=$(hashes) if [ "$hash" != "$new_hash" ]; then - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" --apt-repository-url "$APT_URL" --apt-repository-release "$APT_RELEASE" --apt-repository-archives "$APT_ARCHIVES" hash=$new_hash fi sleep 5 @@ -66,5 +78,5 @@ if [ -n "$ENTR_FALLBACK" ]; then else echo "$(timestamp) Using entr for file monitoring" # Run allowlist configuration now, and again whenever allowlist files are modified - find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" + find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" --apt-repository-url "$APT_URL" --apt-repository-release "$APT_RELEASE" --apt-repository-archives "$APT_ARCHIVES" fi diff --git a/nexus_allowlist/actions.py b/nexus_allowlist/actions.py index 6da674a..0145564 100644 --- a/nexus_allowlist/actions.py +++ b/nexus_allowlist/actions.py @@ -2,13 +2,9 @@ import re from dataclasses import dataclass from pathlib import Path +from typing import Any from nexus_allowlist.nexus import NexusAPI, RepositoryType -from nexus_allowlist.settings import ( - ALLOWED_ARCHIVES, - APT_DISTRO, - APT_REMOTE_URL, -) @dataclass @@ -18,23 +14,24 @@ class Repository: remote_url: str -_NEXUS_REPOSITORIES = { - "pypi_proxy": Repository( - repo_type=RepositoryType.PYPI, - name="pypi-proxy", - remote_url="https://pypi.org", - ), - "cran_proxy": Repository( - repo_type=RepositoryType.CRAN, - name="cran-proxy", - remote_url="https://cran.r-project.org", - ), - "apt_proxy": Repository( - repo_type=RepositoryType.APT, - name="apt-proxy", - remote_url=APT_REMOTE_URL, - ), -} +def get_nexus_repositories(args: argparse.Namespace) -> dict[str, Any]: + return { + "pypi_proxy": Repository( + repo_type=RepositoryType.PYPI, + name="pypi-proxy", + remote_url="https://pypi.org", + ), + "cran_proxy": Repository( + repo_type=RepositoryType.CRAN, + name="cran-proxy", + remote_url="https://cran.r-project.org", + ), + "apt_proxy": Repository( + repo_type=RepositoryType.APT, + name="apt-proxy", + remote_url=args.apt_repository_url, + ), + } def check_package_files(args: argparse.Namespace) -> None: @@ -122,17 +119,21 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: return allowlist -def recreate_repositories(nexus_api: NexusAPI) -> None: +def recreate_repositories( + nexus_api: NexusAPI, + nexus_repositories: dict[str, Any] +) -> None: """ Create PyPI, CRAN and APT proxy repositories in an idempotent manner Args: nexus_api: NexusAPI object + nexus_repositories: A dict of Repository objects """ # Delete all existing repositories nexus_api.delete_all_repositories() - for repository in _NEXUS_REPOSITORIES.values(): + for repository in nexus_repositories.values(): nexus_api.create_proxy_repository( repo_type=repository.repo_type, name=repository.name, @@ -143,9 +144,12 @@ def recreate_repositories(nexus_api: NexusAPI) -> None: def recreate_privileges( packages: str, nexus_api: NexusAPI, + nexus_repositories: dict[str, Any], pypi_allowlist: list[str], cran_allowlist: list[str], apt_allowlist: list[str], + apt_release: str, + apt_archives: list[str], ) -> list[str]: """ Create content selectors and content selector privileges based on the @@ -153,9 +157,12 @@ def recreate_privileges( Args: nexus_api: NexusAPI object + nexus_repositories: A dict of Repository objects pypi_allowlist: List of allowed PyPI packages cran_allowlist: List of allowed CRAN packages apt_allowlist: List of allowed APT packages + apt_release: The APT release + apt_archives: List of allowed APT archives Returns: List of the names of all content selector privileges @@ -179,8 +186,8 @@ def recreate_privileges( name="simple", description="Allow access to 'simple' directory in PyPI repository", expression='format == "pypi" and path=^"/simple"', - repo_type=_NEXUS_REPOSITORIES["pypi_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["pypi_proxy"].name, + repo_type=nexus_repositories["pypi_proxy"].repo_type, + repo=nexus_repositories["pypi_proxy"].name, ) pypi_privilege_names.append(privilege_name) @@ -191,8 +198,8 @@ def recreate_privileges( name="packages", description="Allow access to 'PACKAGES' file in CRAN repository", expression='format == "r" and path=="/src/contrib/PACKAGES"', - repo_type=_NEXUS_REPOSITORIES["cran_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["cran_proxy"].name, + repo_type=nexus_repositories["cran_proxy"].repo_type, + repo=nexus_repositories["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) @@ -203,8 +210,8 @@ def recreate_privileges( name="archive", description="Allow access to 'archive.rds' file in CRAN repository", expression='format == "r" and path=="/src/contrib/Meta/archive.rds"', - repo_type=_NEXUS_REPOSITORIES["cran_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["cran_proxy"].name, + repo_type=nexus_repositories["cran_proxy"].repo_type, + repo=nexus_repositories["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) @@ -214,9 +221,9 @@ def recreate_privileges( nexus_api, name="apt-packages", description="Allow access to 'Packages.gz' file in APT repository", - expression=f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Packages.gz"', - repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + expression=f'format == "apt" and path=~"^/dists/{apt_release}/.*/Packages.gz"', + repo_type=nexus_repositories["apt_proxy"].repo_type, + repo=nexus_repositories["apt_proxy"].name, ) apt_privilege_names.append(privilege_name) @@ -226,9 +233,9 @@ def recreate_privileges( nexus_api, name="inrelease", description="Allow access to 'InRelease' file in APT repository", - expression=f'format == "apt" and path=="/dists/{APT_DISTRO}/InRelease"', - repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + expression=f'format == "apt" and path=="/dists/{apt_release}/InRelease"', + repo_type=nexus_repositories["apt_proxy"].repo_type, + repo=nexus_repositories["apt_proxy"].name, ) apt_privilege_names.append(privilege_name) @@ -239,10 +246,10 @@ def recreate_privileges( name="apt-translation", description="Allow access to 'Translation-*' file in APT repository", expression=( - f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Translation-.*"' + f'format == "apt" and path=~"^/dists/{apt_release}/.*/Translation-.*"' ), - repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + repo_type=nexus_repositories["apt_proxy"].repo_type, + repo=nexus_repositories["apt_proxy"].name, ) apt_privilege_names.append(privilege_name) @@ -255,8 +262,8 @@ def recreate_privileges( name="pypi-all", description="Allow access to all PyPI packages", expression='format == "pypi" and path=^"/packages/"', - repo_type=_NEXUS_REPOSITORIES["pypi_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["pypi_proxy"].name, + repo_type=nexus_repositories["pypi_proxy"].repo_type, + repo=nexus_repositories["pypi_proxy"].name, ) pypi_privilege_names.append(privilege_name) @@ -266,8 +273,8 @@ def recreate_privileges( name="cran-all", description="Allow access to all CRAN packages", expression='format == "r" and path=^"/src/contrib"', - repo_type=_NEXUS_REPOSITORIES["cran_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["cran_proxy"].name, + repo_type=nexus_repositories["cran_proxy"].repo_type, + repo=nexus_repositories["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) @@ -277,11 +284,10 @@ def recreate_privileges( name="apt-all", description="Allow access to all APT packages", expression=( - 'format == "apt" and ' - f'path=~"^/pool/({'|'.join(ALLOWED_ARCHIVES)})/.*"' + f'format == "apt" and path=~"^/pool/({'|'.join(apt_archives)})/.*"' ), - repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + repo_type=nexus_repositories["apt_proxy"].repo_type, + repo=nexus_repositories["apt_proxy"].name, ) apt_privilege_names.append(privilege_name) elif packages == "selected": @@ -292,8 +298,8 @@ def recreate_privileges( name=f"pypi-{package}", description=f"Allow access to {package} on PyPI", expression=f'format == "pypi" and path=^"/packages/{package}/"', - repo_type=_NEXUS_REPOSITORIES["pypi_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["pypi_proxy"].name, + repo_type=nexus_repositories["pypi_proxy"].repo_type, + repo=nexus_repositories["pypi_proxy"].name, ) pypi_privilege_names.append(privilege_name) @@ -308,8 +314,8 @@ def recreate_privileges( f'and (path=^"/src/contrib/{package}_" ' f'or path=^"/src/contrib/Archive/{package}/{package}_")' ), - repo_type=_NEXUS_REPOSITORIES["cran_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["cran_proxy"].name, + repo_type=nexus_repositories["cran_proxy"].repo_type, + repo=nexus_repositories["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) @@ -321,10 +327,10 @@ def recreate_privileges( description=f"Allow access to {packages} APT package", expression=( 'format == "apt" and ' - f'path=~"^/pool/({'|'.join(ALLOWED_ARCHIVES)})/.*/{package}.*"' + f'path=~"^/pool/({'|'.join(apt_archives)})/.*/{package}.*"' ), - repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, - repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + repo_type=nexus_repositories["apt_proxy"].repo_type, + repo=nexus_repositories["apt_proxy"].name, ) apt_privilege_names.append(privilege_name) diff --git a/nexus_allowlist/cli.py b/nexus_allowlist/cli.py index abab00c..a5d0103 100644 --- a/nexus_allowlist/cli.py +++ b/nexus_allowlist/cli.py @@ -81,6 +81,22 @@ def main() -> None: type=Path, help="Path of the file of allowed APT packages, ignored when PACKAGES is all", ) + packages_parser.add_argument( + "--apt-repository-url", + type=str, + help="URL of the upstream APT repository to proxy", + ) + packages_parser.add_argument( + "--apt-repository-release", + type=str, + help="The release name for APT packages", + ) + packages_parser.add_argument( + "--apt-repository-archives", + type=str, + nargs="*", + help="APT repository archives to allow", + ) subparsers = parser.add_subparsers(title="subcommands", required=True) @@ -195,7 +211,10 @@ def initial_configuration(args: argparse.Namespace) -> None: ) # Ensure only desired repositories exist - actions.recreate_repositories(nexus_api) + actions.recreate_repositories( + nexus_api, + actions.get_nexus_repositories(args), + ) # Delete non-default roles nexus_api.delete_all_custom_roles() @@ -246,7 +265,14 @@ def update_allow_lists(args: argparse.Namespace) -> None: # Recreate all content selectors and associated privileges according to the # allowlists privileges = actions.recreate_privileges( - args.packages, nexus_api, pypi_allowlist, cran_allowlist, apt_allowlist + args.packages, + nexus_api, + actions.get_nexus_repositories(args), + pypi_allowlist, + cran_allowlist, + apt_allowlist, + args.apt_release, + args.apt_archives, ) # Grant privileges to the nexus allowlist role diff --git a/nexus_allowlist/nexus.py b/nexus_allowlist/nexus.py index 9045052..9d6e5f6 100644 --- a/nexus_allowlist/nexus.py +++ b/nexus_allowlist/nexus.py @@ -4,8 +4,6 @@ import requests -from nexus_allowlist.settings import APT_DISTRO - _REQUEST_TIMEOUT = 10 diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py deleted file mode 100644 index e7d7958..0000000 --- a/nexus_allowlist/settings.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") -APT_DISTRO = os.getenv("APT_DISTRO", "bookworm") -ALLOWED_ARCHIVES = os.getenv( - "APT_ALLOWED_ARCHIVES", "main,contrib,non-free-firmware,non-free" -).split(",")