From 93f33ed9e325b2278989d48883bd12502788be04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 17:17:29 +0100 Subject: [PATCH] installer: make sure we can always test the installer in ci and locally Just now there is a dependency on cachix, which means we cannot test the installer in CI if forks do not have the necessary secrets set up. We replace this with a simple http server that serves the installer and can be both used in CI and locally. --- .github/workflows/ci.yml | 120 ++++++++----------- flake.nix | 4 +- packaging/hydra.nix | 13 +- scripts/prepare-installer-for-github-actions | 12 +- scripts/serve-installer-for-github-actions | 23 ++++ 5 files changed, 83 insertions(+), 89 deletions(-) create mode 100755 scripts/serve-installer-for-github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3af1e85249f..b5e8d342350 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,6 @@ jobs: - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: - needs: [check_secrets] strategy: fail-fast: false matrix: @@ -34,72 +33,22 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 if: matrix.os == 'ubuntu-latest' - run: scripts/build-checks - - # Steps to test CI automation in your own fork. - # Cachix: - # 1. Sign-up for https://www.cachix.org/ - # 2. Create a cache for $githubuser-nix-install-tests - # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN - # Dockerhub: - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v15 + - run: scripts/prepare-installer-for-github-actions + - name: Upload installer tarball + uses: actions/upload-artifact@v4 + if: needs.get-merge-commit.outputs.mergedSha with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - cachixArgs: '-v' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions + name: installer + path: out/* installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' + needs: [tests] strategy: fail-fast: false matrix: @@ -107,11 +56,18 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - name: Download installer tarball + uses: actions/download-artifact@v4 + with: + name: installer + path: out/* + - name: Serving installer + id: serving_installer + run: scripts/prepare-installer-for-github-actions - uses: cachix/install-nix-action@v30 with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + install_url: 'http://localhost:8000/${{ steps.serving_installer.outputs.system }}' + install_options: "--tarball-url-prefix http://localhost:8000/" - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-latest' - run: brew install fish @@ -123,32 +79,50 @@ jobs: - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + # Steps to test CI automation in your own fork. + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork + check_secrets: + permissions: + contents: none + name: Check Docker secrets present for installer tests + runs-on: ubuntu-latest + outputs: + docker: ${{ steps.secret.outputs.docker }} + steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + docker_push_image: - needs: [check_secrets, tests, vm_tests] + needs: [tests, vm_tests, check_secrets] permissions: contents: read packages: write if: >- + needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' + github.ref_name == 'master' runs-on: ubuntu-latest steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION diff --git a/flake.nix b/flake.nix index 794736af4ec..a5e68609eda 100644 --- a/flake.nix +++ b/flake.nix @@ -186,7 +186,7 @@ }; checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -237,6 +237,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; + binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 17ba720321c..93bf7045ad9 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -123,15 +123,10 @@ in self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; + + installerScriptForGHA = forAllSystems (system: nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + tarballs = self.hydraJobs.binaryTarball.${system}; + }); # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 4b994a7532c..19aa2af39f7 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -1,10 +1,10 @@ #!/usr/bin/env bash -set -e +set -euo pipefail -script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo "$script" | cut -b12-43 -) +nix build -L ".#installerScriptForGHA" ".#binaryTarball" -installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install - -echo "::set-output name=installerURL::$installerURL" +install -D ./result/install "out/install-$(nix eval --raw --impure --expr builtins.currentSystem)" +name="$(basename "$(realpath ./result-1)")" +# everything before the first dash +cp -r ./result-1 "out/${name%%-*}" diff --git a/scripts/serve-installer-for-github-actions b/scripts/serve-installer-for-github-actions new file mode 100755 index 00000000000..eddb85c69e6 --- /dev/null +++ b/scripts/serve-installer-for-github-actions @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [[ ! -d out ]]; then + echo "run prepare-installer-for-github-actions first" + exit 1 +fi +cd out +PORT=${PORT:-8126} +nohup python -m http.server "$PORT" >/dev/null 2>&1 & +pid=$! + +while ! curl -s "http://localhost:$PORT"; do + sleep 1 + if ! kill -0 $pid; then + echo "Failed to start http server" + exit 1 + fi +done +system=$(nix eval --raw --impure --expr builtins.currentSystem) +echo 'To install nix, run the following command:' +echo "sh <(curl http://localhost:$PORT/install-$system) --tarball-url-prefix http://localhost:$PORT/" +echo "::set-output system=$system"