From c17744c97d3d969a74a91f0cb14663ae2834050b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 29 Nov 2023 10:57:59 +0200 Subject: [PATCH 1/3] Add a CI job to build a Docker manifest Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/ci.yaml | 50 +++++++++++++++++++++++++++++++++++---- scripts/deploy-mgrigorov | 17 +++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100755 scripts/deploy-mgrigorov diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 88ccb05..9423399 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -260,10 +260,52 @@ jobs: run: | ./.circleci/run_docker_build.sh - - name: Deploy - if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' + - name: Deploy images + # if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' env: + CFD_QUAY_USER: ${{ secrets.CFD_QUAY_USER }} CFD_QUAY_PASSWORD: ${{ secrets.CFD_QUAY_PASSWORD }} - DH_PASSWORD: ${{ secrets.DH_PASSWORD }} + # DH_PASSWORD: ${{ secrets.DH_PASSWORD }} + DH_USER: ${{ secrets.DOCKER_HUB_USER }} + DH_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWD }} run: | - ./scripts/deploy + ./scripts/deploy-mgrigorov + + build-manifest: + needs: [build] + name: ${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cfg: + - DOCKER_MANIFEST: condaforge/linux-anvil-cos7 + DOCKER_TAG: "latest" + DOCKER_IMAGES: condaforge/linux-anvil-cos7-x86_64:latest,condaforge/linux-anvil-ppc64le:latest,condaforge/linux-anvil-aarch64:latest + + - DOCKER_MANIFEST: condaforge/linux-anvil-alma + DOCKER_TAG: "8" + DOCKER_IMAGES: condaforge/linux-anvil-alma-x86_64:8,condaforge/linux-anvil-alma-ppc64le:8,condaforge/linux-anvil-alma-aarch64:8 + + steps: + + - name: Create Docker manifests + uses: Noelware/docker-manifest-action@v0.3.0 + with: + inputs: ${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} + images: ${{ matrix.cfg.DOCKER_IMAGES }} + push: false + + - name: Deploy manifests + # if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' + env: + CFD_QUAY_USER: ${{ secrets.CFD_QUAY_USER }} + CFD_QUAY_PASSWORD: ${{ secrets.CFD_QUAY_PASSWORD }} + # DH_PASSWORD: ${{ secrets.DH_PASSWORD }} + DH_USER: ${{ secrets.DOCKER_HUB_USER }} + DH_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWD }} + DOCKERIMAGE: ${{ matrix.cfg.DOCKER_MANIFEST }} + DOCKERTAG: ${{ matrix.cfg.DOCKER_TAG }} + run: | + ./scripts/deploy-mgrigorov + \ No newline at end of file diff --git a/scripts/deploy-mgrigorov b/scripts/deploy-mgrigorov new file mode 100755 index 0000000..33248d7 --- /dev/null +++ b/scripts/deploy-mgrigorov @@ -0,0 +1,17 @@ +#!/bin/bash + +set -xe + +docker images + +# dockerhub +echo ${DH_PASSWORD} | docker login -u ${DH_USER} --password-stdin +docker tag condaforge/${DOCKERIMAGE}:${DOCKERTAG} mtgrigorov/${DOCKERIMAGE}:${DOCKERTAG} +docker images +docker push mtgrigorov/${DOCKERIMAGE}:${DOCKERTAG} + +# quay.io +echo ${CFD_QUAY_PASSWORD} | docker login -u ${CFD_QUAY_USER} --password-stdin quay.io +docker tag condaforge/${DOCKERIMAGE}:${DOCKERTAG} quay.io/mgrigorov/${DOCKERIMAGE}:${DOCKERTAG} +docker images +docker push quay.io/mgrigorov/${DOCKERIMAGE}:${DOCKERTAG} From 5ae92d4111b3ba939304733a19d015537bbcac48 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 29 Nov 2023 13:35:34 +0200 Subject: [PATCH 2/3] Test creating manifests Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/ci.yaml | 102 ++++++++++++++++++++++++++++---------- scripts/deploy | 9 ++-- scripts/deploy-mgrigorov | 17 ------- 3 files changed, 79 insertions(+), 49 deletions(-) delete mode 100755 scripts/deploy-mgrigorov diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9423399..28b5f6b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,12 @@ on: - cron: "0 6 * * SUN" # Once weekly on Sunday @ 0600 UTC workflow_dispatch: null +env: + # REGISTRY_DH: docker.io/condaforge + # REGISTRY_QUAY: quay.io/condaforge + REGISTRY_DH: docker.io/mtgrigorov + REGISTRY_QUAY: quay.io/mgrigorov + jobs: build: name: ${{ matrix.cfg.DOCKERIMAGE }}:${{ matrix.cfg.DOCKERTAG }} (${{ matrix.cfg.DISTRO_ARCH }}) @@ -253,6 +259,8 @@ jobs: --build-arg DISTRO_NAME \ --build-arg DISTRO_VER \ -t condaforge/$DOCKERIMAGE:$DOCKERTAG \ + -t ${{ env.REGISTRY_DH }}/$DOCKERIMAGE:$DOCKERTAG \ + -t ${{ env.REGISTRY_QUAY }}/$DOCKERIMAGE:$DOCKERTAG \ -f ${DOCKERFILE:-${DOCKERIMAGE}}/Dockerfile \ --no-cache --squash . @@ -263,49 +271,89 @@ jobs: - name: Deploy images # if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' env: + # CFD_QUAY_USER: conda_forge_daemon CFD_QUAY_USER: ${{ secrets.CFD_QUAY_USER }} CFD_QUAY_PASSWORD: ${{ secrets.CFD_QUAY_PASSWORD }} # DH_PASSWORD: ${{ secrets.DH_PASSWORD }} + # DH_USER: condaforgebot DH_USER: ${{ secrets.DOCKER_HUB_USER }} DH_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWD }} run: | - ./scripts/deploy-mgrigorov + ./scripts/deploy build-manifest: needs: [build] - name: ${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} + # if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' + name: ${{ matrix.docker-registry }}/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: + docker-registry: [docker.io/mtgrigorov, quay.io/mgrigorov] cfg: - - DOCKER_MANIFEST: condaforge/linux-anvil-cos7 + - DOCKER_MANIFEST: linux-anvil-cos7 DOCKER_TAG: "latest" - DOCKER_IMAGES: condaforge/linux-anvil-cos7-x86_64:latest,condaforge/linux-anvil-ppc64le:latest,condaforge/linux-anvil-aarch64:latest - - - DOCKER_MANIFEST: condaforge/linux-anvil-alma + DOCKER_IMAGES: "<>/linux-anvil-cos7-x86_64:<>,<>/linux-anvil-ppc64le:<>,<>/linux-anvil-aarch64:<>" + - DOCKER_MANIFEST: linux-anvil-alma DOCKER_TAG: "8" - DOCKER_IMAGES: condaforge/linux-anvil-alma-x86_64:8,condaforge/linux-anvil-alma-ppc64le:8,condaforge/linux-anvil-alma-aarch64:8 - + DOCKER_IMAGES: "<>/linux-anvil-alma-x86_64:<>,<>/linux-anvil-alma-ppc64le:<>,<>/linux-anvil-alma-aarch64:<>" + - DOCKER_MANIFEST: linux-anvil-ubi-cuda + DOCKER_TAG: "11.0" + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.1" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.2" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.3" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.4" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.5" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.6" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.7" + - DOCKER_MANIFEST: linux-anvil-cos7-cuda + DOCKER_IMAGES: "<>/linux-anvil-cuda:<>,<>/linux-anvil-ppc64le-cuda:<>,<>/linux-anvil-aarch64-cuda:<>" + DOCKER_TAG: "11.8" + steps: - - - name: Create Docker manifests + - uses: actions/checkout@v4 + + - name: Interpolate placeholders + id: interpolate + run: | + set -x + INTERPOLATED=`echo "${{ matrix.cfg.DOCKER_IMAGES }}" | sed "s#<>#${{ matrix.docker-registry }}#g" | sed "s#<>#${{ matrix.cfg.DOCKER_TAG }}#g"` + echo "DOCKER_IMAGES=${INTERPOLATED}" >> "$GITHUB_OUTPUT" + + - name: Login to DockerHub registry + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_PASSWD }} + # username: condaforgebot + # password: ${{ secrets.DH_PASSWORD }} + + - name: Login to Quay.io registry + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.CFD_QUAY_USER }} + password: ${{ secrets.CFD_QUAY_PASSWORD }} + + - name: Push Docker manifest list for ${{ matrix.docker-registry }} uses: Noelware/docker-manifest-action@v0.3.0 with: - inputs: ${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} - images: ${{ matrix.cfg.DOCKER_IMAGES }} - push: false - - - name: Deploy manifests - # if: github.ref == 'refs/heads/main' && github.repository == 'conda-forge/docker-images' - env: - CFD_QUAY_USER: ${{ secrets.CFD_QUAY_USER }} - CFD_QUAY_PASSWORD: ${{ secrets.CFD_QUAY_PASSWORD }} - # DH_PASSWORD: ${{ secrets.DH_PASSWORD }} - DH_USER: ${{ secrets.DOCKER_HUB_USER }} - DH_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWD }} - DOCKERIMAGE: ${{ matrix.cfg.DOCKER_MANIFEST }} - DOCKERTAG: ${{ matrix.cfg.DOCKER_TAG }} - run: | - ./scripts/deploy-mgrigorov - \ No newline at end of file + inputs: ${{ matrix.docker-registry }}/${{ matrix.cfg.DOCKER_MANIFEST }}:${{ matrix.cfg.DOCKER_TAG }} + images: ${{ steps.interpolate.outputs.DOCKER_IMAGES }} + push: true + \ No newline at end of file diff --git a/scripts/deploy b/scripts/deploy index 5545857..7aeeb7b 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -1,10 +1,9 @@ #!/bin/bash # dockerhub -docker login -u condaforgebot -p $DH_PASSWORD -docker push condaforge/$DOCKERIMAGE:$DOCKERTAG +docker login -u ${DH_USER} -p ${DH_PASSWORD} +docker push ${REGISTRY_DH}/$DOCKERIMAGE:$DOCKERTAG # quay.io -docker login -u conda_forge_daemon -p ${CFD_QUAY_PASSWORD} quay.io -docker tag condaforge/$DOCKERIMAGE:$DOCKERTAG quay.io/condaforge/$DOCKERIMAGE:$DOCKERTAG -docker push quay.io/condaforge/$DOCKERIMAGE:$DOCKERTAG +docker login -u ${CFD_QUAY_USER} -p ${CFD_QUAY_PASSWORD} quay.io +docker push ${REGISTRY_QUAY}/$DOCKERIMAGE:$DOCKERTAG diff --git a/scripts/deploy-mgrigorov b/scripts/deploy-mgrigorov deleted file mode 100755 index 33248d7..0000000 --- a/scripts/deploy-mgrigorov +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -xe - -docker images - -# dockerhub -echo ${DH_PASSWORD} | docker login -u ${DH_USER} --password-stdin -docker tag condaforge/${DOCKERIMAGE}:${DOCKERTAG} mtgrigorov/${DOCKERIMAGE}:${DOCKERTAG} -docker images -docker push mtgrigorov/${DOCKERIMAGE}:${DOCKERTAG} - -# quay.io -echo ${CFD_QUAY_PASSWORD} | docker login -u ${CFD_QUAY_USER} --password-stdin quay.io -docker tag condaforge/${DOCKERIMAGE}:${DOCKERTAG} quay.io/mgrigorov/${DOCKERIMAGE}:${DOCKERTAG} -docker images -docker push quay.io/mgrigorov/${DOCKERIMAGE}:${DOCKERTAG} From 426b61256ffcc2b15165cd7f57ea60d0ff230cf2 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 10 Jan 2024 14:00:41 +0200 Subject: [PATCH 3/3] Try to use ${{ vars.** }} in matrix setting Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 28b5f6b..dc2c638 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,8 +12,8 @@ on: env: # REGISTRY_DH: docker.io/condaforge # REGISTRY_QUAY: quay.io/condaforge - REGISTRY_DH: docker.io/mtgrigorov - REGISTRY_QUAY: quay.io/mgrigorov + REGISTRY_DH: ${{ vars.REGISTRY_DH }} + REGISTRY_QUAY: ${{ vars.REGISTRY_QUAY }} jobs: build: @@ -289,7 +289,7 @@ jobs: strategy: fail-fast: false matrix: - docker-registry: [docker.io/mtgrigorov, quay.io/mgrigorov] + docker-registry: [${{ vars.REGISTRY_DH }}, ${{ vars.REGISTRY_QUAY }}] cfg: - DOCKER_MANIFEST: linux-anvil-cos7 DOCKER_TAG: "latest"