Docker updates (#362)

* use docker build action also cross platform building and cache * don't build arm for elasticsearch * mongo-init amd64 only * update version calc * missing quotes * newline * update docker - use newer java version (fixes #361) - build arm version (fixes #322) * fix error in action * fix tags * update tags pushed * special case for clowder image * add names to steps * push * push image - needed for cache * only push to dockerhub * use github.actor * disable fail-fast * explicit set permissions * push to branch tag for cache * remove cache * version bump * Updated CHANGELOG.md. Co-authored-by: Luigi Marini <[email protected]>
clowder-framework · Jun 10, 2022 · b971c44 · b971c44
1 parent 68baf98
commit b971c44
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 83 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -29,8 +29,10 @@ env:
 jobs:
   docker:
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         name:
           - clowder
@@ -42,126 +44,159 @@ jobs:
             FOLDER: "."
             IMAGE: clowder
             README: README.md
+            PLATFORM: "linux/amd64,linux/arm64"
           - name: mongo-init
             FOLDER: scripts/mongo-init
             IMAGE: mongo-init
             README: ""
+            PLATFORM: "linux/amd64"
           - name: monitor
             FOLDER: scripts/monitor
             IMAGE: monitor
             README: ""
+            PLATFORM: "linux/amd64,linux/arm64"
           - name: elasticsearch
             FOLDER: scripts/elasticsearch
             IMAGE: elasticsearch
             README: ""
+            PLATFORM: "linux/amd64"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # calculate some variables that are used later
-      - name: github branch
+      - name: variable setup
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
             BRANCH="${{ github.event.release.target_commitish }}"
+          elif [[ $GITHUB_REF =~ pull ]]; then
+            BRANCH="$(echo $GITHUB_REF | sed 's#refs/pull/\([0-9]*\)/merge#PR-\1#')"
           else
             BRANCH=${GITHUB_REF##*/}
           fi
-          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
+
           if [ "$BRANCH" == "master" ]; then
             version="$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
             tags="latest"
             oldversion=""
             while [ "${oldversion}" != "${version}" ]; do
               oldversion="${version}"
-              tags="${tags},${version}"
+              tags="${tags} ${version}"
               version=${version%.*}
             done
-            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
-            echo "CLOWDER_TAGS=${tags}" >> $GITHUB_ENV
+            version="$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
           elif [ "$BRANCH" == "develop" ]; then
-            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
-            echo "CLOWDER_TAGS=develop" >> $GITHUB_ENV
+            version="develop"
+            tags="develop"
           else
-            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
-            echo "CLOWDER_TAGS=" >> $GITHUB_ENV
+            version="test"
+            tags="${BRANCH}"
           fi
 
-      # build the docker image, this will always run to make sure
-      # the Dockerfile still works.
-      - name: Build image
-        uses: elgohr/[email protected]
-        env:
-          BRANCH: ${{ env.GITHUB_BRANCH }}
-          VERSION: ${{ env.CLOWDER_VERSION }}
-          BUILDNUMBER: ${{ github.run_number }}
-          GITSHA1: ${{ github.sha  }}
+          push_tags=""
+          for tag in ${tags}; do
+            push_tags="${push_tags}${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}:${tag},"
+            push_tags="${push_tags}ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }}:${tag},"
+          done
+          push_tags="${push_tags%,*}"
+
+          echo "BRANCH=${BRANCH}"
+          echo "VERSION=${version}"
+          echo "TAGS=${tags}"
+          echo "PUSH_TAGS=${push_tags}"
+
+          echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
+          echo "VERSION=${version}" >> $GITHUB_ENV
+          echo "TAGS=${push_tags}" >> $GITHUB_ENV
+
+      # setup docker build
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Inspect Builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      # login to registries
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
         with:
-          registry: docker.pkg.github.com
-          name: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ matrix.IMAGE }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
+      # build the clowder docker images
+      - name: Build and push ${{ matrix.IMAGE }}-build
+        if: matrix.IMAGE == 'clowder'
+        uses: docker/build-push-action@v2
+        with:
+          push: true
           context: ${{ matrix.FOLDER }}
-          tags: "${{ env.TAGS }}"
-          buildargs: BRANCH,VERSION,BUILDNUMBER,GITSHA1
-          no_push: true
-
-      # this will publish to github container registry
-      - name: Publish to GitHub
-        if: github.event_name != 'pull_request' && github.repository == env.MASTER_REPO
-        uses: elgohr/[email protected]
-        env:
-          BRANCH: ${{ env.GITHUB_BRANCH }}
-          VERSION: ${{ env.CLOWDER_VERSION }}
-          BUILDNUMBER: ${{ github.run_number }}
-          GITSHA1: ${{ github.sha  }}
+          platforms: ${{ matrix.PLATFORM }}
+          target: ${{ matrix.IMAGE }}-build
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-build-cache:${{ env.BRANCH }}
+          cache-to: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-build-cache:${{ env.BRANCH }},mode=max
+          tags: ${{ env.TAGS }}
+          build-args: |
+            BRANCH=${{ env.BRANCH }}
+            VERSION=${{ env.VERSION }}
+            BUILDNUMBER=${{ github.run_number }}
+            GITSHA1=${{ github.sha  }}
+
+      - name: Build and push ${{ matrix.IMAGE }}-runtime
+        if: matrix.IMAGE == 'clowder'
+        uses: docker/build-push-action@v2
         with:
-          registry: ghcr.io
-          name: ${{ github.repository_owner }}/${{ matrix.IMAGE }}
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_PASSWORD }}
+          push: true
           context: ${{ matrix.FOLDER }}
-          tags: "${{ env.CLOWDER_TAGS }}"
-          buildargs: BRANCH,VERSION,BUILDNUMBER,GITSHA1
+          platforms: ${{ matrix.PLATFORM }}
+          target: ${{ matrix.IMAGE }}-runtime
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-runtime-cache:${{ env.BRANCH }}
+          cache-to: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-runtime-cache:${{ env.BRANCH }}
+          tags: ${{ env.TAGS }}
+          build-args: |
+            BRANCH=${{ env.BRANCH }}
+            VERSION=${{ env.VERSION }}
+            BUILDNUMBER=${{ github.run_number }}
+            GITSHA1=${{ github.sha  }}
 
-      # this will publish to the clowder dockerhub repo
-      - name: Publish to Docker Hub
-        if: github.event_name != 'pull_request' && github.repository == env.MASTER_REPO
-        uses: elgohr/[email protected]
-        env:
-          BRANCH: ${{ env.GITHUB_BRANCH }}
-          VERSION: ${{ env.CLOWDER_VERSION }}
-          BUILDNUMBER: ${{ github.run_number }}
-          GITSHA1: ${{ github.sha  }}
+      # build the other docker images
+      - name: Build and push ${{ matrix.IMAGE }}
+        if: matrix.IMAGE != 'clowder'
+        uses: docker/build-push-action@v2
         with:
-          name: ${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          push: true
           context: ${{ matrix.FOLDER }}
-          tags: "${{ env.CLOWDER_TAGS }}"
-          buildargs: BRANCH,VERSION,BUILDNUMBER,GITSHA1
+          platforms: ${{ matrix.PLATFORM }}
+          cache-from: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-cache:${{ env.BRANCH }}
+          cache-to: type=registry,ref=${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}-cache:${{ env.BRANCH }},mode=max
+          tags: ${{ env.TAGS }}
+          build-args: |
+            BRANCH=${{ env.BRANCH }}
+            VERSION=${{ env.VERSION }}
+            BUILDNUMBER=${{ github.run_number }}
+            GITSHA1=${{ github.sha  }}
 
-      # this will update the README of the dockerhub repo
-      - name: check file
-        id: filecheck
-        run: |
-          if [ "${{ matrix.README }}" != "" ]; then
-            if [ -e "${{ matrix.README }}" ]; then
-              echo "##[set-output name=readme;]${{ matrix.README }}"
-            else
-              echo "##[set-output name=readme;]"
-            fi
-          else
-            if [ -e "${{ matrix.FOLDER }}/README.md" ]; then
-              echo "##[set-output name=readme;]${{ matrix.FOLDER }}/README.md"
-            else
-              echo "##[set-output name=readme;]"
-            fi
-          fi
+      # update README at DockerHub
       - name: Docker Hub Description
-        if: github.event_name == 'push' && github.repository == env.MASTER_REPO && env.BRANCH == 'master' && steps.filecheck.outputs.readme != ''
+        if: matrix.README != '' && github.event_name == 'push' && github.repository == env.MASTER_REPO && env.BRANCH == 'master'
         uses: peter-evans/dockerhub-description@v2
         env:
           DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
           DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
           DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}
-          README_FILEPATH: ${{ steps.filecheck.outputs.readme }}
+          README_FILEPATH: ${{ matrix.README }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.20.3 - 2022-06-10
+
+### Fixed
+- docker images are build using newest java version [#361](https://github.com/clowder-framework/clowder/issues/361)
+
+### Changed
+- docker builds images for amd64 and arm64 [#322](https://github.com/clowder-framework/clowder/issues/322)
+
 ## 1.20.2 - 2022-04-30
 
 ### Fixed

diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 # ----------------------------------------------------------------------
 # BUILD CLOWDER DIST
 # ----------------------------------------------------------------------
-FROM java:jdk-alpine as clowder-build
+FROM openjdk:8-jdk-bullseye as clowder-build
 
 ARG BRANCH="unknown"
 ARG VERSION="unknown"
@@ -30,7 +30,7 @@ RUN rm -rf target/universal/clowder-*.zip clowder clowder-* \
     && ./sbt dist \
     && unzip -q target/universal/clowder-*.zip \
     && mv clowder-* clowder \
-    && apk add --no-cache zip \
+    && apt-get update && apt-get -y install zip \
     && for x in $(find clowder -name \*.jar); do \
          zip -d $x org/apache/log4j/net/JMSAppender.class org/apache/log4j/net/SocketServer.class | grep 'deleting:' && echo "fixed $x"; \
        done; \
@@ -40,10 +40,7 @@ RUN rm -rf target/universal/clowder-*.zip clowder clowder-* \
 # ----------------------------------------------------------------------
 # BUILD CLOWDER
 # ----------------------------------------------------------------------
-FROM java:jre-alpine
-
-# add bash
-RUN apk add --no-cache bash curl
+FROM openjdk:8-jre-bullseye as clowder-runtime
 
 # environemnt variables
 ARG BRANCH="unknown"

diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py
@@ -22,7 +22,7 @@
 author = 'Luigi Marini'
 
 # The full version, including alpha/beta/rc tags
-release = '1.20.2'
+release = '1.20.3'
 
 
 # -- General configuration ---------------------------------------------------

diff --git a/project/Build.scala b/project/Build.scala
@@ -13,7 +13,7 @@ import NativePackagerKeys._
 object ApplicationBuild extends Build {
 
   val appName = "clowder"
-  val version = "1.20.2"
+  val version = "1.20.3"
   val jvm = "1.7"
 
   def appVersion: String = {

diff --git a/public/swagger.yml b/public/swagger.yml
@@ -9,7 +9,7 @@ info:
     Clowder is a customizable and scalable data management system to support any
     data format and multiple research domains. It is under active development
     and deployed for a variety of research projects.
-  version: 1.20.2
+  version: 1.20.3
   termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
   contact:
     name: Clowder