Push PeerScout DAGs Docker image (#991)

* Push Docker image part of elifesciences/data-hub-issues#961 Similar to elifesciences/data-hub-core-airflow-dags#1504 This is just an intermediate step. Actually we will want to build and push an image that doesn't contain Airflow. * Added missing make target `ci-build-main-image`
elifesciences · Aug 21, 2024 · a03d101 · a03d101
1 parent 7ef5048
commit a03d101
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 8 deletions.
diff --git a/.env b/.env
@@ -1,5 +1,8 @@
 COMPOSE_PROJECT_NAME=peerscout-dags
+
+IMAGE_REPO=elifesciences/peerscout-dags
 IMAGE_TAG=develop
+
 PEERSCOUT_DAGS_AIRFLOW_PORT=8084
 PEERSCOUT_DAGS_DEPLOYMENT_ENV=ci
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,12 +1,20 @@
 elifePipeline {
     node('containers-jenkins-plugin') {
-        def commit
+        def image_repo = 'elifesciences/peerscout-dags'
         def jenkins_image_building_ci_pipeline = 'process/process-data-hub-airflow-image-update-repo-list'
+
+        def commit
+        def commitShort
+        def branch
+        def timestamp
         def git_url
 
         stage 'Checkout', {
             checkout scm
             commit = elifeGitRevision()
+            commitShort = elifeGitRevision().substring(0, 8)
+            branch = sh(script: 'git rev-parse --abbrev-ref HEAD', returnStdout: true).trim()
+            timestamp = sh(script: 'date --utc +%Y%m%d.%H%M', returnStdout: true).trim()
             git_url = getGitUrl()
         }
 
@@ -26,14 +34,36 @@ elifePipeline {
             }
         }
 
+        stage 'Build main image', {
+            sh "make IMAGE_REPO=${image_repo} IMAGE_TAG=${commit} ci-build-main-image"
+        }
+
         elifeMainlineOnly {
+            def dev_image_repo = image_repo + '_unstable'
+
             stage 'Merge to master', {
                 elifeGitMoveToBranch commit, 'master'
             }
+
+            stage 'Push image', {
+                sh "make EXISTING_IMAGE_TAG=${commit} EXISTING_IMAGE_REPO=${image_repo} IMAGE_TAG=${commit} IMAGE_REPO=${dev_image_repo} retag-push-image"
+                sh "make EXISTING_IMAGE_TAG=${commit} EXISTING_IMAGE_REPO=${image_repo} IMAGE_TAG=${branch}-${commitShort}-${timestamp} IMAGE_REPO=${dev_image_repo} retag-push-image"
+                sh "make EXISTING_IMAGE_TAG=${commit} EXISTING_IMAGE_REPO=${image_repo} IMAGE_TAG=latest IMAGE_REPO=${dev_image_repo} retag-push-image"
+            }
+
             stage 'Build data pipeline image with latest commit', {
                 triggerImageBuild(jenkins_image_building_ci_pipeline, git_url, commit)
             }
         }
+
+        elifeTagOnly { tagName ->
+            def candidateVersion = tagName - "v"
+
+            stage 'Push release image', {
+                sh "make EXISTING_IMAGE_TAG=${commit} EXISTING_IMAGE_REPO=${image_repo} IMAGE_TAG=latest IMAGE_REPO=${image_repo} retag-push-image"
+                sh "make EXISTING_IMAGE_TAG=${commit} EXISTING_IMAGE_REPO=${image_repo} IMAGE_TAG=${candidateVersion} IMAGE_REPO=${image_repo} retag-push-image"
+            }
+        }
     }
 }
 

diff --git a/Makefile b/Makefile
@@ -155,6 +155,10 @@ end2end-test:
 	$(DOCKER_COMPOSE) run --rm  test-client
 	$(MAKE) clean
 
+ci-build-main-image:
+	$(MAKE) DOCKER_COMPOSE="$(DOCKER_COMPOSE_CI)" \
+		build
+
 ci-build-dev:
 	$(MAKE) DOCKER_COMPOSE="$(DOCKER_COMPOSE_CI)" build-dev
 
@@ -172,3 +176,8 @@ dev-env: airflow-start airflow-logs
 
 ci-clean:
 	$(DOCKER_COMPOSE_CI) down -v
+
+
+retag-push-image:
+	docker tag  $(EXISTING_IMAGE_REPO):$(EXISTING_IMAGE_TAG) $(IMAGE_REPO):$(IMAGE_TAG)
+	docker push  $(IMAGE_REPO):$(IMAGE_TAG)
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
             - ./credentials.json:/tmp/credentials.json
         build:
             context: .
-        image: elifesciences/peerscout-dags
+        image: ${IMAGE_REPO}:${IMAGE_TAG}
         command: ''
 
     peerscout-dags-dev:
@@ -50,7 +50,7 @@ services:
             dockerfile: Dockerfile
             args:
                 install_dev: y
-        image:  elifesciences/peerscout-dags-dev
+        image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         command: /bin/sh -c exit 0
         entrypoint: []
 
@@ -60,12 +60,12 @@ services:
         environment: *airflow-env
         volumes:
             - ./config/webserver_config.py:/opt/airflow/webserver_config.py
-        image:  elifesciences/peerscout-dags-dev
+        image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         entrypoint: /entrypoint
         command: webserver
 
     scheduler:
-        image:  elifesciences/peerscout-dags-dev
+        image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         depends_on:
             - postgres
         environment: *airflow-env
@@ -78,7 +78,7 @@ services:
           - redis
           - scheduler
         volumes: *airflow-volumes
-        image: elifesciences/peerscout-dags-dev
+        image: ${IMAGE_REPO}-dev:${IMAGE_TAG}
         entrypoint: /entrypoint
         hostname: worker
         command: >
@@ -87,7 +87,7 @@ services:
             && airflow celery worker"
 
     test-client:
-        image:  elifesciences/peerscout-dags-dev
+        image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         depends_on:
             - scheduler
             - webserver
@@ -100,7 +100,7 @@ services:
 
     data-hub-pipelines:
         <<: *x-airflow-defaults
-        image:  elifesciences/peerscout-dags
+        image:  ${IMAGE_REPO}-dev:${IMAGE_TAG}
         environment:
             - DEPLOYMENT_ENV=${PEERSCOUT_DAGS_DEPLOYMENT_ENV}
             - GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials.json