Use UBI as base image (#16599)

Logstash Docker images, full and OSS, now use UBI image as its base, replacing the previous Ubuntu base. - change the base image of `full` and `oss` to ubi - Set locale to C.UTF-8 - remove ubi flavour - use go image to build env2yaml - remove redundant and refactor steps - add support to build image in mac aarch64 - allow customizing ELASTIC_VERSION and LOCAL_ARTIFACTS for test purpose
elastic · Nov 8, 2024 · 5826c6f · 5826c6f
1 parent d9ead9a
commit 5826c6f
Show file tree

Hide file tree

Showing 13 changed files with 188 additions and 388 deletions.
diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh
@@ -15,17 +15,11 @@ case "$WORKFLOW_TYPE" in
             rake artifact:docker_oss || error "artifact:docker_oss build failed."
             rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
             rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
         else
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
             # Qualifier is passed from CI as optional field and specify the version postfix
             # in case of alpha or beta releases:
             # e.g: 8.0.0-alpha1
@@ -41,17 +35,11 @@ case "$WORKFLOW_TYPE" in
             RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
             RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
             RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
         else
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
             VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
             # Qualifier is passed from CI as optional field and specify the version postfix
             # in case of alpha or beta releases:
             # e.g: 8.0.0-alpha1
@@ -73,18 +61,14 @@ for file in build/logstash-*; do shasum $file;done
 info "Uploading DRA artifacts in buildkite's artifact store ..."
 # Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
 images="logstash logstash-oss logstash-wolfi"
-if [ "$ARCH" != "aarch64" ]; then
-    # No logstash-ubi8 for AARCH64
-    images="logstash logstash-oss logstash-wolfi logstash-ubi8"
-fi
 for image in ${images}; do
     buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
 done
 
 # Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
 # overwritten when building aarch64 (or viceversa).
 if [ "$ARCH" != "aarch64" ]; then
-    for image in logstash logstash-oss logstash-wolfi logstash-ubi8 logstash-ironbank; do
+    for image in logstash logstash-oss logstash-wolfi logstash-ironbank; do
         buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
     done
 fi

diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh
@@ -11,10 +11,6 @@ function save_docker_tarballs {
     local arch="${1:?architecture required}"
     local version="${2:?stack-version required}"
     local images="logstash logstash-oss logstash-wolfi"
-    if [ "${arch}" != "aarch64" ]; then
-        # No logstash-ubi8 for AARCH64
-        images="logstash logstash-oss logstash-wolfi logstash-ubi8"
-    fi
 
     for image in ${images}; do
         tar_file="${image}-${version}-docker-image-${arch}.tar"

diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh
@@ -46,13 +46,6 @@ if [ "$RELEASE_VER" != "7.17" ]; then
   :
 fi
 
-# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected
-# by the release manager.
-# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle
-# for more details.
-# TODO filter it out when uploading artifacts instead
-rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
-
 info "Downloaded ARTIFACTS sha report"
 for file in build/logstash-*; do shasum $file;done
 

diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh
@@ -15,7 +15,6 @@ fi
 # Can run either a specific flavor, or all flavors -
 # eg `ci/acceptance_tests.sh oss` will run tests for open source container
 #    `ci/acceptance_tests.sh full` will run tests for the default container
-#    `ci/acceptance_tests.sh ubi8` will run tests for the ubi8 based container
 #    `ci/acceptance_tests.sh wolfi` will run tests for the wolfi based container
 #    `ci/acceptance_tests.sh` will run tests for all containers
 SELECTED_TEST_SUITE=$1
@@ -56,16 +55,6 @@ elif [[ $SELECTED_TEST_SUITE == "full" ]]; then
 
   echo "--- Acceptance: Running the tests"
   bundle exec rspec docker/spec/full/*_spec.rb
-elif [[ $SELECTED_TEST_SUITE == "ubi8" ]]; then
-  echo "--- Building $SELECTED_TEST_SUITE docker images"
-  cd $LS_HOME
-  rake artifact:docker_ubi8
-  echo "--- Acceptance: Installing dependencies"
-  cd $QA_DIR
-  bundle install
-
-  echo "--- Acceptance: Running the tests"
-  bundle exec rspec docker/spec/ubi8/*_spec.rb
 elif [[ $SELECTED_TEST_SUITE == "wolfi" ]]; then
   echo "--- Building $SELECTED_TEST_SUITE docker images"
   cd $LS_HOME

diff --git a/docker/Makefile b/docker/Makefile
@@ -2,7 +2,7 @@ SHELL=/bin/bash
 ELASTIC_REGISTRY ?= docker.elastic.co
 
 # Determine the version to build.
-ELASTIC_VERSION := $(shell ../vendor/jruby/bin/jruby bin/elastic-version)
+ELASTIC_VERSION ?= $(shell ../vendor/jruby/bin/jruby bin/elastic-version)
 
 ifdef STAGING_BUILD_NUM
   VERSION_TAG := $(ELASTIC_VERSION)-$(STAGING_BUILD_NUM)
@@ -14,9 +14,13 @@ ifdef DOCKER_ARCHITECTURE
   ARCHITECTURE := $(DOCKER_ARCHITECTURE)
 else
   ARCHITECTURE := $(shell uname -m)
+  # For MacOS
+  ifeq ($(ARCHITECTURE), arm64)
+	ARCHITECTURE := aarch64
+  endif
 endif
 
-IMAGE_FLAVORS ?= oss full ubi8 wolfi
+IMAGE_FLAVORS ?= oss full wolfi
 DEFAULT_IMAGE_FLAVOR ?= full
 
 IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash
@@ -26,7 +30,7 @@ all: build-from-local-artifacts build-from-local-oss-artifacts public-dockerfile
 
 # Build from artifacts on the local filesystem, using an http server (running
 # in a container) to provide the artifacts to the Dockerfile.
-build-from-local-full-artifacts: dockerfile env2yaml
+build-from-local-full-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -36,7 +40,7 @@ build-from-local-full-artifacts: dockerfile env2yaml
 	docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG);
 	docker kill $(HTTPD)
 
-build-from-local-oss-artifacts: dockerfile env2yaml
+build-from-local-oss-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
@@ -45,15 +49,6 @@ build-from-local-oss-artifacts: dockerfile env2yaml
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
-build-from-local-ubi8-artifacts: dockerfile env2yaml
-	docker run --rm -d --name=$(HTTPD) \
-	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
-	           python:3 bash -c 'cd /mnt && python3 -m http.server'
-	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \
-	  (docker kill $(HTTPD); false);
-	-docker kill $(HTTPD)
-
 build-from-local-wolfi-artifacts: dockerfile
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
@@ -66,8 +61,6 @@ build-from-local-wolfi-artifacts: dockerfile
 COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
 COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint
-COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64 
-COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64
 
 $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml
 $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml: data/logstash/config/logstash-oss.yml
@@ -76,8 +69,6 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4
 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
-$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64: data/logstash/env2yaml/env2yaml-arm64
-$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64: data/logstash/env2yaml/env2yaml-amd64
 
 $(ARTIFACTS_DIR)/docker/%:
 	cp -f $< $@
@@ -86,7 +77,6 @@ docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/docker/
 	mkdir -p $(ARTIFACTS_DIR)/docker/bin
 	mkdir -p $(ARTIFACTS_DIR)/docker/config
-	mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/docker/pipeline
 
 COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml
@@ -118,7 +108,7 @@ ironbank_docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline
 
-public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_wolfi public-dockerfiles_ironbank
+public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_ironbank
 
 public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -132,7 +122,7 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
-	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
+	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
 
 public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -146,21 +136,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
-	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
-
-public-dockerfiles_ubi8: templates/Dockerfile.erb docker_paths $(COPY_FILES)
-	../vendor/jruby/bin/jruby -S erb -T "-"\
-		created_date="${BUILD_DATE}" \
-		elastic_version="${ELASTIC_VERSION}" \
-		arch="${ARCHITECTURE}" \
-		version_tag="${VERSION_TAG}" \
-	  release="${RELEASE}" \
-		image_flavor="ubi8" \
-		local_artifacts="false" \
-		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ubi8" && \
-	cd $(ARTIFACTS_DIR)/docker && \
-	cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \
-	tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
+	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
 
 public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -174,9 +150,9 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
-	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
+	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
 
-public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Dockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
+public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 	  elastic_version="${ELASTIC_VERSION}" \
 	  templates/hardening_manifest.yaml.erb > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \
@@ -188,35 +164,11 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Doc
 	  release="${RELEASE}" \
 		image_flavor="ironbank" \
 		local_artifacts="false" \
-		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
+		templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
 	cd $(ARTIFACTS_DIR)/ironbank && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \
 	tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md
 
-# Push the image to the dedicated push endpoint at "push.docker.elastic.co"
-push:
-	$(foreach FLAVOR, $(IMAGE_FLAVORS), \
-	  docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
-	  docker push push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
-	  docker rmi push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
-	)
-	# Also push the default version, with no suffix like '-oss' or '-full'
-	docker tag $(IMAGE_TAG):$(VERSION_TAG) push.$(IMAGE_TAG):$(VERSION_TAG);
-	docker push push.$(IMAGE_TAG):$(VERSION_TAG);
-	docker rmi push.$(IMAGE_TAG):$(VERSION_TAG);
-
-# Compile "env2yaml", the helper for configuring logstash.yml via environment
-# variables.
-env2yaml:
-	docker run --rm \
-	  -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-		-e GOARCH=arm64 -e GOOS=linux \
-		-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-arm64
-	docker run --rm \
-	  -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-		-e GOARCH=amd64 -e GOOS=linux \
-		-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-amd64
-
 # Generate the Dockerfiles from ERB templates.
 dockerfile: templates/Dockerfile.erb
 	$(foreach FLAVOR, $(IMAGE_FLAVORS), \
@@ -226,7 +178,7 @@ dockerfile: templates/Dockerfile.erb
 			arch="${ARCHITECTURE}" \
 			version_tag="${VERSION_TAG}" \
 			image_flavor="${FLAVOR}" \
-			local_artifacts="true" \
+			local_artifacts="${LOCAL_ARTIFACTS}" \
 			templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-${FLAVOR}" ; \
 	)