diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 516e991d668..9310e3e9ea9 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -15,17 +15,11 @@ case "$WORKFLOW_TYPE" in rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi else VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi # Qualifier is passed from CI as optional field and specify the version postfix # in case of alpha or beta releases: # e.g: 8.0.0-alpha1 @@ -41,17 +35,11 @@ case "$WORKFLOW_TYPE" in RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi else VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi # Qualifier is passed from CI as optional field and specify the version postfix # in case of alpha or beta releases: # e.g: 8.0.0-alpha1 @@ -73,10 +61,6 @@ for file in build/logstash-*; do shasum $file;done info "Uploading DRA artifacts in buildkite's artifact store ..." # Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh images="logstash logstash-oss logstash-wolfi" -if [ "$ARCH" != "aarch64" ]; then - # No logstash-ubi8 for AARCH64 - images="logstash logstash-oss logstash-wolfi logstash-ubi8" -fi for image in ${images}; do buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz" done @@ -84,7 +68,7 @@ done # Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be # overwritten when building aarch64 (or viceversa). if [ "$ARCH" != "aarch64" ]; then - for image in logstash logstash-oss logstash-wolfi logstash-ubi8 logstash-ironbank; do + for image in logstash logstash-oss logstash-wolfi logstash-ironbank; do buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz" done fi diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh index 6618c6314ec..e58cef7f2dc 100755 --- a/.buildkite/scripts/dra/common.sh +++ b/.buildkite/scripts/dra/common.sh @@ -11,10 +11,6 @@ function save_docker_tarballs { local arch="${1:?architecture required}" local version="${2:?stack-version required}" local images="logstash logstash-oss logstash-wolfi" - if [ "${arch}" != "aarch64" ]; then - # No logstash-ubi8 for AARCH64 - images="logstash logstash-oss logstash-wolfi logstash-ubi8" - fi for image in ${images}; do tar_file="${image}-${version}-docker-image-${arch}.tar" diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh index 67c6ce895d9..21c7e457026 100755 --- a/.buildkite/scripts/dra/publish.sh +++ b/.buildkite/scripts/dra/publish.sh @@ -46,13 +46,6 @@ if [ "$RELEASE_VER" != "7.17" ]; then : fi -# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected -# by the release manager. -# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle -# for more details. -# TODO filter it out when uploading artifacts instead -rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz - info "Downloaded ARTIFACTS sha report" for file in build/logstash-*; do shasum $file;done diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index 3f4d2de1156..86a721a1596 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -15,7 +15,6 @@ fi # Can run either a specific flavor, or all flavors - # eg `ci/acceptance_tests.sh oss` will run tests for open source container # `ci/acceptance_tests.sh full` will run tests for the default container -# `ci/acceptance_tests.sh ubi8` will run tests for the ubi8 based container # `ci/acceptance_tests.sh wolfi` will run tests for the wolfi based container # `ci/acceptance_tests.sh` will run tests for all containers SELECTED_TEST_SUITE=$1 @@ -56,16 +55,6 @@ elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Acceptance: Running the tests" bundle exec rspec docker/spec/full/*_spec.rb -elif [[ $SELECTED_TEST_SUITE == "ubi8" ]]; then - echo "--- Building $SELECTED_TEST_SUITE docker images" - cd $LS_HOME - rake artifact:docker_ubi8 - echo "--- Acceptance: Installing dependencies" - cd $QA_DIR - bundle install - - echo "--- Acceptance: Running the tests" - bundle exec rspec docker/spec/ubi8/*_spec.rb elif [[ $SELECTED_TEST_SUITE == "wolfi" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME diff --git a/docker/Makefile b/docker/Makefile index 68d8e1fb9d4..8a8fc4b708d 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,7 +2,7 @@ SHELL=/bin/bash ELASTIC_REGISTRY ?= docker.elastic.co # Determine the version to build. -ELASTIC_VERSION := $(shell ../vendor/jruby/bin/jruby bin/elastic-version) +ELASTIC_VERSION ?= $(shell ../vendor/jruby/bin/jruby bin/elastic-version) ifdef STAGING_BUILD_NUM VERSION_TAG := $(ELASTIC_VERSION)-$(STAGING_BUILD_NUM) @@ -14,9 +14,13 @@ ifdef DOCKER_ARCHITECTURE ARCHITECTURE := $(DOCKER_ARCHITECTURE) else ARCHITECTURE := $(shell uname -m) + # For MacOS + ifeq ($(ARCHITECTURE), arm64) + ARCHITECTURE := aarch64 + endif endif -IMAGE_FLAVORS ?= oss full ubi8 wolfi +IMAGE_FLAVORS ?= oss full wolfi DEFAULT_IMAGE_FLAVOR ?= full IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash @@ -26,7 +30,7 @@ all: build-from-local-artifacts build-from-local-oss-artifacts public-dockerfile # Build from artifacts on the local filesystem, using an http server (running # in a container) to provide the artifacts to the Dockerfile. -build-from-local-full-artifacts: dockerfile env2yaml +build-from-local-full-artifacts: dockerfile docker run --rm -d --name=$(HTTPD) \ -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \ python:3 bash -c 'cd /mnt && python3 -m http.server' @@ -36,7 +40,7 @@ build-from-local-full-artifacts: dockerfile env2yaml docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG); docker kill $(HTTPD) -build-from-local-oss-artifacts: dockerfile env2yaml +build-from-local-oss-artifacts: dockerfile docker run --rm -d --name=$(HTTPD) \ -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \ python:3 bash -c 'cd /mnt && python3 -m http.server' @@ -45,15 +49,6 @@ build-from-local-oss-artifacts: dockerfile env2yaml (docker kill $(HTTPD); false); -docker kill $(HTTPD) -build-from-local-ubi8-artifacts: dockerfile env2yaml - docker run --rm -d --name=$(HTTPD) \ - -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \ - python:3 bash -c 'cd /mnt && python3 -m http.server' - timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done' - docker build --progress=plain --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \ - (docker kill $(HTTPD); false); - -docker kill $(HTTPD) - build-from-local-wolfi-artifacts: dockerfile docker run --rm -d --name=$(HTTPD) \ -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \ @@ -66,8 +61,6 @@ build-from-local-wolfi-artifacts: dockerfile COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint -COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64 -COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64 $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml: data/logstash/config/logstash-oss.yml @@ -76,8 +69,6 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint -$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64: data/logstash/env2yaml/env2yaml-arm64 -$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64: data/logstash/env2yaml/env2yaml-amd64 $(ARTIFACTS_DIR)/docker/%: cp -f $< $@ @@ -86,7 +77,6 @@ docker_paths: mkdir -p $(ARTIFACTS_DIR)/docker/ mkdir -p $(ARTIFACTS_DIR)/docker/bin mkdir -p $(ARTIFACTS_DIR)/docker/config - mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml mkdir -p $(ARTIFACTS_DIR)/docker/pipeline COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml @@ -118,7 +108,7 @@ ironbank_docker_paths: mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline -public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_wolfi public-dockerfiles_ironbank +public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_ironbank public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -132,7 +122,7 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ - tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline + tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -146,21 +136,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ - tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline - -public-dockerfiles_ubi8: templates/Dockerfile.erb docker_paths $(COPY_FILES) - ../vendor/jruby/bin/jruby -S erb -T "-"\ - created_date="${BUILD_DATE}" \ - elastic_version="${ELASTIC_VERSION}" \ - arch="${ARCHITECTURE}" \ - version_tag="${VERSION_TAG}" \ - release="${RELEASE}" \ - image_flavor="ubi8" \ - local_artifacts="false" \ - templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ubi8" && \ - cd $(ARTIFACTS_DIR)/docker && \ - cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \ - tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline + tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -174,9 +150,9 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ - tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline + tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline -public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Dockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) +public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ elastic_version="${ELASTIC_VERSION}" \ templates/hardening_manifest.yaml.erb > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \ @@ -188,35 +164,11 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Doc release="${RELEASE}" \ image_flavor="ironbank" \ local_artifacts="false" \ - templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ + templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md -# Push the image to the dedicated push endpoint at "push.docker.elastic.co" -push: - $(foreach FLAVOR, $(IMAGE_FLAVORS), \ - docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \ - docker push push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \ - docker rmi push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \ - ) - # Also push the default version, with no suffix like '-oss' or '-full' - docker tag $(IMAGE_TAG):$(VERSION_TAG) push.$(IMAGE_TAG):$(VERSION_TAG); - docker push push.$(IMAGE_TAG):$(VERSION_TAG); - docker rmi push.$(IMAGE_TAG):$(VERSION_TAG); - -# Compile "env2yaml", the helper for configuring logstash.yml via environment -# variables. -env2yaml: - docker run --rm \ - -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \ - -e GOARCH=arm64 -e GOOS=linux \ - -w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-arm64 - docker run --rm \ - -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \ - -e GOARCH=amd64 -e GOOS=linux \ - -w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-amd64 - # Generate the Dockerfiles from ERB templates. dockerfile: templates/Dockerfile.erb $(foreach FLAVOR, $(IMAGE_FLAVORS), \ @@ -226,7 +178,7 @@ dockerfile: templates/Dockerfile.erb arch="${ARCHITECTURE}" \ version_tag="${VERSION_TAG}" \ image_flavor="${FLAVOR}" \ - local_artifacts="true" \ + local_artifacts="${LOCAL_ARTIFACTS}" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-${FLAVOR}" ; \ ) diff --git a/docker/templates/Dockerfile.erb b/docker/templates/Dockerfile.erb index cf3eecf7559..85296730e1a 100644 --- a/docker/templates/Dockerfile.erb +++ b/docker/templates/Dockerfile.erb @@ -1,6 +1,30 @@ # This Dockerfile was generated from templates/Dockerfile.erb -<% if image_flavor == 'wolfi' -%> -FROM docker.elastic.co/wolfi/go:1-dev as builder-env2yaml +<%# image_flavor 'full', oss', 'wolfi' -%> +<% if local_artifacts == 'false' -%> + <% url_root = 'https://artifacts.elastic.co/downloads/logstash' -%> +<% else -%> + <% url_root = 'http://localhost:8000' -%> +<% end -%> +<% if image_flavor == 'oss' -%> + <% tarball = "logstash-oss-#{elastic_version}-linux-#{arch}.tar.gz" -%> + <% license = 'Apache 2.0' -%> +<% else -%> + <% tarball = "logstash-#{elastic_version}-linux-#{arch}.tar.gz" -%> + <% license = 'Elastic License' -%> +<% end -%> +<% if image_flavor == 'full' || image_flavor == 'oss' -%> + <% base_image = 'docker.elastic.co/ubi8/ubi-minimal' -%> + <% go_image = 'golang:1.23' -%> + <% package_manager = 'microdnf' -%> +<% else -%> + <% base_image = 'docker.elastic.co/wolfi/chainguard-base' -%> + <% go_image = 'docker.elastic.co/wolfi/go:1.23' -%> + <% package_manager = 'apk' -%> +<% end -%> +<% locale = 'C.UTF-8' -%> + +# Build env2yaml +FROM <%= go_image %> as builder-env2yaml COPY env2yaml/env2yaml.go /tmp/go/src/env2yaml/env2yaml.go COPY env2yaml/go.mod /tmp/go/src/env2yaml/go.mod @@ -9,139 +33,55 @@ COPY env2yaml/go.sum /tmp/go/src/env2yaml/go.sum WORKDIR /tmp/go/src/env2yaml RUN go build -<% end -%> - -<% if image_flavor == 'ironbank' -%> -<%# Start image_flavor 'ironbank' %> -ARG BASE_REGISTRY=registry1.dso.mil -ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 -ARG BASE_TAG=9.3 -ARG LOGSTASH_VERSION=<%= elastic_version %> -ARG GOLANG_VERSION=1.21.8 - -# stage 1: build env2yaml -FROM ${BASE_REGISTRY}/google/golang/ubi9/golang-1.21:${GOLANG_VERSION} AS env2yaml -ENV GOPATH=/go - -COPY scripts/go /go - -USER root - -RUN dnf-3 -y upgrade && dnf-3 install -y git && \ - cd /go/src/env2yaml && \ - go build - -# Final stage -FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} - -ARG LOGSTASH_VERSION +# Build main image +# Minimal distributions do not ship with en language packs. +FROM <%= base_image %> ENV ELASTIC_CONTAINER true ENV PATH=/usr/share/logstash/bin:$PATH +ENV LANG=<%= locale %> LC_ALL=<%= locale %> WORKDIR /usr/share -COPY --from=env2yaml /go/src/env2yaml/env2yaml /usr/local/bin/env2yaml -COPY scripts/config/* config/ -COPY scripts/pipeline/default.conf pipeline/logstash.conf -COPY scripts/bin/docker-entrypoint /usr/local/bin/ -COPY logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz /tmp/logstash.tar.gz - -RUN dnf -y upgrade && \ - dnf install -y procps findutils tar gzip which shadow-utils && \ - dnf clean all && \ - groupadd --gid 1000 logstash && \ - adduser --uid 1000 --gid 1000 --home-dir /usr/share/logstash --no-create-home logstash && \ - tar -zxf /tmp/logstash.tar.gz -C /usr/share/ && \ - mv /usr/share/logstash-${LOGSTASH_VERSION} /usr/share/logstash && \ - chown -R 1000:0 /usr/share/logstash && \ - chown --recursive logstash:logstash /usr/share/logstash/ && \ - chown -R logstash:root /usr/share/logstash config/ pipeline/ && \ - chmod -R g=u /usr/share/logstash && \ - mv config/* /usr/share/logstash/config && \ - mv pipeline /usr/share/logstash/pipeline && \ - mkdir /licenses/ && \ - mv /usr/share/logstash/NOTICE.TXT /licenses/NOTICE.TXT && \ - mv /usr/share/logstash/LICENSE.txt /licenses/LICENSE.txt && \ - ln -s /usr/share/logstash /opt/logstash && \ - chmod 0755 /usr/local/bin/docker-entrypoint && \ - rmdir config && \ - rm /tmp/logstash.tar.gz -<%# End image_flavor 'ironbank' %> -<% else -%> -<%# Start image_flavor 'full', oss', 'ubi8', 'wolfi' %> - <% if local_artifacts == 'false' -%> - <% url_root = 'https://artifacts.elastic.co/downloads/logstash' -%> - <% else -%> - <% url_root = 'http://localhost:8000' -%> - <% end -%> - <% if image_flavor == 'oss' -%> - <% tarball = "logstash-oss-#{elastic_version}-linux-$(arch).tar.gz" -%> - <% license = 'Apache 2.0' -%> - <% else -%> - <% tarball = "logstash-#{elastic_version}-linux-$(arch).tar.gz" -%> - <% license = 'Elastic License' -%> - <% end -%> - <% if image_flavor == 'ubi8' %> - <% base_image = 'docker.elastic.co/ubi8/ubi-minimal' -%> - <% package_manager = 'microdnf' -%> - <% arch_command = 'uname -m' -%> - # Minimal distributions do not ship with en language packs. - <% locale = 'C.UTF-8' -%> - <% elsif image_flavor == 'wolfi' %> - <% base_image = 'docker.elastic.co/wolfi/chainguard-base' -%> - <% package_manager = 'apk' -%> - <% arch_command = 'uname -m' -%> - # Minimal distributions do not ship with en language packs. - <% locale = 'C.UTF-8' -%> - <% else -%> - <% base_image = 'ubuntu:20.04' -%> - <% package_manager = 'apt-get' -%> - <% locale = 'en_US.UTF-8' -%> - <% arch_command = 'dpkg --print-architecture' -%> - <% end -%> - -FROM <%= base_image %> +COPY --from=builder-env2yaml /tmp/go/src/env2yaml/env2yaml /usr/local/bin/env2yaml +COPY config/pipelines.yml config/log4j2.properties config/log4j2.file.properties config/ +<% if image_flavor == 'oss' -%> +COPY config/logstash-oss.yml config/logstash.yml +<% else -%><%# 'full', 'wolfi' -%> +COPY config/logstash-full.yml config/logstash.yml +<% end -%> +COPY pipeline/default.conf pipeline/logstash.conf +COPY bin/docker-entrypoint /usr/local/bin/ +# Install packages RUN for iter in {1..10}; do \ -<% if image_flavor == 'wolfi' %> - <%= package_manager %> add --no-cache curl bash && \ -<% else -%> - <% if image_flavor == 'full' || image_flavor == 'oss' -%> - export DEBIAN_FRONTEND=noninteractive && \ - <% end -%> +<% if image_flavor == 'full' || image_flavor == 'oss' -%> <%= package_manager %> update -y && \ <%= package_manager %> upgrade -y && \ <%= package_manager %> install -y procps findutils tar gzip && \ - <% if image_flavor == 'ubi8' -%> - <%= package_manager %> install -y openssl && \ - <% end -%> - <% if image_flavor == 'ubi8' -%> - <%= package_manager %> install -y which shadow-utils && \ - <% else -%> - <%= package_manager %> install -y locales && \ - <% end -%> - <% if image_flavor != 'ubi9' -%> - <%= package_manager %> install -y curl && \ - <% end -%> + <%= package_manager %> install -y openssl && \ + <%= package_manager %> install -y which shadow-utils && \ + <%= package_manager %> install -y curl && \ <%= package_manager %> clean all && \ - <% if image_flavor == 'full' || image_flavor == 'oss' -%> - locale-gen 'en_US.UTF-8' && \ - <%= package_manager %> clean metadata && \ - <% end -%> +<% else -%><%# 'wolfi' -%> + <%= package_manager %> add --no-cache curl bash && \ <% end -%> -exit_code=0 && break || exit_code=$? && \ -echo "packaging error: retry $iter in 10s" && \ -<%= package_manager %> clean all && \ + exit_code=0 && break || \ + exit_code=$? && echo "packaging error: retry $iter in 10s" && \ + <%= package_manager %> clean all && sleep 10; \ + done; \ + (exit $exit_code) + +# Provide a non-root user to run the process +# Add Logstash itself and set permissions <% if image_flavor == 'full' || image_flavor == 'oss' -%> - <%= package_manager %> clean metadata && \ -<% end -%> -sleep 10; done; \ -(exit $exit_code) - -# Provide a non-root user to run the process. -<% if image_flavor == 'wolfi' -%> +RUN groupadd --gid 1000 logstash && \ + adduser --uid 1000 --gid 1000 \ + --home "/usr/share/logstash" \ + --no-create-home \ + logstash && \ +<% else -%><%# 'wolfi' -%> RUN addgroup -g 1000 logstash && \ adduser -u 1000 -G logstash \ --disabled-password \ @@ -149,95 +89,48 @@ RUN addgroup -g 1000 logstash && \ --home "/usr/share/logstash" \ --shell "/sbin/nologin" \ --no-create-home \ - logstash -<% else -%> -RUN groupadd --gid 1000 logstash && \ - adduser --uid 1000 --gid 1000 --home /usr/share/logstash --no-create-home logstash + logstash && \ <% end -%> - -# Add Logstash itself. -RUN curl -Lo - <%= url_root %>/<%= tarball %> | \ + curl -Lo - <%= url_root %>/<%= tarball %> | \ tar zxf - -C /usr/share && \ mv /usr/share/logstash-<%= elastic_version %> /usr/share/logstash && \ - chown --recursive logstash:logstash /usr/share/logstash/ && \ - chown -R logstash:root /usr/share/logstash && \ + chown -R logstash:root /usr/share/logstash config/ pipeline/ && \ chmod -R g=u /usr/share/logstash && \ - mkdir /licenses/ && \ + mv config/* /usr/share/logstash/config && \ + mv pipeline /usr/share/logstash/pipeline && \ + mkdir /licenses && \ mv /usr/share/logstash/NOTICE.TXT /licenses/NOTICE.TXT && \ mv /usr/share/logstash/LICENSE.txt /licenses/LICENSE.txt && \ find /usr/share/logstash -type d -exec chmod g+s {} \; && \ - ln -s /usr/share/logstash /opt/logstash + ln -s /usr/share/logstash /opt/logstash && \ + chmod 0755 /usr/local/bin/docker-entrypoint && \ + rmdir config WORKDIR /usr/share/logstash -ENV ELASTIC_CONTAINER true -ENV PATH=/usr/share/logstash/bin:$PATH - -# Provide a minimal configuration, so that simple invocations will provide -# a good experience. -<% if image_flavor == 'oss' -%> - COPY config/logstash-oss.yml config/logstash.yml -<% else -%> - COPY config/logstash-full.yml config/logstash.yml -<% end -%> -COPY config/pipelines.yml config/log4j2.properties config/log4j2.file.properties config/ -COPY pipeline/default.conf pipeline/logstash.conf - -RUN chown --recursive logstash:root config/ pipeline/ -# Ensure Logstash gets the correct locale by default. -ENV LANG=<%= locale %> LC_ALL=<%= locale %> - -<% if image_flavor == 'wolfi' -%> -COPY --from=builder-env2yaml /tmp/go/src/env2yaml/env2yaml /usr/local/bin/env2yaml -<% else -%> -COPY env2yaml/env2yaml-amd64 env2yaml/env2yaml-arm64 env2yaml/ -# Copy over the appropriate env2yaml artifact -RUN env2yamlarch="$(<%= arch_command %>)"; \ - case "${env2yamlarch}" in \ - 'x86_64'|'amd64') \ - env2yamlarch=amd64; \ - ;; \ - 'aarch64'|'arm64') \ - env2yamlarch=arm64; \ - ;; \ - *) echo >&2 "error: unsupported architecture '$env2yamlarch'"; exit 1 ;; \ - esac; \ - mkdir -p /usr/local/bin; \ - cp env2yaml/env2yaml-${env2yamlarch} /usr/local/bin/env2yaml; \ - rm -rf env2yaml -<% end -%> -# Place the startup wrapper script. -COPY bin/docker-entrypoint /usr/local/bin/ - -RUN chmod 0755 /usr/local/bin/docker-entrypoint -<%# End image_flavor 'full', oss', 'ubi8', 'wolfi' %> -<% end -%> USER 1000 EXPOSE 9600 5044 -<% if image_flavor != 'ironbank' -%> -LABEL org.label-schema.schema-version="1.0" \ - org.label-schema.vendor="Elastic" \ - org.opencontainers.image.vendor="Elastic" \ +LABEL org.label-schema.build-date=<%= created_date %> \ + org.label-schema.license="<%= license %>" \ org.label-schema.name="logstash" \ - org.opencontainers.image.title="logstash" \ - org.label-schema.version="<%= elastic_version %>" \ - org.opencontainers.image.version="<%= elastic_version %>" \ + org.label-schema.schema-version="1.0" \ org.label-schema.url="https://www.elastic.co/products/logstash" \ org.label-schema.vcs-url="https://github.com/elastic/logstash" \ - org.label-schema.license="<%= license %>" \ - org.opencontainers.image.licenses="<%= license %>" \ + org.label-schema.vendor="Elastic" \ + org.label-schema.version="<%= elastic_version %>" \ + org.opencontainers.image.created=<%= created_date %> \ org.opencontainers.image.description="Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" \ - org.label-schema.build-date=<%= created_date %> \ -<% if image_flavor == 'ubi8' -%> license="<%= license %>" \ + org.opencontainers.image.licenses="<%= license %>" \ + org.opencontainers.image.title="logstash" \ + org.opencontainers.image.vendor="Elastic" \ + org.opencontainers.image.version="<%= elastic_version %>" \ description="Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" \ - name="logstash" \ + license="<%= license %>" \ maintainer="info@elastic.co" \ + name="logstash" \ summary="Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" \ - vendor="Elastic" \ -<% end -%> - org.opencontainers.image.created=<%= created_date %> -<% end -%> + vendor="Elastic" ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/docker/templates/IronbankDockerfile.erb b/docker/templates/IronbankDockerfile.erb new file mode 100644 index 00000000000..db810f8307e --- /dev/null +++ b/docker/templates/IronbankDockerfile.erb @@ -0,0 +1,65 @@ +# This Dockerfile was generated from templates/IronbankDockerfile.erb + +ARG BASE_REGISTRY=registry1.dso.mil +ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 +ARG BASE_TAG=9.3 +ARG LOGSTASH_VERSION=<%= elastic_version %> +ARG GOLANG_VERSION=1.21.8 + +# stage 1: build env2yaml +FROM ${BASE_REGISTRY}/google/golang/ubi9/golang-1.21:${GOLANG_VERSION} AS env2yaml + +ENV GOPATH=/go + +COPY scripts/go /go + +USER root + +RUN dnf-3 -y upgrade && dnf-3 install -y git && \ + cd /go/src/env2yaml && \ + go build + +# Final stage +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +ARG LOGSTASH_VERSION + +ENV ELASTIC_CONTAINER true +ENV PATH=/usr/share/logstash/bin:$PATH + +WORKDIR /usr/share + +COPY --from=env2yaml /go/src/env2yaml/env2yaml /usr/local/bin/env2yaml +COPY scripts/config/* config/ +COPY scripts/pipeline/default.conf pipeline/logstash.conf +COPY scripts/bin/docker-entrypoint /usr/local/bin/ +COPY logstash-${LOGSTASH_VERSION}-linux-x86_64.tar.gz /tmp/logstash.tar.gz + +RUN dnf -y upgrade && \ + dnf install -y procps findutils tar gzip which shadow-utils && \ + dnf clean all && \ + groupadd --gid 1000 logstash && \ + adduser --uid 1000 --gid 1000 --home-dir /usr/share/logstash --no-create-home logstash && \ + tar -zxf /tmp/logstash.tar.gz -C /usr/share/ && \ + mv /usr/share/logstash-${LOGSTASH_VERSION} /usr/share/logstash && \ + chown -R 1000:0 /usr/share/logstash && \ + chown --recursive logstash:logstash /usr/share/logstash/ && \ + chown -R logstash:root /usr/share/logstash config/ pipeline/ && \ + chmod -R g=u /usr/share/logstash && \ + mv config/* /usr/share/logstash/config && \ + mv pipeline /usr/share/logstash/pipeline && \ + mkdir /licenses/ && \ + mv /usr/share/logstash/NOTICE.TXT /licenses/NOTICE.TXT && \ + mv /usr/share/logstash/LICENSE.txt /licenses/LICENSE.txt && \ + ln -s /usr/share/logstash /opt/logstash && \ + chmod 0755 /usr/local/bin/docker-entrypoint && \ + rmdir config && \ + rm /tmp/logstash.tar.gz + +WORKDIR /usr/share/logstash + +USER 1000 + +EXPOSE 9600 5044 + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/qa/docker/shared_examples/image_metadata.rb b/qa/docker/shared_examples/image_metadata.rb index 77a995258b9..1750cd0ee51 100644 --- a/qa/docker/shared_examples/image_metadata.rb +++ b/qa/docker/shared_examples/image_metadata.rb @@ -13,24 +13,28 @@ expect(@image.json['Architecture']).to have_correct_architecture end - %w(org.label-schema.license org.opencontainers.image.licenses).each do |label| + %w(license org.label-schema.license org.opencontainers.image.licenses).each do |label| it "should set the license label #{label} correctly" do expect(@labels[label]).to have_correct_license_label(flavor) end end - %w(org.label-schema.name org.opencontainers.image.title).each do |label| + %w(name org.label-schema.name org.opencontainers.image.title).each do |label| it "should set the name label #{label} correctly" do expect(@labels[label]).to eql "logstash" end end - it "should set the vendor label org.opencontainers.image.vendor correctly" do - expect(@labels['org.opencontainers.image.vendor']).to eql "Elastic" + %w(vendor org.opencontainers.image.vendor).each do |label| + it 'should set the vendor label correctly' do + expect(@labels["vendor"]).to eql "Elastic" + end end - it "should set the description label org.opencontainers.image.description correctly" do - expect(@labels['org.opencontainers.image.description']).to eql "Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" + %w(description summary org.opencontainers.image.description).each do |label| + it "should set the description label #{label} correctly" do + expect(@labels[label]).to eql "Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" + end end %w(org.label-schema.version org.opencontainers.image.version).each do |label| @@ -38,4 +42,9 @@ expect(@labels[label]).to eql qualified_version end end + + it 'should set the maintainer label correctly' do + expect(@labels["maintainer"]).to eql "info@elastic.co" + end + end diff --git a/qa/docker/spec/full/container_spec.rb b/qa/docker/spec/full/container_spec.rb index 81a004a39c3..d7f8ae64b09 100644 --- a/qa/docker/spec/full/container_spec.rb +++ b/qa/docker/spec/full/container_spec.rb @@ -20,8 +20,8 @@ cleanup_container(@container) end - it 'has an Ubuntu 20.04 base image' do - expect(exec_in_container(@container, 'cat /etc/os-release').chomp).to match /Ubuntu 20.04/ + it 'should be based on Red Hat Enterprise Linux' do + expect(exec_in_container(@container, 'cat /etc/redhat-release')).to match /Red Hat Enterprise Linux/ end end end diff --git a/qa/docker/spec/oss/container_spec.rb b/qa/docker/spec/oss/container_spec.rb index ceff09510f3..cc824fdb310 100644 --- a/qa/docker/spec/oss/container_spec.rb +++ b/qa/docker/spec/oss/container_spec.rb @@ -18,8 +18,8 @@ cleanup_container(@container) end - it 'has an Ubuntu 20.04 base image' do - expect(exec_in_container(@container, 'cat /etc/os-release').chomp).to match /Ubuntu 20.04/ + it 'should be based on Red Hat Enterprise Linux' do + expect(exec_in_container(@container, 'cat /etc/redhat-release')).to match /Red Hat Enterprise Linux/ end end end diff --git a/qa/docker/spec/ubi8/container_spec.rb b/qa/docker/spec/ubi8/container_spec.rb deleted file mode 100644 index 19e69db1199..00000000000 --- a/qa/docker/spec/ubi8/container_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -require_relative '../spec_helper' -require_relative '../../shared_examples/container_config' -require_relative '../../shared_examples/container_options' -require_relative '../../shared_examples/container' -require_relative '../../shared_examples/xpack' - -describe 'A container running the ubi8 image' do - it_behaves_like 'the container is configured correctly', 'ubi8' - it_behaves_like 'it runs with different configurations', 'ubi8' - it_behaves_like 'it applies settings correctly', 'ubi8' - it_behaves_like 'a container with xpack features', 'ubi8' - - context 'The running container' do - before do - @image = find_image('ubi8') - @container = start_container(@image, {}) - end - - after do - cleanup_container(@container) - end - - it 'should be based on Red Hat Enterprise Linux' do - expect(exec_in_container(@container, 'cat /etc/redhat-release')).to match /Red Hat Enterprise Linux/ - end - end -end diff --git a/qa/docker/spec/ubi8/image_spec.rb b/qa/docker/spec/ubi8/image_spec.rb deleted file mode 100644 index de47bc7daf0..00000000000 --- a/qa/docker/spec/ubi8/image_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -require_relative '../spec_helper' -require_relative '../../shared_examples/image_metadata' - -describe 'An image with the full distribution' do - it_behaves_like 'the metadata is set correctly', 'ubi8' - - context 'the ubi8 image should set its specific labels correctly' do - before do - @image = find_image('ubi8') - @image_config = @image.json['Config'] - @labels = @image_config['Labels'] - end - - %w(license org.label-schema.license org.opencontainers.image.licenses).each do |label| - it "should set the license label #{label} correctly" do - expect(@labels[label]).to have_correct_license_label('ubi8') - end - end - - it 'should set the name label correctly' do - expect(@labels['name']).to eql "logstash" - end - - it 'should set the maintainer label correctly' do - expect(@labels["maintainer"]).to eql "info@elastic.co" - end - - %w(description summary).each do |label| - it "should set the name label #{label} correctly" do - expect(@labels[label]).to eql "Logstash is a free and open server-side data processing pipeline that ingests data from a multitude of sources, transforms it, and then sends it to your favorite 'stash.'" - end - end - - it 'should set the vendor label correctly' do - expect(@labels["vendor"]).to eql "Elastic" - end - end -end diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 807618546ca..1241c3a843b 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -18,6 +18,7 @@ namespace "artifact" do SNAPSHOT_BUILD = ENV["RELEASE"] != "1" VERSION_QUALIFIER = ENV["VERSION_QUALIFIER"] + LOCAL_ARTIFACTS = ENV["LOCAL_ARTIFACTS"] || "true" if VERSION_QUALIFIER PACKAGE_SUFFIX = SNAPSHOT_BUILD ? "-#{VERSION_QUALIFIER}-SNAPSHOT" : "-#{VERSION_QUALIFIER}" else @@ -139,7 +140,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_ubi8", "build_docker_wolfi"] + task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"] desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -329,12 +330,6 @@ namespace "artifact" do build_docker('oss') end - desc "Build UBI8 docker image" - task "docker_ubi8" => %w(prepare generate_build_metadata archives_docker) do - puts("[docker_ubi8] Building UBI docker image") - build_docker('ubi8') - end - desc "Build wolfi docker image" task "docker_wolfi" => %w(prepare generate_build_metadata archives_docker) do puts("[docker_wolfi] Building Wolfi docker image") @@ -346,7 +341,6 @@ namespace "artifact" do puts("[dockerfiles] Building Dockerfiles") build_dockerfile('oss') build_dockerfile('full') - build_dockerfile('ubi8') build_dockerfile('wolfi') build_dockerfile('ironbank') end @@ -363,12 +357,6 @@ namespace "artifact" do build_dockerfile('full') end - desc "Generate Dockerfile for UBI8 images" - task "dockerfile_ubi8" => ["prepare", "generate_build_metadata"] do - puts("[dockerfiles] Building ubi8 Dockerfiles") - build_dockerfile('ubi8') - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") @@ -390,7 +378,6 @@ namespace "artifact" do unless ENV['SKIP_DOCKER'] == "1" Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:docker_ubi8"].invoke Rake::Task["artifact:docker_wolfi"].invoke Rake::Task["artifact:dockerfiles"].invoke Rake::Task["artifact:docker_oss"].invoke @@ -411,11 +398,6 @@ namespace "artifact" do Rake::Task["artifact:dockerfile_oss"].invoke end - task "build_docker_ubi8" => [:generate_build_metadata] do - Rake::Task["artifact:docker_ubi8"].invoke - Rake::Task["artifact:dockerfile_ubi8"].invoke - end - task "build_docker_wolfi" => [:generate_build_metadata] do Rake::Task["artifact:docker_wolfi"].invoke Rake::Task["artifact:dockerfile_wolfi"].invoke @@ -798,7 +780,8 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE + "BUILD_DATE" => BUILD_DATE, + "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") @@ -810,7 +793,8 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE + "BUILD_DATE" => BUILD_DATE, + "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS } Dir.chdir("docker") do |dir| safe_system(env, "make public-dockerfiles_#{flavor}")