Skip to content

Commit

Permalink
zed: build all images + run tests using them
Browse files Browse the repository at this point in the history
Signed-off-by: Mohammed Naser <[email protected]>
  • Loading branch information
mnaser committed Apr 7, 2024
1 parent ee1ff23 commit 16c0fd9
Show file tree
Hide file tree
Showing 84 changed files with 2,757 additions and 1,650 deletions.
188 changes: 27 additions & 161 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,161 +1,27 @@
FROM ubuntu:jammy-20240227 AS ubuntu
LABEL org.opencontainers.image.source=https://github.com/vexxhost/atmosphere

FROM ubuntu AS helm
ARG TARGETOS
ARG TARGETARCH
ARG HELM_VERSION=3.14.0
ADD https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz /helm.tar.gz
RUN tar -xzf /helm.tar.gz
RUN mv /${TARGETOS}-${TARGETARCH}/helm /usr/bin/helm

FROM ubuntu AS ubuntu-cloud-archive
ADD --chmod=644 https://git.launchpad.net/ubuntu/+source/ubuntu-keyring/plain/keyrings/ubuntu-cloud-keyring.gpg /etc/apt/trusted.gpg.d/ubuntu-cloud-keyring.gpg
ARG RELEASE
RUN <<EOF bash -xe
source /etc/os-release
if [ "\${VERSION_CODENAME}" = "jammy" ]; then \
if [ "${RELEASE}" = "yoga" ]; then \
# NOTE: Yoga shipped with 22.04, so no need to add an extra repository.
echo "" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "zed" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/${RELEASE} main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "2023.1" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/antelope main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "2023.2" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/bobcat main" > /etc/apt/sources.list.d/cloudarchive.list; \
elif [ "${RELEASE}" = "master" ]; then \
echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu \${VERSION_CODENAME}-updates/caracal main" > /etc/apt/sources.list.d/cloudarchive.list; \
else \
echo "${RELEASE} is not supported on \${VERSION_CODENAME}"; \
exit 1; \
fi; \
else
echo "Unsupported release"; \
exit 1; \
fi
EOF

FROM alpine/git AS requirements
ARG BRANCH
ADD https://opendev.org/openstack/requirements.git#${BRANCH} /src
RUN <<EOF sh -xe
sed -i 's/cryptography===36.0.2/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/cryptography===40.0.2/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/cryptography===41.0.7/cryptography===42.0.4/' /src/upper-constraints.txt
sed -i 's/Django===3.2.18/Django===3.2.24/' /src/upper-constraints.txt
sed -i 's/Flask===2.2.3/Flask===2.2.5/' /src/upper-constraints.txt
sed -i 's/Jinja2===3.1.2/Jinja2===3.1.3/' /src/upper-constraints.txt
sed -i 's/oauthlib===3.2.0/oauthlib===3.2.2/' /src/upper-constraints.txt
sed -i 's/paramiko===2.11.0/paramiko===3.4.0/' /src/upper-constraints.txt
sed -i 's/paramiko===3.1.0/paramiko===3.4.0/' /src/upper-constraints.txt
sed -i 's/protobuf===4.21.5/protobuf===4.21.6/' /src/upper-constraints.txt
sed -i 's/pyOpenSSL===22.0.0/pyOpenSSL===24.0.0/' /src/upper-constraints.txt
sed -i 's/pyOpenSSL===23.1.1/pyOpenSSL===24.0.0/' /src/upper-constraints.txt
sed -i 's/requests===2.28.1/requests===2.31.0/' /src/upper-constraints.txt
sed -i 's/requests===2.28.2/requests===2.31.0/' /src/upper-constraints.txt
sed -i 's/sqlparse===0.4.2/sqlparse===0.4.4/' /src/upper-constraints.txt
sed -i 's/urllib3===1.26.12/urllib3===1.26.18/' /src/upper-constraints.txt
sed -i 's/urllib3===1.26.15/urllib3===1.26.18/' /src/upper-constraints.txt
sed -i 's/Werkzeug===2.2.2/Werkzeug===2.3.8/' /src/upper-constraints.txt
sed -i 's/Werkzeug===2.2.3/Werkzeug===2.3.8/' /src/upper-constraints.txt
sed -i 's/zstd===1.5.2.5/zstd===1.5.4.0/' /src/upper-constraints.txt
sed -i '/glance-store/d' /src/upper-constraints.txt
sed -i '/horizon/d' /src/upper-constraints.txt
EOF

FROM ubuntu-cloud-archive AS openstack-venv-builder
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
build-essential \
git \
libldap2-dev \
libpcre3-dev \
libsasl2-dev \
libssl-dev \
lsb-release \
openssh-client \
python3 \
python3-dev \
python3-pip \
python3-venv
EOF
RUN <<EOF bash -xe
python3 -m venv --upgrade-deps --system-site-packages /var/lib/openstack
EOF
ENV PATH=/var/lib/openstack/bin:$PATH
COPY --link --from=requirements /src/upper-constraints.txt /upper-constraints.txt
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
cryptography \
pymysql \
python-binary-memcached \
python-memcached \
uwsgi
EOF

FROM ubuntu-cloud-archive AS openstack-runtime
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
ca-certificates \
libpython3.10 \
lsb-release \
python3-distutils \
sudo
EOF
ARG PROJECT
ARG SHELL=/usr/sbin/nologin
RUN \
groupadd -g 42424 ${PROJECT} && \
useradd -u 42424 -g 42424 -M -d /var/lib/${PROJECT} -s ${SHELL} -c "${PROJECT} User" ${PROJECT} && \
mkdir -p /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT} && \
chown -Rv ${PROJECT}:${PROJECT} /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
ENV PATH=/var/lib/openstack/bin:$PATH

FROM alpine/git AS barbican-src
ARG BARBICAN_GIT_REF
ADD --keep-git-dir=true https://opendev.org/openstack/barbican.git#${BARBICAN_GIT_REF} /src
RUN git -C /src fetch --unshallow

FROM openstack-venv-builder AS barbican-build
COPY --from=barbican-src --link /src /src/barbican
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
/src/barbican \
pykmip
EOF

FROM openstack-runtime AS barbican
COPY --from=barbican-build --link /var/lib/openstack /var/lib/openstack

FROM alpine/git AS magnum-src
ARG MAGNUM_GIT_REF
ADD --keep-git-dir=true https://opendev.org/openstack/magnum.git#${MAGNUM_GIT_REF} /src
RUN git -C /src fetch --unshallow
ARG RELEASE
COPY patches/${RELEASE}/magnum /patches
RUN if [ -n "$(ls -A /patches/*.patch)" ]; then git -C /src apply --verbose /patches/*; fi

FROM openstack-venv-builder AS magnum-build
COPY --from=magnum-src --link /src /src/magnum
RUN <<EOF bash -xe
pip3 install \
--constraint /upper-constraints.txt \
/src/magnum \
magnum-cluster-api==0.16.0
EOF

FROM openstack-runtime AS magnum
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
haproxy
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
COPY --from=helm --link /usr/bin/helm /usr/local/bin/helm
COPY --from=magnum-build --link /var/lib/openstack /var/lib/openstack
# Copyright (c) 2024 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

FROM golang AS go-builder
COPY go.mod go.sum /src/
WORKDIR /src
RUN go mod download

FROM go-builder AS libvirt-tls-sidecar-builder
COPY cmd/ /src/cmd/
COPY internal/ /src/internal/
RUN go build -o main ./cmd/libvirt-tls-sidecar/main.go

FROM ubuntu AS libvirt-tls-sidecar
COPY --from=libvirt-tls-sidecar-builder /src/main /usr/bin/libvirt-tls-sidecar
ENTRYPOINT ["/usr/bin/libvirt-tls-sidecar"]
112 changes: 21 additions & 91 deletions Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -36,47 +36,26 @@ unit.go:
SAVE ARTIFACT /src/junit-go.xml AS LOCAL junit-go.xml
END

builder:
FROM ubuntu:jammy
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
build-essential git python3-dev python3-pip python3-venv
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
ARG POETRY_VERSION=1.4.2
RUN pip3 install --no-cache-dir poetry==${POETRY_VERSION}

build.collection:
FROM registry.gitlab.com/pipeline-components/ansible-lint:latest
COPY . /src
RUN ansible-galaxy collection build /src
SAVE ARTIFACT /code/*.tar.gz AS LOCAL dist/

go.build:
FROM golang:1.21
WORKDIR /src
ARG GOOS=linux
ARG GOARCH=amd64
ARG VARIANT
COPY --dir go.mod go.sum ./
RUN go mod download

libvirt-tls-sidecar.build:
FROM +go.build
ARG GOOS=linux
ARG GOARCH=amd64
ARG VARIANT
COPY --dir cmd internal ./
RUN GOARM=${VARIANT#"v"} go build -o main cmd/libvirt-tls-sidecar/main.go
SAVE ARTIFACT ./main

libvirt-tls-sidecar.platform-image:
ARG TARGETPLATFORM
ARG TARGETARCH
ARG TARGETVARIANT
FROM --platform=$TARGETPLATFORM ./images/base+image
COPY \
--platform=linux/amd64 \
(+libvirt-tls-sidecar.build/main --GOARCH=$TARGETARCH --VARIANT=$TARGETVARIANT) /usr/bin/libvirt-tls-sidecar
ENTRYPOINT ["/usr/bin/libvirt-tls-sidecar"]
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
SAVE IMAGE --push ${REGISTRY}/libvirt-tls-sidecar:latest

libvirt-tls-sidecar.image:
BUILD --platform=linux/amd64 --platform=linux/arm64 +libvirt-tls-sidecar.platform-image

build.wheels:
FROM ./images/builder+image
FROM +builder
COPY pyproject.toml poetry.lock ./
ARG --required only
RUN poetry export --only=${only} -f requirements.txt --without-hashes > requirements.txt
Expand Down Expand Up @@ -114,71 +93,22 @@ build.collections:
SAVE IMAGE --cache-hint

image:
ARG RELEASE=2023.1
FROM ./images/cloud-archive-base+image --RELEASE ${RELEASE}
FROM ubuntu:jammy
ENV ANSIBLE_PIPELINING=True
DO ./images+APT_INSTALL --PACKAGES "rsync openssh-client"
RUN <<EOF bash -xe
apt-get update -qq
apt-get install -qq -y --no-install-recommends \
rsync openssh-client
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
COPY +build.venv.runtime/venv /venv
ENV PATH=/venv/bin:$PATH
COPY +build.collections/ /usr/share/ansible
ARG tag=latest
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
SAVE IMAGE --push ${REGISTRY}:${tag}

images:
ARG REGISTRY=ghcr.io/vexxhost/atmosphere
BUILD +libvirt-tls-sidecar.image --REGISTRY=${REGISTRY}
BUILD ./images/cinder+image --REGISTRY=${REGISTRY}
BUILD ./images/cluster-api-provider-openstack+image --REGISTRY=${REGISTRY}
BUILD ./images/designate+image --REGISTRY=${REGISTRY}
BUILD ./images/glance+image --REGISTRY=${REGISTRY}
BUILD ./images/heat+image --REGISTRY=${REGISTRY}
BUILD ./images/horizon+image --REGISTRY=${REGISTRY}
BUILD ./images/ironic+image --REGISTRY=${REGISTRY}
BUILD ./images/keystone+image --REGISTRY=${REGISTRY}
BUILD ./images/kubernetes-entrypoint+image --REGISTRY=${REGISTRY}
BUILD ./images/libvirtd+image --REGISTRY=${REGISTRY}
BUILD ./images/magnum+image --REGISTRY=${REGISTRY}
BUILD ./images/manila+image --REGISTRY=${REGISTRY}
BUILD ./images/netoffload+image --REGISTRY=${REGISTRY}
BUILD ./images/neutron+image --REGISTRY=${REGISTRY}
BUILD ./images/nova-ssh+image --REGISTRY=${REGISTRY}
BUILD ./images/nova+image --REGISTRY=${REGISTRY}
BUILD ./images/octavia+image --REGISTRY=${REGISTRY}
BUILD ./images/openvswitch+image --REGISTRY=${REGISTRY}
BUILD ./images/ovn+images --REGISTRY=${REGISTRY}
BUILD ./images/placement+image --REGISTRY=${REGISTRY}
BUILD ./images/senlin+image --REGISTRY=${REGISTRY}
BUILD ./images/staffeln+image --REGISTRY=${REGISTRY}
BUILD ./images/tempest+image --REGISTRY=${REGISTRY}

SCAN_IMAGE:
FUNCTION
ARG --required IMAGE
# TODO(mnaser): Include secret scanning when it's more reliable.
RUN \
trivy image \
--skip-db-update \
--skip-java-db-update \
--scanners vuln \
--exit-code 1 \
--ignore-unfixed \
--timeout 10m \
${IMAGE}

scan-image:
FROM ./images/trivy+image
ARG --required IMAGE
DO +SCAN_IMAGE --IMAGE ${IMAGE}

scan-images:
FROM ./images/trivy+image
COPY roles/defaults/vars/main.yml /defaults.yml
# TODO(mnaser): Scan all images eventually
FOR IMAGE IN $(cat /defaults.yml | egrep -E 'ghcr.io/vexxhost|registry.atmosphere.dev' | cut -d' ' -f4 | sort | uniq)
BUILD +scan-image --IMAGE ${IMAGE}
END

pin-images:
FROM +build.venv.dev
COPY roles/defaults/vars/main.yml /defaults.yml
Expand Down
Loading

0 comments on commit 16c0fd9

Please sign in to comment.