From 9282784517cb25d3cb6b06a4895626b173177603 Mon Sep 17 00:00:00 2001 From: Andrew Starr-Bochicchio Date: Wed, 14 Aug 2024 13:57:36 -0400 Subject: [PATCH] Clean up build scripts, make targets, and Dockerfile for snap. (#1570) * Clean up build scripts, make targets, and Dockerfile for snap. * Update CONTRIBUTING.md --- CONTRIBUTING.md | 36 ++++++----- Makefile | 40 +----------- dockerfiles/Dockerfile.snap | 92 --------------------------- scripts/_build_snap.sh | 11 ---- scripts/snap.sh | 23 ------- scripts/snap_image.sh | 122 ------------------------------------ 6 files changed, 23 insertions(+), 301 deletions(-) delete mode 100644 dockerfiles/Dockerfile.snap delete mode 100755 scripts/_build_snap.sh delete mode 100755 scripts/snap.sh delete mode 100755 scripts/snap_image.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b384ee328..0b8fb43f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,13 +21,7 @@ - [Build Scripts](#build-scripts) - [Releasing](#releasing) - [Tagging a release](#tagging-a-release) - - [Prerequisites](#prerequisites) - - [If a release fails](#if-a-release-fails) - - [Github Releases & Dockerhub](#github-releases--dockerhub) - - [Prerequisites](#prerequisites-1) - - [Snap](#snap) - - [Prerequisites](#prerequisites-2) - - [Building a new snap base image](#building-a-new-snap-base-image) + - [Snap](#snap) - [Updating Homebrew](#updating-homebrew) @@ -204,8 +198,6 @@ To cut a release, push a new tag (versioning discussed below). ### Tagging a release -##### Prerequisites - 1. Run `make changes` to review the changes since the last release. Based on the changes, decide what kind of release you are doing (bugfix, feature or breaking). @@ -220,14 +212,26 @@ To cut a release, push a new tag (versioning discussed below). The new tag triggers the release. -##### Building a new snap base image +### Snap + +Snap packages are automatically built and uploaded as part of the GitHub Actions +release workflow. + +To build a snap package locally for testing, first install `snapcraft`. + +On Ubuntu, run: + + sudo snap install snapcraft --classic + +Or on MacOS, run: + + brew install snapcraft + +Finally, build the package by running: + + make snap -Occasionally, the snap build will break. When it does, it usually means that you need to update -the custom base image we use to build the snap. The Dockerfile for that image lives in -[dockerfiles/Dockerfile.snap](https://github.com/digitalocean/doctl/blob/main/dockerfiles/Dockerfile.snap). -The header of the Dockerfile has hints for updating the image, as well as instructions for building -the image using `make snap_image`. Once you've built the image, the snap_image target will provide -instructions for next steps. +More details about the snap package can be found in the `snap/snapcraft.yaml` file. ### Updating Homebrew diff --git a/Makefile b/Makefile index a772583de..43ea866e9 100644 --- a/Makefile +++ b/Makefile @@ -122,45 +122,12 @@ gofmt_check: check_focused: @scripts/check_focused_test.sh -.PHONY: snap_image -snap_image: - @echo "==> build docker image for releasing snap" - @echo "" - @scripts/snap_image.sh - -.PHONY: _snap_image_version -_snap_image_version: - @ORIGIN=${ORIGIN} SNAP_IMAGE=true scripts/version.sh - -.PHONY: build_local_snap -build_local_snap: - @echo "==> build local snap using local image tagged doctl-snap-base" - @echo "" - @BUILD=local_snap scripts/snap_image.sh - -.PHONY: prerelease_snap_image -prerelease_snap_image: - @echo "==> tag doctl-snap-base as a prerelease and push to dockerhub as latest" - @echo "" - @BUILD=pre scripts/snap_image.sh - -.PHONY: finalize_snap_image -finalize_snap_image: - @echo "==> tag latest with most recent doctl version push to dockerhub" - @echo "" - @ORIGIN=${ORIGIN} BUILD=finalize scripts/snap_image.sh - -CHANNEL ?= stable - -.PHONY: _build_snap -_build_snap: - @CHANNEL=${CHANNEL} scripts/_build_snap.sh .PHONY: snap -snap: - @echo "==> publish snap (normally done by travis)" +snap: clean + @echo "==> building snap" @echo "" - @CHANNEL=${CHANNEL} scripts/snap.sh + @snapcraft .PHONY: mocks mocks: @@ -234,7 +201,6 @@ _tag_and_release: tag @echo "=> DEPRECATED: BUMP=${BUMP} tag and release" @echo "" @$(MAKE) _release - @$(MAKE) snap .PHONY: release release: diff --git a/dockerfiles/Dockerfile.snap b/dockerfiles/Dockerfile.snap deleted file mode 100644 index c4233f532..000000000 --- a/dockerfiles/Dockerfile.snap +++ /dev/null @@ -1,92 +0,0 @@ -# Note to maintainers: after you make changes to this file, please run `make snap_image`. -# The script will gives instructions to complete the update once it finishes. Be patient, it -# takes a long time to run. -# -# For help with the technical aspects of this Dockerfile, see -# https://snapcraft.io/docs/t/creating-docker-images-for-snapcraft/11739 -# https://github.com/snapcore/snapcraft/blob/main/docker/Dockerfile -# and https://forum.snapcraft.io/. Note that the snapcraft forum does not appear to be indexed -# effectively (at all?) by google. -# -# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ -# for guidance on the style of this Dockerfile -FROM ubuntu:focal as builder - -RUN apt-get update && apt-get install --yes \ - curl \ - jq \ - squashfs-tools - -# Grab the core snap (for backwards compatibility) from the stable channel and unpack it in the proper place -RUN curl -L $(curl -H 'X-Ubuntu-Series: 16' 'https://api.snapcraft.io/api/v1/snaps/details/core' | jq '.download_url' -r) --output core.snap && \ - mkdir -p /snap/core && \ - unsquashfs -d /snap/core/current core.snap - -# Grab the core18 snap (which snapcraft uses as a base) from the stable channel and unpack it in the proper place. -RUN curl -L $(curl -H 'X-Ubuntu-Series: 16' 'https://api.snapcraft.io/api/v1/snaps/details/core18' | jq '.download_url' -r) --output core18.snap && \ - mkdir -p /snap/core18 && \ - unsquashfs -d /snap/core18/current core18.snap - -# Grab the core20 snap from the stable channel and unpack it in the proper place. -RUN curl -L $(curl -H 'X-Ubuntu-Series: 16' 'https://api.snapcraft.io/api/v1/snaps/details/core20' | jq '.download_url' -r) --output core20.snap && \ - mkdir -p /snap/core20 && \ - unsquashfs -d /snap/core20/current core20.snap - -# Grab the snapcraft snap from the stable channel and unpack it in the proper place -RUN curl -L $(curl -H 'X-Ubuntu-Series: 16' 'https://api.snapcraft.io/api/v1/snaps/details/snapcraft?channel=stable' | jq '.download_url' -r) --output snapcraft.snap && \ - mkdir -p /snap/snapcraft && \ - unsquashfs -d /snap/snapcraft/current snapcraft.snap - -# Fix Python3 installation: Make sure we use the interpreter from -# the snapcraft snap: -RUN unlink /snap/snapcraft/current/usr/bin/python3 -RUN ln -s /snap/snapcraft/current/usr/bin/python3.* /snap/snapcraft/current/usr/bin/python3 -RUN echo /snap/snapcraft/current/lib/python3.*/site-packages >> /snap/snapcraft/current/usr/lib/python3/dist-packages/site-packages.pth - -# Create a snapcraft runner (TODO: move version detection to the core of snapcraft) -RUN mkdir -p /snap/bin && \ - echo "#!/bin/sh" > /snap/bin/snapcraft && \ - snap_version="$(awk '/^version:/{print $2}' /snap/snapcraft/current/meta/snap.yaml | tr -d \')" && \ - echo "export SNAP_VERSION=\"$snap_version\"" >> /snap/bin/snapcraft && \ - echo 'exec "$SNAP/usr/bin/python3" "$SNAP/bin/snapcraft" "$@"' >> /snap/bin/snapcraft && \ - chmod +x /snap/bin/snapcraft - -# Grab the golang snap from the stable channel, unpack it in the proper place, and create a runner for it -RUN curl -L $(curl -H 'X-Ubuntu-Series: 16' 'https://api.snapcraft.io/api/v1/snaps/details/go?channel=1.21/stable' | jq '.download_url' -r) --output go.snap && \ - mkdir -p /snap/go && \ - unsquashfs -d /snap/go/current go.snap && \ - cd /snap/bin && \ - ln -s /snap/go/current/bin/go - -# Multi-stage build, only need the snaps from the builder. Copy them one at a -# time so they can be cached. -FROM ubuntu:focal -COPY --from=builder /snap/core /snap/core -COPY --from=builder /snap/core18 /snap/core18 -COPY --from=builder /snap/core20 /snap/core20 -COPY --from=builder /snap/snapcraft /snap/snapcraft -COPY --from=builder /snap/bin/snapcraft /snap/bin/snapcraft -COPY --from=builder /snap/go /snap/go -COPY --from=builder /snap/bin/go /snap/bin/go - -# Install Snap build-time dependencies & generate locale -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get clean && apt-get update && apt-get install --yes \ - sudo \ - locales \ - snapd && \ - locale-gen en_US.UTF-8 && \ - rm -rf /var/lib/apt/lists/* - -# Set the proper environment -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -ENV PATH "/snap/bin:/snap/snapcraft/current/usr/bin:$PATH" -ENV SNAP /snap/snapcraft/current -ENV SNAP_NAME snapcraft -ENV SNAP_ARCH amd64 -ENV GOROOT "/snap/go/current" - -SHELL ["/bin/bash", "-c"] -ENTRYPOINT "snapcraft" diff --git a/scripts/_build_snap.sh b/scripts/_build_snap.sh deleted file mode 100755 index d0173341d..000000000 --- a/scripts/_build_snap.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../" - -make clean - -echo "building snap" -echo "" -cd "$DIR" && docker run --rm -v "$DIR":/build -w /build sammytheshark/doctl-snap-base diff --git a/scripts/snap.sh b/scripts/snap.sh deleted file mode 100755 index 73799b732..000000000 --- a/scripts/snap.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -Eeuo pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../" - -function cleanup { - echo "cleaning up" - echo "" - cd "$DIR" && docker run -v "$DIR":/build -w /build snapcore/snapcraft:stable snapcraft clean -} -trap cleanup EXIT - -make _build_snap - -snap=$(ls doctl_*_amd64.snap) - -CHANNEL=${CHANNEL:-stable} - -echo "releasing snap" -echo "" -cd "$DIR" && docker run -i -v "$DIR":/build -w /build snapcore/snapcraft:stable \ - bash -c "snapcraft login && snapcraft push --release=${CHANNEL} ${snap}" diff --git a/scripts/snap_image.sh b/scripts/snap_image.sh deleted file mode 100755 index c8a56f665..000000000 --- a/scripts/snap_image.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../" - -ORIGIN=${ORIGIN:-origin} - -REPO_NAME=${REPO_NAME:-doctl-snap-base} -LATEST="${REPO_NAME}:latest" -SAMMY_LATEST="sammytheshark/${LATEST}" - -repo_missing() { - [[ $(docker images --format "{{.ID}}" "$1" | wc -l) -eq 0 ]] -} - -build_local_snap() { - docker tag "$REPO_NAME" "sammytheshark/$REPO_NAME" - cd "$DIR" && sudo make clean - make _build_snap -} - -build_pre() { - version="$(ORIGIN=${ORIGIN} SNAP_IMAGE=true "$DIR/scripts/version.sh")" - prerelease="sammytheshark/${REPO_NAME}:${version}" - - docker tag "$REPO_NAME" "$prerelease" && \ - docker push "$prerelease" - - docker tag "$REPO_NAME" "$SAMMY_LATEST" && \ - docker push "$SAMMY_LATEST" -} - -build_finalize() { - version=$(ORIGIN="${ORIGIN}" "$DIR/scripts/version.sh" -s) - release="sammytheshark/${REPO_NAME}:$version" - - docker tag "$SAMMY_LATEST" "$release" && \ - docker push "$release" -} - -build_snap_image() { - < "$DIR/dockerfiles/Dockerfile.snap" docker build --no-cache -t "$REPO_NAME" - - - cat < - make prerelease_snap_image - -Check your work: -- Visit https://hub.docker.com/repository/registry-1.docker.io/sammytheshark/doctl-snap-base/tags?page=1 -- Check the last updated date on the image tagged 'latest' - -### -C. Release a new version of doctl with any changes - -If the only change is a new snap image, fixing a broken snap build, -that's a patch, by the way. :) - -### -D. Rename the prerelease image to the new released version. - -make finalize_snap_image - -###################################################################### -###################################################################### -INSTRUCTIONS -} - -hash docker 2>/dev/null || - { echo >&2 "I require the docker CLI but it's not installed. Please see https://docs.docker.com/install/. Aborting."; exit 1; } - -BUILD=${BUILD:-snap_image} - -case "$BUILD" in - local_snap | pre) - if repo_missing "$LATEST"; then - echo "Could not find ${LATEST}. Did you run 'make snap_image'?" - exit 1 - fi - ;; - - finalize) - if repo_missing "$SAMMY_LATEST"; then - echo "Could not find ${SAMMY_LATEST}. Did you run 'make prerelease_snap_image'?" - exit 1 - fi - ;; -esac - -"build_${BUILD}"