diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c7c037..f8966ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - local: + release: name: Local release runs-on: ubuntu-latest @@ -48,7 +48,7 @@ jobs: run: docker compose up -d - name: Run the local release process for channel ${{ matrix.channel }} - run: ./run.sh ${{ matrix.channel }} + run: ./run.sh release ${{ matrix.channel }} - name: Validate the generated signatures run: docker compose exec -T local /src/local/check-signature.sh ${{ matrix.channel }} @@ -61,6 +61,36 @@ jobs: env: RUSTUP_DIST_SERVER: http://localhost:9000/static + rustup: + name: Local rustup + runs-on: ubuntu-latest + + env: + PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION: 99.0.0 + + strategy: + fail-fast: false + matrix: + channel: [stable, beta] + + steps: + - name: Clone the source code + uses: actions/checkout@v3 + + - name: Ensure Rust Stable is up to date + run: rustup self update && rustup update stable + + - name: Start the local environment + run: docker compose up -d + + - name: Run the local release process for channel ${{ matrix.channel }} + run: ./run.sh rustup ${{ matrix.channel }} + + - name: Update Rustup from the local environment + run: rustup self update + env: + RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + docker: name: Build Docker image runs-on: ubuntu-latest @@ -90,7 +120,7 @@ jobs: permissions: id-token: write - needs: [test, local, docker] + needs: [test, release, rustup, docker] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: diff --git a/local/Dockerfile b/local/Dockerfile index 0087840..406b20c 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -16,13 +16,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ git \ gnupg \ jq \ + libssl-dev \ + openssl \ + pkg-config \ python3 \ socat # Install rustup while removing the pre-installed stable toolchain. -RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ - chmod +x /tmp/rustup-init && \ - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ /root/.cargo/bin/rustup toolchain remove stable ENV PATH=/root/.cargo/bin:$PATH diff --git a/local/run.sh b/local/release.sh similarity index 95% rename from local/run.sh rename to local/release.sh index 45f7b98..7564352 100755 --- a/local/run.sh +++ b/local/release.sh @@ -101,6 +101,14 @@ for file in "${DOWNLOAD_STANDALONE[@]}"; do download "${file}" done +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + echo "==> configuring the environment" # Point to the right GnuPG environment export GNUPGHOME=/persistent/gpg-home diff --git a/local/rustup.sh b/local/rustup.sh new file mode 100755 index 0000000..5d7ff8c --- /dev/null +++ b/local/rustup.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# This script is executed at the start of each local release for Rustup, and +# prepares the environment by copying the artifacts built by CI onto the MinIO +# instance. Then, it starts promote-release with the right flags. + +set -euo pipefail +IFS=$'\n\t' + +RUSTUP_REPO="https://github.com/rust-lang/rustup" +RUSTUP_DEFAULT_BRANCH="master" + +# S3 bucket from which to download the Rustup artifacts +S3_BUCKET="rustup-builds" + +# CDN from which to download the CI artifacts +DOWNLOAD_BASE="https://rustup-builds.rust-lang.org" + +# The artifacts for the following targets will be downloaded and copied during +# the release process. At least one target is required. +DOWNLOAD_TARGETS=( + "x86_64-unknown-linux-gnu" +) + +# The following files will be downloaded and put into the local MinIO instance. +DOWNLOAD_FILES=( + "rustup-init" + "rustup-init.sha256" + "rustup-setup" + "rustup-setup.sha256" +) + +channel="$1" +override_commit="$2" + +if [[ "${override_commit}" = "" ]]; then + echo "==> detecting the last Rustup commit on the default branch" + commit="$(git ls-remote "${RUSTUP_REPO}" | grep "refs/heads/${RUSTUP_DEFAULT_BRANCH}" | awk '{print($1)}')" +else + echo "=>> using overridden commit ${override_commit}" + commit="${override_commit}" +fi + +for target in "${DOWNLOAD_TARGETS[@]}"; do + if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then + echo "==> copying ${target} from S3" + + for file in "${DOWNLOAD_FILES[@]}"; do + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then + mc cp /tmp/component "local/artifacts/builds/${commit}/dist/${target}/${file}" >/dev/null + fi + done + else + echo "==> reusing cached ${target} target" + fi +done + +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + +echo "==> configuring the environment" + +# Release Rustup +export PROMOTE_RELEASE_ACTION="promote-rustup" + +# Point to the right GnuPG environment +export GNUPGHOME=/persistent/gpg-home + +## Environment variables also used in prod releases +export AWS_ACCESS_KEY_ID="access_key" +export AWS_SECRET_ACCESS_KEY="secret_key" +export PROMOTE_RELEASE_CHANNEL="${channel}" +export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID="" +export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID="" +export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" +export PROMOTE_RELEASE_DOWNLOAD_DIR="builds" +export PROMOTE_RELEASE_GPG_KEY_FILE="" +export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" +export PROMOTE_RELEASE_UPLOAD_ADDR="" +export PROMOTE_RELEASE_UPLOAD_BUCKET="static" +export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD" +export PROMOTE_RELEASE_UPLOAD_DIR="rustup" + +## Environment variables used only by local releases +export PROMOTE_RELEASE_S3_ENDPOINT_URL="http://minio:9000" + +# Conditional environment variables +if [[ "${override_commit}" != "" ]]; then + export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" +fi + +echo "==> starting promote-release" +/src/target/release/promote-release /persistent/release "${channel}" diff --git a/local/setup.sh b/local/setup.sh index 2e7e1e0..97219b4 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -11,7 +11,7 @@ MINIO_URL="http://${MINIO_HOST}:${MINIO_PORT}" MINIO_ACCESS_KEY="access_key" MINIO_SECRET_KEY="secret_key" -MINIO_BUCKETS=( "static" "artifacts" ) +MINIO_BUCKETS=( "static" "artifacts" "rustup-builds" ) # Quit immediately when docker-compose receives a Ctrl+C trap exit EXIT @@ -77,9 +77,9 @@ cat < [commit]" +if [[ "$#" -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi +command="$1" + +if [[ "${command}" == "release" ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then + echo "Usage: $0 release [commit]" exit 1 + fi fi -channel="$1" -override_commit="${2-}" + +if [[ "${command}" == "rustup" ]]; then + if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 rustup [commit]" + exit 1 + fi +fi + +channel="$2" +override_commit="${3-}" container_id="$(docker compose ps -q local)" if [[ "${container_id}" == "" ]]; then @@ -27,8 +43,10 @@ if [[ "${container_status}" != "running" ]]; then exit 1 fi -# Ensure the release build is done -cargo build --release +# Pre-built the binary if the host and Docker environments match +if [[ "$(uname)" == "Linux" ]]; then + cargo build --release +fi # Run the command inside the docker environment. -docker compose exec -T local /src/local/run.sh "${channel}" "${override_commit}" +docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" diff --git a/src/config.rs b/src/config.rs index 070ce46..fc3f0fb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,8 @@ impl std::fmt::Display for Channel { } } +// Allow all variant names to start with `Promote` +#[allow(clippy::enum_variant_names)] #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum Action { /// This is the default action, what we'll do if the environment variable @@ -64,6 +66,13 @@ pub(crate) enum Action { /// * Create a rust-lang/cargo branch for the appropriate beta commit. /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. PromoteBranches, + + /// This promotes a new rustup release: + /// + /// * Copy binaries into archives + /// * Copy binaries from dev-static to production + /// * Update dev release number + PromoteRustup, } impl FromStr for Action { @@ -73,7 +82,8 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), - _ => anyhow::bail!("unknown channel: {}", input), + "promote-rustup" => Ok(Action::PromoteRustup), + _ => anyhow::bail!("unknown action: {}", input), } } } diff --git a/src/main.rs b/src/main.rs index 9bafd1c..ceb8fac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod discourse; mod fastly; mod github; mod recompress; +mod rustup; mod sign; mod smoke_test; @@ -76,6 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, + config::Action::PromoteRustup => self.promote_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs new file mode 100644 index 0000000..5c065fb --- /dev/null +++ b/src/rustup.rs @@ -0,0 +1,225 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Error}; +use curl::easy::Easy; +use serde::Deserialize; + +use crate::config::Channel; +use crate::curl_helper::BodyExt; +use crate::{run, Context}; + +impl Context { + /// Promote a `rustup` release + /// + /// The [release process] for `rustup` involves copying existing artifacts from one S3 bucket to + /// another, updating the manifest, and archiving the artifacts for long-term storage. + /// + /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the + /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and + /// copy them into `s3://rustup-builds/builds/${commit-sha}/`. + /// + /// When a new release is cut and this method is invoked, it downloads the artifacts from that + /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is + /// derived from the Cargo.toml file in the `stable` branch. `UPLOAD_BUCKET` can either be the + /// `dev-static` or the `static` bucket. + /// + /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` + /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to + /// download the latest release. + /// + /// Then, the `release-stable.toml` manifest is updated with the new version and copied to + /// `s3://${UPLOAD_BUCKET}/rustup/release-stable.toml`. + /// + /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html + /// [rust-lang/rustup]: https://github.com/rust-lang/rustup + pub fn promote_rustup(&mut self) -> anyhow::Result<()> { + // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly + self.enforce_rustup_channel()?; + + // Get the latest commit from the `stable` branch or use the user-provided override + let head_sha = self.get_commit_sha_for_rustup_release()?; + + // The commit on the `stable` branch is used to determine the version number + let version = self.get_next_rustup_version(&head_sha)?; + + // Download the Rustup artifacts from S3 + let dist_dir = self.download_rustup_artifacts(&head_sha)?; + + // Archive the artifacts + self.archive_rustup_artifacts(&dist_dir, &version)?; + + if self.config.channel == Channel::Stable { + // Promote the artifacts to the release bucket + self.promote_rustup_artifacts(&dist_dir)?; + } + + // Update the release number + self.update_rustup_release(&version)?; + + Ok(()) + } + + fn enforce_rustup_channel(&self) -> anyhow::Result<()> { + println!("Checking channel..."); + + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + Ok(()) + } + + fn get_commit_sha_for_rustup_release(&self) -> anyhow::Result { + match &self.config.override_commit { + Some(sha) => Ok(sha.clone()), + None => self.get_head_sha_for_rustup(), + } + } + + fn get_head_sha_for_rustup(&self) -> anyhow::Result { + #[derive(Deserialize)] + struct Commit { + sha: String, + } + + let mut client = Easy::new(); + client.url("https://api.github.com/repos/rust-lang/rustup/commits/stable")?; + client.useragent("rust-lang/promote-release")?; + + let commit: Commit = client.without_body().send_with_response()?; + + Ok(commit.sha) + } + + fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + // Allow the version to be overridden manually, for example to test the release process + if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { + println!("Using override version: {}", version); + Ok(version) + } else { + self.get_next_rustup_version_from_github(sha) + } + } + + fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { + println!("Getting next Rustup version from Cargo.toml..."); + + #[derive(Deserialize)] + struct Content { + content: String, + } + + #[derive(Deserialize)] + struct CargoToml { + package: Package, + } + + #[derive(Deserialize)] + struct Package { + version: String, + } + + let url = + format!("https://api.github.com/repos/rust-lang/rustup/contents/Cargo.toml?ref={sha}"); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let content: Content = client.without_body().send_with_response()?; + let decoded_content = base64::decode(content.content.replace('\n', ""))?; + let cargo_toml = String::from_utf8(decoded_content)?; + + let toml: CargoToml = toml::from_str(&cargo_toml)?; + + Ok(toml.package.version) + } + + fn download_rustup_artifacts(&mut self, sha: &str) -> Result { + println!("Downloading artifacts from dev-static..."); + + let dl = self.dl_dir().join("dist"); + // Remove the directory if it exists, otherwise just ignore. + let _ = fs::remove_dir_all(&dl); + fs::create_dir_all(&dl)?; + + let download_path = format!("{}/{}", self.config.download_dir, sha); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(self.s3_artifacts_url(&download_path)) + .arg(format!("{}/", dl.display())))?; + + Ok(dl) + } + + fn archive_rustup_artifacts(&mut self, dist_dir: &Path, version: &str) -> Result<(), Error> { + println!("Archiving artifacts for version {version}..."); + + let path = format!("archive/{}/", version); + + self.upload_rustup_artifacts(dist_dir, &path) + } + + fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Promoting artifacts to dist/..."); + + let release_bucket_url = format!( + "s3://{}/{}/dist/", + self.config.upload_bucket, self.config.upload_dir, + ); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(&release_bucket_url)) + } + + fn upload_rustup_artifacts(&mut self, dist_dir: &Path, target_path: &str) -> Result<(), Error> { + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(format!( + "s3://{}/{}/{}", + self.config.upload_bucket, self.config.upload_dir, target_path + ))) + } + + fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { + println!("Updating version and manifest..."); + + let manifest_path = self.dl_dir().join("release-stable.toml"); + let manifest = format!( + r#" +schema-version = '1' +version = '{}' + "#, + version + ); + + fs::write(&manifest_path, manifest)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(manifest_path) + .arg(format!( + "s3://{}/{}/release-stable.toml", + self.config.upload_bucket, self.config.upload_dir + ))) + } +}