Skip to content

Commit

Permalink
Wait and upload for Hydra (#99)
Browse files Browse the repository at this point in the history 
1. Hydra
2. DevX closures
3. Devcontainer
  • Loading branch information
@yvan-sraka
yvan-sraka authored Sep 11, 2023
1 parent 3fef25f commit bd723cf
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 93 deletions.
56 changes: 0 additions & 56 deletions .github/workflows/codespace.yml
View file

This file was deleted.

79 changes: 42 additions & 37 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,41 @@ env:
IMAGE_NAME: ${{ github.repository }}

jobs:
build:
wait-for-hydra-eval:
env:
HYDRA_JOB: ci/eval
GH_TOKEN: ${{ github.token }}
name: "Wait for hydra status"
runs-on: ubuntu-latest
steps:
- name: Get specific check run status
run: |
# start with a random sleep to prevent hitting the api too hard.
while true; do
# For GitHub Apps
# conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --jq '.check_runs[] | select(.name == "ci/hydra-build:$DEV_SHELL") | .conclusion')
# For GitHub Statuses; we need --paginate because there are so many statuses
echo "Querying: gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | select(.context == \"$HYDRA_JOB\") | .state'"
conclusion=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq ".statuses[] | select(.context == \"$HYDRA_JOB\") | .state")
case "$conclusion" in
success)
echo "$HYDRA_JOB succeeded"
exit 0;;
failure)
echo "$HYDRA_JOB failed"
exit 1;;
*)
echo "conclusion is: '$conclusion'"
gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq '.statuses[] | .state+"\t"+.context'|sort
WAIT=$((30 + RANDOM % 30))
echo "$HYDRA_JOB pending. Waiting ${WAIT}s..."
sleep $WAIT;;
esac
done
upload:
needs: wait-for-hydra-eval
name: Container Upload
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -66,39 +100,10 @@ jobs:
- target-platform: "-js"
platform: x86_64-darwin
variant: ""

runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Install Nix with good defaults
uses: cachix/install-nix-action@v20
with:
extra_nix_config: |
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
nix_path: nixpkgs=channel:nixos-unstable
- name: Checkout repository
uses: actions/checkout@v3
- name: Log in to the Container registry
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Work around issue fetching cabal revision files
run: |
# This will cause the revised `.cabal` files used whan building alex
# for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`.
# Later when we want the non `x86_64-linux` version later it will find the
# fixed output of the derivation is already in the `/nix/store` and will
# not try to fetch it using the platform we do not have a builder for.
if [[ "${{ matrix.platform }}" != "x86_64-linux" && "${{ matrix.target-platform }}" = "-js" ]]; then
nix build ".#hydraJobs.x86_64-linux.${{ matrix.compiler-nix-name }}-js-minimal" --show-trace
fi
- name: Compute and upload closure and developer environment to ghcr.io
env:
NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }}
DEV_SHELL: ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env
run: ./extra/ghcr-upload.sh
uses: ./.github/workflows/wait-and-upload.yml
with:
platform: ${{ matrix.platform }}
target-platform: ${{ matrix.target-platform }}
compiler-nix-name: ${{ matrix.compiler-nix-name }}
minimal: ${{ matrix.variant == '-minimal' }}
iog: ${{ matrix.iog == '-iog' }}
139 changes: 139 additions & 0 deletions .github/workflows/wait-and-upload.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
name: Wait and Upload

on:
workflow_call:
inputs:
platform:
required: true
description: 'build platform'
type: string
target-platform:
required: true
description: 'target platform'
type: string
compiler-nix-name:
required: true
description: 'compiler name in nix format. e.g. ghc8107'
type: string
minimal:
description: 'without hls, hlint, ...'
type: boolean
default: true
iog:
description: 'without iog libs: libsodium, libsecp256k1, libblst, ...'
type: boolean
default: false

env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
DEV_SHELL: ${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}${{ inputs.minimal && '-minimal' || '' }}${{ inputs.iog && '-iog' || ''}}-env
DEFAULT_TAG: latest
GH_TOKEN: ${{ github.token }}

jobs:
wait-for-hydra:
name: "Wait for hydra status"
runs-on: ubuntu-latest
steps:
- name: Get specific check run status
run: |
# start with a random sleep to prevent hitting the api too hard.
while true; do
# For GitHub Apps
# conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --jq '.check_runs[] | select(.name == "ci/hydra-build:$DEV_SHELL") | .conclusion')
# For GitHub Statuses; we need --paginate because there are so many statuses
echo "Querying: gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | select(.context == \"ci/hydra-build:$DEV_SHELL\") | .state'"
conclusion=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq ".statuses[] | select(.context == \"ci/hydra-build:$DEV_SHELL\") | .state")
case "$conclusion" in
success)
echo "ci/hydra-build:$DEV_SHELL succeeded"
exit 0;;
failure)
echo "ci/hydra-build:$DEV_SHELL failed"
exit 1;;
*)
echo "conclusion is: '$conclusion'"
gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | .state+"\t"+.context'|sort
WAIT=$((180 + RANDOM % 180))
echo "ci/hydra-build:$DEV_SHELL pending. Waiting ${WAIT}s..."
sleep $WAIT;;
esac
done
ghcr-upload:
needs: wait-for-hydra
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Install Nix with good defaults
uses: cachix/install-nix-action@v20
with:
extra_nix_config: |
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
nix_path: nixpkgs=channel:nixos-unstable
- name: Checkout repository
uses: actions/checkout@v3
- name: Log in to the Container registry
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Work around issue fetching cabal revision files
run: |
# This will cause the revised `.cabal` files used whan building alex
# for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`.
# Later when we want the non `x86_64-linux` version later it will find the
# fixed output of the derivation is already in the `/nix/store` and will
# not try to fetch it using the platform we do not have a builder for.
if [[ "${{ inputs.platform }}" != "x86_64-linux" && "${{ inputs.target-platform }}" = "-js" ]]; then
nix build ".#hydraJobs.x86_64-linux.${{ inputs.compiler-nix-name }}-js-minimal" --show-trace
fi
- name: Compute and upload closure and developer environment to ghcr.io
env:
NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }}
run: ./extra/ghcr-upload.sh

codespace-upload:
env:
IMAGE_NAME: input-output-hk/devx-devcontainer
needs: ghcr-upload
permissions:
packages: write
runs-on: ubuntu-latest
# We want a GitHub Codespace image for each combination of devx developer shell option.
# But, since the purpose of GitHub Codespace is to serve a complete development environment,
# the user is likely to always expect HLS (I don't see the point otherwise).
# Therefore, it doesn't seem useful to build an image on the `-minimal` flavor (without HLS),
# or the `-static` one (especially since the latter currently requires `-minimal` to work).
# Likely, we consider using `-iog` as the default and do not generate other images.
# Then the user choices left would be between native, `-windows` or `-js` target platforms,
# and the GHC version (currently `ghc8107` and `ghc962`).
if: ${{ contains(fromJSON('["x86_64-linux", "aarch64-linux"]'), inputs.platform) && contains(fromJson('["","-windows","-js"]'), inputs.target-platform) && contains(fromJson('["ghc8107","ghc962"]'), inputs.compiler-nix-name) && !inputs.minimal && inputs.iog }}
steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Log in to the Container registry
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: .
push: true
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}-iog
build-args: |
PLATFORM=${{ inputs.platform }}
TARGET_PLATFORM=${{ inputs.target-platform }}
COMPILER_NIX_NAME=${{ inputs.compiler-nix-name }}
MINIMAL=${{ inputs.minimal }}
IOG=${{ inputs.iog }}

0 comments on commit bd723cf

Please sign in to comment.