From 5f146aa0927b2b9949b28bf9ca2862189e0516c0 Mon Sep 17 00:00:00 2001 From: greg pereira Date: Wed, 22 May 2024 22:36:41 -0700 Subject: [PATCH 1/3] Refactor CI for ilab to build via containerfile This implementation should preserve the OCI transfer implementaiton Signed-off-by: greg pereira --- .github/workflows/instructlab.yaml | 40 ++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/instructlab.yaml b/.github/workflows/instructlab.yaml index 7305afc0..a3243312 100644 --- a/.github/workflows/instructlab.yaml +++ b/.github/workflows/instructlab.yaml @@ -4,18 +4,18 @@ on: schedule: # schedule the job to run at 12 AM daily - cron: '0 0 * * *' - # pull_request: - # branches: - # - main - # paths: - # - .github/workflows/instructlab_baseimages_build_push.yaml - # - training/nvidia + pull_request: + branches: + - main + paths: + - .github/workflows/instructlab.yaml + - training/instructlab/Makefile # push: # branches: # - main # paths: - # - .github/workflows/instructlab_baseimages_build_push.yaml - # - training/nvidia + # - .github/workflows/instructlab.yaml + # - training/instructlab/Makefile workflow_dispatch: @@ -28,13 +28,18 @@ env: REGISTRY_ORG: containers jobs: - instructlab-nvidia: - if: "!contains(github.event.pull_request.labels.*.name, 'hold-tests') && github.repository == 'containers-mirror/ai-lab-recipes'" + instructlab-image-builds: + # if: "!contains(github.event.pull_request.labels.*.name, 'hold-tests') && github.repository == 'containers-mirror/ai-lab-recipes'" + if: "!contains(github.event.pull_request.labels.*.name, 'hold-tests')" strategy: matrix: include: - - make_target: nvidia + # - make_target: nvidia + # containerfile_path: training/instructlab/instructlab/containers/cuda/Containerfile - make_target: amd + containerfile_path: training/instructlab/instructlab/containers/rocm/Containerfile + - make_target: intel + containerfile_path: training/instructlab/instructlab/containers/hpu/Containerfile runs-on: ubuntu-latest # runs-on: ubuntu-22.04-2core # starting with minimal option steps: @@ -45,10 +50,19 @@ jobs: - uses: actions/checkout@v4.1.6 + - name: Make Instructlab + run: make instructlab + working-directory: ./training/instructlab + - name: Build Image id: build_image - run: make ${{ matrix.make_target}} - working-directory: ./training/instructlab + uses: redhat-actions/buildah-build@v2 + with: + image: ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target }} + tags: latest ${{ github.sha }} + platforms: linux/amd64 + containerfiles: ${{ matrix.containerfile_path }} + context: training/instructlab/instructlab/ - name: Login to Container Registry if: github.event_name == 'push' && github.ref == 'refs/heads/main' From bccb6a816b2c2919fbfd0e95361a15c473f20fa8 Mon Sep 17 00:00:00 2001 From: greg pereira Date: Wed, 29 May 2024 10:29:44 -0700 Subject: [PATCH 2/3] bump buildah base version for cache mount type Signed-off-by: greg pereira --- .github/workflows/instructlab.yaml | 37 ++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/instructlab.yaml b/.github/workflows/instructlab.yaml index a3243312..bc82e66c 100644 --- a/.github/workflows/instructlab.yaml +++ b/.github/workflows/instructlab.yaml @@ -37,9 +37,9 @@ jobs: # - make_target: nvidia # containerfile_path: training/instructlab/instructlab/containers/cuda/Containerfile - make_target: amd - containerfile_path: training/instructlab/instructlab/containers/rocm/Containerfile + containerfile_path: instructlab/containers/rocm/Containerfile - make_target: intel - containerfile_path: training/instructlab/instructlab/containers/hpu/Containerfile + containerfile_path: instructlab/containers/hpu/Containerfile runs-on: ubuntu-latest # runs-on: ubuntu-22.04-2core # starting with minimal option steps: @@ -54,15 +54,34 @@ jobs: run: make instructlab working-directory: ./training/instructlab + - name: Checkout buildah codebase + id: checkout-buildah + uses: actions/checkout@v4.1.6 + with: + repository: containers/buildah + path: buildah + ref: 'main' + + - name: Setup Buildah v1.36 + working-directory: buildah + run: | + sudo apt-get -y -qq update + sudo apt-get -y install bats btrfs-progs git go-md2man golang libapparmor-dev libglib2.0-dev libgpgme11-dev libseccomp-dev libselinux1-dev make skopeo + make runc all SECURITYTAGS="apparmor seccomp" + sudo make install install.runc + - name: Build Image id: build_image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target }} - tags: latest ${{ github.sha }} - platforms: linux/amd64 - containerfiles: ${{ matrix.containerfile_path }} - context: training/instructlab/instructlab/ + working-directory: ./training/instructlab + run: | + buildah --version + buildah bud --platform linux/amd64 \ + -f ${{ matrix.containerfile_path }} \ + --format docker \ + --tls-verify=true \ + -t ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target}}:latest \ + instructlab/ + - name: Login to Container Registry if: github.event_name == 'push' && github.ref == 'refs/heads/main' From bb44d08e6fe1db7cc98137ded86abdb47d88a804 Mon Sep 17 00:00:00 2001 From: greg pereira Date: Wed, 29 May 2024 11:53:59 -0700 Subject: [PATCH 3/3] refactor to match upstream ilab Signed-off-by: greg pereira --- .github/workflows/instructlab.yaml | 77 ++++++++++++++---------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/.github/workflows/instructlab.yaml b/.github/workflows/instructlab.yaml index bc82e66c..6c25e64d 100644 --- a/.github/workflows/instructlab.yaml +++ b/.github/workflows/instructlab.yaml @@ -49,55 +49,50 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" - uses: actions/checkout@v4.1.6 - - - name: Make Instructlab - run: make instructlab - working-directory: ./training/instructlab - - name: Checkout buildah codebase - id: checkout-buildah - uses: actions/checkout@v4.1.6 - with: - repository: containers/buildah - path: buildah - ref: 'main' + - uses: docker/setup-buildx-action@v3 - - name: Setup Buildah v1.36 - working-directory: buildah + - name: Login to Container Registry + if: github.ref == 'refs/heads/main' run: | - sudo apt-get -y -qq update - sudo apt-get -y install bats btrfs-progs git go-md2man golang libapparmor-dev libglib2.0-dev libgpgme11-dev libseccomp-dev libselinux1-dev make skopeo - make runc all SECURITYTAGS="apparmor seccomp" - sudo make install install.runc + docker login ${{ env.REGISTRY }} -u ${{ secrets.REGISTRY_USER }} -p ${{ secrets.REGISTRY_PASSWORD }} - - name: Build Image - id: build_image + - name: Extract metadata (tags, labels) for ilab image + id: ilab_meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target }} + + - name: Make Instructlab + run: make instructlab working-directory: ./training/instructlab - run: | - buildah --version - buildah bud --platform linux/amd64 \ - -f ${{ matrix.containerfile_path }} \ - --format docker \ - --tls-verify=true \ - -t ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target}}:latest \ - instructlab/ - - - name: Login to Container Registry - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: redhat-actions/podman-login@v1.7 + - name: Build and push ilab images + uses: docker/build-push-action@v5 with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_PASSWORD }} + context: ./training/instructlab/instructlab + platforms: linux/amd64 + build-args: | + GIT_TAG=stable + push: false + tags: ${{ steps.ilab_meta.outputs.tags }} + labels: ${{ steps.ilab_meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + file: ./training/instructlab/${{ matrix.containerfile_path }} - - name: Push image - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: redhat-actions/push-to-registry@v2.8 - with: - image: ${{ steps.build_image.outputs.image }} - tags: ${{ steps.build_image.outputs.tags }} - registry: ${{ env.REGISTRY }} + # - name: Build Image + # id: build_image + # working-directory: ./training/instructlab + # run: | + # buildah --version + # buildah bud --platform linux/amd64 \ + # -f ${{ matrix.containerfile_path }} \ + # --format docker \ + # --tls-verify=true \ + # -t ${{ env.REGISTRY }}/${{ env.REGISTRY_ORG }}/instructlab-${{ matrix.make_target}}:latest \ + # instructlab/ - name: Publish Job Results to Slack id: slack