From 8d285b77c6a0ee3fdf3b7b9783572f4a7ce1ea1f Mon Sep 17 00:00:00 2001 From: Gal Topper Date: Tue, 19 Sep 2023 13:01:47 +0800 Subject: [PATCH] Update GHA files from development --- .github/workflows/pr.yaml | 51 +++++++++-- .github/workflows/push.yaml | 166 ++++++++++++++++++++++++++++-------- 2 files changed, 173 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a97b6c2f..c2c47a9e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,8 +21,8 @@ on: - 'integ_[0-9]+\.[0-9]+' jobs: - test: - name: Lint & test + test-go: + name: Lint & test Go code runs-on: ubuntu-latest steps: @@ -39,17 +39,16 @@ jobs: - name: Dump github ref run: echo "$GITHUB_REF" - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: 1.19.5 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: refs/pull/${{ github.event.number }}/merge -# Linter memory usage seems to be getting CI jobs killed -# - name: Lint -# run: make lint + - name: Lint + run: make lint - name: Run Go tests run: make test-go @@ -57,3 +56,41 @@ jobs: V3IO_API: ${{ secrets.V3IO_API }} V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} V3IO_SESSION: container=bigdata,user=admin + + test-py: + name: Lint & test Python code + runs-on: ubuntu-latest + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/checkout@v3 + + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Install dependencies + run: make python-deps + + - name: Lint Python code + run: make flake8 + + - name: Run Python tests + run: make test-py + env: + V3IO_API: ${{ secrets.V3IO_API }} + V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} + V3IO_SESSION: container=bigdata diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index f63ba3a3..e5405f13 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -17,15 +17,16 @@ name: CI on: push: branches: - - master + - development + - 'integ_[0-9]+\.[0-9]+' release: types: - created jobs: - test: - name: Lint & test + test-go: + name: Lint & test Go code runs-on: ubuntu-latest steps: @@ -42,15 +43,15 @@ jobs: - name: Dump github ref run: echo "$GITHUB_REF" - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: "^1.19.5" - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 -# Linter memory usage seems to be getting CI jobs killed -# - name: Lint -# run: make lint + # This gets killed, probably due to memory usage + # - name: Lint + # run: make lint - name: Run Go tests run: make test-go @@ -59,18 +60,51 @@ jobs: V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} V3IO_SESSION: container=bigdata,user=admin - release: - name: Release + test-py: + name: Lint & test Python code + runs-on: ubuntu-latest + env: + V3IO_API: ${{ secrets.V3IO_API }} + V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} + V3IO_SESSION: container=bigdata + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/checkout@v3 + + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Install dependencies + run: make python-deps + + - name: Lint Python code + run: make flake8 + + - name: Run Python tests + run: make test-py + + release-docker: + name: Release to docker repository runs-on: ubuntu-latest if: github.event_name == 'release' needs: - - test - strategy: - matrix: - go-os: - - linux - - darwin - - windows + - test-go + - test-py steps: - name: Dump GitHub context @@ -86,35 +120,93 @@ jobs: - name: Dump github ref run: echo "$GITHUB_REF" - - name: Extract ref info - id: release_info - run: | - echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/} - echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/} - - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: "^1.19.5" - - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Build binaries for ${{ matrix.go-os }} run: make frames-bin env: - GOPATH: ${{ github.workspace }}/go GOARCH: amd64 - GOOS: ${{ matrix.go-os }} + GOOS: linux FRAMES_TAG: ${{ github.ref_name }} - name: Upload binaries - uses: AButler/upload-release-assets@v2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.ref_name }} framesd-* + + - name: Login to GCR + run: echo "$GCR_JSON_KEY" | docker login -u _json_key --password-stdin https://gcr.io + env: + GCR_JSON_KEY: ${{ secrets.GCR_IGUAZIO_JSON_KEY }} + + - name: Set image tag + run: | + echo FRAMES_TAG=`echo ${{ github.ref_name }} | sed s/^v//` >> $GITHUB_ENV + cat $GITHUB_ENV + + - name: Build docker image + run: make build + env: + GOARCH: amd64 + GOOS: linux + FRAMES_REPOSITORY: gcr.io/iguazio/ + + - name: Push frames image to GCR + run: docker push gcr.io/iguazio/frames:${{ env.FRAMES_TAG }} + + release-docker-internal: + name: Release to internal docker repository + runs-on: [ self-hosted, linux ] + if: github.event_name == 'release' + needs: + - release-docker + + steps: + - name: Login to artifactory + run: docker login -u ${{ secrets.ARTIFACTORY_DOCKER_USERNAME }} -p ${{ secrets.ARTIFACTORY_DOCKER_PASSWORD }} ${{ vars.ARTIFACTORY_REGISTRY }} + + - name: Set image tag + run: | + echo FRAMES_TAG=`echo ${{ github.ref_name }} | sed s/^v//` >> $GITHUB_ENV + cat $GITHUB_ENV + + - name: Pull docker from public repository + run: docker pull gcr.io/iguazio/frames:${{ env.FRAMES_TAG }} + + - name: Tag frames image for artifactory + run: docker tag gcr.io/iguazio/frames:${{ env.FRAMES_TAG }} ${{ vars.ARTIFACTORY_REGISTRY }}/iguazio/frames:${{ env.FRAMES_TAG }} + + - name: Push frames image to artifactory + run: docker push ${{ vars.ARTIFACTORY_REGISTRY }}/iguazio/frames:${{ env.FRAMES_TAG }} + + release-python-library: + name: Release python library + runs-on: ubuntu-latest + if: github.event_name == 'release' + needs: + - release-docker-internal + + steps: + - uses: actions/checkout@v3 + + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Set version + run: make set-version + + - name: Build v3io-frames python distribution + run: make python-dist + + - name: Publish to PyPi + uses: pypa/gh-action-pypi-publish@release/v1 with: - files: framesd-* - repo-token: ${{ secrets.GITHUB_TOKEN }} + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} + packages_dir: clients/py/dist/