Update GHA files from development

.github/workflows/pr.yaml

@@ -21,8 +21,8 @@ on:
       - 'integ_[0-9]+\.[0-9]+'
 
 jobs:
-  test:
-    name: Lint & test
+  test-go:
+    name: Lint & test Go code
     runs-on: ubuntu-latest
 
     steps:
@@ -39,21 +39,58 @@ jobs:
       - name: Dump github ref
         run: echo "$GITHUB_REF"
 
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: 1.19.5
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: refs/pull/${{ github.event.number }}/merge
 
-# Linter memory usage seems to be getting CI jobs killed
-#      - name: Lint
-#        run: make lint
+      - name: Lint
+        run: make lint
 
       - name: Run Go tests
         run: make test-go
         env:
           V3IO_API: ${{ secrets.V3IO_API }}
           V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }}
           V3IO_SESSION: container=bigdata,user=admin
+
+  test-py:
+    name: Lint & test Python code
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dump github context
+        run: echo "$GITHUB_CONTEXT"
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+
+      - name: Dump runner context
+        run: echo "$RUNNER_CONTEXT"
+        env:
+          RUNNER_CONTEXT: ${{ toJson(runner) }}
+
+      - name: Dump github ref
+        run: echo "$GITHUB_REF"
+
+      - uses: actions/checkout@v3
+
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: make python-deps
+
+      - name: Lint Python code
+        run: make flake8
+
+      - name: Run Python tests
+        run: make test-py
+        env:
+          V3IO_API: ${{ secrets.V3IO_API }}
+          V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }}
+          V3IO_SESSION: container=bigdata

.github/workflows/push.yaml

@@ -17,15 +17,16 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - development
+      - 'integ_[0-9]+\.[0-9]+'
 
   release:
     types:
       - created
 
 jobs:
-  test:
-    name: Lint & test
+  test-go:
+    name: Lint & test Go code
     runs-on: ubuntu-latest
 
     steps:
@@ -42,15 +43,15 @@ jobs:
       - name: Dump github ref
         run: echo "$GITHUB_REF"
 
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: "^1.19.5"
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-# Linter memory usage seems to be getting CI jobs killed
-#      - name: Lint
-#        run: make lint
+      # This gets killed, probably due to memory usage
+      # - name: Lint
+      #   run: make lint
 
       - name: Run Go tests
         run: make test-go
@@ -59,18 +60,51 @@ jobs:
           V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }}
           V3IO_SESSION: container=bigdata,user=admin
 
-  release:
-    name: Release
+  test-py:
+    name: Lint & test Python code
+    runs-on: ubuntu-latest
+    env:
+      V3IO_API: ${{ secrets.V3IO_API }}
+      V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }}
+      V3IO_SESSION: container=bigdata
+
+    steps:
+      - name: Dump github context
+        run: echo "$GITHUB_CONTEXT"
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+
+      - name: Dump runner context
+        run: echo "$RUNNER_CONTEXT"
+        env:
+          RUNNER_CONTEXT: ${{ toJson(runner) }}
+
+      - name: Dump github ref
+        run: echo "$GITHUB_REF"
+
+      - uses: actions/checkout@v3
+
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: make python-deps
+
+      - name: Lint Python code
+        run: make flake8
+
+      - name: Run Python tests
+        run: make test-py
+
+  release-docker:
+    name: Release to docker repository
     runs-on: ubuntu-latest
     if: github.event_name == 'release'
     needs:
-      - test
-    strategy:
-      matrix:
-        go-os:
-          - linux
-          - darwin
-          - windows
+      - test-go
+      - test-py
 
     steps:
       - name: Dump GitHub context
@@ -86,35 +120,93 @@ jobs:
       - name: Dump github ref
         run: echo "$GITHUB_REF"
 
-      - name: Extract ref info
-        id: release_info
-        run: |
-          echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/}
-          echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/}
-
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: "^1.19.5"
 
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
       - name: Build binaries for ${{ matrix.go-os }}
         run: make frames-bin
         env:
-          GOPATH: ${{ github.workspace }}/go
           GOARCH: amd64
-          GOOS: ${{ matrix.go-os }}
+          GOOS: linux
           FRAMES_TAG: ${{ github.ref_name }}
 
       - name: Upload binaries
-        uses: AButler/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload ${{ github.ref_name }} framesd-*
+
+      - name: Login to GCR
+        run: echo "$GCR_JSON_KEY" | docker login -u _json_key --password-stdin https://gcr.io
+        env:
+          GCR_JSON_KEY: ${{ secrets.GCR_IGUAZIO_JSON_KEY }}
+
+      - name: Set image tag
+        run: |
+          echo FRAMES_TAG=`echo ${{ github.ref_name }} | sed s/^v//` >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Build docker image
+        run: make build
+        env:
+          GOARCH: amd64
+          GOOS: linux
+          FRAMES_REPOSITORY: gcr.io/iguazio/
+
+      - name: Push frames image to GCR
+        run: docker push gcr.io/iguazio/frames:${{ env.FRAMES_TAG }}
+
+  release-docker-internal:
+    name: Release to internal docker repository
+    runs-on: [ self-hosted, linux ]
+    if: github.event_name == 'release'
+    needs:
+      - release-docker
+
+    steps:
+      - name: Login to artifactory
+        run: docker login -u ${{ secrets.ARTIFACTORY_DOCKER_USERNAME }} -p ${{ secrets.ARTIFACTORY_DOCKER_PASSWORD }} ${{ vars.ARTIFACTORY_REGISTRY }}
+
+      - name: Set image tag
+        run: |
+          echo FRAMES_TAG=`echo ${{ github.ref_name }} | sed s/^v//` >> $GITHUB_ENV
+          cat $GITHUB_ENV
+
+      - name: Pull docker from public repository
+        run: docker pull gcr.io/iguazio/frames:${{ env.FRAMES_TAG }}
+
+      - name: Tag frames image for artifactory
+        run: docker tag gcr.io/iguazio/frames:${{ env.FRAMES_TAG }} ${{ vars.ARTIFACTORY_REGISTRY }}/iguazio/frames:${{ env.FRAMES_TAG }}
+
+      - name: Push frames image to artifactory
+        run: docker push ${{ vars.ARTIFACTORY_REGISTRY }}/iguazio/frames:${{ env.FRAMES_TAG }}
+
+  release-python-library:
+    name: Release python library
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    needs:
+      - release-docker-internal
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+
+      - name: Set version
+        run: make set-version
+
+      - name: Build v3io-frames python distribution
+        run: make python-dist
+
+      - name: Publish to PyPi
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          files: framesd-*
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          user: __token__
+          password: ${{ secrets.PYPI_TOKEN }}
+          packages_dir: clients/py/dist/