diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 00000000..e80ac23a --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,58 @@ +# Copyright 2023 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: CI + +on: + pull_request_target: + branches: + - development + - 'integ_[0-9]+\.[0-9]+' + +jobs: + test: + name: Lint & test + runs-on: ubuntu-latest + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/setup-go@v2 + with: + go-version: 1.19.5 + + - uses: actions/checkout@v2 + with: + ref: refs/pull/${{ github.event.number }}/merge + + - name: Lint + run: make lint + + - name: Run Go tests + run: make test-go + env: + V3IO_API: ${{ secrets.V3IO_API }} + V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} + V3IO_SESSION: container=bigdata,user=admin diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 00000000..10f6de02 --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,118 @@ +# Copyright 2023 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: CI + +on: + push: + branches: + - master + + release: + types: + - created + +jobs: + test: + name: Lint & test + runs-on: ubuntu-latest + + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/setup-go@v2 + with: + go-version: "^1.19.5" + + - uses: actions/checkout@v2 + + - name: Lint + run: make lint + + - name: Run Go tests + run: make test-go + env: + V3IO_API: ${{ secrets.V3IO_API }} + V3IO_ACCESS_KEY: ${{ secrets.V3IO_ACCESS_KEY }} + V3IO_SESSION: container=bigdata,user=admin + + release: + name: Release + runs-on: ubuntu-latest + if: github.event_name == 'release' + needs: + - test + strategy: + matrix: + go-os: + - linux + - darwin + - windows + + steps: + - name: Dump GitHub context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - name: Extract ref info + id: release_info + run: | + echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/} + echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/} + + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "^1.19.5" + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build binaries for ${{ matrix.go-os }} + run: make frames-bin + env: + GOPATH: ${{ github.workspace }}/go + GOARCH: amd64 + GOOS: ${{ matrix.go-os }} + + - name: Upload binaries + uses: AButler/upload-release-assets@v2.0 + with: + files: ${{ github.workspace }}/go/bin/framesd-* + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 6b23702d..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,214 +0,0 @@ -label = "${UUID.randomUUID().toString()}" -git_project = "frames" -git_project_user = "v3io" -git_project_upstream_user = "v3io" -git_deploy_user = "iguazio-prod-git-user" -git_deploy_user_token = "iguazio-prod-git-user-token" -git_deploy_user_private_key = "iguazio-prod-git-user-private-key" - -podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang-python37") { - node("${git_project}-${label}") { - pipelinex = library(identifier: 'pipelinex@development', retriever: modernSCM( - [$class : 'GitSCMSource', - credentialsId: git_deploy_user_private_key, - remote : "git@github.com:iguazio/pipelinex.git"])).com.iguazio.pipelinex - common.notify_slack { - withCredentials([ - string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN'), - string(credentialsId: 'frames-ci-url', variable: 'FRAMES_CI_URL'), - usernamePassword(credentialsId: 'frames-ci-user-credentials', passwordVariable: 'FRAMES_CI_PASSWORD', usernameVariable: 'FRAMES_CI_USERNAME'), - ]) { - github.branch(git_deploy_user, git_project, git_project_user, git_project_upstream_user, true, GIT_TOKEN) { - parallel( - 'test-py': { - container('python37') { - stage('test-py') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("pip install pipenv") - common.shellc("make python-deps") - sh "make test-py" - } - } - } - }, - 'test-go': { - container('golang') { - stage('test-go') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - session = '{"url":"' + FRAMES_CI_URL + '","user":"' + FRAMES_CI_USERNAME + '","password":"' + FRAMES_CI_PASSWORD + '","container":"bigdata"}' - common.shellc("V3IO_SESSION='${session}' make test-go") - } - } - } - }, - 'make lint': { - container('golang') { - stage('make lint') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - sh "make lint" - } - } - } - } - ) - } - github.pr(git_deploy_user, git_project, git_project_user, git_project_upstream_user, true, GIT_TOKEN) { - parallel( - 'test-py': { - container('python37') { - stage('test-py') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("pip install pipenv") - common.shellc("make python-deps") - sh "make test-py" - } - } - } - }, - 'test-go': { - container('golang') { - stage('test-go') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - session = '{"url":"' + FRAMES_CI_URL + '","user":"' + FRAMES_CI_USERNAME + '","password":"' + FRAMES_CI_PASSWORD + '","container":"bigdata"}' - common.shellc("V3IO_SESSION='${session}' make test-go") - } - } - } - }, - 'make lint': { - container('golang') { - stage('make lint') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - sh "make lint" - } - } - } - } - ) - } - github.release(git_deploy_user, git_project, git_project_user, git_project_upstream_user, true, GIT_TOKEN) { - RELEASE_ID = github.get_release_id(git_project, git_project_user, "${github.TAG_VERSION}", GIT_TOKEN) - - parallel( - 'build linux binaries': { - container('golang') { - stage('build linux binaries') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("FRAMES_TAG=${github.TAG_VERSION} GOARCH=amd64 GOOS=linux make frames-bin") - } - } - } - }, - 'build darwin binaries': { - container('golang') { - stage('build darwin binaries') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("FRAMES_TAG=${github.TAG_VERSION} GOARCH=amd64 GOOS=darwin make frames-bin") - } - } - } - }, - 'build windows binaries': { - container('golang') { - stage('build windows binaries') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("FRAMES_TAG=${github.TAG_VERSION} GOARCH=amd64 GOOS=windows make frames-bin") - } - } - } - }, - 'build frames': { - container('docker-cmd') { - stage('build frames') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("FRAMES_REPOSITORY= FRAMES_TAG=${github.DOCKER_TAG_VERSION} make build") - } - } - } - }, - 'build framulate': { - container('docker-cmd') { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - common.shellc("FRAMES_REPOSITORY= FRAMES_TAG=${github.DOCKER_TAG_VERSION} make build-framulate") - } - } - }, - ) - - parallel( - 'upload linux binaries': { - container('jnlp') { - stage('upload linux binaries') { - github.upload_asset(git_project, git_project_user, "framesd-${github.TAG_VERSION}-linux-amd64", RELEASE_ID, GIT_TOKEN, "${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") - } - } - }, - 'upload linux binaries artifactory': { - container('jnlp') { - stage('upload linux binaries artifactory') { - withCredentials([ - string(credentialsId: pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[2], variable: 'PACKAGES_ARTIFACTORY_PASSWORD') - ]) { - common.upload_file_to_artifactory(pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[0], pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[1], PACKAGES_ARTIFACTORY_PASSWORD, "iguazio-devops/k8s", "framesd-${github.TAG_VERSION}-linux-amd64", "${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") - } - } - } - }, - 'upload darwin binaries': { - container('jnlp') { - stage('upload darwin binaries') { - github.upload_asset(git_project, git_project_user, "framesd-${github.TAG_VERSION}-darwin-amd64", RELEASE_ID, GIT_TOKEN, "${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") - } - } - }, - 'upload windows binaries': { - container('jnlp') { - stage('upload windows binaries') { - github.upload_asset(git_project, git_project_user, "framesd-${github.TAG_VERSION}-windows-amd64", RELEASE_ID, GIT_TOKEN, "${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") - } - } - }, - 'upload to pypi': { - container('python37') { - stage('upload to pypi') { - release_body = github.get_release_body("frames", git_project_user, github.TAG_VERSION, GIT_TOKEN) - if (release_body.startsWith("Autorelease")) { - echo "Autorelease is not uploading frames py to pypi." - } else if( "${github.TAG_VERSION}" != "unstable" ) { - withCredentials([ - usernamePassword(credentialsId: "iguazio-prod-pypi-credentials", passwordVariable: 'V3IO_PYPI_PASSWORD', usernameVariable: 'V3IO_PYPI_USER') - ]) { - dir("${github.BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") { - FRAMES_PYPI_VERSION = sh( - script: "echo ${github.DOCKER_TAG_VERSION} | awk -F - '{print \$1}'", - returnStdout: true - ).trim() - common.shellc("pip install pipenv") - common.shellc("make python-deps") - sh "make test-py" - try { - common.shellc("TRAVIS_REPO_SLUG=v3io/frames V3IO_PYPI_USER=${V3IO_PYPI_USER} V3IO_PYPI_PASSWORD=${V3IO_PYPI_PASSWORD} TRAVIS_TAG=${FRAMES_PYPI_VERSION} make pypi") - } catch (err) { - unstable("Failed uploading to pypi") - // Do not continue stages - throw err - } - } - } - } else { - echo "Uploading to pypi only stable version" - } - } - } - }, - ) - container('docker-cmd') { - dockerx.images_push_multi_registries( - ["frames:${github.DOCKER_TAG_VERSION}", "framulate:${github.DOCKER_TAG_VERSION}"], - [pipelinex.DockerRepo.ARTIFACTORY_IGUAZIO, pipelinex.DockerRepo.DOCKER_HUB, pipelinex.DockerRepo.QUAY_IO, pipelinex.DockerRepo.GCR_IO]) - } - } - } - } - } -}