From 8cd0d4441aa77af3156959da9ee6c2174a78703e Mon Sep 17 00:00:00 2001 From: Hiroto Funakoshi Date: Mon, 13 May 2024 12:31:20 +0900 Subject: [PATCH] Introduced two version support feature (#101) * feat: introduce two version support feature Signed-off-by: hlts2 * Update .github/workflows/release.yaml Signed-off-by: Hiroto Funakoshi * Update Makefile Signed-off-by: Hiroto Funakoshi * fix: bugfix pom update timing Signed-off-by: hlts2 * feat: add VALD_SHA Signed-off-by: hlts2 * fix: update secret name Signed-off-by: hlts2 * fix: enable cron and change workflow target branch Signed-off-by: hlts2 * fix: debug job for build and test and publish preapre Signed-off-by: hlts2 * Revert "fix: debug job for build and test and publish preapre" This reverts commit 73cc64e0dca9b061bc981b152c2e70c2d366f273. * Update .github/workflows/sync.yaml Signed-off-by: Hiroto Funakoshi --------- Signed-off-by: hlts2 Signed-off-by: Hiroto Funakoshi --- .github/workflows/build.yml | 222 --------------------------------- .github/workflows/deploy.yml | 85 ------------- .github/workflows/e2e.yaml | 31 +++++ .github/workflows/release.yaml | 30 +++++ .github/workflows/sync.yaml | 30 +++++ .github/workflows/update.yml | 67 ---------- .gitignore | 2 + Makefile | 88 ++++++++++++- VALD_SHA | 1 + 9 files changed, 180 insertions(+), 376 deletions(-) delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/sync.yaml delete mode 100644 .github/workflows/update.yml create mode 100644 VALD_SHA diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index d5db40a..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,222 +0,0 @@ -name: Build native image -on: - push: - branches: - - main - tags: - - '*.*.*' - - 'v*.*.*' - - '*.*.*-*' - - 'v*.*.*-*' - pull_request: - paths: - - .github/** - - src/** - - cmd/** - - project.clj - - Makefile - - VALD_CLIENT_CLJ_VERSION - -jobs: - build-linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install GraalVM - run: | - TMP_GRAALVM_HOME=/tmp/graalvm - GRAALVM_HOME=/tmp/graalvm - GRAALVM_TGZ_URI="https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAALVM_VERSION}/graalvm-ce-${JAVA_VERSION}-linux-amd64-${GRAALVM_VERSION}.tar.gz" - curl -sL $GRAALVM_TGZ_URI --output graalvm.tar.gz - mkdir -p $TMP_GRAALVM_HOME - tar -xf graalvm.tar.gz -C $TMP_GRAALVM_HOME --strip-components=1 - chmod -R a+rwx $TMP_GRAALVM_HOME - echo "PATH=$GRAALVM_HOME/bin:$PATH" >> $GITHUB_ENV - echo "GRAALVM_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - echo "JAVA_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - env: - GRAALVM_VERSION: 21.0.0 - JAVA_VERSION: java11 - - name: Install dependencies - run: | - gu install native-image - curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein - chmod a+x lein - sudo ./lein version - sudo apt-get update && sudo apt-get install -y upx - - name: Build native-image - run: | - make valdcli - # upx --lzma --best valdcli - chmod a+x valdcli - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: valdcli-linux - path: ./valdcli - build-linux-static: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install GraalVM - run: | - TMP_GRAALVM_HOME=/tmp/graalvm - GRAALVM_HOME=/tmp/graalvm - GRAALVM_TGZ_URI="https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAALVM_VERSION}/graalvm-ce-${JAVA_VERSION}-linux-amd64-${GRAALVM_VERSION}.tar.gz" - curl -sL $GRAALVM_TGZ_URI --output graalvm.tar.gz - mkdir -p $TMP_GRAALVM_HOME - tar -xf graalvm.tar.gz -C $TMP_GRAALVM_HOME --strip-components=1 - chmod -R a+rwx $TMP_GRAALVM_HOME - echo "PATH=$GRAALVM_HOME/bin:$PATH" >> $GITHUB_ENV - echo "GRAALVM_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - echo "JAVA_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - env: - GRAALVM_VERSION: 21.0.0 - JAVA_VERSION: java11 - - name: Install dependencies - run: | - gu install native-image - curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein - chmod a+x lein - sudo ./lein version - sudo apt-get update && sudo apt-get install -y upx - - name: Fetch bundle - run: | - curl -L -o musl.tar.gz https://github.com/gradinac/musl-bundle-example/releases/download/v1.0/musl.tar.gz - tar xvzf musl.tar.gz - mv bundle /tmp/musl - - name: Build musl-gcc - run: | - curl -o musl-${MUSL_VERSION}.tar.gz https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz - tar xzvf musl-${MUSL_VERSION}.tar.gz - cd musl-${MUSL_VERSION} - ./configure --disable-shared --prefix=/tmp/musl - make - make install - export PATH=$PATH:/tmp/musl/bin - echo "PATH=/tmp/musl/bin:$PATH" >> $GITHUB_ENV - env: - MUSL_VERSION: 1.2.1 - - name: Build static native-image - run: | - make OPTS="--static --libc=musl" valdcli - # upx --lzma --best valdcli - chmod a+x valdcli - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: valdcli-linux-static - path: ./valdcli - build-macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - name: Install GraalVM - run: | - TMP_GRAALVM_HOME=/tmp/graalvm - GRAALVM_HOME=/tmp/graalvm/Contents/Home - GRAALVM_TGZ_URI="https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAALVM_VERSION}/graalvm-ce-${JAVA_VERSION}-darwin-amd64-${GRAALVM_VERSION}.tar.gz" - curl -sL $GRAALVM_TGZ_URI --output graalvm.tar.gz - mkdir -p $TMP_GRAALVM_HOME - tar -xf graalvm.tar.gz -C $TMP_GRAALVM_HOME --strip-components=1 - chmod -R a+rwx $TMP_GRAALVM_HOME - echo "PATH=$GRAALVM_HOME/bin:$PATH" >> $GITHUB_ENV - echo "GRAALVM_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - echo "JAVA_HOME=$GRAALVM_HOME" >> $GITHUB_ENV - env: - GRAALVM_VERSION: 21.0.0 - JAVA_VERSION: java11 - - name: Install dependencies - run: | - brew install leiningen - gu install native-image - brew install upx - - name: Build native-image - run: | - make valdcli - # upx --lzma --best valdcli - chmod a+x valdcli - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: valdcli-macos - path: ./valdcli - release: - runs-on: ubuntu-latest - needs: - - build-linux - - build-linux-static - - build-macos - steps: - - uses: actions/checkout@v3 - - name: Create release - if: startsWith( github.ref, 'refs/tags/') - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - - name: mkdir - if: startsWith( github.ref, 'refs/tags/') - run: | - mkdir -p tmp/linux - mkdir -p tmp/linux-static - mkdir -p tmp/macos - - uses: actions/download-artifact@v3 - if: startsWith( github.ref, 'refs/tags/') - with: - name: valdcli-linux - path: tmp/linux - - uses: actions/download-artifact@v3 - if: startsWith( github.ref, 'refs/tags/') - with: - name: valdcli-linux-static - path: tmp/linux-static - - uses: actions/download-artifact@v3 - if: startsWith( github.ref, 'refs/tags/') - with: - name: valdcli-macos - path: tmp/macos - - name: zip - if: startsWith( github.ref, 'refs/tags/') - run: | - chmod a+x tmp/linux/valdcli - chmod a+x tmp/linux-static/valdcli - chmod a+x tmp/macos/valdcli - zip --junk-paths valdcli-linux tmp/linux/valdcli - zip --junk-paths valdcli-linux-static tmp/linux-static/valdcli - zip --junk-paths valdcli-macos tmp/macos/valdcli - - name: Upload Release Asset (linux) - if: startsWith( github.ref, 'refs/tags/') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./valdcli-linux.zip - asset_name: valdcli-linux.zip - asset_content_type: application/zip - - name: Upload Release Asset (linux-static) - if: startsWith( github.ref, 'refs/tags/') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./valdcli-linux-static.zip - asset_name: valdcli-linux-static.zip - asset_content_type: application/zip - - name: Upload Release Asset (macos) - if: startsWith( github.ref, 'refs/tags/') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./valdcli-macos.zip - asset_name: valdcli-macos.zip - asset_content_type: application/zip diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index c870de1..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: E2E test and Deploy to Clojars -on: - push: - branches: - - main - tags: - - '*.*.*' - - 'v*.*.*' - - '*.*.*-*' - - 'v*.*.*-*' - pull_request: - -jobs: - e2e: - name: Run E2E Tests - runs-on: ubuntu-latest - steps: - - name: Check out code. - uses: actions/checkout@v3 - - name: Install dependencies - run: | - curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein - chmod a+x lein - sudo ./lein version - - uses: rinx/setup-k3d@v0.0.4 - with: - version: latest - name: vald - agents: 1 - - name: check k3d - run: | - kubectl cluster-info - - uses: azure/setup-helm@v3 - - name: Helm version - run: | - helm version - - name: deploy Vald - run: | - helm repo add vald https://vald.vdaas.org/charts - helm install \ - --values ${VALUES} \ - --set defaults.image.tag=nightly \ - --set agent.ngt.dimension=300 \ - --set agent.ngt.auto_index_length=2 \ - --set agent.minReplicas=1 \ - --set gateway.lb.enabled=false \ - --set discoverer.enabled=false \ - --set manager.index.enabled=false \ - --generate-name vald/vald - sleep 3 - kubectl wait --for=condition=ready pod -l app=vald-agent --timeout=3m - kubectl get pods - env: - VALUES: https://raw.githubusercontent.com/vdaas/vald/main/.github/helm/values/values-lb.yaml - - name: Download data - run: | - curl -OL https://raw.githubusercontent.com/rinx/word2vecjson/master/data/wordvecs1000.json - - name: run tests - run: | - kubectl port-forward statefulset/vald-agent 8081:8081 & - pid=$! - - ./lein test - - kill $pid - deploy: - if: startsWith( github.ref, 'refs/tags/') - needs: - - e2e - name: Deploy to Clojars - runs-on: ubuntu-latest - steps: - - name: Check out code. - uses: actions/checkout@v3 - - name: Install dependencies - run: | - curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein - chmod a+x lein - sudo ./lein version - - name: Deploy - run: | - ./lein deploy clojars - env: - CLOJARS_USER: ${{ secrets.CLOJARS_USER }} - CLOJARS_PASS: ${{ secrets.CLOJARS_PASS }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..5b85dc9 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,31 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Run E2E test" +on: + push: + branches: + - main + pull_request: + +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: vdaas/vald-client-ci/.github/actions/e2e@main + with: + client_type: clj diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..64b3564 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,30 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Run release" +on: + push: + tags: + - '*.*.*' + - 'v*.*.*' + - '*.*.*-*' + - 'v*.*.*-*' +jobs: + release: + uses: vdaas/vald-client-ci/.github/workflows/release.yaml@main + with: + client_type: clj + secrets: inherit diff --git a/.github/workflows/sync.yaml b/.github/workflows/sync.yaml new file mode 100644 index 0000000..41e3646 --- /dev/null +++ b/.github/workflows/sync.yaml @@ -0,0 +1,30 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO NOT EDIT +name: "Sync Vald" +on: + workflow_dispatch: + schedule: + - cron: "0 */3 * * *" +jobs: + sync: + uses: vdaas/vald-client-ci/.github/workflows/sync.yaml@main + with: + client_type: clj + secrets: + CI_USER: ${{ secrets.VALDCLI_USER }} + CI_TOKEN: ${{ secrets.VALDCLI_TOKEN }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml deleted file mode 100644 index d3ca4f1..0000000 --- a/.github/workflows/update.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Update version -on: - schedule: - - cron: 0 * * * * - workflow_dispatch: - -jobs: - update-version: - name: update-version - runs-on: ubuntu-latest - steps: - - name: Check out code. - uses: actions/checkout@v3 - with: - fetch-depth: 0 - persist-credentials: false - - uses: crazy-max/ghaction-import-gpg@v4 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - git_user_signingkey: true - git_commit_gpgsign: true - - name: Run gitwerk semver-auto - run: | - CURRENT_VERSION=`cat VALD_CLIENT_CLJ_VERSION` - GITHUB_LATEST_VERSION=`curl -s https://api.github.com/repos/vdaas/vald-client-java/releases/latest | jq -r '.tag_name'` - if [ "" = "${GITHUB_LATEST_VERSION}" ]; then - echo "the version is empty" - exit 0 - fi - - if [ "${CURRENT_VERSION}" = "v${GITHUB_LATEST_VERSION}" ]; then - echo "vald-client-java latest version in GitHub is ${GITHUB_LATEST_VERSION}. GitHub is not updated" - exit 0 - fi - - STATUS_CODE=`curl -fsSLI https://repo1.maven.org/maven2/org/vdaas/vald/vald-client-java/${GITHUB_LATEST_VERSION} -o /dev/null -w '%{http_code}' 2> /dev/null` - if [ "${STATUS_CODE}" != "200" ]; then - echo "vald-client-java has not been uploaded to maven repository yet" - exit 0 - fi - - VERSION="v${GITHUB_LATEST_VERSION}" - - echo "${VERSION}" > VALD_CLIENT_CLJ_VERSION - - curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein - chmod a+x lein - ./lein pom - rm lein - - git config --global user.name "vdaas-ci" - git config --global user.email "vald@vdaas.org" - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - git checkout main - git add VALD_CLIENT_CLJ_VERSION - git add pom.xml - git commit --signoff -m ":bookmark: Release ${VERSION}" - - git tag ${VERSION} - git remote set-url origin "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - git push origin main - git push origin ${VERSION} - env: - GITHUB_USER: ${{ secrets.VALDCLI_USER }} - GITHUB_TOKEN: ${{ secrets.VALDCLI_TOKEN }} diff --git a/.gitignore b/.gitignore index 511ba82..79f25a0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,7 @@ pom.xml.asc .hgignore .hg/ /valdcli +/vald +lein wordvecs1000.json diff --git a/Makefile b/Makefile index 61625fc..533f369 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,28 @@ XMS = 2g XMX = 6g +REPO ?= vdaas +NAME = vald +VALDREPO = github.com/$(REPO)/$(NAME) +LANGUAGE = clj +PKGNAME = $(NAME)-client-$(LANGUAGE) +PKGREPO = github.com/$(REPO)/$(PKGNAME) + +VALD_DIR = vald +VALD_SHA = VALD_SHA +VALD_CLIENT_CLJ_VERSION = VALD_CLIENT_CLJ_VERSION +VALD_CHECKOUT_REF ?= main + VERSION=$(shell cat VALD_CLIENT_CLJ_VERSION) NATIVE_IMAGE_CONFIG_OUTPUT_DIR=native-config TARGET_JAR=target/vald-client-clj-$(VERSION)-standalone.jar +TEST_DATASET_PATH = wordvecs1000.json + +LEIN_PATH = ./lein + .PHONY: all all: clean @@ -43,12 +59,12 @@ profile/native-image-config: \ $(NATIVE_IMAGE_CONFIG_OUTPUT_DIR): mkdir -p $@ -lein: +$(LEIN_PATH): curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein \ && chmod a+x lein \ && ./lein version -$(TARGET_JAR): src cmd +$(TARGET_JAR): $(LEIN_PATH) src cmd lein with-profile +cmd uberjar valdcli: $(TARGET_JAR) @@ -73,3 +89,71 @@ valdcli: $(TARGET_JAR) -J-Dclojure.spec.skip-macros=true \ -J-Xms$(XMS) \ -J-Xmx$(XMX) + +$(VALD_DIR): + git clone https://$(VALDREPO) $(VALD_DIR) + +.PHONY: pom +## update dependencies +pom: $(LEIN_PATH) + ./lein pom + +.PHONY: proto +## build proto +proto: + @echo "Nothing to do" + +.PHONY: vald/checkout +## checkout vald repository +vald/checkout: $(VALD_DIR) + cd $(VALD_DIR) && git checkout $(VALD_CHECKOUT_REF) + +.PHONY: vald/origin/sha/print +## print origin VALD_SHA value +vald/origin/sha/print: $(VALD_DIR) + @cd $(VALD_DIR) && git rev-parse HEAD | tr -d '\n' + +.PHONY: vald/sha/print +## print VALD_SHA value +vald/sha/print: + @cat $(VALD_SHA) + +.PHONY: vald/sha/update +## update VALD_SHA value +vald/sha/update: $(VALD_DIR) + (cd $(VALD_DIR); git rev-parse HEAD | tr -d '\n' > ../$(VALD_SHA)) + +.PHONY: vald/client/version/print +## print VALD_CLIENT_JAVA_VERSION value +vald/client/version/print: + @cat $(VALD_CLIENT_CLJ_VERSION) + +.PHONY: vald/client/version/update +## update VALD_CLIENT_JAVA_VERSION value +vald/client/version/update: $(VALD_DIR) + cp $(VALD_DIR)/versions/VALD_VERSION $(VALD_CLIENT_CLJ_VERSION) + +.PHONY: test +## Execute test +test: $(LEIN_PATH) $(TEST_DATASET_PATH) + ./lein test + +$(TEST_DATASET_PATH): + curl -L https://raw.githubusercontent.com/rinx/word2vecjson/master/data/wordvecs1000.json -o $(TEST_DATASET_PATH) + +.PHONY: ci/deps/install +## install deps for CI environment +ci/deps/install: $(LEIN_PATH) + +.PHONY: ci/deps/update +## update deps for CI environment +ci/deps/update: pom + +.PHONY: ci/package/prepare +## prepare for publich +ci/package/prepare: ci/deps/install + +.PHONY: ci/package/publish +## publich packages +ci/package/publish: ci/deps/install + ./lein deploy clojars diff --git a/VALD_SHA b/VALD_SHA new file mode 100644 index 0000000..11ece9e --- /dev/null +++ b/VALD_SHA @@ -0,0 +1 @@ +c39130028731aefeb84d03c885e50791b92a8ff3