From 277212b674e8ec69396c1494db4d9d3f6c6021d5 Mon Sep 17 00:00:00 2001 From: Duc Duong <112845152+spartan-ductduong@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:58:40 +0700 Subject: [PATCH 01/15] Set up CI/CD with Jenkins (#1) --- .github/pull_request_template.md | 55 +++++ .github/workflows/auto-generate-release.yml | 51 +++++ .github/workflows/create-major-tag.yml | 45 ++++ .github/workflows/create-minor-tag.yml | 45 ++++ .github/workflows/create-patch-tag.yml | 45 ++++ .github/workflows/deploy-dev.yaml | 221 ++++++++++++++++++++ .github/workflows/deploy-prod-az.yaml | 189 +++++++++++++++++ .github/workflows/docker-publish.yml | 98 --------- .github/workflows/python-publish.yml | 39 ---- Jenkinsfile | 41 ++++ k8s/dev/values.yaml | 188 +++++++++++++++++ k8s/prod-az/values.yaml | 185 ++++++++++++++++ k8s/prod/values.yaml | 185 ++++++++++++++++ 13 files changed, 1250 insertions(+), 137 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/auto-generate-release.yml create mode 100644 .github/workflows/create-major-tag.yml create mode 100644 .github/workflows/create-minor-tag.yml create mode 100644 .github/workflows/create-patch-tag.yml create mode 100644 .github/workflows/deploy-dev.yaml create mode 100644 .github/workflows/deploy-prod-az.yaml delete mode 100644 .github/workflows/docker-publish.yml delete mode 100644 .github/workflows/python-publish.yml create mode 100644 Jenkinsfile create mode 100644 k8s/dev/values.yaml create mode 100644 k8s/prod-az/values.yaml create mode 100644 k8s/prod/values.yaml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..bb5621e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,55 @@ +## Summary + + +### Why + + +### What + + +### Solution + + +## Types of Changes + + +- [ ] ❌ Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] 🚀 New feature (non-breaking change which adds functionality) +- [ ] 🕷 Bug fix (non-breaking change which fixes an issue) +- [ ] 👏 Performance optimization (non-breaking change which addresses a performance issue) +- [ ] 🛠 Refactor (non-breaking change which does not change existing behavior or add new functionality) +- [ ] 📗 Library update (non-breaking change that will update one or more libraries to newer versions) +- [ ] 📝 Documentation (non-breaking change that doesn't change code behavior, can skip testing) +- [ ] ✅ Test (non-breaking change related to testing) +- [ ] 🔒 Security awareness (changes that effect permission scope, security scenarios) + +## Database and Logic Impact + + +- [ ] This update includes DB typing or schema updates that would require a potential DB Migration +- [ ] This update includes logical updates such as changing optional typing to strict typing of core functionality of the SmartSheet/Columns/Enrichments that would require changes fro the User of the product. + - For example, if the user has to re-create a column. This is a breaking change. + +## Test Plan + + +- Test scope: + +- Test steps: + 1. + 2. +- Test scenarios: + - [ ] Verify that users are able to + - [ ] Verify that users are able to + - [ ] Verify that users are able to +- Environment Testing: + + +### Evidence + + +## Related Issues + diff --git a/.github/workflows/auto-generate-release.yml b/.github/workflows/auto-generate-release.yml new file mode 100644 index 0000000..d429a64 --- /dev/null +++ b/.github/workflows/auto-generate-release.yml @@ -0,0 +1,51 @@ +name: Auto Generate Release +run-name: Automatically generate a new release on GitHub + +permissions: + contents: read + +on: + push: + tags: + - v[0-9]+.[0-9]+.* + +jobs: + create-release: + if: github.repository_owner == 'c0x12c' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + token: ${{ steps.github-app-token.outputs.token }} + ref: ${{ github.head_ref }} + + - name: Check if CHANGELOG.md contains "## [Unreleased]" + id: check_changelog + run: | + if grep -q '## \[Unreleased\]' CHANGELOG.md; then + echo "Changelog contains '## [Unreleased]'. Skipping the pipeline." + exit 0 + else + echo "Changelog does not contain '## [Unreleased]'. Continuing the pipeline." + fi + continue-on-error: true + + - uses: taiki-e/create-gh-release-action@v1 + if: ${{ success() }} + with: + changelog: CHANGELOG.md + title: $version + branch: 'master|v[0-9]+' + token: ${{ steps.github-app-token.outputs.token }} diff --git a/.github/workflows/create-major-tag.yml b/.github/workflows/create-major-tag.yml new file mode 100644 index 0000000..581328f --- /dev/null +++ b/.github/workflows/create-major-tag.yml @@ -0,0 +1,45 @@ +name: Create Major Tag + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: 'read' + id-token: 'write' + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + token: ${{ steps.github-app-token.outputs.token }} + ref: ${{ github.head_ref }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install tomlkit + + - name: Bump version + run: | + git config --global --add safe.directory /harness + git config --global user.email "github-actions@github.com" + git config --global user.name "github-actions" + + ./tools/create_release.sh major + env: + GITHUB_TOKEN: ${{ steps.github-app-token.outputs.token }} diff --git a/.github/workflows/create-minor-tag.yml b/.github/workflows/create-minor-tag.yml new file mode 100644 index 0000000..5063672 --- /dev/null +++ b/.github/workflows/create-minor-tag.yml @@ -0,0 +1,45 @@ +name: Create Minor Tag + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: 'read' + id-token: 'write' + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + token: ${{ steps.github-app-token.outputs.token }} + ref: ${{ github.head_ref }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install tomlkit + + - name: Bump version + run: | + git config --global --add safe.directory /harness + git config --global user.email "github-actions@github.com" + git config --global user.name "github-actions" + + ./tools/create_release.sh minor + env: + GITHUB_TOKEN: ${{ steps.github-app-token.outputs.token }} diff --git a/.github/workflows/create-patch-tag.yml b/.github/workflows/create-patch-tag.yml new file mode 100644 index 0000000..9074dd2 --- /dev/null +++ b/.github/workflows/create-patch-tag.yml @@ -0,0 +1,45 @@ +name: Create Patch Tag + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: 'read' + id-token: 'write' + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + token: ${{ steps.github-app-token.outputs.token }} + ref: ${{ github.head_ref }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install tomlkit + + - name: Bump version + run: | + git config --global --add safe.directory /harness + git config --global user.email "github-actions@github.com" + git config --global user.name "github-actions" + + ./tools/create_release.sh patch + env: + GITHUB_TOKEN: ${{ steps.github-app-token.outputs.token }} diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml new file mode 100644 index 0000000..f21215f --- /dev/null +++ b/.github/workflows/deploy-dev.yaml @@ -0,0 +1,221 @@ +name: DEV - Deploy + +on: + push: + branches: [ "main" ] + paths: + - 'k8s/dev/*' + - 'src/**/*' + - 'Dockerfile' + - 'poetry.lock' + workflow_dispatch: + +env: + AWS_EKS_CLUSTER_NAME: ${{ vars.AWS_EKS_CLUSTER_NAME_DEV }} + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID_DEV }} + AWS_REGION: ${{ vars.AWS_REGION_DEV }} + AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME_DEV }} + DD_AGENT_IMAGE: ${{ vars.DD_AGENT_IMAGE_DEV }} + DOCKER_PLATFORM: "linux/amd64" + DOCKER_REPO: "${{ vars.AWS_ACCOUNT_ID_DEV }}.dkr.ecr.${{ vars.AWS_REGION_DEV }}.amazonaws.com/${{ vars.SERVICE_NAME }}" + ENVIRONMENT: dev + HELM_REPO: ${{ vars.HELM_REPO }} + HELM_VERSION: "1.1.2" + SERVICE_NAME: ${{ vars.SERVICE_NAME }} + BASE_DOMAIN: ${{ vars.BASE_DOMAIN_DEV }} + +jobs: + check_disk_space: + name: Check Disk Space + runs-on: self-hosted + steps: + - name: Check available disk space + id: check_space + run: | + # Get the available disk space in GB + available_space=$(df -BG / | tail -1 | awk '{print $4}' | sed 's/G//') + echo "Available space: ${available_space}GB" + # Set an output for the condition in the next step + echo "space=${available_space}" >> $GITHUB_OUTPUT + + - name: Prune Docker if space is low + if: ${{ steps.check_space.outputs.space < 10 }} # Condition to check if space is less than 10GB + run: | + echo "Disk space is low (${available_space}GB). Running docker system prune..." + docker system prune -f + shell: bash + + build_and_push_images: + name: Build and Push Docker Images + runs-on: ["self-hosted", "Linux", "ARM64"] + needs: [check_disk_space] + + permissions: + contents: "read" + id-token: write + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + owner: ${{ github.repository_owner }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + fetch-depth: 3 + ref: ${{ github.head_ref }} + token: ${{ steps.github-app-token.outputs.token }} + # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + +# FIXME: consolidate SQL script files so that we can enable this step +# - name: Run migration filename validation +# run: | +# python src/utils/validate_migration_filenames.py + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_TO_ASSUME }} + role-session-name: gh-actions + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Prepare docker image tags + run: | + SHORT_SHA=$(echo $GITHUB_SHA | cut -c1-8) + CURRENT_DATE=$(date +%Y%m%d) + IMAGE_TAG="${{ env.ENVIRONMENT }}-$CURRENT_DATE-$SHORT_SHA" + + echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV + + - name: Extract project version from the pyproject.toml file + run: | + APP_VERSION=$(grep '^version =' pyproject.toml | awk -F'"' '{print $2}') + echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_ENV + + - name: Build and Push the Docker image + run: | + docker buildx build \ + --platform "$DOCKER_PLATFORM" \ + -t "$DOCKER_REPO":"$IMAGE_TAG" \ + --build-arg APP_VERSION=$APP_VERSION \ + --push . + + - name: Build and Push the flyway Docker image + working-directory: src/service_platform/sql + run: | + DOCKER_MIGRATION_IMAGE_TAG="$IMAGE_TAG"-migration + docker buildx build \ + --platform "$DOCKER_PLATFORM" \ + -t "$DOCKER_REPO":"$DOCKER_MIGRATION_IMAGE_TAG" \ + -f Dockerfile.migration \ + --push . + + - name: "Update values for k8s files" + uses: cschleiden/replace-tokens@v1 + with: + tokenPrefix: "__" + tokenSuffix: "__" + files: '["k8s/dev/**/*.yaml"]' + + - name: Configure Kubernetes client + uses: silverlyra/setup-aws-eks@v0.1 + with: + cluster: ${{ env.AWS_EKS_CLUSTER_NAME }} + + - name: "Helm Install for Service API" + run: | + helm repo add spartan https://x-access-token:${{ steps.github-app-token.outputs.token }}@${{ env.HELM_REPO }} + helm upgrade --install --wait --timeout 600s \ + --namespace ${{ env.SERVICE_NAME }} \ + -f k8s/${{ env.ENVIRONMENT }}/values.yaml \ + --version ${{ env.HELM_VERSION }} \ + ${{ env.SERVICE_NAME }} spartan/spartan + + - name: "Extract Git info" + run: | + echo "GITHUB_AUTHOR=$(git log -1 --pretty=format:'%an <%ae>' | xargs)" >> $GITHUB_ENV + echo "GITHUB_REVISION=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "GITHUB_REPO=$(git config --get remote.origin.url)" >> $GITHUB_ENV + + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + GITHUB_COMMITS=$(git --no-pager log --pretty=format:'%h (%an) %s' -n3 --no-color | while read line || [ -n "$line" ]; do echo -n "- $line \\n"; done) + echo "GITHUB_COMMITS<<$EOF" >> "$GITHUB_ENV" + echo $GITHUB_COMMITS | sed "s/\"/'/g" >> "$GITHUB_ENV" + echo "$EOF" >> "$GITHUB_ENV" + + - name: Slack Notification + id: slack + uses: slackapi/slack-github-action@v1 + with: + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ env.SERVICE_NAME }} - Deploy to DEV*" + } + },{ + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Created by:* ${{ env.GITHUB_AUTHOR }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*API Docs:* https://${{ env.SERVICE_NAME }}.${{ env.BASE_DOMAIN }}/api/docs" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "```${{ env.GITHUB_COMMITS }}```" + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": "View changes" + }, + "style": "primary", + "url": "${{ env.GITHUB_REPO }}/commit/${{ env.GITHUB_REVISION}}" + }, + { + "type": "button", + "text": { + "type": "plain_text", + "text": "View Run" + }, + "style": "primary", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + ] + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/deploy-prod-az.yaml b/.github/workflows/deploy-prod-az.yaml new file mode 100644 index 0000000..e88c5d4 --- /dev/null +++ b/.github/workflows/deploy-prod-az.yaml @@ -0,0 +1,189 @@ +name: Prod AZ - Deploy + +on: + push: + tags: + - 'v*.*.*' + +env: + AWS_EKS_CLUSTER_NAME: ${{ vars.AWS_EKS_CLUSTER_NAME_PROD_AZ }} + AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID_PROD_AZ }} + AWS_REGION: "us-west-2" + AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME_PROD_AZ }} + DD_AGENT_IMAGE: ${{ vars.DD_AGENT_IMAGE_PROD_AZ }} + DOCKER_PLATFORM: "linux/amd64" + DOCKER_REPO: "${{ vars.AWS_ACCOUNT_ID_PROD_AZ }}.dkr.ecr.${{ vars.AWS_REGION_PROD_AZ }}.amazonaws.com" + DOCKER_REPO_DEV: "${{ vars.AWS_ACCOUNT_ID_DEV }}.dkr.ecr.${{ vars.AWS_REGION_DEV }}.amazonaws.com" + ENVIRONMENT: prod-az + HELM_REPO: ${{ vars.HELM_REPO }} + HELM_VERSION: "1.1.2" + SERVICE_NAME: ${{ vars.SERVICE_NAME }} + BASE_DOMAIN: ${{ vars.BASE_DOMAIN_PROD_AZ }} + +jobs: + build_and_push_images: + name: Build and Push Docker Images + runs-on: ubuntu-latest + + permissions: + contents: "read" + id-token: write + + steps: + - name: Create GitHub App Token + uses: actions/create-github-app-token@v1 + id: github-app-token + with: + app-id: ${{ vars.APP_ID }} + owner: ${{ github.repository_owner }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Checkout source + uses: actions/checkout@v4 + with: + fetch-depth: 3 + ref: ${{ github.head_ref }} + token: ${{ steps.github-app-token.outputs.token }} + # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Prepare docker image tags + run: | + IMAGE_TAG=$(echo $GITHUB_SHA | cut -c1-8) + echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV + echo "SERVICE_IMAGE_DEV=$DOCKER_REPO_DEV/$SERVICE_NAME:$IMAGE_TAG" >> $GITHUB_ENV + echo "SERVICE_IMAGE=$DOCKER_REPO/$SERVICE_NAME:$IMAGE_TAG" >> $GITHUB_ENV + + - name: Configure AWS Credentials for DEV + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_DEV }} + role-session-name: gh-actions + aws-region: ${{ vars.AWS_REGION_DEV }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Pull images from DEV for promotion to Prod AZ + run: | + docker pull "$SERVICE_IMAGE_DEV" + docker pull "$SERVICE_IMAGE_DEV"-migration + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_TO_ASSUME }} + role-session-name: gh-actions + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Tag and Push the Docker images + run: | + docker tag "$SERVICE_IMAGE_DEV" "$SERVICE_IMAGE" + docker push "$SERVICE_IMAGE" + + docker tag "$SERVICE_IMAGE_DEV"-migration "$SERVICE_IMAGE"-migration + docker push "$SERVICE_IMAGE"-migration + + - name: "Update values for k8s files" + uses: cschleiden/replace-tokens@v1 + with: + tokenPrefix: "__" + tokenSuffix: "__" + files: '["k8s/${{ env.ENVIRONMENT }}/**/*.yaml"]' + + - name: Configure Kubernetes client + uses: silverlyra/setup-aws-eks@v0.1 + with: + cluster: ${{ env.AWS_EKS_CLUSTER_NAME }} + + - name: "Helm Install for Service API" + run: | + helm repo add spartan https://x-access-token:${{ steps.github-app-token.outputs.token }}@${{ env.HELM_REPO }} + helm upgrade --install --wait --timeout 600s \ + --namespace ${{ env.SERVICE_NAME }} \ + -f k8s/${{ env.ENVIRONMENT }}/values.yaml \ + --version ${{ env.HELM_VERSION }} \ + ${{ env.SERVICE_NAME }} spartan/spartan + + - name: "Extract Git info" + run: | + echo "GITHUB_AUTHOR=$(git log -1 --pretty=format:'%an <%ae>' | xargs)" >> $GITHUB_ENV + echo "GITHUB_REVISION=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo "GITHUB_REPO=$(git config --get remote.origin.url)" >> $GITHUB_ENV + + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + GITHUB_COMMITS=$(git --no-pager log --pretty=format:'%h (%an) %s' -n3 --no-color | while read line || [ -n "$line" ]; do echo -n "- $line \\n"; done) + echo "GITHUB_COMMITS<<$EOF" >> "$GITHUB_ENV" + echo $GITHUB_COMMITS | sed "s/\"/'/g" >> "$GITHUB_ENV" + echo "$EOF" >> "$GITHUB_ENV" + + - name: Slack Notification + id: slack + uses: slackapi/slack-github-action@v1 + with: + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ env.SERVICE_NAME }} - Deploy to PROD AstraZeneca*" + } + },{ + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*Created by:* ${{ env.GITHUB_AUTHOR }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*API Docs:* https://${{ env.SERVICE_NAME }}.${{ env.BASE_DOMAIN }}/api/docs" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "```${{ env.GITHUB_COMMITS }}```" + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": "View changes" + }, + "style": "primary", + "url": "${{ env.GITHUB_REPO }}/commit/${{ env.GITHUB_REVISION}}" + }, + { + "type": "button", + "text": { + "type": "plain_text", + "text": "View Run" + }, + "style": "primary", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + ] + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index fed119c..0000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Docker - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -on: - release: - types: [published] - -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - - -jobs: - build: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - # This is used to complete the identity challenge - # with sigstore/fulcio when running outside of PRs. - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Install the cosign tool except on PR - # https://github.com/sigstore/cosign-installer - - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v3.5.0 - with: - cosign-release: 'v2.2.4' - - # Setup QEMU for multi-platform build support - # https://docs.docker.com/build/ci/github-actions/multi-platform/ - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 #v3.0.0 - - # Set up BuildKit Docker container builder to be able to build - # multi-platform images and export cache - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 - - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - # Sign the resulting Docker image digest except on PRs. - # This will only write to the public Rekor transparency log when the Docker - # repository is public to avoid leaking data. If you would like to publish - # transparency data even for private images, pass --force to cosign below. - # https://github.com/sigstore/cosign - - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} - env: - # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable - TAGS: ${{ steps.meta.outputs.tags }} - DIGEST: ${{ steps.build-and-push.outputs.digest }} - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index bdaab28..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..2d34c73 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,41 @@ +@Library('atrix@master') _ + +makeBuildPipeline { + serviceConfigurations = [ + name : 'nlm-ingestor', + clusterNamePrefix : 'atrix-eks-', + helmRepo : 'spartan', + chartPath : 'spartan/spartan', + chartVersion : '1.1.3', + namespace : 'service-platform' + ] + + dockerFilePath = '/' + dockerFileName = 'Dockerfile' + + nodeBuildLabel = 'heavy' + + helmStageTimeout = 30 + + testCommand = 'jenkins-test' + + informStageEnabled = true + + codeQualityStageEnabled = false + + promoteImageEnabled = true + + devDeploymentEnabled = { ctx, buildEnv -> + buildEnv.getBranchName() == "main" || buildEnv.getBranchName() == "master" + } + + additionalContainerConfig = { ctx, buildEnv -> + if (buildEnv.isPullRequestBuild()) { + [] + } else { + [ + kaniko: [:] + ] + } + } +} diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml new file mode 100644 index 0000000..c1ef278 --- /dev/null +++ b/k8s/dev/values.yaml @@ -0,0 +1,188 @@ +replicaCount: 1 +image: + repository: __DOCKER_REPO__/__SERVICE_NAME__ + pullPolicy: IfNotPresent + tag: "__IMAGE_TAG__" + +containerPort: 9090 + +imagePullSecrets: [ ] +nameOverride: "" +fullnameOverride: "__SERVICE_NAME__" +containerName: "__SERVICE_NAME__" + +serviceAccount: + create: false + annotations: { } + name: "__SERVICE_NAME__" + +podAnnotations: + ad.datadoghq.com/__SERVICE_NAME__.check_names: '["__SERVICE_NAME__"]' + ad.datadoghq.com/__SERVICE_NAME__.init_configs: '[{}]' + ad.datadoghq.com/__SERVICE_NAME__.instances: | + [ + { + "host": "%%host%%", + "port": "9090" + } + ] + ad.datadoghq.com/__SERVICE_NAME__.logs: | + [{ + "type": "file", + "path": "/var/log/application/app.log", + "source": "eks-cluster", + "service": "__SERVICE_NAME__", + "env" : "__ENVIRONMENT__", + "auto_multi_line_detection": true + }] +podSecurityContext: { } + +securityContext: { } + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: true + className: "alb" + annotations: + alb.ingress.kubernetes.io/group.name: "external" + alb.ingress.kubernetes.io/healthcheck-path: "/api/health/" + alb.ingress.kubernetes.io/listen-ports: "[{\"HTTP\": 80}, {\"HTTPS\": 443}]" + alb.ingress.kubernetes.io/scheme: "internet-facing" + alb.ingress.kubernetes.io/target-type: "ip" + kubernetes.io/ingress.class: "alb" + hosts: + - host: "__SERVICE_NAME__.__BASE_DOMAIN__" + paths: + - path: /* + pathType: ImplementationSpecific + +livenessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 +readinessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 + +resources: + limits: + cpu: 950m + memory: 3Gi + ephemeral-storage: 50Gi + requests: + cpu: 950m + memory: 3Gi + ephemeral-storage: 50Gi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 600 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 30 + +nodeSelector: {} + # usage: gpu + # team: ml-team + # kubernetes.io/instance-type: c8g.medium + +tolerations: [] +# - key: "doc_parser" +# operator: "Equal" +# value: "true" +# effect: "NoSchedule" + +affinity: { } + +testConnection: + enabled: false + +secret: + externalSecretEnv: + enabled: true + name: __SERVICE_NAME__-secret + +configMap: + externalConfigMapEnv: + enabled: true + name: __SERVICE_NAME__-config-map + +extraEnvs: + - name: DD_APM_FILTER_TAGS_REGEX_REJECT + value: "http.route:/health" + - name: DD_CLUSTER_AGENT_URL + value: "https://datadog-cluster-agent.datadog:5005" + - name: DD_EKS_FARGATE + value: "true" + - name: DD_ENV + value: "__ENVIRONMENT__" + - name: DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL + value: "true" + - name: DD_LOGS_ENABLED + value: "true" + - name: DD_LOGS_INJECTION + value: "true" + - name: DD_PROCESS_AGENT_ENABLED + value: "true" + - name: DD_PROFILING_ENABLED + value: "true" + - name: DD_SERVICE + value: "__SERVICE_NAME__" + - name: DD_VERSION + value: "__DD_VERSION__" + - name: LOG_FILE + value: "/var/log/application/app.log" + +sidecars: + - name: datadog-agent + image: "__DD_AGENT_IMAGE__" + ports: + - containerPort: 8125 + name: dogstatsdport + protocol: UDP + - containerPort: 8126 + name: traceport + protocol: TCP + sharedVolume: + mountPath: /var/log/application/ + resources: + limits: + cpu: 200m + memory: 0.25Gi + requests: + cpu: 200m + memory: 0.25Gi + +datadog: + enabled: true + +sentry: + enabled: true diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml new file mode 100644 index 0000000..40f33d2 --- /dev/null +++ b/k8s/prod-az/values.yaml @@ -0,0 +1,185 @@ +replicaCount: 1 +image: + repository: __DOCKER_REPO__/__SERVICE_NAME__ + pullPolicy: IfNotPresent + tag: "__IMAGE_TAG__" + +containerPort: 9090 + +imagePullSecrets: [ ] +nameOverride: "" +fullnameOverride: "__SERVICE_NAME__" +containerName: "__SERVICE_NAME__" + +serviceAccount: + create: false + annotations: { } + name: "__SERVICE_NAME__" + +podAnnotations: + ad.datadoghq.com/__SERVICE_NAME__.check_names: '["__SERVICE_NAME__"]' + ad.datadoghq.com/__SERVICE_NAME__.init_configs: '[{}]' + ad.datadoghq.com/__SERVICE_NAME__.instances: | + [ + { + "host": "%%host%%", + "port": "9090" + } + ] + ad.datadoghq.com/__SERVICE_NAME__.logs: | + [{ + "type": "file", + "path": "/var/log/application/app.log", + "source": "eks-cluster", + "service": "__SERVICE_NAME__", + "env" : "__ENVIRONMENT__", + "auto_multi_line_detection": true + }] +podSecurityContext: { } + +securityContext: { } + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: true + className: "alb" + annotations: + alb.ingress.kubernetes.io/group.name: "external" + alb.ingress.kubernetes.io/healthcheck-path: "/api/health/" + alb.ingress.kubernetes.io/listen-ports: "[{\"HTTP\": 80}, {\"HTTPS\": 443}]" + alb.ingress.kubernetes.io/scheme: "internet-facing" + alb.ingress.kubernetes.io/target-type: "ip" + kubernetes.io/ingress.class: "alb" + hosts: + - host: "__SERVICE_NAME__.__BASE_DOMAIN__" + paths: + - path: /* + pathType: ImplementationSpecific + +livenessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 +readinessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 + +resources: + limits: + cpu: 1950m + memory: 6Gi + ephemeral-storage: 50Gi + requests: + cpu: 1950m + memory: 6Gi + ephemeral-storage: 50Gi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 600 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 30 + +nodeSelector: {} + # usage: gpu + # team: ml-team + # kubernetes.io/instance-type: c8g.medium + +tolerations: [] +# - key: "doc_parser" +# operator: "Equal" +# value: "true" +# effect: "NoSchedule" + +affinity: { } + +testConnection: + enabled: false + +secret: + externalSecretEnv: + enabled: true + name: __SERVICE_NAME__-secret + +configMap: + externalConfigMapEnv: + enabled: true + name: __SERVICE_NAME__-config-map + +extraEnvs: + - name: DD_APM_FILTER_TAGS_REGEX_REJECT + value: "http.route:/health" + - name: DD_CLUSTER_AGENT_URL + value: "https://datadog-cluster-agent.datadog:5005" + - name: DD_EKS_FARGATE + value: "true" + - name: DD_ENV + value: "__ENVIRONMENT__" + - name: DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL + value: "true" + - name: DD_LOGS_ENABLED + value: "true" + - name: DD_LOGS_INJECTION + value: "true" + - name: DD_PROCESS_AGENT_ENABLED + value: "true" + - name: DD_PROFILING_ENABLED + value: "true" + - name: DD_SERVICE + value: "__SERVICE_NAME__" + - name: DD_VERSION + value: "__DD_VERSION__" + - name: LOG_FILE + value: "/var/log/application/app.log" + +sidecars: + - name: datadog-agent + image: "__DD_AGENT_IMAGE__" + ports: + - containerPort: 8125 + name: dogstatsdport + protocol: UDP + - containerPort: 8126 + name: traceport + protocol: TCP + sharedVolume: + mountPath: /var/log/application/ + resources: + limits: + cpu: 200m + memory: 0.25Gi + requests: + cpu: 200m + memory: 0.25Gi + +datadog: + enabled: true diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml new file mode 100644 index 0000000..40f33d2 --- /dev/null +++ b/k8s/prod/values.yaml @@ -0,0 +1,185 @@ +replicaCount: 1 +image: + repository: __DOCKER_REPO__/__SERVICE_NAME__ + pullPolicy: IfNotPresent + tag: "__IMAGE_TAG__" + +containerPort: 9090 + +imagePullSecrets: [ ] +nameOverride: "" +fullnameOverride: "__SERVICE_NAME__" +containerName: "__SERVICE_NAME__" + +serviceAccount: + create: false + annotations: { } + name: "__SERVICE_NAME__" + +podAnnotations: + ad.datadoghq.com/__SERVICE_NAME__.check_names: '["__SERVICE_NAME__"]' + ad.datadoghq.com/__SERVICE_NAME__.init_configs: '[{}]' + ad.datadoghq.com/__SERVICE_NAME__.instances: | + [ + { + "host": "%%host%%", + "port": "9090" + } + ] + ad.datadoghq.com/__SERVICE_NAME__.logs: | + [{ + "type": "file", + "path": "/var/log/application/app.log", + "source": "eks-cluster", + "service": "__SERVICE_NAME__", + "env" : "__ENVIRONMENT__", + "auto_multi_line_detection": true + }] +podSecurityContext: { } + +securityContext: { } + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: true + className: "alb" + annotations: + alb.ingress.kubernetes.io/group.name: "external" + alb.ingress.kubernetes.io/healthcheck-path: "/api/health/" + alb.ingress.kubernetes.io/listen-ports: "[{\"HTTP\": 80}, {\"HTTPS\": 443}]" + alb.ingress.kubernetes.io/scheme: "internet-facing" + alb.ingress.kubernetes.io/target-type: "ip" + kubernetes.io/ingress.class: "alb" + hosts: + - host: "__SERVICE_NAME__.__BASE_DOMAIN__" + paths: + - path: /* + pathType: ImplementationSpecific + +livenessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 +readinessProbe: + httpGet: + path: /api/health/ + port: 9090 + initialDelaySeconds: 120 + timeoutSeconds: 3 + failureThreshold: 6 + +resources: + limits: + cpu: 1950m + memory: 6Gi + ephemeral-storage: 50Gi + requests: + cpu: 1950m + memory: 6Gi + ephemeral-storage: 50Gi + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 600 + policies: + - type: Percent + value: 100 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 30 + +nodeSelector: {} + # usage: gpu + # team: ml-team + # kubernetes.io/instance-type: c8g.medium + +tolerations: [] +# - key: "doc_parser" +# operator: "Equal" +# value: "true" +# effect: "NoSchedule" + +affinity: { } + +testConnection: + enabled: false + +secret: + externalSecretEnv: + enabled: true + name: __SERVICE_NAME__-secret + +configMap: + externalConfigMapEnv: + enabled: true + name: __SERVICE_NAME__-config-map + +extraEnvs: + - name: DD_APM_FILTER_TAGS_REGEX_REJECT + value: "http.route:/health" + - name: DD_CLUSTER_AGENT_URL + value: "https://datadog-cluster-agent.datadog:5005" + - name: DD_EKS_FARGATE + value: "true" + - name: DD_ENV + value: "__ENVIRONMENT__" + - name: DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL + value: "true" + - name: DD_LOGS_ENABLED + value: "true" + - name: DD_LOGS_INJECTION + value: "true" + - name: DD_PROCESS_AGENT_ENABLED + value: "true" + - name: DD_PROFILING_ENABLED + value: "true" + - name: DD_SERVICE + value: "__SERVICE_NAME__" + - name: DD_VERSION + value: "__DD_VERSION__" + - name: LOG_FILE + value: "/var/log/application/app.log" + +sidecars: + - name: datadog-agent + image: "__DD_AGENT_IMAGE__" + ports: + - containerPort: 8125 + name: dogstatsdport + protocol: UDP + - containerPort: 8126 + name: traceport + protocol: TCP + sharedVolume: + mountPath: /var/log/application/ + resources: + limits: + cpu: 200m + memory: 0.25Gi + requests: + cpu: 200m + memory: 0.25Gi + +datadog: + enabled: true From 26ee6393609681205ac2d28652b4fb893ed25036 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Mon, 2 Dec 2024 13:15:40 +0700 Subject: [PATCH 02/15] Update Helm values path --- Jenkinsfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 2d34c73..866d10d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,4 +38,24 @@ makeBuildPipeline { ] } } + + helmValuesPath = { ctx, buildEnv -> + if (buildEnv.isDevDeploymentEnabled()) { + 'k8s/dev/values.yaml' + } else if (buildEnv.isProdDeploymentEnabled()) { + 'k8s/prod/values.yaml' + } else { + ctx.error 'unknown environment!! Abort!!' + } + } + + helmValuesPath = { ctx, buildEnv -> + if (buildEnv.isDevDeploymentEnabled()) { + 'k8s/dev/values.yaml' + } else if (buildEnv.isProdDeploymentEnabled()) { + 'k8s/prod/values.yaml' + } else { + ctx.error 'unknown environment!! Abort!!' + } + } } From cabc100a9b31a4456cc17d1f01f81830b7f813ad Mon Sep 17 00:00:00 2001 From: Duc Duong <112845152+spartan-ductduong@users.noreply.github.com> Date: Mon, 2 Dec 2024 13:50:13 +0700 Subject: [PATCH 03/15] Update port and healthcheck endpoint (#2) --- k8s/dev/values.yaml | 2 +- k8s/prod-az/values.yaml | 2 +- k8s/prod/values.yaml | 2 +- nlm_ingestor/ingestion_daemon/__main__.py | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml index c1ef278..b6301ec 100644 --- a/k8s/dev/values.yaml +++ b/k8s/dev/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 9090 +containerPort: 5001 imagePullSecrets: [ ] nameOverride: "" diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index 40f33d2..85fd561 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 9090 +containerPort: 5001 imagePullSecrets: [ ] nameOverride: "" diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index 40f33d2..85fd561 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 9090 +containerPort: 5001 imagePullSecrets: [ ] nameOverride: "" diff --git a/nlm_ingestor/ingestion_daemon/__main__.py b/nlm_ingestor/ingestion_daemon/__main__.py index 29e73cc..630adc2 100644 --- a/nlm_ingestor/ingestion_daemon/__main__.py +++ b/nlm_ingestor/ingestion_daemon/__main__.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) logger.setLevel(cfg.log_level()) -@app.route('/', methods=['GET']) +@app.route('/api/health', methods=['GET']) def health_check(): return 'Service is running', 200 @@ -76,5 +76,3 @@ def main(): if __name__ == "__main__": main() - - From ef3a33b6e0e9336185110b73e418cb3ee367ff93 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Mon, 2 Dec 2024 14:41:30 +0700 Subject: [PATCH 04/15] Fix port values in Helm values files --- k8s/dev/values.yaml | 6 +++--- k8s/prod-az/values.yaml | 6 +++--- k8s/prod/values.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml index b6301ec..3dca225 100644 --- a/k8s/dev/values.yaml +++ b/k8s/dev/values.yaml @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "9090" + "port": "5001" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index 85fd561..2677df1 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "9090" + "port": "5001" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index 85fd561..2677df1 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "9090" + "port": "5001" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 9090 + port: 5001 initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 From 1b36de0040394b29f7005557521e81c0f7add19a Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Mon, 2 Dec 2024 15:55:57 +0700 Subject: [PATCH 05/15] Use gunicorn to run the server --- Dockerfile.prod | 38 +++++++++++++++++++++++ Jenkinsfile | 2 +- nlm_ingestor/ingestion_daemon/__main__.py | 1 + pull_request_template.md | 27 ---------------- run_prod.sh | 5 +++ 5 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 Dockerfile.prod delete mode 100644 pull_request_template.md create mode 100755 run_prod.sh diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..803b4be --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:experimental +FROM python:3.11-bookworm +RUN apt-get update && apt-get -y --no-install-recommends install libgomp1 +ENV APP_HOME /app +# install Java +RUN mkdir -p /usr/share/man/man1 && \ + apt-get update -y && \ + apt-get install -y openjdk-17-jre-headless +# install essential packages +RUN apt-get install -y \ + libxml2-dev libxslt-dev \ + build-essential libmagic-dev +# install tesseract +RUN apt-get install -y \ + tesseract-ocr \ + lsb-release \ + && echo "deb https://notesalexp.org/tesseract-ocr5/$(lsb_release -cs)/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/notesalexp.list > /dev/null \ + && apt-get update -oAcquire::AllowInsecureRepositories=true \ + && apt-get install notesalexp-keyring -oAcquire::AllowInsecureRepositories=true -y --allow-unauthenticated \ + && apt-get update \ + && apt-get install -y \ + tesseract-ocr libtesseract-dev \ + && wget -P /usr/share/tesseract-ocr/5/tessdata/ https://github.com/tesseract-ocr/tessdata/raw/main/eng.traineddata +RUN apt-get install unzip -y && \ + apt-get install git -y && \ + apt-get autoremove -y +WORKDIR ${APP_HOME} +COPY . ./ +RUN pip install --upgrade pip setuptools +RUN apt-get install -y libmagic1 +RUN mkdir -p -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts +RUN pip install -r requirements.txt +RUN python -m nltk.downloader stopwords +RUN python -m nltk.downloader punkt +RUN python -c "import tiktoken; tiktoken.get_encoding(\"cl100k_base\")" +RUN chmod +x run.sh +EXPOSE 5001 +CMD ./run.sh diff --git a/Jenkinsfile b/Jenkinsfile index 866d10d..e9224a7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ makeBuildPipeline { ] dockerFilePath = '/' - dockerFileName = 'Dockerfile' + dockerFileName = 'Dockerfile.prod' nodeBuildLabel = 'heavy' diff --git a/nlm_ingestor/ingestion_daemon/__main__.py b/nlm_ingestor/ingestion_daemon/__main__.py index 630adc2..b820e64 100644 --- a/nlm_ingestor/ingestion_daemon/__main__.py +++ b/nlm_ingestor/ingestion_daemon/__main__.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) logger.setLevel(cfg.log_level()) +@app.route('/', methods=['GET']) @app.route('/api/health', methods=['GET']) def health_check(): return 'Service is running', 200 diff --git a/pull_request_template.md b/pull_request_template.md deleted file mode 100644 index 2a08ef8..0000000 --- a/pull_request_template.md +++ /dev/null @@ -1,27 +0,0 @@ -## Description of the change - -> Description here - -## Type of change -- [ ] Bug fix (non-breaking change that fixes an issue) -- [ ] New feature (non-breaking change that adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - -## Related issues - -> Fix [#1]() - -## Checklists - -### Development - -- [ ] Lint rules pass locally -- [ ] The code changed/added as part of this pull request has been covered with tests -- [ ] All tests related to the changed code pass in development - -### Code review - -- [ ] This pull request has a descriptive title and information useful to a reviewer. There may be a screenshot or screencast attached -- [ ] "Ready for review" label attached to the PR and reviewers mentioned in a comment -- [ ] Changes have been reviewed by at least one other engineer -- [ ] Issue from task tracker has a link to this pull request diff --git a/run_prod.sh b/run_prod.sh new file mode 100755 index 0000000..92eb6ff --- /dev/null +++ b/run_prod.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# latest version of java and a python environment where requirements are installed is required +nohup java -jar jars/tika-server-standard-nlm-modified-2.9.2_v2.jar > /dev/null 2>&1 & + +gunicorn -b 0.0.0.0:5001 nlm_ingestor.ingestion_daemon.__main__:app From d7134c260c244bfcc03e8ebc03ccc1e02f10bd3c Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Mon, 2 Dec 2024 19:51:57 +0700 Subject: [PATCH 06/15] Update health endpoint's URL --- nlm_ingestor/ingestion_daemon/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nlm_ingestor/ingestion_daemon/__main__.py b/nlm_ingestor/ingestion_daemon/__main__.py index b820e64..80827ce 100644 --- a/nlm_ingestor/ingestion_daemon/__main__.py +++ b/nlm_ingestor/ingestion_daemon/__main__.py @@ -16,6 +16,7 @@ @app.route('/', methods=['GET']) @app.route('/api/health', methods=['GET']) +@app.route('/api/health/', methods=['GET']) def health_check(): return 'Service is running', 200 From 3588546f027d5d8e268b994ea653a85c23e40b2d Mon Sep 17 00:00:00 2001 From: Duc Duong <112845152+spartan-ductduong@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:34:24 +0700 Subject: [PATCH 07/15] Use environment variable for port configuration (#3) --- .github/workflows/auto-generate-release.yml | 12 ------------ .github/workflows/deploy-dev.yaml | 2 +- .github/workflows/deploy-prod-az.yaml | 3 ++- k8s/dev/values.yaml | 11 ++++------- k8s/prod-az/values.yaml | 8 ++++---- k8s/prod/values.yaml | 8 ++++---- nlm_ingestor/ingestion_daemon/__main__.py | 6 ++++-- 7 files changed, 19 insertions(+), 31 deletions(-) diff --git a/.github/workflows/auto-generate-release.yml b/.github/workflows/auto-generate-release.yml index d429a64..2a8dd32 100644 --- a/.github/workflows/auto-generate-release.yml +++ b/.github/workflows/auto-generate-release.yml @@ -31,21 +31,9 @@ jobs: token: ${{ steps.github-app-token.outputs.token }} ref: ${{ github.head_ref }} - - name: Check if CHANGELOG.md contains "## [Unreleased]" - id: check_changelog - run: | - if grep -q '## \[Unreleased\]' CHANGELOG.md; then - echo "Changelog contains '## [Unreleased]'. Skipping the pipeline." - exit 0 - else - echo "Changelog does not contain '## [Unreleased]'. Continuing the pipeline." - fi - continue-on-error: true - - uses: taiki-e/create-gh-release-action@v1 if: ${{ success() }} with: - changelog: CHANGELOG.md title: $version branch: 'master|v[0-9]+' token: ${{ steps.github-app-token.outputs.token }} diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index f21215f..6e46471 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -20,7 +20,7 @@ env: DOCKER_REPO: "${{ vars.AWS_ACCOUNT_ID_DEV }}.dkr.ecr.${{ vars.AWS_REGION_DEV }}.amazonaws.com/${{ vars.SERVICE_NAME }}" ENVIRONMENT: dev HELM_REPO: ${{ vars.HELM_REPO }} - HELM_VERSION: "1.1.2" + HELM_VERSION: "1.1.3" SERVICE_NAME: ${{ vars.SERVICE_NAME }} BASE_DOMAIN: ${{ vars.BASE_DOMAIN_DEV }} diff --git a/.github/workflows/deploy-prod-az.yaml b/.github/workflows/deploy-prod-az.yaml index e88c5d4..f7827f3 100644 --- a/.github/workflows/deploy-prod-az.yaml +++ b/.github/workflows/deploy-prod-az.yaml @@ -16,7 +16,8 @@ env: DOCKER_REPO_DEV: "${{ vars.AWS_ACCOUNT_ID_DEV }}.dkr.ecr.${{ vars.AWS_REGION_DEV }}.amazonaws.com" ENVIRONMENT: prod-az HELM_REPO: ${{ vars.HELM_REPO }} - HELM_VERSION: "1.1.2" + HELM_VERSION: "1.1.3" + PORT: ${{ vars.PORT }} SERVICE_NAME: ${{ vars.SERVICE_NAME }} BASE_DOMAIN: ${{ vars.BASE_DOMAIN_PROD_AZ }} diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml index 3dca225..7f557e9 100644 --- a/k8s/dev/values.yaml +++ b/k8s/dev/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 5001 +containerPort: "__PORT__" imagePullSecrets: [ ] nameOverride: "" @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "5001" + "port": "__PORT__" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 @@ -183,6 +183,3 @@ sidecars: datadog: enabled: true - -sentry: - enabled: true diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index 2677df1..a7b13e2 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 5001 +containerPort: "__PORT__" imagePullSecrets: [ ] nameOverride: "" @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "5001" + "port": "__PORT__" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index 2677df1..a7b13e2 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -4,7 +4,7 @@ image: pullPolicy: IfNotPresent tag: "__IMAGE_TAG__" -containerPort: 5001 +containerPort: "__PORT__" imagePullSecrets: [ ] nameOverride: "" @@ -23,7 +23,7 @@ podAnnotations: [ { "host": "%%host%%", - "port": "5001" + "port": "__PORT__" } ] ad.datadoghq.com/__SERVICE_NAME__.logs: | @@ -62,14 +62,14 @@ ingress: livenessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 readinessProbe: httpGet: path: /api/health/ - port: 5001 + port: __PORT__ initialDelaySeconds: 120 timeoutSeconds: 3 failureThreshold: 6 diff --git a/nlm_ingestor/ingestion_daemon/__main__.py b/nlm_ingestor/ingestion_daemon/__main__.py index 80827ce..83f1a23 100644 --- a/nlm_ingestor/ingestion_daemon/__main__.py +++ b/nlm_ingestor/ingestion_daemon/__main__.py @@ -8,6 +8,8 @@ from nlm_ingestor.ingestor import ingestor_api from nlm_utils.utils import file_utils +PORT = int(os.environ.get('PORT', 5001)) + app = Flask(__name__) # initialize logging @@ -73,8 +75,8 @@ def parse_document( return make_response(jsonify({"status": status, "reason": msg}), rc) def main(): - logger.info("Starting ingestor service..") - app.run(host="0.0.0.0", port=5001, debug=False) + logger.info(f"Starting ingestor service on port {PORT}..") + app.run(host="0.0.0.0", port=PORT, debug=False) if __name__ == "__main__": main() From 042ec4077443a38d06704ac3e964948c6dba3ab0 Mon Sep 17 00:00:00 2001 From: Duc Duong <112845152+spartan-ductduong@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:53:31 +0700 Subject: [PATCH 08/15] Update env variable name for port config (#4) --- .github/workflows/deploy-dev.yaml | 1 + .github/workflows/deploy-prod-az.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index 6e46471..77cc080 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -21,6 +21,7 @@ env: ENVIRONMENT: dev HELM_REPO: ${{ vars.HELM_REPO }} HELM_VERSION: "1.1.3" + PORT: ${{ vars.PORT_DEV }} SERVICE_NAME: ${{ vars.SERVICE_NAME }} BASE_DOMAIN: ${{ vars.BASE_DOMAIN_DEV }} diff --git a/.github/workflows/deploy-prod-az.yaml b/.github/workflows/deploy-prod-az.yaml index f7827f3..03455cb 100644 --- a/.github/workflows/deploy-prod-az.yaml +++ b/.github/workflows/deploy-prod-az.yaml @@ -17,7 +17,7 @@ env: ENVIRONMENT: prod-az HELM_REPO: ${{ vars.HELM_REPO }} HELM_VERSION: "1.1.3" - PORT: ${{ vars.PORT }} + PORT: ${{ vars.PORT_PROD_AZ }} SERVICE_NAME: ${{ vars.SERVICE_NAME }} BASE_DOMAIN: ${{ vars.BASE_DOMAIN_PROD_AZ }} From 0e3d67469fcc81276bb7c4536a0b15e6d7ed1a3d Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Mon, 2 Dec 2024 22:01:09 +0700 Subject: [PATCH 09/15] Update deployVarsMode to k8s --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e9224a7..1041c38 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,6 +13,8 @@ makeBuildPipeline { dockerFilePath = '/' dockerFileName = 'Dockerfile.prod' + deployVarsMode = 'k8s' + nodeBuildLabel = 'heavy' helmStageTimeout = 30 From 57a8a7cad4dc6137f8c9790f511c7144bfb09114 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Tue, 3 Dec 2024 01:30:47 +0700 Subject: [PATCH 10/15] Add release tools --- tools/create_release.sh | 23 ++ tools/semtag | 729 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 752 insertions(+) create mode 100755 tools/create_release.sh create mode 100755 tools/semtag diff --git a/tools/create_release.sh b/tools/create_release.sh new file mode 100755 index 0000000..d8a26ba --- /dev/null +++ b/tools/create_release.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SEMTAG='./tools/semtag' +ACTION=${1:-patch} + +git fetch origin --tags + +RELEASE_VERSION="$($SEMTAG final -s $ACTION -o)" + +echo "Next release version: $RELEASE_VERSION" + +if test -f "pyproject.toml"; then + PROJECT_VERSION=$(echo $RELEASE_VERSION | sed 's/^v//') + python src/utils/update_project_version.py "$PROJECT_VERSION" + + git config --global user.name 'github-actions[bot]' + git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add pyproject.toml + git commit -m "Bump version to $RELEASE_VERSION [skip ci]" + git push +fi + +$SEMTAG final -s $ACTION -v "$RELEASE_VERSION" diff --git a/tools/semtag b/tools/semtag new file mode 100755 index 0000000..b841a19 --- /dev/null +++ b/tools/semtag @@ -0,0 +1,729 @@ +#!/usr/bin/env bash + +PROG=semtag +PROG_VERSION="v0.1.0" + +SEMVER_REGEX="^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(\-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$" +IDENTIFIER_REGEX="^\-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*$" +NUMERIC_REGEX='^[0-9]+$' + +# Global variables +FIRST_VERSION="v0.0.0" +finalversion=$FIRST_VERSION +lastversion=$FIRST_VERSION +hasversiontag="false" +scope="patch" +displayonly="false" +forcetag="false" +forcedversion= +versionname= +identifier= + +HELP="\ +Usage: + $PROG + $PROG getlast + $PROG getfinal + $PROG (final|alpha|beta|candidate) [-s (major|minor|patch|auto) | -o] + $PROG --help + $PROG --version +Options: + -s The scope that must be increased, can be major, minor or patch. + The resulting version will match X.Y.Z(-PRERELEASE)(+BUILD) + where X, Y and Z are positive integers, PRERELEASE is an optionnal + string composed of alphanumeric characters describing if the build is + a release candidate, alpha or beta version, with a number. + BUILD is also an optional string composed of alphanumeric + characters and hyphens. + Setting the scope as 'auto', the script will chose the scope between + 'minor' and 'patch', depending on the amount of lines added (<10% will + choose patch). + -v Specifies manually the version to be tagged, must be a valid semantic version + in the format X.Y.Z where X, Y and Z are positive integers. + -o Output the version only, shows the bumped version, but doesn't tag. + -f Forces to tag, even if there are unstaged or uncommited changes. +Commands: + --help Print this help message. + --version Prints the program's version. + get Returns both current final version and last tagged version. + getlast Returns the latest tagged version. + getfinal Returns the latest tagged final version. + getcurrent Returns the current version, based on the latest one, if there are uncommited or + unstaged changes, they will be reflected in the version, adding the number of + pending commits, current branch and commit hash. + final Tags the current build as a final version, this only can be done on the default branch. + candidate Tags the current build as a release candidate, the tag will contain all + the commits from the last final version. + alpha Tags the current build as an alpha version, the tag will contain all + the commits from the last final version. + beta Tags the current build as a beta version, the tag will contain all + the commits from the last final version." + +# Commands and options +ACTION="getlast" +ACTION="$1" +shift + +# We get the parameters +while getopts "v:s:of" opt; do + case $opt in + v) + forcedversion="$OPTARG" + ;; + s) + scope="$OPTARG" + ;; + o) + displayonly="true" + ;; + f) + forcetag="true" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +# Try to programmatically fetch the default branch. Go by the first remote HEAD found, otherwise default to `master`. +# $1 The variable to store the result +function get_default_branch { + local __result=$1 + + local __remotes=$(git remote) + if [[ -n $__remotes ]]; then + for __remote in $__remotes; do + local __default_branch_ref=$(git symbolic-ref --quiet refs/remotes/${__remote}/HEAD || true) + local __default_branch=${__default_branch_ref#refs/remotes/${__remote}/} + if [[ -n ${__default_branch} ]]; then + break + fi + done + fi + + eval "${__result}=${__default_branch:-master}" +} + +# Gets a string with the version and returns an array of maximum size of 5 with all the parts of the sematinc version +# $1 The string containing the version in semantic format +# $2 The variable to store the result array: +# position 0: major number +# position 1: minor number +# position 2: patch number +# position 3: identifier (or prerelease identifier) +# position 4: build info +function explode_version { + local __version=$1 + local __result=$2 + if [[ $__version =~ $SEMVER_REGEX ]] ; then + local __major=${BASH_REMATCH[1]} + local __minor=${BASH_REMATCH[2]} + local __patch=${BASH_REMATCH[3]} + local __prere=${BASH_REMATCH[4]} + local __build=${BASH_REMATCH[5]} + eval "$__result=(\"$__major\" \"$__minor\" \"$__patch\" \"$__prere\" \"$__build\")" + else + eval "$__result=" + fi +} + +# Compare two versions and returns -1, 0 or 1 +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the result +function compare_versions { + local __first + local __second + explode_version $1 __first + explode_version $2 __second + local lv=$3 + + # Compares MAJOR, MINOR and PATCH + for i in 0 1 2; do + local __numberfirst=${__first[$i]} + local __numbersecond=${__second[$i]} + case $(($__numberfirst - $__numbersecond)) in + 0) + ;; + -[0-9]*) + eval "$lv=-1" + return 0 + ;; + [0-9]*) + eval "$lv=1" + return 0 + ;; + esac + done + + # Identifiers should compare with the ASCII order. + local __identifierfirst=${__first[3]} + local __identifiersecond=${__second[3]} + compare_identifiers "${__first[3]}" "${__second[3]}" compareresult + eval "$lv=$compareresult" +} + + +# Returns the number comparison +# $1 The first number to compare +# $2 The second number to compare +# $3 The variable where to store the result +function compare_numeric { + local __first=$1 + local __second=$2 + local __result=$3 + + if (( "$__first" < "$__second" )) ; then + eval "$__result=-1" + elif (( "$__first" > "$__second" )) ; then + eval "$__result=1" + else + eval "$__result=0" + fi +} + +# Returns the alpanumeric comparison +# $1 The first alpanumeric to compare +# $2 The second alpanumeric to compare +# $3 The variable where to store the result +function compare_alphanumeric { + local __first=$1 + local __second=$2 + local __result=$3 + + if [[ "$__first" < "$__second" ]] ; then + eval "$__result=-1" + elif [[ "$__first" > "$__second" ]] ; then + eval "$__result=1" + else + eval "$__result=0" + fi +} + +# Returns the last version of two +# $1 The first version to compare +# $2 The second version to compare +# $3 The variable where to store the last one +function get_latest_of_two { + local __first=$1 + local __second=$2 + local __result + local __latest=$3 + + compare_versions $__first $__second __result + case $__result in + 0) + eval "$__latest=$__second" + ;; + -1) + eval "$__latest=$__second" + ;; + 1) + eval "$__latest=$__first" + ;; + esac +} + +# Returns comparison of two identifier parts +# $1 The first part to compare +# $2 The second part to compare +# $3 The variable where to store the compare result +function compare_identifier_part { + local __first=$1 + local __second=$2 + local __result=$3 + local compareresult + + if [[ "$__first" =~ $NUMERIC_REGEX ]] && [[ "$__second" =~ $NUMERIC_REGEX ]] ; then + compare_numeric "$__first" "$__second" compareresult + eval "$__result=$compareresult" + return 0 + fi + + + compare_alphanumeric "$__first" "$__second" compareresult + eval "$__result=$compareresult" +} + +# Returns comparison of two identifiers +# $1 The first identifier to compare +# $2 The second identifier to compare +# $3 The variable where to store the compare result +function compare_identifiers { + local __first=$1 + local __second=$2 + local __result=$3 + local partresult + local arraylengths + if [[ -n "$__first" ]] && [[ -n "$__second" ]]; then + explode_identifier "${__first}" explodedidentifierfirst + explode_identifier "${__second}" explodedidentifiersecond + + firstsize=${#explodedidentifierfirst[@]} + secondsize=${#explodedidentifiersecond[@]} + minlength=$(( $firstsize<$secondsize ? $firstsize : $secondsize )) + for (( i = 0 ; i < $minlength ; i++ )); do + compare_identifier_part "${explodedidentifierfirst[$i]}" "${explodedidentifiersecond[$i]}" partresult + case $partresult in + 0) + ;; + *) + eval "$__result=$partresult" + return 0 + ;; + esac + done + compare_numeric $firstsize $secondsize arraylengths + eval "$__result=$arraylengths" + return 0 + elif [[ -z "$__first" ]] && [[ -n "$__second" ]]; then + eval "$__result=1" + return 0 + elif [[ -n "$__first" ]] && [[ -z "$__second" ]]; then + eval "$__result=-1" + return 0 + fi + + eval "$__result=0" +} + +# Assigns a 2 size array with the identifier, having the identifier at pos 0, and the number in pos 1 +# $1 The identifier in the format -id.# +# $2 The vferiable where to store the 2 size array +function explode_identifier { + local __identifier=$1 + local __result=$2 + if [[ $__identifier =~ $IDENTIFIER_REGEX ]] ; then + IFS='-.' read -ra identifierparts <<< $__identifier + eval "$__result=( ${identifierparts[@]} )" + else + eval "$__result=" + fi +} + +# Gets a list of tags and assigns the base and latest versions +# Receives an array with the tags containing the versions +# Assigns to the global variables finalversion and lastversion the final version and the latest version +function get_latest { + local __taglist=("$@") + local __tagsnumber=${#__taglist[@]} + local __current + case $__tagsnumber in + 0) + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + ;; + 1) + __current=${__taglist[0]} + explode_version $__current ver + if [ -n "$ver" ]; then + if [ -n "${ver[3]}" ]; then + finalversion=$FIRST_VERSION + else + finalversion=$__current + fi + lastversion=$__current + else + finalversion=$FIRST_VERSION + lastversion=$FIRST_VERSION + fi + ;; + *) + local __lastpos=$(($__tagsnumber-1)) + for i in $(seq 0 $__lastpos) + do + __current=${__taglist[i]} + explode_version ${__taglist[i]} ver + if [ -n "$ver" ]; then + if [ -z "${ver[3]}" ]; then + get_latest_of_two $finalversion $__current finalversion + get_latest_of_two $lastversion $finalversion lastversion + else + get_latest_of_two $lastversion $__current lastversion + fi + fi + done + ;; + esac + + if git rev-parse -q --verify "refs/tags/$lastversion" >/dev/null; then + hasversiontag="true" + else + hasversiontag="false" + fi +} + +# Gets the next version given the provided scope +# $1 The version that is going to be bumped +# $2 The scope to bump +# $3 The variable where to stoer the result +function get_next_version { + local __exploded + local __fromversion=$1 + local __scope=$2 + local __result=$3 + explode_version $__fromversion __exploded + case $__scope in + major) + __exploded[0]=$((${__exploded[0]}+1)) + __exploded[1]=0 + __exploded[2]=0 + ;; + minor) + __exploded[1]=$((${__exploded[1]}+1)) + __exploded[2]=0 + ;; + patch) + __exploded[2]=$((${__exploded[2]}+1)) + ;; + esac + + eval "$__result=v${__exploded[0]}.${__exploded[1]}.${__exploded[2]}" +} + +function bump_version { + ## First we try to get the next version based on the existing last one + if [ "$scope" == "auto" ]; then + get_scope_auto scope + fi + + local __candidatefromlast=$FIRST_VERSION + local __explodedlast + explode_version $lastversion __explodedlast + if [[ -n "${__explodedlast[3]}" ]]; then + # Last version is not final + local __idlast + explode_identifier ${__explodedlast[3]} __idlast + + # We get the last, given the desired id based on the scope + __candidatefromlast="v${__explodedlast[0]}.${__explodedlast[1]}.${__explodedlast[2]}" + if [[ -n "$identifier" ]]; then + local __nextid="$identifier.1" + if [ "$identifier" == "${__idlast[0]}" ]; then + # We target the same identifier as the last so we increase one + __nextid="$identifier.$(( ${__idlast[1]}+1 ))" + __candidatefromlast="$__candidatefromlast-$__nextid" + else + # Different identifiers, we make sure we are assigning a higher identifier, if not, we increase the version + __candidatefromlast="$__candidatefromlast-$__nextid" + local __comparedwithlast + compare_versions $__candidatefromlast $lastversion __comparedwithlast + if [ "$__comparedwithlast" == -1 ]; then + get_next_version $__candidatefromlast $scope __candidatefromlast + __candidatefromlast="$__candidatefromlast-$__nextid" + fi + fi + fi + fi + + # Then we try to get the version based on the latest final one + local __candidatefromfinal=$FIRST_VERSION + get_next_version $finalversion $scope __candidatefromfinal + if [[ -n "$identifier" ]]; then + __candidatefromfinal="$__candidatefromfinal-$identifier.1" + fi + + # Finally we compare both candidates + local __resultversion + local __result + compare_versions $__candidatefromlast $__candidatefromfinal __result + case $__result in + 0) + __resultversion=$__candidatefromlast + ;; + -1) + __resultversion="$__candidatefromfinal" + ;; + 1) + __resultversion=$__candidatefromlast + ;; + esac + + eval "$1=$__resultversion" +} + +function increase_version { + local __version= + + if [ -z $forcedversion ]; then + bump_version __version + else + if [[ $forcedversion =~ $SEMVER_REGEX ]] ; then + compare_versions $forcedversion $lastversion __result + if [ $__result -le 0 ]; then + echo "Version can't be lower than last version: $lastversion" + exit 1 + fi + else + echo "Non valid version to bump" + exit 1 + fi + __version=$forcedversion + fi + + if [ "$displayonly" == "true" ]; then + echo "$__version" + else + if [ "$forcetag" == "false" ]; then + check_git_dirty_status + fi + local __commitlist + if [ "$finalversion" == "$FIRST_VERSION" ] || [ "$hasversiontag" != "true" ]; then + __commitlist="$(git log --pretty=oneline | cat)" + else + __commitlist="$(git log --pretty=oneline $finalversion... | cat)" + fi + + # If we are forcing a bump, we add bump to the commit list + if [[ -z $__commitlist && "$forcetag" == "true" ]]; then + __commitlist="bump" + fi + + if [[ -z $__commitlist ]]; then + echo "No commits since the last final version, not bumping version" + else + if [[ -z $versionname ]]; then + versionname=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + fi + local __message="$versionname +$__commitlist" + + # We check we have info on the user + local __username=$(git config user.name) + if [ -z "$__username" ]; then + __username=$(id -u -n) + git config user.name $__username + fi + local __useremail=$(git config user.email) + if [ -z "$__useremail" ]; then + __useremail=$(hostname) + git config user.email "$__username@$__useremail" + fi + + git tag -a $__version -m "$__message" + + # If we have a remote, we push there + local __remotes=$(git remote) + if [[ -n $__remotes ]]; then + for __remote in $__remotes; do + git push $__remote $__version > /dev/null + if [ $? -eq 0 ]; then + echo "$__version pushed to $__remote" + else + echo "Error pushing the tag $__version to $__remote" + exit 1 + fi + done + else + echo "$__version" + fi + fi + fi +} + +function check_git_dirty_status { + local __repostatus= + get_work_tree_status __repostatus + + if [ "$__repostatus" == "uncommitted" ]; then + echo "ERROR: You have uncommitted changes" + git status --porcelain + exit 1 + fi + + if [ "$__repostatus" == "unstaged" ]; then + echo "ERROR: You have unstaged changes" + git status --porcelain + exit 1 + fi +} + +# Get the total amount of lines of code in the repo +function get_total_lines { + local __empty_id="$(git hash-object -t tree /dev/null)" + local __changes="$(git diff --numstat $__empty_id | cat)" + local __added_deleted=$1 + get_changed_lines "$__changes" $__added_deleted +} + +# Get the total amount of lines of code since the provided tag +function get_sincetag_lines { + local __sincetag=$1 + local __changes="$(git diff --numstat $__sincetag | cat)" + local __added_deleted=$2 + get_changed_lines "$__changes" $__added_deleted +} + +function get_changed_lines { + local __changes_numstat=$1 + local __result=$2 + IFS=$'\n' read -rd '' -a __changes_array <<<"$__changes_numstat" + local __diff_regex="^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+.+$" + + local __total_added=0 + local __total_deleted=0 + for i in "${__changes_array[@]}" + do + if [[ $i =~ $__diff_regex ]] ; then + local __added=${BASH_REMATCH[1]} + local __deleted=${BASH_REMATCH[2]} + __total_added=$(( $__total_added+$__added )) + __total_deleted=$(( $__total_deleted+$__deleted )) + fi + done + eval "$2=( $__total_added $__total_deleted )" +} + +function get_scope_auto { + local __verbose=$2 + local __total=0 + local __since=0 + local __scope= + + get_total_lines __total + get_sincetag_lines $finalversion __since + + local __percentage=0 + if [ "$__total" != "0" ]; then + local __percentage=$(( 100*$__since/$__total )) + if [ $__percentage -gt "10" ]; then + __scope="minor" + else + __scope="patch" + fi + fi + + eval "$1=$__scope" + if [[ -n "$__verbose" ]]; then + echo "[Auto Scope] Percentage of lines changed: $__percentage" + echo "[Auto Scope] : $__scope" + fi +} + +function get_work_tree_status { + # Update the index + git update-index -q --ignore-submodules --refresh > /dev/null + eval "$1=" + + if ! git diff-files --quiet --ignore-submodules -- > /dev/null + then + eval "$1=unstaged" + fi + + if ! git diff-index --cached --quiet HEAD --ignore-submodules -- > /dev/null + then + eval "$1=uncommitted" + fi +} + +function get_current { + if [ "$hasversiontag" == "true" ]; then + local __commitcount="$(git rev-list $lastversion.. --count)" + else + local __commitcount="$(git rev-list --count HEAD)" + fi + local __status= + get_work_tree_status __status + + if [ "$__commitcount" == "0" ] && [ -z "$__status" ]; then + eval "$1=$lastversion" + else + local __buildinfo="$(git rev-parse --short HEAD)" + local __currentbranch="$(git rev-parse --abbrev-ref HEAD)" + get_default_branch default_branch + if [ "$__currentbranch" != "$default_branch" ]; then + __buildinfo="$__currentbranch.$__buildinfo" + fi + + local __suffix= + if [ "$__commitcount" != "0" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__commitcount" + fi + if [ -n "$__status" ]; then + if [ -n "$__suffix" ]; then + __suffix="$__suffix." + fi + __suffix="$__suffix$__status" + fi + + __suffix="$__suffix+$__buildinfo" + if [ "$lastversion" == "$finalversion" ]; then + scope="patch" + identifier= + local __bumped= + bump_version __bumped + eval "$1=$__bumped-dev.$__suffix" + else + eval "$1=$lastversion.$__suffix" + fi + fi +} + +function init { + git fetch > /dev/null + TAGS="$(git tag)" + IFS=$'\n' read -rd '' -a TAG_ARRAY <<<"$TAGS" + + get_latest ${TAG_ARRAY[@]} + currentbranch="$(git rev-parse --abbrev-ref HEAD)" +} + +case $ACTION in + --help) + echo -e "$HELP" + ;; + --version) + echo -e "${PROG}: $PROG_VERSION" + ;; + final) + init + get_default_branch default_branch + diff=$(git diff $default_branch | cat) + if [ "$forcetag" == "false" ]; then + if [ -n "$diff" ]; then + echo "ERROR: Branch must be updated with $default_branch for final versions" + exit 1 + fi + fi + increase_version + ;; + alpha|beta) + init + identifier="$ACTION" + increase_version + ;; + candidate) + init + identifier="rc" + increase_version + ;; + getlast) + init + echo "$lastversion" + ;; + getfinal) + init + echo "$finalversion" + ;; + getcurrent) + init + get_current current + echo "$current" + ;; + get) + init + echo "Current final version: $finalversion" + echo "Last tagged version: $lastversion" + ;; + *) + echo "'$ACTION' is not a valid command, see --help for available commands." + ;; +esac From 704101e66d45f30343db5894afea03e03a18aad9 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Tue, 3 Dec 2024 01:52:50 +0700 Subject: [PATCH 11/15] Remove Flyway Docker image build and push steps --- .github/workflows/deploy-dev.yaml | 10 ---------- .github/workflows/deploy-prod-az.yaml | 4 ---- 2 files changed, 14 deletions(-) diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml index 77cc080..207eacd 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-dev.yaml @@ -115,16 +115,6 @@ jobs: --build-arg APP_VERSION=$APP_VERSION \ --push . - - name: Build and Push the flyway Docker image - working-directory: src/service_platform/sql - run: | - DOCKER_MIGRATION_IMAGE_TAG="$IMAGE_TAG"-migration - docker buildx build \ - --platform "$DOCKER_PLATFORM" \ - -t "$DOCKER_REPO":"$DOCKER_MIGRATION_IMAGE_TAG" \ - -f Dockerfile.migration \ - --push . - - name: "Update values for k8s files" uses: cschleiden/replace-tokens@v1 with: diff --git a/.github/workflows/deploy-prod-az.yaml b/.github/workflows/deploy-prod-az.yaml index 03455cb..88ad72c 100644 --- a/.github/workflows/deploy-prod-az.yaml +++ b/.github/workflows/deploy-prod-az.yaml @@ -73,7 +73,6 @@ jobs: - name: Pull images from DEV for promotion to Prod AZ run: | docker pull "$SERVICE_IMAGE_DEV" - docker pull "$SERVICE_IMAGE_DEV"-migration - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -90,9 +89,6 @@ jobs: docker tag "$SERVICE_IMAGE_DEV" "$SERVICE_IMAGE" docker push "$SERVICE_IMAGE" - docker tag "$SERVICE_IMAGE_DEV"-migration "$SERVICE_IMAGE"-migration - docker push "$SERVICE_IMAGE"-migration - - name: "Update values for k8s files" uses: cschleiden/replace-tokens@v1 with: From cc1241a0d525cced143e8e190734c205d8bea834 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Tue, 3 Dec 2024 02:16:31 +0700 Subject: [PATCH 12/15] Fix namespace in Helm Install step --- .github/workflows/deploy-prod-az.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-prod-az.yaml b/.github/workflows/deploy-prod-az.yaml index 88ad72c..2c488a8 100644 --- a/.github/workflows/deploy-prod-az.yaml +++ b/.github/workflows/deploy-prod-az.yaml @@ -17,6 +17,7 @@ env: ENVIRONMENT: prod-az HELM_REPO: ${{ vars.HELM_REPO }} HELM_VERSION: "1.1.3" + NAMESPACE: "service-platform" PORT: ${{ vars.PORT_PROD_AZ }} SERVICE_NAME: ${{ vars.SERVICE_NAME }} BASE_DOMAIN: ${{ vars.BASE_DOMAIN_PROD_AZ }} @@ -105,7 +106,7 @@ jobs: run: | helm repo add spartan https://x-access-token:${{ steps.github-app-token.outputs.token }}@${{ env.HELM_REPO }} helm upgrade --install --wait --timeout 600s \ - --namespace ${{ env.SERVICE_NAME }} \ + --namespace ${{ env.NAMESPACE }} \ -f k8s/${{ env.ENVIRONMENT }}/values.yaml \ --version ${{ env.HELM_VERSION }} \ ${{ env.SERVICE_NAME }} spartan/spartan From bc14435355df2c16383e61e51f31a377155c66e1 Mon Sep 17 00:00:00 2001 From: Duc Duong Date: Tue, 3 Dec 2024 03:16:12 +0700 Subject: [PATCH 13/15] Reduce minReplicas to 1 --- k8s/prod-az/values.yaml | 2 +- k8s/prod/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index a7b13e2..9902c16 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -86,7 +86,7 @@ resources: autoscaling: enabled: true - minReplicas: 2 + minReplicas: 1 maxReplicas: 4 metrics: - type: Resource diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index a7b13e2..9902c16 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -86,7 +86,7 @@ resources: autoscaling: enabled: true - minReplicas: 2 + minReplicas: 1 maxReplicas: 4 metrics: - type: Resource From 65ada7f60ef636681ffda83b196d85194ddc4776 Mon Sep 17 00:00:00 2001 From: Duc Duong <112845152+spartan-ductduong@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:51:55 +0700 Subject: [PATCH 14/15] Reduce pod resource allocations (#5) --- k8s/dev/values.yaml | 12 ++++++------ k8s/prod-az/values.yaml | 12 ++++++------ k8s/prod/values.yaml | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml index 7f557e9..7c6b13b 100644 --- a/k8s/dev/values.yaml +++ b/k8s/dev/values.yaml @@ -76,13 +76,13 @@ readinessProbe: resources: limits: - cpu: 950m - memory: 3Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi requests: - cpu: 950m - memory: 3Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi autoscaling: enabled: true diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index 9902c16..7c6b13b 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -76,13 +76,13 @@ readinessProbe: resources: limits: - cpu: 1950m - memory: 6Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi requests: - cpu: 1950m - memory: 6Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi autoscaling: enabled: true diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index 9902c16..7c6b13b 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -76,13 +76,13 @@ readinessProbe: resources: limits: - cpu: 1950m - memory: 6Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi requests: - cpu: 1950m - memory: 6Gi - ephemeral-storage: 50Gi + cpu: 500m + memory: 2Gi + ephemeral-storage: 30Gi autoscaling: enabled: true From 59b5a836da0e602eb08c3fe689e5c7135f43e182 Mon Sep 17 00:00:00 2001 From: vuongnguyen-spartan Date: Wed, 4 Dec 2024 22:18:03 +0700 Subject: [PATCH 15/15] Upscale resource --- k8s/dev/values.yaml | 8 ++++---- k8s/prod-az/values.yaml | 8 ++++---- k8s/prod/values.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/k8s/dev/values.yaml b/k8s/dev/values.yaml index 7c6b13b..637177a 100644 --- a/k8s/dev/values.yaml +++ b/k8s/dev/values.yaml @@ -76,12 +76,12 @@ readinessProbe: resources: limits: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 4Gi ephemeral-storage: 30Gi requests: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 4Gi ephemeral-storage: 30Gi autoscaling: diff --git a/k8s/prod-az/values.yaml b/k8s/prod-az/values.yaml index 7c6b13b..f3540e6 100644 --- a/k8s/prod-az/values.yaml +++ b/k8s/prod-az/values.yaml @@ -76,12 +76,12 @@ readinessProbe: resources: limits: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 6Gi ephemeral-storage: 30Gi requests: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 6Gi ephemeral-storage: 30Gi autoscaling: diff --git a/k8s/prod/values.yaml b/k8s/prod/values.yaml index 7c6b13b..f3540e6 100644 --- a/k8s/prod/values.yaml +++ b/k8s/prod/values.yaml @@ -76,12 +76,12 @@ readinessProbe: resources: limits: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 6Gi ephemeral-storage: 30Gi requests: - cpu: 500m - memory: 2Gi + cpu: 950m + memory: 6Gi ephemeral-storage: 30Gi autoscaling: