.gitlab-ci.yml

#file: noinspection YAMLSchemaValidation
---

include:
  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'

stages:
  - 🧪 test
  - 📋 lint
  - 🏗 build
  - 📦 publish
  - 🚀 deploy
  - 📣 release

variables:
  # CI config
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  # Postgres Service
  POSTGRES_HOST: "postgres"
  POSTGRES_DB: "assets-tracking-test"
  POSTGRES_USER: "postgres"
  POSTGRES_PASSWORD: "password"

  # App
  APP_DB_DATABASE: "assets-tracking-dev"  # intentionally different to service DB as not used for tests
  ASSETS_TRACKING_SERVICE_DB_DSN: "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$APP_DB_DATABASE"

  # Packaging
  PACKAGE_ACCESS_IDENTIFIER: "public_access"  # ('read_package_registry` scope)
  PACKAGE_ACCESS_TOKEN: "pygX3Ar5sRL7BXhk_SyN"

  # Dependencies
  ATS_AIR_ACCESS_IDENTIFIER: "gitlab_ci"

  # Deployment
  SSH_PRIVATE_KEY: "id_ed25519.pem"
  DEPLOY_HOST: "bslws01.nerc-bas.ac.uk"
  DEPLOY_USER: "geoweb"
  DEPLOY_HOME: "/users/$DEPLOY_USER"
  DEPLOY_APP_NAME: "assets-tracking-service"
  DEPLOY_APP_PIP_PACKAGE: "$DEPLOY_APP_NAME"
  DEPLOY_APP_PIP_EXTRA_INDEX: "https://$PACKAGE_ACCESS_IDENTIFIER:$PACKAGE_ACCESS_TOKEN@gitlab.data.bas.ac.uk/api/v4/projects/$CI_PROJECT_ID/packages/pypi/simple"
  DEPLOY_ATS_AIR_PIP_EXTRA_INDEX: "https://$ATS_AIR_ACCESS_IDENTIFIER:$ATS_AIR_ACCESS_TOKEN@gitlab.data.bas.ac.uk/api/v4/projects/1216/packages/pypi/simple"
  DEPLOY_VENV: "/users/$DEPLOY_USER/.venv/$DEPLOY_APP_NAME"
  DEPLOY_MODULE_BASE: "/users/$DEPLOY_USER/Modules/modulefiles/$DEPLOY_APP_NAME"

  # Secrets (set in GitLab CI/CD settings)
  # - PROJECT_ACCESS_TOKEN
  #   - https://gitlab.data.bas.ac.uk/MAGIC/assets-tracking-service/-/settings/access_tokens
  #   - (name: 'gitlab_ci' scope: 'read_api', role: 'reporter')
  # - ATS_AIR_ACCESS_TOKEN
  #   - https://gitlab.data.bas.ac.uk/MAGIC/assets-tracking-service/-/settings/repository
  #   - (name: 'GitLab CI', username: 'gitlab_ci', scope: 'read_package_registry')
  # - SAFETY_API_KEY
  #   - https://start.1password.com/open/i?a=QSB6V7TUNVEOPPPWR6G7S2ARJ4&v=k34cpwfkqaxp2r56u4aklza6ni&i=swbuhnii4ego6qycyqknvtk7gi&h=magic.1password.eu
  # - OP_SERVICE_ACCOUNT_TOKEN
  #   - https://start.1password.com/open/i?a=QSB6V7TUNVEOPPPWR6G7S2ARJ4&v=k34cpwfkqaxp2r56u4aklza6ni&i=hpszegtzzarkgua2ckg57gxpxe&h=magic.1password.eu

cache:
  paths:
    - .cache/pip  # for pipx/pip
    - .venv/      # for project dependencies

image: python:3.11-slim

.before_script_common: &before_script_common
  - apt-get update

.before_script_python: &before_script_python
  - *before_script_common

  # setup python
  - python --version
  - python -m pip install --upgrade pip

  # setup poetry
  - python -m pip install pipx
  - python -m pipx install poetry==1.8.2
  - python -m pipx ensurepath
  - export PATH=/root/.local/bin:$PATH
  - poetry --version
  - poetry config virtualenvs.in-project true
  - poetry config repositories.gitlab "$CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/packages/pypi"
  - poetry config http-basic.gitlab gitlab-ci-token "$CI_JOB_TOKEN" --local
  - poetry config http-basic.ats-air gitlab-ci-token "$CI_JOB_TOKEN" --local
  - poetry check

.before_script_python_app_deps: &before_script_python_app_deps
  - *before_script_python

  # setup gdal
  - apt-get install -y build-essential gdal-bin libgdal-dev
  - ogrinfo --version

  # setup app deps
  - apt-get install -y libkrb5-dev
  - poetry install --no-interaction --no-ansi
  - poetry run python -m pip install --no-deps arcgis

.before_script_postgres_testing: &before_script_postgres_testing
  - *before_script_common

  # setup postgres (testing)
  - apt-get install -y postgresql-client
  - export PGPASSWORD=$POSTGRES_PASSWORD    # best practice way to provide password to psql
  - psql -h $POSTGRES_HOST -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT 'OK' AS status;"
  - psql -h $POSTGRES_HOST -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "DROP EXTENSION IF EXISTS postgis_tiger_geocoder;"
  - psql -h $POSTGRES_HOST -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "DROP EXTENSION IF EXISTS postgis_topology;"
  - poetry run python -c 'from assets_tracking_service.config import Config; c = Config(); print(c.db_dsn_safe)'

.before_script_deploy: &before_script_deploy
  - *before_script_common

  # setup 1Password CLI
  - apt-get install -y curl unzip
  - curl -sSfo op.zip https://cache.agilebits.com/dist/1P/op2/pkg/v2.28.0/op_linux_amd64_v2.28.0.zip
  - unzip -od /usr/local/bin/ op.zip

  # setup SSH
  - apt-get install -y openssh-client
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - op read --out-file ~/.ssh/$SSH_PRIVATE_KEY "op://Infrastructure/Assets Tracking Service - GitLab CI private key/private key?ssh-format=openssh"
  - eval $(ssh-agent -s)
  - chmod 400 ~/.ssh/$SSH_PRIVATE_KEY
  - ssh-add ~/.ssh/$SSH_PRIVATE_KEY
  - ssh-keyscan $DEPLOY_HOST >> ~/.ssh/known_hosts
  - chmod 644 ~/.ssh/known_hosts

  # get secrets
  - export DEPLOY_APP_DB_DSN=$(op read "op://Infrastructure/Assets Tracking Service - Staging DB/PSQL connection string")
  - export DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_USERNAME=$(op read "op://Infrastructure/vgyikhtrssx4h23vordfqduvuy/username")
  - export DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_PASSWORD=$(op read "op://Infrastructure/vgyikhtrssx4h23vordfqduvuy/password")
  - export DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_API_KEY=$(op read "op://Infrastructure/vgyikhtrssx4h23vordfqduvuy/API/API Key")
  - export DEPLOY_APP_PROVIDER_GEOTAB_USERNAME=$(op read "op://Infrastructure/Assets Tracking Service - Geotab User/username")
  - export DEPLOY_APP_PROVIDER_GEOTAB_PASSWORD=$(op read "op://Infrastructure/Assets Tracking Service - Geotab User/password")
  - export DEPLOY_APP_PROVIDER_GEOTAB_DATABASE=$(op read "op://Infrastructure/Assets Tracking Service - Geotab User/database")
  - export DEPLOY_APP_EXPORTER_ARCGIS_USERNAME=$(op read "op://Infrastructure/vcadxkix3qwguf4trgspkcdxr4/username")
  - export DEPLOY_APP_EXPORTER_ARCGIS_PASSWORD=$(op read "op://Infrastructure/vcadxkix3qwguf4trgspkcdxr4/password")

.before_script_release: &before_script_release
  - apk add --no-cache curl jq

# Jobs

pytest:
  stage: 🧪 test
  needs: []
  services:
    - name: postgis/postgis:16-3.4-alpine
      alias: postgres
  before_script:
    - *before_script_python_app_deps
    - *before_script_postgres_testing
  script:
    - poetry run pytest -o junit_family=xunit2 --junitxml=test-results.xml --cov --cov-report=html --continue-on-collection-errors
    # --continue-on-collection-errors counteracts `-x` flag set in pyproject.toml
  coverage: '/Total coverage: \d+\.\d+/'
  artifacts:
    when: always
    reports:
      junit: test-results.xml
    paths:
      - htmlcov
    expire_in: 1 month
  rules:
    -
      changes:
        - '**/*.py'
        - 'src/assets_tracking_service/resources/db_migrations/**/*.sql'
        - 'pyproject.toml'
        - 'poetry.lock'
      if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'

ruff:
  stage: 📋 lint
  needs: []
  before_script:
    - *before_script_python_app_deps
  script:
    - poetry run ruff check src/ tests/
    - poetry run ruff format --check src/ tests/
  rules:
    -
      changes:
        - '**/*.py'
      if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'

safety:
  stage: 📋 lint
  needs: []
  before_script:
    - *before_script_python_app_deps
  script:
    - poetry run safety --stage cicd scan --detailed-output
  rules:
    -
      changes:
        - '.safety-policy.yml'
        - 'poetry.lock'
      if: '$CI_COMMIT_BRANCH != "main" && $CI_COMMIT_TAG == null'

build:
  stage: 🏗 build
  needs: []
  before_script:
    - *before_script_python
  script:
    - poetry build
  artifacts:
    paths:
      - dist/
    expire_in: 1 month
  rules:
    -
      changes:
        - '**/*.py'
        - 'pyproject.toml'
        - 'poetry.lock'
      if: '$CI_COMMIT_BRANCH != "main" || $CI_COMMIT_TAG != null'

publish:
  stage: 📦 publish
  needs:
    - job: build
      artifacts: true
  before_script:
    - *before_script_python
  script:
    - poetry publish --no-interaction --repository gitlab
  rules:
    - if: $CI_COMMIT_TAG

deploy:
  stage: 🚀 deploy
  needs:
    - job: publish
  before_script:
    - *before_script_deploy
    - export DEPLOY_APP_VERSION=$(echo $CI_COMMIT_TAG | cut -c 2-)
  script:
    - ssh $DEPLOY_USER@$DEPLOY_HOST "$DEPLOY_VENV/bin/python -m pip install --extra-index $DEPLOY_APP_PIP_EXTRA_INDEX --extra-index $DEPLOY_ATS_AIR_PIP_EXTRA_INDEX --upgrade $DEPLOY_APP_PIP_PACKAGE==$DEPLOY_APP_VERSION"
    - |
      cat > module.tpl << EOF
      #%Module1.0

      proc ModulesHelp { } {
        puts stderr "Adds the control CLI for the Assets Tracking Service to your environment\n Run `ats-ctl --help` for further help."
      }
      module-whatis "Control CLI for the Assets Tracking Service.\n"

      set basedir "$DEPLOY_VENV"

      setenv ASSETS_TRACKING_SERVICE_ENABLE_FEATURE_SENTRY true
      setenv ASSETS_TRACKING_SERVICE_SENTRY_ENVIRONMENT production
      setenv ASSETS_TRACKING_SERVICE_DB_DSN $DEPLOY_APP_DB_DSN
      setenv ASSETS_TRACKING_SERVICE_ENABLE_PROVIDER_GEOTAB true
      setenv ASSETS_TRACKING_SERVICE_ENABLE_PROVIDER_AIRCRAFT_TRACKING true
      setenv ASSETS_TRACKING_SERVICE_ENABLE_EXPORTER_ARCGIS true
      setenv ASSETS_TRACKING_SERVICE_ENABLE_EXPORTER_GEOJSON true
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_AIRCRAFT_TRACKING_USERNAME $DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_USERNAME
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_AIRCRAFT_TRACKING_PASSWORD $DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_PASSWORD
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_AIRCRAFT_TRACKING_API_KEY $DEPLOY_APP_PROVIDER_AIRCRAFT_TRACKING_API_KEY
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_GEOTAB_USERNAME $DEPLOY_APP_PROVIDER_GEOTAB_USERNAME
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_GEOTAB_PASSWORD $DEPLOY_APP_PROVIDER_GEOTAB_PASSWORD
      setenv ASSETS_TRACKING_SERVICE_PROVIDER_GEOTAB_DATABASE $DEPLOY_APP_PROVIDER_GEOTAB_DATABASE
      setenv ASSETS_TRACKING_SERVICE_EXPORTER_ARCGIS_USERNAME $DEPLOY_APP_EXPORTER_ARCGIS_USERNAME
      setenv ASSETS_TRACKING_SERVICE_EXPORTER_ARCGIS_PASSWORD $DEPLOY_APP_EXPORTER_ARCGIS_PASSWORD
      setenv ASSETS_TRACKING_SERVICE_EXPORTER_ARCGIS_ITEM_ID c85b4a215ef045aeb452c5ae28d6644d
      setenv ASSETS_TRACKING_SERVICE_EXPORTER_GEOJSON_OUTPUT_PATH $DEPLOY_HOME/projects/$DEPLOY_APP_NAME/exports/summary.geojson

      prepend-path PATH \${basedir}/bin
      EOF
    - cat module.tpl
    - ssh $DEPLOY_USER@$DEPLOY_HOST "mkdir -p $DEPLOY_MODULE_BASE"
    - scp module.tpl $DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_MODULE_BASE/latest
  rules:
    - if: $CI_COMMIT_TAG

release:
  stage: 📣 release
  needs:
    - job: publish
  image: registry.gitlab.com/gitlab-org/release-cli:latest
  before_script:
    - *before_script_release

    - export TAG_NO_PREFIX=$(echo $CI_COMMIT_TAG | cut -c 2-)
    # for a string v0.8.13, replace last digit to always be 0
    - export TAG_NO_PATCH=$(echo $CI_COMMIT_TAG | sed 's/[0-9]$/0/')

    - 'curl -s -H "Authorization: Bearer $PROJECT_ACCESS_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages" > data.json'
    - export PACKAGE_ID=$(cat data.json | jq -r ".[] | select(.version==\"$TAG_NO_PREFIX\") | .id") && rm data.json

    - 'curl -s -H "Authorization: Bearer $PROJECT_ACCESS_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/milestones?title=$CI_COMMIT_TAG" > milestone_exact.json'
    - 'curl -s -H "Authorization: Bearer $PROJECT_ACCESS_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/milestones?title=$TAG_NO_PATCH" > milestone-minor.json'
    - export MILESTONE_TITLE_EXACT=$(cat milestone_exact.json | jq -r ".[0] | .title") && rm milestone_exact.json
    - export MILESTONE_TITLE_MINOR=$(cat milestone-minor.json | jq -r ".[0] | .title") && rm milestone-minor.json
    - >
      if [ "$MILESTONE_TITLE_EXACT" != "null" ]; then
          export MILESTONE_TITLE=$MILESTONE_TITLE_EXACT
      elif [ "$MILESTONE_TITLE_MINOR" != "null" ]; then
          export MILESTONE_TITLE=$MILESTONE_TITLE_MINOR
      else
          export MILESTONE_TITLE=""
      fi

    - curl -s -L -O https://github.com/taiki-e/parse-changelog/releases/download/v0.6.8/parse-changelog-x86_64-unknown-linux-musl.tar.gz
    - tar -xzf parse-changelog-x86_64-unknown-linux-musl.tar.gz -C /usr/local/bin/
    - parse-changelog CHANGELOG.md "$TAG_NO_PREFIX" > changelog.txt

    # the release section cannot access variables defined in a script but can read from a file :|
    - echo "$TAG_NO_PREFIX" > tag_no_prefix.txt
    - echo "$PACKAGE_ID" > package_id.txt
    - echo "$MILESTONE_TITLE" > milestone_title.txt
  script:
    - echo 'releasing'
  release:
    name: $(cat tag_no_prefix.txt)
    tag_name: $CI_COMMIT_TAG
    milestones:
      - $(cat milestone_title.txt)
    description: $(cat changelog.txt)
    assets:
      links:
        - name: README
          url: '$CI_PROJECT_URL/-/blob/$CI_COMMIT_TAG/README.md'
          link_type: runbook
        - name: Python Package
          url: '$CI_PROJECT_URL/-/packages/$(cat package_id.txt)'
          link_type: package
  rules:
    - if: $CI_COMMIT_TAG