From 58836d9dca6276d862df6e7cbffaf88906152381 Mon Sep 17 00:00:00 2001 From: Ramkumar Vengadakrishnan Date: Tue, 7 Nov 2023 15:28:10 -0600 Subject: [PATCH] feat: Updated CI to migrate from toolsmith to shepherd Authored-by: Ramkumar Vengadakrishnan [#186418591] - Update tile-generator CI to move away from using env app concourse resource to shepherd v2 --- ci/generate_pipeline_yml.py | 2 +- ci/pipeline.yml.jinja2 | 126 +++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 59 deletions(-) diff --git a/ci/generate_pipeline_yml.py b/ci/generate_pipeline_yml.py index e38c2c3..982e611 100755 --- a/ci/generate_pipeline_yml.py +++ b/ci/generate_pipeline_yml.py @@ -3,7 +3,7 @@ import os from jinja2 import Template -clusters = ['2_11_lts2', '2_12', '2_13', '3_0', '4_0', '5_0_prerelease'] +clusters = ['-2_11', '-2_12', '-2_13', '_three', '_four', '-5_0'] # Commenting out this as we only have one example and it breaks tiles = [] # [d for d in os.listdir('../examples') if os.path.isdir(os.path.join('../examples', d))] diff --git a/ci/pipeline.yml.jinja2 b/ci/pipeline.yml.jinja2 index c308a44..30fe2a1 100644 --- a/ci/pipeline.yml.jinja2 +++ b/ci/pipeline.yml.jinja2 @@ -13,10 +13,10 @@ groups: {% endfor %}{% raw %} - build-tile - acceptance-tests - {% endraw %}{% for cluster in clusters %}- claim-{{ cluster }} - - deploy-tile-{{ cluster }} - - deployment-tests-{{ cluster }} - - remove-tile-{{ cluster }}{% if not loop.last %} + {% endraw %}{% for cluster in clusters %}- claim{{ cluster }} + - deploy-tile{{ cluster }} + - deployment-tests{{ cluster }} + - remove-tile{{ cluster }}{% if not loop.last %} {% endif %}{% endfor %}{% raw %} - name: release @@ -45,10 +45,11 @@ resource_types: repository: harbor-repo.vmware.com/dockerhub-proxy-cache/pivotalcf/pivnet-resource tag: latest-final -- name: pcf-pool - type: docker-image +- name: shepherd + type: registry-image source: - repository: harbor-repo.vmware.com/dockerhub-proxy-cache/cftoolsmiths/toolsmiths-envs-resource + repository: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource + tag: v1 resources: @@ -146,12 +147,17 @@ resources: username: ((harbor.username)) password: ((harbor.token)) -{% endraw %}{% for cluster in clusters %}- name: pcf-environment-{{ cluster }} - type: pcf-pool +{% endraw %}{% for cluster in clusters %}- name: pcf-environment{{ cluster }} + type: shepherd source: - api_token: {% raw %}((toolsmiths_api_token)){% endraw %} - hostname: environments.toolsmiths.cf-app.com - pool_name: us_{{cluster}}{% if not loop.last %} + url: https://v2.shepherd.run + service-account-key: ((shepherd.svc-tile-generator)) + lease: + namespace: tas-partner-tiles + pool: + namespace: official + name: tas{{ cluster }}{% if not loop.last %} + compatibility-mode: environments-app {% endif %}{% endfor %} @@ -489,7 +495,7 @@ jobs: text: 'Acceptance tests failed, check for details' username: Tile Generator -{% endraw %}{% for cluster in clusters %}- name: claim-{{ cluster }} +{% endraw %}{% for cluster in clusters %}- name: claim{{ cluster }} plan: - in_parallel: - get: tile-generator-docker-image @@ -505,29 +511,32 @@ jobs: passed: [ acceptance-tests ] - get: tile-generator-package-release passed: [ acceptance-tests ] - - put: pcf-environment-{{ cluster }} + - put: pcf-environment{{ cluster }} params: - action: claim + action: create + duration: 6h + resource: pcf-environment{{ cluster }} + timeout: 2h -- name: deploy-tile-{{ cluster }} +- name: deploy-tile{{ cluster }} disable_manual_trigger: true plan: - in_parallel: - get: tile-generator-docker-image - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] - get: generated-tile - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] - get: tile-generator-repo - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] - get: version - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] trigger: true - get: tile-generator-package - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] - get: tile-generator-package-release - passed: [ claim-{{ cluster }} ] - - get: pcf-environment-{{ cluster }} - passed: [ claim-{{ cluster }} ] + passed: [ claim{{ cluster }} ] + - get: pcf-environment{{ cluster }} + passed: [ claim{{ cluster }} ] trigger: true - task: run-deploy-tile image: tile-generator-docker-image @@ -535,7 +544,7 @@ jobs: platform: linux inputs: - name: tile-generator-repo - - name: pcf-environment-{{ cluster }} + - name: pcf-environment{{ cluster }} - name: generated-tile run: path: bash @@ -550,7 +559,7 @@ jobs: unzip -o test-tile-*.pivotal STEMCELL_OS=`cat metadata/test-tile.yml | grep stemcell_criteria -A 3 | grep os | tr -d ' ' | cut -d ':' -f 2` STEMCELL_VERSION=`cat metadata/test-tile.yml | grep stemcell_criteria -A 3 | grep version | tr -d " '" | cut -d ':' -f 2` - pushd ../pcf-environment-* + pushd ../pcf-environment* STEMCELL_AVAILABLE=`pcf stemcells` popd if [[ $STEMCELL_AVAILABLE == *"stemcell-$STEMCELL_VERSION"*"$STEMCELL_OS"* ]]; then @@ -559,12 +568,12 @@ jobs: wget https://bosh.io/stemcells/bosh-google-kvm-$STEMCELL_OS-go_agent -O bosh.io_stemcells STEMCELL_URL=`cat bosh.io_stemcells | grep light | grep $STEMCELL_VERSION | cut -d '"' -f 2 | head -n 1` wget $STEMCELL_URL - pushd ../pcf-environment-* + pushd ../pcf-environment* pcf upload-stemcell ../generated-tile/light-bosh-stemcell-* popd fi cd .. - tile-generator-repo/ci/scripts/run-deploytile.sh generated-tile pcf-environment-{{ cluster }} + tile-generator-repo/ci/scripts/run-deploytile.sh generated-tile pcf-environment{{ cluster }} on_failure: in_parallel: - put: slack-alert @@ -574,33 +583,33 @@ jobs: text: 'Deploy tile failed, check for details' username: Tile Generator -- name: deployment-tests-{{ cluster }} +- name: deployment-tests{{ cluster }} disable_manual_trigger: true plan: - in_parallel: - get: tile-generator-docker-image - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] - get: generated-tile - passed: [ deploy-tile-{{ cluster }} ] - - get: pcf-environment-{{ cluster }} - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] + - get: pcf-environment{{ cluster }} + passed: [ deploy-tile{{ cluster }} ] trigger: true - get: tile-generator-repo - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] - get: version - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] trigger: true - get: tile-generator-package - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] - get: tile-generator-package-release - passed: [ deploy-tile-{{ cluster }} ] + passed: [ deploy-tile{{ cluster }} ] - task: run-deployment-tests image: tile-generator-docker-image config: platform: linux inputs: - name: tile-generator-repo - - name: pcf-environment-{{ cluster }} + - name: pcf-environment{{ cluster }} - name: generated-tile run: path: bash @@ -610,7 +619,7 @@ jobs: command -v virtualenv >/dev/null 2>&1 || pip install virtualenv # Our env use weak ssl ciphers :( so downgrade the minimum version sed -i 's/DEFAULT@SECLEVEL=2/DEFAULT@SECLEVEL=1/g' /etc/ssl/openssl.cnf - tile-generator-repo/ci/scripts/run-deploymenttests.sh generated-tile pcf-environment-{{ cluster }} + tile-generator-repo/ci/scripts/run-deploymenttests.sh generated-tile pcf-environment{{ cluster }} on_failure: in_parallel: - put: slack-alert @@ -620,33 +629,33 @@ jobs: text: 'Deployment tests failed, check for details' username: Tile Generator -- name: remove-tile-{{ cluster }} +- name: remove-tile{{ cluster }} disable_manual_trigger: true plan: - in_parallel: - get: tile-generator-docker-image - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] - get: generated-tile - passed: [ deployment-tests-{{ cluster }} ] - - get: pcf-environment-{{ cluster }} - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] + - get: pcf-environment{{ cluster }} + passed: [ deployment-tests{{ cluster }} ] trigger: true - get: tile-generator-repo - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] - get: version - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] trigger: true - get: tile-generator-package - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] - get: tile-generator-package-release - passed: [ deployment-tests-{{ cluster }} ] + passed: [ deployment-tests{{ cluster }} ] - task: remove-tile image: tile-generator-docker-image config: platform: linux inputs: - name: tile-generator-repo - - name: pcf-environment-{{ cluster }} + - name: pcf-environment{{ cluster }} - name: generated-tile run: path: bash @@ -655,7 +664,7 @@ jobs: - | # Our env use weak ssl ciphers :( so downgrade the minimum version sed -i 's/DEFAULT@SECLEVEL=2/DEFAULT@SECLEVEL=1/g' /etc/ssl/openssl.cnf - tile-generator-repo/ci/scripts/run-removetile.sh generated-tile pcf-environment-{{ cluster }} + tile-generator-repo/ci/scripts/run-removetile.sh generated-tile pcf-environment{{ cluster }} on_failure: put: slack-alert params: @@ -663,10 +672,11 @@ jobs: icon_emoji: ':concourse:' text: 'Remove tile failed, check for details' username: Tile Generator - - put: pcf-environment-{{ cluster }} + - put: pcf-environment{{ cluster }} params: - action: unclaim - env_file: pcf-environment-{{ cluster }}/metadata{% if not loop.last %} + inputs: [] + action: release + resource: pcf-environment{{ cluster }}{% if not loop.last %} {% endif %}{% endfor %}{% raw %} @@ -675,15 +685,15 @@ jobs: plan: - in_parallel: - get: tile-generator-docker-image - passed: [ {% endraw %}{% for cluster in clusters %}remove-tile-{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] + passed: [ {% endraw %}{% for cluster in clusters %}remove-tile{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] - get: tile-generator-repo - passed: [ {% endraw %}{% for cluster in clusters %}remove-tile-{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] + passed: [ {% endraw %}{% for cluster in clusters %}remove-tile{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] - get: tile-generator-package - passed: [ {% endraw %}{% for cluster in clusters %}remove-tile-{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] + passed: [ {% endraw %}{% for cluster in clusters %}remove-tile{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] - get: tile-generator-package-release - passed: [ {% endraw %}{% for cluster in clusters %}remove-tile-{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] + passed: [ {% endraw %}{% for cluster in clusters %}remove-tile{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] - get: version - passed: [ {% endraw %}{% for cluster in clusters %}remove-tile-{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] + passed: [ {% endraw %}{% for cluster in clusters %}remove-tile{{ cluster }}{% if not loop.last %}, {% endif %}{% endfor %}{% raw %} ] trigger: true - name: create-release