From 4f6d56484911274672a908ca82e3aa2eaf922082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 9 Sep 2023 22:00:09 +0200 Subject: [PATCH 01/24] eessi-hpc.org -> eessi.io --- prepare-client-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 55a93ce8..5696fb34 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -40,10 +40,10 @@ - name: Make EESSI CVMFS domain configuration file copy: content: | - # Stratum 1 servers for the eessi-hpc.org domain + # Stratum 1 servers for the eessi.io domain CVMFS_SERVER_URL="{{ item.urls|join(';') }}" - # Public keys for the eessi-hpc.org domain + # Public keys for the eessi.io domain CVMFS_KEYS_DIR="/etc/cvmfs/keys/{{ item.domain }}" # Enable or disable the Geo API, based on the configuration file From 5c64985f87e62ce9cc8aa6f6bfdbf925467b3bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sat, 9 Sep 2023 22:00:26 +0200 Subject: [PATCH 02/24] prepare for eessi.io domain --- inventory/group_vars/all.yml | 77 ++++-------------------------------- 1 file changed, 7 insertions(+), 70 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 1ff0b8bf..6cae5577 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -8,7 +8,7 @@ eessi_cvmfs_repos_enabled: true eessi_email: eessi@list.rug.nl # -# Defaults for eessi-hpc.org repos. +# Defaults for eessi.io repos. # # We do not use a config repo anymore. @@ -17,57 +17,13 @@ eessi_cvmfs_config_repo: {} # Public keys for the repositories, which you can find on the Stratum 0 at: # /etc/cvmfs/keys/*.pub # Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys. -eessi_cvmfs_keys: - - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub - key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA - F20UjePR7mJyWB13eTBXnjq7PLKKilfk2llLKetPIRMEmbvp97NiQ9ALe/tiGDzT - 1Wg82Mf9oO4DNQYrTfE/MP5inIrMtSv1WnYJYV4Bgm24s8IhHXytIf1D4qt0x1v5 - 0yRui0dwaZrErWKRKufbJ6ehmEDHuVSEAujrORyig/jYFKVyTaCsXnmwFDJmX2Uw - RGCaf0M9ZLYs9t/SWA0s08rle5kH82hgBsIvUr7PLT7hbU407VK6+Y02tuxbI0JR - +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk - BwIDAQAB - -----END PUBLIC KEY----- - - path: /etc/cvmfs/keys/eessi-hpc.org/pilot.eessi-hpc.org.pub - key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg - 8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8 - v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo - 4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ - uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm - HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby - FQIDAQAB - -----END PUBLIC KEY----- - # The following (duplicate) key can be removed after this PR has been merged: - # https://github.com/galaxyproject/ansible-cvmfs/pull/32 - - path: /etc/cvmfs/keys/eessi-hpc.org/data.eessi-hpc.org.pub - key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg - 8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8 - v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo - 4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ - uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm - HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby - FQIDAQAB - -----END PUBLIC KEY----- - - path: /etc/cvmfs/keys/eessi-hpc.org/ci.eessi-hpc.org.pub - key: | - -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg - 8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8 - v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo - 4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ - uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm - HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby - FQIDAQAB - -----END PUBLIC KEY----- +eessi_cvmfs_keys: [] +# - path: /etc/cvmfs/keys/eessi.io/software.eessi.io +# key: | # URLs for all the Stratum 1 servers. eessi_cvmfs_server_urls: - - domain: eessi-hpc.org + - domain: eessi.io use_geoapi: true urls: - "http://rug-nl.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@" @@ -77,35 +33,16 @@ eessi_cvmfs_server_urls: # Configuration of all the repositories. eessi_cvmfs_repositories: - - repository: pilot.eessi-hpc.org + - repository: software.eessi.io stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi-hpc.org + key_dir: /etc/cvmfs/keys/eessi.io server_options: - CVMFS_AUTO_GC=false - CVMFS_AUTO_TAG_TIMESPAN="30 days ago" - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: false - - repository: data.eessi-hpc.org - stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org - owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi-hpc.org - server_options: - - CVMFS_AUTO_GC=false - - CVMFS_AUTO_TAG_TIMESPAN="30 days ago" - - CVMFS_GARBAGE_COLLECTION=true - client_options: [] - use_for_ci: false - - repository: ci.eessi-hpc.org - stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org - owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi-hpc.org - server_options: - - CVMFS_AUTO_GC=false - - CVMFS_GARBAGE_COLLECTION=true - client_options: [] - use_for_ci: true # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. From 2b2b7828861c27a00d73587644889ef410959e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 15:41:30 +0200 Subject: [PATCH 03/24] add (temporary) servers and keys for eessi.io --- inventory/group_vars/all.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 6cae5577..27913924 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -17,24 +17,31 @@ eessi_cvmfs_config_repo: {} # Public keys for the repositories, which you can find on the Stratum 0 at: # /etc/cvmfs/keys/*.pub # Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys. -eessi_cvmfs_keys: [] -# - path: /etc/cvmfs/keys/eessi.io/software.eessi.io -# key: | +eessi_cvmfs_keys: + - path: /etc/cvmfs/keys/eessi.io/software.eessi.io.pub + key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/MGKNYeXgAzNGRmyy6c + AHPzXOfA+MPEdDsYNxz4FWCq24HqwDGCdrGFWuAPPnocumMpVqU6G4nhjYNEf7HY + VBXub9sAS7cl7jycN+n82A7hgoZqkTlRpBTCv30zAII7MsCfSakxXx5PLIWFe10i + 8Cs7heb2ZAdDXPCJE1CQMnhknAu1ucKfZz5mB6DP6pGiH30rGyFiEH7M5HsfiA9T + VkfWMPmd10YmboA9GsNhng8SWoWX8rMDIupAmtYTxiS3bdslrAa7hThpymtx6Jcb + ZhEaIPp0ACctyk4+B6ZXnZX0HHtqqpyP6mo2QMMc6fQI9RlvO9hboqoESza5FzJ5 + FQIDAQAB + -----END PUBLIC KEY----- # URLs for all the Stratum 1 servers. eessi_cvmfs_server_urls: - domain: eessi.io use_geoapi: true urls: - - "http://rug-nl.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@" - - "http://bgo-no.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@" - - "http://azure-us-east1.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@" - - "http://aws-eu-west1.stratum1.cvmfs.eessi-infra.org/cvmfs/@fqrn@" + - "http://eessi-cvmfs.s3.eu-west-1.amazonaws.com/cvmfs/@fqrn@" # Configuration of all the repositories. eessi_cvmfs_repositories: - repository: software.eessi.io - stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org + #stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org + stratum0: 195.169.23.15 owner: "{{ cvmfs_repo_owner | default('root') }}" key_dir: /etc/cvmfs/keys/eessi.io server_options: From 85d08702355d23b9b945d1254deb336004bfcead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 15:44:43 +0200 Subject: [PATCH 04/24] replace pilot.eessi-hpc.org by software.eessi.io --- .../build-test-release-client-packages.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test-release-client-packages.yml b/.github/workflows/build-test-release-client-packages.yml index 0faf0a4e..856c3d60 100644 --- a/.github/workflows/build-test-release-client-packages.yml +++ b/.github/workflows/build-test-release-client-packages.yml @@ -147,12 +147,12 @@ jobs: run: sudo cvmfs_config setup - name: Test repository access - run: ls /cvmfs/pilot.eessi-hpc.org/ + run: ls /cvmfs/software.eessi.io/ test-rpm-package: needs: build-linux-packages runs-on: ubuntu-latest - container: + container: image: centos:centos7 options: --device /dev/fuse --privileged @@ -183,10 +183,10 @@ jobs: - name: Mount the repositories #run: cvmfs_config setup - run: mkdir -p /cvmfs/pilot.eessi-hpc.org && mount -t cvmfs pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org + run: mkdir -p /cvmfs/software.eessi.io && mount -t cvmfs software.eessi.io /cvmfs/software.eessi.io - name: Test repository access - run: ls /cvmfs/pilot.eessi-hpc.org/ + run: ls /cvmfs/software.eessi.io/ test-tar-package: needs: build-linux-packages @@ -223,7 +223,7 @@ jobs: run: sudo cvmfs_config setup - name: Test repository access - run: ls /cvmfs/pilot.eessi-hpc.org/ + run: ls /cvmfs/software.eessi.io/ # test-macos-package: # needs: build-macos-package @@ -254,10 +254,10 @@ jobs: # run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local # # - name: Mount the repositories -# run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org +# run: sudo mkdir -p /Users/Shared/cvmfs/software.eessi.io && sudo mount -t cvmfs software.eessi.io /Users/Shared/cvmfs/software.eessi.io # # - name: Test repository access -# run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/ +# run: ls /Users/Shared/cvmfs/software.eessi.io/ release: #needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package, test-tar-package] From 1f5e649e99f6a212af7d2f5e54d67eee0597b9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 29 Sep 2023 15:48:51 +0200 Subject: [PATCH 05/24] remove commented line with old stratum 0 --- inventory/group_vars/all.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 27913924..188b3d42 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -40,7 +40,6 @@ eessi_cvmfs_server_urls: # Configuration of all the repositories. eessi_cvmfs_repositories: - repository: software.eessi.io - #stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org stratum0: 195.169.23.15 owner: "{{ cvmfs_repo_owner | default('root') }}" key_dir: /etc/cvmfs/keys/eessi.io From 9a6400785b9f125445df6c25e8407aa8fddec071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Oct 2023 14:36:29 +0200 Subject: [PATCH 06/24] ingest to new software.eessi.io repo --- scripts/ingest-tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ingest-tarball.sh b/scripts/ingest-tarball.sh index 25e3883b..27899cfc 100755 --- a/scripts/ingest-tarball.sh +++ b/scripts/ingest-tarball.sh @@ -12,7 +12,7 @@ # Only if it passes these checks, the tarball gets ingested to the base dir in the repository specified below. -repo=pilot.eessi-hpc.org +repo=software.eessi.io basedir=versions decompress="gunzip -c" # list of supported architectures for compat and software layers From 52aa5af397489da515081bb35ff91f533dbdebb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Oct 2023 15:00:33 +0200 Subject: [PATCH 07/24] update comment about generic and vendorless targets --- roles/create_cvmfs_content_structure/files/.cvmfsdirtab | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/create_cvmfs_content_structure/files/.cvmfsdirtab b/roles/create_cvmfs_content_structure/files/.cvmfsdirtab index 89e34c63..191fbe8c 100644 --- a/roles/create_cvmfs_content_structure/files/.cvmfsdirtab +++ b/roles/create_cvmfs_content_structure/files/.cvmfsdirtab @@ -9,7 +9,7 @@ /versions/*/software/*/*/*/*/software /versions/*/software/*/*/*/*/software/*/* /versions/*/software/*/*/*/*/modules -# generic and graviton2 are one level less deep +# generic and some (aarch64) targets are one level less deep /versions/*/software/*/*/*/software /versions/*/software/*/*/*/software/*/* /versions/*/software/*/*/*/modules From 1bafe5b1fcc133ef09ddb4bd154176a9da27d73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 2 Oct 2023 15:02:02 +0200 Subject: [PATCH 08/24] define files and (variant) symlinks for software.eessi.io --- .../vars/software.eessi.io | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 roles/create_cvmfs_content_structure/vars/software.eessi.io diff --git a/roles/create_cvmfs_content_structure/vars/software.eessi.io b/roles/create_cvmfs_content_structure/vars/software.eessi.io new file mode 100644 index 00000000..409376dd --- /dev/null +++ b/roles/create_cvmfs_content_structure/vars/software.eessi.io @@ -0,0 +1,11 @@ +# Specifications of files and symlinks for the software.eessi.io CVMFS repository. +# Paths for files and symlinks should be relative to the root of the repository. +--- +files: + - name: .cvmfsdirtab + dest: '' + mode: '644' + +symlinks: + latest: versions/2023.06 + host_injections: '$(EESSI_HOST_INJECTIONS:-/opt/eessi)' From 476594c335c175dda45fa31c7c8afca6f52e0c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Oct 2023 11:50:38 +0200 Subject: [PATCH 09/24] add .yml extension --- .../vars/{software.eessi.io => software.eessi.io.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/create_cvmfs_content_structure/vars/{software.eessi.io => software.eessi.io.yml} (100%) diff --git a/roles/create_cvmfs_content_structure/vars/software.eessi.io b/roles/create_cvmfs_content_structure/vars/software.eessi.io.yml similarity index 100% rename from roles/create_cvmfs_content_structure/vars/software.eessi.io rename to roles/create_cvmfs_content_structure/vars/software.eessi.io.yml From da9603788d9152315ed5d629769e24a5b51483be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Oct 2023 13:52:16 +0200 Subject: [PATCH 10/24] fix loop and run as repo owner --- roles/create_cvmfs_content_structure/tasks/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/create_cvmfs_content_structure/tasks/main.yml b/roles/create_cvmfs_content_structure/tasks/main.yml index 6e71bb60..e14c6f6f 100644 --- a/roles/create_cvmfs_content_structure/tasks/main.yml +++ b/roles/create_cvmfs_content_structure/tasks/main.yml @@ -6,16 +6,17 @@ ansible.builtin.stat: path: "{{ role_path }}/vars/{{ item.repository }}.yml" with_items: "{{ eessi_cvmfs_repositories }}" - register: repo_vars_file + register: repo_vars_files - name: "Apply do_repo for each repository with a vars file" ansible.builtin.include_tasks: do_repo.yml vars: cvmfs_repo: "{{ repo_vars_file.item.repository }}" when: repo_vars_file.stat.exists - with_items: "{{ repo_vars_file.results }}" + with_items: "{{ repo_vars_files.results }}" loop_control: loop_var: repo_vars_file args: apply: - become: "{{ 'yes' if repo_vars_file.item.owner == 'root' else 'no' }}" + become: "{{ 'true' if repo_vars_file.item.owner != lookup('env', 'USER') else 'false' }}" + become_user: "{{ repo_vars_file.item.owner }}" From d3dedc5c57590219a6f0566b196148ddc524c22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Oct 2023 14:44:12 +0200 Subject: [PATCH 11/24] run create_cvmfs_content_structure role on stratum0 --- stratum0.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/stratum0.yml b/stratum0.yml index 0e234861..6bcde8ed 100644 --- a/stratum0.yml +++ b/stratum0.yml @@ -16,3 +16,4 @@ - role: geerlingguy.repo-epel when: ansible_facts['os_family'] == 'RedHat' - galaxyproject.cvmfs + - create_cvmfs_content_structure From 73d931275081d42b8a8c67917554e446dfe755b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 3 Nov 2023 14:10:27 +0100 Subject: [PATCH 12/24] use DNS names instad of IPs --- inventory/group_vars/all.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 188b3d42..2bae46bf 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -35,12 +35,13 @@ eessi_cvmfs_server_urls: - domain: eessi.io use_geoapi: true urls: - - "http://eessi-cvmfs.s3.eu-west-1.amazonaws.com/cvmfs/@fqrn@" + - "http://aws-eu-central-s1.eessi.science/cvmfs/@fqrn@" + - "http://azure-us-east-s1.eessi.science/cvmfs/@fqrn@" # Configuration of all the repositories. eessi_cvmfs_repositories: - repository: software.eessi.io - stratum0: 195.169.23.15 + stratum0: rug-nl-s0.eessi.science owner: "{{ cvmfs_repo_owner | default('root') }}" key_dir: /etc/cvmfs/keys/eessi.io server_options: From 83d02b04c650d502aa324a003be1c6f39786452c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 13:18:54 +0100 Subject: [PATCH 13/24] remove 'latest' symlink --- roles/create_cvmfs_content_structure/vars/software.eessi.io.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/create_cvmfs_content_structure/vars/software.eessi.io.yml b/roles/create_cvmfs_content_structure/vars/software.eessi.io.yml index 409376dd..35ff95e6 100644 --- a/roles/create_cvmfs_content_structure/vars/software.eessi.io.yml +++ b/roles/create_cvmfs_content_structure/vars/software.eessi.io.yml @@ -7,5 +7,4 @@ files: mode: '644' symlinks: - latest: versions/2023.06 host_injections: '$(EESSI_HOST_INJECTIONS:-/opt/eessi)' From 7f9b6c1a7cb50e9c21d8f2baea9bdcb56752fbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 13:22:13 +0100 Subject: [PATCH 14/24] add description to software.eessi.io repo --- inventory/group_vars/all.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 2bae46bf..935e918d 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -50,6 +50,7 @@ eessi_cvmfs_repositories: - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: false + description: "Repository that contains the EESSI software stacks." # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. From 009087070849ac8b908aabc6c295fae8a080b9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 14:08:15 +0100 Subject: [PATCH 15/24] only use CI-enabled repos for all playbooks --- .github/workflows/test-playbook.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index d6db6487..9e825071 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -16,12 +16,9 @@ echo ' - 127.0.0.1' >> inventory/local_site_specific_vars.yml export CVMFS_GEO_DB_FILE=NONE # Only test CI suitable repos on the Stratum 1, as the other ones may be very large. -if [ $playbook == "stratum1.yml" ] -then - cat >> inventory/local_site_specific_vars.yml <<-EOF - cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}" - EOF -fi +cat >> inventory/local_site_specific_vars.yml <<-EOF + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}" +EOF # Install the Ansible dependencies. ansible-galaxy role install -r requirements.yml -p ./roles From f6383af1f9280e56ce2a5a90017599be412e4636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 14:14:16 +0100 Subject: [PATCH 16/24] add ci repo --- inventory/group_vars/all.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 947b785b..d38cc93f 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -51,6 +51,16 @@ eessi_cvmfs_repositories: client_options: [] use_for_ci: false description: "Repository that contains the EESSI software stacks." + - repository: ci.eessi.io + stratum0: rug-nl-s0.eessi.science + owner: "{{ cvmfs_repo_owner | default('root') }}" + key_dir: /etc/cvmfs/keys/eessi.io + server_options: + - CVMFS_AUTO_GC=false + - CVMFS_GARBAGE_COLLECTION=true + client_options: [] + use_for_ci: true + description: "Small-sized repository used for CI purposes." # Meta information for CVMFS Stratum servers cvmfs_server_meta_administrator: "EESSI CVMFS Administrators" From 2de1d94e7b0211bb24def7fa814fb8e582ab030c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 14:27:12 +0100 Subject: [PATCH 17/24] first check cvmfs_repositories, then use eessi_cvmfs_repositories --- roles/cvmfs_server_meta_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml index 53a1a55a..6ab5038a 100644 --- a/roles/cvmfs_server_meta_info/tasks/main.yml +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -17,6 +17,6 @@ recommended-stratum0: "http://{{ item.stratum0 }}/cvmfs/{{ item.repository }}" recommended-stratum1s: "{{ eessi_cvmfs_server_urls | selectattr('domain', 'in', item.repository) | map(attribute='urls') | map('regex_replace', '@fqrn@', item.repository) }}" - with_items: "{{ eessi_cvmfs_repositories }}" + with_items: "{{ cvmfs_repositories | default(eessi_cvmfs_repositories) }}" when: "'cvmfsstratum0servers' in group_names or cvmfs_role == 'stratum0'" ... From b04dd46bda35d1412c385d9ec5b0f067591fe40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 16:34:43 +0100 Subject: [PATCH 18/24] remove ci.eessi.io for now --- inventory/group_vars/all.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index d38cc93f..947b785b 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -51,16 +51,6 @@ eessi_cvmfs_repositories: client_options: [] use_for_ci: false description: "Repository that contains the EESSI software stacks." - - repository: ci.eessi.io - stratum0: rug-nl-s0.eessi.science - owner: "{{ cvmfs_repo_owner | default('root') }}" - key_dir: /etc/cvmfs/keys/eessi.io - server_options: - - CVMFS_AUTO_GC=false - - CVMFS_GARBAGE_COLLECTION=true - client_options: [] - use_for_ci: true - description: "Small-sized repository used for CI purposes." # Meta information for CVMFS Stratum servers cvmfs_server_meta_administrator: "EESSI CVMFS Administrators" From cb4faeef6d8e8aa4bd1c81ba7ac9381b91e84823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 16:34:57 +0100 Subject: [PATCH 19/24] only add the metadata for servers with at least one repo --- roles/cvmfs_server_meta_info/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml index 6ab5038a..93c7ce02 100644 --- a/roles/cvmfs_server_meta_info/tasks/main.yml +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -7,6 +7,7 @@ group: root mode: 0644 become: true + when: (cvmfs_repositories | length) > 0 - name: Create meta information for each CVMFS repository ansible.builtin.include_tasks: repo_meta_info.yml From d32ecbaab02293ce273d9ec0ca909e7a1bbd87ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 16:44:00 +0100 Subject: [PATCH 20/24] first check cvmfs_repositories, then use eessi_cvmfs_repositories --- roles/create_cvmfs_content_structure/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/create_cvmfs_content_structure/tasks/main.yml b/roles/create_cvmfs_content_structure/tasks/main.yml index e14c6f6f..39c238b6 100644 --- a/roles/create_cvmfs_content_structure/tasks/main.yml +++ b/roles/create_cvmfs_content_structure/tasks/main.yml @@ -5,7 +5,7 @@ delegate_to: localhost ansible.builtin.stat: path: "{{ role_path }}/vars/{{ item.repository }}.yml" - with_items: "{{ eessi_cvmfs_repositories }}" + with_items: "{{ cvmfs_repositories | default(eessi_cvmfs_repositories) }}" register: repo_vars_files - name: "Apply do_repo for each repository with a vars file" From 27bc4916143711635c8e0443e44fd6a8c657adf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 9 Nov 2023 17:04:44 +0100 Subject: [PATCH 21/24] revert removal that checks for stratum1 playbook --- .github/workflows/test-playbook.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-playbook.sh b/.github/workflows/test-playbook.sh index 9e825071..d6db6487 100755 --- a/.github/workflows/test-playbook.sh +++ b/.github/workflows/test-playbook.sh @@ -16,9 +16,12 @@ echo ' - 127.0.0.1' >> inventory/local_site_specific_vars.yml export CVMFS_GEO_DB_FILE=NONE # Only test CI suitable repos on the Stratum 1, as the other ones may be very large. -cat >> inventory/local_site_specific_vars.yml <<-EOF - cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}" -EOF +if [ $playbook == "stratum1.yml" ] +then + cat >> inventory/local_site_specific_vars.yml <<-EOF + cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}" + EOF +fi # Install the Ansible dependencies. ansible-galaxy role install -r requirements.yml -p ./roles From 45aa2ff9c9adeb4197e6627b456887d8cd6cb558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 10 Nov 2023 09:23:07 +0100 Subject: [PATCH 22/24] use localhost, since this is always running on stratum 0 --- roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml index 20afe2a8..120f7fd5 100644 --- a/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml +++ b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml @@ -18,7 +18,7 @@ - name: Get the current meta information of this repository from the server ansible.builtin.command: - cmd: "cvmfs_swissknife info -M -r {{ this_cvmfs_repo_meta['recommended-stratum0'] }}" + cmd: "cvmfs_swissknife info -M -r http://localhost:{{ cvmfs_stratum0_http_ports[0] | default('80') }}/cvmfs/{{ this_cvmfs_repo.repository }}" changed_when: false register: current_repo_meta From 3e21537680c96a87a1fc347dcf77d2ca18500dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 10 Nov 2023 09:28:34 +0100 Subject: [PATCH 23/24] set default repo owner to root --- roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml index 120f7fd5..56a8a59c 100644 --- a/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml +++ b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml @@ -26,7 +26,7 @@ ansible.builtin.command: cmd: "cvmfs_server update-repoinfo -f {{ tmp_json_file.path }} {{ this_cvmfs_repo.repository }}" when: (current_repo_meta.stdout | checksum) != json_file_stat.stat.checksum - become_user: "{{ cvmfs_repo_owner }}" + become_user: "{{ cvmfs_repo_owner | default('root') }}" - name: Remove temporary json file ansible.builtin.file: From e4f085bb04176147446ea008bc4da5e2affccf58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 10 Nov 2023 14:33:16 +0100 Subject: [PATCH 24/24] use new public key --- inventory/group_vars/all.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 947b785b..0a73acf6 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -21,13 +21,13 @@ eessi_cvmfs_keys: - path: /etc/cvmfs/keys/eessi.io/software.eessi.io.pub key: | -----BEGIN PUBLIC KEY----- - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/MGKNYeXgAzNGRmyy6c - AHPzXOfA+MPEdDsYNxz4FWCq24HqwDGCdrGFWuAPPnocumMpVqU6G4nhjYNEf7HY - VBXub9sAS7cl7jycN+n82A7hgoZqkTlRpBTCv30zAII7MsCfSakxXx5PLIWFe10i - 8Cs7heb2ZAdDXPCJE1CQMnhknAu1ucKfZz5mB6DP6pGiH30rGyFiEH7M5HsfiA9T - VkfWMPmd10YmboA9GsNhng8SWoWX8rMDIupAmtYTxiS3bdslrAa7hThpymtx6Jcb - ZhEaIPp0ACctyk4+B6ZXnZX0HHtqqpyP6mo2QMMc6fQI9RlvO9hboqoESza5FzJ5 - FQIDAQAB + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyau1UFUcoiqpE5U9StON + W0Trc3PM02AA5kYgknrqZJdusj5PcNg7rhOnrd+SX8BIiVtVMr6mqORXsJ2FNydJ + lfm512GcmurM154m6Hz4I+UttcpVLe1CvlyxjjCrN0siSiPnLDONy3Ih1PWyRlA/ + sobSaIhx3Vyn1IzC0nbjstKObAYJs2kjokS+cipX2oIXJSoxkhvXcxmiMCi2GQIs + KTQDT+bATs9eK+ZqY4n5x8VX87AlMBQuSCP5mA8VQPx4aCf8AjHWAn3qmPrhk3od + Z/Cl9vrYzoWJT1HOyumv7aEv7UlY81++3w/AwYKXLJI6ioop2AotjrUm3w2U3+k7 + WQIDAQAB -----END PUBLIC KEY----- # URLs for all the Stratum 1 servers.