From 1bbe56f6c4fd628944d0d8402938c0221f307f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 7 Jun 2021 22:50:49 +0200 Subject: [PATCH 01/22] add step that changes gentoo overlay from using rsync to git --- .../compatibility_layer/tasks/add_overlay.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 28a81364..e6f69337 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -58,6 +58,28 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +- name: Use git instead of rsync for the Gentoo overlay + copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/gentoo.conf" + mode: "0644" + content: | + [DEFAULT] + main-repo = gentoo + sync-git-pull-extra-opts = --quiet + + [gentoo] + priority = 1 + location = {{ gentoo_prefix_path }}/var/db/repos/gentoo + sync-uri = https://github.com/gentoo/gentoo.git + sync-type = git + auto-sync = Yes + clone-depth = 1 + +- name: Remove the old contents of the Gentoo overlay + file: + state: absent + path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" + - name: Sync the repositories portage: sync: 'yes' From a1c31829ff171f96eb8311f35eeb951d4554f56c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:13:52 +0200 Subject: [PATCH 02/22] add step for generating post-sync hooks --- .../roles/compatibility_layer/tasks/add_overlay.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index e6f69337..363a1a4d 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -75,6 +75,18 @@ auto-sync = Yes clone-depth = 1 +- name: Add post-sync hooks for Gentoo overlay + template: + src: "{{ item }}.j2" + dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}" + with_items: + - sync_gentoo_cache + - sync_gentoo_dtd + - sync_gentoo_glsa + - sync_gentoo_news + - sync_gentoo_projects_xml + - sync_overlay_cache + - name: Remove the old contents of the Gentoo overlay file: state: absent From 6f2ff40cb00e028508cedd92880731f27c1dc85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:15:41 +0200 Subject: [PATCH 03/22] extra step for creating postsync dir --- .../roles/compatibility_layer/tasks/add_overlay.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 363a1a4d..7ae73129 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -75,6 +75,12 @@ auto-sync = Yes clone-depth = 1 +- name: Create repo.postsync.d directory + file: + path: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d" + state: directory + mode: '0755' + - name: Add post-sync hooks for Gentoo overlay template: src: "{{ item }}.j2" From bf7c37c9f698a7a8686d6b6f64c58b6ed1101cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:16:40 +0200 Subject: [PATCH 04/22] rename step --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 7ae73129..cae02a86 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -93,7 +93,7 @@ - sync_gentoo_projects_xml - sync_overlay_cache -- name: Remove the old contents of the Gentoo overlay +- name: Remove the old Gentoo overlay directory file: state: absent path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" From 66482ca5fdaaabebbd74a3c6ab3cf702f6b63ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:23:23 +0200 Subject: [PATCH 05/22] add mode for template task --- .../roles/compatibility_layer/tasks/add_overlay.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index cae02a86..14fdb6ea 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -66,7 +66,7 @@ [DEFAULT] main-repo = gentoo sync-git-pull-extra-opts = --quiet - + [gentoo] priority = 1 location = {{ gentoo_prefix_path }}/var/db/repos/gentoo @@ -85,13 +85,14 @@ template: src: "{{ item }}.j2" dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}" + mode: 0755 with_items: - sync_gentoo_cache - sync_gentoo_dtd - sync_gentoo_glsa - sync_gentoo_news - - sync_gentoo_projects_xml - - sync_overlay_cache + #- sync_gentoo_projects_xml + #- sync_overlay_cache - name: Remove the old Gentoo overlay directory file: From fd52eb74ebc5bb4d456fcbf8ac9c2243254396ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:33:24 +0200 Subject: [PATCH 06/22] also generate metadata for custom overlays --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 14fdb6ea..534be4d7 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -91,8 +91,8 @@ - sync_gentoo_dtd - sync_gentoo_glsa - sync_gentoo_news - #- sync_gentoo_projects_xml - #- sync_overlay_cache + - sync_gentoo_projects_xml + - sync_overlay_cache - name: Remove the old Gentoo overlay directory file: From d98d6ed6d9f22db0a639c62f49aa1e2f3211b7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:36:46 +0200 Subject: [PATCH 07/22] add postsync scripts --- .../templates/sync_gentoo_cache.j2 | 34 +++++++++++++++++++ .../templates/sync_gentoo_dtd.j2 | 17 ++++++++++ .../templates/sync_gentoo_glsa.j2 | 17 ++++++++++ .../templates/sync_gentoo_news.j2 | 17 ++++++++++ .../templates/sync_overlay_cache.j2 | 14 ++++++++ 5 files changed, 99 insertions(+) create mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 create mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 create mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 create mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 create mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 new file mode 100755 index 00000000..f1f572d2 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 @@ -0,0 +1,34 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +if [[ -f ${repository_path}/metadata/timestamp.x ]]; then + portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) +else + portage_current_timestamp=0 +fi + +ebegin "Fetching metadata timestamp for ${repository_name}" +rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x +eend $? +portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) + +if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then + ebegin "Fetching pre-generated metadata cache for ${repository_name}" + rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/ + eend $? +else + einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)" +fi + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 new file mode 100755 index 00000000..81d8ff0f --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +DTDDIR="${repository_path}"/metadata/dtd +ebegin "Updating DTDs" +if [[ -e ${DTDDIR} ]]; then + git -C "${DTDDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 new file mode 100755 index 00000000..220e3aea --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +GLSADIR="${repository_path}"/metadata/glsa +ebegin "Updating GLSAs" +if [[ -e ${GLSADIR} ]]; then + git -C "${GLSADIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 new file mode 100755 index 00000000..dbab56ca --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +NEWSDIR="${repository_path}"/metadata/news +ebegin "Updating news items" +if [[ -e ${NEWSDIR} ]]; then + git -C "${NEWSDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}" +fi +eend $? "Try to remove ${NEWSDIR}" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 new file mode 100755 index 00000000..e373c810 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +repository_name="${1}" + +[[ ${repository_name} == "gentoo" ]] && exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? From e433003bd98541b677f049ca1452c167f84335fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:38:50 +0200 Subject: [PATCH 08/22] add other script, which we won't use for now --- .../templates/sync_gentoo_projects_xml.j2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 new file mode 100644 index 00000000..07067e5b --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 @@ -0,0 +1,12 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh + +ebegin "Updating projects.xml" +wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml +eend $? From 2f1b85e1339058438ef78f49ad9fa221e2f0a31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 9 Jun 2021 17:39:25 +0200 Subject: [PATCH 09/22] dont run sync_gentoo_projects_xml for now --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 534be4d7..aabc0f90 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -91,8 +91,8 @@ - sync_gentoo_dtd - sync_gentoo_glsa - sync_gentoo_news - - sync_gentoo_projects_xml - sync_overlay_cache + #- sync_gentoo_projects_xml - name: Remove the old Gentoo overlay directory file: From c88d68a9a93e3c342cff9e8b1b95527c9bc87081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 10 Jun 2021 09:39:01 +0200 Subject: [PATCH 10/22] use copy instead of template for post-sync scripts --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index aabc0f90..e3d05bc8 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -82,8 +82,8 @@ mode: '0755' - name: Add post-sync hooks for Gentoo overlay - template: - src: "{{ item }}.j2" + copy: + src: "{{ item }}" dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}" mode: 0755 with_items: From e2e694ef52b2449517fe4b8ebcff4a82a15e0759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 10 Jun 2021 09:42:37 +0200 Subject: [PATCH 11/22] use portageq to find EPREFIX, remove .j2 extensions --- .../files/sync_gentoo_cache | 34 +++++++++++++++++++ .../compatibility_layer/files/sync_gentoo_dtd | 17 ++++++++++ .../files/sync_gentoo_glsa | 17 ++++++++++ .../files/sync_gentoo_news | 17 ++++++++++ .../files/sync_gentoo_projects_xml | 12 +++++++ .../files/sync_overlay_cache | 14 ++++++++ 6 files changed, 111 insertions(+) create mode 100755 ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache create mode 100755 ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd create mode 100755 ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa create mode 100755 ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news create mode 100644 ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml create mode 100755 ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache new file mode 100755 index 00000000..65b4f709 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache @@ -0,0 +1,34 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +if [[ -f ${repository_path}/metadata/timestamp.x ]]; then + portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) +else + portage_current_timestamp=0 +fi + +ebegin "Fetching metadata timestamp for ${repository_name}" +rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x +eend $? +portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) + +if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then + ebegin "Fetching pre-generated metadata cache for ${repository_name}" + rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/ + eend $? +else + einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)" +fi + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd new file mode 100755 index 00000000..c12f6c3b --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +DTDDIR="${repository_path}"/metadata/dtd +ebegin "Updating DTDs" +if [[ -e ${DTDDIR} ]]; then + git -C "${DTDDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa new file mode 100755 index 00000000..e2a2face --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +GLSADIR="${repository_path}"/metadata/glsa +ebegin "Updating GLSAs" +if [[ -e ${GLSADIR} ]]; then + git -C "${GLSADIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news new file mode 100755 index 00000000..92c69a63 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +NEWSDIR="${repository_path}"/metadata/news +ebegin "Updating news items" +if [[ -e ${NEWSDIR} ]]; then + git -C "${NEWSDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}" +fi +eend $? "Try to remove ${NEWSDIR}" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml new file mode 100644 index 00000000..fe1ac0e9 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml @@ -0,0 +1,12 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +ebegin "Updating projects.xml" +wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache b/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache new file mode 100755 index 00000000..e7f5ecf0 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache @@ -0,0 +1,14 @@ +#!/bin/bash + +repository_name="${1}" + +[[ ${repository_name} == "gentoo" ]] && exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? From c01cf2d321c01c99bccfc3f7bca701affb5c183a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 11 Jun 2021 16:26:31 +0200 Subject: [PATCH 12/22] add reframe test for validating that gentoo overlay uses git --- test/compat_layer.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/compat_layer.py b/test/compat_layer.py index 132f2009..17e0f500 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -182,3 +182,21 @@ def __init__(self): sn.assert_eq(self.exit_code, 0), sn.assert_found(f'\n/{self.symlink_to_host}\n', self.stdout), ]) + + +@rfm.simple_test +class GentooOverlayGitTest(RunInGentooPrefixTest): + def __init__(self): + # the switch to git was made in pilot version 2021.06 + self.skip_if(self.eessi_version == '2021.03') + + super().__init__() + self.descr = 'Verify that the Gentoo overlay is synced using git.' + self.command = f'emerge --info' + + gentoo_git_repo_info = '''gentoo + location: /cvmfs/pilot.eessi-hpc.org/2021.06/compat/linux/x86_64/var/db/repos/gentoo + sync-type: git + sync-uri: https://github.com/gentoo/gentoo.git''' + + self.sanity_patterns = sn.assert_found(gentoo_git_repo_info, self.stdout) From 7a08570eb8bb54ea4e12b64545ba033dc02b70a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 11 Jun 2021 18:03:19 +0200 Subject: [PATCH 13/22] fix hardcoded path --- test/compat_layer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/compat_layer.py b/test/compat_layer.py index 17e0f500..dcd6f461 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -194,9 +194,10 @@ def __init__(self): self.descr = 'Verify that the Gentoo overlay is synced using git.' self.command = f'emerge --info' + gentoo_repo_dir = os.path.join(self.compat_dir, 'var', 'db', 'repos', 'gentoo') gentoo_git_repo_info = '''gentoo - location: /cvmfs/pilot.eessi-hpc.org/2021.06/compat/linux/x86_64/var/db/repos/gentoo + location: %s sync-type: git - sync-uri: https://github.com/gentoo/gentoo.git''' + sync-uri: https://github.com/gentoo/gentoo.git''' % gentoo_repo_dir self.sanity_patterns = sn.assert_found(gentoo_git_repo_info, self.stdout) From 0e64d56a24580233942c58fe55c6dccbe3cdc7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 20 Jun 2021 14:21:59 +0200 Subject: [PATCH 14/22] remove old versions of scripts in template dir --- .../templates/sync_gentoo_cache.j2 | 34 ------------------- .../templates/sync_gentoo_dtd.j2 | 17 ---------- .../templates/sync_gentoo_glsa.j2 | 17 ---------- .../templates/sync_gentoo_news.j2 | 17 ---------- .../templates/sync_gentoo_projects_xml.j2 | 12 ------- .../templates/sync_overlay_cache.j2 | 14 -------- 6 files changed, 111 deletions(-) delete mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 delete mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 delete mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 delete mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 delete mode 100644 ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 delete mode 100755 ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 deleted file mode 100755 index f1f572d2..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_cache.j2 +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -repository_name="${1}" -repository_path="${3}" - -[[ ${repository_name} == "gentoo" ]] || exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -# Number of jobs for egencache, default is number or processors. -parallel_jobs="$(nproc)" - -if [[ -f ${repository_path}/metadata/timestamp.x ]]; then - portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) -else - portage_current_timestamp=0 -fi - -ebegin "Fetching metadata timestamp for ${repository_name}" -rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x -eend $? -portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) - -if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then - ebegin "Fetching pre-generated metadata cache for ${repository_name}" - rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/ - eend $? -else - einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)" -fi - -ebegin "Updating metadata cache for ${repository_name}" -egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc -eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 deleted file mode 100755 index 81d8ff0f..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_dtd.j2 +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -repository_name="${1}" -repository_path="${3}" - -[[ ${repository_name} == "gentoo" ]] || exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -DTDDIR="${repository_path}"/metadata/dtd -ebegin "Updating DTDs" -if [[ -e ${DTDDIR} ]]; then - git -C "${DTDDIR}" pull -q --ff-only -else - git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}" -fi -eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 deleted file mode 100755 index 220e3aea..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_glsa.j2 +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -repository_name="${1}" -repository_path="${3}" - -[[ ${repository_name} == "gentoo" ]] || exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -GLSADIR="${repository_path}"/metadata/glsa -ebegin "Updating GLSAs" -if [[ -e ${GLSADIR} ]]; then - git -C "${GLSADIR}" pull -q --ff-only -else - git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}" -fi -eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 deleted file mode 100755 index dbab56ca..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_news.j2 +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -repository_name="${1}" -repository_path="${3}" - -[[ ${repository_name} == "gentoo" ]] || exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -NEWSDIR="${repository_path}"/metadata/news -ebegin "Updating news items" -if [[ -e ${NEWSDIR} ]]; then - git -C "${NEWSDIR}" pull -q --ff-only -else - git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}" -fi -eend $? "Try to remove ${NEWSDIR}" diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 deleted file mode 100644 index 07067e5b..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_gentoo_projects_xml.j2 +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -repository_name="${1}" -repository_path="${3}" - -[[ ${repository_name} == "gentoo" ]] || exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -ebegin "Updating projects.xml" -wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml -eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 b/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 deleted file mode 100755 index e373c810..00000000 --- a/ansible/playbooks/roles/compatibility_layer/templates/sync_overlay_cache.j2 +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -repository_name="${1}" - -[[ ${repository_name} == "gentoo" ]] && exit 0 - -source {{ gentoo_prefix_path }}/lib/gentoo/functions.sh - -# Number of jobs for egencache, default is number or processors. -parallel_jobs="$(nproc)" - -ebegin "Updating metadata cache for ${repository_name}" -egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc -eend $? From 2c5b58f14bc2cff6ecd2f62ee734aba6356e9e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 20 Jun 2021 14:23:07 +0200 Subject: [PATCH 15/22] remove commented sync_gentoo_project_xml --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index e3d05bc8..523782bd 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -92,7 +92,6 @@ - sync_gentoo_glsa - sync_gentoo_news - sync_overlay_cache - #- sync_gentoo_projects_xml - name: Remove the old Gentoo overlay directory file: From 2c1a333eb4f3780c0c7286ed5970559c3e5c2d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 13:12:33 +0200 Subject: [PATCH 16/22] Add a system update step for existing Prefix --- .../roles/compatibility_layer/tasks/add_overlay.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 523782bd..0d22db1c 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -13,12 +13,21 @@ path: /etc/group regexp: portage line: portage::250:portage - + - name: Install equery command (dependency for the portage module) command: cmd: emerge gentoolkit creates: "{{ gentoo_prefix_path }}/usr/bin/equery" +- name: Update system set if we are running in an existing Prefix installation + portage: + changed_use: yes + deep: yes + package: system + update: yes + + when: not startprefix.stat.exists + - name: Create repos directory file: path: "{{ gentoo_prefix_path }}/etc/portage/repos.conf" From 397ac8d8658058e36835b0056534e763b1e7478a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 13:16:17 +0200 Subject: [PATCH 17/22] remove trailing spaces --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 0d22db1c..4e58a827 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -13,7 +13,7 @@ path: /etc/group regexp: portage line: portage::250:portage - + - name: Install equery command (dependency for the portage module) command: cmd: emerge gentoolkit @@ -25,7 +25,6 @@ deep: yes package: system update: yes - when: not startprefix.stat.exists - name: Create repos directory From 9db7b4ba55e34955fe2883e40283465e1f05dd90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 14:10:00 +0200 Subject: [PATCH 18/22] remove wrong 'not' operator --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 4e58a827..3339d979 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -25,7 +25,7 @@ deep: yes package: system update: yes - when: not startprefix.stat.exists + when: startprefix.stat.exists - name: Create repos directory file: From 973081fb887c4e1ae0580aae49c8713188c00854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 14:36:01 +0200 Subject: [PATCH 19/22] update world instead of system --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 3339d979..239dc860 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -19,11 +19,11 @@ cmd: emerge gentoolkit creates: "{{ gentoo_prefix_path }}/usr/bin/equery" -- name: Update system set if we are running in an existing Prefix installation +- name: Update world set if we are running in an existing Prefix installation portage: changed_use: yes deep: yes - package: system + package: "@world" update: yes when: startprefix.stat.exists From 3044dff1676188b7c8212e6edf36bca26cbe6ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 15:00:23 +0200 Subject: [PATCH 20/22] verbose output for ansible-playbook --- .github/actions/install_compatibility_layer/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install_compatibility_layer/entrypoint.sh b/.github/actions/install_compatibility_layer/entrypoint.sh index aca33c96..b4904ddd 100755 --- a/.github/actions/install_compatibility_layer/entrypoint.sh +++ b/.github/actions/install_compatibility_layer/entrypoint.sh @@ -5,7 +5,7 @@ cat << EOF > hosts 127.0.0.1 eessi_host_arch=$(uname -m) eessi_host_os=linux EOF -ansible-playbook --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml +ansible-playbook -v --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml # A successful installation should at least have Lmod and archspec, # so let's check if we can use them. From cdeb9e72c1c9d8878d72e9cc521f46d33a6f2321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 15:18:21 +0200 Subject: [PATCH 21/22] Update of system set should go after the sync... --- .../compatibility_layer/tasks/add_overlay.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 239dc860..3733afae 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -19,14 +19,6 @@ cmd: emerge gentoolkit creates: "{{ gentoo_prefix_path }}/usr/bin/equery" -- name: Update world set if we are running in an existing Prefix installation - portage: - changed_use: yes - deep: yes - package: "@world" - update: yes - when: startprefix.stat.exists - - name: Create repos directory file: path: "{{ gentoo_prefix_path }}/etc/portage/repos.conf" @@ -125,3 +117,11 @@ force: yes with_items: "{{ find_configs.results | rejectattr('files', 'equalto', []) | map(attribute='files') | list }}" + +- name: Update system set if we are running in an existing Prefix installation + portage: + changed_use: yes + deep: yes + package: "@system" + update: yes + when: startprefix.stat.exists From 254a1b82232e573d58fa50ef4e116d0b4b3b1879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 21 Jun 2021 15:52:59 +0200 Subject: [PATCH 22/22] update world instead of system --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 3733afae..0454e1aa 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -122,6 +122,6 @@ portage: changed_use: yes deep: yes - package: "@system" + package: "@world" update: yes when: startprefix.stat.exists