Merge branch 'main' into playbook_test_step

.github/actions/install_compatibility_layer/entrypoint.sh

@@ -5,4 +5,4 @@ cat << EOF > hosts
 127.0.0.1 eessi_host_arch=$(uname -m) eessi_host_os=linux
 EOF
 
-ansible-playbook --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml
+ansible-playbook -v --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml

ansible/playbooks/roles/compatibility_layer/defaults/main.yml

@@ -1,6 +1,6 @@
 # Defaults file for the compatibility layer role.
 ---
-eessi_version: "2021.03"
+eessi_version: "2021.06"
 
 custom_overlays:
   - name: eessi
@@ -17,17 +17,15 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/{{ eessi_version }}/compat/{{
 
 # How to build the prefix
 prefix_required_space: 15 GB
-prefix_snapshot_url: http://cvmfs-s0.eessi-hpc.org/snapshots
-prefix_snapshot_version: 20210322
-prefix_python_targets: python3_8
+prefix_snapshot_url: https://eessi-gentoo-snapshot.s3-eu-west-1.amazonaws.com
+prefix_snapshot_version: 20210607
 prefix_user_defined_trusted_dirs:
   - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib"
 prefix_singularity_command: "singularity run -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}"
 prefix_source: "docker://eessi/bootstrap-prefix:centos8-{{ eessi_host_arch }}"
 prefix_source_options: "{{ gentoo_prefix_path }} noninteractive"
 prefix_install: >-
     SINGULARITYENV_USE_CPU_CORES={{ ansible_processor_vcpus }}
-    SINGULARITYENV_PYTHON_TARGETS="{{ prefix_python_targets }}"
     SINGULARITYENV_CUSTOM_SNAPSHOT_URL="{{ prefix_snapshot_url }}"
     SINGULARITYENV_CUSTOM_SNAPSHOT_VERSION="{{ prefix_snapshot_version }}"
     {{ prefix_singularity_command }} {{ prefix_source }} {{ prefix_source_options }}

ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+repository_name="${1}"
+repository_path="${3}"
+
+[[ ${repository_name} == "gentoo" ]] || exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+# Number of jobs for egencache, default is number or processors.
+parallel_jobs="$(nproc)"
+
+if [[ -f ${repository_path}/metadata/timestamp.x ]]; then
+	portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" )
+else
+	portage_current_timestamp=0
+fi
+
+ebegin "Fetching metadata timestamp for ${repository_name}"
+rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x
+eend $?
+portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" )
+
+if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then
+	ebegin "Fetching pre-generated metadata cache for ${repository_name}"
+	rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/
+	eend $?
+else
+	einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)"
+fi
+
+ebegin "Updating metadata cache for ${repository_name}"
+egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc
+eend $?

ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+repository_name="${1}"
+repository_path="${3}"
+
+[[ ${repository_name} == "gentoo" ]] || exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+DTDDIR="${repository_path}"/metadata/dtd
+ebegin "Updating DTDs"
+if [[ -e ${DTDDIR} ]]; then
+	git -C "${DTDDIR}" pull -q --ff-only
+else
+	git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}"
+fi
+eend "$?"

ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+repository_name="${1}"
+repository_path="${3}"
+
+[[ ${repository_name} == "gentoo" ]] || exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+GLSADIR="${repository_path}"/metadata/glsa
+ebegin "Updating GLSAs"
+if [[ -e ${GLSADIR} ]]; then
+	git -C "${GLSADIR}" pull -q --ff-only
+else
+	git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}"
+fi
+eend "$?"

ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+repository_name="${1}"
+repository_path="${3}"
+
+[[ ${repository_name} == "gentoo" ]] || exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+NEWSDIR="${repository_path}"/metadata/news
+ebegin "Updating news items"
+if [[ -e ${NEWSDIR} ]]; then
+	git -C "${NEWSDIR}" pull -q --ff-only
+else
+	git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}"
+fi
+eend $? "Try to remove ${NEWSDIR}"

ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+repository_name="${1}"
+repository_path="${3}"
+
+[[ ${repository_name} == "gentoo" ]] || exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+ebegin "Updating projects.xml"
+wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml
+eend $?

ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+repository_name="${1}"
+
+[[ ${repository_name} == "gentoo" ]] && exit 0
+
+source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh
+
+# Number of jobs for egencache, default is number or processors.
+parallel_jobs="$(nproc)"
+
+ebegin "Updating metadata cache for ${repository_name}"
+egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc
+eend $?

ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml

@@ -58,6 +58,46 @@
         selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ')
       }}
 
+- name: Use git instead of rsync for the Gentoo overlay
+  copy:
+    dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/gentoo.conf"
+    mode: "0644"
+    content: |
+      [DEFAULT]
+      main-repo = gentoo
+      sync-git-pull-extra-opts = --quiet
+
+      [gentoo]
+      priority  = 1
+      location  = {{ gentoo_prefix_path }}/var/db/repos/gentoo
+      sync-uri  = https://github.com/gentoo/gentoo.git
+      sync-type = git
+      auto-sync = Yes
+      clone-depth = 1
+
+- name: Create repo.postsync.d directory
+  file:
+    path: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d"
+    state: directory
+    mode: '0755'
+
+- name: Add post-sync hooks for Gentoo overlay
+  copy:
+    src: "{{ item }}"
+    dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}"
+    mode: 0755
+  with_items:
+    - sync_gentoo_cache
+    - sync_gentoo_dtd
+    - sync_gentoo_glsa
+    - sync_gentoo_news
+    - sync_overlay_cache
+
+- name: Remove the old Gentoo overlay directory
+  file:
+    state: absent
+    path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo"
+
 - name: Sync the repositories
   portage:
     sync: 'yes'
@@ -77,3 +117,11 @@
     force: yes
   with_items:
     "{{ find_configs.results | rejectattr('files', 'equalto', []) | map(attribute='files') | list }}"
+
+- name: Update system set if we are running in an existing Prefix installation
+  portage:
+    changed_use: yes
+    deep: yes
+    package: "@world"
+    update: yes
+  when: startprefix.stat.exists

ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml

@@ -1,27 +1,5 @@
 # Install a specified list of sets and packages.
 ---
-- name: Find all strings in libc library
-  command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a"
-  register: libc_strings
-  when: eessi_host_os == "linux"
-
-- name: Find user defined trusted dirs in libc strings output
-  set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}'
-  with_items: "{{ prefix_user_defined_trusted_dirs }}"
-  register: trusted_dirs_in_libc
-
-- name: (Re)install glibc with the user-defined-trusted-dirs option
-  portage:
-    package: sys-libs/glibc
-    noreplace: no
-    oneshot: yes
-  become: no
-  environment:
-    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
-  when:
-    - eessi_host_os == "linux"
-    - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0
-
 - name: Install package set {{ package_sets }}
   portage:
     package: "@{{ item }}"

ansible/playbooks/roles/compatibility_layer/tasks/main.yml

@@ -24,8 +24,8 @@
     args:
       apply:
         become: False
-        environment:
-          PYTHON_TARGETS: "{{ prefix_python_targets }}"
+
+  - include_tasks: set_glibc_trusted_dirs.yml
 
   - include_tasks: install_packages.yml
 

ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml

@@ -0,0 +1,45 @@
+# Make sure that glibc is always compiled with a user-defined-trusted-dirs option
+---
+- name: Find all strings in libc library
+  command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a"
+  register: libc_strings
+  when: eessi_host_os == "linux"
+
+- name: Find user defined trusted dirs in libc strings output
+  set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}'
+  with_items: "{{ prefix_user_defined_trusted_dirs }}"
+  register: trusted_dirs_in_libc
+
+- name: (Re)install glibc with the user-defined-trusted-dirs option
+  portage:
+    package: sys-libs/glibc
+    noreplace: no
+    oneshot: yes
+  become: no
+  environment:
+    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+  when:
+    - eessi_host_os == "linux"
+    - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0
+
+- name: Create portage env directory
+  file:
+    path: "{{ gentoo_prefix_path }}/etc/portage/env"
+    state: directory
+    mode: 0755
+
+- name: Add env file for glibc to make sure the user-defined-trusted-dirs is always used
+  copy:
+    dest: "{{ gentoo_prefix_path }}/etc/portage/env/glibc-user-defined-trusted-dirs.conf"
+    mode: 0644
+    content: |
+      EXTRA_EMAKE="user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+
+- name: Add glibc env file to package.env
+  lineinfile:
+    path: "{{ gentoo_prefix_path }}/etc/portage/package.env"
+    create: yes
+    mode: 0644
+    line: sys-libs/glibc glibc-user-defined-trusted-dirs.conf
+    state: present
+

bootstrap-prefix.sh

@@ -282,7 +282,7 @@ configure_toolchain() {
 			;;
 	esac
 
-	return
+	return 0
 }
 
 bootstrap_setup() {
@@ -450,6 +450,12 @@ bootstrap_setup() {
 	ACCEPT_KEYWORDS="~${ARCH}-linux"
 	EOF
 
+	# bug #788613 avoid gcc-11 during stage 2/3 prior sync/emerge -e
+	is-rap && cat >> "${ROOT}"/etc/portage/make.profile/package.mask <<-EOF
+	# during bootstrap mask, bug #788613
+	>=sys-devel/gcc-11
+	EOF
+
 	# Use package.use to disable in the portage tree to be shared between
 	# stage2 and stage3. The hack will be undone during tree sync in stage3.
 	cat >> "${ROOT}"/etc/portage/make.profile/package.use <<-EOF
@@ -1355,8 +1361,8 @@ bootstrap_libressl() {
 
 bootstrap_stage_host_gentoo() {
 	if ! is-rap ; then
-		einfo "Shortcut only supports prefix-standalone, but we are bootstrapping"
-		einfo "prefix-rpath. Do nothing."
+		einfo "Shortcut only supports prefix-standalone, but we "
+		einfo "are bootstrapping prefix-rpath.  Do nothing."
 		return 0
 	fi
 
@@ -1447,6 +1453,7 @@ bootstrap_stage1() {
 	# we're working with now, bug #650284
 	[[ -x ${ROOT}/tmp/usr/bin/bash ]] \
 		|| (bootstrap_bash) || return 1
+
 	# Some host tools need to be wrapped to be useful for us.
 	# We put them in tmp/usr/local/bin, to not accidentally
 	# be identified as stage1-installed like in bug #615410.
@@ -1475,10 +1482,14 @@ bootstrap_stage1() {
 				# We need to direct the system gcc to find the system binutils.
 				cat >> "${ROOT}"/tmp/usr/local/bin/gcc <<-EOF
 					#! /bin/sh
-					PATH="${STAGE1_PATH}" export PATH
-					exec "\${0##*/}" "\$@"
+					PATH="${ORIGINAL_PATH}" export PATH
+					exec "$(type -P gcc)" "\$@"
+				EOF
+				cat >> "${ROOT}"/tmp/usr/local/bin/g++ <<-EOF
+					#! /bin/sh
+					PATH="${ORIGINAL_PATH}" export PATH
+					exec "$(type -P g++)" "\$@"
 				EOF
-				cp "${ROOT}"/tmp/usr/local/bin/g{cc,++}
 				chmod 755 "${ROOT}"/tmp/usr/local/bin/g{cc,++}
 			fi
 			;;
@@ -1905,7 +1916,11 @@ bootstrap_stage3() {
 	}
 
 	# pre_emerge_pkgs relies on stage 2 portage.
-	pre_emerge_pkgs() { is-rap && without_stack_emerge_pkgs "$@" || with_stack_emerge_pkgs "$@"; }
+	pre_emerge_pkgs() {
+		is-rap \
+			&& without_stack_emerge_pkgs "$@" \
+			|| with_stack_emerge_pkgs "$@"
+	}
 
 	# Some packages fail to properly depend on sys-apps/texinfo.
 	# We don't really need that package, so we fake it instead,
@@ -2768,7 +2783,6 @@ EOF
 		# location seems ok
 		break
 	done
-	export STAGE1_PATH=${PATH}
 	export PATH="$EPREFIX/usr/bin:$EPREFIX/bin:$EPREFIX/tmp/usr/bin:$EPREFIX/tmp/bin:$EPREFIX/tmp/usr/local/bin:${PATH}"
 
 	cat << EOF
@@ -3078,9 +3092,6 @@ case ${CHOST} in
 	powerpc-*darwin*)
 		DARWIN_USE_GCC=1  # must use GCC, Clang is impossible
 		;;
-#	arm64-*darwin*)
-#		DARWIN_USE_GCC=0  # cannot use GCC yet (needs silicon support)
-#		;;
 	*-darwin*)
 		# normalise value of DARWIN_USE_GCC
 		case ${DARWIN_USE_GCC} in
@@ -3116,6 +3127,9 @@ case ${CHOST}:${LC_ALL}:${LANG} in
 	;;
 esac
 
+# save original path, need this before interactive, #788334
+ORIGINAL_PATH="${PATH}"
+
 # Just guessing a prefix is kind of scary.  Hence, to make it a bit less
 # scary, we force the user to give the prefix location here.  This also
 # makes the script a bit less dangerous as it will die when just run to