Merge branch 'master' of github.com:eessi/compatibility-layer into ex…

…plain_hosts_file

.github/workflows/ansible-lint.yml

@@ -23,7 +23,7 @@ jobs:
         # targets: |
         #   playbook_1.yml
         #   playbook_2.yml
-        targets: "playbooks/*.yml"
+        targets: "ansible/playbooks/*.yml"
         # [optional]
         # Arguments to override a package and its version to be set explicitly.
         # Must follow the example syntax.

README.md

@@ -35,25 +35,44 @@ or simply:
 ./bootstrap-prefix.sif
 ```
 
+Our version of the script allows you to pick a custom snapshot for the Portage tree. This can be done by setting `SNAPSHOT_URL` to
+a URL that points to a directory, and setting `CUSTOM_SNAPSHOT` to the name of a snapshot file (must be a bzip2 archive). For instance:
+```
+env SNAPSHOT_URL="http://cvmfs-s0.eessi-hpc.org/snapshots" CUSTOM_SNAPSHOT="portage-20200909.tar.bz2" ./bootstrap-prefix.sif
+```
+
 If you want to run your own version of the bootstrap script, use:
 ```
 singularity exec bootstrap-prefix.sif ./bootstrap-prefix.sh
 ```
 
 After starting the bootstrap have a long coffee...
 
-### Adding EESSI overlay
+Once the bootstrap is completed, run the script to replace some paths with symlinks into the host OS:
+
+```
+scripts/prefix-symlink-host-paths.sh
+```
+
+### Adding the EESSI overlay and packages
 Additional packages are added in the EESSI overlay, which is based on ComputeCanada.
+You can add them manually or in an automated way by using Ansible, being Ansible the preferred way. Below you can find the two options explained.
+
+#### Ansible playbook (Option 1)
+The installation of the EESSI-specific parts can be automatically executed by running the Ansible playbook `install.yml` inside the folder `ansible/playbooks`. 
+This playbook will install the [EESSI Gentoo overlay](https://github.com/EESSI/gentoo-overlay) and a set of packages, including `Lmod` and `archspec`. See the `README` in the `ansible/playbooks` folder for more details.
+
+#### Manually (Option 2)
 To add the overlay: 
 
 Start the prefix
 ```
 startprefix
 ```
-Ensure the configuration directory exists, with $(EPREFIX) the path to your prefix installation.
+Ensure the configuration directory exists, with ${EPREFIX} the path to your prefix installation.
 ```
 export EPREFIX=path/to/your/prefix
-mkdir $(EPREFIX)/etc/portage/repos.conf
+mkdir ${EPREFIX}/etc/portage/repos.conf
 ```
 Configure the overlay. 
 ```
@@ -65,10 +84,16 @@ Sync the overlay
 emerge --sync
 ```
 
-#### Ansible playbook
+After synchronizing the overlay, add the EESSI package set(s) that you would like to install, e.g. for set `2020.08`:
+```
+mkdir ${EPREFIX}/etc/portage/sets/
+ln -s  ${EPREFIX}/var/db/repos/eessi/etc/portage/sets/2020.08 ${EPREFIX}/etc/portage/sets/
+```
 
-The steps described above can be automatically executed by running the Ansible playbook `install.yml` inside the folder `playbooks`. 
-See the `README` in that folder for more details.
+Finally, install the package set(s) defined at `${EPREFIX}/etc/portage/sets/`, e.g.:
+```
+emerge @2020.08
+```
 
 ### Updating the Prefix
 #### Packages
@@ -82,7 +107,7 @@ Try to mask latest versions:
 
 Create a mask file if not existing and mask newer versions from thin provisioning tools greater or equal to 0.7.6:
 ```
-echo ">=sys-block/thin-provisioning-tools-0.7.6" >> $(EPREFIX)/etc/portage/package.mask
+echo ">=sys-block/thin-provisioning-tools-0.7.6" >> ${EPREFIX}/etc/portage/package.mask
 ```
 
 #### Portage

bootstrap-prefix.sh

@@ -569,7 +569,9 @@ bootstrap_tree() {
 	# RAP uses the latest gentoo main repo snapshot to bootstrap.
 	is-rap && LATEST_TREE_YES=1
 	local PV="20200607"
-	if [[ -n ${LATEST_TREE_YES} ]]; then
+	if [[ -n ${CUSTOM_SNAPSHOT} ]]; then
+		do_tree "${SNAPSHOT_URL}" "${CUSTOM_SNAPSHOT}"
+	elif [[ -n ${LATEST_TREE_YES} ]]; then
 		do_tree "${SNAPSHOT_URL}" portage-latest.tar.bz2
 	else
 		do_tree http://dev.gentoo.org/~grobian/distfiles prefix-overlay-${PV}.tar.bz2

scripts/prefix-symlink-host-paths.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+function echo_green() {
+    echo -e "\e[32m$1\e[0m"
+}
+
+function echo_red() {
+    echo -e "\e[31m$1\e[0m"
+}
+
+function echo_yellow() {
+    echo -e "\e[33m$1\e[0m"
+}
+
+if [ -z $EPREFIX ]; then
+    echo_red "ERROR: \$EPREFIX not defined" >&2
+    exit 1
+fi
+
+EXPECTED_START='/cvmfs/pilot.eessi-hpc.org'
+if [[ $EPREFIX != $EXPECTED_START/* ]]; then
+    echo_red "ERROR: \$EPREFIX does not start with '$EXPECTED_START': EPREFIX=$EPREFIX" >&2
+    exit 1
+fi
+
+# /etc/passwd: required to ensure local users are known (see https://github.com/EESSI/compatibility-layer/issues/15)
+# /etc/group: required to ensure local user groups are known
+for path in /etc/passwd /etc/group; do
+    echo ">> checking $path ..."
+    ls -ld ${EPREFIX}$path | grep " -> $path" > /dev/null
+    ec=$?
+    if [ $ec -ne 0 ]; then
+        echo_yellow ">> [CHANGE] ${EPREFIX}$path is *not* a symlink to $path, fixing that..."
+        rm ${EPREFIX}$path
+        ln -s $path ${EPREFIX}$path
+    else
+        echo_green ">> [OK] ${EPREFIX}$path is already a symlink to $path"
+    fi
+done