diff --git a/.github/workflows/Dockerfile-ubuntu-20.04 b/.github/workflows/Dockerfile-ubuntu-20.04 index b5d59a41..1f77e4d3 100644 --- a/.github/workflows/Dockerfile-ubuntu-20.04 +++ b/.github/workflows/Dockerfile-ubuntu-20.04 @@ -6,7 +6,7 @@ RUN apt-get update RUN apt-get install -y cron gpg python3-pip git RUN env DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get install -y systemd -RUN pip3 install ansible +RUN pip3 install "ansible<9.0.0" COPY ./.github/workflows/test-playbook.sh /test-playbook.sh diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 4ee4fc1d..c28ff134 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -18,7 +18,7 @@ on: # this often means that we have a new CVMFS configuration package release: types: [published] - + # Declare default permissions as read only. permissions: read-all @@ -32,7 +32,7 @@ jobs: contents: read strategy: matrix: - tag: ['client-pilot:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11'] platform: [amd64, arm64, ppc64le] steps: @@ -91,7 +91,7 @@ jobs: contents: read strategy: matrix: - tag: ['client-pilot:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' diff --git a/.github/workflows/test-containers.yml b/.github/workflows/test-containers.yml index 1a8aea15..5262b5ca 100644 --- a/.github/workflows/test-containers.yml +++ b/.github/workflows/test-containers.yml @@ -11,4 +11,4 @@ jobs: steps: - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Build the Docker image - run: docker build . --file containers/Dockerfile.EESSI-client-pilot-centos7 + run: docker build . --file containers/Dockerfile.EESSI-client-centos7 diff --git a/README.md b/README.md index 10ec3f80..1f366f4c 100644 --- a/README.md +++ b/README.md @@ -197,15 +197,16 @@ Finally, run `cvmfs_config setup` to set up CVMFS. ### Client -Once the client has been installed, you should be able to access all repositories under /cvmfs. They might not immediately show up in that directory before you have actually used them, so you might first have to run ls, e.g.: +Once the client has been installed, you should be able to access all repositories under `/cvmfs`. +They might not immediately show up in that directory before you have actually used them, so you might first have to run ls, e.g.: ``` -ls /cvmfs/pilot.eessi-hpc.org +ls /cvmfs/software.eessi.io ``` On the client machines you can use the `cvmfs_config` tool for different operations. For instance, you can verify the file system by running: ``` -$ sudo cvmfs_config probe pilot.eessi-hpc.org -Probing /cvmfs/pilot.eessi-hpc.org... OK +$ sudo cvmfs_config probe software.eessi.io +Probing /cvmfs/software.eessi.io... OK ``` Checking for misconfigurations can be done with: @@ -224,7 +225,7 @@ In order to test your local proxy and/or Stratum 1, even without a client instal curl. Leave out the `--proxy http://url-to-your-proxy:3128` if you do not use a proxy. ``` -curl --proxy http://url-to-your-proxy:3128 --head http://url-to-your-stratum1/cvmfs/pilot.eessi-hpc.org/.cvmfspublished +curl --proxy http://url-to-your-proxy:3128 --head http://url-to-your-stratum1/cvmfs/software.eessi.io/.cvmfspublished ``` This should return: ``` @@ -237,19 +238,19 @@ The second time you run it, you should get a cache hit: X-Cache: HIT from url-to-your-proxy ``` -Example with the Norwegian Stratum 1: +Example with an EESSI Stratum 1 server: ``` -curl --head http://bgo-no.stratum1.cvmfs.eessi-infra.org/cvmfs/pilot.eessi-hpc.org/.cvmfspublished +curl --head http://aws-eu-central-s1.eessi.science/cvmfs/software.eessi.io/.cvmfspublished ``` ### Using the CVMFS infrastructure When the infrastructure seems to work, you can try publishing some new files. This can be done by starting a transaction on the Stratum 0, adding some files, and publishing the transaction: ``` -sudo cvmfs_server transaction pilot.eessi-hpc.org -mkdir /cvmfs/pilot.eessi-hpc.org/testdir -touch /cvmfs/pilot.eessi-hpc.org/testdir/testfile -sudo cvmfs_server publish pilot.eessi-hpc.org +sudo cvmfs_server transaction software.eessi.io +mkdir /cvmfs/software.eessi.io/testdir +touch /cvmfs/software.eessi.io/testdir/testfile +sudo cvmfs_server publish software.eessi.io ``` It might take a few minutes, but then the new file should show up at the clients. diff --git a/containers/Dockerfile.EESSI-build-node-debian11 b/containers/Dockerfile.EESSI-build-node-debian11 index 5fb1d9b7..a0c85abe 100644 --- a/containers/Dockerfile.EESSI-build-node-debian11 +++ b/containers/Dockerfile.EESSI-build-node-debian11 @@ -1,5 +1,5 @@ -ARG cvmfsversion=2.10.1 -ARG awscliversion=1.27.146 +ARG cvmfsversion=2.11.2 +ARG awscliversion=1.30.2 ARG fuseoverlayfsversion=1.10 FROM debian:11.7 AS prepare-deb @@ -21,6 +21,7 @@ RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 RUN apt-get install -y python3-pip jq RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-config-default_latest_all.deb \ /root/deb/cvmfs-config-eessi_latest_all.deb @@ -32,13 +33,13 @@ RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \ && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \ && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local -RUN mkdir -p /cvmfs/pilot.eessi-hpc.org +RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) RUN pip3 install archspec awscli==${awscliversion} -RUN curl -OL https://raw.githubusercontent.com/EESSI/infrastructure/main/eessi-upload-to-staging \ +RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ && mv eessi-upload-to-staging /usr/bin \ && chmod a+x /usr/bin/eessi-upload-to-staging diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-centos7 similarity index 90% rename from containers/Dockerfile.EESSI-client-pilot-centos7 rename to containers/Dockerfile.EESSI-client-centos7 index f4904819..52229109 100644 --- a/containers/Dockerfile.EESSI-client-pilot-centos7 +++ b/containers/Dockerfile.EESSI-client-centos7 @@ -1,7 +1,7 @@ -ARG cvmfsversion=2.9.4 +ARG cvmfsversion=2.11.2 # Stick to old version of fuse-overlayfs due to issues with newer versions # (cfr. https://github.com/containers/fuse-overlayfs/issues/232) -ARG fuseoverlayfsversion=0.3 +ARG fuseoverlayfsversion=1.10 FROM centos:7 AS prepare-rpm ARG cvmfsversion @@ -27,6 +27,7 @@ COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fu RUN yum install -y sudo vim openssh-clients lsof RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \ /root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \ + /root/rpmbuild/RPMS/$(uname -m)/cvmfs-libs-${cvmfsversion}-1.el7.$(uname -m).rpm \ http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm @@ -38,6 +39,6 @@ RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \ && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \ && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local -RUN mkdir -p /cvmfs/pilot.eessi-hpc.org +RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi diff --git a/containers/README.md b/containers/README.md index e113828c..654faa1c 100644 --- a/containers/README.md +++ b/containers/README.md @@ -4,8 +4,8 @@ This directory contains recipes for containers that are useful in the scope of t ## Client container -Container to provide easy access to EESSI pilot repository, -see https://github.com/users/EESSI/packages/container/package/client-pilot and https://eessi.github.io/docs/pilot. +Container to provide easy access to EESSI software stack, +see https://github.com/users/EESSI/packages/container/package/client and https://www.eessi.io/docs/getting_access/eessi_container/. This container image is based on CentOS 7, and gets automatically built and pushed to the GitHub Container Registry when one of its source files (the Dockerfile or the script that generates the CernVM-FS RPMs) gets changed, or when a new version of the filesystem-layer repository is released. @@ -16,13 +16,13 @@ or when a new version of the filesystem-layer repository is released. mkdir -p /tmp/$USER/{var-lib-cvmfs,var-run-cvmfs,home} export SINGULARITY_BIND="/tmp/$USER/var-run-cvmfs:/var/run/cvmfs,/tmp/$USER/var-lib-cvmfs:/var/lib/cvmfs" export SINGULARITY_HOME="/tmp/$USER/home:/home/$USER" -export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org" -singularity shell --fusemount "$EESSI_PILOT" docker://ghcr.io/eessi/client-pilot:centos7 +export EESSI_STACK="container:cvmfs2 software.eessi.io /cvmfs/software.eessi.io" +singularity shell --fusemount "$EESSI_STACK" docker://ghcr.io/eessi/client:centos7 ``` ## Build node container -Container that can be used to build and install software to /cvmfs by leveraging `fuse-overlayfs` for +Container that can be used to build and install software to `/cvmfs` by leveraging `fuse-overlayfs` for providing a writable overlay. The container image is based on Debian 10.6, and gets automatically built and pushed to the GitHub Container Registry when one of its source files (the Dockerfile or the script that generates the CernVM-FS deb packages) gets changed, @@ -37,9 +37,9 @@ mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs} export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache export SINGULARITY_BIND="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs" export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER" -export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org" -export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org" -singularity shell --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" docker://ghcr.io/eessi/build-node:debian11 +export EESSI_STACK_READONLY="container:cvmfs2 software.eessi.io /cvmfs_ro/software.eessi.io" +export EESSI_STACK_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/software.eessi.io -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/software.eessi.io" +singularity shell --fusemount "$EESSI_STACK_READONLY" --fusemount "$EESSI_STACK_WRITABLE_OVERLAY" docker://ghcr.io/eessi/build-node:debian11 ``` For more details about building software, see: https://eessi.github.io/docs/software_layer/build_nodes/ diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index bf76f1ce..071fa867 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -23,6 +23,7 @@ else cd /root/deb wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs_${cvmfsversion}~1+${os}_${arch}.deb wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3_${cvmfsversion}~1+${os}_${arch}.deb + wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-libs_${cvmfsversion}~1+${os}_${arch}.deb fi cd /root/deb diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh index 078c4f71..2e955768 100644 --- a/containers/build-or-download-cvmfs-rpms.sh +++ b/containers/build-or-download-cvmfs-rpms.sh @@ -16,4 +16,5 @@ else cd /root/rpmbuild/RPMS/${arch} wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.${arch}.rpm wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3-${cvmfsversion}-1.el7.${arch}.rpm + wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-libs-${cvmfsversion}-1.el7.${arch}.rpm fi diff --git a/scripts/check-stratum-servers.py b/scripts/check-stratum-servers.py index 2e2a4bd6..de4270d9 100755 --- a/scripts/check-stratum-servers.py +++ b/scripts/check-stratum-servers.py @@ -11,7 +11,7 @@ # Default location for EESSI's Ansible group vars file containing the CVMFS settings. DEFAULT_ANSIBLE_GROUP_VARS_LOCATION = 'https://raw.githubusercontent.com/EESSI/filesystem-layer/main/inventory/group_vars/all.yml' # Default fully qualified CVMFS repository name -DEFAULT_CVMFS_FQRN = 'pilot.eessi-hpc.org' +DEFAULT_CVMFS_FQRN = 'software.eessi.io' # Maximum amount of time (in minutes) that a Stratum 1 is allowed to not having performed a snapshot. DEFAULT_MAX_SNAPSHOT_DELAY = 30 # Maximum amount of time (in minutes) allowed between the snapshots of any two Stratum 1 servers.