From 6c47a073fd7c53bcbfb478d556c9ba9414d057ad Mon Sep 17 00:00:00 2001 From: Bob Date: Fri, 26 Jun 2020 17:07:05 +0200 Subject: [PATCH 01/35] Initial version, only deb for now --- build-packages.yml | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 build-packages.yml diff --git a/build-packages.yml b/build-packages.yml new file mode 100644 index 00000000..69f423e5 --- /dev/null +++ b/build-packages.yml @@ -0,0 +1,65 @@ +# Build Debian (deb) and Redhat (rpm) packages for CVMFS clients. +--- +- name: Build deb and yum packages. + hosts: localhost + vars: + deb_package_dependencies: [] +# - cvmfs +# - cvmfs-config-none + build_debian_package_files_directories: + - src: "{{ temp_buildinput.path }}/etc/" + dest: "etc" + build_debian_package_control_fields: + Package: cvmfs-config-eessi + Version: 0.1 + Priority: optional + Architecture: all + Depends: "{{ deb_package_dependencies|join(',') }}" + Maintainer: Bob Dröge + Description: CVMFS config repository package for EESSI. + pre_tasks: + # Create a temporary directory where we generate the contents of the packages. + - name: Create temporary directory + tempfile: + state: directory + suffix: buildinput + register: temp_buildinput + + # + - name: Create directory structure for the CVMFS config + file: + path: "{{ item }}" + state: directory + mode: 0775 + with_items: + - "{{ temp_buildinput.path }}" + - "{{ temp_buildinput.path }}/etc" + - "{{ temp_buildinput.path }}/etc/cvmfs" + - "{{ temp_buildinput.path }}/etc/cvmfs/keys" + - "{{ temp_buildinput.path }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" + - "{{ temp_buildinput.path }}/etc/cvmfs/config.d" + - "{{ temp_buildinput.path }}/etc/cvmfs/default.d" + + - name: Make EESSI CVMFS public key file + copy: + content: "{{ eessi_cvmfs_config_repo.key.key }}" + dest: "{{ temp_buildinput.path }}{{ eessi_cvmfs_config_repo.key.path }}" + + - name: Make EESSI CVMFS configuration file + copy: + content: | + ## This file is maintained by Ansible - CHANGES WILL BE OVERWRITTEN + CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" + CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" + dest: "{{ temp_buildinput.path }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" + + - name: Make EESSI CVMFS default configuration file + copy: + content: | + CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" + CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" + dest: "{{ temp_buildinput.path }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" + + roles: + - role: freedomofpress.build-debian-package + tags: build From 0bf7c69337ca94db7a7c9cc07bb0316097e1ad2e Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 14:57:14 +0200 Subject: [PATCH 02/35] Add deb packaging workflow --- .github/workflows/packages.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/packages.yml diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 00000000..1435dcb7 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,18 @@ +name: Build packages + +on: [push, pull_request] + +jobs: + debian: + + runs-on: ubuntu-latest + + steps: + # Important: This sets up your GITHUB_WORKSPACE environment variable + - uses: actions/checkout@v2 + + - name: Build Debian package + uses: dawidd6/action-debian-package@v1 + with: + source_directory: templates # lolcat # optional, relative to workspace directory + artifacts_directory: output # optional, relative to workspace directory From 5ae0a48f61f75bbb98c136f02a57741d331df7d0 Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 16:22:39 +0200 Subject: [PATCH 03/35] Change CI for building packages --- .github/workflows/packages.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 1435dcb7..2cec1d6f 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -3,7 +3,7 @@ name: Build packages on: [push, pull_request] jobs: - debian: + build: runs-on: ubuntu-latest @@ -11,8 +11,9 @@ jobs: # Important: This sets up your GITHUB_WORKSPACE environment variable - uses: actions/checkout@v2 - - name: Build Debian package - uses: dawidd6/action-debian-package@v1 + - name: ansible check with debian:stable + uses: roles-ansible/check-ansible-debian-stable-action@master with: - source_directory: templates # lolcat # optional, relative to workspace directory - artifacts_directory: output # optional, relative to workspace directory + targets: "./build-packages.yml" + hosts: "localhost" + requirements: "freedomofpress.build-debian-package" From aad4c70ef378cd2b4770d47462cacc25238ab4c4 Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 16:35:57 +0200 Subject: [PATCH 04/35] Add freedomofpress.build-debian-package role --- .gitmodules | 3 +++ roles/freedomofpress.build-debian-package | 1 + 2 files changed, 4 insertions(+) create mode 160000 roles/freedomofpress.build-debian-package diff --git a/.gitmodules b/.gitmodules index 54e03522..7dbc3813 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "roles/geerlingguy.repo-epel"] path = roles/geerlingguy.repo-epel url = https://github.com/geerlingguy/ansible-role-repo-epel +[submodule "roles/freedomofpress.build-debian-package"] + path = roles/freedomofpress.build-debian-package + url = https://github.com/EESSI/ansible-role-build-debian-package.git diff --git a/roles/freedomofpress.build-debian-package b/roles/freedomofpress.build-debian-package new file mode 160000 index 00000000..880f6147 --- /dev/null +++ b/roles/freedomofpress.build-debian-package @@ -0,0 +1 @@ +Subproject commit 880f6147b5abd1fdb0242ff6d3da7f737f848c4e From 9133e0656d855723573845266251632ca827302c Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 16:48:02 +0200 Subject: [PATCH 05/35] Use Debian stretch for CI --- .github/workflows/packages.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 2cec1d6f..8f46e2a4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -11,9 +11,8 @@ jobs: # Important: This sets up your GITHUB_WORKSPACE environment variable - uses: actions/checkout@v2 - - name: ansible check with debian:stable - uses: roles-ansible/check-ansible-debian-stable-action@master + - name: ansible check with debian:stretch + uses: roles-ansible/check-ansible-debian-stretch-action@master with: targets: "./build-packages.yml" hosts: "localhost" - requirements: "freedomofpress.build-debian-package" From d60f96219d1b20255be3a1d693b8a08aaf4d786c Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 16:58:21 +0200 Subject: [PATCH 06/35] Upload packages --- .github/workflows/packages.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 8f46e2a4..dacd3edb 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -1,9 +1,17 @@ name: Build packages -on: [push, pull_request] +on: + # Trigger the workflow on push or pull request, + # but only for the master branch + push: + branches: + - master + pull_request: + branches: + - master jobs: - build: + build_deb: runs-on: ubuntu-latest @@ -16,3 +24,9 @@ jobs: with: targets: "./build-packages.yml" hosts: "localhost" + + - name: upload package as artifact + uses: actions/upload-artifact@v2 + with: + name: deb_package + path: ${{ github.workspace }} From 3204d8f3381befb878389727a51e3ac1251e5be7 Mon Sep 17 00:00:00 2001 From: Bob Date: Tue, 30 Jun 2020 17:04:25 +0200 Subject: [PATCH 07/35] Upload only the package --- .github/workflows/packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index dacd3edb..9ccf3a7f 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -28,5 +28,5 @@ jobs: - name: upload package as artifact uses: actions/upload-artifact@v2 with: - name: deb_package - path: ${{ github.workspace }} + #name: deb_package + path: ${{ github.workspace }}/build/*.deb From d29a7fe797c2fef93d64aa0506e556cdc3ce0bc1 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 12:39:57 +0200 Subject: [PATCH 08/35] test rpmbuild command --- build-packages.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-packages.yml b/build-packages.yml index 69f423e5..93b45185 100644 --- a/build-packages.yml +++ b/build-packages.yml @@ -60,6 +60,9 @@ CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ temp_buildinput.path }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" + - name: rpmbuild + command: rpmbuild + roles: - role: freedomofpress.build-debian-package tags: build From 64d3561f1ad0d4309df05e7a5c45aeec6f23dfed Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 12:45:10 +0200 Subject: [PATCH 09/35] install rpmbuild --- build-packages.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build-packages.yml b/build-packages.yml index 93b45185..a05b79d4 100644 --- a/build-packages.yml +++ b/build-packages.yml @@ -60,7 +60,12 @@ CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ temp_buildinput.path }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" - - name: rpmbuild + - name: install rpmbuild + apt: + name: rpm + state: present + + - name: run rpmbuild tool command: rpmbuild roles: From 80c1e1e5825737349243ccfb15e9d3c8cc804a73 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:18:58 +0200 Subject: [PATCH 10/35] Added role for making rpms --- build-packages.yml | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/build-packages.yml b/build-packages.yml index a05b79d4..2696e497 100644 --- a/build-packages.yml +++ b/build-packages.yml @@ -3,20 +3,9 @@ - name: Build deb and yum packages. hosts: localhost vars: - deb_package_dependencies: [] -# - cvmfs -# - cvmfs-config-none - build_debian_package_files_directories: - - src: "{{ temp_buildinput.path }}/etc/" - dest: "etc" - build_debian_package_control_fields: - Package: cvmfs-config-eessi - Version: 0.1 - Priority: optional - Architecture: all - Depends: "{{ deb_package_dependencies|join(',') }}" - Maintainer: Bob Dröge - Description: CVMFS config repository package for EESSI. + package_name: cvmfs-config-eessi + package_version: 0.1 + package_description: CVMFS config repository package for EESSI. pre_tasks: # Create a temporary directory where we generate the contents of the packages. - name: Create temporary directory @@ -60,14 +49,26 @@ CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ temp_buildinput.path }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" - - name: install rpmbuild - apt: - name: rpm - state: present - - - name: run rpmbuild tool - command: rpmbuild - roles: - role: freedomofpress.build-debian-package tags: build + vars: + deb_package_dependencies: [] +# - cvmfs +# - cvmfs-config-none + build_debian_package_files_directories: + - src: "{{ temp_buildinput.path }}/etc/" + dest: "etc" + build_debian_package_control_fields: + Package: "{{ package_name }}" + Version: "{{ package_version }}" + Priority: optional + Architecture: all + Depends: "{{ deb_package_dependencies|join(',') }}" + Maintainer: Bob Dröge + Description: "{{ package_description }}" + - role: build-rpm-package + tags: build + vars: + build_rpm_package_directory: /tmp/rpmbuild + build_rpm_package_files_directories: "{{ temp_buildinput.path }}" From be58e41fb11ca480d002e0f936218c9c2aec3600 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:19:31 +0200 Subject: [PATCH 11/35] Add build dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6ff331c7..39af2bac 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +build hosts From 9add7773fc82c857a10059645e99fb86297009fc Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:20:35 +0200 Subject: [PATCH 12/35] Role for making RPMs --- roles/build-rpm-package/defaults/main.yml | 6 ++++ .../tasks/build_rpm_package.yml | 8 +++++ .../tasks/create_directories.yml | 32 +++++++++++++++++++ roles/build-rpm-package/tasks/main.yml | 10 ++++++ roles/build-rpm-package/tasks/packages.yml | 9 ++++++ roles/build-rpm-package/templates/specfile.j2 | 19 +++++++++++ 6 files changed, 84 insertions(+) create mode 100644 roles/build-rpm-package/defaults/main.yml create mode 100644 roles/build-rpm-package/tasks/build_rpm_package.yml create mode 100644 roles/build-rpm-package/tasks/create_directories.yml create mode 100644 roles/build-rpm-package/tasks/main.yml create mode 100644 roles/build-rpm-package/tasks/packages.yml create mode 100644 roles/build-rpm-package/templates/specfile.j2 diff --git a/roles/build-rpm-package/defaults/main.yml b/roles/build-rpm-package/defaults/main.yml new file mode 100644 index 00000000..105be7db --- /dev/null +++ b/roles/build-rpm-package/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# defaults file for build-rpm-package +build_rpm_package_local_directory: build + +# Build directory. +build_rpm_package_directory: /tmp/rpmbuild diff --git a/roles/build-rpm-package/tasks/build_rpm_package.yml b/roles/build-rpm-package/tasks/build_rpm_package.yml new file mode 100644 index 00000000..b544fc92 --- /dev/null +++ b/roles/build-rpm-package/tasks/build_rpm_package.yml @@ -0,0 +1,8 @@ +--- +- name: Build RPM package. + command: "rpmbuild --define \"_topdir {{ build_rpm_package_directory }}\" --define \"_rpmdir {{ build_rpm_package_local_directory }}\" -bb {{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec" + +#- name: Copy package to local directory +# copy: +# src: "{{ build_rpm_package_directory }}/RPMS/*/*.rpm" +# dest: "{{ build_rpm_package_local_directory }}" diff --git a/roles/build-rpm-package/tasks/create_directories.yml b/roles/build-rpm-package/tasks/create_directories.yml new file mode 100644 index 00000000..301010aa --- /dev/null +++ b/roles/build-rpm-package/tasks/create_directories.yml @@ -0,0 +1,32 @@ +--- +- name: Clean up build files. + file: + state: absent + path: "{{ build_rpm_package_directory }}" + +- name: Create base build directory. + file: + state: directory + path: "{{ item }}" + with_items: + - "{{ build_rpm_package_directory }}" + - "{{ build_rpm_package_directory }}/SOURCES" + - "{{ build_rpm_package_directory }}/{{ package_name }}-{{ package_version }}" + - "{{ build_rpm_package_directory }}/SPECS" + +#- name: Copy source files +# copy: +# src: "{{ build_rpm_package_files_directories }}/" +# dest: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}" + +- name: Make tarball of source files + archive: + #path: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}" + path: "{{ build_rpm_package_files_directories }}/" + dest: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz" + +- name: Make Spec file + template: + src: specfile.j2 + dest: "{{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec" + diff --git a/roles/build-rpm-package/tasks/main.yml b/roles/build-rpm-package/tasks/main.yml new file mode 100644 index 00000000..b992d124 --- /dev/null +++ b/roles/build-rpm-package/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# tasks file for build-rpm-package +- include: packages.yml + tags: packages + +- include: create_directories.yml + tags: directories + +- include: build_rpm_package.yml + tags: build diff --git a/roles/build-rpm-package/tasks/packages.yml b/roles/build-rpm-package/tasks/packages.yml new file mode 100644 index 00000000..190c6e31 --- /dev/null +++ b/roles/build-rpm-package/tasks/packages.yml @@ -0,0 +1,9 @@ +--- +- name: Install build dependencies. + apt: + name: "{{ item }}" + state: present + update_cache: yes + cache_valid_time: 3600 + with_items: + - rpm diff --git a/roles/build-rpm-package/templates/specfile.j2 b/roles/build-rpm-package/templates/specfile.j2 new file mode 100644 index 00000000..c8976c68 --- /dev/null +++ b/roles/build-rpm-package/templates/specfile.j2 @@ -0,0 +1,19 @@ +Name: {{ package_name }} +Version: {{ package_version }} +Release: 1 +Summary: {{ package_description }} +License: MIT + +BuildArch: noarch + +%description +{{ package_description }} + +%prep +%install +mkdir -p %{buildroot} +#/{{ package_name }}-{{ package_version }} +tar -C %{buildroot} -zxvf {{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz + +%files +/* From 14ee0aa96741dd1821e5461afba648480affa25f Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:22:37 +0200 Subject: [PATCH 13/35] Updated git action, upload rpm --- .github/workflows/packages.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 9ccf3a7f..dc777ef7 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -25,8 +25,14 @@ jobs: targets: "./build-packages.yml" hosts: "localhost" - - name: upload package as artifact + - name: upload deb package as artifact uses: actions/upload-artifact@v2 with: - #name: deb_package + name: Deb package path: ${{ github.workspace }}/build/*.deb + + - name: upload rpm package as artifact + uses: actions/upload-artifact@v2 + with: + name: RPM package + path: ${{ github.workspace }}/build/noarch/*.rpm From fb6a445fdf7c6f9460ed2c9b142180876bf09a82 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:37:18 +0200 Subject: [PATCH 14/35] Fix lint issues --- roles/build-rpm-package/tasks/build_rpm_package.yml | 9 ++++++++- roles/build-rpm-package/tasks/create_directories.yml | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/build-rpm-package/tasks/build_rpm_package.yml b/roles/build-rpm-package/tasks/build_rpm_package.yml index b544fc92..e18f4e92 100644 --- a/roles/build-rpm-package/tasks/build_rpm_package.yml +++ b/roles/build-rpm-package/tasks/build_rpm_package.yml @@ -1,6 +1,13 @@ --- - name: Build RPM package. - command: "rpmbuild --define \"_topdir {{ build_rpm_package_directory }}\" --define \"_rpmdir {{ build_rpm_package_local_directory }}\" -bb {{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec" + command: + cmd: | + rpmbuild + --define "_topdir {{ build_rpm_package_directory }}" + --define "_rpmdir {{ build_rpm_package_local_directory }}" + -bb + {{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec + creates: "{{ build_rpm_package_local_directory }}/noarch/{{ package_name }}-{{ package_version }}*.rpm" #- name: Copy package to local directory # copy: diff --git a/roles/build-rpm-package/tasks/create_directories.yml b/roles/build-rpm-package/tasks/create_directories.yml index 301010aa..507e8849 100644 --- a/roles/build-rpm-package/tasks/create_directories.yml +++ b/roles/build-rpm-package/tasks/create_directories.yml @@ -29,4 +29,3 @@ template: src: specfile.j2 dest: "{{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec" - From b44a5d4d3396ce8c3a455a65b112312e97f65544 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 14:37:28 +0200 Subject: [PATCH 15/35] Rename task --- .github/workflows/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index dc777ef7..adb7226b 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -11,7 +11,7 @@ on: - master jobs: - build_deb: + build-packages: runs-on: ubuntu-latest From 9d391eddb253802cd5301636061a56b38738c6e9 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 15:45:26 +0200 Subject: [PATCH 16/35] Improved spec file template --- roles/build-rpm-package/templates/specfile.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/build-rpm-package/templates/specfile.j2 b/roles/build-rpm-package/templates/specfile.j2 index c8976c68..a9753f6a 100644 --- a/roles/build-rpm-package/templates/specfile.j2 +++ b/roles/build-rpm-package/templates/specfile.j2 @@ -12,8 +12,9 @@ BuildArch: noarch %prep %install mkdir -p %{buildroot} -#/{{ package_name }}-{{ package_version }} tar -C %{buildroot} -zxvf {{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz %files -/* +{% for file in rpmcontents.files %} +{{ file.path | regex_replace(build_rpm_package_files_directories) }} +{% endfor %} From 055038abab5bb8940d3e08c270461c14dd6ea4eb Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 1 Jul 2020 15:45:39 +0200 Subject: [PATCH 17/35] Generate list of files for spec file --- .../build-rpm-package/tasks/create_directories.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/roles/build-rpm-package/tasks/create_directories.yml b/roles/build-rpm-package/tasks/create_directories.yml index 507e8849..637a9871 100644 --- a/roles/build-rpm-package/tasks/create_directories.yml +++ b/roles/build-rpm-package/tasks/create_directories.yml @@ -14,17 +14,18 @@ - "{{ build_rpm_package_directory }}/{{ package_name }}-{{ package_version }}" - "{{ build_rpm_package_directory }}/SPECS" -#- name: Copy source files -# copy: -# src: "{{ build_rpm_package_files_directories }}/" -# dest: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}" - - name: Make tarball of source files archive: - #path: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}" path: "{{ build_rpm_package_files_directories }}/" dest: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz" +- name: Find files that will be included in the RPM package + find: + paths: "{{ build_rpm_package_files_directories }}/" + recurse: yes + file_type: file + register: rpmcontents + - name: Make Spec file template: src: specfile.j2 From 1d38350af0e35e0983019b8f455f456fc4516d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 15:56:16 +0200 Subject: [PATCH 18/35] Remove custom build-rpm-package role --- roles/build-rpm-package/defaults/main.yml | 6 ---- .../tasks/build_rpm_package.yml | 15 --------- .../tasks/create_directories.yml | 32 ------------------- roles/build-rpm-package/tasks/main.yml | 10 ------ roles/build-rpm-package/tasks/packages.yml | 9 ------ roles/build-rpm-package/templates/specfile.j2 | 20 ------------ roles/freedomofpress.build-debian-package | 2 +- 7 files changed, 1 insertion(+), 93 deletions(-) delete mode 100644 roles/build-rpm-package/defaults/main.yml delete mode 100644 roles/build-rpm-package/tasks/build_rpm_package.yml delete mode 100644 roles/build-rpm-package/tasks/create_directories.yml delete mode 100644 roles/build-rpm-package/tasks/main.yml delete mode 100644 roles/build-rpm-package/tasks/packages.yml delete mode 100644 roles/build-rpm-package/templates/specfile.j2 diff --git a/roles/build-rpm-package/defaults/main.yml b/roles/build-rpm-package/defaults/main.yml deleted file mode 100644 index 105be7db..00000000 --- a/roles/build-rpm-package/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# defaults file for build-rpm-package -build_rpm_package_local_directory: build - -# Build directory. -build_rpm_package_directory: /tmp/rpmbuild diff --git a/roles/build-rpm-package/tasks/build_rpm_package.yml b/roles/build-rpm-package/tasks/build_rpm_package.yml deleted file mode 100644 index e18f4e92..00000000 --- a/roles/build-rpm-package/tasks/build_rpm_package.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Build RPM package. - command: - cmd: | - rpmbuild - --define "_topdir {{ build_rpm_package_directory }}" - --define "_rpmdir {{ build_rpm_package_local_directory }}" - -bb - {{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec - creates: "{{ build_rpm_package_local_directory }}/noarch/{{ package_name }}-{{ package_version }}*.rpm" - -#- name: Copy package to local directory -# copy: -# src: "{{ build_rpm_package_directory }}/RPMS/*/*.rpm" -# dest: "{{ build_rpm_package_local_directory }}" diff --git a/roles/build-rpm-package/tasks/create_directories.yml b/roles/build-rpm-package/tasks/create_directories.yml deleted file mode 100644 index 637a9871..00000000 --- a/roles/build-rpm-package/tasks/create_directories.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Clean up build files. - file: - state: absent - path: "{{ build_rpm_package_directory }}" - -- name: Create base build directory. - file: - state: directory - path: "{{ item }}" - with_items: - - "{{ build_rpm_package_directory }}" - - "{{ build_rpm_package_directory }}/SOURCES" - - "{{ build_rpm_package_directory }}/{{ package_name }}-{{ package_version }}" - - "{{ build_rpm_package_directory }}/SPECS" - -- name: Make tarball of source files - archive: - path: "{{ build_rpm_package_files_directories }}/" - dest: "{{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz" - -- name: Find files that will be included in the RPM package - find: - paths: "{{ build_rpm_package_files_directories }}/" - recurse: yes - file_type: file - register: rpmcontents - -- name: Make Spec file - template: - src: specfile.j2 - dest: "{{ build_rpm_package_directory }}/SPECS/{{ package_name }}-{{ package_version }}.spec" diff --git a/roles/build-rpm-package/tasks/main.yml b/roles/build-rpm-package/tasks/main.yml deleted file mode 100644 index b992d124..00000000 --- a/roles/build-rpm-package/tasks/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# tasks file for build-rpm-package -- include: packages.yml - tags: packages - -- include: create_directories.yml - tags: directories - -- include: build_rpm_package.yml - tags: build diff --git a/roles/build-rpm-package/tasks/packages.yml b/roles/build-rpm-package/tasks/packages.yml deleted file mode 100644 index 190c6e31..00000000 --- a/roles/build-rpm-package/tasks/packages.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Install build dependencies. - apt: - name: "{{ item }}" - state: present - update_cache: yes - cache_valid_time: 3600 - with_items: - - rpm diff --git a/roles/build-rpm-package/templates/specfile.j2 b/roles/build-rpm-package/templates/specfile.j2 deleted file mode 100644 index a9753f6a..00000000 --- a/roles/build-rpm-package/templates/specfile.j2 +++ /dev/null @@ -1,20 +0,0 @@ -Name: {{ package_name }} -Version: {{ package_version }} -Release: 1 -Summary: {{ package_description }} -License: MIT - -BuildArch: noarch - -%description -{{ package_description }} - -%prep -%install -mkdir -p %{buildroot} -tar -C %{buildroot} -zxvf {{ build_rpm_package_directory }}/SOURCES/{{ package_name }}-{{ package_version }}.tar.gz - -%files -{% for file in rpmcontents.files %} -{{ file.path | regex_replace(build_rpm_package_files_directories) }} -{% endfor %} diff --git a/roles/freedomofpress.build-debian-package b/roles/freedomofpress.build-debian-package index 880f6147..b52e748e 160000 --- a/roles/freedomofpress.build-debian-package +++ b/roles/freedomofpress.build-debian-package @@ -1 +1 @@ -Subproject commit 880f6147b5abd1fdb0242ff6d3da7f737f848c4e +Subproject commit b52e748e8fb1e671b66bf99682acc23c34f01ed0 From aef3885919c0302b29ad0238679a86a714b34116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 15:57:14 +0200 Subject: [PATCH 19/35] Rename playbook build-packages to prepare-client-packages --- build-packages.yml | 74 ------------------------------------- prepare-client-packages.yml | 54 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 74 deletions(-) delete mode 100644 build-packages.yml create mode 100644 prepare-client-packages.yml diff --git a/build-packages.yml b/build-packages.yml deleted file mode 100644 index 2696e497..00000000 --- a/build-packages.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Build Debian (deb) and Redhat (rpm) packages for CVMFS clients. ---- -- name: Build deb and yum packages. - hosts: localhost - vars: - package_name: cvmfs-config-eessi - package_version: 0.1 - package_description: CVMFS config repository package for EESSI. - pre_tasks: - # Create a temporary directory where we generate the contents of the packages. - - name: Create temporary directory - tempfile: - state: directory - suffix: buildinput - register: temp_buildinput - - # - - name: Create directory structure for the CVMFS config - file: - path: "{{ item }}" - state: directory - mode: 0775 - with_items: - - "{{ temp_buildinput.path }}" - - "{{ temp_buildinput.path }}/etc" - - "{{ temp_buildinput.path }}/etc/cvmfs" - - "{{ temp_buildinput.path }}/etc/cvmfs/keys" - - "{{ temp_buildinput.path }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" - - "{{ temp_buildinput.path }}/etc/cvmfs/config.d" - - "{{ temp_buildinput.path }}/etc/cvmfs/default.d" - - - name: Make EESSI CVMFS public key file - copy: - content: "{{ eessi_cvmfs_config_repo.key.key }}" - dest: "{{ temp_buildinput.path }}{{ eessi_cvmfs_config_repo.key.path }}" - - - name: Make EESSI CVMFS configuration file - copy: - content: | - ## This file is maintained by Ansible - CHANGES WILL BE OVERWRITTEN - CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" - CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" - dest: "{{ temp_buildinput.path }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" - - - name: Make EESSI CVMFS default configuration file - copy: - content: | - CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" - CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" - dest: "{{ temp_buildinput.path }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" - - roles: - - role: freedomofpress.build-debian-package - tags: build - vars: - deb_package_dependencies: [] -# - cvmfs -# - cvmfs-config-none - build_debian_package_files_directories: - - src: "{{ temp_buildinput.path }}/etc/" - dest: "etc" - build_debian_package_control_fields: - Package: "{{ package_name }}" - Version: "{{ package_version }}" - Priority: optional - Architecture: all - Depends: "{{ deb_package_dependencies|join(',') }}" - Maintainer: Bob Dröge - Description: "{{ package_description }}" - - role: build-rpm-package - tags: build - vars: - build_rpm_package_directory: /tmp/rpmbuild - build_rpm_package_files_directories: "{{ temp_buildinput.path }}" diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml new file mode 100644 index 00000000..06811a07 --- /dev/null +++ b/prepare-client-packages.yml @@ -0,0 +1,54 @@ +# Prepare the input files for the Debian (deb) and Redhat (rpm) packages for CVMFS clients. +--- +- name: Prepare Deb and RPM packages + hosts: localhost + vars: + package_vendor: EESSI + package_maintainer: EESSI + package_description: CVMFS config repository package for EESSI. + package_source_dir: "{{ lookup('env', 'GITHUB_WORKSPACE') | default('/tmp', True) }}/package" + tasks: + - name: Create directory structure for the CVMFS config + file: + path: "{{ item }}" + state: directory + mode: 0775 + with_items: + - "{{ package_source_dir }}" + - "{{ package_source_dir }}/etc" + - "{{ package_source_dir }}/etc/cvmfs" + - "{{ package_source_dir }}/etc/cvmfs/keys" + - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}" + - "{{ package_source_dir }}/etc/cvmfs/config.d" + - "{{ package_source_dir }}/etc/cvmfs/default.d" + + - name: Make EESSI CVMFS public key file + copy: + content: "{{ eessi_cvmfs_config_repo.key.key }}" + dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" + + - name: Make EESSI CVMFS configuration file + copy: + content: | + CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" + CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" + dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" + + - name: Make EESSI CVMFS default configuration file + copy: + content: | + CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" + CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" + dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" + + - name: Register all files to be included in the package + find: + paths: "{{ package_source_dir }}/" + recurse: yes + file_type: file + register: pkg_files + + - name: Make package metadata file + template: + src: client_packages_metadata.j2 + dest: "{{ package_source_dir }}/client_packages_metadata.yaml" From cdd0b2001de830157a04d3bff4c27198662febb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 15:57:43 +0200 Subject: [PATCH 20/35] spec template file for building packages --- templates/client_packages_metadata.j2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 templates/client_packages_metadata.j2 diff --git a/templates/client_packages_metadata.j2 b/templates/client_packages_metadata.j2 new file mode 100644 index 00000000..7b1c120b --- /dev/null +++ b/templates/client_packages_metadata.j2 @@ -0,0 +1,12 @@ +# This file is used as metadata file for the generation of client packages. +# See: https://github.com/kentik/pkg +meta: + description: {{ package_description }} + vendor: {{ package_vendor }} + maintainer: {{ package_maintainer }} +files: +{% for file in pkg_files.files %} + "{{ file.path | regex_replace(package_source_dir) }}": + file: "{{ file.path | regex_replace(package_source_dir + '/package/') }}" + user: "root" +{% endfor %} From b361f83560484cb85cfba94524c7bb21b684f81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 15:58:23 +0200 Subject: [PATCH 21/35] Rename and change workflow --- .github/workflows/build-client-packages.yml | 123 ++++++++++++++++++++ .github/workflows/packages.yml | 38 ------ 2 files changed, 123 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/build-client-packages.yml delete mode 100644 .github/workflows/packages.yml diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml new file mode 100644 index 00000000..6fe12cd2 --- /dev/null +++ b/.github/workflows/build-client-packages.yml @@ -0,0 +1,123 @@ +name: Build client packages + +on: + push: + branches: + - master + - build_packages + tags: + - 'v*' + pull_request: + branches: + - master + - build_packages + +jobs: + + build-packages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Get the version number for the packages + id: get_version + # If this is a tag, use the tag name (e.g. v1.2.3) without v as version number. + # Otherwise, just use 0.0. + run: | + VERSION=0.0 + REF_NAME=${{ github.ref }} + [[ $REF_NAME == refs/tags/v* ]] && VERSION=${REF_NAME/refs\/tags\/v/} + echo ::set-output name=version::${VERSION} + + - name: Prepare package source + uses: roles-ansible/check-ansible-debian-stretch-action@master + with: + targets: "./prepare-client-packages.yml" + hosts: "localhost" + + - name: Build RPM package + id: build-rpm + uses: kentik/pkg@v1.0.0-rc6 + with: + name: cvmfs-config-eessi + version: ${{ steps.get_version.outputs.version }} + arch: x86_64 + format: rpm + package: package/client_packages_metadata.yaml + + - name: Build Deb package + id: build-deb + uses: kentik/pkg@v1.0.0-rc6 + with: + name: cvmfs-config-eessi + version: ${{ steps.get_version.outputs.version }} + arch: x86_64 + format: deb + package: package/client_packages_metadata.yaml + + - name: Upload Deb package as artifact + uses: actions/upload-artifact@v2 + with: + name: Deb package + path: ${{ steps.build-deb.outputs.package }} + + - name: Upload RPM package as artifact + uses: actions/upload-artifact@v2 + with: + name: RPM package + path: ${{ steps.build-rpm.outputs.package }} + + release: + needs: build-packages + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - name: Download Deb package + uses: actions/download-artifact@v2 + with: + name: Deb package + + - name: Download RPM package + uses: actions/download-artifact@v2 + with: + name: RPM package + + - name: Find filenames of downloaded packages + id: find_filenames + shell: bash + run: | + rpmfile="$(ls -1 *.rpm)" + debfile="$(ls -1 *.deb)" + echo ::set-output name=rpmfile::${rpmfile} + echo ::set-output name=debfile::${debfile} + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Filesystem Layer ${{ github.ref }} + draft: false + prerelease: false + + - name: Upload RPM as release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.find_filenames.outputs.rpmfile }} + asset_name: ${{ steps.find_filenames.outputs.rpmfile }} + asset_content_type: application/x-rpm + + - name: Upload Deb as release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.find_filenames.outputs.debfile }} + asset_name: ${{ steps.find_filenames.outputs.debfile }} + asset_content_type: application/x-debian-package diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml deleted file mode 100644 index adb7226b..00000000 --- a/.github/workflows/packages.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build packages - -on: - # Trigger the workflow on push or pull request, - # but only for the master branch - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - build-packages: - - runs-on: ubuntu-latest - - steps: - # Important: This sets up your GITHUB_WORKSPACE environment variable - - uses: actions/checkout@v2 - - - name: ansible check with debian:stretch - uses: roles-ansible/check-ansible-debian-stretch-action@master - with: - targets: "./build-packages.yml" - hosts: "localhost" - - - name: upload deb package as artifact - uses: actions/upload-artifact@v2 - with: - name: Deb package - path: ${{ github.workspace }}/build/*.deb - - - name: upload rpm package as artifact - uses: actions/upload-artifact@v2 - with: - name: RPM package - path: ${{ github.workspace }}/build/noarch/*.rpm From 1d0ae6aa6acc655677b7a35d56d2e40b36f1c1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 16:00:10 +0200 Subject: [PATCH 22/35] Don't run CI on non-master branches --- .github/workflows/build-client-packages.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index 6fe12cd2..e0a59ac3 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -4,13 +4,11 @@ on: push: branches: - master - - build_packages tags: - 'v*' pull_request: branches: - master - - build_packages jobs: From e058f49288e96a92b326b540cca8b56cdfd3657a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 16:05:40 +0200 Subject: [PATCH 23/35] Set file permissions on generated/copied files --- prepare-client-packages.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 06811a07..47c61c02 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -26,6 +26,7 @@ copy: content: "{{ eessi_cvmfs_config_repo.key.key }}" dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" + mode: 0644 - name: Make EESSI CVMFS configuration file copy: @@ -33,6 +34,7 @@ CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" + mode: 0644 - name: Make EESSI CVMFS default configuration file copy: @@ -40,6 +42,7 @@ CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" + mode: 0644 - name: Register all files to be included in the package find: @@ -52,3 +55,4 @@ template: src: client_packages_metadata.j2 dest: "{{ package_source_dir }}/client_packages_metadata.yaml" + mode: 0644 From 89b8658c7f5ebdfd4312f5332d55c3db45902c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 16:11:50 +0200 Subject: [PATCH 24/35] remove roles dir --- roles/freedomofpress.build-debian-package | 1 - 1 file changed, 1 deletion(-) delete mode 160000 roles/freedomofpress.build-debian-package diff --git a/roles/freedomofpress.build-debian-package b/roles/freedomofpress.build-debian-package deleted file mode 160000 index b52e748e..00000000 --- a/roles/freedomofpress.build-debian-package +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b52e748e8fb1e671b66bf99682acc23c34f01ed0 From 24493a1db6f9193af4826084f86188686d24ef9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 16:51:43 +0200 Subject: [PATCH 25/35] Fix for not finding group_vars --- .github/workflows/build-client-packages.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index e0a59ac3..b9d80850 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -27,6 +27,11 @@ jobs: [[ $REF_NAME == refs/tags/v* ]] && VERSION=${REF_NAME/refs\/tags\/v/} echo ::set-output name=version::${VERSION} + # The next step uses a custom Ansible inventory, and due to that it cannot find + # the group_vars folder inside the inventory folder. This symlink fixes that. + - name: Make symlink to group_vars + run: ln -s inventory/group_vars + - name: Prepare package source uses: roles-ansible/check-ansible-debian-stretch-action@master with: From 99481e2069f791df6bdcd15a3124e02da09b11cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:11:37 +0200 Subject: [PATCH 26/35] remove spec file for packages --- templates/client_packages_metadata.j2 | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 templates/client_packages_metadata.j2 diff --git a/templates/client_packages_metadata.j2 b/templates/client_packages_metadata.j2 deleted file mode 100644 index 7b1c120b..00000000 --- a/templates/client_packages_metadata.j2 +++ /dev/null @@ -1,12 +0,0 @@ -# This file is used as metadata file for the generation of client packages. -# See: https://github.com/kentik/pkg -meta: - description: {{ package_description }} - vendor: {{ package_vendor }} - maintainer: {{ package_maintainer }} -files: -{% for file in pkg_files.files %} - "{{ file.path | regex_replace(package_source_dir) }}": - file: "{{ file.path | regex_replace(package_source_dir + '/package/') }}" - user: "root" -{% endfor %} From 88579b0e12622975b3ae99c13528577bb4bf84c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:12:49 +0200 Subject: [PATCH 27/35] use fpm instead of nfpm --- .github/workflows/build-client-packages.yml | 38 ++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index b9d80850..f88509dc 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -4,11 +4,13 @@ on: push: branches: - master + - build_packages tags: - 'v*' pull_request: branches: - master + - build_packages jobs: @@ -27,11 +29,6 @@ jobs: [[ $REF_NAME == refs/tags/v* ]] && VERSION=${REF_NAME/refs\/tags\/v/} echo ::set-output name=version::${VERSION} - # The next step uses a custom Ansible inventory, and due to that it cannot find - # the group_vars folder inside the inventory folder. This symlink fixes that. - - name: Make symlink to group_vars - run: ln -s inventory/group_vars - - name: Prepare package source uses: roles-ansible/check-ansible-debian-stretch-action@master with: @@ -40,35 +37,38 @@ jobs: - name: Build RPM package id: build-rpm - uses: kentik/pkg@v1.0.0-rc6 + uses: bpicode/github-action-fpm@master with: - name: cvmfs-config-eessi - version: ${{ steps.get_version.outputs.version }} - arch: x86_64 - format: rpm - package: package/client_packages_metadata.yaml + fpm_args: "etc" + fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package" - name: Build Deb package id: build-deb - uses: kentik/pkg@v1.0.0-rc6 + uses: bpicode/github-action-fpm@master with: - name: cvmfs-config-eessi - version: ${{ steps.get_version.outputs.version }} - arch: x86_64 - format: deb - package: package/client_packages_metadata.yaml + fpm_args: "etc" + fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package" + + - name: Find filenames of downloaded packages + id: find_filenames + shell: bash + run: | + rpmfile="$(ls -1 *.rpm)" + debfile="$(ls -1 *.deb)" + echo ::set-output name=rpmfile::${rpmfile} + echo ::set-output name=debfile::${debfile} - name: Upload Deb package as artifact uses: actions/upload-artifact@v2 with: name: Deb package - path: ${{ steps.build-deb.outputs.package }} + path: ${{ steps.find_filenames.outputs.debfile }} - name: Upload RPM package as artifact uses: actions/upload-artifact@v2 with: name: RPM package - path: ${{ steps.build-rpm.outputs.package }} + path: ${{ steps.find_filenames.outputs.rpmfile }} release: needs: build-packages From f157b95da35fb76fd6007678092b7cb9145a912f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:13:09 +0200 Subject: [PATCH 28/35] use fpm instead of nfpm --- prepare-client-packages.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 47c61c02..7a20e2e9 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -26,7 +26,6 @@ copy: content: "{{ eessi_cvmfs_config_repo.key.key }}" dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" - mode: 0644 - name: Make EESSI CVMFS configuration file copy: @@ -34,7 +33,6 @@ CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" - mode: 0644 - name: Make EESSI CVMFS default configuration file copy: @@ -42,17 +40,3 @@ CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" - mode: 0644 - - - name: Register all files to be included in the package - find: - paths: "{{ package_source_dir }}/" - recurse: yes - file_type: file - register: pkg_files - - - name: Make package metadata file - template: - src: client_packages_metadata.j2 - dest: "{{ package_source_dir }}/client_packages_metadata.yaml" - mode: 0644 From 3548c6655f0cd7062c2282a4642aae12ce7773a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:15:11 +0200 Subject: [PATCH 29/35] Add description to package --- .github/workflows/build-client-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index f88509dc..1476f7f7 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -40,14 +40,14 @@ jobs: uses: bpicode/github-action-fpm@master with: fpm_args: "etc" - fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package" + fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" - name: Build Deb package id: build-deb uses: bpicode/github-action-fpm@master with: fpm_args: "etc" - fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package" + fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'" - name: Find filenames of downloaded packages id: find_filenames From 1f6bd7da0c2f47ddb94826207777e043b62ae37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:20:03 +0200 Subject: [PATCH 30/35] add modes for copy tasks --- prepare-client-packages.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/prepare-client-packages.yml b/prepare-client-packages.yml index 7a20e2e9..25dd3003 100644 --- a/prepare-client-packages.yml +++ b/prepare-client-packages.yml @@ -26,6 +26,7 @@ copy: content: "{{ eessi_cvmfs_config_repo.key.key }}" dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}" + mode: 0644 - name: Make EESSI CVMFS configuration file copy: @@ -33,6 +34,7 @@ CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}" CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}" dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf" + mode: 0644 - name: Make EESSI CVMFS default configuration file copy: @@ -40,3 +42,4 @@ CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}" CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}" dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf" + mode: 0644 From ab498e6a88bbffb5c7b85be7eb8b7b21d5674f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 20:31:11 +0200 Subject: [PATCH 31/35] Fix for not finding group_vars --- .github/workflows/build-client-packages.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index 1476f7f7..6e8fd191 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -29,6 +29,11 @@ jobs: [[ $REF_NAME == refs/tags/v* ]] && VERSION=${REF_NAME/refs\/tags\/v/} echo ::set-output name=version::${VERSION} + # The next step uses a custom Ansible inventory, and due to that it cannot find + # the group_vars folder inside the inventory folder. This symlink fixes that. + - name: Make symlink to group_vars + run: ln -s inventory/group_vars + - name: Prepare package source uses: roles-ansible/check-ansible-debian-stretch-action@master with: From bc70d8946d13594d65b990b23e2e411c57e01e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 29 Sep 2020 21:14:50 +0200 Subject: [PATCH 32/35] small syntax fix --- .github/workflows/build-client-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-client-packages.yml b/.github/workflows/build-client-packages.yml index 6e8fd191..96f10431 100644 --- a/.github/workflows/build-client-packages.yml +++ b/.github/workflows/build-client-packages.yml @@ -33,7 +33,7 @@ jobs: # the group_vars folder inside the inventory folder. This symlink fixes that. - name: Make symlink to group_vars run: ln -s inventory/group_vars - + - name: Prepare package source uses: roles-ansible/check-ansible-debian-stretch-action@master with: From 01f915b000bf33596cf5863a105816a7a3550e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 30 Sep 2020 09:37:35 +0200 Subject: [PATCH 33/35] Update README for client packages --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index bf8bd381..1b015039 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,8 @@ ansible-playbook -b -K -e @inventory/local_site_specific_vars.yml localproxy.yml ``` ### Clients + +#### Method 1: Ansible Make sure that your hosts file contains the list of hosts where the CVMFS client should be installed. Furthermore, you can define a list of (local) proxy servers that your clients should use in `inventory/local_site_specific_vars.yml` using the parameter `local_cvmfs_http_proxies`. @@ -146,6 +148,30 @@ Finally, run the playbook: ansible-playbook -b -K -e @inventory/local_site_specific_vars.yml client.yml ``` +#### Method 2: Packages +On many operating systems the CVMFS client can be installed through your package manager. +For details, see the [Getting Started page](https://cvmfs.readthedocs.io/en/stable/cpt-quickstart.html) +in the documentation. + +After installing the client, you will have to configure it. +For this you can use the CVMFS configuration packages that we provide for clients. +These packages can be found on the [Releases](https://github.com/eessi/filesystem-layer/releases) page. +Download the package for your operating system, and install it, e.g.: +``` +rpm -i cvmfs-config-eessi-*.rpm +dpkg -i cvmfs-config-eessi-*.rpm +``` + +Finally, you need to make a file `/etc/cvmfs/default.local` manually; this file is used for local settings and +contains, for instance, the URL to your local proxy and the size of the local cache. As an example, you can put +the following in this file, which corresponds to not using a proxy and setting the local quota limit to 40000MB: +``` +CVMFS_HTTP_PROXY=DIRECT +CVMFS_QUOTA_LIMIT=40000 +``` + +Admin note: for building the client configuration packages, see + ## Verification and usage ### Client @@ -198,3 +224,12 @@ touch /cvmfs/pilot.eessi-hpc.org/testdir/testfile sudo cvmfs_server publish pilot.eessi-hpc.org ``` It might take a few minutes, but then the new file should show up at the clients. + + +## Building the CVMFS configuration packages + +For each push and pull request to the master branch, packages are automatically built by a Github Action. +The resulting (unversioned) packages can be found as build artifacts on the page of each run of this action. +When a new tag is created to mark a versioned release of the repository (e.g. `v1.2.3`, where the `v` is required!), +the action builds a package with the same version number, creates a release, and stores the packages +as release assets. From f614ebd3fb47ec901e2e02dea4f6e77da8ab3e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Wed, 30 Sep 2020 09:41:51 +0200 Subject: [PATCH 34/35] Update README for client packages --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1b015039..871d950c 100644 --- a/README.md +++ b/README.md @@ -162,15 +162,18 @@ rpm -i cvmfs-config-eessi-*.rpm dpkg -i cvmfs-config-eessi-*.rpm ``` -Finally, you need to make a file `/etc/cvmfs/default.local` manually; this file is used for local settings and +Next, you need to make a file `/etc/cvmfs/default.local` manually; this file is used for local settings and contains, for instance, the URL to your local proxy and the size of the local cache. As an example, you can put the following in this file, which corresponds to not using a proxy and setting the local quota limit to 40000MB: ``` CVMFS_HTTP_PROXY=DIRECT CVMFS_QUOTA_LIMIT=40000 ``` +For more details about configuring your client, see https://cvmfs.readthedocs.io/en/stable/cpt-configure.html. -Admin note: for building the client configuration packages, see +Finally, run `cvmfs_config setup` to set up CVMFS. + +*Admin note: for building the client configuration packages, see [this section](#building-the-cvmfs-configuration-packages).* ## Verification and usage From 09a24efa93a571d92ca2923978e12a6bc220559f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 1 Oct 2020 09:03:25 +0200 Subject: [PATCH 35/35] rpm -> deb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 871d950c..839ee2e1 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ These packages can be found on the [Releases](https://github.com/eessi/filesyste Download the package for your operating system, and install it, e.g.: ``` rpm -i cvmfs-config-eessi-*.rpm -dpkg -i cvmfs-config-eessi-*.rpm +dpkg -i cvmfs-config-eessi-*.deb ``` Next, you need to make a file `/etc/cvmfs/default.local` manually; this file is used for local settings and