From b7346d64ad5e73fc84aa45b4212d4c718c3dc544 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Tue, 29 Oct 2024 11:26:22 -0400 Subject: [PATCH 01/11] .gitignore: Ignore ansible inventory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f19c79c..3391d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /unstable/ /clones/ /work/ +/inventory/hosts.yml \ No newline at end of file From dc39dfeb688d968347152ac7116427004b5d3e23 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Tue, 29 Oct 2024 11:29:52 -0400 Subject: [PATCH 02/11] Break out mirror.tar creation into a separate script --- archive_screenshots.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 archive_screenshots.sh diff --git a/archive_screenshots.sh b/archive_screenshots.sh new file mode 100644 index 0000000..d72706f --- /dev/null +++ b/archive_screenshots.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +if [[ ! -d "work/output" ]]; then + echo "No output directory found, have you ran appstream-builder?" +fi + +pushd work/output + +if [[ ! -d "mirror/source" ]]; then + mkdir -p mirror/source +fi + +for i in 112x63 224x126 1248x702 624x351 1504x846 752x423; do + cp -R $i mirror/. +done + +pushd mirror/source +tar xf ../../*screenshots*.tar + +popd + +tar cvf mirror.tar mirror/ \ No newline at end of file From fa50edf3d6f2a67e83fc79f7cf9fc9ec8a10ea0e Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Tue, 29 Oct 2024 11:35:31 -0400 Subject: [PATCH 03/11] Add an ansible playbook which automates appstream generation --- create_symlinks.py | 78 +++++++++++++++++++ inventory/hosts.yml.example | 5 ++ playbook.yml | 149 ++++++++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 create_symlinks.py create mode 100644 inventory/hosts.yml.example create mode 100644 playbook.yml diff --git a/create_symlinks.py b/create_symlinks.py new file mode 100644 index 0000000..bf0e822 --- /dev/null +++ b/create_symlinks.py @@ -0,0 +1,78 @@ +# Create a directory of symlinks to optimize and correct the appstream build + +import pathlib +import argparse +import os +from xml.etree import ElementTree + + +def main(): + parser = argparse.ArgumentParser( + prog='create_symlinks', + description='Create symbolic links to eopkg files to optimize and correct our appstream build' + ) + parser.add_argument( + "eopkg_index", + action="store", + help="Path to the eopkg index file to be validated against. This must be an uncompressed XML file.", + type=pathlib.Path, + ) + parser.add_argument( + "packages_directory", + action="store", + help="Path to the packages (input) directory.", + type=pathlib.Path, + ) + parser.add_argument( + "symlinks_directory", + action="store", + help="Path to the symlinks (output) directory.", + type=pathlib.Path, + ) + args = parser.parse_args() + eopkg_packages = get_packages_from_eopkg_index(args.eopkg_index) + symlinks_created = 0 + for package_file in args.packages_directory.glob('**/*.eopkg'): + if check_file(package_file, eopkg_packages): + # Create a symlink for this package, we like it + os.symlink(package_file, pathlib.Path.joinpath(args.symlinks_directory, pathlib.Path(package_file.name))) + symlinks_created += 1 + print(f'Created {symlinks_created} symlinks.') + + +def get_packages_from_eopkg_index(xml_path: pathlib.Path) -> dict: + print('gothere') + solus_xml = open(xml_path, 'r') + tree = ElementTree.parse(solus_xml) + root = tree.getroot() + packages = { + package.find("Name").text: package.find("History").findall("Update")[0].attrib['release'] + for package in root.findall("Package") + } + # pprint(packages) + return packages + + +def parse_package_filename(package_filename: str) -> dict: + package_split = package_filename.rsplit('-', 4) + output = { + 'name': package_split[0], + 'release': package_split[2], + 'dbginfo': 'dbginfo' in package_filename + } + return output + + +def check_file(path: pathlib.Path, eopkg_packages: dict) -> bool: + eopkg_info = parse_package_filename(path.name) + print(eopkg_info) + if (eopkg_info['name'] in eopkg_packages.keys() and eopkg_info['release'] == eopkg_packages[eopkg_info['name']])\ + and not eopkg_info['dbginfo']\ + : + return True + else: + return False + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/inventory/hosts.yml.example b/inventory/hosts.yml.example new file mode 100644 index 0000000..0cb8237 --- /dev/null +++ b/inventory/hosts.yml.example @@ -0,0 +1,5 @@ +teaparty: + hosts: + packages.getsol.us: + ansible_user: + # Set this to your username diff --git a/playbook.yml b/playbook.yml new file mode 100644 index 0000000..6b103b6 --- /dev/null +++ b/playbook.yml @@ -0,0 +1,149 @@ +--- +- name: Ensure requirements are met for building appstream data + hosts: teaparty + become: yes + tags: + - setup + + tasks: + - name: Check for appstream-builder + ansible.builtin.raw: which appstream-builder + check_mode: false + changed_when: false + failed_when: which_res.rc > 1 + register: which_res + + - name: Ensure other dependencies are installed + ansible.builtin.dnf: + name: + - librsvg2 + - gtk3 + - libpng + state: present + + - name: Remove old work directory to avoid permissions issues + ansible.builtin.file: + path: "/srv/appstream-data/work/" + state: absent + +- name: Actually build appstream data + hosts: teaparty + become: no + tags: + - generate + + tasks: + - name: Ensure work/output directories exist + ansible.builtin.file: + path: "/srv/appstream-data/{{ item }}" + state: directory + loop: + - work/ + - work/output + - work/cache + - work/logs + - work/icons + - work/tmp + - work/package-symlinks + + - name: Select package files to parse using appstream-builder + ansible.builtin.command: + argv: + - python3 + - /srv/appstream-data/create_symlinks.py + - /srv/ferryd/root/repo/unstable/eopkg-index.xml + - /srv/ferryd/root/repo/unstable/ + - /srv/appstream-data/work/package-symlinks + + - name: Run appstream-builder against the entire repository + ansible.builtin.command: + argv: + - appstream-builder + - --packages-dir=./package-symlinks + - --output-dir=./output + - --cache-dir=./cache + - --log-dir=./logs + - --include-failed + - --basename=solus-1 + - --origin=solus + - --veto-ignore=missing-parents + - --veto-ignore=add-default-icons + chdir: /srv/appstream-data/work/ + + - name: Gather screenshots based on appstream data + ansible.builtin.command: + argv: + - appstream-util + - mirror-screenshots + - output/solus-1.xml.gz + - https://screenshots.getsol.us + - ./cache + - ./output + chdir: /srv/appstream-data/work/ + +- name: Check generated appstream data + hosts: teaparty + become: no + tags: + - test_metadata + + tasks: + - name: Check appstream metadata against eopkg index + ansible.builtin.command: + argv: + - /srv/appstream-data/check_packages_exist.py + - /srv/appstream-data/work/output/solus-1.xml.gz + - /srv/ferryd/root/repo/unstable/eopkg-index.xml + + # TODO: need to test contents of screenshots archive + +- name: Install screenshots + hosts: teaparty + become: yes + tags: + - install_screenshots + + tasks: + - name: Create a backup of existing screenshots + community.general.archive: + dest: /srv/www/screenshots/mirror.tar.backup + format: "tar" + path: /srv/www/screenshots/* + exclude_path: + - /srv/www/screenshots/mirror.tar + - /srv/www/screenshots/mirror.tar.backup + + - name: Archive new screenshots + ansible.builtin.command: + argv: + - ./archive_screenshots.sh + chdir: /srv/appstream-data + + - name: Extract new screenshots + ansible.builtin.unarchive: + remote_src: true + src: /srv/appstream-data/work/output/mirror.tar + dest: /srv/www/screenshots/ + mode: "0755" + extra_opts: + - --strip-components=1 + +- name: Fetch generated appstream data + hosts: teaparty + become: no + tags: + - fetch_appstream_data + + tasks: + - name: Fetch appstream data + ansible.builtin.fetch: + src: "/srv/appstream-data/work/output/{{ item }}" + dest: "{{ playbook_dir }}/" + flat: true + loop: + - "solus-1-failed.xml.gz" + - "solus-1-icons.tar.gz" + - "solus-1-ignore.xml.gz" + - "solus-1-screenshots.tar" + - "solus-1.xml.gz" + From c6e5c0f2328909fa0a8e9bee08ea32d5e7dd1864 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Thu, 21 Nov 2024 01:05:18 -0500 Subject: [PATCH 04/11] Add a python script which prints the next git tag --- get-next-tag.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 get-next-tag.py diff --git a/get-next-tag.py b/get-next-tag.py new file mode 100755 index 0000000..20e7b1b --- /dev/null +++ b/get-next-tag.py @@ -0,0 +1,10 @@ +#!/bin/python3 +# A script to determine the next tag for getsolus/solus-appstream-data. +# This behavior is likely fairly specific to this repository. + +import subprocess + +git_tags = subprocess.check_output(['git', '--no-pager', 'tag', '--sort=-creatordate']).split(b'\n') +latest_number = int(git_tags[0].strip(b'v')) +next_tag = f'v{latest_number + 1}' +print(next_tag) From a53332406d019b57a5fa840205a0c8f0b81087a8 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Thu, 21 Nov 2024 02:12:45 -0500 Subject: [PATCH 05/11] Add Taskfile.yml --- Taskfile.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Taskfile.yml diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..4cf4e46 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,29 @@ +version: 3 +vars: + NEXT_TAG: + sh: '{{.TASKFILE_DIR}}/get-next-tag.py' + +tasks: + generate: + label: "Generate Appstream metadata and download it" + cmds: + - "ansible-playbook -i inventory/hosts.yml -K playbook.yml" + + commit: + label: "Add, commit, and tag new appstream data" + cmds: + - "git add solus-1-failed.xml.gz solus-1-icons.tar.gz solus-1-ignore.xml.gz solus-1-screenshots.tar solus-1.xml.gz" + - "git commit -m 'Refresh Appstream metainfo'" + - "git tag {{.NEXT_TAG}}" + + push: + label: "Push changes to getsolus/solus-appstream-data" + cmds: + - "git push --tags" + + full-process: + label: "Perform the entire appstream generation process" + cmds: + - task: generate + - task: commit + - task: push From 3dd44b3437c98f79820adeafc9b2c5ee8078bf09 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Thu, 21 Nov 2024 12:38:23 -0500 Subject: [PATCH 06/11] Follow consistent script naming practice --- Taskfile.yml | 2 +- get-next-tag.py => get_next_tag.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename get-next-tag.py => get_next_tag.py (100%) diff --git a/Taskfile.yml b/Taskfile.yml index 4cf4e46..8712059 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 vars: NEXT_TAG: - sh: '{{.TASKFILE_DIR}}/get-next-tag.py' + sh: '{{.TASKFILE_DIR}}/get_next_tag.py' tasks: generate: diff --git a/get-next-tag.py b/get_next_tag.py similarity index 100% rename from get-next-tag.py rename to get_next_tag.py From ea8938ad05a62b5634586a46932179a5d2eb46d9 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Thu, 21 Nov 2024 12:49:12 -0500 Subject: [PATCH 07/11] Add script to automate creating ansible inventory --- setup_inventory.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 setup_inventory.py diff --git a/setup_inventory.py b/setup_inventory.py new file mode 100644 index 0000000..65962a9 --- /dev/null +++ b/setup_inventory.py @@ -0,0 +1,25 @@ +import yaml +import argparse +import pathlib + +def main(): + parser = argparse.ArgumentParser( + prog='setup_inventory', + description='Set up the Ansible inventory for Solus appstream data generation' + ) + parser.add_argument( + "inventory_file", + action="store", + help="Path to the desired location of the inventory file", + type=pathlib.Path, + ) + args = parser.parse_args() + username = input("Your username on teaparty (must have sudo access): ") + print(f'Writing Ansible inventory to {args.inventory_file}') + inventory = {'teaparty': {'hosts': {'packages.getsol.us': {'ansible_user': username}}}} + + with open(args.inventory_file, 'w') as f: + yaml.safe_dump(inventory, f) + +if __name__ == '__main__': + main() From 408e252da5b6a35df68fd053076a16cf1f6c1d71 Mon Sep 17 00:00:00 2001 From: Hans Kelson Date: Thu, 21 Nov 2024 16:10:38 -0500 Subject: [PATCH 08/11] Document and somewhat automate the setup process --- HISTORICAL-README.md | 63 ++++++++++++++++++++++++++++++++++ README.md | 81 ++++++++++++-------------------------------- Taskfile.yml | 6 ++++ 3 files changed, 90 insertions(+), 60 deletions(-) create mode 100644 HISTORICAL-README.md diff --git a/HISTORICAL-README.md b/HISTORICAL-README.md new file mode 100644 index 0000000..bd59fdf --- /dev/null +++ b/HISTORICAL-README.md @@ -0,0 +1,63 @@ +## Old process - Regenerating appstream data on your local machine +> [!WARNING] +> This process should still be functional, but is much more tedious and time-consuming than the new playbook method. These directions are kept around for historical purposes only. + +1. Install `appstream-glib`, `rsync` + +2. Run `./clone.sh` to clone all packages from packages.getsol.us + +3. Run `./mvem.sh` to remove unneccessary eopkgs for appstream-builder to chew through + +4. Run `./do_stream.sh` to actually generate the appstream data + +5. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository + +- Hint: Run `diff --stat` to get a general idea of the size differences +- Hint: Look through `solus-1-failed.xml.gz` to see if there any easily fixable failures +- Hint: Add + ``` + [diff "gz"] + textconv = gzip -dc + binary = true + ``` + to your git config to diff the .gz files. + +6. Run `./install.sh` to test out locally. + +7. Upload `./work/output/mirror.tar` to `packages.getsol.us:/srv/www/screenshots/`, create a tar backup of the old screenshots and untar in place, make sure the folder permissions are 0755. +8. Load up software centre and check out some packages with metadata to confirm all is well + +7. Git commit along with a new tag + +8. Build `appstream-data` with the new tag and publish to the people. + +## Old Process - regenerating from teaparty server directly +> [!WARNING] +> This process should still be functional, but is much more tedious and time-consuming than the new playbook method. These directions are kept around for historical purposes only. + +If you have ssh access to the teaparty repo server (you have if you've done a sync), you can regenerate the appstream data directly from there without the need to clone the binary repo first. + +1. SSH into teaparty and go to `/srv/appstream-data` which is a shared directory of this repository. + +2. Run `./do_stream_teaparty.sh` to actually generate the appstream data. Try to choose a quiet time when the server isn't under much load. + +3. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository. + +4. Installing screenshots: + - Ensure the recently generated `mirror.tar` archive looks correct `tar -tvf ./work/output/mirror.tar`, it should contain screenshots + - Create a backup of the previous screenshots: `pushd /srv/www/screenshots/; sudo tar --create --file=mirror.tar.backup $(ls -d */) && popd` + - Move the screenshots archive to screenshots: `sudo mv work/output/mirror.tar /srv/www/screenshots/` + - Untar in place, ensure permissions are correct: `pushd /srv/www/screenshots; sudo tar -xvf mirror.tar --strip-components=1 && chmod 0755 $(ls -d */); popd` + +5. Sync the artefacts to this repo locally: + - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.gz .` + - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.tar .` + +6. Run `./install.sh` to try out and load up the software centre and check out some packages with metadata to confirm all is well + +7. Git commit along with a new tag + +8. Build `appstream-data` with the new tag and publish to the people. + +TODO: + - Make appstream-builder ignore -devel and -dbginfo packages \ No newline at end of file diff --git a/README.md b/README.md index 4c022e4..b775bb8 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,26 @@ # Solus Appstream Generation -## Regenerating appstream data - -1. Install `appstream-glib`, `rsync` - -2. Run `./clone.sh` to clone all packages from packages.getsol.us - -3. Run `./mvem.sh` to remove unneccessary eopkgs for appstream-builder to chew through - -4. Run `./do_stream.sh` to actually generate the appstream data - -5. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository - -- Hint: Run `diff --stat` to get a general idea of the size differences -- Hint: Look through `solus-1-failed.xml.gz` to see if there any easily fixable failures -- Hint: Add - ``` - [diff "gz"] - textconv = gzip -dc - binary = true - ``` - to your git config to diff the .gz files. - -6. Run `./install.sh` to test out locally. - -7. Upload `./work/output/mirror.tar` to `packages.getsol.us:/srv/www/screenshots/`, create a tar backup of the old screenshots and untar in place, make sure the folder permissions are 0755. - -8. Load up software centre and check out some packages with metadata to confirm all is well - -7. Git commit along with a new tag - -8. Build `appstream-data` with the new tag and publish to the people. - -## Regenerating from teaparty server directly - -If you have ssh access to the teaparty repo server (you have if you've done a sync), you can regenerate the appstream data directly from there without the need to clone the binary repo first. - -1. SSH into teaparty and go to `/srv/appstream-data` which is a shared directory of this repository. - -2. Run `./do_stream_teaparty.sh` to actually generate the appstream data. Try to choose a quiet time when the server isn't under much load. - -3. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository. - -4. Installing screenshots: - - Ensure the recently generated `mirror.tar` archive looks correct `tar -tvf ./work/output/mirror.tar`, it should contain screenshots - - Create a backup of the previous screenshots: `pushd /srv/www/screenshots/; sudo tar --create --file=mirror.tar.backup $(ls -d */) && popd` - - Move the screenshots archive to screenshots: `sudo mv work/output/mirror.tar /srv/www/screenshots/` - - Untar in place, ensure permissions are correct: `pushd /srv/www/screenshots; sudo tar -xvf mirror.tar --strip-components=1 && chmod 0755 $(ls -d */); popd` - -5. Sync the artefacts to this repo locally: - - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.gz .` - - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.tar .` - -6. Run `./install.sh` to try out and load up the software centre and check out some packages with metadata to confirm all is well - -7. Git commit along with a new tag - -8. Build `appstream-data` with the new tag and publish to the people. - -TODO: - - Make appstream-builder ignore -devel and -dbginfo packages +## Usage +### Initial Setup +These directions will guide you through configuring your system to generate appstream data. You should only need to do this once. +> [!NOTE] +> This assumes you have already completed the [Prepare for Packaging](https://help.getsol.us/docs/packaging/prepare-for-packaging) steps listed in the help center. You will need a fully-configured git setup for this tooling to work. +> Additionally, your GitHub account must have push access to this repository (should be true for all staff). +1. Ensure that your local clone of this repository is set up to be able to push (Cloning via GitHub CLI recommended). +2. Ensure you have `go-task` installed. All interaction with this tooling should be possible through the `Taskfile.yml` in this repository. +3. Run `go-task appstream-init`. This task will install `pyyaml` and `ansible` on your system and walk you through setting up the Ansible inventory you need for the playbook. +### Generating Appstream Data +This is the actual process of generating appstream metadata from our repository, which should ideally be done each week (after deprecations, but before sync). Make sure you've correctly completed all the Initial Setup steps first. +1. Run `go-task full-process` from this directory. That will automatically: + - Generate appstream data, + - Check it against the eopkg index, + - Download new metadata to your local clone of the repo, + - Commit the changes, + - Add a new tag to the repository, + - and push the changes to github. +> [!NOTE] +> This would be a good time to take a break and do something else. Just make sure your computer doesn't go to sleep. It takes a while (about 30 minutes). +2. Go to your clone of the packages monorepo and update the `appstream-data` package to use the newly-tagged version of this repository. Follow standard packaging procedure to get those changes into the repository. ## Debugging Failures diff --git a/Taskfile.yml b/Taskfile.yml index 8712059..3d1ab77 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,3 +27,9 @@ tasks: - task: generate - task: commit - task: push + + appstream-init: + label: "Set up the local system to run this tooling" + cmds: + - "sudo eopkg install pyyaml ansible" + - "{{.TASKFILE_DIR}}/setup_inventory.py" From 58e7e2b3c468bbcb70ded665de8eae8b104e35b7 Mon Sep 17 00:00:00 2001 From: Hans K Date: Wed, 1 Jan 2025 09:58:29 -0500 Subject: [PATCH 09/11] setup_inventory: Add shebang line and execute bit --- setup_inventory.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 setup_inventory.py diff --git a/setup_inventory.py b/setup_inventory.py old mode 100644 new mode 100755 index 65962a9..1f626cd --- a/setup_inventory.py +++ b/setup_inventory.py @@ -1,3 +1,5 @@ +#!/bin/python3 + import yaml import argparse import pathlib From d24a5453191ce14b496f84d5fd8c864ffebe264a Mon Sep 17 00:00:00 2001 From: Hans K Date: Wed, 1 Jan 2025 09:59:50 -0500 Subject: [PATCH 10/11] appstream-init: Install community.general and pass filename to setup_inventory.py --- Taskfile.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3d1ab77..d37c292 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -32,4 +32,5 @@ tasks: label: "Set up the local system to run this tooling" cmds: - "sudo eopkg install pyyaml ansible" - - "{{.TASKFILE_DIR}}/setup_inventory.py" + - "ansible-galaxy collection install community.general" + - "{{.TASKFILE_DIR}}/setup_inventory.py {{.TASKFILE_DIR}}/inventory/hosts.yml" From d18a34268b12fd517ead31c8129df58633cd52ef Mon Sep 17 00:00:00 2001 From: Hans K Date: Wed, 1 Jan 2025 10:01:50 -0500 Subject: [PATCH 11/11] Add executable bit to archive_screenshots.sh --- archive_screenshots.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 archive_screenshots.sh diff --git a/archive_screenshots.sh b/archive_screenshots.sh old mode 100644 new mode 100755