diff --git a/.gitignore b/.gitignore index f19c79c..3391d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /unstable/ /clones/ /work/ +/inventory/hosts.yml \ No newline at end of file diff --git a/HISTORICAL-README.md b/HISTORICAL-README.md new file mode 100644 index 0000000..bd59fdf --- /dev/null +++ b/HISTORICAL-README.md @@ -0,0 +1,63 @@ +## Old process - Regenerating appstream data on your local machine +> [!WARNING] +> This process should still be functional, but is much more tedious and time-consuming than the new playbook method. These directions are kept around for historical purposes only. + +1. Install `appstream-glib`, `rsync` + +2. Run `./clone.sh` to clone all packages from packages.getsol.us + +3. Run `./mvem.sh` to remove unneccessary eopkgs for appstream-builder to chew through + +4. Run `./do_stream.sh` to actually generate the appstream data + +5. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository + +- Hint: Run `diff --stat` to get a general idea of the size differences +- Hint: Look through `solus-1-failed.xml.gz` to see if there any easily fixable failures +- Hint: Add + ``` + [diff "gz"] + textconv = gzip -dc + binary = true + ``` + to your git config to diff the .gz files. + +6. Run `./install.sh` to test out locally. + +7. Upload `./work/output/mirror.tar` to `packages.getsol.us:/srv/www/screenshots/`, create a tar backup of the old screenshots and untar in place, make sure the folder permissions are 0755. +8. Load up software centre and check out some packages with metadata to confirm all is well + +7. Git commit along with a new tag + +8. Build `appstream-data` with the new tag and publish to the people. + +## Old Process - regenerating from teaparty server directly +> [!WARNING] +> This process should still be functional, but is much more tedious and time-consuming than the new playbook method. These directions are kept around for historical purposes only. + +If you have ssh access to the teaparty repo server (you have if you've done a sync), you can regenerate the appstream data directly from there without the need to clone the binary repo first. + +1. SSH into teaparty and go to `/srv/appstream-data` which is a shared directory of this repository. + +2. Run `./do_stream_teaparty.sh` to actually generate the appstream data. Try to choose a quiet time when the server isn't under much load. + +3. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository. + +4. Installing screenshots: + - Ensure the recently generated `mirror.tar` archive looks correct `tar -tvf ./work/output/mirror.tar`, it should contain screenshots + - Create a backup of the previous screenshots: `pushd /srv/www/screenshots/; sudo tar --create --file=mirror.tar.backup $(ls -d */) && popd` + - Move the screenshots archive to screenshots: `sudo mv work/output/mirror.tar /srv/www/screenshots/` + - Untar in place, ensure permissions are correct: `pushd /srv/www/screenshots; sudo tar -xvf mirror.tar --strip-components=1 && chmod 0755 $(ls -d */); popd` + +5. Sync the artefacts to this repo locally: + - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.gz .` + - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.tar .` + +6. Run `./install.sh` to try out and load up the software centre and check out some packages with metadata to confirm all is well + +7. Git commit along with a new tag + +8. Build `appstream-data` with the new tag and publish to the people. + +TODO: + - Make appstream-builder ignore -devel and -dbginfo packages \ No newline at end of file diff --git a/README.md b/README.md index 4c022e4..b775bb8 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,26 @@ # Solus Appstream Generation -## Regenerating appstream data - -1. Install `appstream-glib`, `rsync` - -2. Run `./clone.sh` to clone all packages from packages.getsol.us - -3. Run `./mvem.sh` to remove unneccessary eopkgs for appstream-builder to chew through - -4. Run `./do_stream.sh` to actually generate the appstream data - -5. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository - -- Hint: Run `diff --stat` to get a general idea of the size differences -- Hint: Look through `solus-1-failed.xml.gz` to see if there any easily fixable failures -- Hint: Add - ``` - [diff "gz"] - textconv = gzip -dc - binary = true - ``` - to your git config to diff the .gz files. - -6. Run `./install.sh` to test out locally. - -7. Upload `./work/output/mirror.tar` to `packages.getsol.us:/srv/www/screenshots/`, create a tar backup of the old screenshots and untar in place, make sure the folder permissions are 0755. - -8. Load up software centre and check out some packages with metadata to confirm all is well - -7. Git commit along with a new tag - -8. Build `appstream-data` with the new tag and publish to the people. - -## Regenerating from teaparty server directly - -If you have ssh access to the teaparty repo server (you have if you've done a sync), you can regenerate the appstream data directly from there without the need to clone the binary repo first. - -1. SSH into teaparty and go to `/srv/appstream-data` which is a shared directory of this repository. - -2. Run `./do_stream_teaparty.sh` to actually generate the appstream data. Try to choose a quiet time when the server isn't under much load. - -3. Run `./publish.sh` to copy the generated artefacts into the root directory of the git repository. - -4. Installing screenshots: - - Ensure the recently generated `mirror.tar` archive looks correct `tar -tvf ./work/output/mirror.tar`, it should contain screenshots - - Create a backup of the previous screenshots: `pushd /srv/www/screenshots/; sudo tar --create --file=mirror.tar.backup $(ls -d */) && popd` - - Move the screenshots archive to screenshots: `sudo mv work/output/mirror.tar /srv/www/screenshots/` - - Untar in place, ensure permissions are correct: `pushd /srv/www/screenshots; sudo tar -xvf mirror.tar --strip-components=1 && chmod 0755 $(ls -d */); popd` - -5. Sync the artefacts to this repo locally: - - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.gz .` - - `rsync -avPHL user@packages.getsol.us:/srv/appstream-data/*.tar .` - -6. Run `./install.sh` to try out and load up the software centre and check out some packages with metadata to confirm all is well - -7. Git commit along with a new tag - -8. Build `appstream-data` with the new tag and publish to the people. - -TODO: - - Make appstream-builder ignore -devel and -dbginfo packages +## Usage +### Initial Setup +These directions will guide you through configuring your system to generate appstream data. You should only need to do this once. +> [!NOTE] +> This assumes you have already completed the [Prepare for Packaging](https://help.getsol.us/docs/packaging/prepare-for-packaging) steps listed in the help center. You will need a fully-configured git setup for this tooling to work. +> Additionally, your GitHub account must have push access to this repository (should be true for all staff). +1. Ensure that your local clone of this repository is set up to be able to push (Cloning via GitHub CLI recommended). +2. Ensure you have `go-task` installed. All interaction with this tooling should be possible through the `Taskfile.yml` in this repository. +3. Run `go-task appstream-init`. This task will install `pyyaml` and `ansible` on your system and walk you through setting up the Ansible inventory you need for the playbook. +### Generating Appstream Data +This is the actual process of generating appstream metadata from our repository, which should ideally be done each week (after deprecations, but before sync). Make sure you've correctly completed all the Initial Setup steps first. +1. Run `go-task full-process` from this directory. That will automatically: + - Generate appstream data, + - Check it against the eopkg index, + - Download new metadata to your local clone of the repo, + - Commit the changes, + - Add a new tag to the repository, + - and push the changes to github. +> [!NOTE] +> This would be a good time to take a break and do something else. Just make sure your computer doesn't go to sleep. It takes a while (about 30 minutes). +2. Go to your clone of the packages monorepo and update the `appstream-data` package to use the newly-tagged version of this repository. Follow standard packaging procedure to get those changes into the repository. ## Debugging Failures diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..d37c292 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,36 @@ +version: 3 +vars: + NEXT_TAG: + sh: '{{.TASKFILE_DIR}}/get_next_tag.py' + +tasks: + generate: + label: "Generate Appstream metadata and download it" + cmds: + - "ansible-playbook -i inventory/hosts.yml -K playbook.yml" + + commit: + label: "Add, commit, and tag new appstream data" + cmds: + - "git add solus-1-failed.xml.gz solus-1-icons.tar.gz solus-1-ignore.xml.gz solus-1-screenshots.tar solus-1.xml.gz" + - "git commit -m 'Refresh Appstream metainfo'" + - "git tag {{.NEXT_TAG}}" + + push: + label: "Push changes to getsolus/solus-appstream-data" + cmds: + - "git push --tags" + + full-process: + label: "Perform the entire appstream generation process" + cmds: + - task: generate + - task: commit + - task: push + + appstream-init: + label: "Set up the local system to run this tooling" + cmds: + - "sudo eopkg install pyyaml ansible" + - "ansible-galaxy collection install community.general" + - "{{.TASKFILE_DIR}}/setup_inventory.py {{.TASKFILE_DIR}}/inventory/hosts.yml" diff --git a/archive_screenshots.sh b/archive_screenshots.sh new file mode 100755 index 0000000..d72706f --- /dev/null +++ b/archive_screenshots.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +if [[ ! -d "work/output" ]]; then + echo "No output directory found, have you ran appstream-builder?" +fi + +pushd work/output + +if [[ ! -d "mirror/source" ]]; then + mkdir -p mirror/source +fi + +for i in 112x63 224x126 1248x702 624x351 1504x846 752x423; do + cp -R $i mirror/. +done + +pushd mirror/source +tar xf ../../*screenshots*.tar + +popd + +tar cvf mirror.tar mirror/ \ No newline at end of file diff --git a/create_symlinks.py b/create_symlinks.py new file mode 100644 index 0000000..bf0e822 --- /dev/null +++ b/create_symlinks.py @@ -0,0 +1,78 @@ +# Create a directory of symlinks to optimize and correct the appstream build + +import pathlib +import argparse +import os +from xml.etree import ElementTree + + +def main(): + parser = argparse.ArgumentParser( + prog='create_symlinks', + description='Create symbolic links to eopkg files to optimize and correct our appstream build' + ) + parser.add_argument( + "eopkg_index", + action="store", + help="Path to the eopkg index file to be validated against. This must be an uncompressed XML file.", + type=pathlib.Path, + ) + parser.add_argument( + "packages_directory", + action="store", + help="Path to the packages (input) directory.", + type=pathlib.Path, + ) + parser.add_argument( + "symlinks_directory", + action="store", + help="Path to the symlinks (output) directory.", + type=pathlib.Path, + ) + args = parser.parse_args() + eopkg_packages = get_packages_from_eopkg_index(args.eopkg_index) + symlinks_created = 0 + for package_file in args.packages_directory.glob('**/*.eopkg'): + if check_file(package_file, eopkg_packages): + # Create a symlink for this package, we like it + os.symlink(package_file, pathlib.Path.joinpath(args.symlinks_directory, pathlib.Path(package_file.name))) + symlinks_created += 1 + print(f'Created {symlinks_created} symlinks.') + + +def get_packages_from_eopkg_index(xml_path: pathlib.Path) -> dict: + print('gothere') + solus_xml = open(xml_path, 'r') + tree = ElementTree.parse(solus_xml) + root = tree.getroot() + packages = { + package.find("Name").text: package.find("History").findall("Update")[0].attrib['release'] + for package in root.findall("Package") + } + # pprint(packages) + return packages + + +def parse_package_filename(package_filename: str) -> dict: + package_split = package_filename.rsplit('-', 4) + output = { + 'name': package_split[0], + 'release': package_split[2], + 'dbginfo': 'dbginfo' in package_filename + } + return output + + +def check_file(path: pathlib.Path, eopkg_packages: dict) -> bool: + eopkg_info = parse_package_filename(path.name) + print(eopkg_info) + if (eopkg_info['name'] in eopkg_packages.keys() and eopkg_info['release'] == eopkg_packages[eopkg_info['name']])\ + and not eopkg_info['dbginfo']\ + : + return True + else: + return False + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/get_next_tag.py b/get_next_tag.py new file mode 100755 index 0000000..20e7b1b --- /dev/null +++ b/get_next_tag.py @@ -0,0 +1,10 @@ +#!/bin/python3 +# A script to determine the next tag for getsolus/solus-appstream-data. +# This behavior is likely fairly specific to this repository. + +import subprocess + +git_tags = subprocess.check_output(['git', '--no-pager', 'tag', '--sort=-creatordate']).split(b'\n') +latest_number = int(git_tags[0].strip(b'v')) +next_tag = f'v{latest_number + 1}' +print(next_tag) diff --git a/inventory/hosts.yml.example b/inventory/hosts.yml.example new file mode 100644 index 0000000..0cb8237 --- /dev/null +++ b/inventory/hosts.yml.example @@ -0,0 +1,5 @@ +teaparty: + hosts: + packages.getsol.us: + ansible_user: + # Set this to your username diff --git a/playbook.yml b/playbook.yml new file mode 100644 index 0000000..6b103b6 --- /dev/null +++ b/playbook.yml @@ -0,0 +1,149 @@ +--- +- name: Ensure requirements are met for building appstream data + hosts: teaparty + become: yes + tags: + - setup + + tasks: + - name: Check for appstream-builder + ansible.builtin.raw: which appstream-builder + check_mode: false + changed_when: false + failed_when: which_res.rc > 1 + register: which_res + + - name: Ensure other dependencies are installed + ansible.builtin.dnf: + name: + - librsvg2 + - gtk3 + - libpng + state: present + + - name: Remove old work directory to avoid permissions issues + ansible.builtin.file: + path: "/srv/appstream-data/work/" + state: absent + +- name: Actually build appstream data + hosts: teaparty + become: no + tags: + - generate + + tasks: + - name: Ensure work/output directories exist + ansible.builtin.file: + path: "/srv/appstream-data/{{ item }}" + state: directory + loop: + - work/ + - work/output + - work/cache + - work/logs + - work/icons + - work/tmp + - work/package-symlinks + + - name: Select package files to parse using appstream-builder + ansible.builtin.command: + argv: + - python3 + - /srv/appstream-data/create_symlinks.py + - /srv/ferryd/root/repo/unstable/eopkg-index.xml + - /srv/ferryd/root/repo/unstable/ + - /srv/appstream-data/work/package-symlinks + + - name: Run appstream-builder against the entire repository + ansible.builtin.command: + argv: + - appstream-builder + - --packages-dir=./package-symlinks + - --output-dir=./output + - --cache-dir=./cache + - --log-dir=./logs + - --include-failed + - --basename=solus-1 + - --origin=solus + - --veto-ignore=missing-parents + - --veto-ignore=add-default-icons + chdir: /srv/appstream-data/work/ + + - name: Gather screenshots based on appstream data + ansible.builtin.command: + argv: + - appstream-util + - mirror-screenshots + - output/solus-1.xml.gz + - https://screenshots.getsol.us + - ./cache + - ./output + chdir: /srv/appstream-data/work/ + +- name: Check generated appstream data + hosts: teaparty + become: no + tags: + - test_metadata + + tasks: + - name: Check appstream metadata against eopkg index + ansible.builtin.command: + argv: + - /srv/appstream-data/check_packages_exist.py + - /srv/appstream-data/work/output/solus-1.xml.gz + - /srv/ferryd/root/repo/unstable/eopkg-index.xml + + # TODO: need to test contents of screenshots archive + +- name: Install screenshots + hosts: teaparty + become: yes + tags: + - install_screenshots + + tasks: + - name: Create a backup of existing screenshots + community.general.archive: + dest: /srv/www/screenshots/mirror.tar.backup + format: "tar" + path: /srv/www/screenshots/* + exclude_path: + - /srv/www/screenshots/mirror.tar + - /srv/www/screenshots/mirror.tar.backup + + - name: Archive new screenshots + ansible.builtin.command: + argv: + - ./archive_screenshots.sh + chdir: /srv/appstream-data + + - name: Extract new screenshots + ansible.builtin.unarchive: + remote_src: true + src: /srv/appstream-data/work/output/mirror.tar + dest: /srv/www/screenshots/ + mode: "0755" + extra_opts: + - --strip-components=1 + +- name: Fetch generated appstream data + hosts: teaparty + become: no + tags: + - fetch_appstream_data + + tasks: + - name: Fetch appstream data + ansible.builtin.fetch: + src: "/srv/appstream-data/work/output/{{ item }}" + dest: "{{ playbook_dir }}/" + flat: true + loop: + - "solus-1-failed.xml.gz" + - "solus-1-icons.tar.gz" + - "solus-1-ignore.xml.gz" + - "solus-1-screenshots.tar" + - "solus-1.xml.gz" + diff --git a/setup_inventory.py b/setup_inventory.py new file mode 100755 index 0000000..1f626cd --- /dev/null +++ b/setup_inventory.py @@ -0,0 +1,27 @@ +#!/bin/python3 + +import yaml +import argparse +import pathlib + +def main(): + parser = argparse.ArgumentParser( + prog='setup_inventory', + description='Set up the Ansible inventory for Solus appstream data generation' + ) + parser.add_argument( + "inventory_file", + action="store", + help="Path to the desired location of the inventory file", + type=pathlib.Path, + ) + args = parser.parse_args() + username = input("Your username on teaparty (must have sudo access): ") + print(f'Writing Ansible inventory to {args.inventory_file}') + inventory = {'teaparty': {'hosts': {'packages.getsol.us': {'ansible_user': username}}}} + + with open(args.inventory_file, 'w') as f: + yaml.safe_dump(inventory, f) + +if __name__ == '__main__': + main()