Merge branch 'feat/game-downloader' into cooker-0.8.1b

functions/global.sh

@@ -8,6 +8,7 @@ source /app/libexec/compression.sh
 source /app/libexec/dialogs.sh
 source /app/libexec/logger.sh
 source /app/libexec/functions.sh
+source /app/libexec/romhack_downloader_wrapper.sh
 source /app/libexec/multi_user.sh
 source /app/libexec/framework.sh
 source /app/libexec/post_update.sh

functions/romhack_downloader/db_setup.sql

@@ -0,0 +1,55 @@
+.nullvalue NULL
+
+DROP TABLE IF EXISTS base;
+DROP TABLE IF EXISTS rhack;
+
+CREATE TABLE base (
+        system TEXT NOT NULL, -- e.g. 'nes' or 'n64'
+        name TEXT NOT NULL, -- full name, e.g. "Super Mario Bros."
+        region TEXT NOT NULL, -- 'U' (USA), 'E' (Europe), 'J' (Japan) or 'W' (World)
+        version TEXT NOT NULL, -- normally '1.0'; revision 1 is '1.1' etc.
+        hash TEXT NOT NULL PRIMARY KEY, -- crc32
+        local_path TEXT
+);
+
+CREATE TABLE rhack (
+        id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+        base_hash TEXT NOT NULL,
+        name TEXT NOT NULL,
+        author TEXT NOT NULL,
+        type TEXT, -- e.g. 'English translation'; put NULL when original hack
+        released TEXT, -- 'YYYY-MM-DD'; if e.g. only year and month is available do 'YYYY-MM'
+        version TEXT NOT NULL, -- e.g. '1.0'
+        retro_achievements TEXT NOT NULL, -- 'yes' or 'no'
+        url TEXT, -- direct download link; always prefer patches provided by RetroAchievements, if available
+        archive_path TEXT, -- path of the patch file inside the archive, e.g. 'patches/v1.bps'
+        description TEXT, -- place the whole text on a single line, no line break
+        FOREIGN KEY (base_hash) REFERENCES base (hash)
+);
+
+-----------------------------------------------------------
+--- ROM Hacks
+-----------------------------------------------------------
+
+-- Base ROMs. Order these alphabetically; left-most element is most important for ordering, then next to left-most etc.
+INSERT INTO base (system, name, region, version, hash) VALUES
+('gba', 'Tomato Adventure', 'J', '1.0', 'e37ca939'),
+('n64', 'Super Mario 64', 'U', '1.0', '3ce60709'),
+('n64', 'The Legend of Zelda - Ocarina of Time', 'U', '1.2', 'cd16c529'),
+('nes', 'Super Mario Bros.', 'W', '1.0', '393a432f'),
+('snes', 'Super Mario World', 'U', '1.0', 'b19ed489')
+;
+
+-- ROM Hacks. Follow ordering of base table above, so group by system first, then name of base game etc.
+INSERT INTO rhack (base_hash, name, type, version, author, released, retro_achievements, url, archive_path, description) VALUES
+-- Super Mario 64
+('3ce60709', 'Super Mario 64 (U)', 'Randomizer', '1.1.2', 'Arthurtilly', NULL, 'yes', 'https://github.com/RetroAchievements/RAPatches/raw/main/N64/Hacks/Super%20Mario%2064/10509-SM64-Randomizer.zip', 'SM64 - Randomizer (v1.1.2) (Arthurtilly).bps', NULL),
+('3ce60709', 'Super Mario Bros. 64', NULL, '1.1', 'Kaze Emanuar', '2018-12-21', 'yes', 'https://github.com/RetroAchievements/RAPatches/raw/main/N64/Hacks/Super%20Mario%2064/13831-SM64-SMB64.7z', 'SM64 - Super Mario Bros. 64 (Kaze Emanuar).bps', 'Super Mario Bros. 64 allows you to play through 30 classic NES Super Mario Bros recreated in the Mario 64 game engine. You get infinite lives to play through the game, but are given a ‘Par’ for each level, referring to the amount of lives an average player should lose per level, and you earn points for losing as few lives as possible. There are four playable characters (Mario, Luigi, Wario and Luigi), each of which has their own unique jump height which can make the game harder or easier (we’d recommend Wario or Luigi for your first playthrough).'),
+-- Super Mario Bros.
+('393a432f', 'Super Mario Unlimited Deluxe', NULL, '2.4', 'frantik', '2021-03-26', 'yes', 'https://github.com/RetroAchievements/RAPatches/raw/main/NES/Hacks/Super%20Mario%20Bros/9904-SMB1-UnlimitedDeluxe.zip', 'SMB1 - Super Mario Unlimited - Deluxe (v2.4) (frantik).ips', 'Super Mario Unlimited Deluxe is a traditional-style Mario hack with difficulty ramping up from beginner to expert. It is based on the Super Mario Bros engine, but has been completely reworked into a whole new adventure.'),
+-- Super Mario World
+('b19ed489', 'New Super Mario World 2: Around The World', NULL, '1.3', 'Pink Gold Peach', '2019-12-10', 'yes', 'https://github.com/RetroAchievements/RAPatches/raw/main/SNES/Hacks/Super%20Mario%20World/16121-NSMW2AroundtheWorld.zip', 'SMW - New Super Mario World 2 - Around the World (v1.3) (Pink Gold Peach).bps', 'The sequel to NSMW1 The 12 Magic Orbs, this hack features 16 different worlds and 90+ unique levels filled with challenge and secrets. The hack uses a lot of ASM like custom sprites, blocks, uberASM effects and other stuff like that. Aesthetically it has a choconilla style with most of the graphics being from the original SMW with some new custom graphics.'),
+('b19ed489', 'Yoshi''s Strange Quest', NULL, '1.3', 'Yoshifanatic', '2015-03-07', 'no', 'https://github.com/RetroAchievements/RAPatches/raw/main/SNES/Hacks/Super%20Mario%20World/8366-YoshisStrangeQuest.zip', 'SMW - Yoshi''s Strange Quest (v1.3) (Yoshifanatic).bps', 'This is the sequel to Mario''s Strange Quest. Picking up where Mario''s Strange Quest left off, it turns out that the part where Yoshi''s eggs hatched at the end of MSQ didn''t actually happen. What really happened after Mario beat Bowser, rescued Yoshi''s eggs, and saved the princess was that Yoshi and his sleepy friend decided to move to a new land so that he can protect his eggs from Bowser before they really hatched. So, both Yoshis do so and they find themselves in the land of Weirdonia. However, it seems that Bowser apparently insists on stealing Yoshi''s eggs, since Yoshi''s eggs were stolen again while Yoshi was out shopping. Since Mario isn''t around to help this time, Yoshi goes on a quest by himself to retrieve his eggs. However, just like Mario''s Strange Quest, this isn''t your ordinary quest. The land of Weirdonia is a strange land filled with bizarre gimmicks, weird themes, and possibly jelly filled donuts and pizza. Expect the unexpected during Yoshi''s journey.'),
+-- Tomato Adventure
+('e37ca939', 'Tomato Adventure (J)', 'English translation', '1.1.1', 'Unknown W. Brackets', '2021-06-17', 'yes', 'https://github.com/RetroAchievements/RAPatches/raw/main/GBA/Translation/English/9802-TomatoAdv-EnglishTranslation.zip', 'Tomato Adventure (Japan) (En) (v1.1.1) (Unknown W. Brackets).bps', NULL)
+;

functions/romhack_downloader/download_patch.py

@@ -0,0 +1,34 @@
+# Copyright 2024 Libretto
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os, pathlib, requests
+from pyunpack import Archive
+from urllib.parse import urlparse
+
+
+# return absolute path of patch
+def get_patch_from_archive(archive_path, location_in_archive):
+    os.system("rm -rf /tmp/extract_patch")
+    extract_dir = "/tmp/extract_patch"
+    os.makedirs(extract_dir, exist_ok=True)
+
+    Archive(archive_path).extractall(extract_dir)
+    return os.path.join(extract_dir, location_in_archive)
+
+
+def directly_download(url):
+    file_extension = ''.join(pathlib.Path(url).suffixes)
+    archive_path = f"/tmp/patch_archive{file_extension}"
+
+    request = requests.get(url)
+    with open(archive_path, 'wb') as file:
+        file.write(request.content)
+    return archive_path
+
+
+# "main" function of this module
+def download_patch(url, location_in_archive):
+    match urlparse(url).netloc: # domain
+        case _: # direct download possible
+            archive_path = directly_download(url)
+            return get_patch_from_archive(archive_path, location_in_archive)

functions/romhack_downloader/main.py

@@ -0,0 +1,70 @@
+# Copyright 2024 Libretto
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import argparse, os, pathlib, sqlite3
+
+from scan_roms import scan_avail_base_roms
+from download_patch import download_patch
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fetch-compatible-hacks', action='store_const', const=True)
+    parser.add_argument('-i', '--install')
+    parser.add_argument('-r', '--roms-folder', required=True)
+    return parser.parse_args()
+
+
+def reset_db(db):
+    with open('db_setup.sql', 'r') as file:
+        db.executescript(file.read())
+
+
+def construct_rhack_path(roms_folder, system, rhack_name, base_path, rhack_version, rhack_type):
+    rhack_dir = os.path.join(roms_folder, system, 'ROM Hacks')
+    os.makedirs(rhack_dir, exist_ok=True)
+
+    rhack_extension = pathlib.Path(base_path).suffix
+    if rhack_type:
+        rhack_filename = f"{rhack_name}[{rhack_type} v{rhack_version}]{rhack_extension}"
+    else:
+        rhack_filename = f"{rhack_name}[v{rhack_version}]{rhack_extension}"
+
+    rhack_path = os.path.join(rhack_dir, rhack_filename)
+    print(f"Path of the ROM Hack: {rhack_path}")
+    return rhack_path
+
+
+def install_rhack(db, id, roms_folder):
+    db.execute((
+        "SELECT rhack.url, base.local_path, base.system, rhack.name, rhack.archive_path, rhack.version, rhack.type "
+        "FROM base JOIN rhack ON base.hash = rhack.base_hash "
+        f"WHERE rhack.id = {id}"
+    ))
+    url, base_path, system, rhack_name, archive_path, rhack_version, rhack_type = db.fetchone()
+
+    rhack_path = construct_rhack_path(roms_folder, system, rhack_name, base_path, rhack_version, rhack_type)
+
+    patch_path = download_patch(url, archive_path)
+    if patch_path: os.system(f'flatpak run com.github.Alcaro.Flips --apply "{patch_path}" "{base_path}" "{rhack_path}"')
+
+    # cleanup
+    os.system("rm -rf /tmp/patch_archive* /tmp/extract_patch")
+
+
+def main():
+    db_connection = sqlite3.connect('romhacks.db')
+    db = db_connection.cursor()
+
+    args = parse_arguments()
+
+    if args.fetch_compatible_hacks:
+        reset_db(db)
+        scan_avail_base_roms(db, args.roms_folder)
+
+    if args.install: 
+        install_rhack(db, args.install, args.roms_folder)
+
+    db_connection.close()
+
+
+main()

functions/romhack_downloader/scan_roms.py

@@ -0,0 +1,42 @@
+# Copyright 2024 Libretto
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os, zlib
+
+
+def crc32(file):
+    prev = 0
+    for each_line in open(file, 'rb'):
+        prev = zlib.crc32(each_line, prev)
+    return ("%X"%(prev & 0xFFFFFFFF)).lower()
+
+
+def add_base_path_to_db(db, path, hash):
+    sanitized_path = path.replace("'", "''")
+    db.execute(f"UPDATE base SET local_path = '{sanitized_path}' WHERE hash = '{hash}'")
+
+
+def scan_and_add(db, root_search_path, avail_systems):
+    for search_dir, avail_dirs, avail_files in os.walk(root_search_path):
+
+        # only look at consoles that appear in our patch db 
+        if os.path.basename(search_dir) in avail_systems:
+
+            for file in avail_files:
+                if file.endswith('.txt'): continue
+
+                rom_path = os.path.join(search_dir, file)
+                rom_hash = crc32(rom_path)
+
+                add_base_path_to_db(db, rom_path, rom_hash)
+
+
+# return a list of consoles for which patches are available
+def get_avail_systems(db):
+    db.execute("SELECT DISTINCT system FROM base")
+    return [tuple[0] for tuple in db.fetchall()]
+
+
+# "main" function of this module
+def scan_avail_base_roms(db, search_path):
+    scan_and_add(db, search_path, get_avail_systems(db))

functions/romhack_downloader_wrapper.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+romhack_downloader_wrapper() {
+  # turn ~ into path
+  eval expanded_roms_folder="$roms_folder"
+
+  cd /app/libexec/romhack_downloader
+  python3 main.py --roms-folder "$expanded_roms_folder" "$@"
+}

net.retrodeck.retrodeck.yml

@@ -63,6 +63,9 @@ modules:
   #- rd-submodules/shared-modules/libusb/libusb.json # 1.0.26
   # we added the libusb 1.0.27 as Dolphin is breaking with 1.0.27, when bot will be aligned we can go back to the submodule
 
+  # pip modules for romhack downloader
+  - rd-submodules/python3-pip/python3-modules.json
+
   # This module is used to define the RetroDECK version
   # If the version is set as cooker it will automatically generate the version tag based on the date
   # else it will just put what is written, "v" is not needed

rd-submodules/python3-pip/python3-modules.json

@@ -0,0 +1,79 @@
+{
+    "name": "python3-modules",
+    "buildsystem": "simple",
+    "build-commands": [],
+    "modules": [
+        {
+            "name": "python3-requests",
+            "buildsystem": "simple",
+            "build-commands": [
+                "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"requests\" --no-build-isolation"
+            ],
+            "sources": [
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl",
+                    "sha256": "dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz",
+                    "sha256": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl",
+                    "sha256": "82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl",
+                    "sha256": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.whl",
+                    "sha256": "450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"
+                }
+            ]
+        },
+        {
+            "name": "python3-pyunpack",
+            "buildsystem": "simple",
+            "build-commands": [
+                "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"pyunpack\" --no-build-isolation"
+            ],
+            "sources": [
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/9c/cf/27d1f4b3bae5e566f94fc716e048120128cf603d5163638d22bcd0fc92d8/EasyProcess-1.1-py3-none-any.whl",
+                    "sha256": "82eed523a0a5eb12a81fa4eacd9f342caeb3f900eb4b798740e6696ad07e63f9"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/42/ee/84c8990b08efa0265bd10fc8781ef26e3157715bf0dfa47ee3c056b513d4/entrypoint2-1.1-py2.py3-none-any.whl",
+                    "sha256": "eeb8c327bdb65cdd1668c023a6b110b7e3d1a046fb05e043861ebd9264b3a257"
+                },
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/72/20/5a6dcb0d28529ce6efe850755994c80817279eecf08620003775fda3b914/pyunpack-0.3-py2.py3-none-any.whl",
+                    "sha256": "8f517cfc71215f37f74cf3a7668028828c68dc76f4d02e7a69f227ce978d51a3"
+                }
+            ]
+        },
+        {
+            "name": "python3-patool",
+            "buildsystem": "simple",
+            "build-commands": [
+                "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"patool\" --no-build-isolation"
+            ],
+            "sources": [
+                {
+                    "type": "file",
+                    "url": "https://files.pythonhosted.org/packages/e6/64/e9dd887985305d4cc88b09bcaaaafe5053197e5ebbeba62473f8c3cf6d80/patool-2.2.0-py2.py3-none-any.whl",
+                    "sha256": "21db6cc2fcd77acd37768258d1ad5aa3df0f676331fd80dfb1eb628626bc9155"
+                }
+            ]
+        }
+    ]
+}