Merge pull request #74 from Haidra-Org/main

.git-blame-ignore-revs

@@ -4,4 +4,10 @@ d834bf44ec4b63e80a7e100e0493ba0f24ec0d14
 f74a0f3deb14b21a8155d4133030a4871ffef84b
 
 # Medium style/lint/minor refactor only changes 4/11/23
-1cb70d2eb3c219bff304c4ed0ac34f7456946281
+1cb70d2eb3c219bff304c4ed0ac34f7456946281
+
+# Precommit 8/30/23
+46af8ec00f93edde1a4953936ca25fb9c9fc369e
+
+# Catchup and included hordelib/nodes/
+23cbab10b5895072678a070beb7a6dc0b8640722

.github/workflows/maintests.yml

@@ -10,7 +10,7 @@ on:
       - 'tox.ini'
       - '.github/workflows/maintests.yml'
       - '.github/workflows/prtests.yml'
-      - '.github/workflows/release.yml' 
+      - '.github/workflows/release.yml'
 jobs:
   build:
 
@@ -29,13 +29,11 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
       - name: Install tox and any other packages
-        run: | 
+        run: |
           python -m pip install --upgrade pip
           pip install --upgrade -r requirements.dev.txt
-      - name: Run format check
-        run: tox -e format
-      - name: Run lint check
-        run: tox -e lint-weak-check
+      - name: Run lint/format/mypy check
+        run: tox -e pre-commit
       - name: Check build_helper.py hordelib imports have no breaking dependency changes
         run: tox -e test-build-helper
       - name: Build unit test environment, confirm CUDA is available on host

.github/workflows/prtests.yml

@@ -13,7 +13,7 @@ on:
       - 'tox.ini'
       - '.github/workflows/maintests.yml'
       - '.github/workflows/prtests.yml'
-      - '.github/workflows/release.yml' 
+      - '.github/workflows/release.yml'
 jobs:
   build:
 
@@ -34,13 +34,11 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
       - name: Install tox and any other packages
-        run: | 
+        run: |
           python -m pip install --upgrade pip
           pip install --upgrade -r requirements.dev.txt
-      - name: Run format check
-        run: tox -e format
-      - name: Run lint check
-        run: tox -e lint-weak-check
+      - name: Run lint/format/mypy check
+        run: tox -e pre-commit
       - name: Check build_helper.py hordelib imports have no breaking dependency changes
         run: tox -e test-build-helper
       - name: Build unit test environment, confirm CUDA is available on host

.github/workflows/release.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - releases
-      
+
 permissions:
   contents: write
   pull-requests: read
@@ -53,7 +53,7 @@ jobs:
       if: ${{ steps.release.outputs.version != '' }}
       run: |
         python -m pip install --upgrade pip
-        python -m pip install build psutil typing-extensions loguru strenum --user
+        python -m pip install build psutil typing-extensions loguru strenum python-dotenv --user
 
     - name: "✏️ Install changelog dependencies"
       if: ${{ steps.release.outputs.version != '' }}
@@ -110,4 +110,3 @@ jobs:
     #   with:
     #     webhook-url: ${{ secrets.DISCORD_WEBHOOK_URL }}
     #     content: "New version of hordelib has been published to pypi: ${{ steps.release.outputs.version }}. You can update your worker by running `./update-runtime.sh --hordelib` (Linux) or `update-runtime.cmd --hordelib` (Windows). Changelog: https://t.ly/z2vQ"
-

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+    -   id: black
+        exclude: ^hordelib/nodes/.*\..*$
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.282
+    hooks:
+    -    id: ruff
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.4.1'
+    hooks:
+    -   id: mypy
+        exclude: ^examples/.*$ # FIXME
+        additional_dependencies: [pydantic, strenum, types-colorama, types-docutils, types-Pillow, types-psutil, types-Pygments, types-pywin32, types-PyYAML, types-regex, types-requests, types-setuptools, types-tabulate, types-tqdm, types-urllib3]

MANIFEST.in

@@ -1,3 +1,8 @@
+recursive-include hordelib *.typed
+
+exclude images/*
+exclude images_expected/*
+
 # Although it looks like these files don't exist, they are created
 # by the build_helper just prior to pypi packaging.
 graft hordelib/_comfyui

README.md

@@ -78,7 +78,7 @@ data = {
     "n_iter": 1,
     "model": "Deliberate",
 }
-pil_image = generate.basic_inference(data)
+pil_image = generate.basic_inference_single_image(data)
 pil_image.save("test.png")
 ```
 
@@ -113,7 +113,7 @@ Custom nodes for ComfyUI providing face restoration.
 
 ## DMCA Abuse
 
-On 26th May 2023 an [individual](https://github.com/hlky) issued a [DMCA takedown notice](https://github.com/github/dmca/blob/master/2023/05/2023-05-26-nataili.md) to Github against `hordelib` which claimed their name had been removed from the copyright header in the AGPL license in the 7 files listed in the takedown notice. This claim was true, and this attribution had been removed by a `hordelib` contributor prior to being committed into the `hordelib` repository. 
+On 26th May 2023 an [individual](https://github.com/hlky) issued a [DMCA takedown notice](https://github.com/github/dmca/blob/master/2023/05/2023-05-26-nataili.md) to Github against `hordelib` which claimed their name had been removed from the copyright header in the AGPL license in the 7 files listed in the takedown notice. This claim was true, and this attribution had been removed by a `hordelib` contributor prior to being committed into the `hordelib` repository.
 
 Unfortunately, it appears the individual making the DMCA claim was acting in bad faith, and even when their name was restored to the copyright attribution in the files, they persisted to press the DMCA takedown claim, which, due to the nature of the Github process, resulted in hordelib being subject to a DMCA takedown on Github.
 

examples/download_all_sd_models.py

@@ -0,0 +1,13 @@
+import hordelib.initialisation
+from hordelib.shared_model_manager import SharedModelManager
+
+
+def main():
+    hordelib.initialisation.initialise(setup_logging=True, logging_verbosity=5)
+
+    SharedModelManager.load_model_managers(["compvis"])
+    SharedModelManager.manager.compvis.download_all_models()
+
+
+if __name__ == "__main__":
+    main()

examples/kudos.py

@@ -2,7 +2,6 @@
 import sys
 
 import torch
-import torch.nn as nn
 
 
 class KudosModel:
@@ -132,9 +131,7 @@ def calculate_kudos(self, payload, basis_adjustment=0, basis_scale=1):
         kudos = kudos * basis_scale
 
         # Scale our kudos by the time the job will take to complete
-        kudos = job_ratio * kudos
-
-        return kudos
+        return job_ratio * kudos
 
     @classmethod
     def one_hot_encode(cls, strings, unique_strings):
@@ -211,7 +208,6 @@ def calculate_basis_time(self):
 
 
 if __name__ == "__main__":
-
     if len(sys.argv) != 2:
         print("Syntax: kudos.py <model_filename>")
 

examples/lora_downloader.py

@@ -88,6 +88,7 @@ def _get_json(self, url):
                 # Failed badly
                 logger.error(f"LORA download failed {e}")
                 return None
+        return None
 
     def _get_more_items(self):
         if not self._data:
@@ -116,7 +117,7 @@ def _sanitise_filename(self, filename):
         # First remove exotic unicode characters
         filename = filename.encode("ascii", "ignore").decode("ascii")
         # Now exploit characters
-        valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+        valid_chars = f"-_.() {string.ascii_letters}{string.digits}"
         return "".join(c for c in filename if c in valid_chars)
 
     def _parse_civitai_lora_data(self, item):
@@ -149,7 +150,7 @@ def _parse_civitai_lora_data(self, item):
                 break
         # If we don't have everything required, fail
         if not lora.get("sha256") or not lora.get("filename") or not lora.get("url") or not lora.get("triggers"):
-            return
+            return None
         # Fixup A1111 centric triggers
         for i, trigger in enumerate(lora["triggers"]):
             if re.match("<lora:(.*):.*>", trigger):
@@ -181,7 +182,7 @@ def _download_thread(self):
                         with open(hashfile) as infile:
                             try:
                                 hashdata = infile.read().split()[0]
-                            except (IndexError, OSError, IOError, PermissionError):
+                            except (IndexError, OSError, PermissionError):
                                 hashdata = ""
                         if hashdata.lower() == lora["sha256"].lower():
                             # we already have this lora, consider it downloaded
@@ -205,7 +206,7 @@ def _download_thread(self):
                         with open(filename, "wb") as outfile:
                             outfile.write(response.content)
                         # Save the hash file
-                        with open(hashfile, "wt") as outfile:
+                        with open(hashfile, "w") as outfile:
                             outfile.write(f"{sha256} *{lora['filename']}")
 
                         # Shout about it
@@ -217,9 +218,9 @@ def _download_thread(self):
                             if self._downloaded_mb > self._max_disk:
                                 self.done = True
                         break
-                    else:
-                        logger.debug(f"Downloaded LORA file for {lora['filename']} didn't match hash")
-                        pass  # we will retry
+
+                    logger.debug(f"Downloaded LORA file for {lora['filename']} didn't match hash")
+                    pass  # we will retry
 
                 except (requests.HTTPError, requests.ConnectionError, requests.Timeout, json.JSONDecodeError) as e:
                     logger.debug(f"Error downloading {lora['filename']} {e}")
@@ -262,11 +263,9 @@ def _process_items(self):
                 self._download_lora(lora)
 
     def _start_processing(self):
-
         self.done = False
 
         while not self.done:
-
             # Get some items to download
             self._get_more_items()
 
@@ -292,11 +291,10 @@ def download(self, wait=False):
 
         # Save the final model data index
         filename = os.path.join(self._download_dir, "loras.json")
-        with open(filename, "wt", encoding="utf-8", errors="ignore") as outfile:
+        with open(filename, "w", encoding="utf-8", errors="ignore") as outfile:
             outfile.write(json.dumps(self.model_data, indent=4))
 
 
 if __name__ == "__main__":
-
     downloader = LoraDownloader(allowed_storage=1024)  # MB
     downloader.download(wait=True)

examples/run_all_models.py

@@ -28,7 +28,7 @@ def add_model(model_name):
 def do_inference(model_name, iterations=1):
     """Do some work on the GPU"""
     horde = HordeLib()
-    for i in range(iterations):
+    for _ in range(iterations):
         data = {
             "sampler_name": "k_euler",
             "cfg_scale": 6.5,
@@ -52,7 +52,7 @@ def do_inference(model_name, iterations=1):
             "n_iter": 1,
             "model": model_name,
         }
-        pil_image = horde.basic_inference(data)
+        pil_image = horde.basic_inference_single_image(data)
         if not pil_image:
             logger.error("Inference is failing to generate images")
         else:

examples/run_all_stress_tests.py

@@ -1,4 +1,3 @@
-import os
 import subprocess
 import sys
 

examples/run_controlnet.py

@@ -1,14 +1,12 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_controlnet
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 from hordelib.consts import MODEL_CATEGORY_NAMES
 
 
 def main():
-
     hordelib.initialise()
 
     from PIL import Image
@@ -46,7 +44,7 @@ def main():
             # Not valid for normal image input test
             continue
         data["control_type"] = preproc
-        pil_image = generate.basic_inference(data)
+        pil_image = generate.basic_inference_single_image(data)
         pil_image.save(f"images/run_controlnet_{preproc}.webp", quality=90)
 
 

examples/run_controlnet_annotator.py

@@ -1,14 +1,12 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_controlnet
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 from hordelib.consts import MODEL_CATEGORY_NAMES
 
 
 def main():
-
     hordelib.initialise()
 
     from PIL import Image
@@ -45,7 +43,7 @@ def main():
             # Not valid for normal image input test
             continue
         data["control_type"] = preproc
-        pil_image = generate.basic_inference(data)
+        pil_image = generate.basic_inference_single_image(data)
         pil_image.save(f"images/run_annotator_{preproc}.webp", quality=90)
 
 

examples/run_facefix.py

@@ -1,7 +1,6 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_facefix
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 

examples/run_img2img.py

@@ -1,7 +1,6 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_img2img
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 from hordelib.consts import MODEL_CATEGORY_NAMES
@@ -39,7 +38,7 @@ def main():
         "model": "Deliberate",
         "source_image": Image.open("images/test_db0.jpg"),
     }
-    pil_image = generate.basic_inference(data)
+    pil_image = generate.basic_inference_single_image(data)
     pil_image.save("images/run_img2img.webp", quality=90)
 
 

examples/run_img2img_hires.py

@@ -1,14 +1,12 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_img2img_hires
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 from hordelib.consts import MODEL_CATEGORY_NAMES
 
 
 def main():
-
     hordelib.initialise()
 
     from PIL import Image
@@ -40,7 +38,7 @@ def main():
         "model": "Deliberate",
         "source_image": Image.open("images/test_db0.jpg"),
     }
-    pil_image = generate.basic_inference(data)
+    pil_image = generate.basic_inference_single_image(data)
     pil_image.save("images/run_img2img_hires.webp", quality=90)
 
 

examples/run_img2img_inpaint.py

@@ -1,7 +1,6 @@
 # This tests running hordelib standalone, as an external caller would use it.
 # Call with: python -m test.run_img2img_inpaint
 # You need all the deps in whatever environment you are running this.
-import os
 
 import hordelib
 from hordelib.consts import MODEL_CATEGORY_NAMES
@@ -40,7 +39,7 @@ def main():
         "source_image": Image.open("images/test_inpaint.png"),
         "source_processing": "inpainting",
     }
-    pil_image = generate.basic_inference(data)
+    pil_image = generate.basic_inference_single_image(data)
     pil_image.save("images/run_img2img_inpaint.webp", quality=90)