From 33281fb23057ff84a2fc949067a1d9e8e888bae5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:47:38 +0200 Subject: [PATCH 01/11] added package.py --- package.py | 16 ++++++++++++++++ server/__init__.py | 9 --------- 2 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 package.py diff --git a/package.py b/package.py new file mode 100644 index 0000000..fbcc40d --- /dev/null +++ b/package.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""Package declaring addon version.""" +name = "syncsketch" +title = "SyncSketch" +version = "0.2.0" +client_dir = "ayon_syncsketch" + +# TODO: need to make sure image is published to docker hub +services = { + "processor": {"image": f"ynput/ayon-syncsketch-processor:{version}"} +} + +ayon_required_addons = { + "core": ">=0.4.0", +} +ayon_compatible_addons = {} diff --git a/server/__init__.py b/server/__init__.py index 339c372..ebb5dfe 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -15,21 +15,12 @@ from .common import constants from .settings import SyncsketchSettings, DEFAULT_VALUES -from .version import __version__ class SyncsketchAddon(BaseServerAddon): - name = "syncsketch" - title = "SyncSketch" - version = __version__ settings_model: Type[SyncsketchSettings] = SyncsketchSettings - # TODO: need to make sure image is published to docker hub - services = { - "processor": {"image": f"ynput/ayon-syncsketch-processor:{version}"} - } async def resolved_secrets(self): - addon_settings = await self.get_studio_settings() syncsk_server_config = addon_settings.syncsketch_server_config syncsk_server_config = dict(syncsk_server_config) From 57e9a584df79963efc7e8e515005c18ef1162574 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:48:07 +0200 Subject: [PATCH 02/11] remove version.py from root --- version.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 version.py diff --git a/version.py b/version.py deleted file mode 100644 index a242f0e..0000000 --- a/version.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- -"""Package declaring addon version.""" -__version__ = "0.1.1" From 12b8a53a744e6670edb4d815b4c980ab4d6759d7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:48:44 +0200 Subject: [PATCH 03/11] removed pyproject toml --- client/pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 client/pyproject.toml diff --git a/client/pyproject.toml b/client/pyproject.toml deleted file mode 100644 index 3772116..0000000 --- a/client/pyproject.toml +++ /dev/null @@ -1,4 +0,0 @@ -[tool.poetry.dependencies] -python = "^3.9" -aioresponses = "0.7.4" -pytest-asyncio = "0.21.0" \ No newline at end of file From 515124d691f6279356e0f38b1f7dcf2aa786cff4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:48:55 +0200 Subject: [PATCH 04/11] remove py2 compatibility --- syncsketch_common/server_handler.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/syncsketch_common/server_handler.py b/syncsketch_common/server_handler.py index bec30e6..5a12b17 100644 --- a/syncsketch_common/server_handler.py +++ b/syncsketch_common/server_handler.py @@ -3,16 +3,10 @@ import json import os import time -import requests import logging +from urllib.parse import urlencode -try: - # Python 3 - from urllib.parse import urlparse, urlencode -except ImportError: - # Python 2 - from urlparse import urlparse - from urllib import urlencode +import requests class ServerCommunication: From 9c0bb84f54fe05bbe5c03cd24a09edbf1c405f86 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:59:33 +0200 Subject: [PATCH 05/11] use correct method name --- server/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/__init__.py b/server/__init__.py index ebb5dfe..4fcd12c 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -41,7 +41,7 @@ async def get_default_settings(self): return settings_model_cls(**DEFAULT_VALUES) async def setup(self): - need_restart = await self.create_applications_attribute() + need_restart = await self.create_syncsketch_id_attribute() if need_restart: self.request_server_restart() @@ -178,7 +178,7 @@ async def _syncsketch_event(self, request: dict[str, Any]): f"Received a SyncSketch event that we don't handle. {request}" ) - async def create_applications_attribute(self) -> bool: + async def create_syncsketch_id_attribute(self) -> bool: """Make sure there are required attributes which ftrack addon needs. Returns: From 5b2294d4fa8c435621832e7380cd48e782519eb9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:00:02 +0200 Subject: [PATCH 06/11] added ruff linting with action --- .github/workflows/pr_linting.yml | 24 ++++++++++ ruff.toml | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 .github/workflows/pr_linting.yml create mode 100644 ruff.toml diff --git a/.github/workflows/pr_linting.yml b/.github/workflows/pr_linting.yml new file mode 100644 index 0000000..3d2431b --- /dev/null +++ b/.github/workflows/pr_linting.yml @@ -0,0 +1,24 @@ +name: 📇 Code Linting + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number}} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..f5e7dd8 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,77 @@ +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 79 +indent-width = 4 + +[lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E", "F", "W"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[lint.extend-per-file-ignores] +"tests/*" = ["F401", "F811", "F821"] + +[format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" From cba39a84e107403acf10b4a81a592ce5a4641074 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:00:21 +0200 Subject: [PATCH 07/11] fix ruff validations --- client/ayon_syncsketch/__init__.py | 4 +++- client/ayon_syncsketch/addon.py | 7 +++---- .../plugins/publish/integrate_reviewables.py | 13 ++++++++----- .../plugins/publish/validate_server_connection.py | 3 ++- create_package.py | 2 +- server/__init__.py | 1 - service/processor/__init__.py | 1 - .../event_handlers/review_session_end.py | 7 ++++--- service/processor/lib/__init__.py | 2 +- service/processor/lib/event_abstraction.py | 4 +++- service/processor/processor.py | 10 ++++++---- syncsketch_common/config.py | 2 +- syncsketch_common/constants.py | 2 +- syncsketch_common/server_handler.py | 15 +++++++++++---- .../ayon_syncsketch/api/test_server_handler.py | 10 +++++++--- tests/lib.py | 14 ++++++++------ 16 files changed, 59 insertions(+), 38 deletions(-) diff --git a/client/ayon_syncsketch/__init__.py b/client/ayon_syncsketch/__init__.py index c9e43f2..0a5fa62 100644 --- a/client/ayon_syncsketch/__init__.py +++ b/client/ayon_syncsketch/__init__.py @@ -2,4 +2,6 @@ SyncsketchAddon, ) -__all__ = ("SyncsketchAddon",) +__all__ = ( + "SyncsketchAddon", +) diff --git a/client/ayon_syncsketch/addon.py b/client/ayon_syncsketch/addon.py index 96a356c..3abc041 100644 --- a/client/ayon_syncsketch/addon.py +++ b/client/ayon_syncsketch/addon.py @@ -2,15 +2,14 @@ from ayon_core.addon import AYONAddon, IPluginPaths from ayon_syncsketch.common import config -from .version import __version__ +from .version import __version__ -SYNCSKETCH_MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) +SYNCSKETCH_ADDON_ROOT = os.path.dirname(os.path.abspath(__file__)) class SyncsketchAddon(AYONAddon, IPluginPaths): name = "syncsketch" - enabled = True version = __version__ def get_syncsketch_config(self, project_name): @@ -27,7 +26,7 @@ def get_syncsketch_config(self, project_name): def get_plugin_paths(self): return { "publish": [ - os.path.join(SYNCSKETCH_MODULE_DIR, "plugins", "publish") + os.path.join(SYNCSKETCH_ADDON_ROOT, "plugins", "publish") ] } diff --git a/client/ayon_syncsketch/plugins/publish/integrate_reviewables.py b/client/ayon_syncsketch/plugins/publish/integrate_reviewables.py index b5a4fc8..1bc21dc 100644 --- a/client/ayon_syncsketch/plugins/publish/integrate_reviewables.py +++ b/client/ayon_syncsketch/plugins/publish/integrate_reviewables.py @@ -4,7 +4,10 @@ import json from copy import deepcopy from pprint import pformat + import pyblish.api +import requests + from ayon_core.pipeline import KnownPublishError, AYONPyblishPluginMixin from ayon_core.lib import ( StringTemplate, @@ -12,8 +15,7 @@ filter_profiles, prepare_template_data ) -import ayon_syncsketch.common.server_handler import ServerCommunication # noqa: E501 -import requests +from ayon_syncsketch.common.server_handler import ServerCommunication class IntegrateReviewables(pyblish.api.InstancePlugin, @@ -32,8 +34,9 @@ class IntegrateReviewables(pyblish.api.InstancePlugin, review_item_profiles = [] def filter_review_item_profiles( - self, family, host_name, task_name, task_type): - if self.review_item_profiles is []: + self, family, host_name, task_name, task_type + ): + if not self.review_item_profiles: return [] filtering_criteria = { @@ -320,4 +323,4 @@ def get_attribute_defs(cls): default=True, label="SyncSketch Upload" ) - ] \ No newline at end of file + ] diff --git a/client/ayon_syncsketch/plugins/publish/validate_server_connection.py b/client/ayon_syncsketch/plugins/publish/validate_server_connection.py index e9d9ace..df73c98 100644 --- a/client/ayon_syncsketch/plugins/publish/validate_server_connection.py +++ b/client/ayon_syncsketch/plugins/publish/validate_server_connection.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """Validate a SyncSketch connection.""" import pyblish.api -from ayon_syncsketch.common.server_handler import ServerCommunication # noqa: E501 import requests +from ayon_syncsketch.common.server_handler import ServerCommunication + class ValidateServerConnection(pyblish.api.ContextPlugin): """Validate SyncSketch server connection. diff --git a/create_package.py b/create_package.py index 7250719..e802d2e 100644 --- a/create_package.py +++ b/create_package.py @@ -343,4 +343,4 @@ def main(output_dir=None, skip_zip=False, keep_sources=False): ) args = parser.parse_args(sys.argv[1:]) - main(args.output_dir, args.skip_zip, args.keep_sources) \ No newline at end of file + main(args.output_dir, args.skip_zip, args.keep_sources) diff --git a/server/__init__.py b/server/__init__.py index 4fcd12c..d8ede45 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,4 +1,3 @@ -import os import json from pprint import pformat import socket diff --git a/service/processor/__init__.py b/service/processor/__init__.py index 748e2d4..e9e7e9f 100644 --- a/service/processor/__init__.py +++ b/service/processor/__init__.py @@ -4,4 +4,3 @@ __all__ = ( "SyncSketchProcessor", ) - diff --git a/service/processor/event_handlers/review_session_end.py b/service/processor/event_handlers/review_session_end.py index 0ab7540..ba23f50 100644 --- a/service/processor/event_handlers/review_session_end.py +++ b/service/processor/event_handlers/review_session_end.py @@ -6,7 +6,8 @@ class SyncsketchReviewSessionEnd(FtrackNoteSyncing): """ SyncSketch Review Session End Event Handler. - This class is responsible for processing SyncSketch Review Session End events. + This class is responsible for processing SyncSketch Review Session End + events. """ def __init__(self, addon_settings): @@ -25,8 +26,8 @@ def process(self, payload): Task, if all is found, we try to update the Task's notes with the ones from SyncSketch that are not already there. - Notes are published as the same user as in SyncSketch if the user has the - username in Ftrack otherwise it defaults to the API username. + Notes are published as the same user as in SyncSketch if the user has + the username in Ftrack otherwise it defaults to the API username. Payload example: { diff --git a/service/processor/lib/__init__.py b/service/processor/lib/__init__.py index 51bd350..a12d76a 100644 --- a/service/processor/lib/__init__.py +++ b/service/processor/lib/__init__.py @@ -2,4 +2,4 @@ __all__ = [ "EventProcessor", -] \ No newline at end of file +] diff --git a/service/processor/lib/event_abstraction.py b/service/processor/lib/event_abstraction.py index 58c0449..87b9249 100644 --- a/service/processor/lib/event_abstraction.py +++ b/service/processor/lib/event_abstraction.py @@ -37,7 +37,9 @@ def __init__(self, addon_settings): try: self.statuses_mapping = addon_settings["statuses_mapping"] - self.syncsk_server_config = addon_settings["syncsketch_server_config"] + self.syncsk_server_config = ( + addon_settings["syncsketch_server_config"] + ) self.all_resolved_secrets = get_resolved_secrets( self.syncsk_server_config) diff --git a/service/processor/processor.py b/service/processor/processor.py index 7e414c5..aa68675 100644 --- a/service/processor/processor.py +++ b/service/processor/processor.py @@ -46,8 +46,8 @@ def start_processing(self): """ Main loop enrolling on AYON events. We look for events of the topic `syncsketch.event` and process them by - issuing events of topic `syncsketch.proc` which run the `_upload_review_ - notes_to_ftrack` method against the event payload. + issuing events of topic `syncsketch.proc` which run the + `_upload_review_ notes_to_ftrack` method against the event payload. """ logging.info("Starting the SyncSketch Processor.") root_dir = os.path.dirname(__file__) @@ -56,7 +56,9 @@ def start_processing(self): # Get a list of all the Python files in the folder handler_files = [ - f[:-3] for f in os.listdir(self.event_handlers_folder) if f.endswith(".py") + f[:-3] + for f in os.listdir(self.event_handlers_folder) + if f.endswith(".py") ] # Import each module and get the class @@ -131,4 +133,4 @@ def start_processing(self): except Exception as err: logging.error(f"Unable to enroll for Ayon events: {err}") time.sleep(self.event_sleep_time) - continue \ No newline at end of file + continue diff --git a/syncsketch_common/config.py b/syncsketch_common/config.py index a7e5dbc..60d0137 100644 --- a/syncsketch_common/config.py +++ b/syncsketch_common/config.py @@ -1,6 +1,6 @@ - import ayon_api from ayon_api import get_addon_project_settings + from .constants import required_secret_keys diff --git a/syncsketch_common/constants.py b/syncsketch_common/constants.py index c61646a..a7e185e 100644 --- a/syncsketch_common/constants.py +++ b/syncsketch_common/constants.py @@ -1,4 +1,4 @@ required_secret_keys = [ "auth_token", "auth_user", "account_id", "ftrack_api_key", "ftrack_username" -] \ No newline at end of file +] diff --git a/syncsketch_common/server_handler.py b/syncsketch_common/server_handler.py index 5a12b17..b61fe59 100644 --- a/syncsketch_common/server_handler.py +++ b/syncsketch_common/server_handler.py @@ -628,8 +628,15 @@ def update_review_item(self, item_id, data=None): patch_data=data ) - def upload_review_item(self, review_id, filepath, artist_name="", file_name="", - no_convert_flag=False, item_parent_id=False): + def upload_review_item( + self, + review_id, + filepath, + artist_name="", + file_name="", + no_convert_flag=False, + item_parent_id=False + ): """ Convenience function to upload a file to a review. It will automatically create an Item and attach it to the review. @@ -644,8 +651,8 @@ def upload_review_item(self, review_id, filepath, artist_name="", file_name="", no_convert_flag (bool, optional): The video you are uploading is already in a browser compatible format. Defaults to False. item_parent_id (int, optional): Set when you want to add a new - version of an item. item_parent_id is the id of the item you want - to upload a new version for. Defaults to False. + version of an item. item_parent_id is the id of the item you + want to upload a new version for. Defaults to False. Returns: dict: The response from the API call. diff --git a/tests/client/ayon_syncsketch/api/test_server_handler.py b/tests/client/ayon_syncsketch/api/test_server_handler.py index 6ef4b9a..01bccf6 100644 --- a/tests/client/ayon_syncsketch/api/test_server_handler.py +++ b/tests/client/ayon_syncsketch/api/test_server_handler.py @@ -14,7 +14,9 @@ def test_upload_image_file(self): review_id=os.environ["SYNCSKETCH_REVIEW_ID"], filepath=os.path.join(THIS_DIR, "test_image.jpg"), artist_name="John.Smith", - file_name="shot | episode | sequence | shot01 | compositing | v11 .mp4", + file_name=( + "shot | episode | sequence | shot01 | compositing | v11 .mp4" + ), no_convert_flag=True, item_parent_id=False ) @@ -29,8 +31,10 @@ def test_upload_video_file(self): review_id=os.environ["SYNCSKETCH_REVIEW_ID"], filepath=os.path.join(THIS_DIR, "test_video.mp4"), artist_name="John.Smith", - file_name="shot | episode | sequence | shot01 | compositing | v11 .mp4", + file_name=( + "shot | episode | sequence | shot01 | compositing | v11 .mp4" + ), no_convert_flag=True, item_parent_id=False ) - print(response) \ No newline at end of file + print(response) diff --git a/tests/lib.py b/tests/lib.py index 3fe0305..ac71e02 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -1,10 +1,10 @@ -from unittest.mock import Base -import pytest import os import sys -import responses from pathlib import Path +import responses +import pytest +from unittest.mock import Base from .config_tests import set_environment @@ -27,7 +27,7 @@ class BaseTest: @pytest.fixture(scope="package") def ayon_addons_manager(self): - import ayon_start + import ayon_start # noqa F401 from ayon_core.addon import AddonsManager yield AddonsManager @@ -35,7 +35,9 @@ def ayon_addons_manager(self): # This is the pytest fixture that creates a mock server @pytest.fixture def mock_server(self): - with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + with responses.RequestsMock( + assert_all_requests_are_fired=False + ) as rsps: yield rsps @@ -63,4 +65,4 @@ class Context(PyblishContext): "syncsketch": syncsketch_addon } } - yield Context \ No newline at end of file + yield Context From a04d942f21dfec59dca7bdd95332a3e16ddd16e5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:10:17 +0200 Subject: [PATCH 08/11] updated create package script --- create_package.py | 493 ++++++++++++++++++++++++++++++---------------- 1 file changed, 325 insertions(+), 168 deletions(-) diff --git a/create_package.py b/create_package.py index e802d2e..d8e9b80 100644 --- a/create_package.py +++ b/create_package.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """Prepares server package from addon repo to upload to server. Requires Python 3.9. (Or at least 3.8+). @@ -18,23 +20,45 @@ Package contains server side files directly, client side code zipped in `private` subfolder. """ + import os import sys import re -import json -import platform +import io import shutil +import platform import argparse import logging import collections import zipfile +import subprocess +from typing import Optional, Iterable, Pattern, Union, List, Tuple + +import package +FileMapping = Tuple[Union[str, io.BytesIO], str] +ADDON_NAME: str = package.name +ADDON_VERSION: str = package.version +ADDON_CLIENT_DIR: Union[str, None] = getattr(package, "client_dir", None) COMMON_DIR_NAME: str = "syncsketch_common" -ADDON_NAME = "syncsketch" -ADDON_CLIENT_DIR = "ayon_syncsketch" + +CURRENT_ROOT: str = os.path.dirname(os.path.abspath(__file__)) +SERVER_ROOT: str = os.path.join(CURRENT_ROOT, "server") +FRONTEND_ROOT: str = os.path.join(CURRENT_ROOT, "frontend") +FRONTEND_DIST_ROOT: str = os.path.join(FRONTEND_ROOT, "dist") +DST_DIST_DIR: str = os.path.join("frontend", "dist") +PRIVATE_ROOT: str = os.path.join(CURRENT_ROOT, "private") +PUBLIC_ROOT: str = os.path.join(CURRENT_ROOT, "public") +CLIENT_ROOT: str = os.path.join(CURRENT_ROOT, "client") +COMMON_DIR_ROOT: str = os.path.join(CURRENT_ROOT, COMMON_DIR_NAME) + +VERSION_PY_CONTENT = f'''# -*- coding: utf-8 -*- +"""Package declaring AYON addon '{ADDON_NAME}' version.""" +__version__ = "{ADDON_VERSION}" +''' # Patterns of directories to be skipped for server part of addon -IGNORE_DIR_PATTERNS = [ +IGNORE_DIR_PATTERNS: List[Pattern] = [ re.compile(pattern) for pattern in { # Skip directories starting with '.' @@ -45,7 +69,7 @@ ] # Patterns of files to be skipped for server part of addon -IGNORE_FILE_PATTERNS = [ +IGNORE_FILE_PATTERNS: List[Pattern] = [ re.compile(pattern) for pattern in { # Skip files starting with '.' @@ -75,12 +99,28 @@ def _extract_member(self, member, tpath, pwd): else: tpath = "\\\\?\\" + tpath - return super(ZipFileLongPaths, self)._extract_member( - member, tpath, pwd - ) + return super()._extract_member(member, tpath, pwd) + + +def _get_yarn_executable() -> Union[str, None]: + cmd = "which" + if platform.system().lower() == "windows": + cmd = "where" + for line in subprocess.check_output( + [cmd, "yarn"], encoding="utf-8" + ).splitlines(): + if not line or not os.path.exists(line): + continue + try: + subprocess.call([line, "--version"]) + return line + except OSError: + continue + return None -def safe_copy_file(src_path, dst_path): + +def safe_copy_file(src_path: str, dst_path: str): """Copy file and make sure destination directory exists. Ignore if destination already contains directories from source. @@ -93,223 +133,329 @@ def safe_copy_file(src_path, dst_path): if src_path == dst_path: return - dst_dir = os.path.dirname(dst_path) - try: - os.makedirs(dst_dir) - except Exception: - pass + dst_dir: str = os.path.dirname(dst_path) + os.makedirs(dst_dir, exist_ok=True) shutil.copy2(src_path, dst_path) -def _value_match_regexes(value, regexes): - for regex in regexes: - if regex.search(value): - return True - return False +def _value_match_regexes(value: str, regexes: Iterable[Pattern]) -> bool: + return any( + regex.search(value) + for regex in regexes + ) def find_files_in_subdir( - src_path, - ignore_file_patterns=None, - ignore_dir_patterns=None -): + src_path: str, + ignore_file_patterns: Optional[List[Pattern]] = None, + ignore_dir_patterns: Optional[List[Pattern]] = None +) -> List[Tuple[str, str]]: + """Find all files to copy in subdirectories of given path. + + All files that match any of the patterns in 'ignore_file_patterns' will + be skipped and any directories that match any of the patterns in + 'ignore_dir_patterns' will be skipped with all subfiles. + + Args: + src_path (str): Path to directory to search in. + ignore_file_patterns (Optional[list[Pattern]]): List of regexes + to match files to ignore. + ignore_dir_patterns (Optional[list[Pattern]]): List of regexes + to match directories to ignore. + + Returns: + list[tuple[str, str]]: List of tuples with path to file and parent + directories relative to 'src_path'. + """ + if ignore_file_patterns is None: ignore_file_patterns = IGNORE_FILE_PATTERNS if ignore_dir_patterns is None: ignore_dir_patterns = IGNORE_DIR_PATTERNS - output = [] + output: List[Tuple[str, str]] = [] + if not os.path.exists(src_path): + return output - hierarchy_queue = collections.deque() + hierarchy_queue: collections.deque = collections.deque() hierarchy_queue.append((src_path, [])) while hierarchy_queue: - item = hierarchy_queue.popleft() + item: Tuple[str, str] = hierarchy_queue.popleft() dirpath, parents = item for name in os.listdir(dirpath): - path = os.path.join(dirpath, name) + path: str = os.path.join(dirpath, name) if os.path.isfile(path): if not _value_match_regexes(name, ignore_file_patterns): - items = list(parents) + items: List[str] = list(parents) items.append(name) output.append((path, os.path.sep.join(items))) continue if not _value_match_regexes(name, ignore_dir_patterns): - items = list(parents) + items: List[str] = list(parents) items.append(name) hierarchy_queue.append((path, items)) return output -def copy_server_content(addon_output_dir, current_dir, log): - """Copies server side folders to 'addon_package_dir' +def update_client_version(logger): + """Update version in client code if version.py is present.""" + if not ADDON_CLIENT_DIR: + return - Args: - addon_output_dir (str): package dir in addon repo dir - current_dir (str): addon repo dir - log (logging.Logger) - """ + version_path: str = os.path.join( + CLIENT_ROOT, ADDON_CLIENT_DIR, "version.py" + ) + if not os.path.exists(version_path): + logger.debug("Did not find version.py in client directory") + return - log.info("Copying server content") + logger.info("Updating client version") + with open(version_path, "w") as stream: + stream.write(VERSION_PY_CONTENT) - server_dirpath = os.path.join(current_dir, "server") - common_dir: str = os.path.join(current_dir, COMMON_DIR_NAME) - filepaths_to_copy: list[tuple[str, str]] = [ - ( - os.path.join(current_dir, "version.py"), - os.path.join(addon_output_dir, "version.py") - ), - # Copy constants needed for attributes creation - ( - os.path.join(common_dir, "server_handler.py"), - os.path.join(addon_output_dir, "common", "server_handler.py") - ), - ( - os.path.join(common_dir, "constants.py"), - os.path.join(addon_output_dir, "common", "constants.py") - ), - ] +def build_frontend(): + yarn_executable = _get_yarn_executable() + if yarn_executable is None: + raise RuntimeError("Yarn executable was not found.") - for item in find_files_in_subdir(server_dirpath): - src_path, dst_subpath = item - dst_path = os.path.join(addon_output_dir, dst_subpath) - filepaths_to_copy.append((src_path, dst_path)) + subprocess.run([yarn_executable, "install"], cwd=FRONTEND_ROOT) + subprocess.run([yarn_executable, "build"], cwd=FRONTEND_ROOT) + if not os.path.exists(FRONTEND_DIST_ROOT): + raise RuntimeError( + "Frontend build failed. Did not find 'dist' folder." + ) - # Copy files - for src_path, dst_path in filepaths_to_copy: - safe_copy_file(src_path, dst_path) +def get_client_files_mapping() -> List[Tuple[str, str]]: + """Mapping of source client code files to destination paths. -def zip_client_side(addon_package_dir, current_dir, log): - """Copy and zip `client` content into 'addon_package_dir'. + Example output: + [ + ( + "C:/addons/MyAddon/version.py", + "my_addon/version.py" + ), + ( + "C:/addons/MyAddon/client/my_addon/__init__.py", + "my_addon/__init__.py" + ) + ] + + Returns: + list[tuple[str, str]]: List of path mappings to copy. The destination + path is relative to expected output directory. - Args: - addon_package_dir (str): Output package directory path. - current_dir (str): Directory path of addon source. - log (logging.Logger): Logger object. """ - client_dir = os.path.join(current_dir, "client") - common_dir: str = os.path.join(current_dir, COMMON_DIR_NAME) + # Add client code content to zip + client_code_dir: str = os.path.join(CLIENT_ROOT, ADDON_CLIENT_DIR) + mapping = [ + (path, os.path.join(ADDON_CLIENT_DIR, sub_path)) + for path, sub_path in find_files_in_subdir(client_code_dir) + ] - if not os.path.isdir(client_dir): - log.info("Client directory was not found. Skipping") - return + for path, sub_path in find_files_in_subdir(COMMON_DIR_ROOT): + dst_path = "/".join((ADDON_CLIENT_DIR, "common", sub_path)) + mapping.append((path, dst_path)) + license_path = os.path.join(CURRENT_ROOT, "LICENSE") + if os.path.exists(license_path): + mapping.append((license_path, f"{ADDON_CLIENT_DIR}/LICENSE")) + return mapping + + +def get_client_zip_content(log) -> io.BytesIO: log.info("Preparing client code zip") - private_dir = os.path.join(addon_package_dir, "private") - - if not os.path.exists(private_dir): - os.makedirs(private_dir) - - src_version_path = os.path.join(current_dir, "version.py") - dst_version_path = os.path.join(ADDON_CLIENT_DIR, "version.py") - - zip_filepath = os.path.join(os.path.join(private_dir, "client.zip")) - with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf: - # Add client code content to zip - for path, sub_path in find_files_in_subdir(client_dir): - if ( - "common" in path - or "version.py" in path - or "settings.py" in path - ): - # skip common for case the folder is preset during development - print(path, sub_path) - continue + files_mapping: List[Tuple[str, str]] = get_client_files_mapping() + stream = io.BytesIO() + with ZipFileLongPaths(stream, "w", zipfile.ZIP_DEFLATED) as zipf: + for src_path, subpath in files_mapping: + zipf.write(src_path, subpath) + stream.seek(0) + return stream + + +def get_base_files_mapping() -> List[FileMapping]: + filepaths_to_copy: List[FileMapping] = [ + ( + os.path.join(CURRENT_ROOT, "package.py"), + "package.py" + ) + ] + # Add license file to package if exists + license_path = os.path.join(CURRENT_ROOT, "LICENSE") + if os.path.exists(license_path): + filepaths_to_copy.append((license_path, "LICENSE")) + + for filename in ( + "server_handler.py", + "constants.py", + ): + src_path = os.path.join(COMMON_DIR_ROOT, filename) + dst_path = os.path.join("server", "common", filename) + filepaths_to_copy.append((src_path, dst_path)) + + # Go through server, private and public directories and find all files + for dirpath in (SERVER_ROOT, PRIVATE_ROOT, PUBLIC_ROOT): + if not os.path.exists(dirpath): + continue + + dirname = os.path.basename(dirpath) + for src_file, subpath in find_files_in_subdir(dirpath): + dst_subpath = os.path.join(dirname, subpath) + filepaths_to_copy.append((src_file, dst_subpath)) + + if os.path.exists(FRONTEND_DIST_ROOT): + for src_file, subpath in find_files_in_subdir(FRONTEND_DIST_ROOT): + dst_subpath = os.path.join(DST_DIST_DIR, subpath) + filepaths_to_copy.append((src_file, dst_subpath)) + + pyproject_toml = os.path.join(CLIENT_ROOT, "pyproject.toml") + if os.path.exists(pyproject_toml): + filepaths_to_copy.append( + (pyproject_toml, "private/pyproject.toml") + ) - zipf.write(path, sub_path) + return filepaths_to_copy + + +def copy_client_code(output_dir: str, log: logging.Logger): + """Copies server side folders to 'addon_package_dir' + + Args: + output_dir (str): Output directory path. + log (logging.Logger) + + """ + log.info(f"Copying client for {ADDON_NAME}-{ADDON_VERSION}") - for path, sub_path in find_files_in_subdir(common_dir): - dst_path = "/".join((ADDON_CLIENT_DIR, "common", sub_path)) - zipf.write(path, dst_path) + full_output_path = os.path.join( + output_dir, f"{ADDON_NAME}_{ADDON_VERSION}" + ) + if os.path.exists(full_output_path): + shutil.rmtree(full_output_path) + os.makedirs(full_output_path, exist_ok=True) - # Add 'version.py' to client code - zipf.write(src_version_path, dst_version_path) + for src_path, dst_subpath in get_client_files_mapping(): + dst_path = os.path.join(full_output_path, dst_subpath) + safe_copy_file(src_path, dst_path) + log.info("Client copy finished") -def create_server_package(output_dir, addon_output_dir, addon_version, log): - """Create server package zip file. - The zip file can be installed to a server using UI or rest api endpoints. +def copy_addon_package( + output_dir: str, + files_mapping: List[FileMapping], + log: logging.Logger +): + """Copy client code to output directory. Args: - output_dir (str): Directory path to output zip file. - addon_output_dir (str): Directory path to addon output directory. - addon_version (str): Version of addon. + output_dir (str): Directory path to output client code. + files_mapping (List[FileMapping]): List of tuples with source file + and destination subpath. log (logging.Logger): Logger object. + """ + log.info(f"Copying package for {ADDON_NAME}-{ADDON_VERSION}") - log.info("Creating server package") - output_path = os.path.join( - output_dir, f"{ADDON_NAME}-{addon_version}.zip" + # Add addon name and version to output directory + addon_output_dir: str = os.path.join( + output_dir, ADDON_NAME, ADDON_VERSION ) - manifest_data: dict[str, str] = { - "addon_name": ADDON_NAME, - "addon_version": addon_version - } - with ZipFileLongPaths(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: - # Write a manifest to zip - zipf.writestr("manifest.json", json.dumps(manifest_data, indent=4)) + if os.path.isdir(addon_output_dir): + log.info(f"Purging {addon_output_dir}") + shutil.rmtree(addon_output_dir) + + os.makedirs(addon_output_dir, exist_ok=True) + + # Copy server content + for src_file, dst_subpath in files_mapping: + dst_path: str = os.path.join(addon_output_dir, dst_subpath) + dst_dir: str = os.path.dirname(dst_path) + os.makedirs(dst_dir, exist_ok=True) + if isinstance(src_file, io.BytesIO): + with open(dst_path, "wb") as stream: + stream.write(src_file.getvalue()) + else: + safe_copy_file(src_file, dst_path) + + log.info("Package copy finished") + + +def create_addon_package( + output_dir: str, + files_mapping: List[FileMapping], + log: logging.Logger +): + log.info(f"Creating package for {ADDON_NAME}-{ADDON_VERSION}") - # Move addon content to zip into 'addon' directory - addon_output_dir_offset = len(addon_output_dir) + 1 - for root, _, filenames in os.walk(addon_output_dir): - if not filenames: - continue + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join( + output_dir, f"{ADDON_NAME}-{ADDON_VERSION}.zip" + ) - dst_root = "addon" - if root != addon_output_dir: - dst_root = os.path.join( - dst_root, root[addon_output_dir_offset:] - ) - for filename in filenames: - src_path = os.path.join(root, filename) - dst_path = os.path.join(dst_root, filename) - zipf.write(src_path, dst_path) + with ZipFileLongPaths(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: + # Copy server content + for src_file, dst_subpath in files_mapping: + if isinstance(src_file, io.BytesIO): + zipf.writestr(dst_subpath, src_file.getvalue()) + else: + zipf.write(src_file, dst_subpath) - log.info(f"Output package can be found: {output_path}") + log.info("Package created") -def main(output_dir=None, skip_zip=False, keep_sources=False): - log = logging.getLogger("create_package") - log.info("Start creating package") +def main( + output_dir: Optional[str] = None, + skip_zip: Optional[bool] = False, + only_client: Optional[bool] = False +): + log: logging.Logger = logging.getLogger("create_package") + log.info("Package creation started") - current_dir = os.path.dirname(os.path.abspath(__file__)) if not output_dir: - output_dir = os.path.join(current_dir, "package") - - version_filepath = os.path.join(current_dir, "version.py") - version_content = {} - with open(version_filepath, "r") as stream: - exec(stream.read(), version_content) - addon_version = version_content["__version__"] - - addon_output_root = os.path.join(output_dir, ADDON_NAME) - addon_output_dir = os.path.join(addon_output_root, addon_version) - if os.path.isdir(addon_output_root): - log.info(f"Purging {addon_output_root}") - shutil.rmtree(addon_output_root) + output_dir = os.path.join(CURRENT_ROOT, "package") + + has_client_code = bool(ADDON_CLIENT_DIR) + if has_client_code: + client_dir: str = os.path.join(CLIENT_ROOT, ADDON_CLIENT_DIR) + if not os.path.exists(client_dir): + raise RuntimeError( + f"Client directory was not found '{client_dir}'." + " Please check 'client_dir' in 'package.py'." + ) + update_client_version(log) + + if only_client: + if not has_client_code: + raise RuntimeError("Client code is not available. Skipping") + + copy_client_code(output_dir, log) + return - os.makedirs(addon_output_dir) + log.info(f"Preparing package for {ADDON_NAME}-{ADDON_VERSION}") - log.info(f"Preparing package for {ADDON_NAME}-{addon_version}") + if os.path.exists(FRONTEND_ROOT): + build_frontend() - copy_server_content(addon_output_dir, current_dir, log) + files_mapping: List[FileMapping] = [] + files_mapping.extend(get_base_files_mapping()) - zip_client_side(addon_output_dir, current_dir, log) + if has_client_code: + files_mapping.append( + (get_client_zip_content(log), "private/client.zip") + ) # Skip server zipping - if not skip_zip: - create_server_package( - output_dir, addon_output_dir, addon_version, log - ) - # Remove sources only if zip file is created - if not keep_sources: - log.info("Removing source files for server package") - shutil.rmtree(addon_output_root) + if skip_zip: + copy_addon_package(output_dir, files_mapping, log) + else: + create_addon_package(output_dir, files_mapping, log) + log.info("Package creation finished") @@ -324,14 +470,6 @@ def main(output_dir=None, skip_zip=False, keep_sources=False): " server folder structure." ) ) - parser.add_argument( - "--keep-sources", - dest="keep_sources", - action="store_true", - help=( - "Keep folder structure when server package is created." - ) - ) parser.add_argument( "-o", "--output", dest="output_dir", @@ -341,6 +479,25 @@ def main(output_dir=None, skip_zip=False, keep_sources=False): " (Will be purged if already exists!)" ) ) + parser.add_argument( + "--only-client", + dest="only_client", + action="store_true", + help=( + "Extract only client code. This is useful for development." + " Requires '-o', '--output' argument to be filled." + ) + ) + parser.add_argument( + "--debug", + dest="debug", + action="store_true", + help="Debug log messages." + ) args = parser.parse_args(sys.argv[1:]) - main(args.output_dir, args.skip_zip, args.keep_sources) + level = logging.INFO + if args.debug: + level = logging.DEBUG + logging.basicConfig(level=level) + main(args.output_dir, args.skip_zip, args.only_client) From 2c37e52d5b7129bcf1a36d1810281ea3ee7ae10f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:12:02 +0200 Subject: [PATCH 09/11] lowered version and push client version --- .gitignore | 4 ---- client/ayon_syncsketch/version.py | 3 +++ package.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 client/ayon_syncsketch/version.py diff --git a/.gitignore b/.gitignore index 4b08f46..15e6d75 100644 --- a/.gitignore +++ b/.gitignore @@ -161,12 +161,8 @@ Temporary Items .apdisk # development environment -server/version.py server/common/ client/ayon_syncsketch/common/ service/processor/common/ - service/poetry.lock - -client/ayon_syncsketch/version.py diff --git a/client/ayon_syncsketch/version.py b/client/ayon_syncsketch/version.py new file mode 100644 index 0000000..32063d3 --- /dev/null +++ b/client/ayon_syncsketch/version.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +"""Package declaring AYON addon 'syncsketch' version.""" +__version__ = "0.1.2-dev.1" diff --git a/package.py b/package.py index fbcc40d..067de06 100644 --- a/package.py +++ b/package.py @@ -2,7 +2,7 @@ """Package declaring addon version.""" name = "syncsketch" title = "SyncSketch" -version = "0.2.0" +version = "0.1.2-dev.1" client_dir = "ayon_syncsketch" # TODO: need to make sure image is published to docker hub From 7a71a7e89d94c6681465610baa5c30a39e03ad83 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:17:07 +0200 Subject: [PATCH 10/11] update service version on create package --- create_package.py | 24 ++++++++++++++++++++++++ service/docker-compose.yml | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/create_package.py b/create_package.py index d8e9b80..fffdf2e 100644 --- a/create_package.py +++ b/create_package.py @@ -217,6 +217,28 @@ def update_client_version(logger): stream.write(VERSION_PY_CONTENT) +def update_service_version(logger): + docker_compose_path = os.path.join( + CURRENT_ROOT, "service", "docker-compose.yml" + ) + with open(docker_compose_path, "r") as stream: + content = stream.readlines() + + new_lines = [] + sep = "image: ynput/ayon-syncsketch-processor" + for line in content: + if sep in line: + head, _ = line.split(sep) + new_line = f"{head}{sep}:{ADDON_VERSION}\n" + if new_line != line: + logger.info(f"Updating service version to {ADDON_VERSION}") + + new_lines.append(line) + + with open(docker_compose_path, "w") as stream: + stream.write("".join(new_lines)) + + def build_frontend(): yarn_executable = _get_yarn_executable() if yarn_executable is None: @@ -430,6 +452,8 @@ def main( ) update_client_version(log) + update_service_version(log) + if only_client: if not has_client_code: raise RuntimeError("Client code is not available. Skipping") diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 142973b..71d08b7 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -2,6 +2,6 @@ name: ayon-syncsketch-services services: processor: container_name: processor - image: ynput/ayon-syncsketch-processor:0.1.1 + image: ynput/ayon-syncsketch-processor:0.1.2-dev.1 restart: unless-stopped env_file: .env From d586dbd3867dc920276b62f3d38d3fe3c2413a8b Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:35:46 +0200 Subject: [PATCH 11/11] updated release trigger --- .github/workflows/release_trigger.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_trigger.yml b/.github/workflows/release_trigger.yml index 01a3b3a..caa1075 100644 --- a/.github/workflows/release_trigger.yml +++ b/.github/workflows/release_trigger.yml @@ -2,10 +2,23 @@ name: 🚀 Release Trigger on: workflow_dispatch: + inputs: + draft: + type: boolean + description: "Create Release Draft" + required: false + default: true + release_overwrite: + type: string + description: "Set Version Release Tag" + required: false jobs: call-release-trigger: - uses: ynput/ops-repo-automation/.github/workflows/release_trigger.yml@main + uses: ynput/ops-repo-automation/.github/workflows/release_trigger.yml@develop + with: + draft: ${{ inputs.draft }} + release_overwrite: ${{ inputs.release_overwrite }} secrets: token: ${{ secrets.YNPUT_BOT_TOKEN }} email: ${{ secrets.CI_EMAIL }}