From 0bf29952a7ac3d3665575731592647d20e6aa2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 19 Nov 2024 17:39:39 +0100 Subject: [PATCH 1/3] :wrench: refactoring rendering on farm wip --- .../plugins/create/create_render.py | 34 +++++- .../publish/collect_render_instances.py | 92 -------------- .../plugins/publish/collect_render_target.py | 25 ++++ ...der.py => create_farm_render_instances.py} | 113 +++++++++++++++--- 4 files changed, 153 insertions(+), 111 deletions(-) delete mode 100644 client/ayon_unreal/plugins/publish/collect_render_instances.py create mode 100644 client/ayon_unreal/plugins/publish/collect_render_target.py rename client/ayon_unreal/plugins/publish/{collect_farm_render.py => create_farm_render_instances.py} (67%) diff --git a/client/ayon_unreal/plugins/create/create_render.py b/client/ayon_unreal/plugins/create/create_render.py index 1a6373fb..27800f38 100644 --- a/client/ayon_unreal/plugins/create/create_render.py +++ b/client/ayon_unreal/plugins/create/create_render.py @@ -45,6 +45,14 @@ def create_instance( instance_data, pre_create_data) + def update_instance(self, instance, instance_data, pre_create_data): + instance_data["label"] = f'instance_data.get("folderPath") - {instance_data.get("productName")}' + super(CreateRender, self).update_instance( + instance, + instance_data, + pre_create_data) + + def create_with_new_sequence( self, product_name, instance_data, pre_create_data ): @@ -236,6 +244,10 @@ def create_from_existing_sequence( selected_asset_path, master_seq, master_lvl, seq_data) def create(self, product_name, instance_data, pre_create_data): + instance_data["label"] = f'{instance_data.get("folderPath")} - {product_name}' + if not instance_data.get("creator_attributes"): + instance_data["creator_attributes"] = {} + instance_data["creator_attributes"]["render_target"] = pre_create_data.get("render_target") if pre_create_data.get("create_seq"): self.create_with_new_sequence( product_name, instance_data, pre_create_data) @@ -299,6 +311,26 @@ def get_instance_attr_defs(self): return [ EnumDef( "render_target", items=rendering_targets, - label="Render target" + label="Render target", + default="local" + ), + BoolDef( + "review", + label="Generate review", + default=True ), ] + + def _on_value_changed(self, event): + for changed_item in event["changes"]: + instance = changed_item["instance"] + changes = changed_item["changes"] + if ( + instance is not None + and "folderPath" in changes + and instance.creator_identifier == self.identifier + ): + instance.data["label"] = f'{instance.data.get("folderPath")} - {instance.data.get("productName")}' + + def register_callbacks(self): + self.create_context.add_value_changed_callback(self._on_value_changed) diff --git a/client/ayon_unreal/plugins/publish/collect_render_instances.py b/client/ayon_unreal/plugins/publish/collect_render_instances.py deleted file mode 100644 index e1053ef2..00000000 --- a/client/ayon_unreal/plugins/publish/collect_render_instances.py +++ /dev/null @@ -1,92 +0,0 @@ -import unreal - -from ayon_unreal.api import pipeline -import pyblish.api - - -class CollectRenderInstances(pyblish.api.InstancePlugin): - """ Marks instance to be rendered locally or on the farm - - """ - order = pyblish.api.CollectorOrder - hosts = ["unreal"] - families = ["render"] - label = "Collect Render Instances" - - def process(self, instance): - self.log.debug("Preparing Rendering Instances") - - render_target = (instance.data["creator_attributes"]. - get("render_target")) - if render_target == "farm": - instance.data["families"].append("render.farm") - instance.data["farm"] = True - self.preparing_rendering_instance(instance) - - else: - instance.data["families"].append("render.local") - - def preparing_rendering_instance(self, instance): - context = instance.context - - data = instance.data - data['remove'] = True - - ar = unreal.AssetRegistryHelpers.get_asset_registry() - - sequence = ar.get_asset_by_object_path( - data.get('sequence')).get_asset() - - sequences = [{ - "sequence": sequence, - "output": data.get('output'), - "frame_range": ( - data.get('frameStart'), data.get('frameEnd')) - }] - - for s in sequences: - self.log.debug(f"Processing: {s.get('sequence').get_name()}") - subscenes = pipeline.get_subsequences(s.get('sequence')) - - if subscenes: - for ss in subscenes: - sequences.append({ - "sequence": ss.get_sequence(), - "output": (f"{s.get('output')}/" - f"{ss.get_sequence().get_name()}"), - "frame_range": ( - ss.get_start_frame(), ss.get_end_frame() - 1) - }) - else: - # Avoid creating instances for camera sequences - if "_camera" not in s.get('sequence').get_name(): - seq = s.get('sequence') - seq_name = seq.get_name() - - product_type = "render" - new_product_name = f"{data.get('productName')}_{seq_name}" - new_instance = context.create_instance( - new_product_name - ) - new_instance[:] = seq_name - - new_data = new_instance.data - - new_data["folderPath"] = instance.data["folderPath"] - new_data["setMembers"] = seq_name - new_data["productName"] = new_product_name - new_data["productType"] = product_type - new_data["family"] = product_type - new_data["families"] = [product_type, "review"] - new_data["parent"] = data.get("parent") - new_data["level"] = data.get("level") - new_data["output"] = s['output'] - new_data["fps"] = seq.get_display_rate().numerator - new_data["frameStart"] = int(s.get('frame_range')[0]) - new_data["frameEnd"] = int(s.get('frame_range')[1]) - new_data["sequence"] = seq.get_path_name() - new_data["master_sequence"] = data["master_sequence"] - new_data["master_level"] = data["master_level"] - new_data["farm"] = instance.data.get("farm", False) - - self.log.debug(f"new instance data: {new_data}") diff --git a/client/ayon_unreal/plugins/publish/collect_render_target.py b/client/ayon_unreal/plugins/publish/collect_render_target.py new file mode 100644 index 00000000..d6c228e8 --- /dev/null +++ b/client/ayon_unreal/plugins/publish/collect_render_target.py @@ -0,0 +1,25 @@ +"""Collect current project path.""" +import unreal # noqa +import pyblish.api + + +class CollectRenderTarget(pyblish.api.InstancePlugin): + """Inject the current working file into context.""" + + order = pyblish.api.CollectorOrder - 0.5 + label = "Collect Render Target" + hosts = ["unreal"] + families = ["render"] + + def process(self, instance): + """Inject the current working file.""" + render_target = (instance.data["creator_attributes"]. + get("render_target")) + if render_target == "farm": + self.log.debug("Rendering on farm") + instance.data["farm"] = True + return + + self.log.debug("Using locally renderer files") + instance.data["families"].append("render.local") + instance.data["farm"] = False diff --git a/client/ayon_unreal/plugins/publish/collect_farm_render.py b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py similarity index 67% rename from client/ayon_unreal/plugins/publish/collect_farm_render.py rename to client/ayon_unreal/plugins/publish/create_farm_render_instances.py index 5bac1f3e..dfc651f8 100644 --- a/client/ayon_unreal/plugins/publish/collect_farm_render.py +++ b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py @@ -4,9 +4,11 @@ import pyblish.api -from ayon_core.pipeline import publish +from ayon_core.pipeline import publish, PublishError from ayon_core.pipeline.publish import RenderInstance +from ayon_unreal.api import pipeline + from ayon_unreal.api.pipeline import UNREAL_VERSION from ayon_unreal.api.rendering import ( SUPPORTED_EXTENSION_MAP, @@ -30,12 +32,77 @@ class UnrealRenderInstance(RenderInstance): render_queue_path = attr.ib(default=None) -class CollectUnrealRemoteRender(publish.AbstractCollectRender): +class CreateFarmRenderInstances(publish.AbstractCollectRender): - order = pyblish.api.CollectorOrder + 0.405 - label = "Collect Farm Expected files" + order = pyblish.api.CollectorOrder + 0.21 + label = "Create Farm Render Instances" hosts = ["unreal"] - families = ["render.farm"] + families = ["render"] + + def preparing_rendering_instance(self, instance): + context = instance.context + + data = instance.data + data["remove"] = True + + ar = unreal.AssetRegistryHelpers.get_asset_registry() + + sequence = ar.get_asset_by_object_path( + data.get("sequence")).get_asset() + + sequences = [{ + "sequence": sequence, + "output": data.get("output"), + "frame_range": ( + data.get("frameStart"), data.get("frameEnd")) + }] + + for s in sequences: + self.log.debug(f"Processing: {s.get('sequence').get_name()}") + subscenes = pipeline.get_subsequences(s.get('sequence')) + + if subscenes: + for ss in subscenes: + sequences.append({ + "sequence": ss.get_sequence(), + "output": (f"{s.get('output')}/" + f"{ss.get_sequence().get_name()}"), + "frame_range": ( + ss.get_start_frame(), ss.get_end_frame() - 1) + }) + else: + # Avoid creating instances for camera sequences + if "_camera" not in s.get('sequence').get_name(): + seq = s.get('sequence') + seq_name = seq.get_name() + + product_type = "render" + new_product_name = f"{data.get('productName')}_{seq_name}" + new_instance = context.create_instance( + new_product_name + ) + new_instance[:] = seq_name + + new_data = new_instance.data + + new_data["folderPath"] = instance.data["folderPath"] + new_data["setMembers"] = seq_name + new_data["productName"] = new_product_name + new_data["productType"] = product_type + new_data["family"] = product_type + new_data["families"] = [product_type, "review"] + new_data["parent"] = data.get("parent") + new_data["level"] = data.get("level") + new_data["output"] = s['output'] + new_data["fps"] = seq.get_display_rate().numerator + new_data["frameStart"] = int(s.get('frame_range')[0]) + new_data["frameEnd"] = int(s.get('frame_range')[1]) + new_data["sequence"] = seq.get_path_name() + new_data["master_sequence"] = data["master_sequence"] + new_data["master_level"] = data["master_level"] + new_data["farm"] = instance.data.get("farm", False) + + self.log.debug(f"new instance data: {new_data}") def get_instances(self, context): instances = [] @@ -70,6 +137,9 @@ def get_instances(self, context): fps = f"{output_fps.denominator}.{output_fps.numerator}" for inst in context: + instance_families = inst.data.get("families", []) + product_name = inst.data["productName"] + if not inst.data.get("active", True): continue @@ -77,13 +147,19 @@ def get_instances(self, context): if family not in ["render"]: continue + # skip if locar render instances + if "render.local" in instance_families: + continue + render_queue_path = ( project_settings["unreal"]["render_queue_path"] ) if not unreal.EditorAssetLibrary.does_asset_exist( render_queue_path): - # TODO temporary until C++ blueprint is created as it is not - # possible to create renderQueue + # TODO: temporary until C++ blueprint is created as it is not + # possible to create renderQueue. Also, we could + # use Render Graph from UE 5.4 + master_level = inst.data["master_level"] sequence = inst.data["sequence"] msg = (f"Please create `Movie Pipeline Queue` " @@ -91,19 +167,17 @@ def get_instances(self, context): f"Set it Sequence to `{sequence}`, " f"Map to `{master_level}` and " f"Settings to `{config_path}` ") - raise RuntimeError(msg) + raise PublishError(msg) - instance_families = inst.data.get("families", []) - product_name = inst.data["productName"] # backward compatibility task_name = inst.data.get("task") or inst.data.get("task_name") - self.log.info(f"task_name::{task_name}") + self.log.debug(f"Task name:{task_name}") ar = unreal.AssetRegistryHelpers.get_asset_registry() sequence = (ar.get_asset_by_object_path(inst.data["sequence"]). get_asset()) if not sequence: - raise RuntimeError(f"Cannot find {inst.data['sequence']}") + raise PublishError(f"Cannot find {inst.data['sequence']}") # current frame range - might be different from created frame_start = sequence.get_playback_start() @@ -123,13 +197,15 @@ def get_instances(self, context): publish_attributes = {} - instance = UnrealRenderInstance( + review = bool(inst.data["creator_attributes"].get("review")) + + new_instance = UnrealRenderInstance( family="render", families=["render.farm"], version=version, time="", source=current_file, - label="{} - {}".format(product_name, family), + label=f"{product_name} - {family}", productName=product_name, productType="render", folderPath=inst.data["folderPath"], @@ -144,7 +220,7 @@ def get_instances(self, context): tileRendering=False, tilesX=0, tilesY=0, - review="review" in instance_families, + review=review, frameStart=frame_start, frameEnd=frame_end, frameStep=1, @@ -156,14 +232,15 @@ def get_instances(self, context): config_path=config_path, master_level=inst.data["master_level"], render_queue_path=render_queue_path, - deadline=inst.data.get("deadline") + deadline=inst.data.get("deadline"), ) - instance.farm = True + new_instance.farm = True - instances.append(instance) + instances.append(new_instance) instances_to_remove.append(inst) for instance in instances_to_remove: + self.log.debug(f"Removing instance: {instance}") context.remove(instance) return instances From e515c8fa52daa85ee051b1a227a23a1a210bec25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 20 Nov 2024 00:28:47 +0100 Subject: [PATCH 2/3] :recycle: force multipart to get the reviews --- .../plugins/publish/create_farm_render_instances.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/ayon_unreal/plugins/publish/create_farm_render_instances.py b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py index dfc651f8..004c9a69 100644 --- a/client/ayon_unreal/plugins/publish/create_farm_render_instances.py +++ b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py @@ -100,6 +100,7 @@ def preparing_rendering_instance(self, instance): new_data["sequence"] = seq.get_path_name() new_data["master_sequence"] = data["master_sequence"] new_data["master_level"] = data["master_level"] + new_data["review"] = instance.data.get("review", False) new_data["farm"] = instance.data.get("farm", False) self.log.debug(f"new instance data: {new_data}") @@ -233,6 +234,7 @@ def get_instances(self, context): master_level=inst.data["master_level"], render_queue_path=render_queue_path, deadline=inst.data.get("deadline"), + multipartExr=True, ) new_instance.farm = True @@ -253,14 +255,14 @@ def _get_expected_file_name(self, file_name_format, ext, frame_placeholder) return f"{file_name_format}.{ext}" - def get_expected_files(self, render_instance): + def get_expected_files(self, render_instance: UnrealRenderInstance): """ Returns list of rendered files that should be created by Deadline. These are not published directly, they are source for later 'submit_publish_job'. Args: - render_instance (RenderInstance): to pull anatomy and parts used + render_instance (UnrealRenderInstance): to pull anatomy and parts used in url Returns: @@ -288,7 +290,7 @@ def _get_output_dir(self, render_instance): """ Returns dir path of rendered files, used in submit_publish_job for metadata.json location. - Should be in separate folder inside of work area. + Should be in separate folder inside work area. Args: render_instance (RenderInstance): From 2f09a7d333deb45392eb14c7e05764ce8fcbbeb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 20 Nov 2024 17:05:16 +0100 Subject: [PATCH 3/3] :bug: fix bugs --- .../ayon_unreal/plugins/create/create_render.py | 5 ++++- .../plugins/publish/collect_render_files.py | 1 - .../publish/create_farm_render_instances.py | 12 ++++++++---- ...t_render_target.py => switch_render_target.py} | 15 ++++++--------- 4 files changed, 18 insertions(+), 15 deletions(-) rename client/ayon_unreal/plugins/publish/{collect_render_target.py => switch_render_target.py} (60%) diff --git a/client/ayon_unreal/plugins/create/create_render.py b/client/ayon_unreal/plugins/create/create_render.py index 27800f38..8af230ac 100644 --- a/client/ayon_unreal/plugins/create/create_render.py +++ b/client/ayon_unreal/plugins/create/create_render.py @@ -330,7 +330,10 @@ def _on_value_changed(self, event): and "folderPath" in changes and instance.creator_identifier == self.identifier ): - instance.data["label"] = f'{instance.data.get("folderPath")} - {instance.data.get("productName")}' + instance.data["label"] = ( + f'{instance.data.get("folderPath")} - ' + f'{instance.data.get("productName")}' + ) def register_callbacks(self): self.create_context.add_value_changed_callback(self._on_value_changed) diff --git a/client/ayon_unreal/plugins/publish/collect_render_files.py b/client/ayon_unreal/plugins/publish/collect_render_files.py index cc6055d7..64eab110 100644 --- a/client/ayon_unreal/plugins/publish/collect_render_files.py +++ b/client/ayon_unreal/plugins/publish/collect_render_files.py @@ -15,7 +15,6 @@ class CollectRenderFiles(pyblish.api.InstancePlugin): add them as representation. """ order = pyblish.api.CollectorOrder + 0.001 - hosts = ["unreal"] families = ["render.local"] label = "Collect Render Files" diff --git a/client/ayon_unreal/plugins/publish/create_farm_render_instances.py b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py index 004c9a69..7c6916ff 100644 --- a/client/ayon_unreal/plugins/publish/create_farm_render_instances.py +++ b/client/ayon_unreal/plugins/publish/create_farm_render_instances.py @@ -36,8 +36,6 @@ class CreateFarmRenderInstances(publish.AbstractCollectRender): order = pyblish.api.CollectorOrder + 0.21 label = "Create Farm Render Instances" - hosts = ["unreal"] - families = ["render"] def preparing_rendering_instance(self, instance): context = instance.context @@ -152,6 +150,10 @@ def get_instances(self, context): if "render.local" in instance_families: continue + if not inst.data.get("farm", False): + self.log.info("Skipping local render instance") + continue + render_queue_path = ( project_settings["unreal"]["render_queue_path"] ) @@ -198,7 +200,10 @@ def get_instances(self, context): publish_attributes = {} - review = bool(inst.data["creator_attributes"].get("review")) + try: + review = bool(inst.data["creator_attributes"].get("review")) + except KeyError: + review = inst.data.get("review", False) new_instance = UnrealRenderInstance( family="render", @@ -234,7 +239,6 @@ def get_instances(self, context): master_level=inst.data["master_level"], render_queue_path=render_queue_path, deadline=inst.data.get("deadline"), - multipartExr=True, ) new_instance.farm = True diff --git a/client/ayon_unreal/plugins/publish/collect_render_target.py b/client/ayon_unreal/plugins/publish/switch_render_target.py similarity index 60% rename from client/ayon_unreal/plugins/publish/collect_render_target.py rename to client/ayon_unreal/plugins/publish/switch_render_target.py index d6c228e8..7c373c52 100644 --- a/client/ayon_unreal/plugins/publish/collect_render_target.py +++ b/client/ayon_unreal/plugins/publish/switch_render_target.py @@ -1,22 +1,19 @@ -"""Collect current project path.""" -import unreal # noqa import pyblish.api -class CollectRenderTarget(pyblish.api.InstancePlugin): - """Inject the current working file into context.""" - - order = pyblish.api.CollectorOrder - 0.5 - label = "Collect Render Target" - hosts = ["unreal"] +class SwitchRenderTargets(pyblish.api.InstancePlugin): + """Switch between farm and local render targets.""" + order = pyblish.api.CollectorOrder - 0.499 families = ["render"] + label = "Switch Render Targets" def process(self, instance): - """Inject the current working file.""" + self.log.debug(instance.data["creator_attributes"]) render_target = (instance.data["creator_attributes"]. get("render_target")) if render_target == "farm": self.log.debug("Rendering on farm") + instance.data["families"].append("render.farm") instance.data["farm"] = True return