From 9f73b1857af9a82579905b9ea3374c1f7c21bd89 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Sep 2024 15:34:02 +0200 Subject: [PATCH 01/47] Added creation of separate sequence category Creates additional `sequence` subfolder for seq assets to be created. --- .../match_shotgrid_hierarchy_in_ayon.py | 10 +++ services/shotgrid_common/utils.py | 74 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index e9af92a5..62e48e99 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -21,6 +21,7 @@ from utils import ( get_sg_entities, get_asset_category, + get_sequence_category, update_ay_entity_custom_attributes, ) @@ -105,6 +106,8 @@ def match_shotgrid_hierarchy_in_ayon( ay_entity = child break + log.info(f"SHOTGRID_TYPE_ATTRIB::{sg_ay_dict['attribs'].get(SHOTGRID_TYPE_ATTRIB)}") + log.info(f"ay_entity::{ay_entity}") # If we couldn't find it we create it. if ay_entity is None: if sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "AssetCategory": # noqa @@ -114,6 +117,13 @@ def match_shotgrid_hierarchy_in_ayon( sg_ay_dict ) + if sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Sequence": # noqa + ay_parent_entity = get_sequence_category( + entity_hub, + ay_parent_entity, + sg_ay_dict + ) + if not ay_entity: ay_entity = _create_new_entity( entity_hub, diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 7cb4a3b1..e46aecff 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -472,6 +472,80 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict): return None +def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): + """Look for existing "Sequence" folders in AYON. + + Asset categories are not entities per se in ShotGrid, they are + a "string" field in the `Asset` type, which is then used to visually + group `Asset`s; here we attempt to find any `AssetCategory` folder + type that already matches the one in ShotGrid. + + Args: + entity_hub (ayon_api.EntityHub): The project's entity hub. + parent_entity: Ayon parent entity. + sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + + """ + # just in case the asset type doesn't exist yet + log.info(f'sg_ay_dict::{sg_ay_dict}') + + sequence_category_name = slugify_string(sg_ay_dict["folder_type"]).lower() + log.info(f'sequence_category_name::{sequence_category_name}') + sequence_categories = [ + entity + for entity in parent_entity.get_children() + if ( + entity.entity_type == "folder" + and entity.folder_type == "Sequence" + and entity.name == sequence_category_name + ) + ] + log.info(f'sequence_categories::{sequence_categories}') + for sequence_category in sequence_categories: + return sequence_category + + try: + return create_sequence_category(entity_hub, parent_entity, sg_ay_dict) + except Exception: + log.error("Unable to create Sequence.", exc_info=True) + + return None + + +def create_sequence_category(entity_hub, parent_entity, sg_ay_dict): + """Create an "Sequences" folder in AYON. + + Args: + entity_hub (ayon_api.EntityHub): The project's entity hub. + parent_entity: AYON parent entity. + sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + """ + sequence_category = sg_ay_dict["folder_type"] + # asset category entity name + cat_ent_name = slugify_string(sequence_category).lower() + + sequence_category_entity = { + "label": cat_ent_name, + "name": cat_ent_name, + "attribs": { + SHOTGRID_ID_ATTRIB: slugify_string(sequence_category).lower(), + SHOTGRID_TYPE_ATTRIB: "Sequence", + }, + "parent_id": parent_entity.id, + "data": { + CUST_FIELD_CODE_ID: None, + CUST_FIELD_CODE_SYNC: None, + }, + "folder_type": "Sequence", + } + + sequence_category_entity = entity_hub.add_new_folder(**sequence_category_entity) + + log.info(f"Created Sequence: {sequence_category_entity}") + return sequence_category_entity + + + def get_or_create_sg_field( sg_session: shotgun_api3.Shotgun, sg_entity_type: str, From ac4e85e5f19de4c83ad82a66d632816986fb00f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Sep 2024 16:13:57 +0200 Subject: [PATCH 02/47] Added shot subfolder Only for shots not in sequence, these are added under `sequence/` subfolder. --- .../match_shotgrid_hierarchy_in_ayon.py | 14 ++++- services/shotgrid_common/utils.py | 63 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 62e48e99..56241cdb 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -22,6 +22,7 @@ get_sg_entities, get_asset_category, get_sequence_category, + get_shot_category, update_ay_entity_custom_attributes, ) @@ -106,8 +107,6 @@ def match_shotgrid_hierarchy_in_ayon( ay_entity = child break - log.info(f"SHOTGRID_TYPE_ATTRIB::{sg_ay_dict['attribs'].get(SHOTGRID_TYPE_ATTRIB)}") - log.info(f"ay_entity::{ay_entity}") # If we couldn't find it we create it. if ay_entity is None: if sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "AssetCategory": # noqa @@ -117,13 +116,22 @@ def match_shotgrid_hierarchy_in_ayon( sg_ay_dict ) - if sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Sequence": # noqa + if (sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Sequence" + and not ay_parent_entity): ay_parent_entity = get_sequence_category( entity_hub, ay_parent_entity, sg_ay_dict ) + if (sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Shot" + and not ay_parent_entity): + ay_parent_entity = get_shot_category( + entity_hub, + ay_parent_entity, + sg_ay_dict + ) + if not ay_entity: ay_entity = _create_new_entity( entity_hub, diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index e46aecff..ccad1d2f 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -545,6 +545,69 @@ def create_sequence_category(entity_hub, parent_entity, sg_ay_dict): return sequence_category_entity +def get_shot_category(entity_hub, parent_entity, sg_ay_dict): + """Look for existing "shot" folders in AYON. + + Args: + entity_hub (ayon_api.EntityHub): The project's entity hub. + parent_entity: Ayon parent entity. + sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + + """ + shot_category_name = slugify_string(sg_ay_dict["folder_type"]).lower() + shot_categories = [ + entity + for entity in parent_entity.get_children() + if ( + entity.entity_type == "folder" + and entity.folder_type == "Shot" + and entity.name == shot_category_name + ) + ] + + for shot_category in shot_categories: + return shot_category + + try: + return create_shot_category(entity_hub, parent_entity, sg_ay_dict) + except Exception: + log.error("Unable to create Shot.", exc_info=True) + + return None + + +def create_shot_category(entity_hub, parent_entity, sg_ay_dict): + """Create "shot" subfolder in AYON. + + Args: + entity_hub (ayon_api.EntityHub): The project's entity hub. + parent_entity: AYON parent entity. + sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + """ + shot_category = sg_ay_dict["folder_type"] + # asset category entity name + cat_ent_name = slugify_string(shot_category).lower() + + sequence_category_entity = { + "label": cat_ent_name, + "name": cat_ent_name, + "attribs": { + SHOTGRID_ID_ATTRIB: slugify_string(shot_category).lower(), + SHOTGRID_TYPE_ATTRIB: "Shot", + }, + "parent_id": parent_entity.id, + "data": { + CUST_FIELD_CODE_ID: None, + CUST_FIELD_CODE_SYNC: None, + }, + "folder_type": "Shot", + } + + shot_category_entity = entity_hub.add_new_folder(**sequence_category_entity) + + log.info(f"Created shot: {shot_category_entity}") + return shot_category_entity + def get_or_create_sg_field( sg_session: shotgun_api3.Shotgun, From 8907ec64a58bfdbbddfceb98836bd812c514d385 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 11:46:07 +0200 Subject: [PATCH 03/47] Refactored similar code --- .../match_shotgrid_hierarchy_in_ayon.py | 11 +- services/shotgrid_common/utils.py | 199 ++++++------------ 2 files changed, 75 insertions(+), 135 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 56241cdb..dd1940e7 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -109,23 +109,24 @@ def match_shotgrid_hierarchy_in_ayon( # If we couldn't find it we create it. if ay_entity is None: - if sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "AssetCategory": # noqa + parent_is_project = isinstance(ay_parent_entity, ProjectEntity) + + shotgrid_type = sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) + if shotgrid_type == "AssetCategory": ay_entity = get_asset_category( entity_hub, ay_parent_entity, sg_ay_dict ) - if (sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Sequence" - and not ay_parent_entity): + if shotgrid_type == "Sequence" and parent_is_project: ay_parent_entity = get_sequence_category( entity_hub, ay_parent_entity, sg_ay_dict ) - if (sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) == "Shot" - and not ay_parent_entity): + if shotgrid_type == "Shot" and parent_is_project: ay_parent_entity = get_shot_category( entity_hub, ay_parent_entity, diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index ccad1d2f..49e9b8f5 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -397,39 +397,6 @@ def create_sg_entities_in_ay( return sg_folder_entities, sg_steps -def create_asset_category(entity_hub, parent_entity, sg_ay_dict): - """Create an "AssetCategory" folder in AYON. - - Args: - entity_hub (ayon_api.EntityHub): The project's entity hub. - parent_entity: AYON parent entity. - sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. - """ - asset_category = sg_ay_dict["data"]["sg_asset_type"] - # asset category entity name - cat_ent_name = slugify_string(asset_category).lower() - - asset_category_entity = { - "label": asset_category, - "name": cat_ent_name, - "attribs": { - SHOTGRID_ID_ATTRIB: slugify_string(asset_category).lower(), - SHOTGRID_TYPE_ATTRIB: "AssetCategory", - }, - "parent_id": parent_entity.id, - "data": { - CUST_FIELD_CODE_ID: None, - CUST_FIELD_CODE_SYNC: None, - }, - "folder_type": "AssetCategory", - } - - asset_category_entity = entity_hub.add_new_folder(**asset_category_entity) - - log.info(f"Created AssetCategory: {asset_category_entity}") - return asset_category_entity - - def get_asset_category(entity_hub, parent_entity, sg_ay_dict): """Look for existing "AssetCategory" folders in AYON. @@ -451,25 +418,13 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict): asset_category_name = slugify_string( sg_ay_dict["data"]["sg_asset_type"]).lower() - asset_categories = [ - entity - for entity in parent_entity.get_children() - if ( - entity.entity_type == "folder" - and entity.folder_type == "AssetCategory" - and entity.name == asset_category_name - ) - ] - - for asset_category in asset_categories: - return asset_category - - try: - return create_asset_category(entity_hub, parent_entity, sg_ay_dict) - except Exception: - log.error("Unable to create AssetCategory.", exc_info=True) - - return None + return _get_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + asset_category_name, + "AssetCategory" + ) def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): @@ -486,127 +441,111 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ - # just in case the asset type doesn't exist yet - log.info(f'sg_ay_dict::{sg_ay_dict}') - - sequence_category_name = slugify_string(sg_ay_dict["folder_type"]).lower() - log.info(f'sequence_category_name::{sequence_category_name}') - sequence_categories = [ - entity - for entity in parent_entity.get_children() - if ( - entity.entity_type == "folder" - and entity.folder_type == "Sequence" - and entity.name == sequence_category_name - ) - ] - log.info(f'sequence_categories::{sequence_categories}') - for sequence_category in sequence_categories: - return sequence_category - - try: - return create_sequence_category(entity_hub, parent_entity, sg_ay_dict) - except Exception: - log.error("Unable to create Sequence.", exc_info=True) - - return None + return _get_special_category(entity_hub, parent_entity, sg_ay_dict) -def create_sequence_category(entity_hub, parent_entity, sg_ay_dict): - """Create an "Sequences" folder in AYON. +def get_shot_category(entity_hub, parent_entity, sg_ay_dict): + """Look for existing "shot" folders in AYON under "parent_entity". Args: entity_hub (ayon_api.EntityHub): The project's entity hub. - parent_entity: AYON parent entity. + parent_entity: Ayon parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. - """ - sequence_category = sg_ay_dict["folder_type"] - # asset category entity name - cat_ent_name = slugify_string(sequence_category).lower() - sequence_category_entity = { - "label": cat_ent_name, - "name": cat_ent_name, - "attribs": { - SHOTGRID_ID_ATTRIB: slugify_string(sequence_category).lower(), - SHOTGRID_TYPE_ATTRIB: "Sequence", - }, - "parent_id": parent_entity.id, - "data": { - CUST_FIELD_CODE_ID: None, - CUST_FIELD_CODE_SYNC: None, - }, - "folder_type": "Sequence", - } - - sequence_category_entity = entity_hub.add_new_folder(**sequence_category_entity) - - log.info(f"Created Sequence: {sequence_category_entity}") - return sequence_category_entity + """ + return _get_special_category(entity_hub, parent_entity, sg_ay_dict) -def get_shot_category(entity_hub, parent_entity, sg_ay_dict): - """Look for existing "shot" folders in AYON. +def _get_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + category_name=None, + folder_type=None +): + """Returns or creates special subfolders (shot|sequence|AssetCategory). Args: entity_hub (ayon_api.EntityHub): The project's entity hub. - parent_entity: Ayon parent entity. + parent_entity: AYON parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. - + category_name (Optional[str]): force this category name + folder_type (Optional[str]): force this folder type + Returns: + (FolderEntity) """ - shot_category_name = slugify_string(sg_ay_dict["folder_type"]).lower() - shot_categories = [ + if not folder_type: + folder_type = slugify_string(sg_ay_dict["folder_type"]) + if not category_name: + category_name = folder_type.lower() + categories = [ entity for entity in parent_entity.get_children() if ( entity.entity_type == "folder" - and entity.folder_type == "Shot" - and entity.name == shot_category_name + and entity.folder_type == folder_type + and entity.name == category_name ) ] - - for shot_category in shot_categories: - return shot_category + for category in categories: + return category try: - return create_shot_category(entity_hub, parent_entity, sg_ay_dict) + return _create_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + category_name, + folder_type + ) except Exception: - log.error("Unable to create Shot.", exc_info=True) + log.error(f"Unable to create {folder_type}.", exc_info=True) return None -def create_shot_category(entity_hub, parent_entity, sg_ay_dict): - """Create "shot" subfolder in AYON. +def _create_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + category_name=None, + folder_type=None +): + """Creates special subfolders (shot|sequence|AssetCategory) in AYON Args: entity_hub (ayon_api.EntityHub): The project's entity hub. parent_entity: AYON parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + category_name (Optional[str]): force this category name + folder_type (Optional[str]): force this folder type + Returns: + (FolderEntity) """ - shot_category = sg_ay_dict["folder_type"] - # asset category entity name - cat_ent_name = slugify_string(shot_category).lower() - - sequence_category_entity = { - "label": cat_ent_name, - "name": cat_ent_name, + if not folder_type: + folder_type = slugify_string(sg_ay_dict["folder_type"]) + if not category_name: + category_name = folder_type.lower() + + category_entity = { + "label": category_name, + "name": category_name, "attribs": { - SHOTGRID_ID_ATTRIB: slugify_string(shot_category).lower(), - SHOTGRID_TYPE_ATTRIB: "Shot", + SHOTGRID_ID_ATTRIB: category_name, + SHOTGRID_TYPE_ATTRIB: folder_type, }, "parent_id": parent_entity.id, "data": { CUST_FIELD_CODE_ID: None, CUST_FIELD_CODE_SYNC: None, }, - "folder_type": "Shot", + "folder_type": folder_type, } - shot_category_entity = entity_hub.add_new_folder(**sequence_category_entity) + category_entity = entity_hub.add_new_folder(**category_entity) - log.info(f"Created shot: {shot_category_entity}") - return shot_category_entity + log.info(f"Created {folder_type}: {category_entity}") + return category_entity def get_or_create_sg_field( From 2c1682f857eebe62e9c7e87f4d75f148eb02bdb6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:01:30 +0200 Subject: [PATCH 04/47] Added new folder types Special folders 'shot', 'sequence' --- services/shotgrid_common/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 49e9b8f5..a3dad19f 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -358,7 +358,11 @@ def create_sg_entities_in_ay( new_folder_types = sg_folder_entities + project_entity.folder_types # So we can have a specific folder for AssetCategory - new_folder_types.append({"name": "AssetCategory"}) + new_folder_types.extend([ + {"name": "AssetCategory"}, + {"name": "ShotCategory"}, + {"name": "SequenceCategory"}, + ]) # Make sure list items are unique new_folder_types = list({ From c12dc8a2b6a48d742e4f5acbf861e7435469225e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:02:00 +0200 Subject: [PATCH 05/47] Use new folder types for shot, sequence folders --- services/shotgrid_common/utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index a3dad19f..28edae8d 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -445,7 +445,13 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ - return _get_special_category(entity_hub, parent_entity, sg_ay_dict) + return _get_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + category_name="sequence", + folder_type="SequenceCategory" + ) def get_shot_category(entity_hub, parent_entity, sg_ay_dict): @@ -457,7 +463,13 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ - return _get_special_category(entity_hub, parent_entity, sg_ay_dict) + return _get_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + category_name="shot", + folder_type="ShotCategory" + ) def _get_special_category( From 9ee2c8c6fb16bb5c8f462e5e27f149e3e8d29cb9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:03:28 +0200 Subject: [PATCH 06/47] Skip non integer id Contain 'shot','sequence' values, not real SG entities, they are only attributes there --- .../ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py index ee691aa0..9d94bd3e 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py @@ -124,6 +124,11 @@ def match_ayon_hierarchy_in_shotgrid( continue elif sg_entity_id: # convert sg_entity_id to int if exists + if not isinstance(sg_entity_id, int): + log.warning( + f"Entity '{ay_entity.name}' not real Shotgrid entity, skipping..." # noqa + ) + continue sg_entity_id = int(sg_entity_id) if sg_entity_type == "AssetCategory": From bda159a6af6ba12c14fd08232f3c6014927abffa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:03:51 +0200 Subject: [PATCH 07/47] Skip non integer id Contain 'shot','sequence' values, not real SG entities, they are only attributes there --- .../shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py index 56de6067..2693819b 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py @@ -271,6 +271,12 @@ def remove_sg_entity_from_ayon_event( ) return + if not isinstance(sg_id, int): + log.warning( + f"Entity '{ay_entity_path}' is not Shotgrid real entity, skipping." + ) + return + sg_type = ayon_event["payload"]["entityData"]["attrib"]["shotgridType"] if not sg_type: From df445bfda55f719c1956de4b70a22c4baf36f188 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:04:22 +0200 Subject: [PATCH 08/47] Handles special folders when creating entity in SG --- .../ayon_shotgrid_hub/update_from_ayon.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py index 2693819b..72f45fc4 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py @@ -331,16 +331,19 @@ def _create_sg_entity( sg_field_name = "code" sg_step = None - # parent AssetCategory should not be created in Shotgrid - # it is only used for grouping Asset types + special_folder_types =["AssetCategory", "ShotCategory", "SequenceCategory"] + # parent special folder like AssetCategory should not be created in + # Shotgrid it is only used for grouping Asset types + is_parent_project_entity = isinstance(ay_entity.parent, ProjectEntity) if ( - isinstance(ay_entity.parent, ProjectEntity) - and ay_entity.folder_type == "AssetCategory" + is_parent_project_entity + and ay_entity.folder_type in special_folder_types ): return - elif ay_entity.parent.folder_type == "AssetCategory": + elif (not is_parent_project_entity and + ay_entity.parent.folder_type in special_folder_types): sg_parent_id = None - sg_parent_type = "AssetCategory" + sg_parent_type = ay_entity.parent.folder_type else: sg_parent_id = ay_entity.parent.attribs.get(SHOTGRID_ID_ATTRIB) sg_parent_type = ay_entity.parent.attribs.get(SHOTGRID_TYPE_ATTRIB) From 9483b83fc2f68aa939b2cad9c9ffcfd54e8ff159 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:04:56 +0200 Subject: [PATCH 09/47] Use only real parents, not grouping folders like shot --- .../ayon_shotgrid_hub/update_from_ayon.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py index 72f45fc4..69e28aad 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py @@ -422,13 +422,14 @@ def _create_sg_entity( else: data = { "project": sg_project, - parent_field: { - "type": sg_parent_type, - "id": int(sg_parent_id) - }, sg_field_name: ay_entity.name, CUST_FIELD_CODE_ID: ay_entity.id, } + if isinstance(sg_parent_id, int): + data[parent_field] = { + "type": sg_parent_type, + "id": int(sg_parent_id) + } # Fill up data with any extra attributes from Ayon we want to sync to SG data.update(get_sg_custom_attributes_data( From 0523d584b2aea2e7591963a70530d8d0235b06bf Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:05:49 +0200 Subject: [PATCH 10/47] Handles creation of shot/sequence only from events --- .../ayon_shotgrid_hub/update_from_shotgrid.py | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index e342debb..b79a432f 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -29,6 +29,8 @@ from utils import ( get_asset_category, + get_shot_category, + get_sequence_category, get_sg_entity_as_ay_dict, get_sg_entity_parent_field, update_ay_entity_custom_attributes, @@ -84,6 +86,9 @@ def create_ay_entity_from_sg_event( extra_fields.append("sg_asset_type") sg_parent_field = "sg_asset_type" + if sg_event["entity_type"] == "Shot": + sg_parent_field = "sg_sequence" + sg_ay_dict = get_sg_entity_as_ay_dict( sg_session, sg_event["entity_type"], @@ -131,14 +136,11 @@ def create_ay_entity_from_sg_event( return ay_entity - # INFO: Parent entity might not be added in SG so this needs to be handled - # with optional way. - if sg_ay_dict["data"].get(sg_parent_field) is None: - # Parent is the project - log.debug(f"ShotGrid Parent is the Project: {sg_project}") - ay_parent_entity = ayon_entity_hub.project_entity - elif ( - sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] == "Asset" + shotgrid_type = sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] + sg_parent = sg_ay_dict["data"].get(sg_parent_field) + + if ( + shotgrid_type == "Asset" and sg_ay_dict["data"].get("sg_asset_type") ): log.debug("ShotGrid Parent is an Asset category.") @@ -148,12 +150,35 @@ def create_ay_entity_from_sg_event( sg_ay_dict, ) + elif(shotgrid_type == "Sequence"): + log.info("ShotGrid Parent is an Sequence category.") + ay_parent_entity = get_sequence_category( + ayon_entity_hub, + ayon_entity_hub.project_entity, + sg_ay_dict, + ) + + elif(shotgrid_type == "Shot") and not sg_parent: + log.info("ShotGrid Parent is an Shot category.") + ay_parent_entity = get_shot_category( + ayon_entity_hub, + ayon_entity_hub.project_entity, + sg_ay_dict, + ) + + # INFO: Parent entity might not be added in SG so this needs to be handled + # with optional way. + elif sg_parent is None: + # Parent is the project + log.debug(f"ShotGrid Parent is the Project: {sg_project}") + ay_parent_entity = ayon_entity_hub.project_entity + else: # Find parent entity ID sg_parent_entity_dict = get_sg_entity_as_ay_dict( sg_session, - sg_ay_dict["data"][sg_parent_field]["type"], - sg_ay_dict["data"][sg_parent_field]["id"], + sg_parent["type"], + sg_parent["id"], project_code_field, ) From 7fc70a76c5864fbb2deba1a2f32b31d0c6b81e99 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Sep 2024 14:22:04 +0200 Subject: [PATCH 11/47] Removed storing unnecessary sg_id shot and sequence AYON folder doesn't need to store anything as SG_ID >> there is no need to check that value is convertible into int. Last place left might be actually necessary. --- .../ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py | 6 ------ .../shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py | 6 ------ services/shotgrid_common/utils.py | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py index 9d94bd3e..b27db6e8 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py @@ -123,12 +123,6 @@ def match_ayon_hierarchy_in_shotgrid( ) continue elif sg_entity_id: - # convert sg_entity_id to int if exists - if not isinstance(sg_entity_id, int): - log.warning( - f"Entity '{ay_entity.name}' not real Shotgrid entity, skipping..." # noqa - ) - continue sg_entity_id = int(sg_entity_id) if sg_entity_type == "AssetCategory": diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py index 69e28aad..fde33fb7 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py @@ -271,12 +271,6 @@ def remove_sg_entity_from_ayon_event( ) return - if not isinstance(sg_id, int): - log.warning( - f"Entity '{ay_entity_path}' is not Shotgrid real entity, skipping." - ) - return - sg_type = ayon_event["payload"]["entityData"]["attrib"]["shotgridType"] if not sg_type: diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 28edae8d..4d4ba301 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -547,7 +547,7 @@ def _create_special_category( "label": category_name, "name": category_name, "attribs": { - SHOTGRID_ID_ATTRIB: category_name, + SHOTGRID_ID_ATTRIB: None, SHOTGRID_TYPE_ATTRIB: folder_type, }, "parent_id": parent_entity.id, From 3b97998030142f018ddcece17a26ce75e0933c0b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Sep 2024 17:28:57 +0200 Subject: [PATCH 12/47] Special folders are configured in Settings Supports multilevel folder structure --- .../ayon_shotgrid_hub/__init__.py | 4 +- .../match_shotgrid_hierarchy_in_ayon.py | 12 ++- .../ayon_shotgrid_hub/update_from_shotgrid.py | 7 +- services/shotgrid_common/utils.py | 87 ++++++++++++------- 4 files changed, 71 insertions(+), 39 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py index 680d6dab..a5b989f9 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py @@ -280,7 +280,8 @@ def synchronize_projects(self, source="ayon"): self._sg, self.sg_enabled_entities, self.sg_project_code_field, - self.custom_attribs_map + self.custom_attribs_map, + self.settings ) case _: @@ -321,6 +322,7 @@ def react_to_shotgrid_event(self, sg_event_meta): self.sg_enabled_entities, self.sg_project_code_field, self.custom_attribs_map, + self.settings ) case "attribute_change": diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index dd1940e7..7183b657 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -38,7 +38,8 @@ def match_shotgrid_hierarchy_in_ayon( sg_session: shotgun_api3.Shotgun, sg_enabled_entities: List[str], project_code_field: str, - custom_attribs_map: Dict[str, str] + custom_attribs_map: Dict[str, str], + addon_settings: Dict[str, str] ): """Replicate a Shotgrid project into AYON. @@ -116,21 +117,24 @@ def match_shotgrid_hierarchy_in_ayon( ay_entity = get_asset_category( entity_hub, ay_parent_entity, - sg_ay_dict + sg_ay_dict, + addon_settings ) if shotgrid_type == "Sequence" and parent_is_project: ay_parent_entity = get_sequence_category( entity_hub, ay_parent_entity, - sg_ay_dict + sg_ay_dict, + addon_settings ) if shotgrid_type == "Shot" and parent_is_project: ay_parent_entity = get_shot_category( entity_hub, ay_parent_entity, - sg_ay_dict + sg_ay_dict, + addon_settings ) if not ay_entity: diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index b79a432f..0a5542df 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -56,7 +56,8 @@ def create_ay_entity_from_sg_event( ayon_entity_hub: ayon_api.entity_hub.EntityHub, sg_enabled_entities: List[str], project_code_field: str, - custom_attribs_map: Optional[Dict[str, str]] = None + custom_attribs_map: Optional[Dict[str, str]] = None, + addon_settings: Optional[Dict[str, str]] = None ): """Create an AYON entity from a ShotGrid Event. @@ -69,6 +70,7 @@ def create_ay_entity_from_sg_event( project_code_field (str): The Shotgrid project code field. custom_attribs_map (Optional[dict]): A dictionary that maps ShotGrid attributes to Ayon attributes. + addon_settings (Optional[dict]): A dictionary of Settings Returns: ay_entity (ayon_api.entity_hub.EntityHub.Entity): The newly @@ -148,6 +150,7 @@ def create_ay_entity_from_sg_event( ayon_entity_hub, ayon_entity_hub.project_entity, sg_ay_dict, + addon_settings ) elif(shotgrid_type == "Sequence"): @@ -156,6 +159,7 @@ def create_ay_entity_from_sg_event( ayon_entity_hub, ayon_entity_hub.project_entity, sg_ay_dict, + addon_settings ) elif(shotgrid_type == "Shot") and not sg_parent: @@ -164,6 +168,7 @@ def create_ay_entity_from_sg_event( ayon_entity_hub, ayon_entity_hub.project_entity, sg_ay_dict, + addon_settings ) # INFO: Parent entity might not be added in SG so this needs to be handled diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 4d4ba301..8ca1422c 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -401,7 +401,7 @@ def create_sg_entities_in_ay( return sg_folder_entities, sg_steps -def get_asset_category(entity_hub, parent_entity, sg_ay_dict): +def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): """Look for existing "AssetCategory" folders in AYON. Asset categories are not entities per se in ShotGrid, they are @@ -413,6 +413,7 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict): entity_hub (ayon_api.EntityHub): The project's entity hub. parent_entity: Ayon parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. + addon_settings (dict): Settings """ # just in case the asset type doesn't exist yet @@ -422,16 +423,22 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict): asset_category_name = slugify_string( sg_ay_dict["data"]["sg_asset_type"]).lower() + folder_path = (addon_settings["compatibility_settings"] + ["folder_locations"] + ["asset_folder"]) + + folder_path = "/".join([folder_path, asset_category_name]) + return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - asset_category_name, - "AssetCategory" + folder_path=folder_path, + folder_type="AssetCategory" ) -def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): +def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): """Look for existing "Sequence" folders in AYON. Asset categories are not entities per se in ShotGrid, they are @@ -445,16 +452,19 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ + folder_path = (addon_settings["compatibility_settings"] + ["folder_locations"] + ["sequence_folder"]) return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - category_name="sequence", + folder_path=folder_path, folder_type="SequenceCategory" ) -def get_shot_category(entity_hub, parent_entity, sg_ay_dict): +def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): """Look for existing "shot" folders in AYON under "parent_entity". Args: @@ -463,11 +473,14 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ + folder_path = (addon_settings["compatibility_settings"] + ["folder_locations"] + ["shot_folder"]) return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - category_name="shot", + folder_path=folder_path, folder_type="ShotCategory" ) @@ -476,7 +489,7 @@ def _get_special_category( entity_hub, parent_entity, sg_ay_dict, - category_name=None, + folder_path, folder_type=None ): """Returns or creates special subfolders (shot|sequence|AssetCategory). @@ -485,39 +498,47 @@ def _get_special_category( entity_hub (ayon_api.EntityHub): The project's entity hub. parent_entity: AYON parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. - category_name (Optional[str]): force this category name + folder_path (str): where folder should be located folder_type (Optional[str]): force this folder type Returns: (FolderEntity) """ if not folder_type: folder_type = slugify_string(sg_ay_dict["folder_type"]) - if not category_name: - category_name = folder_type.lower() - categories = [ - entity - for entity in parent_entity.get_children() - if ( - entity.entity_type == "folder" - and entity.folder_type == folder_type - and entity.name == category_name - ) - ] - for category in categories: - return category - try: - return _create_special_category( - entity_hub, - parent_entity, - sg_ay_dict, - category_name, - folder_type - ) - except Exception: - log.error(f"Unable to create {folder_type}.", exc_info=True) + if not folder_path: + return parent_entity + + folders = collections.deque(folder_path.split("/")) + + while folders: + found_folder = None + folder_name = folders.popleft() - return None + for entity in parent_entity.get_children(): + if ( + entity.entity_type == "folder" + and entity.folder_type == folder_type + and entity.name == folder_name + ): + parent_entity = entity + found_folder = entity + break + + if not found_folder: + try: + found_folder = _create_special_category( + entity_hub, + parent_entity, + sg_ay_dict, + folder_name, + folder_type + ) + parent_entity = found_folder + except Exception: + log.error(f"Unable to create {folder_type}.", exc_info=True) + + return found_folder def _create_special_category( From e02228ab99d60c9d5368c1d23d986591758c77f6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Sep 2024 18:00:18 +0200 Subject: [PATCH 13/47] Added configuration for special folder prefixes --- server/settings/main.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/server/settings/main.py b/server/settings/main.py index 64382c29..1f5c6504 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -113,6 +113,14 @@ class AttributesMappingModel(BaseSettingsModel): ) +class FolderLocationsModel(BaseSettingsModel): + """AYON folders to store separate of SG types""" + asset_folder: str = SettingsField(title="Assets", default="assets") + sequence_folder: str = SettingsField(title="Sequences", + default="sequences") + shot_folder: str = SettingsField(title="Shots", default="shots") + + class ShotgridCompatibilitySettings(BaseSettingsModel): """ Settings to define relationships between ShotGrid and AYON. """ @@ -136,6 +144,15 @@ class ShotgridCompatibilitySettings(BaseSettingsModel): ), ) + folder_locations: FolderLocationsModel = SettingsField( + title="Folder locations", + default_factory=FolderLocationsModel, + description=( + "Locations of AYON folders matching to SG types." + "Eg. where SG assets will be stored, where SG shots and sequences." + ), + ) + class ClientLoginDetailsModel(BaseSettingsModel): _layout = "expanded" @@ -218,13 +235,13 @@ class ShotgridSettings(BaseSettingsModel): shotgrid_project_code_field: str = SettingsField( default="code", title="ShotGrid Project Code field name", + disabled=True, description=( "In order to create AYON projects, we need a Project Code, you " "can specify here which field in the ShotGrid Project " "entity represents it." ), - example="sg_code", - scope=["studio"], + example="sg_code" ) enable_shotgrid_local_storage: bool = SettingsField( default=True, From 5ae2744ce9d0b78e43a765ab44512c3a5c3e4cf6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Sep 2024 15:55:21 +0200 Subject: [PATCH 14/47] Revert unwanted changes Changes not relevant to this PR. --- server/settings/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/settings/main.py b/server/settings/main.py index 1f5c6504..5cda9083 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -235,13 +235,13 @@ class ShotgridSettings(BaseSettingsModel): shotgrid_project_code_field: str = SettingsField( default="code", title="ShotGrid Project Code field name", - disabled=True, description=( "In order to create AYON projects, we need a Project Code, you " "can specify here which field in the ShotGrid Project " "entity represents it." ), - example="sg_code" + example="sg_code", + scope=["studio"], ) enable_shotgrid_local_storage: bool = SettingsField( default=True, From 16b728f11ca67b8846f6537766a074eb0720ee87 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 14 Nov 2024 15:39:49 +0100 Subject: [PATCH 15/47] Add folder types enum and implement default ShotGrid reparenting entities. - Added `folder_types_enum` to settings. - Implemented function for default ShotGrid reparenting entities. - Created models for folder reparenting in AYON. --- server/settings/main.py | 94 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 5cda9083..bf2023c9 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -1,6 +1,10 @@ from ayon_server.entities.core.attrib import attribute_library from ayon_server.settings import BaseSettingsModel, SettingsField -from ayon_server.settings.enum import secrets_enum, anatomy_presets_enum +from ayon_server.settings.enum import ( + secrets_enum, + anatomy_presets_enum, + folder_types_enum +) def default_shotgrid_entities(): @@ -30,6 +34,17 @@ def default_shotgrid_enabled_entities(): ] +def default_shotgrid_reparenting_entities(): + """The entity types in ShotGrid that are enabled by default in AYON.""" + return [ + "Episode", + "Sequence", + "Shot", + "AssetCategory", + "Asset", + ] + + def get_default_folder_attributes(): """Get AYON's Folder attributes @@ -121,6 +136,78 @@ class FolderLocationsModel(BaseSettingsModel): shot_folder: str = SettingsField(title="Shots", default="shots") +class FolderReparentingParentsModel(BaseSettingsModel): + sg_entity_type: str = SettingsField( + "Asset", + title="ShotGrid Entity Type", + enum_resolver=( + lambda: default_shotgrid_reparenting_entities() + ["< None >"] + ), + description=( + "Type of the ShotGrid entity to parent in AYON. " + "' < None > ' means no SG type and can be used just as folder." + ), + ) + folder_type: str = SettingsField( + "asset", + title="Parent Folder Type", + enum_resolver=folder_types_enum, + description="Type of the parent folder in AYON", + ) + folder_name: str = SettingsField( + "assets", + title="Parent Folder Name", + description="Name of the parent folder in AYON", + ) + + +class FolderReparentingPresetsModel(BaseSettingsModel): + + reparenting_type: str = SettingsField( + "root_relocate", + title="Re-parenting Type", + enum_resolver=lambda: [ + {"value": "root_relocate", "label": "Root relocation"}, + {"value": "type_grouping", "label": "Type grouping"}, + ], + description="Type of the parenting", + ) + filter_by_sg_entity_type: str = SettingsField( + "Asset", + title="Filter by ShotGrid Entity Type", + enum_resolver=default_shotgrid_reparenting_entities, + description=("Type of the ShotGrid entity to filter preset on."), + ) + parents: list[FolderReparentingParentsModel] = SettingsField( + title="Parents", + default_factory=list, + description=( + "List of parent folders. If empty default behavior will be used. " + "The order of the parents from top to bottom is important." + "Within 'Root relocation' type the first parent will be " + "the root folder." + ), + ) + + +class FolderReparentingModel(BaseSettingsModel): + """Re-parent folders for AYON folders matching to SG types""" + enabled: bool = SettingsField( + False, + title="Enabled", + description="Enable or disable the re-parenting", + ) + + presets: list[FolderReparentingPresetsModel] = SettingsField( + title="Presets", + default_factory=list, + description=( + "List of presets for re-parenting. " + "If empty default behavior will be used." + ), + ) + + class ShotgridCompatibilitySettings(BaseSettingsModel): """ Settings to define relationships between ShotGrid and AYON. """ @@ -152,6 +239,11 @@ class ShotgridCompatibilitySettings(BaseSettingsModel): "Eg. where SG assets will be stored, where SG shots and sequences." ), ) + folder_parenting: FolderReparentingModel = SettingsField( + title="Folder re-parenting", + default_factory=FolderReparentingModel, + description=("Parent folders for AYON folders matching to SG types."), + ) class ClientLoginDetailsModel(BaseSettingsModel): From a2c56b209371443c74c925de5684da9e054760e4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 14 Nov 2024 15:41:07 +0100 Subject: [PATCH 16/47] removing settings --- server/settings/main.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/server/settings/main.py b/server/settings/main.py index bf2023c9..80fe4ee2 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -127,15 +127,6 @@ class AttributesMappingModel(BaseSettingsModel): enum_resolver=default_shotgrid_entities ) - -class FolderLocationsModel(BaseSettingsModel): - """AYON folders to store separate of SG types""" - asset_folder: str = SettingsField(title="Assets", default="assets") - sequence_folder: str = SettingsField(title="Sequences", - default="sequences") - shot_folder: str = SettingsField(title="Shots", default="shots") - - class FolderReparentingParentsModel(BaseSettingsModel): sg_entity_type: str = SettingsField( "Asset", @@ -231,14 +222,6 @@ class ShotgridCompatibilitySettings(BaseSettingsModel): ), ) - folder_locations: FolderLocationsModel = SettingsField( - title="Folder locations", - default_factory=FolderLocationsModel, - description=( - "Locations of AYON folders matching to SG types." - "Eg. where SG assets will be stored, where SG shots and sequences." - ), - ) folder_parenting: FolderReparentingModel = SettingsField( title="Folder re-parenting", default_factory=FolderReparentingModel, From eee98f3d6f9f9dd8071ef5af2eee57804774ca47 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 14 Nov 2024 15:44:03 +0100 Subject: [PATCH 17/47] Refactor folder path retrieval for asset, sequence, and shot categories Update folder path retrieval to align with new re-parenting structure from `folder_parenting` presets. --- services/shotgrid_common/utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index d0739204..5e576597 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -424,6 +424,9 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): asset_category_name = slugify_string( sg_ay_dict["data"]["sg_asset_type"]).lower() + # TODO: this needs to be changed to implement the new re parenting + # structure from `folder_parenting` presets + # addon_settings["compatibility_settings"]["folder_parenting"] folder_path = (addon_settings["compatibility_settings"] ["folder_locations"] ["asset_folder"]) @@ -453,6 +456,9 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings) sg_ay_dict (dict): The ShotGrid entity ready for AYON consumption. """ + # TODO: this needs to be changed to implement the new re parenting + # structure from `folder_parenting` presets + # addon_settings["compatibility_settings"]["folder_parenting"] folder_path = (addon_settings["compatibility_settings"] ["folder_locations"] ["sequence_folder"]) @@ -474,6 +480,9 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ + # TODO: this needs to be changed to implement the new re parenting + # structure from `folder_parenting` presets + # addon_settings["compatibility_settings"]["folder_parenting"] folder_path = (addon_settings["compatibility_settings"] ["folder_locations"] ["shot_folder"]) From 39e880eddab095fd90e5cac6ca558421c1cd06b2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Nov 2024 16:36:13 +0100 Subject: [PATCH 18/47] Do not put shots in sequence to different folder type Previously shots without sequence were in `shots`, shots with sequence under `sequences`. --- .../shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 75776d04..8d9ba02d 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -228,7 +228,7 @@ def _get_ayon_parent_entity(ayon_entity_hub, project_code_field, sg_ay_dict, addon_settings ) - elif(shotgrid_type == "Shot") and not sg_parent: + elif(shotgrid_type == "Shot"): log.info("ShotGrid Parent is an Shot category.") ay_parent_entity = get_shot_category( ayon_entity_hub, From 0edc9036f050e07c69ba8c47b29868827f357462 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Nov 2024 16:36:53 +0100 Subject: [PATCH 19/47] Removed unnecessary code Will be resolved lower if necessary --- .../ayon_shotgrid_hub/update_from_shotgrid.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 8d9ba02d..d87019d2 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -201,13 +201,7 @@ def _get_ayon_parent_entity(ayon_entity_hub, project_code_field, sg_ay_dict, shotgrid_type = sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] sg_parent = sg_ay_dict["data"].get(sg_parent_field) - # INFO: Parent entity might not be added in SG so this needs to be handled - # with optional way. - if sg_ay_dict["data"].get(sg_parent_field) is None: - # Parent is the project - log.debug(f"ShotGrid Parent is the Project: {sg_project}") - ay_parent_entity = ayon_entity_hub.project_entity - elif ( + if ( shotgrid_type == "Asset" and sg_ay_dict["data"].get("sg_asset_type") ): From ea292b10e3ef1cf99d87cfa263ed312bb13db816 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Nov 2024 16:46:20 +0100 Subject: [PATCH 20/47] Added docstring --- .../ayon_shotgrid_hub/update_from_shotgrid.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index d87019d2..a3ca8d3a 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -196,8 +196,31 @@ def create_ay_entity_from_sg_event( return ay_entity -def _get_ayon_parent_entity(ayon_entity_hub, project_code_field, sg_ay_dict, - sg_parent_field, sg_project, sg_session, addon_settings): +def _get_ayon_parent_entity( + ayon_entity_hub, + project_code_field, + sg_ay_dict, + sg_parent_field, + sg_project, + sg_session, + addon_settings +): + """Tries to find parent entity in AYON + + Args: + ayon_entity_hub (ayon_api.entity_hub.EntityHub): The AYON EntityHub. + project_code_field (str): The Shotgrid project code field. + sg_ay_dict (dict): The ShotGrid entity ready for AYON consumption.: + sg_parent_field (str): 'project'|'sequence' + sg_project (dict): The ShotGrid project. + sg_session (shotgun_api3.Shotgun): The ShotGrid API session. + addon_settings (Optional[dict]): A dictionary of Settings. Used to + query location of custom folders (`shots`, `sequences`) + + Returns: + ay_entity (ayon_api.entity_hub.EntityHub.Entity): + FolderEntity|ProjectEntity + """ shotgrid_type = sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] sg_parent = sg_ay_dict["data"].get(sg_parent_field) From dc1196ac96a9e9bcc6a611a7513d4ec21c110d5a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Nov 2024 17:30:44 +0100 Subject: [PATCH 21/47] Fix import --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index ac430f1b..cf853471 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -14,6 +14,8 @@ SHOTGRID_TYPE_ATTRIB, ) +from ayon_api.entity_hub import ProjectEntity + from utils import ( create_new_ayon_entity, get_sg_entities, From ec7608a061df075950d3b1e140adec8ab4e2408f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 15 Nov 2024 15:11:10 +0100 Subject: [PATCH 22/47] Remove AssetCategory, update folder naming conventions, and add re-parenting models for Root relocation and Type grouping. --- server/settings/main.py | 66 +++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/server/settings/main.py b/server/settings/main.py index 80fe4ee2..40035507 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -40,7 +40,6 @@ def default_shotgrid_reparenting_entities(): "Episode", "Sequence", "Shot", - "AssetCategory", "Asset", ] @@ -127,42 +126,27 @@ class AttributesMappingModel(BaseSettingsModel): enum_resolver=default_shotgrid_entities ) + class FolderReparentingParentsModel(BaseSettingsModel): - sg_entity_type: str = SettingsField( - "Asset", - title="ShotGrid Entity Type", - enum_resolver=( - lambda: default_shotgrid_reparenting_entities() + ["< None >"] - ), - description=( - "Type of the ShotGrid entity to parent in AYON. " - "' < None > ' means no SG type and can be used just as folder." - ), - ) folder_type: str = SettingsField( "asset", - title="Parent Folder Type", + title="Parent Ayon Folder Type", enum_resolver=folder_types_enum, description="Type of the parent folder in AYON", ) folder_name: str = SettingsField( "assets", - title="Parent Folder Name", - description="Name of the parent folder in AYON", + title="Parent Ayon Folder Name", + description=( + "Name of the parent folder in AYON. Anatomy presets can be used." + "`sg_` prefix can be used to refer to ShotGrid entities. Example: " + "`{sg_asset[type]}` will be replaced with the ShotGrid Asset Type." + ), ) class FolderReparentingPresetsModel(BaseSettingsModel): - reparenting_type: str = SettingsField( - "root_relocate", - title="Re-parenting Type", - enum_resolver=lambda: [ - {"value": "root_relocate", "label": "Root relocation"}, - {"value": "type_grouping", "label": "Type grouping"}, - ], - description="Type of the parenting", - ) filter_by_sg_entity_type: str = SettingsField( "Asset", title="Filter by ShotGrid Entity Type", @@ -181,8 +165,26 @@ class FolderReparentingPresetsModel(BaseSettingsModel): ) -class FolderReparentingModel(BaseSettingsModel): - """Re-parent folders for AYON folders matching to SG types""" +class FolderReparentingRelocateModel(BaseSettingsModel): + """Re-parent folders with Root relocation""" + enabled: bool = SettingsField( + False, + title="Enabled", + description="Enable or disable the re-parenting", + ) + + presets: list[FolderReparentingPresetsModel] = SettingsField( + title="Presets", + default_factory=list, + description=( + "List of presets for re-parenting. " + "If empty default behavior will be used." + ), + ) + + +class FolderReparentingTypeGroupingModel(BaseSettingsModel): + """Re-parent folders with Type grouping""" enabled: bool = SettingsField( False, title="Enabled", @@ -199,6 +201,18 @@ class FolderReparentingModel(BaseSettingsModel): ) +class FolderReparentingModel(BaseSettingsModel): + + root_relocate: FolderReparentingRelocateModel = SettingsField( + default_factory=FolderReparentingRelocateModel, + title="Root relocation", + ) + + type_grouping: FolderReparentingTypeGroupingModel = SettingsField( + default_factory=FolderReparentingTypeGroupingModel, + title="Type grouping", + ) + class ShotgridCompatibilitySettings(BaseSettingsModel): """ Settings to define relationships between ShotGrid and AYON. """ From e97919da28aa7710af161f1dcac1fd87a5b4aa37 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Nov 2024 16:48:50 +0100 Subject: [PATCH 23/47] Fix use project setting Previously used only Studio settings --- services/shotgrid_common/ayon_shotgrid_hub/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py index 88c04b44..b09d6917 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/__init__.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/__init__.py @@ -82,7 +82,7 @@ def __init__(self, custom_attribs_types=None, sg_enabled_entities=None, ): - self.settings = ayon_api.get_service_addon_settings() + self.settings = ayon_api.get_service_addon_settings(project_name) self._sg = sg_connection From e697216bffe154157eb8960da2d699d7103b0a5c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Nov 2024 12:48:52 +0100 Subject: [PATCH 24/47] Implemented type_grouping use case This allows to separate each Asset/Shot/Sequence into separate hiearchies. --- .../match_shotgrid_hierarchy_in_ayon.py | 2 +- services/shotgrid_common/utils.py | 99 ++++++++++--------- 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index cf853471..f75eb6a4 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -113,7 +113,7 @@ def match_shotgrid_hierarchy_in_ayon( shotgrid_type = sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) if shotgrid_type == "AssetCategory": - ay_entity = get_asset_category( + ay_parent_entity = get_asset_category( entity_hub, ay_parent_entity, sg_ay_dict, diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 5e576597..808fc36f 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -417,28 +417,15 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. addon_settings (dict): Settings """ - # just in case the asset type doesn't exist yet - if not sg_ay_dict["data"].get("sg_asset_type"): - sg_ay_dict["data"]["sg_asset_type"] = sg_ay_dict["name"] - - asset_category_name = slugify_string( - sg_ay_dict["data"]["sg_asset_type"]).lower() - - # TODO: this needs to be changed to implement the new re parenting - # structure from `folder_parenting` presets - # addon_settings["compatibility_settings"]["folder_parenting"] - folder_path = (addon_settings["compatibility_settings"] - ["folder_locations"] - ["asset_folder"]) - - folder_path = "/".join([folder_path, asset_category_name]) + transfer_type = "type_grouping" + folders_and_types = _get_parents_and_types( + addon_settings, transfer_type, "Asset") return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - folder_path=folder_path, - folder_type="AssetCategory" + folders_and_types=folders_and_types ) @@ -456,18 +443,14 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings) sg_ay_dict (dict): The ShotGrid entity ready for AYON consumption. """ - # TODO: this needs to be changed to implement the new re parenting - # structure from `folder_parenting` presets - # addon_settings["compatibility_settings"]["folder_parenting"] - folder_path = (addon_settings["compatibility_settings"] - ["folder_locations"] - ["sequence_folder"]) + transfer_type = "type_grouping" + folders_and_types = _get_parents_and_types( + addon_settings, transfer_type, "Sequence") return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - folder_path=folder_path, - folder_type="SequenceCategory" + folders_and_types=folders_and_types ) @@ -480,18 +463,14 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ - # TODO: this needs to be changed to implement the new re parenting - # structure from `folder_parenting` presets - # addon_settings["compatibility_settings"]["folder_parenting"] - folder_path = (addon_settings["compatibility_settings"] - ["folder_locations"] - ["shot_folder"]) + transfer_type = "type_grouping" + folders_and_types = _get_parents_and_types( + addon_settings, transfer_type, "Shot") return _get_special_category( entity_hub, parent_entity, sg_ay_dict, - folder_path=folder_path, - folder_type="ShotCategory" + folders_and_types=folders_and_types ) @@ -499,8 +478,7 @@ def _get_special_category( entity_hub, parent_entity, sg_ay_dict, - folder_path, - folder_type=None + folders_and_types=None ): """Returns or creates special subfolders (shot|sequence|AssetCategory). @@ -508,27 +486,24 @@ def _get_special_category( entity_hub (ayon_api.EntityHub): The project's entity hub. parent_entity: AYON parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. - folder_path (str): where folder should be located - folder_type (Optional[str]): force this folder type + folders_and_types (deque(([str], [str]))) Returns: (FolderEntity) """ - if not folder_type: - folder_type = slugify_string(sg_ay_dict["folder_type"]) - - if not folder_path: - return parent_entity - - folders = collections.deque(folder_path.split("/")) - - while folders: + while folders_and_types: found_folder = None - folder_name = folders.popleft() + parent = folders_and_types.popleft() + folder_name, folder_type = parent + placeholders = {"shotgrid_type": sg_ay_dict["attribs"]["shotgridType"]} + try: + folder_name = folder_name.format(**placeholders) + except KeyError: + # ignore superfluous placeholders + pass for entity in parent_entity.get_children(): if ( - entity.entity_type == "folder" - and entity.folder_type == folder_type + entity.folder_type == folder_type and entity.name == folder_name ): parent_entity = entity @@ -595,6 +570,32 @@ def _create_special_category( return category_entity +def _get_parents_and_types(addon_settings, transfer_type, sg_entity_type): + parents_presets = (addon_settings["compatibility_settings"] + ["folder_parenting"] + [transfer_type]) + + if not parents_presets["enabled"]: + return + + found_preset = None + for preset in parents_presets["presets"]: + if preset["filter_by_sg_entity_type"] != sg_entity_type: + continue + found_preset = preset + + if not found_preset: + return + + folders_and_types = collections.deque() + for parent in found_preset["parents"]: + folders_and_types.append( + (parent["folder_name"], parent["folder_type"]) + ) + + return folders_and_types + + def get_or_create_sg_field( sg_session: shotgun_api3.Shotgun, sg_entity_type: str, From 986e646a5d5863299fa60e0384d9b0638bb1afce Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:54:03 +0100 Subject: [PATCH 25/47] Fix access to local variable --- .../shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 24d822cc..8ee365ef 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -219,6 +219,7 @@ def _get_ayon_parent_entity( """ shotgrid_type = sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] sg_parent = sg_ay_dict["data"].get(sg_parent_field) + ay_parent_entity = None if ( shotgrid_type == "Asset" From dbb0f16781c877ed3a30cf455c688b0e1f8ac1ac Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:54:48 +0100 Subject: [PATCH 26/47] Fix non parented entities --- .../shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 8ee365ef..136a76c9 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -137,6 +137,10 @@ def create_ay_entity_from_sg_event( addon_settings ) + sg_parent = sg_ay_dict["data"].get(sg_parent_field) + if not ay_parent_entity and not sg_parent: + ay_parent_entity = ayon_entity_hub.project_entity + if not ay_parent_entity: sg_ay_parent_dict = get_sg_entity_as_ay_dict( sg_session, From 59feada624beeaa26ce9194a581b6f415cae72b7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:55:50 +0100 Subject: [PATCH 27/47] Fix cases where no parent found --- .../ayon_shotgrid_hub/update_from_shotgrid.py | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 136a76c9..67f5d902 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -255,33 +255,34 @@ def _get_ayon_parent_entity( addon_settings ) - # INFO: Parent entity might not be added in SG so this needs to be handled - # with optional way. - elif sg_parent is None: - # Parent is the project - log.debug(f"ShotGrid Parent is the Project: {sg_project}") - ay_parent_entity = ayon_entity_hub.project_entity + if ay_parent_entity is None: + # INFO: Parent entity might not be added in SG so this needs to + # be handled with optional way. + if sg_parent is None: + # Parent is the project + log.debug(f"ShotGrid Parent is the Project: {sg_project}") + ay_parent_entity = ayon_entity_hub.project_entity - else: - # Find parent entity ID - sg_parent_entity_dict = get_sg_entity_as_ay_dict( - sg_session, - sg_parent["type"], - sg_parent["id"], - project_code_field, - ) + else: + # Find parent entity ID + sg_parent_entity_dict = get_sg_entity_as_ay_dict( + sg_session, + sg_parent["type"], + sg_parent["id"], + project_code_field, + ) - log.debug(f"ShotGrid Parent entity: {sg_parent_entity_dict}") - ay_parent_entity = ayon_entity_hub.get_or_query_entity_by_id( - sg_parent_entity_dict["data"].get(CUST_FIELD_CODE_ID), - [ - ( - "task" - if sg_parent_entity_dict["type"] == "task" - else "folder" - ) - ], - ) + log.debug(f"ShotGrid Parent entity: {sg_parent_entity_dict}") + ay_parent_entity = ayon_entity_hub.get_or_query_entity_by_id( + sg_parent_entity_dict["data"].get(CUST_FIELD_CODE_ID), + [ + ( + "task" + if sg_parent_entity_dict["type"] == "task" + else "folder" + ) + ], + ) return ay_parent_entity From b08ed553dc9c48473aa8c1cb062fab0a3931c11f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:57:49 +0100 Subject: [PATCH 28/47] Introduced parenting transfer workflows 2 workflows - root relocate - put hierarchy to AYON only additional folders - type grouping - all shots/sequences/assets together separately --- services/shotgrid_common/constants.py | 6 ++++ services/shotgrid_common/utils.py | 48 ++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/services/shotgrid_common/constants.py b/services/shotgrid_common/constants.py index 54157b5f..cd1d5b88 100644 --- a/services/shotgrid_common/constants.py +++ b/services/shotgrid_common/constants.py @@ -156,3 +156,9 @@ "session_uuid", "created_at", ] + + +class FOLDER_REPARENTING_TYPE: + ROOT_RELOCATE = "root_relocate" + TYPE_GROUPING = "type_grouping" + diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 808fc36f..ac6a61d7 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -15,6 +15,7 @@ SG_PROJECT_ATTRS, SHOTGRID_ID_ATTRIB, SHOTGRID_TYPE_ATTRIB, + FOLDER_REPARENTING_TYPE ) from ayon_api.entity_hub import ( @@ -417,7 +418,7 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. addon_settings (dict): Settings """ - transfer_type = "type_grouping" + transfer_type = _get_parenting_transfer_type(addon_settings) folders_and_types = _get_parents_and_types( addon_settings, transfer_type, "Asset") @@ -443,7 +444,7 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings) sg_ay_dict (dict): The ShotGrid entity ready for AYON consumption. """ - transfer_type = "type_grouping" + transfer_type = _get_parenting_transfer_type(addon_settings) folders_and_types = _get_parents_and_types( addon_settings, transfer_type, "Sequence") return _get_special_category( @@ -463,9 +464,24 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. """ - transfer_type = "type_grouping" + sg_entity_type = "Shot" + transfer_type = _get_parenting_transfer_type(addon_settings) + parent_sequence = None + if transfer_type == FOLDER_REPARENTING_TYPE.ROOT_RELOCATE: + # TODO what if non standard mapping of shots + sg_parent = sg_ay_dict["data"].get("sg_sequence") + if sg_parent: + sg_entity_type = "Sequence" # look for custom parents of Sequence + parent_sequence = (sg_parent["name"], sg_parent["type"]) + else: + return entity_hub.project_entity + folders_and_types = _get_parents_and_types( - addon_settings, transfer_type, "Shot") + addon_settings, transfer_type, sg_entity_type) + + if parent_sequence: + folders_and_types.append(parent_sequence) + return _get_special_category( entity_hub, parent_entity, @@ -595,6 +611,30 @@ def _get_parents_and_types(addon_settings, transfer_type, sg_entity_type): return folders_and_types +def _get_parenting_transfer_type(addon_settings): + """Select which workflow is enabled. + + TODO refactor to single object with type selector not two object + current implementation will be only for development and easier testing + + Returns: + (str): + "root_relocate" - keep SG hierachy, put in additional AYON folder + "type_grouping" - separate SG objects into AYON folders + """ + folder_parenting = (addon_settings["compatibility_settings"] + ["folder_parenting"]) + + enabled_transfer_type = None + for transfer_type, transfer_type_info in folder_parenting.items(): + if transfer_type_info["enabled"]: + if enabled_transfer_type: + raise RuntimeError("Both types cannot be enabled. Please " + "disable one.") + enabled_transfer_type = transfer_type + + return enabled_transfer_type + def get_or_create_sg_field( sg_session: shotgun_api3.Shotgun, From 7f17ded053c4917813cf57e2ac0e5b8afdefe97a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:58:46 +0100 Subject: [PATCH 29/47] Renamed placeholder value Dont have access to `sg_asset` in code --- server/settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 40035507..f1286a92 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -140,7 +140,7 @@ class FolderReparentingParentsModel(BaseSettingsModel): description=( "Name of the parent folder in AYON. Anatomy presets can be used." "`sg_` prefix can be used to refer to ShotGrid entities. Example: " - "`{sg_asset[type]}` will be replaced with the ShotGrid Asset Type." + "`{shotgrid_type}` will be replaced with the ShotGrid Asset Type." ), ) From c1a2e931fe377c77a3833081d4a5a572c2801981 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:59:16 +0100 Subject: [PATCH 30/47] Implemented pulling name from AssetCategory as folder name --- services/shotgrid_common/utils.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index ac6a61d7..7846135f 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -506,11 +506,14 @@ def _get_special_category( Returns: (FolderEntity) """ + found_folder = None + + placeholders = _get_placeholders(sg_ay_dict) while folders_and_types: found_folder = None parent = folders_and_types.popleft() folder_name, folder_type = parent - placeholders = {"shotgrid_type": sg_ay_dict["attribs"]["shotgridType"]} + try: folder_name = folder_name.format(**placeholders) except KeyError: @@ -542,6 +545,27 @@ def _get_special_category( return found_folder +def _get_placeholders(sg_ay_dict): + """Returns dynamic values for placeholders used in folder name. + + Currently implemented only `shotgrid_type` which points to name of + AssetCategory + TODO probably refactor `shotgrid_type` to different name if ShotCategory + removed + """ + placeholders = {} + # regular update process + sg_asset_type = sg_ay_dict["data"].get("sg_asset_type") + if sg_asset_type: + placeholders["shotgrid_type"] = sg_asset_type.lower() + else: + # AssetCategory for match_ + sg_asset_type = sg_ay_dict["attribs"].get("shotgridType") + if sg_asset_type: + placeholders["shotgrid_type"] = sg_ay_dict["name"] + return placeholders + + def _create_special_category( entity_hub, parent_entity, @@ -599,6 +623,7 @@ def _get_parents_and_types(addon_settings, transfer_type, sg_entity_type): if preset["filter_by_sg_entity_type"] != sg_entity_type: continue found_preset = preset + break if not found_preset: return From 5e263df63cc80b66ca36ce77154acadc64e595c8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 13:59:40 +0100 Subject: [PATCH 31/47] Fixes for parent transfer --- .../match_shotgrid_hierarchy_in_ayon.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index f75eb6a4..d51e92ed 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -101,16 +101,6 @@ def match_shotgrid_hierarchy_in_ayon( # If we haven't found the ay_entity by its id, check by its name # to avoid creating duplicates and erroring out if ay_entity is None: - name = slugify_string(sg_ay_dict["name"]) - for child in ay_parent_entity.children: - if child.name.lower() == name.lower(): - ay_entity = child - break - - # If we couldn't find it we create it. - if ay_entity is None: - parent_is_project = isinstance(ay_parent_entity, ProjectEntity) - shotgrid_type = sg_ay_dict["attribs"].get(SHOTGRID_TYPE_ATTRIB) if shotgrid_type == "AssetCategory": ay_parent_entity = get_asset_category( @@ -119,8 +109,12 @@ def match_shotgrid_hierarchy_in_ayon( sg_ay_dict, addon_settings ) + # If the entity has children, add it to the deck + for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): + sg_ay_dicts_deck.append((ay_parent_entity, sg_child_id)) + continue - if shotgrid_type == "Sequence" and parent_is_project: + elif shotgrid_type == "Sequence": ay_parent_entity = get_sequence_category( entity_hub, ay_parent_entity, @@ -128,7 +122,7 @@ def match_shotgrid_hierarchy_in_ayon( addon_settings ) - if shotgrid_type == "Shot" and parent_is_project: + elif shotgrid_type == "Shot": ay_parent_entity = get_shot_category( entity_hub, ay_parent_entity, @@ -136,6 +130,14 @@ def match_shotgrid_hierarchy_in_ayon( addon_settings ) + name = slugify_string(sg_ay_dict["name"]) + for child in ay_parent_entity.children: + if child.name.lower() == name.lower(): + ay_entity = child + break + + # If we couldn't find it we create it. + if ay_entity is None: if not ay_entity: ay_entity = create_new_ayon_entity( sg_session, @@ -197,10 +199,6 @@ def match_shotgrid_hierarchy_in_ayon( ) ay_entity.data.update(update_data) - # If the entity has children, add it to the deck - for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): - sg_ay_dicts_deck.append((ay_entity, sg_child_id)) - # Sync project attributes from Shotgrid to AYON entity_hub.project_entity.attribs.set( SHOTGRID_ID_ATTRIB, From e7865c85c3c148c574047d0138513af2c854f59e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 15:12:48 +0100 Subject: [PATCH 32/47] Added parent sequence info to shot sg_ay_dict --- services/shotgrid_common/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 7846135f..3ef2c7b6 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -162,6 +162,10 @@ def _sg_to_ay_dict( CUST_FIELD_CODE_ID: sg_entity.get(CUST_FIELD_CODE_ID), } } + + if sg_entity["type"] == "Shot" and sg_entity.get("sg_sequence"): + sg_ay_dict["data"]["sg_sequence"] = sg_entity["sg_sequence"] + if custom_attribs_map: for ay_attrib, sg_attrib in custom_attribs_map.items(): sg_value = sg_entity.get(sg_attrib) or sg_entity.get(f"sg_{sg_attrib}") From 0c690cce9a1c57522321205486e6424cb8ee4005 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Nov 2024 15:13:38 +0100 Subject: [PATCH 33/47] Fix missing Task entities Code was removed during refactor erroneously --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index d51e92ed..113f0071 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -199,6 +199,10 @@ def match_shotgrid_hierarchy_in_ayon( ) ay_entity.data.update(update_data) + # If the entity has children, add it to the deck + for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): + sg_ay_dicts_deck.append((ay_parent_entity, sg_child_id)) + # Sync project attributes from Shotgrid to AYON entity_hub.project_entity.attribs.set( SHOTGRID_ID_ATTRIB, From 15eb5a037e626b27c0c8323d80823db1dcd535a6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 13:24:34 +0100 Subject: [PATCH 34/47] Extracted _sync_project_attributes --- .../match_shotgrid_hierarchy_in_ayon.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 113f0071..4d46b6ed 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -203,7 +203,39 @@ def match_shotgrid_hierarchy_in_ayon( for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): sg_ay_dicts_deck.append((ay_parent_entity, sg_child_id)) - # Sync project attributes from Shotgrid to AYON + _sync_project_attributes(entity_hub, custom_attribs_map, sg_project) + + try: + entity_hub.commit_changes() + except Exception: + log.error( + "Unable to commit all entities to AYON!", exc_info=True) + + log.info( + "Processed entities successfully!. " + f"Amount of entities: {len(processed_ids)}" + ) + + # Update Shotgrid project with AYON ID and sync status + sg_session.update( + "Project", + sg_project["id"], + { + CUST_FIELD_CODE_ID: entity_hub.project_entity.id, + CUST_FIELD_CODE_SYNC: sg_project_sync_status + } + ) + + +def _sync_project_attributes(entity_hub, custom_attribs_map, sg_project): + """Sync project attributes from Shotgrid to AYON + + Args: + entity_hub (ayon_api.entity_hub.EntityHub): The AYON EntityHub. + custom_attribs_map (dict): A dictionary mapping AYON attributes to + Shotgrid fields, without the `sg_` prefix. + sg_project (dict): The Shotgrid project. + """ entity_hub.project_entity.attribs.set( SHOTGRID_ID_ATTRIB, sg_project["id"] @@ -214,7 +246,7 @@ def match_shotgrid_hierarchy_in_ayon( ) for ay_attrib, sg_attrib in custom_attribs_map.items(): attrib_value = sg_project.get(sg_attrib) \ - or sg_project.get(f"sg_{sg_attrib}") + or sg_project.get(f"sg_{sg_attrib}") if attrib_value is None: continue @@ -229,27 +261,6 @@ def match_shotgrid_hierarchy_in_ayon( attrib_value ) - try: - entity_hub.commit_changes() - except Exception: - log.error( - "Unable to commit all entities to AYON!", exc_info=True) - - log.info( - "Processed entities successfully!. " - f"Amount of entities: {len(processed_ids)}" - ) - - # Update Shotgrid project with AYON ID and sync status - sg_session.update( - "Project", - sg_project["id"], - { - CUST_FIELD_CODE_ID: entity_hub.project_entity.id, - CUST_FIELD_CODE_SYNC: sg_project_sync_status - } - ) - def _add_tags(project_name, tags): """Add tags to AYON project. From af1ad652953e6c756bb1fb171cbd9b171fbba7df Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 13:43:49 +0100 Subject: [PATCH 35/47] Extracted _update_sg_entity Pushes/updates AYON entity id to matching SG entity --- .../match_shotgrid_hierarchy_in_ayon.py | 84 ++++++++++++------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 4d46b6ed..5aa231f4 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -170,34 +170,15 @@ def match_shotgrid_hierarchy_in_ayon( log.error(f"Entity {sg_ay_dict} not found in AYON.") continue - # Update SG entity with new created data - sg_ay_dict["data"][CUST_FIELD_CODE_ID] = ay_entity.id - sg_ay_dicts[sg_entity_id] = sg_ay_dict - - # If the entity is not a "Folder" or "AssetCategory" we update the - # entity ID and sync status in Shotgrid and AYON - if ( - sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] not in [ - "Folder", "AssetCategory" - ] - and ( - sg_ay_dict["data"][CUST_FIELD_CODE_ID] != ay_entity.id - or sg_ay_dict["data"][CUST_FIELD_CODE_SYNC] != sg_entity_sync_status # noqa - ) - ): - log.debug( - "Updating AYON entity ID and sync status in SG and AYON") - update_data = { - CUST_FIELD_CODE_ID: ay_entity.id, - CUST_FIELD_CODE_SYNC: sg_entity_sync_status - } - # Update Shotgrid entity with AYON ID and sync status - sg_session.update( - sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB], - sg_entity_id, - update_data - ) - ay_entity.data.update(update_data) + # pass AYON id to SG + _update_sg_entity( + ay_entity, + sg_ay_dict, + sg_ay_dicts, + sg_entity_id, + sg_entity_sync_status, + sg_session + ) # If the entity has children, add it to the deck for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): @@ -227,6 +208,53 @@ def match_shotgrid_hierarchy_in_ayon( ) +def _update_sg_entity( + ay_entity, + sg_ay_dict, + sg_ay_dicts, + sg_entity_id, + sg_entity_sync_status, + sg_session +): + """Update SG entity with new created data id + + Args: + ay_entity (ayon_api.entity_hub.EntityHub.Entity): new AYON entity + sg_ay_dict (dict): info about SG entity convert to AYON dict + sg_ay_dicts (list[dict]): all processed SG entities + sg_entity_id (int): id of currently processed SG entity + sg_entity_sync_status (str): 'Synched'|'Failed' + sg_session (shotgun_api3.Shotgun): + """ + sg_ay_dict["data"][CUST_FIELD_CODE_ID] = ay_entity.id + sg_ay_dicts[sg_entity_id] = sg_ay_dict + + # If the entity is not a "Folder" or "AssetCategory" we update the + # entity ID and sync status in Shotgrid and AYON + if ( + sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] not in [ + "Folder", "AssetCategory" + ] + and ( + sg_ay_dict["data"][CUST_FIELD_CODE_ID] != ay_entity.id or + sg_ay_dict["data"][CUST_FIELD_CODE_SYNC] != sg_entity_sync_status + ) + ): + log.debug( + "Updating AYON entity ID and sync status in SG and AYON") + update_data = { + CUST_FIELD_CODE_ID: ay_entity.id, + CUST_FIELD_CODE_SYNC: sg_entity_sync_status + } + # Update Shotgrid entity with AYON ID and sync status + sg_session.update( + sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB], + sg_entity_id, + update_data + ) + ay_entity.data.update(update_data) + + def _sync_project_attributes(entity_hub, custom_attribs_map, sg_project): """Sync project attributes from Shotgrid to AYON From f3ad1c5e4c99ef0902beccbaa88727ded14916d5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 15:08:18 +0100 Subject: [PATCH 36/47] Removed parent_entity It should always start from project_entity and add layers. --- services/shotgrid_common/utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 3ef2c7b6..940a2b0d 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -428,7 +428,6 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): return _get_special_category( entity_hub, - parent_entity, sg_ay_dict, folders_and_types=folders_and_types ) @@ -453,7 +452,6 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings) addon_settings, transfer_type, "Sequence") return _get_special_category( entity_hub, - parent_entity, sg_ay_dict, folders_and_types=folders_and_types ) @@ -488,28 +486,26 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): return _get_special_category( entity_hub, - parent_entity, sg_ay_dict, folders_and_types=folders_and_types ) def _get_special_category( - entity_hub, - parent_entity, - sg_ay_dict, - folders_and_types=None + entity_hub, + sg_ay_dict, + folders_and_types=None ): """Returns or creates special subfolders (shot|sequence|AssetCategory). Args: entity_hub (ayon_api.EntityHub): The project's entity hub. - parent_entity: AYON parent entity. sg_ay_dict (dict): The ShotGrid entity ready for Ayon consumption. folders_and_types (deque(([str], [str]))) Returns: (FolderEntity) """ + parent_entity = entity_hub.project_entity found_folder = None placeholders = _get_placeholders(sg_ay_dict) From 8b3c017e26f31747db1de32e93faace3c2488619 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 15:08:43 +0100 Subject: [PATCH 37/47] Added note --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 5aa231f4..9b6f5ece 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -112,6 +112,8 @@ def match_shotgrid_hierarchy_in_ayon( # If the entity has children, add it to the deck for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): sg_ay_dicts_deck.append((ay_parent_entity, sg_child_id)) + + # AssetCategory is not "real" entity to create or update ids continue elif shotgrid_type == "Sequence": From 8120182e78aae14eae3671803ba53b91a09a7491 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 16:37:30 +0100 Subject: [PATCH 38/47] Fix wrong parents for Tasks --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 9b6f5ece..9726c2cd 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -184,7 +184,7 @@ def match_shotgrid_hierarchy_in_ayon( # If the entity has children, add it to the deck for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []): - sg_ay_dicts_deck.append((ay_parent_entity, sg_child_id)) + sg_ay_dicts_deck.append((ay_entity, sg_child_id)) _sync_project_attributes(entity_hub, custom_attribs_map, sg_project) From f802f26f01747a0333bfaa88130e8bb669d3a97e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 16:38:10 +0100 Subject: [PATCH 39/47] Removed unnecessary ShotCategory, SequenceCategory --- .../shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py | 2 +- services/shotgrid_common/utils.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py index 8f68f0b0..cb4f4c9d 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_ayon.py @@ -319,7 +319,7 @@ def _create_sg_entity( sg_field_name = "code" sg_step = None - special_folder_types =["AssetCategory", "ShotCategory", "SequenceCategory"] + special_folder_types = ["AssetCategory"] # parent special folder like AssetCategory should not be created in # Shotgrid it is only used for grouping Asset types is_parent_project_entity = isinstance(ay_entity.parent, ProjectEntity) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 940a2b0d..e8893fb6 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -366,9 +366,7 @@ def create_sg_entities_in_ay( new_folder_types = sg_folder_entities + project_entity.folder_types # So we can have a specific folder for AssetCategory new_folder_types.extend([ - {"name": "AssetCategory"}, - {"name": "ShotCategory"}, - {"name": "SequenceCategory"}, + {"name": "AssetCategory"} ]) # Make sure list items are unique @@ -550,8 +548,7 @@ def _get_placeholders(sg_ay_dict): Currently implemented only `shotgrid_type` which points to name of AssetCategory - TODO probably refactor `shotgrid_type` to different name if ShotCategory - removed + TODO probably refactor `shotgrid_type` to different name """ placeholders = {} # regular update process From e29611f2d501037fc0bcb22a6f88ad8728beee7b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 17:31:16 +0100 Subject: [PATCH 40/47] Fix missing data on Task entities --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 9726c2cd..fb29667e 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -254,6 +254,8 @@ def _update_sg_entity( sg_entity_id, update_data ) + if not ay_entity.data: + ay_entity.data = {} ay_entity.data.update(update_data) From cbff7e9ec324ee7a581ac6dbd95b9237eec49720 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 17:42:38 +0100 Subject: [PATCH 41/47] Ruff --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index fb29667e..48c75121 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -14,8 +14,6 @@ SHOTGRID_TYPE_ATTRIB, ) -from ayon_api.entity_hub import ProjectEntity - from utils import ( create_new_ayon_entity, get_sg_entities, From 1d77f0e66529afcf5be0749404461c1b9fc0afd1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 22 Nov 2024 11:57:23 +0100 Subject: [PATCH 42/47] Fix explicit argument New ayon_python_api has modified arguments --- services/shotgrid_common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index e8893fb6..7e3634c1 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -1313,7 +1313,7 @@ def create_new_ayon_entity( return ay_entity = entity_hub.add_new_task( - sg_ay_dict["task_type"], + task_type=sg_ay_dict["task_type"], name=sg_ay_dict["name"], label=sg_ay_dict["label"], entity_id=sg_ay_dict["data"][CUST_FIELD_CODE_ID], @@ -1322,7 +1322,7 @@ def create_new_ayon_entity( ) else: ay_entity = entity_hub.add_new_folder( - sg_ay_dict["folder_type"], + folder_type=sg_ay_dict["folder_type"], name=sg_ay_dict["name"], label=sg_ay_dict["label"], entity_id=sg_ay_dict["data"][CUST_FIELD_CODE_ID], From c3ad7bb2f9d55d84f07d28b7e669d4a091ced890 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Nov 2024 16:26:48 +0100 Subject: [PATCH 43/47] Removed parent entity from arguments Removed from underlying call, not necessary here anymore --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 3 --- .../shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index 48c75121..b9d0f1bf 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -103,7 +103,6 @@ def match_shotgrid_hierarchy_in_ayon( if shotgrid_type == "AssetCategory": ay_parent_entity = get_asset_category( entity_hub, - ay_parent_entity, sg_ay_dict, addon_settings ) @@ -117,7 +116,6 @@ def match_shotgrid_hierarchy_in_ayon( elif shotgrid_type == "Sequence": ay_parent_entity = get_sequence_category( entity_hub, - ay_parent_entity, sg_ay_dict, addon_settings ) @@ -125,7 +123,6 @@ def match_shotgrid_hierarchy_in_ayon( elif shotgrid_type == "Shot": ay_parent_entity = get_shot_category( entity_hub, - ay_parent_entity, sg_ay_dict, addon_settings ) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py index 67f5d902..16bdfea7 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py @@ -232,7 +232,6 @@ def _get_ayon_parent_entity( log.debug("ShotGrid Parent is an Asset category.") ay_parent_entity = get_asset_category( ayon_entity_hub, - ayon_entity_hub.project_entity, sg_ay_dict, addon_settings ) @@ -241,7 +240,6 @@ def _get_ayon_parent_entity( log.info("ShotGrid Parent is an Sequence category.") ay_parent_entity = get_sequence_category( ayon_entity_hub, - ayon_entity_hub.project_entity, sg_ay_dict, addon_settings ) @@ -250,7 +248,6 @@ def _get_ayon_parent_entity( log.info("ShotGrid Parent is an Shot category.") ay_parent_entity = get_shot_category( ayon_entity_hub, - ayon_entity_hub.project_entity, sg_ay_dict, addon_settings ) From ccc1f1cb4459b775f5e0396f972b63e60bc7c7ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Nov 2024 16:27:21 +0100 Subject: [PATCH 44/47] Removed parent entity from arguments Removed from underlying call, not necessary here anymore --- services/shotgrid_common/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index 7e3634c1..f258fa68 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -406,7 +406,7 @@ def create_sg_entities_in_ay( return sg_folder_entities, sg_steps -def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): +def get_asset_category(entity_hub, sg_ay_dict, addon_settings): """Look for existing "AssetCategory" folders in AYON. Asset categories are not entities per se in ShotGrid, they are @@ -431,7 +431,7 @@ def get_asset_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): ) -def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): +def get_sequence_category(entity_hub, sg_ay_dict, addon_settings): """Look for existing "Sequence" folders in AYON. Asset categories are not entities per se in ShotGrid, they are @@ -455,7 +455,7 @@ def get_sequence_category(entity_hub, parent_entity, sg_ay_dict, addon_settings) ) -def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): +def get_shot_category(entity_hub, sg_ay_dict, addon_settings): """Look for existing "shot" folders in AYON under "parent_entity". Args: @@ -473,8 +473,6 @@ def get_shot_category(entity_hub, parent_entity, sg_ay_dict, addon_settings): if sg_parent: sg_entity_type = "Sequence" # look for custom parents of Sequence parent_sequence = (sg_parent["name"], sg_parent["type"]) - else: - return entity_hub.project_entity folders_and_types = _get_parents_and_types( addon_settings, transfer_type, sg_entity_type) @@ -507,6 +505,9 @@ def _get_special_category( found_folder = None placeholders = _get_placeholders(sg_ay_dict) + if not folders_and_types: + return parent_entity + while folders_and_types: found_folder = None parent = folders_and_types.popleft() @@ -613,7 +614,7 @@ def _get_parents_and_types(addon_settings, transfer_type, sg_entity_type): [transfer_type]) if not parents_presets["enabled"]: - return + return [] found_preset = None for preset in parents_presets["presets"]: @@ -623,7 +624,7 @@ def _get_parents_and_types(addon_settings, transfer_type, sg_entity_type): break if not found_preset: - return + return [] folders_and_types = collections.deque() for parent in found_preset["parents"]: From 18143101a775098babd5a7b934a8675d1e0a83bf Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 26 Nov 2024 16:21:35 +0100 Subject: [PATCH 45/47] Updated logging --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index b9d0f1bf..a4bd97b7 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -238,7 +238,8 @@ def _update_sg_entity( ) ): log.debug( - "Updating AYON entity ID and sync status in SG and AYON") + f"Updating AYON entity ID '{ay_entity.id}' and " + f"sync status in SG '{sg_ay_dict['name']}' and AYON") update_data = { CUST_FIELD_CODE_ID: ay_entity.id, CUST_FIELD_CODE_SYNC: sg_entity_sync_status From d89372e7bc45c73dec3312ccc2394e48b0a80ee6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 26 Nov 2024 17:20:49 +0100 Subject: [PATCH 46/47] Update data only if exist Tasks don't have data --- .../ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py index a4bd97b7..da26ea84 100644 --- a/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py +++ b/services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py @@ -250,9 +250,8 @@ def _update_sg_entity( sg_entity_id, update_data ) - if not ay_entity.data: - ay_entity.data = {} - ay_entity.data.update(update_data) + if ay_entity.data: + ay_entity.data.update(update_data) def _sync_project_attributes(entity_hub, custom_attribs_map, sg_project): From 281a19293946de61953c468e9ba3ec8c161d87aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 26 Nov 2024 17:31:23 +0100 Subject: [PATCH 47/47] Set data directly when create entity --- services/shotgrid_common/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/shotgrid_common/utils.py b/services/shotgrid_common/utils.py index f258fa68..2bdc08a5 100644 --- a/services/shotgrid_common/utils.py +++ b/services/shotgrid_common/utils.py @@ -1319,7 +1319,8 @@ def create_new_ayon_entity( label=sg_ay_dict["label"], entity_id=sg_ay_dict["data"][CUST_FIELD_CODE_ID], parent_id=parent_entity.id, - attribs=sg_ay_dict["attribs"] + attribs=sg_ay_dict["attribs"], + data=sg_ay_dict["data"] ) else: ay_entity = entity_hub.add_new_folder( @@ -1328,7 +1329,8 @@ def create_new_ayon_entity( label=sg_ay_dict["label"], entity_id=sg_ay_dict["data"][CUST_FIELD_CODE_ID], parent_id=parent_entity.id, - attribs=sg_ay_dict["attribs"] + attribs=sg_ay_dict["attribs"], + data=sg_ay_dict["data"] ) log.debug(f"Created new AYON entity: {ay_entity}")