Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Load and manage products from a library project #1005

Draft
wants to merge 26 commits into
base: develop
Choose a base branch
from
+487 −333
Draft

Load and manage products from a library project #1005

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
aaadaff
refactoring the load container so that it can load the library project
moonyuet Nov 11, 2024
e0ff639
Merge branch 'develop' into enhancement/Load-and-manage-products-from…
moonyuet Nov 11, 2024
dcb838e
resolve the project root during updating container
moonyuet Nov 11, 2024
735409f
do not get the container item from self.get_container
moonyuet Nov 12, 2024
0bec953
query the product id per project
moonyuet Nov 14, 2024
574ea35
codes clean up & make sure it supports mulitple asset loading per pro…
moonyuet Nov 15, 2024
8965a88
clean up code and add project name row into scene inventory
moonyuet Nov 18, 2024
0866c00
code tweaks - kuba's comment
moonyuet Nov 18, 2024
004e962
big roy comment - refactoring the dict per repre_id per project
moonyuet Nov 18, 2024
3078ba2
kuba's comment - add current project name as argument in the get_cont…
moonyuet Nov 19, 2024
d9c1a29
loading the asset per repre_id and per project
moonyuet Nov 21, 2024
ec56335
Merge branch 'develop' into enhancement/Load-and-manage-products-from…
moonyuet Nov 21, 2024
7a22491
remove unused variable
moonyuet Nov 21, 2024
a70135b
implement switch and set version by repre_id/product_id per project
moonyuet Nov 21, 2024
77e5317
remove unused variable
moonyuet Nov 21, 2024
e625901
big roy's comment - code tweak
moonyuet Nov 22, 2024
7edc759
remove unused variables
iLLiCiTiT Nov 22, 2024
ef26dc2
added empty lines for readability
iLLiCiTiT Nov 22, 2024
7935ed3
site sync expects project name
iLLiCiTiT Nov 22, 2024
b25715e
use project name in site sync calls
iLLiCiTiT Nov 22, 2024
a30698e
refactor view codebase
iLLiCiTiT Nov 22, 2024
562f2ed
site sync is fully project specific
iLLiCiTiT Nov 22, 2024
eea31b6
don't slow down project name getter
iLLiCiTiT Nov 22, 2024
e21c7a1
use project name to get correct status icons
iLLiCiTiT Nov 22, 2024
87907b5
fix switch version
iLLiCiTiT Nov 22, 2024
b28f4b0
comment out unused variables
iLLiCiTiT Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions client/ayon_core/pipeline/load/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,7 @@ def update_container(container, version=-1):
from ayon_core.pipeline import get_current_project_name

# Compute the different version from 'representation'
project_name = get_current_project_name()
project_name = container.get("project_name", get_current_project_name())
repre_id = container["representation"]
if not _is_valid_representation_id(repre_id):
raise ValueError(
Expand Down Expand Up @@ -542,9 +542,6 @@ def update_container(container, version=-1):
)
)

path = get_representation_path(new_representation)
if not path or not os.path.exists(path):
raise ValueError("Path {} doesn't exist".format(path))
project_entity = ayon_api.get_project(project_name)
context = {
"project": project_entity,
Expand All @@ -553,6 +550,9 @@ def update_container(container, version=-1):
"version": new_version,
"representation": new_representation,
}
path = get_representation_path_from_context(context)
if not path or not os.path.exists(path):
raise ValueError("Path {} doesn't exist".format(path))

return Loader().update(container, context)

Expand Down Expand Up @@ -588,7 +588,7 @@ def switch_container(container, representation, loader_plugin=None):
)

# Get the new representation to switch to
project_name = get_current_project_name()
project_name = container.get("project_name", get_current_project_name())

context = get_representation_context(
project_name, representation["id"]
Expand Down
7 changes: 4 additions & 3 deletions client/ayon_core/tools/sceneinventory/control.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from ayon_core.host import HostBase
from ayon_core.pipeline import (
registered_host,
get_current_context,
get_current_context
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
)
from ayon_core.tools.common_models import HierarchyModel, ProjectsModel

Expand Down Expand Up @@ -110,8 +110,9 @@ def get_representation_info_items(self, representation_ids):
representation_ids
)

def get_version_items(self, product_ids):
return self._containers_model.get_version_items(product_ids)
def get_version_items(self, product_ids, representation_ids):
return self._containers_model.get_version_items(
product_ids, representation_ids)

# Site Sync methods
def is_sitesync_enabled(self):
Expand Down
5 changes: 4 additions & 1 deletion client/ayon_core/tools/sceneinventory/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,15 +130,18 @@ def refresh(self, selected=None):
self._clear_items()

items_by_repre_id = {}
project_names = set()
for container_item in container_items:
# if (
# selected is not None
# and container_item.item_id not in selected
# ):
# continue
repre_id = container_item.representation_id
project_name = container_item.project_name
items = items_by_repre_id.setdefault(repre_id, [])
items.append(container_item)
project_names.add(project_name)

repre_id = set(items_by_repre_id.keys())
repre_info_by_id = self._controller.get_representation_info_items(
Expand All @@ -150,7 +153,7 @@ def refresh(self, selected=None):
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
product_ids
product_ids, project_names
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
)
Copy link
Collaborator

@BigRoy BigRoy Nov 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is the wrong approach because we'd need to know which product id belongs to which project. What @iLLiCiTiT mentioined before is to separate the query to self._controller.get_version_items() to have separate calls per project. So above, we'd get the product ids per project and then here we would do:

version_items_by_product_id = self._controller.get_version_items(project_name, product_id)

As stated before - as far as I know on the backend nothing is keeping the database from having a product entity with the same id across different projects. Is that correct @martastain ? (It's unlikely by just creating products to happen, but when cloning e.g. projects I suppose it may very well share the same ids?) As such, we should still make sure to be explicit about which project each entry belongs to.

The same goes for version_items_by_product_id dict. That should technically also be "per project" because otherwise product_id may overlap accidentally.

@iLLiCiTiT thoughts?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes

# SiteSync addon information
progress_by_id = self._controller.get_representations_site_progress(
Expand Down
132 changes: 78 additions & 54 deletions client/ayon_core/tools/sceneinventory/models/containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,15 @@ def __init__(
loader_name,
namespace,
object_name,
item_id
item_id,
project_name
):
self.representation_id = representation_id
self.loader_name = loader_name
self.object_name = object_name
self.namespace = namespace
self.item_id = item_id
self.project_name = project_name

@classmethod
def from_container_data(cls, container):
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -109,6 +111,7 @@ def from_container_data(cls, container):
namespace=container["namespace"],
object_name=container["objectName"],
item_id=uuid.uuid4().hex,
project_name=container.get("project_name", None)
)


Expand Down Expand Up @@ -189,15 +192,21 @@ def __init__(self, controller):
self._items_cache = None
self._containers_by_id = {}
self._container_items_by_id = {}
self._container_items_by_project = {}
self._project_name_by_repre_id = {}
self._version_items_by_product_id = {}
self._repre_info_by_id = {}
self._product_id_by_project = {}

def reset(self):
self._items_cache = None
self._containers_by_id = {}
self._container_items_by_id = {}
self._container_items_by_project = {}
self._project_name_by_repre_id = {}
self._version_items_by_product_id = {}
self._repre_info_by_id = {}
self._product_id_by_project = {}

def get_containers(self):
self._update_cache()
Expand All @@ -221,70 +230,81 @@ def get_container_items_by_id(self, item_ids):

def get_representation_info_items(self, representation_ids):
output = {}
missing_repre_ids = set()
missing_repre_ids_by_project = {}
Copy link
Collaborator

@BigRoy BigRoy Nov 15, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Funnily enough - this now doesn't need to be "per project" anymore - because the full method operates within the project's context. (by having a project_name input)

Only as soon as you touch the variables on the instance itself self then you'd need to ensure to access them for the relevant project.

If it makes more sense, we could also just change those global cache dicts by moving them into one per project cache.

self._project_cache = defaultdict(dict)

And then we could do:

self._project_cache[project_name]["container_items_by_id"]

If that makes more sense to you. But that may just be overkill.

Anyway, it all boils down to: whenever we cache something to the entity id - we should do it by the hash of (project_name, entity_id) otherwise it's not unique.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we can do this in separate PR as enhancement.
Not sure if there would be lots of changes in regard to this implementation.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we can do this in separate PR as enhancement.

We should implement the support correctly. It should not be that hard, it requires a lot of changes in code, but in practice it means we just have to consider project name as one hierarchy level above everything else (everywhere). If you struggle to achieve that, let me know.

current_project_name = self._controller.get_current_project_name()
for repre_id in representation_ids:
try:
uuid.UUID(repre_id)
except ValueError:
output[repre_id] = RepresentationInfo.new_invalid()
continue

project_name = self._project_name_by_repre_id.get(repre_id)
if project_name is None:
project_name = current_project_name
repre_info = self._repre_info_by_id.get(repre_id)
if repre_info is None:
missing_repre_ids.add(repre_id)
missing_repre_ids_by_project.setdefault(
project_name, set()
).add(repre_id)
else:
output[repre_id] = repre_info

if not missing_repre_ids:
if not missing_repre_ids_by_project:
return output

project_name = self._controller.get_current_project_name()
repre_hierarchy_by_id = get_representations_hierarchy(
project_name, missing_repre_ids
)
for repre_id, repre_hierarchy in repre_hierarchy_by_id.items():
kwargs = {
"folder_id": None,
"folder_path": None,
"product_id": None,
"product_name": None,
"product_type": None,
"product_group": None,
"version_id": None,
"representation_name": None,
}
folder = repre_hierarchy.folder
product = repre_hierarchy.product
version = repre_hierarchy.version
repre = repre_hierarchy.representation
if folder:
kwargs["folder_id"] = folder["id"]
kwargs["folder_path"] = folder["path"]
if product:
group = product["attrib"]["productGroup"]
kwargs["product_id"] = product["id"]
kwargs["product_name"] = product["name"]
kwargs["product_type"] = product["productType"]
kwargs["product_group"] = group
if version:
kwargs["version_id"] = version["id"]
if repre:
kwargs["representation_name"] = repre["name"]

repre_info = RepresentationInfo(**kwargs)
self._repre_info_by_id[repre_id] = repre_info
output[repre_id] = repre_info
for project_name, missing_ids in missing_repre_ids_by_project.items():
repre_hierarchy_by_id = get_representations_hierarchy(
project_name, missing_ids
)
for repre_id, repre_hierarchy in repre_hierarchy_by_id.items():
kwargs = {
"folder_id": None,
"folder_path": None,
"product_id": None,
"product_name": None,
"product_type": None,
"product_group": None,
"version_id": None,
"representation_name": None,
}
folder = repre_hierarchy.folder
product = repre_hierarchy.product
version = repre_hierarchy.version
repre = repre_hierarchy.representation
if folder:
kwargs["folder_id"] = folder["id"]
kwargs["folder_path"] = folder["path"]
if product:
group = product["attrib"]["productGroup"]
kwargs["product_id"] = product["id"]
kwargs["product_name"] = product["name"]
kwargs["product_type"] = product["productType"]
kwargs["product_group"] = group
if version:
kwargs["version_id"] = version["id"]
if repre:
kwargs["representation_name"] = repre["name"]

repre_info = RepresentationInfo(**kwargs)
self._repre_info_by_id[repre_id] = repre_info
self._product_id_by_project[project_name] = repre_info.product_id
output[repre_id] = repre_info
return output

def get_version_items(self, product_ids):
def get_version_items(self, product_ids, project_names):
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
if not product_ids:
return {}

missing_ids = {
product_id
for product_id in product_ids
if product_id not in self._version_items_by_product_id
}
moonyuet marked this conversation as resolved.
Show resolved Hide resolved

product_ids_by_project = {
project_name: self._product_id_by_project.get(project_name)
for project_name in project_names
}
if missing_ids:
status_items_by_name = {
status_item.name: status_item
Expand All @@ -293,25 +313,27 @@ def get_version_items(self, product_ids):

def version_sorted(entity):
return entity["version"]

project_name = self._controller.get_current_project_name()
version_entities_list = []
version_entities_by_product_id = {
product_id: []
for product_id in missing_ids
}

version_entities = list(ayon_api.get_versions(
project_name,
product_ids=missing_ids,
fields={"id", "version", "productId", "status"}
))
version_entities.sort(key=version_sorted)
for version_entity in version_entities:
for project_name, product_id in product_ids_by_project.items():
if product_id not in missing_ids:
continue
version_entities = list(ayon_api.get_versions(
project_name,
product_ids={product_id},
fields={"id", "version", "productId", "status"}
))

version_entities_list.extend(version_entities)
version_entities_list.sort(key=version_sorted)
for version_entity in version_entities_list:
product_id = version_entity["productId"]
version_entities_by_product_id[product_id].append(
version_entity
)

for product_id, version_entities in (
version_entities_by_product_id.items()
):
Expand All @@ -337,7 +359,6 @@ def version_sorted(entity):
self._version_items_by_product_id[product_id] = (
version_items_by_id
)

return {
product_id: dict(self._version_items_by_product_id[product_id])
for product_id in product_ids
Expand All @@ -358,6 +379,7 @@ def _update_cache(self):
container_items = []
containers_by_id = {}
container_items_by_id = {}
project_name_by_repre_id = {}
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
invalid_ids_mapping = {}
for container in containers:
try:
Expand All @@ -381,8 +403,10 @@ def _update_cache(self):

containers_by_id[item.item_id] = container
container_items_by_id[item.item_id] = item
project_name_by_repre_id[item.representation_id] = item.project_name
container_items.append(item)

self._containers_by_id = containers_by_id
self._container_items_by_id = container_items_by_id
self._project_name_by_repre_id = project_name_by_repre_id
self._items_cache = container_items
19 changes: 13 additions & 6 deletions client/ayon_core/tools/sceneinventory/view.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,13 +208,15 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index):
filtered_items = []
product_ids = set()
version_ids = set()
project_names = set()
moonyuet marked this conversation as resolved.
Show resolved Hide resolved
for container_item in container_items_by_id.values():
repre_id = container_item.representation_id
repre_info = repre_info_by_id.get(repre_id)
if repre_info and repre_info.is_valid:
filtered_items.append(container_item)
version_ids.add(repre_info.version_id)
product_ids.add(repre_info.product_id)
project_names.add(container_item.project_name)

# remove
remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR)
Expand All @@ -228,8 +230,7 @@ def _build_item_menu_for_selection(self, menu, indexes, active_index):
return

version_items_by_product_id = self._controller.get_version_items(
product_ids
)
product_ids, project_names)
has_outdated = False
has_loaded_hero_versions = False
has_available_hero_version = False
Expand Down Expand Up @@ -739,6 +740,10 @@ def _show_version_dialog(self, item_ids, active_repre_id):
container_item.representation_id
for container_item in container_items_by_id.values()
}
project_names = {
container_item.project_name
for container_item in container_items_by_id.values()
}
repre_info_by_id = self._controller.get_representation_info_items(
repre_ids
)
Expand All @@ -751,8 +756,7 @@ def _show_version_dialog(self, item_ids, active_repre_id):
active_version_id = active_repre_info.version_id
active_product_id = active_repre_info.product_id
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
product_ids, project_names)
version_items = list(
version_items_by_product_id[active_product_id].values()
)
Expand Down Expand Up @@ -934,6 +938,10 @@ def _on_switch_to_versioned(self, item_ids):
container_item.representation_id
for container_item in containers_items_by_id.values()
}
project_names = {
container_item.project_name
for container_item in containers_items_by_id.values()
}
repre_info_by_id = self._controller.get_representation_info_items(
repre_ids
)
Expand All @@ -943,8 +951,7 @@ def _on_switch_to_versioned(self, item_ids):
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
product_ids, project_names)

update_containers = []
update_versions = []
Expand Down