Merge pull request #2682 from pypeclub/bugfix/OP-2610_Ftrack-hiearchical-attributes-sync-uses-default-values

Ftrack: Hierarchical attributes are queried properly

openpype/modules/default_modules/ftrack/event_handlers_server/action_push_frame_values_to_task.py

@@ -2,7 +2,10 @@ import sys
 import json
 import collections
 import ftrack_api
-from openpype_modules.ftrack.lib import ServerAction
+from openpype_modules.ftrack.lib import (
+    ServerAction,
+    query_custom_attributes
+)
 
 
 class PushHierValuesToNonHier(ServerAction):
@@ -51,10 +54,6 @@ class PushHierValuesToNonHier(ServerAction):
         " from CustomAttributeConfiguration"
         " where key in ({})"
     )
-    cust_attr_value_query = (
-        "select value, entity_id from CustomAttributeValue"
-        " where entity_id in ({}) and configuration_id in ({})"
-    )
 
     # configurable
     settings_key = "sync_hier_entity_attributes"
@@ -344,25 +343,11 @@ class PushHierValuesToNonHier(ServerAction):
                 all_ids_with_parents.add(parent_id)
                 _entity_id = parent_id
 
-        joined_entity_ids = self.join_query_keys(all_ids_with_parents)
-
-        hier_attr_ids = self.join_query_keys(
-            tuple(hier_attr["id"] for hier_attr in hier_attrs)
-        )
+        hier_attr_ids = tuple(hier_attr["id"] for hier_attr in hier_attrs)
         hier_attrs_key_by_id = {
             hier_attr["id"]: hier_attr["key"]
             for hier_attr in hier_attrs
         }
-        call_expr = [{
-            "action": "query",
-            "expression": self.cust_attr_value_query.format(
-                joined_entity_ids, hier_attr_ids
-            )
-        }]
-        if hasattr(session, "call"):
-            [values] = session.call(call_expr)
-        else:
-            [values] = session._call(call_expr)
 
         values_per_entity_id = {}
         for entity_id in all_ids_with_parents:
@@ -370,7 +355,10 @@ class PushHierValuesToNonHier(ServerAction):
             for key in hier_attrs_key_by_id.values():
                 values_per_entity_id[entity_id][key] = None
 
-        for item in values["data"]:
+        values = query_custom_attributes(
+            session, all_ids_with_parents, hier_attr_ids, True
+        )
+        for item in values:
             entity_id = item["entity_id"]
             key = hier_attrs_key_by_id[item["configuration_id"]]
 

openpype/modules/default_modules/ftrack/event_handlers_server/event_push_frame_values_to_task.py

@@ -17,11 +17,6 @@ class PushFrameValuesToTaskEvent(BaseEvent):
         " (object_type_id in ({}) or is_hierarchical is true)"
     )
 
-    cust_attr_query = (
-        "select value, entity_id from ContextCustomAttributeValue "
-        "where entity_id in ({}) and configuration_id in ({})"
-    )
-
     _cached_task_object_id = None
     _cached_interest_object_ids = None
     _cached_user_id = None
@@ -273,16 +268,23 @@ class PushFrameValuesToTaskEvent(BaseEvent):
             hier_attr_ids.append(attr_id)
 
         conf_ids = list(hier_attr_ids)
+        task_conf_ids = []
         for key, attr_id in task_attrs.items():
             attr_key_by_id[attr_id] = key
             nonhier_id_by_key[key] = attr_id
             conf_ids.append(attr_id)
+            task_conf_ids.append(attr_id)
 
         # Query custom attribute values
         # - result does not contain values for all entities only result of
         #   query callback to ftrack server
         result = query_custom_attributes(
-            session, conf_ids, whole_hierarchy_ids
+            session, list(hier_attr_ids), whole_hierarchy_ids, True
+        )
+        result.extend(
+            query_custom_attributes(
+                session, task_conf_ids, whole_hierarchy_ids, False
+            )
         )
 
         # Prepare variables where result will be stored
@@ -547,7 +549,7 @@ class PushFrameValuesToTaskEvent(BaseEvent):
         )
         attr_ids = set(attr_id_to_key.keys())
 
-        current_values_by_id = self.current_values(
+        current_values_by_id = self.get_current_values(
             session, attr_ids, entity_ids, task_entity_ids, hier_attrs
         )
 
@@ -642,27 +644,17 @@ class PushFrameValuesToTaskEvent(BaseEvent):
 
         return interesting_data, changed_keys_by_object_id
 
-    def current_values(
+    def get_current_values(
         self, session, attr_ids, entity_ids, task_entity_ids, hier_attrs
     ):
         current_values_by_id = {}
         if not attr_ids or not entity_ids:
             return current_values_by_id
-        joined_conf_ids = self.join_query_keys(attr_ids)
-        joined_entity_ids = self.join_query_keys(entity_ids)
 
-        call_expr = [{
-            "action": "query",
-            "expression": self.cust_attr_query.format(
-                joined_entity_ids, joined_conf_ids
-            )
-        }]
-        if hasattr(session, "call"):
-            [values] = session.call(call_expr)
-        else:
-            [values] = session._call(call_expr)
-
-        for item in values["data"]:
+        values = query_custom_attributes(
+            session, attr_ids, entity_ids, True
+        )
+        for item in values:
             entity_id = item["entity_id"]
             attr_id = item["configuration_id"]
             if entity_id in task_entity_ids and attr_id in hier_attrs:

openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_links.py

@@ -128,7 +128,7 @@ class SyncLinksToAvalon(BaseEvent):
 
     def _get_mongo_ids_by_ftrack_ids(self, session, attr_id, ftrack_ids):
         output = query_custom_attributes(
-            session, [attr_id], ftrack_ids
+            session, [attr_id], ftrack_ids, True
         )
         mongo_id_by_ftrack_id = {}
         for item in output:

openpype/modules/default_modules/ftrack/event_handlers_server/event_sync_to_avalon.py

@@ -17,6 +17,7 @@ from avalon.api import AvalonMongoDB
 
 from openpype_modules.ftrack.lib import (
     get_openpype_attr,
+    query_custom_attributes,
     CUST_ATTR_ID_KEY,
     CUST_ATTR_AUTO_SYNC,
 
@@ -2130,22 +2131,12 @@ class SyncToAvalonEvent(BaseEvent):
         for key in hier_cust_attrs_keys:
             configuration_ids.add(hier_attr_id_by_key[key])
 
-        entity_ids_joined = self.join_query_keys(cust_attrs_ftrack_ids)
-        attributes_joined = self.join_query_keys(configuration_ids)
-
-        queries = [{
-            "action": "query",
-            "expression": (
-                "select value, entity_id, configuration_id"
-                " from CustomAttributeValue "
-                "where entity_id in ({}) and configuration_id in ({})"
-            ).format(entity_ids_joined, attributes_joined)
-        }]
-
-        if hasattr(self.process_session, "call"):
-            [values] = self.process_session.call(queries)
-        else:
-            [values] = self.process_session._call(queries)
+        values = query_custom_attributes(
+            self.process_session,
+            configuration_ids,
+            cust_attrs_ftrack_ids,
+            True
+        )
 
         ftrack_project_id = self.cur_project["id"]
 
@@ -2170,7 +2161,7 @@ class SyncToAvalonEvent(BaseEvent):
 
         # PREPARE DATA BEFORE THIS
         avalon_hier = []
-        for item in values["data"]:
+        for item in values:
             value = item["value"]
             if value is None:
                 continue

openpype/modules/default_modules/ftrack/event_handlers_user/action_clean_hierarchical_attributes.py

@@ -19,8 +19,8 @@ class CleanHierarchicalAttrsAction(BaseAction):
         " from TypedContext where project_id is \"{}\""
     )
     cust_attr_query = (
-        "select value, entity_id from CustomAttributeValue "
-        "where entity_id in ({}) and configuration_id is \"{}\""
+        "select value, entity_id from CustomAttributeValue"
+        " where entity_id in ({}) and configuration_id is \"{}\""
     )
     settings_key = "clean_hierarchical_attr"
 
@@ -65,17 +65,14 @@ class CleanHierarchicalAttrsAction(BaseAction):
                 )
             )
             configuration_id = attr["id"]
-            call_expr = [{
-                "action": "query",
-                "expression": self.cust_attr_query.format(
+            values = session.query(
+                self.cust_attr_query.format(
                     entity_ids_joined, configuration_id
                 )
-            }]
-
-            [values] = self.session.call(call_expr)
+            ).all()
 
             data = {}
-            for item in values["data"]:
+            for item in values:
                 value = item["value"]
                 if value is None:
                     data[item["entity_id"]] = value
@@ -90,10 +87,10 @@ class CleanHierarchicalAttrsAction(BaseAction):
                 len(data), configuration_key
             ))
             for entity_id, value in data.items():
-                entity_key = collections.OrderedDict({
-                    "configuration_id": configuration_id,
-                    "entity_id": entity_id
-                })
+                entity_key = collections.OrderedDict((
+                    ("configuration_id", configuration_id),
+                    ("entity_id", entity_id)
+                ))
                 session.recorded_operations.push(
                     ftrack_api.operation.DeleteEntityOperation(
                         "CustomAttributeValue",

openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py

@@ -306,8 +306,8 @@ class CustomAttributes(BaseAction):
         }
 
         cust_attr_query = (
-            "select value, entity_id from ContextCustomAttributeValue "
-            "where configuration_id is {}"
+            "select value, entity_id from CustomAttributeValue"
+            " where configuration_id is {}"
         )
         for attr_def in object_type_attrs:
             attr_ent_type = attr_def["entity_type"]
@@ -328,21 +328,14 @@ class CustomAttributes(BaseAction):
             self.log.debug((
                 "Converting Avalon MongoID attr for Entity type \"{}\"."
             ).format(entity_type_label))
-
-            call_expr = [{
-                "action": "query",
-                "expression": cust_attr_query.format(attr_def["id"])
-            }]
-            if hasattr(session, "call"):
-                [values] = session.call(call_expr)
-            else:
-                [values] = session._call(call_expr)
-
-            for value in values["data"]:
-                table_values = collections.OrderedDict({
-                    "configuration_id": hierarchical_attr["id"],
-                    "entity_id": value["entity_id"]
-                })
+            values = session.query(
+                cust_attr_query.format(attr_def["id"])
+            ).all()
+            for value in values:
+                table_values = collections.OrderedDict([
+                    ("configuration_id", hierarchical_attr["id"]),
+                    ("entity_id", value["entity_id"])
+                ])
 
                 session.recorded_operations.push(
                     ftrack_api.operation.UpdateEntityOperation(

openpype/modules/default_modules/ftrack/ftrack_module.py

@@ -303,9 +303,10 @@ class FtrackModule(
             # TODO add add permissions check
             # TODO add value validations
             # - value type and list items
-            entity_key = collections.OrderedDict()
-            entity_key["configuration_id"] = configuration["id"]
-            entity_key["entity_id"] = project_id
+            entity_key = collections.OrderedDict([
+                ("configuration_id", configuration["id"]),
+                ("entity_id", project_id)
+            ])
 
             session.recorded_operations.push(
                 ftrack_api.operation.UpdateEntityOperation(

openpype/modules/default_modules/ftrack/lib/avalon_sync.py

@@ -1,11 +1,8 @@
-import os
 import re
 import json
 import collections
 import copy
 
-import six
-
 from avalon.api import AvalonMongoDB
 
 import avalon
@@ -18,7 +15,7 @@ from openpype.api import (
 from openpype.lib import ApplicationManager
 
 from .constants import CUST_ATTR_ID_KEY
-from .custom_attributes import get_openpype_attr
+from .custom_attributes import get_openpype_attr, query_custom_attributes
 
 from bson.objectid import ObjectId
 from bson.errors import InvalidId
@@ -235,33 +232,19 @@ def get_hierarchical_attributes_values(
 
     entity_ids = [item["id"] for item in entity["link"]]
 
-    join_ent_ids = join_query_keys(entity_ids)
-    join_attribute_ids = join_query_keys(attr_key_by_id.keys())
-
-    queries = []
-    queries.append({
-        "action": "query",
-        "expression": (
-            "select value, configuration_id, entity_id"
-            " from CustomAttributeValue"
-            " where entity_id in ({}) and configuration_id in ({})"
-        ).format(join_ent_ids, join_attribute_ids)
-    })
-
-    if hasattr(session, "call"):
-        [values] = session.call(queries)
-    else:
-        [values] = session._call(queries)
+    values = query_custom_attributes(
+        session, list(attr_key_by_id.keys()), entity_ids, True
+    )
 
     hier_values = {}
     for key, val in defaults.items():
         hier_values[key] = val
 
-    if not values["data"]:
+    if not values:
         return hier_values
 
     values_by_entity_id = collections.defaultdict(dict)
-    for item in values["data"]:
+    for item in values:
         value = item["value"]
         if value is None:
             continue
@@ -861,33 +844,6 @@ class SyncEntitiesFactory:
 
             self.entities_dict[parent_id]["children"].remove(ftrack_id)
 
-    def _query_custom_attributes(self, session, conf_ids, entity_ids):
-        output = []
-        # Prepare values to query
-        attributes_joined = join_query_keys(conf_ids)
-        attributes_len = len(conf_ids)
-        chunk_size = int(5000 / attributes_len)
-        for idx in range(0, len(entity_ids), chunk_size):
-            entity_ids_joined = join_query_keys(
-                entity_ids[idx:idx + chunk_size]
-            )
-
-            call_expr = [{
-                "action": "query",
-                "expression": (
-                    "select value, entity_id from ContextCustomAttributeValue "
-                    "where entity_id in ({}) and configuration_id in ({})"
-                ).format(entity_ids_joined, attributes_joined)
-            }]
-            if hasattr(session, "call"):
-                [result] = session.call(call_expr)
-            else:
-                [result] = session._call(call_expr)
-
-            for item in result["data"]:
-                output.append(item)
-        return output
-
     def set_cutom_attributes(self):
         self.log.debug("* Preparing custom attributes")
         # Get custom attributes and values
@@ -994,7 +950,7 @@ class SyncEntitiesFactory:
                     copy.deepcopy(prepared_avalon_attr_ca_id)
                 )
 
-        items = self._query_custom_attributes(
+        items = query_custom_attributes(
             self.session,
             list(attribute_key_by_id.keys()),
             sync_ids
@@ -1082,10 +1038,11 @@ class SyncEntitiesFactory:
             for key, val in prepare_dict_avalon.items():
                 entity_dict["avalon_attrs"][key] = val
 
-        items = self._query_custom_attributes(
+        items = query_custom_attributes(
             self.session,
             list(attribute_key_by_id.keys()),
-            sync_ids
+            sync_ids,
+            True
         )
 
         avalon_hier = []
@@ -1806,10 +1763,10 @@ class SyncEntitiesFactory:
                 configuration_id = self.entities_dict[ftrack_id][
                     "avalon_attrs_id"][CUST_ATTR_ID_KEY]
 
-                _entity_key = collections.OrderedDict({
-                    "configuration_id": configuration_id,
-                    "entity_id": ftrack_id
-                })
+                _entity_key = collections.OrderedDict([
+                    ("configuration_id", configuration_id),
+                    ("entity_id", ftrack_id)
+                ])
 
                 self.session.recorded_operations.push(
                     ftrack_api.operation.UpdateEntityOperation(

openpype/modules/default_modules/ftrack/lib/custom_attributes.py

@@ -88,26 +88,36 @@ def join_query_keys(keys):
     return ",".join(["\"{}\"".format(key) for key in keys])
 
 
-def query_custom_attributes(session, conf_ids, entity_ids, table_name=None):
+def query_custom_attributes(
+    session, conf_ids, entity_ids, only_set_values=False
+):
     """Query custom attribute values from ftrack database.
 
     Using ftrack call method result may differ based on used table name and
     version of ftrack server.
 
+    For hierarchical attributes you shou always use `only_set_values=True`
+    otherwise result will be default value of custom attribute and it would not
+    be possible to differentiate if value is set on entity or default value is
+    used.
+
     Args:
         session(ftrack_api.Session): Connected ftrack session.
         conf_id(list, set, tuple): Configuration(attribute) ids which are
             queried.
         entity_ids(list, set, tuple): Entity ids for which are values queried.
-        table_name(str): Table nam from which values are queried. Not
-            recommended to change until you know what it means.
+        only_set_values(bool): Entities that don't have explicitly set
+            value won't return a value. If is set to False then default custom
+            attribute value is returned if value is not set.
     """
     output = []
     # Just skip
     if not conf_ids or not entity_ids:
         return output
 
-    if table_name is None:
+    if only_set_values:
+        table_name = "CustomAttributeValue"
+    else:
         table_name = "ContextCustomAttributeValue"
 
     # Prepare values to query
@@ -122,19 +132,16 @@ def query_custom_attributes(session, conf_ids, entity_ids, table_name=None):
         entity_ids_joined = join_query_keys(
             entity_ids[idx:idx + chunk_size]
         )
-
-        call_expr = [{
-            "action": "query",
-            "expression": (
-                "select value, entity_id from {}"
-                " where entity_id in ({}) and configuration_id in ({})"
-            ).format(table_name, entity_ids_joined, attributes_joined)
-        }]
-        if hasattr(session, "call"):
-            [result] = session.call(call_expr)
-        else:
-            [result] = session._call(call_expr)
-
-        for item in result["data"]:
-            output.append(item)
+        output.extend(
+            session.query(
+                (
+                    "select value, entity_id from {}"
+                    " where entity_id in ({}) and configuration_id in ({})"
+                ).format(
+                    table_name,
+                    entity_ids_joined,
+                    attributes_joined
+                )
+            ).all()
+        )
     return output