Synchronisation issues

service_tools/requirements.txt

@@ -1,6 +1,6 @@
 annotated-types==0.6.0
 appdirs==1.4.4
-ayon-python-api==1.0.4
+ayon-python-api>=1.0.4
 certifi==2024.2.2
 charset-normalizer==3.3.2
 colorama==0.4.6

services/leecher/leecher/listener.py

@@ -38,8 +38,6 @@ def __init__(self, func: Union[Callable, None] = None):
         else:
             self.func = func
 
-        logging.debug(f"Callback method is {self.func}.")
-
         try:
             ayon_api.init_service()
             self.settings = ayon_api.get_service_addon_settings()
@@ -71,8 +69,10 @@ def __init__(self, func: Union[Callable, None] = None):
                 for attr in self.settings["compatibility_settings"]["custom_attribs_map"]  # noqa: E501
                 if attr["sg"]
             }
+
+            # TODO: implement a way to handle status_list and tags
             self.custom_attribs_map.update({
-                "status": "status_list",
+                # "status": "status_list",
                 "tags": "tags"
             })
 
@@ -171,8 +171,6 @@ def _get_last_event_processed(self, sg_filters):
             )
             last_event_id = last_event["id"]
 
-        logging.debug(f"Last non-processed SG Event is {last_event}")
-
         return last_event_id
 
     def start_listening(self):
@@ -221,7 +219,6 @@ def start_listening(self):
                         event["event_type"].endswith("_Change")
                         and event["attribute_name"].replace("sg_", "") not in list(self.custom_attribs_map.values())
                     ):
-                        logging.debug(f"Skipping event for attribute change '{event['attribute_name'].replace('sg_', '')}', as we can't handle it.")
                         last_event_id = event.get("id", None)
                         continue
 

services/processor/processor/processor.py

@@ -96,8 +96,6 @@ def __init__(self):
         self.handlers_map = self._get_handlers()
         if not self.handlers_map:
             logging.error("No handlers found for the processor, aborting.")
-        else:
-            logging.debug(f"Found these handlers: {self.handlers_map}")
 
     def _get_handlers(self):
         """ Import the handlers found in the `handlers` directory.
@@ -145,10 +143,6 @@ def start_processing(self):
         will trigger the `handlers/project_sync.py` since that one has the attribute
         REGISTER_EVENT_TYPE = ["create-project"]
         """
-        logging.debug(
-            "Querying for `shotgrid.event` events "
-            f"every {self.sg_polling_frequency} seconds..."
-        )
         while True:
             try:
                 event = ayon_api.enroll_event_job(

services/shotgrid_common/ayon_shotgrid_hub/__init__.py

@@ -67,6 +67,12 @@ class AyonShotgridHub:
         custom_attribs_types (dict): A dictionary mapping AYON attribute types
             to Shotgrid field types.
     """
+
+    custom_attribs_map = {
+        "status": "status_list",
+        "tags": "tags"
+    }
+
     def __init__(self,
         project_name,
         project_code,
@@ -100,15 +106,11 @@ def __init__(self,
         else:
             self.sg_project_code_field = "code"
 
-        self.custom_attribs_map = {
-            "status": "status_list",
-            "tags": "tags"
-        }
+        # add custom attributes from settings
         if custom_attribs_map:
             self.custom_attribs_map.update(custom_attribs_map)
 
-        if custom_attribs_types:
-            self.custom_attribs_types = custom_attribs_types
+        self.custom_attribs_types = custom_attribs_types
 
         if sg_enabled_entities:
             self.sg_enabled_entities = sg_enabled_entities
@@ -363,8 +365,10 @@ def react_to_shotgrid_event(self, sg_event):
             case "attribute_change":
                 update_ayon_entity_from_sg_event(
                     sg_event,
+                    self._sg_project,
                     self._sg,
                     self._ay_project,
+                    self.sg_enabled_entities,
                     self.sg_project_code_field,
                     self.custom_attribs_map,
                 )

services/shotgrid_common/ayon_shotgrid_hub/match_ayon_hierarchy_in_shotgrid.py

@@ -66,7 +66,6 @@ def match_ayon_hierarchy_in_shotgrid(
 
     while sg_ay_dicts_deck:
         (sg_ay_parent_entity, ay_entity) = sg_ay_dicts_deck.popleft()
-        logging.debug(f"Processing {ay_entity})")
 
         sg_ay_dict = None
 
@@ -85,7 +84,8 @@ def match_ayon_hierarchy_in_shotgrid(
 
             if sg_entity_id in sg_ay_dicts:
                 sg_ay_dict = sg_ay_dicts[sg_entity_id]
-                logging.info(f"Entity already exists in Shotgrid {sg_ay_dict}")
+                logging.info(
+                    f"Entity already exists in Shotgrid {sg_ay_dict['name']}")
 
                 if sg_ay_dict["data"][CUST_FIELD_CODE_ID] != ay_entity.id:
                     logging.error(
@@ -148,7 +148,8 @@ def match_ayon_hierarchy_in_shotgrid(
                 SHOTGRID_TYPE_ATTRIB,
                 sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB]
             )
-            entity_hub.commit_changes()
+
+        entity_hub.commit_changes()
 
         if sg_ay_dict is None:
             # Shotgrid doesn't have the concept of "Folders"
@@ -268,9 +269,6 @@ def _create_new_entity(
         log_traceback(e)
         raise e
 
-    logging.debug(f"Created new entity: {sg_entity}")
-    logging.debug(f"Parent is: {sg_parent_entity}")
-
     sg_ay_dict = get_sg_entity_as_ay_dict(
         sg_session,
         sg_entity["type"],

services/shotgrid_common/ayon_shotgrid_hub/match_shotgrid_hierarchy_in_ayon.py

@@ -29,9 +29,9 @@ def match_shotgrid_hierarchy_in_ayon(
     """Replicate a Shotgrid project into AYON.
 
     This function creates a "deck" which we keep increasing while traversing
-    the Shotgrid project and finding new childrens, this is more efficient than
-    creating a dictionary with the whle Shotgrid project structure since we
-    `popleft` the elements when procesing them.
+    the Shotgrid project and finding new children, this is more efficient than
+    creating a dictionary with the while Shotgrid project structure since we
+    `popleft` the elements when processing them.
 
     Args:
         entity_hub (ayon_api.entity_hub.EntityHub): The AYON EntityHub.
@@ -51,14 +51,34 @@ def match_shotgrid_hierarchy_in_ayon(
     sg_ay_dicts_deck = collections.deque()
 
     # Append the project's direct children.
-    for sg_ay_dict_child in sg_ay_dicts_parents[sg_project["id"]]:
-        sg_ay_dicts_deck.append((entity_hub.project_entity, sg_ay_dict_child))
+    for sg_ay_dict_child_id in sg_ay_dicts_parents[sg_project["id"]]:
+        sg_ay_dicts_deck.append(
+            (entity_hub.project_entity, sg_ay_dict_child_id)
+        )
 
     sg_project_sync_status = "Synced"
+    processed_ids = set()
 
     while sg_ay_dicts_deck:
-        (ay_parent_entity, sg_ay_dict) = sg_ay_dicts_deck.popleft()
-        logging.debug(f"Processing {sg_ay_dict} with parent {ay_parent_entity}")
+        (ay_parent_entity, sg_ay_dict_child_id) = sg_ay_dicts_deck.popleft()
+        sg_ay_dict = sg_ay_dicts[sg_ay_dict_child_id]
+        sg_entity_id = sg_ay_dict["attribs"][SHOTGRID_ID_ATTRIB]
+        if sg_entity_id in processed_ids:
+            msg = (
+                f"Entity {sg_entity_id} already processed, skipping..."
+                f"Sg Ay Dict: {sg_ay_dict} - "
+                f"Ay Parent Entity: {ay_parent_entity}"
+            )
+            logging.warning(msg)
+
+            # append msg to temp file for debugging
+            with open("/service/processed_ids.txt", "a") as f:
+                f.write(f"{msg}\n")
+            continue
+
+        processed_ids.add(sg_entity_id)
+
+        logging.info(f"Deck size: {len(sg_ay_dicts_deck)}")
 
         ay_entity = None
         sg_entity_sync_status = "Synced"
@@ -75,9 +95,6 @@ def match_shotgrid_hierarchy_in_ayon(
             for child in ay_parent_entity.children:
                 if child.name.lower() == name.lower():
                     ay_entity = child
-                    logging.debug(
-                        f"Found another entity with the same name: {ay_entity.name} <{ay_entity.id}>."
-                    )
                     break
 
         # If we couldn't find it we create it.
@@ -86,7 +103,7 @@ def match_shotgrid_hierarchy_in_ayon(
                 ay_entity = get_asset_category(
                     entity_hub,
                     ay_parent_entity,
-                    sg_ay_dict["name"]
+                    sg_ay_dict
                 )
 
             if not ay_entity:
@@ -96,20 +113,16 @@ def match_shotgrid_hierarchy_in_ayon(
                     sg_ay_dict
                 )
         else:
-            logging.debug(
-                f"Entity {ay_entity.name} <{ay_entity.id}> exists in AYON. "
-                "Making sure the stored ShotGrid Data matches."
-            )
-
             ay_sg_id_attrib = ay_entity.attribs.get(
                 SHOTGRID_ID_ATTRIB
             )
 
-            if ay_sg_id_attrib != str(sg_ay_dict["attribs"][SHOTGRID_ID_ATTRIB]): # noqa
+            # If the ShotGrid ID in AYON doesn't match the one in ShotGrid
+            if str(ay_sg_id_attrib) != str(sg_entity_id):  # noqa
                 logging.error(
                     f"The AYON entity {ay_entity.name} <{ay_entity.id}> has the "  # noqa
                     f"ShotgridId {ay_sg_id_attrib}, while the ShotGrid ID "  # noqa
-                    f"should be {sg_ay_dict['attribs'][SHOTGRID_ID_ATTRIB]}"
+                    f"should be {sg_entity_id}"
                 )
                 sg_entity_sync_status = "Failed"
                 sg_project_sync_status = "Failed"
@@ -118,47 +131,55 @@ def match_shotgrid_hierarchy_in_ayon(
                     ay_entity, sg_ay_dict, custom_attribs_map
                 )
 
+        # skip if no ay_entity is found
+        # perhaps due Task with project entity as parent
+        if not ay_entity:
+            logging.error(f"Entity {sg_ay_dict} not found in AYON.")
+            continue
+
         # Update SG entity with new created data
         sg_ay_dict["data"][CUST_FIELD_CODE_ID] = ay_entity.id
-        sg_ay_dicts[sg_ay_dict["attribs"][SHOTGRID_ID_ATTRIB]] = sg_ay_dict
-
-        entity_id = sg_ay_dict["name"]
+        sg_ay_dicts[sg_entity_id] = sg_ay_dict
 
         # If the entity is not a "Folder" or "AssetCategory" we update the
         # entity ID and sync status in Shotgrid and AYON
-        if sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] not in [
-            "Folder", "AssetCategory"
-        ]:
-            if (
+        if (
+            sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB] not in [
+                "Folder", "AssetCategory"
+            ]
+            and (
                 sg_ay_dict["data"][CUST_FIELD_CODE_ID] != ay_entity.id
                 or sg_ay_dict["data"][CUST_FIELD_CODE_SYNC] != sg_entity_sync_status  # noqa
-            ):
-                logging.debug("Updating AYON entity ID and sync status in SG and AYON")
-                update_data = {
-                    CUST_FIELD_CODE_ID: ay_entity.id,
-                    CUST_FIELD_CODE_SYNC: sg_entity_sync_status
-                }
-                sg_session.update(
-                    sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB],
-                    sg_ay_dict["attribs"][SHOTGRID_ID_ATTRIB],
-                    update_data
-                )
-                ay_entity.data.update(
-                    update_data
-                )
-
-            entity_id = sg_ay_dict["attribs"][SHOTGRID_ID_ATTRIB]
-
-        try:
-            entity_hub.commit_changes()
-        except Exception as e:
-            logging.error(f"Unable to create entity {sg_ay_dict} in AYON!")
-            log_traceback(e)
+            )
+        ):
+            # logging.debug(
+            #     "Updating AYON entity ID and sync status in SG and AYON")
+            update_data = {
+                CUST_FIELD_CODE_ID: ay_entity.id,
+                CUST_FIELD_CODE_SYNC: sg_entity_sync_status
+            }
+            # Update Shotgrid entity with Ayon ID and sync status
+            sg_session.update(
+                sg_ay_dict["attribs"][SHOTGRID_TYPE_ATTRIB],
+                sg_entity_id,
+                update_data
+            )
+            ay_entity.data.update(update_data)
 
         # If the entity has children, add it to the deck
-        for sg_child in sg_ay_dicts_parents.get(entity_id, []):
-            sg_ay_dicts_deck.append((ay_entity, sg_child))
-
+        for sg_child_id in sg_ay_dicts_parents.get(sg_entity_id, []):
+            sg_ay_dicts_deck.append((ay_entity, sg_child_id))
+
+    try:
+        entity_hub.commit_changes()
+    except Exception as e:
+        logging.error("Unable to commit all entities to AYON!")
+        log_traceback(e)
+
+    logging.info(
+        "Processed entities successfully!. "
+        f"Amount of entities: {len(processed_ids)}"
+    )
     # Sync project attributes from Shotgrid to AYON
     entity_hub.project_entity.attribs.set(
         SHOTGRID_ID_ATTRIB,
@@ -171,8 +192,7 @@ def match_shotgrid_hierarchy_in_ayon(
     for ay_attrib, sg_attrib in custom_attribs_map.items():
         attrib_value = sg_project.get(sg_attrib) \
             or sg_project.get(f"sg_{sg_attrib}")
-
-        logging.debug(f"Checking {sg_attrib} -> {attrib_value}")
+
         if attrib_value is None:
             continue
 
@@ -210,6 +230,14 @@ def _create_new_entity(entity_hub, parent_entity, sg_ay_dict):
         sg_ay_dict (dict): Ayon ShotGrid entity to create.
     """
     if sg_ay_dict["type"].lower() == "task":
+        # only create if parent_entity type is not project
+        if parent_entity.entity_type == "project":
+            logging.warning(
+                f"Can't create task '{sg_ay_dict['name']}' under project "
+                "'{parent_entity.name}'. Parent should not be project it self!"
+            )
+            return
+
         ay_entity = entity_hub.add_new_task(
             sg_ay_dict["task_type"],
             name=sg_ay_dict["name"],
@@ -229,16 +257,22 @@ def _create_new_entity(entity_hub, parent_entity, sg_ay_dict):
             attribs=sg_ay_dict["attribs"],
             data=sg_ay_dict["data"],
         )
-    
+
     # TODO: this doesn't work yet
     status = sg_ay_dict["attribs"].get("status")
     if status:
-        ay_entity.status = status
+        # TODO: Implement status update
+        try:
+            # INFO: it was causing error so trying to set status directly
+            ay_entity.status = status
+        except ValueError as e:
+            # `ValueError: Status ip is not available on project.`
+            # logging.warning(f"Status sync not implemented: {e}")
+            pass
 
     tags = sg_ay_dict["attribs"].get("tags")
     if tags:
         ay_entity.tags = [tag["name"] for tag in tags]
 
-    logging.debug(f"Created new entity: {ay_entity.name} ({ay_entity.id})")
-    logging.debug(f"Parent is: {parent_entity.name} ({parent_entity.id})")
+    logging.info(f"Created new entity: {ay_entity.name} ({ay_entity.id})")
     return ay_entity