feat(submissions)!: add submission unique identifier (meta/rootUuid) in data API response TASK-1458

jsapp/js/actions/submissions.ts

@@ -4,7 +4,11 @@
 
 import Reflux from 'reflux';
 import {dataInterface} from 'js/dataInterface';
-import {notify} from 'js/utils';
+import {
+  notify,
+  matchUuid,
+  addDefaultUuidPrefix,
+} from 'js/utils';
 import {ROOT_URL} from 'js/constants';
 import type {
   GetSubmissionsOptions,
@@ -80,7 +84,7 @@ submissionsActions.getSubmissionByUuid.listen((assetUid: string, submissionUuid:
   // `meta/rootUuid` remains consistent across edits.
   const query = JSON.stringify({
     '$or': [
-      {'meta/rootUuid': submissionUuid},
+      {'meta/rootUuid': addDefaultUuidPrefix(submissionUuid)},
       {'_uuid': submissionUuid},
     ],
   });
@@ -92,7 +96,7 @@ submissionsActions.getSubmissionByUuid.listen((assetUid: string, submissionUuid:
     .done((response: PaginatedResponse<SubmissionResponse>) => {
       // preferentially return a result matching the persistent UUID
       submissionsActions.getSubmissionByUuid.completed(
-        response.results.find((sub) => sub['meta/rootUuid'] === submissionUuid) ||
+        response.results.find((sub) => matchUuid(sub['meta/rootUuid'], submissionUuid)) ||
         response.results[0]
       );
     })

jsapp/js/components/processing/singleProcessingStore.ts

@@ -36,7 +36,10 @@ import {
   ProcessingTab,
 } from 'js/components/processing/routes.utils';
 import type {KoboSelectOption} from 'js/components/common/koboSelect';
-import {getExponentialDelayTime} from 'jsapp/js/utils';
+import {
+  getExponentialDelayTime,
+  removeDefaultUuidPrefix,
+} from 'jsapp/js/utils';
 import envStore from 'jsapp/js/envStore';
 
 export enum StaticDisplays {
@@ -535,10 +538,10 @@ class SingleProcessingStore extends Reflux.Store {
               if (rowName) {
                 // `meta/rootUuid` is persistent across edits while `_uuid` is not;
                 // use the persistent identifier if present.
-                let uuid = result['meta/rootUuid'];
-                if (uuid === undefined) {
-                  uuid = result['_uuid'];
-                }
+                const uuid = result['meta/rootUuid']
+                  ? removeDefaultUuidPrefix(result['meta/rootUuid'])
+                  : result['_uuid'];
+
                 submissionsEditIds[xpath].push({
                   editId: uuid,
                   hasResponse: Object.keys(result).includes(flatPaths[rowName]),

jsapp/js/components/submissions/table.tsx

@@ -34,7 +34,10 @@ import {
 } from 'js/constants';
 import type {AnyRowTypeName} from 'js/constants';
 import {PERMISSIONS_CODENAMES} from 'js/components/permissions/permConstants';
-import {formatTimeDateShort} from 'js/utils';
+import {
+  formatTimeDateShort,
+  removeDefaultUuidPrefix,
+} from 'js/utils';
 import type {SurveyFlatPaths} from 'js/assetUtils';
 import {
   getRowName,
@@ -970,7 +973,9 @@ export class DataTable extends React.Component<DataTableProps, DataTableState> {
                 q.type === QUESTION_TYPES.audio.id ||
                 q.type === QUESTION_TYPES['background-audio'].id
               ) {
-                const submissionEditId = row.original['meta/rootUuid'] || row.original._uuid;
+                const submissionEditId =
+                  removeDefaultUuidPrefix(row.original['meta/rootUuid'])
+                  || row.original._uuid;
 
                 if (mediaAttachment !== null && q.$xpath !== undefined) {
                   return (

jsapp/js/dataInterface.ts

@@ -215,6 +215,7 @@ export interface SubmissionResponse {
   end?: string;
   'formhub/uuid': string;
   'meta/instanceID': string;
+  'meta/rootUuid': string;
   phonenumber?: string;
   start?: string;
   today?: string;

jsapp/js/utils.ts

@@ -532,3 +532,59 @@ export function getExponentialDelayTime(
 
   return count;
 }
+
+// Submission UUID comparison helpers
+
+/**
+ * Add a default 'uuid:' prefix to the provided identifier,
+ * if there's no prefix already.
+ *
+ *  Examples:
+ *    'kobotoolbox.org:123456789' -> unchanged
+ *    'uuid:123456789' -> unchanged
+ *    '123456789' -> 'uuid:123456789'
+ *
+ * 🐍 Equivalent to add_uuid_prefix on the backend.
+ */
+export function addDefaultUuidPrefix(uuid: string) {
+  return uuid.includes(':') ? uuid : `uuid:${uuid}`;
+}
+
+/**
+ * Remove the default 'uuid:' prefix from a provided identifier,
+ * while preserving any custom prefixes (e.g. 'kobotoolbox.org:123456789'),
+ * which are allowed by OpenRosa to reduce the chance of ID collisions.
+ *
+ * 🐍 Equivalent to remove_uuid_prefix on the backend.
+ */
+export function removeDefaultUuidPrefix(uuid: string) {
+  return uuid.replace(/^uuid:/, '');
+}
+
+/**
+ * Compare any two uuid's, accounting for the presence or absence of
+ * the default `'uuid:'` prefix in `meta/instanceId` and `meta/rootUuid`.
+ *
+ * Use this when comparing a `_uuid` with one of those `meta/` fields,
+ * since the meta fields include the `uuid:` prefix but the _uuid field
+ * strips them.
+ *
+ * Usage examples:
+ *
+ *     matchUuid( _uuid, rootUuid )   // ✅ true if equivalent
+ *     matchUuid( _uuid, instanceId ) // ✅ true if equivalent
+ *
+ *     matchUuid( instanceId, rootUuid )  // ✔️ this works too
+ *
+ *     matchUuid( 'some-uuid-that-here-exists',
+ *           'uuid:some-uuid-that-here-exists') // ✔️ match true
+ *
+ *     matchUuid(     'uuid-collision',
+ *        'org.example:uuid-collision') // false (different namespace)
+ */
+export function matchUuid(uuidA: string, uuidB: string) {
+  return (
+    addDefaultUuidPrefix(uuidA) ===
+    addDefaultUuidPrefix(uuidB)
+  );
+}

kobo/apps/openrosa/apps/logger/xform_instance_parser.py

@@ -18,12 +18,18 @@
 from kpi.utils.log import logging
 
 
+def add_uuid_prefix(uuid_: str) -> str:
+    if ':' in uuid_:
+        return uuid_
+    return f'uuid:{uuid_}'
+
+
 def get_meta_node_from_xml(
     xml_str: str, meta_name: str
 ) -> Union[None, tuple[str, minidom.Document]]:
     xml = clean_and_parse_xml(xml_str)
     children = xml.childNodes
-    # children ideally contains a single element
+    # children ideally contain a single element
     # that is the parent of all survey elements
     if children.length == 0:
         raise ValueError(t('XML string must have a survey element.'))
@@ -58,37 +64,27 @@ def get_meta_from_xml(xml_str: str, meta_name: str) -> str:
 
 def get_uuid_from_xml(xml):
 
-    def _uuid_only(uuid):
-        """
-        Strips the 'uuid:' prefix from the provided identifier if it exists.
-        This preserves any custom ID schemes (e.g., 'kobotoolbox.org:123456789')
-        while ensuring only the 'uuid:' prefix is removed. This approach
-        adheres to the OpenRosa spec, allowing custom prefixes to be stored
-        intact in the database to prevent potential ID collisions.
-        """
-        return re.sub(r'^uuid:', '', uuid)
-
     uuid = get_meta_from_xml(xml, 'instanceID')
     if uuid:
-        return _uuid_only(uuid)
+        return remove_uuid_prefix(uuid)
     # check in survey_node attributes
     xml = clean_and_parse_xml(xml)
     children = xml.childNodes
-    # children ideally contains a single element
+    # children ideally contain a single element
     # that is the parent of all survey elements
     if children.length == 0:
         raise ValueError(t('XML string must have a survey element.'))
     survey_node = children[0]
     uuid = survey_node.getAttribute('instanceID')
     if uuid != '':
-        return _uuid_only(uuid)
+        return remove_uuid_prefix(uuid)
     return None
 
 
 def get_root_uuid_from_xml(xml):
     root_uuid = get_meta_from_xml(xml, 'rootUuid')
     if root_uuid:
-        return root_uuid
+        return remove_uuid_prefix(root_uuid)
 
     # If no rootUuid, fall back to instanceID
     return get_uuid_from_xml(xml)
@@ -98,7 +94,7 @@ def get_submission_date_from_xml(xml) -> Optional[datetime]:
     # check in survey_node attributes
     xml = clean_and_parse_xml(xml)
     children = xml.childNodes
-    # children ideally contains a single element
+    # children ideally contain a single element
     # that is the parent of all survey elements
     if children.length == 0:
         raise ValueError(t('XML string must have a survey element.'))
@@ -139,6 +135,17 @@ def set_meta(xml_str: str, meta_name: str, new_value: str) -> str:
     return root.toxml()
 
 
+def remove_uuid_prefix(uuid_: str) -> str:
+    """
+    Strips the 'uuid:' prefix from the provided identifier if it exists.
+    This preserves any custom ID schemes (e.g., 'kobotoolbox.org:123456789')
+    while ensuring only the 'uuid:' prefix is removed. This approach
+    adheres to the OpenRosa spec, allowing custom prefixes to be stored
+    intact in the database to prevent potential ID collisions.
+    """
+    return re.sub(r'^uuid:', '', uuid_)
+
+
 def _xml_node_to_dict(node: Node, repeats: list = []) -> dict:
     assert isinstance(node, Node)
     if len(node.childNodes) == 0:
@@ -346,9 +353,11 @@ def _set_attributes(self):
             try:
                 assert key not in self._attributes
             except AssertionError:
-                logging.debug(
-                    f'Skipping duplicate attribute: {key} with value {value}'
-                )
+                pass
+                # Lines commented below to avoid flooding dev console
+                # logging.debug(
+                #     f'Skipping duplicate attribute: {key} with value {value}'
+                # )
             else:
                 self._attributes[key] = value
 

kobo/apps/openrosa/apps/viewer/models/parsed_instance.py

@@ -10,10 +10,12 @@
 
 from kobo.apps.hook.utils.services import call_services
 from kobo.apps.openrosa.apps.logger.models import Instance, Note, XForm
+from kobo.apps.openrosa.apps.logger.xform_instance_parser import add_uuid_prefix
 from kobo.apps.openrosa.libs.utils.common_tags import (
     ATTACHMENTS,
     GEOLOCATION,
     ID,
+    META_ROOT_UUID,
     MONGO_STRFTIME,
     NOTES,
     SUBMISSION_TIME,
@@ -285,6 +287,7 @@ def to_dict_for_mongo(self):
 
         data = {
             UUID: self.instance.uuid,
+            META_ROOT_UUID: add_uuid_prefix(self.instance.root_uuid),
             ID: self.instance.id,
             ATTACHMENTS: _get_attachments_from_instance(self.instance),
             self.STATUS: self.instance.status,

kobo/apps/openrosa/apps/viewer/tests/test_pandas_mongo_bridge.py

@@ -219,6 +219,7 @@ def test_csv_columns_for_gps_within_groups(self):
             'gps_group/_gps_altitude',
             'gps_group/_gps_precision',
             'meta/instanceID',
+            'meta/rootUuid',
             'web_browsers/firefox',
             'web_browsers/chrome',
             'web_browsers/ie',
@@ -259,6 +260,7 @@ def test_format_mongo_data_for_csv(self):
             'kids/kids_details[2]/kids_name': 'Cain',
             'kids/kids_details[2]/kids_age': '76',
             'meta/instanceID': 'uuid:435f173c688e482463a486617004534df',
+            'meta/rootUuid': 'uuid:435f173c688e482463a486617004534df',
             'web_browsers/chrome': True,
             'web_browsers/ie': True,
             'web_browsers/safari': False,
@@ -573,7 +575,7 @@ def test_csv_column_indices_in_groups_within_repeats(self):
         # remove dynamic fields
         ignore_list = [
             '_uuid', 'meta/instanceID', 'formhub/uuid', '_submission_time',
-            '_id']
+            '_id', 'meta/rootUuid']
         for item in ignore_list:
             data_0.pop(item)
         expected_data_0 = {

kobo/apps/openrosa/libs/utils/common_tags.py

@@ -25,23 +25,24 @@
 NAME = "name"
 
 # extra fields that we're adding to our mongo doc
-XFORM_ID_STRING = "_xform_id_string"
-STATUS = "_status"
-ATTACHMENTS = "_attachments"
-UUID = "_uuid"
-USERFORM_ID = "_userform_id"
-DATE = "_date"
-GEOLOCATION = "_geolocation"
+XFORM_ID_STRING = '_xform_id_string'
+STATUS = '_status'
+ATTACHMENTS = '_attachments'
+UUID = '_uuid'
+USERFORM_ID = '_userform_id'
+DATE = '_date'
+GEOLOCATION = '_geolocation'
 SUBMISSION_TIME = '_submission_time'
-DELETEDAT = "_deleted_at"  # no longer used but may persist in old submissions
-SUBMITTED_BY = "_submitted_by"
-VALIDATION_STATUS = "_validation_status"
+DELETEDAT = '_deleted_at'  # no longer used but may persist in old submissions
+SUBMITTED_BY = '_submitted_by'
+VALIDATION_STATUS = '_validation_status'
 
-INSTANCE_ID = "instanceID"
-META_INSTANCE_ID = "meta/instanceID"
-INDEX = "_index"
-PARENT_INDEX = "_parent_index"
-PARENT_TABLE_NAME = "_parent_table_name"
+INSTANCE_ID = 'instanceID'
+META_INSTANCE_ID = 'meta/instanceID'
+META_ROOT_UUID = 'meta/rootUuid'
+INDEX = '_index'
+PARENT_INDEX = '_parent_index'
+PARENT_TABLE_NAME = '_parent_table_name'
 
 # datetime format that we store in mongo
 MONGO_STRFTIME = '%Y-%m-%dT%H:%M:%S'

kobo/apps/subsequences/migrations/0004_increase_subsequences_submission_uuid.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2025-02-10 18:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('subsequences', '0003_alter_submissionextras_date_created_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='submissionextras',
+            name='submission_uuid',
+            field=models.CharField(max_length=249),
+        ),
+    ]

kobo/apps/subsequences/models.py

@@ -9,8 +9,7 @@
 
 class SubmissionExtras(AbstractTimeStampedModel):
 
-    # FIXME: uuid on the KoboCAT logger.Instance model has max_length 249
-    submission_uuid = models.CharField(max_length=40)
+    submission_uuid = models.CharField(max_length=249)
     content = models.JSONField(default=dict)
     asset = models.ForeignKey(
         Asset,

kobo/apps/subsequences/utils/__init__.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from copy import deepcopy
 
+from kobo.apps.openrosa.apps.logger.xform_instance_parser import remove_uuid_prefix
 from ..actions.automatic_transcription import AutomaticTranscriptionAction
 from ..actions.qual import QualAction
 from ..actions.translation import TranslationAction
@@ -162,7 +163,7 @@ def stream_with_extras(submission_stream, asset):
 
     for submission in submission_stream:
         if SUBMISSION_UUID_FIELD in submission:
-            uuid = submission[SUBMISSION_UUID_FIELD]
+            uuid = remove_uuid_prefix(submission[SUBMISSION_UUID_FIELD])
         else:
             uuid = submission['_uuid']