Merge pull request #2729 from kobotoolbox/kobotoolbox/formpack#219

Handle (undocumented) XLSForm `disabled` column appropriately

dependencies/pip/dev_requirements.txt

@@ -5,7 +5,7 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@18462f320b39e6504a0788f379a0dca5651e9748#egg=formpack  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/formpack.git@d8d7a64acd69f6c6fa56638588dbe38c7a97a1e0#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu

dependencies/pip/external_services.txt

@@ -5,7 +5,7 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@18462f320b39e6504a0788f379a0dca5651e9748#egg=formpack  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/formpack.git@d8d7a64acd69f6c6fa56638588dbe38c7a97a1e0#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu

dependencies/pip/requirements.in

@@ -2,7 +2,7 @@
 # https://github.com/bndr/pipreqs is a handy utility, too.
 
 # formpack
--e git+https://github.com/kobotoolbox/formpack.git@18462f320b39e6504a0788f379a0dca5651e9748#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@d8d7a64acd69f6c6fa56638588dbe38c7a97a1e0#egg=formpack
 
 # More up-to-date version of django-digest than PyPI seems to have.
 # Also, python-digest is an unlisted dependency thereof.

dependencies/pip/requirements.txt

@@ -5,7 +5,7 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@18462f320b39e6504a0788f379a0dca5651e9748#egg=formpack  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/formpack.git@d8d7a64acd69f6c6fa56638588dbe38c7a97a1e0#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu

dependencies/pip/travis_ci.txt

@@ -5,7 +5,7 @@
 #    make pip_compile
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest  # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@18462f320b39e6504a0788f379a0dca5651e9748#egg=formpack  # via -r dependencies/pip/requirements.in
+-e git+https://github.com/kobotoolbox/formpack.git@d8d7a64acd69f6c6fa56638588dbe38c7a97a1e0#egg=formpack  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest  # via -r dependencies/pip/requirements.in
 -e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect  # via -r dependencies/pip/requirements.in
 amqp==2.5.2               # via -r dependencies/pip/requirements.in, kombu

kpi/tests/test_mock_data_exports.py

@@ -204,20 +204,25 @@ def setUp(self):
                 self.asset.owner.id)
         )
 
-    def run_csv_export_test(self, expected_lines, export_options=None, user=None):
+    def run_csv_export_test(
+        self, expected_lines, export_options=None, asset=None, user=None
+    ):
         """
         Repeat yourself less while writing CSV export tests.
 
         `expected_lines`: a list of strings *without* trailing newlines whose
                           UTF-8 encoded representation should match the export
                           result
-        `export_options`: a list of extra options for `ExportTask.data`. Do not
-                          include `source` or `type`
+        `export_options`: (optional) a list of extra options for
+                          `ExportTask.data`. Do not include `source` or `type`
+        `asset`: (optional) the asset to export. Defaults to `self.asset`
+        `user`: (optional) the user to own the export. Defaults to `self.user`
         """
         export_task = ExportTask()
+        asset = self.asset if asset is None else asset
         export_task.user = self.user if user is None else user
         export_task.data = {
-            'source': reverse('asset-detail', args=[self.asset.uid]),
+            'source': reverse('asset-detail', args=[asset.uid]),
             'type': 'csv'
         }
         if export_options:
@@ -617,3 +622,25 @@ def test_export_exceeding_api_submission_limit(self):
         export_task._run_task(messages)
         # Don't forget to add one for the header row!
         self.assertEqual(len(list(export_task.result)), limit + excess + 1)
+
+    def test_export_with_disabled_questions(self):
+        asset = Asset.objects.create(
+            name='Form with undocumented `disabled` column',
+            owner=self.asset.owner,
+            content={'survey': [
+                {'name': 'q', 'type': 'integer'},
+                {'name': 'ignore', 'type': 'select_one nope', 'disabled': True},
+            ]},
+        )
+        asset.deploy(backend='mock', active=True)
+        asset.deployment.mock_submissions(
+            [{'__version__': asset.latest_deployed_version.uid, 'q': 123,}]
+        )
+        # observe that `ignore` does not appear!
+        expected_lines = [
+            '"q";"_id";"_uuid";"_submission_time";"_validation_status";"_index"',
+            '"123";"";"";"";"";"1"',
+        ]
+        # fails with `KeyError` prior to fix for kobotoolbox/formpack#219
+        self.run_csv_export_test(expected_lines, asset=asset)
+