Merge pull request #60 from bodleian/depthmap_fixes

Depthmap fixes

docs/jp2_colour_management.rst

@@ -58,6 +58,11 @@ The ``kdu_compress`` :ref:`command options <kdu_compress-options>` we use do not
 
 As very few of our input images have an alpha channel, and we have not encountered failing cases in live data (as opposed to constructed test data), this is not a priority for us. We just make sure to check RGBA JP2 conversions are lossless, so we will catch any future failing cases.
 
+RGBX colour mode
+~~~~~~~~~~~~~~~~~~
+
+I could not find a simple way to convert an RGBX TIFF to an RGBX JP2, though it's possible more work would produce one. In our particular use case it's always clear whether the image should be interpreted as RGBA or RGBX, so we convert RGBX images to RGBA, preserving all of the pixel data but not the colour mode. The program logs a warning when converting RGBX files because of this.
+
 Colour profiles / modes not supported by JP2
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

image_processing/conversion.py

@@ -47,10 +47,16 @@ def convert_to_jpg(self, input_filepath, output_filepath, resize=None, quality=N
         """
         with Image.open(input_filepath) as input_pil:
             icc_profile = input_pil.info.get('icc_profile')
-            if input_pil.mode == 'RGBA':
+            if input_pil.mode in ['RGBA', 'RGBX']:
                 self.logger.warning(
-                    'Image is RGBA - the alpha channel will be removed from the JPEG derivative image')
+                    'Image is %s - the fourth channel will be removed from the JPEG derivative image', input_pil.mode)
                 input_pil = input_pil.convert(mode="RGB")
+            if input_pil.mode == 'I;16':
+                # JPEG doesn't support 16bit
+                self.logger.warning(
+                    'Image is 16bpp - will be downsampled to 8bpp')
+                input_pil = input_pil.convert(mode="RGB")
+
             # libjpeg has a maximum dimension of 65,500, which is lower than the actual maximum jpeg dimension of 65,535
             # JPEG2000 does not have the restriction
             # if we're not already scaling the jpeg, then clamp the thumbnail to the max supported dimensions

image_processing/derivative_files_generator.py

@@ -216,6 +216,11 @@ def generate_jp2_from_tiff(self, tiff_file, jp2_filepath):
             if tiff_pil.mode == 'RGBA':
                 if kakadu.ALPHA_OPTION not in kakadu_options:
                     kakadu_options += [kakadu.ALPHA_OPTION]
+            elif tiff_pil.mode == 'RGBX':
+                self.log.warning('Input tiff has colour mode RGBX. It will be converted to RGBA')
+                if kakadu.ALPHA_OPTION not in kakadu_options:
+                    kakadu_options += [kakadu.ALPHA_OPTION]
+
 
         self.kakadu.kdu_compress(tiff_file, jp2_filepath, kakadu_options=kakadu_options)
         self.log.debug('Lossless jp2 file {0} generated'.format(jp2_filepath))

image_processing/validation.py

@@ -14,7 +14,7 @@
 GREYSCALE = 'L'
 BITONAL = '1'
 MONOTONE_COLOUR_MODES = [GREYSCALE, BITONAL]
-ACCEPTED_COLOUR_MODES = ['RGB', 'RGBA', GREYSCALE, BITONAL]
+ACCEPTED_COLOUR_MODES = ['RGB', 'RGBA', 'RGBX', 'I;16', GREYSCALE, BITONAL]
 
 
 def validate_jp2(image_file, output_file=None):
@@ -154,10 +154,11 @@ def check_colour_profiles_match(source_filepath, converted_filepath):
             if source_image.mode != converted_image.mode:
                 if source_image.mode == BITONAL and converted_image.mode == GREYSCALE:
                     logger.info('Converted image is greyscale, not bitonal. This is expected')
+                elif source_image.mode == 'RGBX' and converted_image.mode == 'RGBA':
+                    logger.info('Converted image in RGBA space, but was converted from RGBX. This is expected.')
                 else:
                     raise exceptions.ValidationError(
-                        'Converted file {0} has different colour mode from {1}'
-                        .format(converted_filepath, source_filepath)
+                        f'Converted file {converted_filepath} has different colour mode ({converted_image.mode}) from {source_filepath} ({source_image.mode})'
                     )
 
             source_icc = source_image.info.get('icc_profile')
@@ -188,7 +189,10 @@ def check_image_suitable_for_jp2_conversion(image_filepath, require_icc_profile_
         if colour_mode not in ACCEPTED_COLOUR_MODES:
             raise exceptions.ValidationError("Unsupported colour mode {0} for {1}".format(colour_mode, image_filepath))
 
-        if colour_mode == 'RGBA':
+        if colour_mode == 'RGBX':
+            logger.warning("{0} is RGBX and will convert to a RGBA jp2, preserving the pixel information but losing the colour mode".format(image_filepath))
+
+        if colour_mode in ['RGBA', 'RGBX']:
             # In some cases alpha channel data is stored in a way that means it would be lost in the conversion back to
             # tiff from jp2.
             # "Kakadu Warning:
@@ -198,9 +202,10 @@ def check_image_suitable_for_jp2_conversion(image_filepath, require_icc_profile_
 
             # As we rarely encounter RGBA files, and mostly ones without any alpha channel data, we just warn here
             # the visually identical check should pick up any problems
-            logger.warning("You must double check the jp2 conversion is lossless. "
-                        "{0} is an RGBA image, and the resulting jp2 may convert back to an RGB tiff "
-                        "if the alpha channel is unassociated".format(image_filepath))
+            logger.warning("You must check the jp2 conversion is lossless. "
+                            "{0} will convert to a RGBA jp2, and may convert back to an RGB tiff "
+                            "if the alpha channel is unassociated."
+                           "The usual visually identical check will detect this if run".format(image_filepath))
 
         icc_needed = (require_icc_profile_for_greyscale and colour_mode == GREYSCALE) \
             or (require_icc_profile_for_colour and colour_mode not in MONOTONE_COLOUR_MODES)

tests/test_conversion.py

@@ -44,6 +44,20 @@ def test_converts_tif_to_jpeg(self):
             assert os.path.isfile(output_file)
             assert image_files_match(output_file, filepaths.HIGH_QUALITY_JPG_FROM_STANDARD_TIF)
 
+    def test_converts_depthmap_tif_to_jpeg(self):
+        with temporary_folder() as output_folder:
+            output_file = os.path.join(output_folder, 'output.jpg')
+            conversion.Converter().convert_to_jpg(filepaths.DEPTHMAP_TIF, output_file, resize=None,
+                                                  quality=derivative_files_generator.DEFAULT_JPG_HIGH_QUALITY_VALUE)
+            assert os.path.isfile(output_file)
+
+    def test_converts_normalmap_tif_to_jpeg(self):
+        with temporary_folder() as output_folder:
+            output_file = os.path.join(output_folder, 'output.jpg')
+            conversion.Converter().convert_to_jpg(filepaths.NORMALMAP_TIF, output_file, resize=None,
+                                                  quality=derivative_files_generator.DEFAULT_JPG_HIGH_QUALITY_VALUE)
+            assert os.path.isfile(output_file)
+
     @mark.skipif(not cmd_is_executable('/opt/kakadu/kdu_compress'), reason="requires kakadu installed")
     def test_converts_tif_to_lossy_jpeg2000(self):
         with temporary_folder() as output_folder:

tests/test_derivatives_generator.py

@@ -225,3 +225,28 @@ def test_creates_correct_files_from_jpg(self):
             assert image_files_match(jpg_file, filepaths.STANDARD_JPG)
             assert image_files_match(jp2_file, filepaths.LOSSLESS_JP2_FROM_STANDARD_JPG_XMP)
             assert xmp_files_match(embedded_metadata_file, filepaths.STANDARD_JPG_XMP)
+
+    def test_converts_i_16_tif_to_jpeg2000_losslessly(self):
+        """
+        Check transformations of 16 bit depthmap images do not lose data
+        :return:
+        """
+        with temporary_folder() as output_folder:
+            generator = get_derivatives_generator()
+            output_file = os.path.join(output_folder, 'output.jp2')
+            generator.generate_jp2_from_tiff(filepaths.DEPTHMAP_TIF, output_file)
+            assert os.path.isfile(output_file)
+            generator.check_conversion_was_lossless(filepaths.DEPTHMAP_TIF, output_file)
+
+    def test_converts_rgbx_tif_to_jpeg2000_losslessly(self):
+        """
+        Check transformations of RGBX normal map images do not lose data
+        :return:
+        """
+        with temporary_folder() as output_folder:
+            generator = get_derivatives_generator()
+            output_file = os.path.join(output_folder, 'output.jp2')
+            # note that this method has specific handling for files with four channels that the direct converter method does not
+            generator.generate_jp2_from_tiff(filepaths.NORMALMAP_TIF, output_file)
+            assert os.path.isfile(output_file)
+            generator.check_conversion_was_lossless(filepaths.NORMALMAP_TIF, output_file)

tests/test_utils/filepaths.py

@@ -49,11 +49,15 @@
 
 NO_PROFILE_TIF = 'tests/data/no_profile.tif'
 
+# tifs from our archiox project, which can have unusual colour modes
+DEPTHMAP_TIF = 'tests/data/depth_map.tif'  # 16 bit (I;16) archiox image
+NORMALMAP_TIF = 'tests/data/normal_map.tif'  # RGBX archiox image (compressed, as decompression with Image Magick converts it to RGBA)
+
 # just truncated files
 INVALID_JP2 = 'tests/data/invalid.jp2'
 INVALID_TIF = 'tests/data/invalid.tif'
 
 KAKADU_BASE_PATH = '/opt/kakadu'
 DEFAULT_IMAGE_MAGICK_PATH = '/usr/bin/'
 
-SRGB_ICC_PROFILE = 'tests/data/sRGB_v4_ICC_preference.icc'
+SRGB_ICC_PROFILE = 'tests/data/sRGB_v4_ICC_preference.icc'