Fix azimuth noise with holes bug in SAR-C reader

satpy/readers/sar_c_safe.py

@@ -35,10 +35,10 @@
 """
 
 import logging
-import xml.etree.ElementTree as ET
 from functools import lru_cache
 from threading import Lock
 
+import defusedxml.ElementTree as ET
 import numpy as np
 import rasterio
 import rioxarray
@@ -281,17 +281,21 @@ def _create_dask_slices_from_blocks(self, chunks):
 
     def _create_dask_slice_from_block_line(self, current_line, chunks):
         """Create a dask slice from the blocks at the current line."""
-        current_blocks = self._find_blocks_covering_line(current_line)
-        current_blocks.sort(key=(lambda x: x.coords['x'][0]))
-
-        next_line = min((arr.coords['y'][-1] for arr in current_blocks))
-        current_y = np.arange(current_line, next_line + 1)
-
-        pieces = [arr.sel(y=current_y) for arr in current_blocks]
+        pieces = self._get_array_pieces_for_current_line(current_line)
         dask_pieces = self._get_padded_dask_pieces(pieces, chunks)
         new_slice = da.hstack(dask_pieces)
+
         return new_slice
 
+    def _get_array_pieces_for_current_line(self, current_line):
+        """Get the array pieces that cover the current line."""
+        current_blocks = self._find_blocks_covering_line(current_line)
+        current_blocks.sort(key=(lambda x: x.coords['x'][0]))
+        next_line = self._get_next_start_line(current_blocks, current_line)
+        current_y = np.arange(current_line, next_line)
+        pieces = [arr.sel(y=current_y) for arr in current_blocks]
+        return pieces
+
     def _find_blocks_covering_line(self, current_line):
         """Find the blocks covering a given line."""
         current_blocks = []
@@ -300,13 +304,38 @@ def _find_blocks_covering_line(self, current_line):
                 current_blocks.append(block)
         return current_blocks
 
+    def _get_next_start_line(self, current_blocks, current_line):
+        next_line = min((arr.coords['y'][-1] for arr in current_blocks)) + 1
+        blocks_starting_soon = [block for block in self.blocks if current_line < block.coords["y"][0] < next_line]
+        if blocks_starting_soon:
+            next_start_line = min((arr.coords["y"][0] for arr in blocks_starting_soon))
+            next_line = min(next_line, next_start_line)
+        return next_line
+
     def _get_padded_dask_pieces(self, pieces, chunks):
         """Get the padded pieces of a slice."""
-        dask_pieces = [piece.data for piece in pieces]
+        pieces = sorted(pieces, key=(lambda x: x.coords['x'][0]))
+        dask_pieces = self._get_full_dask_pieces(pieces, chunks)
         self._pad_dask_pieces_before(pieces, dask_pieces, chunks)
         self._pad_dask_pieces_after(pieces, dask_pieces, chunks)
         return dask_pieces
 
+    @staticmethod
+    def _get_full_dask_pieces(pieces, chunks):
+        """Get the dask pieces without wholes."""
+        dask_pieces = []
+        for i, piece in enumerate(pieces[:-1]):
+            dask_pieces.append(piece.data)
+            previous_x_end = piece.coords['x'][-1]
+            next_x_start = pieces[i + 1].coords['x'][0]
+            if previous_x_end != next_x_start - 1:
+                missing_x = np.arange(previous_x_end, next_x_start - 1)
+                missing_y = piece.coords['y']
+                new_piece = da.full((len(missing_y), len(missing_x)), np.nan, chunks=chunks)
+                dask_pieces.append(new_piece)
+        dask_pieces.append(pieces[-1].data)
+        return dask_pieces
+
     @staticmethod
     def _pad_dask_pieces_before(pieces, dask_pieces, chunks):
         """Pad the dask pieces before."""

satpy/tests/reader_tests/test_sar_c_safe.py

@@ -626,8 +626,6 @@ def setUp(self):
         filename_info = dict(start_time=None, end_time=None, polarization="vv")
         self.annotation_fh = SAFEXMLAnnotation(BytesIO(annotation_xml), filename_info, mock.MagicMock())
         self.noise_fh = SAFEXMLNoise(BytesIO(noise_xml), filename_info, mock.MagicMock(), self.annotation_fh)
-        self.noise_fh_with_holes = SAFEXMLNoise(BytesIO(noise_xml_with_holes), filename_info, mock.MagicMock(),
-                                                self.annotation_fh)
 
         self.expected_azimuth_noise = np.array([[np.nan, 1, 1, 1, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
                                                 [np.nan, 1, 1, 1, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
@@ -653,6 +651,8 @@ def setUp(self):
                                               [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
                                               ])
 
+        self.noise_fh_with_holes = SAFEXMLNoise(BytesIO(noise_xml_with_holes), filename_info, mock.MagicMock(),
+                                                self.annotation_fh)
         self.expected_azimuth_noise_with_holes = np.array(
             [[np.nan, np.nan, np.nan, 1, 1, 1, np.nan, np.nan, np.nan, np.nan],
              [2, 2, np.nan, 1, 1, 1, np.nan, np.nan, np.nan, np.nan],

setup.py

@@ -37,7 +37,7 @@
 
 test_requires = ['behave', 'h5py', 'netCDF4', 'pyhdf', 'imageio', 'pylibtiff',
                  'rasterio', 'geoviews', 'trollimage', 'fsspec', 'bottleneck',
-                 'rioxarray', 'pytest', 'pytest-lazy-fixture']
+                 'rioxarray', 'pytest', 'pytest-lazy-fixture', 'defusedxml']
 
 extras_require = {
     # Readers:
@@ -55,7 +55,7 @@
     'hrit_msg': ['pytroll-schedule'],
     'msi_safe': ['rioxarray', "bottleneck", "python-geotiepoints"],
     'nc_nwcsaf_msg': ['netCDF4 >= 1.1.8'],
-    'sar_c': ['python-geotiepoints >= 1.1.7', 'rasterio', 'rioxarray'],
+    'sar_c': ['python-geotiepoints >= 1.1.7', 'rasterio', 'rioxarray', 'defusedxml'],
     'abi_l1b': ['h5netcdf'],
     'seviri_l1b_hrit': ['pyorbital >= 1.3.1'],
     'seviri_l1b_native': ['pyorbital >= 1.3.1'],