Patch for 6.1.4

Inventory.py

@@ -26,8 +26,7 @@
 
 import Constants
 import FileUtils
-from DatastripReader import DatastripReader
-from InventoryMetadataReader import InventoryMetadataReader
+
 from JobOrderInventoryReader import JobOrderInventoryReader
 from TileMtdReader import TileMtdReader
 
@@ -88,12 +87,14 @@ def split_granules_jobs(self, granules, working, output, ds_input, acquisition_s
             real_nb_tasks = len(granules)
         step = int(math.ceil(float(len(granules)) / real_nb_tasks))
         self.logger.debug("Nb granule per instance : "+str(step))
+        nb_used=0
         for i in range(real_nb_tasks):
+            step = int((len(granules) - nb_used) / (real_nb_tasks - i))
             inventory_job_order = os.path.join(working, "JobOrderInventory_"+str(i+1)+".xml")
-            if i < nb_tasks:
-                job_granules = granules[i*step:(i+1)*step]
-            else:
-                job_granules = granules[i * step:]
+            self.logger.debug("i: "+str(i)+", "+str(nb_used)+":"+str(nb_used+step))
+            job_granules = granules[nb_used:nb_used+step]
+            nb_used += step
+            self.logger.debug("len: "+str(len(job_granules)))
             if granule_type.startswith("L1"):
                 self._write_gr_l1_job_order(inventory_job_order, job_granules, working,
                                             acquisition_station,
@@ -228,28 +229,3 @@ def _copy_olqc_report(self, folder, list_of_reports, output_folder):
             if report_file is None:
                 raise Exception("No "+report+" in folder : " + folder)
             FileUtils.copy_file(report_file, os.path.join(output_folder, os.path.basename(report_file)))
-
-    def _patch_inventory_dates_datastrip(self, datastrip_out):
-        self.logger.debug("Patching inventory mtd dates for datastrip : "+datastrip_out)
-        # read mtd ds
-        datastrip_xml_out = glob.glob(os.path.join(datastrip_out, "S2*DS*xml"))[0]
-        reader_mt = DatastripReader(datastrip_xml_out)
-        sensing_start = reader_mt.get_sensing_start_utc()
-        sensing_stop = reader_mt.get_sensing_stop_utc()
-        inventory_xml_out = os.path.join(datastrip_out, "Inventory_Metadata.xml")
-        reader_inventory = InventoryMetadataReader(inventory_xml_out)
-        reader_inventory.set_val_start_date(sensing_start)
-        reader_inventory.set_val_stop_date(sensing_stop)
-        reader_inventory.write_to_file(inventory_xml_out)
-
-    def _patch_inventory_dates_granule(self, granule_out):
-        self.logger.debug("Patching inventory mtd dates for granule : "+granule_out)
-        # read mtd gr
-        granule_xml_out = glob.glob(os.path.join(granule_out, "S2*xml"))[0]
-        reader_mt = DatastripReader(granule_xml_out)
-        sensing_time = reader_mt.get_sensing_time_utc()
-        inventory_xml_out = os.path.join(granule_out, "Inventory_Metadata.xml")
-        reader_inventory = InventoryMetadataReader(inventory_xml_out)
-        reader_inventory.set_val_start_date(sensing_time)
-        reader_inventory.set_val_stop_date(sensing_time)
-        reader_inventory.write_to_file(inventory_xml_out)

InventoryL1A.py

@@ -103,8 +103,6 @@ def _inventory_l1a_ds(self, context_manager, dateoflogs, gipp_folder,
         # Check if inventory has been generated
         if not os.path.exists(os.path.join(datastrip_out, "Inventory_Metadata.xml")):
             raise Exception("No Inventory_Metadata.xml file generated in "+datastrip_out)
-        # Patch dates
-        self._patch_inventory_dates_datastrip(datastrip_out)
         # tar datastrip
         datastrip_filename = os.path.join(output_folder_inventory, os.path.basename(datastrip_out))
         FileUtils.copy_directory_recursive(datastrip_out, datastrip_filename)
@@ -168,8 +166,6 @@ def _inventory_l1a_gr(self, context_manager, dateoflogs,
             # Check if inventory has been generated
             if not os.path.exists(os.path.join(gr, "Inventory_Metadata.xml")):
                 raise Exception("No Inventory_Metadata.xml file generated in " + gr)
-            # Patch dates
-            self._patch_inventory_dates_granule(gr)
             granule_filename = os.path.join(output_folder_inventory, os.path.basename(gr))
             # Copy OLQC reports
             self.__copy_olqc_granule(gr, output_folder_inventory)

InventoryL1B.py

@@ -21,9 +21,7 @@
 
 import Constants
 import FileUtils
-from DatastripReader import DatastripReader
 from Inventory import Inventory
-from InventoryMetadataReader import InventoryMetadataReader
 from JobOrderInventoryReader import JobOrderInventoryReader
 
 
@@ -104,8 +102,6 @@ def _inventory_l1b_ds(self, context_manager, dateoflogs, gipp_folder, acquisitio
         # Check if inventory has been generated
         if not os.path.exists(os.path.join(datastrip_out, "Inventory_Metadata.xml")):
             raise Exception("No Inventory_Metadata.xml file generated in " + datastrip_out)
-        # Patch the dates
-        self._patch_inventory_dates_datastrip(datastrip_out)
         # Tar the datastrip
         ds_basename = os.path.basename(datastrip_out)
         out_file = os.path.join(output_folder_inventory, ds_basename)
@@ -146,7 +142,7 @@ def _inventory_l1b_ds(self, context_manager, dateoflogs, gipp_folder, acquisitio
 
     def _inventory_l1b_gr(self, context_manager, dateoflogs, acquisition_station, working_dir, nb_tasks):
         self.logger.info("Doing inventory for L1B Granules")
-        pdi_ds_output_folder = context_manager.get_element("PROC.PDI_DS")
+        pdi_ds_output_folder = context_manager.get_element("PROC.L1B.PDI_DS")
         if pdi_ds_output_folder is None:
             raise Exception("No PROC.PDI_DS in context !!!")
         datastrips_out = glob.glob(os.path.join(pdi_ds_output_folder[0], "S2*DS*"))
@@ -171,15 +167,14 @@ def _inventory_l1b_gr(self, context_manager, dateoflogs, acquisition_station, wo
             FileUtils.create_directory(output_folder_inventory)
         # Create the job orders
         jobs = self.split_granules_jobs(granules_out, inventory_working_dir, output_folder_inventory, datastrip_out,
-                                        acquisition_station, len(granules_out), "L1B")#nb_tasks, "L1B")
+                                        acquisition_station, nb_tasks, "L1B")
         # Launch process
         self._launch_inventory_process(jobs, "GR_L1B", self._gr_inventory_script)
         # Tar the granules
         for gr in granules_out:
             # Check if inventory has been generated
             if not os.path.exists(os.path.join(gr, "Inventory_Metadata.xml")):
                 raise Exception("No Inventory_Metadata.xml file generated in " + gr)
-            self._patch_inventory_dates_granule(gr)
             granule_filename = os.path.join(output_folder_inventory, os.path.basename(gr))
             self.logger.debug("Compressing "+gr+" to "+granule_filename)
             # Copy OLQC reports

InventoryL1C.py

@@ -114,8 +114,6 @@ def _inventory_l1c_ds(self, context_manager, dateoflogs, gipp_folder, acquisitio
         # Check if inventory has been generated
         if not os.path.exists(os.path.join(datastrip_out, "Inventory_Metadata.xml")):
             raise Exception("No Inventory_Metadata.xml file generated in " + datastrip_out)
-        # Patch the dates
-        self._patch_inventory_dates_datastrip(datastrip_out)
         # Tar the datastrip
         ds_basename = os.path.basename(datastrip_out)
         out_file = os.path.join(output_folder_inventory, ds_basename)
@@ -207,8 +205,6 @@ def _inventory_l1c_tl(self, context_manager, tiles_names, dateoflogs, acquisitio
             inventory_mtd_reader = InventoryMetadataReader(os.path.join(tl, "Inventory_Metadata.xml"))
             if inventory_mtd_reader.get_file_id() is None or len(inventory_mtd_reader.get_file_id()) == 0:
                 raise Exception("Inventory_Metadata.xml is not correctly generated for " + tl)
-            # Patch the dates
-            self._patch_inventory_dates_granule(tl)
             # Remove VECTOR folder
             if os.path.isdir(os.path.join(tl, "QI_DATA", "VECTOR")):
                 shutil.rmtree(os.path.join(tl, "QI_DATA", "VECTOR"))

VersionManager.py

@@ -63,6 +63,13 @@ def __init__(self, probas, exe_file, trigger_file, from_probas, loglevel):
             self._exe_json["OLQC"] = self._probas_version_map["OLQC"][0]
             if "GSE" in self._probas_version_map.keys():
                 self._exe_json["GSE"] = self._probas_version_map["GSE"][0]
+            if "INV_MTD_L1" in self._probas_version_map.keys():
+                self._exe_json["INVENTORY_L1_SOFT_VERSION"] = self._probas_version_map["INV_MTD_L1"][0]
+                self._exe_json["INVENTORY_L1_SOFT_FOLDER"] = os.path.join(Constants.IDPSC_EXE_DIR, "INV_MTD_L1",
+                                                                          self._exe_json["INVENTORY_L1_SOFT_VERSION"],
+                                                                          "scripts")
+                self._logger.info("Using proba inventory L1 version "+self._exe_json["INVENTORY_L1_SOFT_VERSION"]+" in "
+                                  +self._exe_json["INVENTORY_L1_SOFT_FOLDER"])
             for chain in self._exe_json["CHAINS"]:
                 for idp in self._exe_json["CHAINS"][chain]:
                     self._exe_json["CHAINS"][chain][idp] = self._probas_version_map[idp][0]
@@ -106,7 +113,7 @@ def get_inventory_l0_gr_script_name(self):
         return self._exe_json["INVENTORY_L0_GR_SCRIPT_NAME"]
 
     def get_inventory_l0_soft_version(self):
-        return self._exe_json["INVENTORY_L1_SOFT_VERSION"]
+        return self._exe_json["INVENTORY_L0_SOFT_VERSION"]
 
     def get_inventory_l0_ds_script_path(self):
         return os.path.join( self.get_inventory_l0_soft_folder(),self.get_inventory_l0_ds_script_name())

config/6.1/orchestratorICD.json

@@ -997,7 +997,7 @@
       ],
       [
         "LAND_WATER_MASKS_PROCESS",
-        "true"
+        "false"
       ],
       [
         "PARALLEL_TILE",

config/6.1/orchestratorICD_L0C.json

@@ -263,6 +263,18 @@
         "PROCESSING_CENTER_ORG",
         "ESA"
       ],
+      [
+        "PROCESSING_CENTER_NAME",
+        "EDRS"
+      ],
+      [
+        "PROCESSING_CENTER_COUNTRY",
+        "Italy"
+      ],
+      [
+        "PROCESSING_CENTER_ORG",
+        "ESA"
+      ],
       [
         "CREATION_DATE",
         "2014-06-30T14:00:00Z"

config/IDPSC_EXE_export.json

@@ -4,11 +4,11 @@
   "INVENTORY_L0_SOFT_FOLDER" : "/usr/local/components/facilities/DPC-CORE-l0pack-inv-mtd/DPC-CORE-l0pack-inv-mtd-3.0.3/inventory_metadata_l0/scripts/",
   "INVENTORY_L0_DS_SCRIPT_NAME" : "inventory_metadata_l0_ds.sh",
   "INVENTORY_L0_GR_SCRIPT_NAME" : "inventory_metadata_l0_gr.sh",
-  "INVENTORY_L0_SOFT_VERSION" : "3.0.3",
-  "INVENTORY_L1_SOFT_FOLDER" : "/dpc/app/s2ipf/INV_MTD_L1/06.01.00/scripts/",
+  "INVENTORY_L0_SOFT_VERSION" : "03.00.03",
+  "INVENTORY_L1_SOFT_FOLDER" : "/dpc/app/s2ipf/INV_MTD_L1/06.01.04/scripts/",
   "INVENTORY_L1_DS_SCRIPT_NAME" : "INV_MTD_L1.bash",
   "INVENTORY_L1_GR_SCRIPT_NAME" : "INV_MTD_L1.bash",
-  "INVENTORY_L1_SOFT_VERSION" : "6.1.0",
+  "INVENTORY_L1_SOFT_VERSION" : "06.01.04",
   "OLQC" : "06.01.00",
   "GSE": "06.01.00",
   "CHAINS" : {
@@ -29,11 +29,11 @@
       "FORMAT_IMG_L1B": "06.01.00",
       "FORMAT_IMG_L1C": "06.01.00",
       "INIT_LOC_L1": "06.01.00",
-      "INIT_VS_GEO": "06.01.00",
+      "INIT_VS_GEO": "06.01.01",
       "MASK_S2": "06.01.00",
       "FORMAT_IMG_PVI_TCI": "06.01.00",
       "FORMAT_METADATA_DS_L1A": "06.01.00",
-      "FORMAT_METADATA_DS_L1B": "06.01.00",
+      "FORMAT_METADATA_DS_L1B": "06.01.04",
       "FORMAT_METADATA_DS_L1C": "06.01.00",
       "FORMAT_METADATA_GR_L1A": "06.01.00",
       "FORMAT_METADATA_GR_L1B": "06.01.00",
@@ -44,12 +44,12 @@
       "GET_TILE_LIST": "06.01.00",
       "RADIO_AB": "06.01.00",
       "RADIO_FINALIZE": "06.01.00",
-      "RESAMPLE_TO_VS": "06.01.00",
-      "SPATIO": "06.01.00",
+      "RESAMPLE_TO_VS": "06.01.01",
+      "SPATIO": "06.01.01",
       "TILE_FINALIZE": "06.01.00",
-      "TILE_INIT": "06.01.00",
-      "TP_COLLECT": "06.01.00",
-      "TP_FILTER": "06.01.00",
+      "TILE_INIT": "06.01.03",
+      "TP_COLLECT": "06.01.01",
+      "TP_FILTER": "06.01.01",
       "UNFORMAT_GRI": "06.01.00",
       "QL_CLOUD_MASK": "06.01.00",
       "QL_DECOMP": "06.01.00",
@@ -60,5 +60,3 @@
     }
   }
 }
-
-

config/MainSchemeTriggers.json

@@ -1,4 +1,8 @@
 {
+  "RESAMPLE_TO_VS":
+  {
+     "06.01.01" : "6.1"
+  },
   "RESAMPLE_TO_VS":
   {
      "06.01.00" : "6.1"

data/6.1/L1A/TaskTable_L1A.xml

@@ -459,7 +459,22 @@
                                     <Retrieval_Mode>ValIntersect</Retrieval_Mode>
                                     <T0>0</T0>
                                     <T1>0</T1>
-                                    <File_Type>DEM_GLOBE</File_Type>
+                                    <File_Type>DEM_GEOID</File_Type>
+                                    <File_Name_Type>Directory</File_Name_Type>
+                                </Alternative>
+                            </List_of_Alternatives>
+                        </Input>
+                        <Input>
+                            <Mode>ALWAYS</Mode>
+                            <Mandatory>Yes</Mandatory>
+                            <List_of_Alternatives count="1">
+                                <Alternative>
+                                    <Order>1</Order>
+                                    <Origin>DB</Origin>
+                                    <Retrieval_Mode>ValIntersect</Retrieval_Mode>
+                                    <T0>0</T0>
+                                    <T1>0</T1>
+                                    <File_Type>DEM_SRTM</File_Type>
                                     <File_Name_Type>Directory</File_Name_Type>
                                 </Alternative>
                             </List_of_Alternatives>
@@ -634,7 +649,7 @@
                     <Critical>true</Critical>
                     <Criticality_Level>1</Criticality_Level>
                     <File_Name>$INIT_LOC_L1_EXE_DIR</File_Name>
-                    <List_of_Inputs count="15">
+                    <List_of_Inputs count="14">
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>
@@ -800,21 +815,6 @@
                                 </Alternative>
                             </List_of_Alternatives>
                         </Input>
-                        <Input>
-                            <Mode>ALWAYS</Mode>
-                            <Mandatory>Yes</Mandatory>
-                            <List_of_Alternatives count="1">
-                                <Alternative>
-                                    <Order>1</Order>
-                                    <Origin>DB</Origin>
-                                    <Retrieval_Mode>ValIntersect</Retrieval_Mode>
-                                    <T0>0</T0>
-                                    <T1>0</T1>
-                                    <File_Type>DEM_GLOBE</File_Type>
-                                    <File_Name_Type>Directory</File_Name_Type>
-                                </Alternative>
-                            </List_of_Alternatives>
-                        </Input>
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>

data/6.1/L1AFormatDS/TaskTable_L1AFormatDS.xml

@@ -37,7 +37,7 @@
                     <Critical>true</Critical>
                     <Criticality_Level>1</Criticality_Level>
                     <File_Name>$FORMAT_METADATA_DS_L1A_EXE_DIR</File_Name>
-                    <List_of_Inputs count="29">
+                    <List_of_Inputs count="28">
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>
@@ -428,21 +428,6 @@
                                 </Alternative>
                             </List_of_Alternatives>
                         </Input>
-                        <Input>
-                            <Mode>ALWAYS</Mode>
-                            <Mandatory>Yes</Mandatory>
-                            <List_of_Alternatives count="1">
-                                <Alternative>
-                                    <Order>1</Order>
-                                    <Origin>DB</Origin>
-                                    <Retrieval_Mode>ValIntersect</Retrieval_Mode>
-                                    <T0>0</T0>
-                                    <T1>0</T1>
-                                    <File_Type>DEM_GLOBE</File_Type>
-                                    <File_Name_Type>Directory</File_Name_Type>
-                                </Alternative>
-                            </List_of_Alternatives>
-                        </Input>
                     </List_of_Inputs>
                     <List_of_Outputs count="2">
                         <Output>

data/6.1/L1B/TaskTable_L1B.xml

@@ -1722,7 +1722,7 @@
                     <Critical>true</Critical>
                     <Criticality_Level>1</Criticality_Level>
                     <File_Name>$FORMAT_METADATA_DS_L1B_EXE_DIR</File_Name>
-                    <List_of_Inputs count="38">
+                    <List_of_Inputs count="37">
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>
@@ -2203,21 +2203,6 @@
                                 </Alternative>
                             </List_of_Alternatives>
                         </Input>
-                        <Input>
-                            <Mode>ALWAYS</Mode>
-                            <Mandatory>Yes</Mandatory>
-                            <List_of_Alternatives count="1">
-                                <Alternative>
-                                    <Order>1</Order>
-                                    <Origin>DB</Origin>
-                                    <Retrieval_Mode>ValIntersect</Retrieval_Mode>
-                                    <T0>0</T0>
-                                    <T1>0</T1>
-                                    <File_Type>DEM_GLOBE</File_Type>
-                                    <File_Name_Type>Directory</File_Name_Type>
-                                </Alternative>
-                            </List_of_Alternatives>
-                        </Input>
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>

data/6.1/L1BNoGRI/TaskTable_L1BNoGRI.xml

@@ -264,7 +264,7 @@
                     <Critical>true</Critical>
                     <Criticality_Level>1</Criticality_Level>
                     <File_Name>$FORMAT_METADATA_DS_L1B_EXE_DIR</File_Name>
-                    <List_of_Inputs count="38">
+                    <List_of_Inputs count="37">
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>
@@ -745,21 +745,6 @@
                                 </Alternative>
                             </List_of_Alternatives>
                         </Input>
-                        <Input>
-                            <Mode>ALWAYS</Mode>
-                            <Mandatory>Yes</Mandatory>
-                            <List_of_Alternatives count="1">
-                                <Alternative>
-                                    <Order>1</Order>
-                                    <Origin>DB</Origin>
-                                    <Retrieval_Mode>ValIntersect</Retrieval_Mode>
-                                    <T0>0</T0>
-                                    <T1>0</T1>
-                                    <File_Type>DEM_GLOBE</File_Type>
-                                    <File_Name_Type>Directory</File_Name_Type>
-                                </Alternative>
-                            </List_of_Alternatives>
-                        </Input>
                         <Input>
                             <Mode>ALWAYS</Mode>
                             <Mandatory>Yes</Mandatory>