Temporary fix to avoid #1469

metaspace/engine/sm/engine/annotation_lithops/load_ds.py

@@ -119,17 +119,28 @@ def _upload_imzml_browser_files(
     browser_storage: Storage,
     uuid: str,
 ) -> None:
-    """Save imzml browser files on the object storage"""
+    """Save imzML browser files on the object storage"""
 
     def upload_file(data: np.array, key: str):
         save_cobj(browser_storage, data, key=key)
 
+    # TODO: need reimplement save_cobj for file > 5 GB
+    # https://github.com/metaspace2020/metaspace/issues/1469
+    if mzs.itemsize > 4:
+        mzs = mzs.astype('f')
+    if ints.itemsize > 4:
+        ints = ints.astype('f')
+
+    if any([o.nbytes >= 5 * 1024 ** 3 for o in (mzs, ints, sp_idxs)]):
+        print('At least one object has a size of more than 5 GB')
+        return
+
     keys = [f'{uuid}/{k}' for k in ['mzs.npy', 'ints.npy', 'sp_idxs.npy']]
     with ThreadPoolExecutor(3) as executor:
         cobjs = list(executor.map(upload_file, [mzs, ints, sp_idxs], keys))
 
     chunk_records_number = 1024
-    mz_index = mzs[::chunk_records_number].astype('f')
+    mz_index = mzs[::chunk_records_number]
     cobjs.append(save_cobj(browser_storage, mz_index, key=f'{uuid}/mz_index.npy'))
 
     key = f'{uuid}/portable_spectrum_reader.pickle'