fix a little bug in columns check for ts data

hydromodel/datasets/data_preprocess.py

@@ -1,7 +1,7 @@
 """
 Author: Wenyu Ouyang
 Date: 2022-10-25 21:16:22
-LastEditTime: 2024-03-25 17:06:13
+LastEditTime: 2024-03-25 17:19:38
 LastEditors: Wenyu Ouyang
 Description: preprocess data for models in hydro-model-xaj
 FilePath: \hydro-model-xaj\hydromodel\datasets\data_preprocess.py
@@ -57,11 +57,10 @@ def check_tsdata_format(file_path):
     try:
         data = pd.read_csv(file_path)
 
+        data_columns = [remove_unit_from_name(col) for col in data.columns]
         # Check required columns
         missing_required_columns = [
-            column
-            for column in data.columns
-            if remove_unit_from_name(column) not in required_columns
+            column for column in required_columns if column not in data_columns
         ]
 
         if missing_required_columns:
@@ -71,11 +70,8 @@ def check_tsdata_format(file_path):
             return False
 
         # Check optional columns
-        for column in data.columns:
-            if (
-                remove_unit_from_name(column) not in required_columns
-                and remove_unit_from_name(column) not in optional_columns
-            ):
+        for column in optional_columns:
+            if column not in data_columns:
                 print(
                     f"Optional column '{column}' not found in file: {file_path}, but it's okay."
                 )

scripts/calibrate_xaj.py

@@ -11,7 +11,6 @@
 from hydroutils import hydro_file
 
 sys.path.append(os.path.dirname(Path(os.path.abspath(__file__)).parent))
-import definitions
 from hydromodel.trainers.calibrate_sceua import calibrate_by_sceua
 from hydromodel.datasets.data_postprocess import (
     renormalize_params,
@@ -23,7 +22,6 @@
 from hydromodel.trainers.train_utils import show_calibrate_result, show_test_result
 from hydromodel.models.xaj import xaj
 from hydromodel.trainers.calibrate_ga import calibrate_by_ga, show_ga_result
-from hydromodel.utils import units
 
 
 def calibrate(args):
@@ -32,7 +30,6 @@ def calibrate(args):
     model_info = args.model
     algo_info = args.algorithm
     comment = args.comment
-    data_dir = os.path.join(definitions.ROOT_DIR, "hydromodel", "example", exp)
     data_dir = os.path.join('D:/研究生/毕业论文/new毕业论文/预答辩/碧流河水库/模型运行/')
     kfold = [
         int(f_name[len("data_info_fold") : -len("_test.json")])

scripts/datapreprocess4calibrate.py

@@ -1,12 +1,13 @@
 """
 Author: Wenyu Ouyang
 Date: 2022-11-19 17:27:05
-LastEditTime: 2023-06-03 16:21:17
+LastEditTime: 2024-03-25 17:10:51
 LastEditors: Wenyu Ouyang
 Description: the script to preprocess data for models in hydro-model-xaj
-FilePath: /hydro-model-xaj/scripts/datapreprocess4calibrate.py
+FilePath: \hydro-model-xaj\scripts\datapreprocess4calibrate.py
 Copyright (c) 2021-2022 Wenyu Ouyang. All rights reserved.
 """
+
 import numpy as np
 import hydrodataset
 import argparse
@@ -15,9 +16,7 @@
 from pathlib import Path
 
 sys.path.append(os.path.dirname(Path(os.path.abspath(__file__)).parent))
-import definitions
 from hydromodel.datasets.data_preprocess import (
-    trans_camels_format_to_xaj_format,
     cross_valid_data,
     split_train_test,
 )