From a3791dd9a9757a9ee733ef03cfd320e3173c0b1b Mon Sep 17 00:00:00 2001
From: ouyangwenyu <wenyuouyang@outlook.com>
Date: Mon, 25 Mar 2024 17:23:50 +0800
Subject: [PATCH] fix a little bug in columns check for ts data

---
 hydromodel/datasets/data_preprocess.py | 14 +++++---------
 scripts/calibrate_xaj.py               |  3 ---
 scripts/datapreprocess4calibrate.py    |  7 +++----
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/hydromodel/datasets/data_preprocess.py b/hydromodel/datasets/data_preprocess.py
index e1ed836..850a20d 100644
--- a/hydromodel/datasets/data_preprocess.py
+++ b/hydromodel/datasets/data_preprocess.py
@@ -1,7 +1,7 @@
 """
 Author: Wenyu Ouyang
 Date: 2022-10-25 21:16:22
-LastEditTime: 2024-03-25 17:06:13
+LastEditTime: 2024-03-25 17:19:38
 LastEditors: Wenyu Ouyang
 Description: preprocess data for models in hydro-model-xaj
 FilePath: \hydro-model-xaj\hydromodel\datasets\data_preprocess.py
@@ -57,11 +57,10 @@ def check_tsdata_format(file_path):
     try:
         data = pd.read_csv(file_path)
 
+        data_columns = [remove_unit_from_name(col) for col in data.columns]
         # Check required columns
         missing_required_columns = [
-            column
-            for column in data.columns
-            if remove_unit_from_name(column) not in required_columns
+            column for column in required_columns if column not in data_columns
         ]
 
         if missing_required_columns:
@@ -71,11 +70,8 @@ def check_tsdata_format(file_path):
             return False
 
         # Check optional columns
-        for column in data.columns:
-            if (
-                remove_unit_from_name(column) not in required_columns
-                and remove_unit_from_name(column) not in optional_columns
-            ):
+        for column in optional_columns:
+            if column not in data_columns:
                 print(
                     f"Optional column '{column}' not found in file: {file_path}, but it's okay."
                 )
diff --git a/scripts/calibrate_xaj.py b/scripts/calibrate_xaj.py
index 3c89722..a558833 100644
--- a/scripts/calibrate_xaj.py
+++ b/scripts/calibrate_xaj.py
@@ -11,7 +11,6 @@
 from hydroutils import hydro_file
 
 sys.path.append(os.path.dirname(Path(os.path.abspath(__file__)).parent))
-import definitions
 from hydromodel.trainers.calibrate_sceua import calibrate_by_sceua
 from hydromodel.datasets.data_postprocess import (
     renormalize_params,
@@ -23,7 +22,6 @@
 from hydromodel.trainers.train_utils import show_calibrate_result, show_test_result
 from hydromodel.models.xaj import xaj
 from hydromodel.trainers.calibrate_ga import calibrate_by_ga, show_ga_result
-from hydromodel.utils import units
 
 
 def calibrate(args):
@@ -32,7 +30,6 @@ def calibrate(args):
     model_info = args.model
     algo_info = args.algorithm
     comment = args.comment
-    data_dir = os.path.join(definitions.ROOT_DIR, "hydromodel", "example", exp)
     data_dir = os.path.join('D:/研究生/毕业论文/new毕业论文/预答辩/碧流河水库/模型运行/')
     kfold = [
         int(f_name[len("data_info_fold") : -len("_test.json")])
diff --git a/scripts/datapreprocess4calibrate.py b/scripts/datapreprocess4calibrate.py
index 590896c..54b1b37 100644
--- a/scripts/datapreprocess4calibrate.py
+++ b/scripts/datapreprocess4calibrate.py
@@ -1,12 +1,13 @@
 """
 Author: Wenyu Ouyang
 Date: 2022-11-19 17:27:05
-LastEditTime: 2023-06-03 16:21:17
+LastEditTime: 2024-03-25 17:10:51
 LastEditors: Wenyu Ouyang
 Description: the script to preprocess data for models in hydro-model-xaj
-FilePath: /hydro-model-xaj/scripts/datapreprocess4calibrate.py
+FilePath: \hydro-model-xaj\scripts\datapreprocess4calibrate.py
 Copyright (c) 2021-2022 Wenyu Ouyang. All rights reserved.
 """
+
 import numpy as np
 import hydrodataset
 import argparse
@@ -15,9 +16,7 @@
 from pathlib import Path
 
 sys.path.append(os.path.dirname(Path(os.path.abspath(__file__)).parent))
-import definitions
 from hydromodel.datasets.data_preprocess import (
-    trans_camels_format_to_xaj_format,
     cross_valid_data,
     split_train_test,
 )