diff --git a/config/augmentations/postprocessing/to_tensor copy.yaml b/config/augmentations/postprocessing/to_tensor_torchvision.yaml similarity index 100% rename from config/augmentations/postprocessing/to_tensor copy.yaml rename to config/augmentations/postprocessing/to_tensor_torchvision.yaml diff --git a/config/augmentations_train/osl.yaml b/config/augmentations_train/osl.yaml index 90bf5c89..fcb9c080 100644 --- a/config/augmentations_train/osl.yaml +++ b/config/augmentations_train/osl.yaml @@ -8,7 +8,7 @@ defaults: - random_horizontal_flip - /augmentations/color: - /augmentations/postprocessing: - - to_tensor + - to_tensor_torchvision - normalize # task: # - one-shot-learning diff --git a/config/augmentations_val/osl.yaml b/config/augmentations_val/osl.yaml index 90bf5c89..fcb9c080 100644 --- a/config/augmentations_val/osl.yaml +++ b/config/augmentations_val/osl.yaml @@ -8,7 +8,7 @@ defaults: - random_horizontal_flip - /augmentations/color: - /augmentations/postprocessing: - - to_tensor + - to_tensor_torchvision - normalize # task: # - one-shot-learning diff --git a/config/datasets/detection/detection_lungs.yaml b/config/datasets/detection/detection_lungs.yaml index 6cb44e93..880ab353 100644 --- a/config/datasets/detection/detection_lungs.yaml +++ b/config/datasets/detection/detection_lungs.yaml @@ -7,7 +7,7 @@ description: Набор данных содержит РГ снимки легк markup_info: Набор данных содержит разметку bounding box легких и патологий. date_time: 18.07.2022 -_target_: innofw.core.datamodules.lightning_datamodules.detection.YOLOv5DataModule +_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter train: source: https://api.blackhole.ai.innopolis.university/public-datasets/lungs_detection/train.zip diff --git a/config/experiments/classification/KG_090323_wpng92_mnist.yaml b/config/experiments/classification/KG_090323_wpng92_mnist.yaml index 4e3170ad..c0beebad 100644 --- a/config/experiments/classification/KG_090323_wpng92_mnist.yaml +++ b/config/experiments/classification/KG_090323_wpng92_mnist.yaml @@ -2,25 +2,19 @@ defaults: - override /models: classification/resnet.yaml - override /datasets: classification/classification_mnist.yaml - - override /augmentations_train: none + - override /augmentations_train: none #classification.yaml - override /augmentations_val: none - override /augmentations_test: none - override /losses: log_loss.yaml - - override /optimizers: lion -# - override /callbacks: classification.yaml - + - override /optimizers: adam + - override /schedulers: project: "mnist_classification" task: "image-classification" random_seed: 42 -original_work_dir: ${hydra:runtime.cwd} -weights_freq: 1 -batch_size: 32 -stop_param: 1 -epochs: 10 - -#wandb: -# enable: True -# project: mnist -# group: 160323_just_testing_wandb -# job_type: training +batch_size: 128 +epochs: 5 +accelerator: "gpu" +devices: 2 +num_workers: 1 +weights_freq: 7 diff --git a/config/experiments/one-shot-learning/IM_190722_vwer3f23_oneshotlearning.yaml b/config/experiments/one-shot-learning/IM_190722_vwer3f23_oneshotlearning.yaml index ab707cff..05f3efda 100644 --- a/config/experiments/one-shot-learning/IM_190722_vwer3f23_oneshotlearning.yaml +++ b/config/experiments/one-shot-learning/IM_190722_vwer3f23_oneshotlearning.yaml @@ -12,4 +12,7 @@ project: "faces" task: "one-shot-learning" random_seed: 42 weights_freq: 1 -epochs: 20 +epochs: 50 +accelerator: "gpu" +devices: 1 +batch_size: 64 diff --git a/config/experiments/semantic-segmentation/KG_041223_ingieq921_unet.yaml b/config/experiments/semantic-segmentation/KG_041223_ingieq921_unet.yaml new file mode 100644 index 00000000..58085b6a --- /dev/null +++ b/config/experiments/semantic-segmentation/KG_041223_ingieq921_unet.yaml @@ -0,0 +1,21 @@ +# @package _global_ +defaults: + - override /models: semantic-segmentation/unet_smp.yaml + - override /datasets: semantic-segmentation/segmentation_arable.yaml + - override /optimizers: adam.yaml + - override /losses: segmentation_losses + +trainers: + overfit_batches: 20 # fit only 20 val batches and 40 train batches + +accelerator: gpu + +models: + in_channels: 4 + + +project: "segmentation" +task: "image-segmentation" +random_seed: 42 +weights_freq: 1 +batch_size: 10 \ No newline at end of file diff --git a/examples/measure_anomaly_detection.sh b/examples/measure_anomaly_detection.sh new file mode 100644 index 00000000..27c59515 --- /dev/null +++ b/examples/measure_anomaly_detection.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=anomaly-detection/IM_040822_rbe23lls_anomaly_detection_timeseries_lstm.yaml epochs=100 optimizers=adam accelerator=gpu & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID \ No newline at end of file diff --git a/examples/measure_complexing.sh b/examples/measure_complexing.sh new file mode 100644 index 00000000..2d855fd6 --- /dev/null +++ b/examples/measure_complexing.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python innofw/utils/data_utils/preprocessing/band_composer.py\ + --src_type sentinel2\ + --src_path tests/data/images/other/satellite_cropped/sentinel2/one\ + --channels "[\"RED\", \"GRN\", \"BLU\", \"NIR\"]" & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID diff --git a/examples/measure_detection_yolov5.sh b/examples/measure_detection_yolov5.sh new file mode 100644 index 00000000..7bd073e5 --- /dev/null +++ b/examples/measure_detection_yolov5.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=detection/KA_120722_8adfcdaa_yolov5.yaml epochs=100 optimizers=adam accelerator=gpu & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log+mem_log and nvidiasmi_log" +echo $PID + +while true +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv >> nvidiasmi_log + sleep 1 +done + +kill $PID \ No newline at end of file diff --git a/examples/measure_image_classification.sh b/examples/measure_image_classification.sh new file mode 100644 index 00000000..547f750d --- /dev/null +++ b/examples/measure_image_classification.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=classification/KG_090323_wpng92_mnist.yaml epochs=100 & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID \ No newline at end of file diff --git a/examples/measure_lin_regression.sh b/examples/measure_lin_regression.sh new file mode 100644 index 00000000..430bb600 --- /dev/null +++ b/examples/measure_lin_regression.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=regression/KA_130722_9f7134db_linear_regression.yaml & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID diff --git a/examples/measure_oneshot_learning.sh b/examples/measure_oneshot_learning.sh new file mode 100644 index 00000000..70329ae0 --- /dev/null +++ b/examples/measure_oneshot_learning.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=one-shot-learning/IM_190722_vwer3f23_oneshotlearning.yaml & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID diff --git a/examples/measure_segmentation.sh b/examples/measure_segmentation.sh new file mode 100644 index 00000000..7e81d331 --- /dev/null +++ b/examples/measure_segmentation.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=semantic-segmentation/KG_041223_ingieq921_unet.yaml epochs=100 optimizers=adam accelerator=gpu & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID \ No newline at end of file diff --git a/examples/measure_table_clustering.sh b/examples/measure_table_clustering.sh new file mode 100644 index 00000000..9919c7b0 --- /dev/null +++ b/examples/measure_table_clustering.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=clustering/KA_280722_9379c32c_kmeans_credit_cards & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID + diff --git a/examples/measure_xgboost.sh b/examples/measure_xgboost.sh new file mode 100644 index 00000000..db4c3a7a --- /dev/null +++ b/examples/measure_xgboost.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# run like this: sudo -E env "PATH=$PATH" bash examples/measure_image_classification.sh +sudo apt-get install -y iotop > /dev/null +PWD=$(pwd) + +current_dir=${PWD##*/} + +var2="examples" +if [ "$current_dir" = "$var2" ]; then + cd .. +fi +echo "" > nohup.out +echo "" > nvidiasmi_log +echo "" > mem_log +echo "" > cpu_log + +export NO_CLI=True + +INITIAL_PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $INITIAL_PROCESSES + + +nohup time sudo -E env "PATH=$PATH" python train.py experiments=regression/KA_190722_0a70ef39_xgbregressor_cc.yaml & +PID=$! +echo "Saving cpu+ram info and nvidia-smi to cpu_log + mem_log and nvidiasmi_log" +echo $PID + +PROCESSES=$(sudo lsof nohup.out | wc -l) && echo $PROCESSES +while [ $PROCESSES -gt $INITIAL_PROCESSES ] +do + top -b n1 | grep -E 'Cpu' >> cpu_log + top -b n1 | grep -E 'MiB Mem' >> mem_log + sudo nvidia-smi --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv | grep "00000000:25:00.0" >> nvidiasmi_log + sleep 1 + PROCESSES=$(sudo lsof nohup.out | wc -l) # && echo $PROCESSES +done + +sudo -E env "PATH=$PATH" python examples/measurements_compaction.py + +sudo kill -9 $PID diff --git a/examples/measurements_compaction.py b/examples/measurements_compaction.py new file mode 100644 index 00000000..306bf6e3 --- /dev/null +++ b/examples/measurements_compaction.py @@ -0,0 +1,96 @@ +import numpy as np +from dateutil import parser + +RAM_LOGS_FILE = './mem_log' +CPU_LOGS_FILE = './cpu_log' +GPU_LOGS_FILE = './nvidiasmi_log' +RAM_LOGS_COMPACT_FILE = './mem_log_comp' +CPU_LOGS_COMPACT_FILE = './cpu_log_comp' +GPU_LOGS_COMPACT_FILE = './nvidiasmi_log_comp' + + +def compact_ram_logs(): + used_ram = [] + cached_ram = [] + with open(RAM_LOGS_FILE, 'r') as f: + lines = f.readlines() + for line in lines[1:]: + data = line.split(', ') + used = float(data[-2].split()[0]) + cached = float(data[-1].split()[0]) + used_ram.append(used); + cached_ram.append(cached) + with open(RAM_LOGS_COMPACT_FILE, 'w') as f: + print('Measured in MB', file=f) + print( + f'Min RAM usage = {min(used_ram)} | Max RAM usage = {max(used_ram)} | Mean RAM usage = {np.mean(used_ram)} with std = {np.std(used_ram)} ', + file=f) + print( + f'Min RAM cache = {min(cached_ram)} | Max RAM cache = {max(cached_ram)} | Mean RAM cache = {np.mean(cached_ram)} with std = {np.std(cached_ram)} ', + file=f) + + +def compact_cpu_logs(): + cpu_us = [] + cpu_sy = [] + cpu_id = [] + with open(CPU_LOGS_FILE, 'r') as f: + lines = f.readlines() + for line in lines[1:]: + data = line.split(', ') + cpu_us.append(float(data[0].split()[1])) + cpu_sy.append(float(data[1].split()[0])) + cpu_id.append(float(data[3].split()[0])) + cpu_us, cpu_sy, cpu_id = np.array(cpu_us), np.array(cpu_sy), np.array(cpu_id) + used_cpu = cpu_sy + cpu_us + + with open(CPU_LOGS_COMPACT_FILE, 'w') as f: + print( + f'Min idle CPU% = {min(cpu_id)} | Max idle CPU% = {max(cpu_id)} | Mean idle CPU% = {np.mean(cpu_id)} with std = {np.std(cpu_id)} ', + file=f) + print( + f'Min user CPU% = {min(cpu_us)} | Max user CPU% = {max(cpu_us)} | Mean user CPU% = {np.mean(cpu_us)} with std = {np.std(cpu_us)} ', + file=f) + print( + f'Min system CPU% = {min(cpu_sy)} | Max system CPU% = {max(cpu_sy)} | Mean system CPU% = {np.mean(cpu_sy)} with std = {np.std(cpu_sy)} ', + file=f) + print( + f'Min used CPU% = {min(used_cpu)} | Max used CPU% = {max(used_cpu)} | Mean used CPU% = {np.mean(used_cpu)} with std = {np.std(used_cpu)} ', + file=f) + + +def compact_gpu_logs(): + timestamps = [] + gpu_utilization = [] + memory_utilization = [] + occupied_memory = [] + with open(GPU_LOGS_FILE, 'r') as f: + lines = f.readlines() + for line in lines[1:]: + data = line.split(', ') + timestamps.append(data[0]) + occupied_memory.append(float(data[-1].split()[0])) + memory_utilization.append(float(data[-4].split()[0])) + gpu_utilization.append(float(data[-5].split()[0])) + + gpu_util, mem_util, occ_mem = np.array(gpu_utilization), np.array(memory_utilization), np.array(occupied_memory) + start_time, end_time = parser.parse(timestamps[0]), parser.parse(timestamps[-1]) + with open(GPU_LOGS_COMPACT_FILE, 'w') as f: + print( + f'Min util of GPU% = {min(gpu_util)} | Max util of GPU% = {max(gpu_util)} | Mean util of GPU% = {np.mean(gpu_util)} with std = {np.std(gpu_util)} ', + file=f) + print( + f'Min util of GPU memory % = {min(mem_util)} | Max util of GPU memory % = {max(mem_util)} | Mean util of GPU memory % = {np.mean(mem_util)} with std = {np.std(mem_util)} ', + file=f) + print( + f'Min occupied GPU memory, MB = {min(occ_mem)} | Max occupied GPU memory, MB = {max(occ_mem)} | Mean occupied GPU memory, MB = {np.mean(occ_mem)} with std = {np.std(occ_mem)} ', + file=f) + print(f'Total runtime = {end_time - start_time}', file=f) + + +if __name__ == '__main__': + compact_ram_logs() + compact_cpu_logs() + compact_gpu_logs() + print(f'Saved compacted statistics to {RAM_LOGS_COMPACT_FILE}, {RAM_LOGS_COMPACT_FILE}, {GPU_LOGS_COMPACT_FILE}') + print(f'Algorithm runtime was saved to {GPU_LOGS_COMPACT_FILE}') \ No newline at end of file diff --git a/innofw/utils/framework.py b/innofw/utils/framework.py index 3d22faf1..6e838077 100755 --- a/innofw/utils/framework.py +++ b/innofw/utils/framework.py @@ -143,7 +143,7 @@ def is_albu(aug): return isinstance(aug, A.Compose) or isinstance( aug, A.core.transforms_interface.BasicTransform ) - + cfg = cfg.get('augmentations', cfg) if cfg is not None else cfg for key in keys: try: for v in cfg[key].values():