"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
}
],
"source": [
@@ -128,7 +141,7 @@
},
{
"cell_type": "code",
- "execution_count": 59,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -216,21 +229,165 @@
"metadata": {},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "====================\n",
- "torch.Size([1001]) torch.float32 cpu\n",
- "torch.Size([1001, 4000, 2]) torch.float32 cpu\n",
- "torch.Size([1001, 4000, 2]) torch.float32 cpu\n",
- "torch.Size([4000, 2]) torch.float32 cpu\n",
- "torch.Size([4000, 2]) torch.float32 cpu\n",
- "====================\n",
- "torch.Size([4000000, 1, 2]) torch.float32 cpu\n",
- "torch.Size([4000000, 1, 1]) torch.float32 cpu\n",
- "torch.Size([4000000, 1, 2]) torch.float32 cpu\n",
- "torch.Size([4000000, 1, 2]) torch.float32 cpu\n"
- ]
+ "data": {
+ "text/html": [
+ "====================\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "====================\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([1001])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1001\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([1001, 4000, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1001\u001b[0m, \u001b[1;36m4000\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([1001, 4000, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1001\u001b[0m, \u001b[1;36m4000\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "====================\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "====================\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000000, 1, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000000\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000000, 1, 1])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000000\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000000, 1, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000000\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "torch.Size([4000000, 1, 2])\n",
+ "torch.float32 cpu\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m4000000\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n",
+ "torch.float32 cpu\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
"data": {
@@ -263,22 +420,18 @@
"metadata": {},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "==========model==========\n",
- "batch_szie:1000, channel:1, length:7\n",
- "MLP(\n",
- " (fcin): Linear(in_features=5, out_features=512, bias=True)\n",
- " (fcs): ModuleList(\n",
- " (0): Linear(in_features=512, out_features=512, bias=True)\n",
- " (1): Linear(in_features=512, out_features=512, bias=True)\n",
- " (2): Linear(in_features=512, out_features=512, bias=True)\n",
- " (3): Linear(in_features=512, out_features=512, bias=True)\n",
- " )\n",
- " (fcout): Linear(in_features=512, out_features=2, bias=True)\n",
- " (relu): ReLU()\n",
- ")\n"
+ "ename": "RuntimeError",
+ "evalue": "CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[7], line 9\u001b[0m\n\u001b[1;32m 6\u001b[0m train_ds \u001b[39m=\u001b[39m BBdataset(raw_data)\n\u001b[1;32m 7\u001b[0m train_dl \u001b[39m=\u001b[39m DataLoader(train_ds, batch_size\u001b[39m=\u001b[39mbatch_size, shuffle\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m----> 9\u001b[0m model \u001b[39m=\u001b[39m MLP(input_dim\u001b[39m=\u001b[39;49m\u001b[39m5\u001b[39;49m, output_dim\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, hidden_layers\u001b[39m=\u001b[39;49m\u001b[39m4\u001b[39;49m, hidden_dim\u001b[39m=\u001b[39;49m\u001b[39m512\u001b[39;49m)\u001b[39m.\u001b[39;49mto(device)\n\u001b[1;32m 10\u001b[0m optimizer \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39moptim\u001b[39m.\u001b[39mAdam(model\u001b[39m.\u001b[39mparameters(), lr\u001b[39m=\u001b[39mlr)\n\u001b[1;32m 11\u001b[0m loss_fn \u001b[39m=\u001b[39m nn\u001b[39m.\u001b[39mMSELoss()\n",
+ "File \u001b[0;32m~/miniconda3/envs/sde/lib/python3.11/site-packages/torch/nn/modules/module.py:1145\u001b[0m, in \u001b[0;36mModule.to\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1141\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 1142\u001b[0m non_blocking, memory_format\u001b[39m=\u001b[39mconvert_to_format)\n\u001b[1;32m 1143\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m, non_blocking)\n\u001b[0;32m-> 1145\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_apply(convert)\n",
+ "File \u001b[0;32m~/miniconda3/envs/sde/lib/python3.11/site-packages/torch/nn/modules/module.py:797\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 795\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_apply\u001b[39m(\u001b[39mself\u001b[39m, fn):\n\u001b[1;32m 796\u001b[0m \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mchildren():\n\u001b[0;32m--> 797\u001b[0m module\u001b[39m.\u001b[39;49m_apply(fn)\n\u001b[1;32m 799\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[1;32m 800\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[1;32m 801\u001b[0m \u001b[39m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[1;32m 802\u001b[0m \u001b[39m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 807\u001b[0m \u001b[39m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[1;32m 808\u001b[0m \u001b[39m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
+ "File \u001b[0;32m~/miniconda3/envs/sde/lib/python3.11/site-packages/torch/nn/modules/module.py:820\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m 816\u001b[0m \u001b[39m# Tensors stored in modules are graph leaves, and we don't want to\u001b[39;00m\n\u001b[1;32m 817\u001b[0m \u001b[39m# track autograd history of `param_applied`, so we have to use\u001b[39;00m\n\u001b[1;32m 818\u001b[0m \u001b[39m# `with torch.no_grad():`\u001b[39;00m\n\u001b[1;32m 819\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[0;32m--> 820\u001b[0m param_applied \u001b[39m=\u001b[39m fn(param)\n\u001b[1;32m 821\u001b[0m should_use_set_data \u001b[39m=\u001b[39m compute_should_use_set_data(param, param_applied)\n\u001b[1;32m 822\u001b[0m \u001b[39mif\u001b[39;00m should_use_set_data:\n",
+ "File \u001b[0;32m~/miniconda3/envs/sde/lib/python3.11/site-packages/torch/nn/modules/module.py:1143\u001b[0m, in \u001b[0;36mModule.to..convert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m 1140\u001b[0m \u001b[39mif\u001b[39;00m convert_to_format \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m t\u001b[39m.\u001b[39mdim() \u001b[39min\u001b[39;00m (\u001b[39m4\u001b[39m, \u001b[39m5\u001b[39m):\n\u001b[1;32m 1141\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 1142\u001b[0m non_blocking, memory_format\u001b[39m=\u001b[39mconvert_to_format)\n\u001b[0;32m-> 1143\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39;49mto(device, dtype \u001b[39mif\u001b[39;49;00m t\u001b[39m.\u001b[39;49mis_floating_point() \u001b[39mor\u001b[39;49;00m t\u001b[39m.\u001b[39;49mis_complex() \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m, non_blocking)\n",
+ "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: out of memory\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"
]
}
],
@@ -475,7 +628,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.16"
+ "version": "3.11.4"
}
},
"nbformat": 4,
diff --git a/test.py b/test.py
index 5e197ae..40e81bd 100644
--- a/test.py
+++ b/test.py
@@ -48,7 +48,8 @@ def main():
parser.add_argument('--batch_size', type=int, default=8000)
parser.add_argument('-n','--normalize', action='store_true')
parser.add_argument('--tarined_data', action='store_true')
-
+ parser.add_argument('--filter_number', type=int)
+
args = parser.parse_args()
check_model_task(args)
@@ -58,7 +59,13 @@ def main():
torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
- experiment_name = args.task
+ experiment_name = args.task
+ if args.change_epsilons:
+ experiment_name += '_change_epsilons'
+ if args.filter_number is not None and 'mnist' in args.task:
+ experiment_name += f'_filter{args.filter_number}'
+
+
log_dir = Path('experiments') / experiment_name / 'test' / tt.strftime("%Y-%m-%d/%H_%M_%S/")
ds_cached_dir = Path('experiments') / experiment_name / 'data'
log_dir.mkdir(parents=True, exist_ok=True)
diff --git a/train.py b/train.py
index 4c248da..409dda8 100644
--- a/train.py
+++ b/train.py
@@ -144,12 +144,12 @@ def main_worker(args):
TimeElapsedColumn(),
transient=False,
) as progress:
- task1 = progress.add_task("[gold]Training whole dataset (lr: X) (loss=X)", total=ds_info['nums_sub_ds']*args.epoch_nums)
+ task1 = progress.add_task("[red]Training whole dataset (lr: X) (loss=X)", total=ds_info['nums_sub_ds']*args.epoch_nums)
while not progress.finished:
if ds_info['nums_sub_ds'] == 1:
new_dl = read_ds_from_pkl(args,
real_metadata,
- args.ds_cached_dir / f"new_ds_{int(iter%ds_info['nums_sub_ds'])}.pkl"
+ args.ds_cached_dir / f"new_ds_0.pkl"
)
for iter in range(ds_info['nums_sub_ds']*args.epoch_nums):
if ds_info['nums_sub_ds'] > 1:
@@ -158,7 +158,7 @@ def main_worker(args):
args.ds_cached_dir / f"new_ds_{int(iter%ds_info['nums_sub_ds'])}.pkl"
)
- task2 = progress.add_task(f"[green]Training sub dataset {int(iter%ds_info['nums_sub_ds'])}", total=args.iter_nums)
+ task2 = progress.add_task(f"[dark_orange]Training sub dataset {int(iter%ds_info['nums_sub_ds'])}", total=args.iter_nums)
for _ in range(args.iter_nums):
now_loss = train(args, model ,new_dl, optimizer, scheduler, loss_fn, before_train, after_train)
loss_list.append(now_loss)
@@ -167,8 +167,8 @@ def main_worker(args):
progress.update(task2, visible=False)
progress.remove_task(task2)
torch.save(model.state_dict(), args.log_dir / f'model_{model.__class__.__name__}_{int(iter)}.pth')
- progress.update(task1, advance=1, description="[red]Training whole dataset (l r: %2.5f) (loss=%2.5f)" % (cur_lr, now_loss))
-
+ progress.update(task1, advance=1, description="[red]Training whole dataset (lr: %2.5f) (loss=%2.5f)" % (cur_lr, now_loss))
+ progress.log(f"[green]sub dataset {int(iter%ds_info['nums_sub_ds'])} finished; Loss: {now_loss}")
# Draw loss curve
fig, ax = plt.subplots(figsize=(10, 5))
ax.plot(loss_list)