minor changes

.gitignore

@@ -1,6 +1,8 @@
 checkpoints
 **/wandb/**
 wandb
+ignore
+experimental_data
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.vscode/launch.json

@@ -12,14 +12,16 @@
             "console": "integratedTerminal",
             "justMyCode": true,
             "args": [
-                "--model_config", "peft_pretraining/llama_small.json",
+                "--model_config", "configs/llama_9m.json",
                 "--use_peft",
+                "--lora_r", "32",
                 "--relora", "100",
-                "--train_ln",
+                "--reset_optimizer_on_relora", "False",
                 "--device", "cuda:0",
                 "--lr", "0.0005",
-                "--batch_size", "48",
-                "--dtype", "bfloat16"
+                "--batch_size", "240",
+                "--dtype", "bfloat16",
+                "--tags", "vscode_debugger",
             ]
         }
     ]

README.md

@@ -10,7 +10,7 @@ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/
 bash miniconda.sh -b
 /root/miniconda3/bin/conda init
 
-bash
+bash  # to enable conda
 
 git clone https://github.com/guitaricet/peft_pretraining
 cd peft_pretraining