update configurations for fullfinetune and peftfinetune

tanganke · Nov 26, 2024 · 0bbd246 · 0bbd246
1 parent a3cda5c
commit 0bbd246
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 9 deletions.
diff --git a/config/method/lm_finetune/fullfinetune_sft.yaml b/config/method/lm_finetune/fullfinetune_sft.yaml
@@ -8,9 +8,9 @@ optimizer:
   lr: 5e-5
 
 lr_scheduler:
-  _target_: torchtune.training.lr_schedulers.get_cosine_schedule_with_warmup
-  num_warmup_steps: 5
-  num_training_steps: _T_max_ # this will be replaced by the expected number of training steps
+  _target_: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: _T_max_ # this will be replaced by the expected number of training steps
+  eta_min: 1e-6
 
 dataloader_kwargs:
   # per-gpu batch size

diff --git a/config/method/lm_finetune/peftfinetune_sft.yaml b/config/method/lm_finetune/peftfinetune_sft.yaml
@@ -8,9 +8,9 @@ optimizer:
   lr: 5e-5
 
 lr_scheduler:
-  _target_: torchtune.training.lr_schedulers.get_cosine_schedule_with_warmup
-  num_warmup_steps: 5
-  num_training_steps: _T_max_ # this will be replaced by the expected number of training steps
+  _target_: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: _T_max_ # this will be replaced by the expected number of training steps
+  eta_min: 1e-6
 
 dataloader_kwargs:
   # per-gpu batch size
@@ -22,9 +22,14 @@ peft_config:
   _target_: peft.LoraConfig
   task_type: peft.TaskType.CAUSAL_LM
   target_modules:
-    - query
-    - value
-  r: 16
+    # lora attention modules
+    - q_proj
+    - v_proj
+    # lora mlp modules
+    - gate_proj
+    - down_proj
+    - up_proj
+  r: 64
   lora_alpha: 16
   lora_dropout: 0
   bias: none