Merge branch 'main' of https://github.com/t1101675/LMOps

microsoft · Oct 28, 2024 · 0390cd2 · 0390cd2
2 parents c38563c + 6944b4c
commit 0390cd2
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 5 deletions.
diff --git a/data_selection/README.md b/data_selection/README.md
@@ -3,8 +3,11 @@
 [paper](https://arxiv.org/abs/2410.07064) | [huggingface](https://huggingface.co/Data-Selection)
 
 <div>Theory Overview:</div>
+
 <img src="./figures/theory.png" width="70%"/>
+<br>
 <div>Training Framwork PDS:</div>
+
 <img src="./figures/method.png" width="70%"/>
 
 ## Overview of the Training Framework
@@ -14,7 +17,11 @@
 4. Filter CC with the scores.
 5. Pre-train the model.
 
-## Selected Data and Pre-Trained Models
+## Pre-Trained Models
++ [Models](https://huggingface.co/collections/Data-Selection/baseline-models-670550972a59015f6c8870ab) Trained on Redpajama CC (Conventional Pre-Training, Baselines)
++ [Models](https://huggingface.co/collections/Data-Selection/pds-models-6705504096a78d10a30837c0) Trained PDS-Selected Data
+
+## Selected Data
 TODO
 
 ## Details of the Pipeline & How to run

diff --git a/data_selection/install.sh b/data_selection/install.sh
@@ -9,7 +9,6 @@ pip3 install rich
 pip3 install accelerate
 pip3 install datasets
 pip3 install sentencepiece
-pip3 install peft
 pip3 install matplotlib
 pip3 install wandb
 pip3 install cvxpy

diff --git a/data_selection/scripts/data_scorer/infer.sh b/data_selection/scripts/data_scorer/infer.sh
@@ -14,8 +14,8 @@ DISTRIBUTED_ARGS="--num_gpus $GPUS_PER_NODE \
 
 # model
 BASE_PATH=${1-"/home/MiniLLM"}
-CKPT="${BASE_PATH}/results/data_scorer/cc-sgd100-160M-10k-lima-163840/fairseq_125M/e5-w10-bs16-lr0.0001cosine1e-07-G2-N16-NN2/mean-bias-linear/best"
-CKPT_NAME="cc-sgd100-160M-10k-lima"
+CKPT="${BASE_PATH}/results/data_scorer/"
+CKPT_NAME="cc-160M-lima"
 # data
 DATA_DIR="${BASE_PATH}/processed_data/data_scorer_infer/cc/mistral-fairseq-1024"
 # hp

diff --git a/data_selection/scripts/pmp_solver/160M.sh b/data_selection/scripts/pmp_solver/160M.sh
@@ -27,7 +27,7 @@ OPTS=""
 OPTS+=" --type pmp_solver"
 # model
 OPTS+=" --model-type mistral"
-OPTS+=" --model-path ${BASE_PATH}/results/pretrain/cc/mistral_160M/t100K-w2K-bs8-lr0.0006cosine6e-05-G4-N16-NN2-scr/10000"
+OPTS+=" --model-path ${BASE_PATH}/results/pretrain/mistral_160M-10K/"
 OPTS+=" --base-path ${BASE_PATH}"
 OPTS+=" --ckpt-name 160M-10k"
 OPTS+=" --n-gpu ${GPUS_PER_NODE}"