fix llama100M

Guitaricet · Aug 5, 2023 · ac3083c · ac3083c
1 parent 6c74b61
commit ac3083c
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/configs/llama_100m.json b/configs/llama_100m.json
@@ -5,13 +5,13 @@
     "bos_token_id": 0,
     "eos_token_id": 1,
     "hidden_act": "silu",
-    "hidden_size": 512,
-    "intermediate_size": 1368,
+    "hidden_size": 640,
+    "intermediate_size": 1708,
     "initializer_range": 0.02,
     "max_sequence_length": 1024,
     "model_type": "llama",
-    "num_attention_heads": 16,
-    "num_hidden_layers": 21,
+    "num_attention_heads": 10,
+    "num_hidden_layers": 12,
     "pad_token_id": -1,
     "rms_norm_eps": 1e-06,
     "transformers_version": "4.28.1",