ADLR/megatron-lm!2610 - Update config for llama 3.1 8b vision projection

NVIDIA · Jan 31, 2025 · 42a76b9 · 42a76b9
1 parent bc12efb
commit 42a76b9
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/examples/multimodal/config.py b/examples/multimodal/config.py
@@ -186,6 +186,9 @@ def get_vision_projection_config(config, hidden_size):
     elif config.language_model_type == "llama3.1_8b":
         config.ffn_hidden_size = 4096
         config.activation_func = torch.nn.functional.gelu
+        config.layernorm_epsilon = 1e-5
+        config.add_bias_linear = True
+        config.normalization = "LayerNorm"
     elif config.language_model_type == "mistral_7b":
         config.ffn_hidden_size = 14336
         config.activation_func = torch.nn.functional.gelu