DeepSeek-V2-MoE transformer config adapt to mcore0.6.0 (alibaba#253)

Co-authored-by: one_game <[email protected]>
guoshengjun · Jun 10, 2024 · 942c562 · 942c562
1 parent b9d128d
commit 942c562
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/megatron_patch/model/deepseek_v2/transformer_config.py b/megatron_patch/model/deepseek_v2/transformer_config.py
@@ -26,3 +26,7 @@ class DeepSeekV2TransformerConfig(TransformerConfig):
     rotary_base: int = None
 
     rotary_scaling_factor: int = None
+
+    max_position_embeddings: int = None
+
+    moe_aux_loss_coeff: float = 0.0