VeOmni/configs/pretrain/deepseek.yaml at main · princepride/VeOmni · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
model:
  # model_path: deepseek-ai/DeepSeek-V3-0324
  attn_implementation: flash_attention_2
  moe_implementation: fused

data:
  train_path: fineweb_100BT
  train_size: 1000000000000
  dataloader_type: native
  datasets_type: iterable
  data_type: plaintext
  max_seq_len: 8192
  text_keys: text
  drop_last: true

train:
  output_dir: deepseek_v3
  data_parallel_mode: fsdp1
  ulysses_parallel_size: 1
  expert_parallel_size: 8
  global_batch_size: 32
  micro_batch_size: 1
  rmpad: false
  rmpad_with_pos_ids: true
  bsz_warmup_ratio: 0.007
  dyn_bsz_margin: 0
  dyn_bsz_buffer_size: 200
  optimizer: adamw
  lr: 3.0e-4
  lr_warmup_ratio: 0.007
  lr_decay_style: constant
  lr_decay_ratio: 1.0
  weight_decay: 0.01
  max_grad_norm: 1.0
  enable_mixed_precision: true
  enable_gradient_checkpointing: true
  enable_full_shard: true
  enable_fsdp_offload: false
  enable_activation_offload: false
  init_device: meta
  enable_full_determinism: false
  empty_cache_steps: 500
  ckpt_manager: dcp
  load_checkpoint_path: ""
  save_steps: 100
  save_hf_weights: true
  wandb_project: DeepSeek-V3
  wandb_name: dsv3_pretrain_ep8