7b.yaml

# Copyright 2026 FlagOS Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

system:
  tensor_model_parallel_size: 1
  pipeline_model_parallel_size: 1
  disable_bias_linear: True
  use_flash_attn: True
  use_distributed_optimizer: True
  precision:
    fp16: True
    initial_loss_scale: 522893
    min_loss_scale: 1.0
    attention_softmax_in_fp32: True
    accumulate_allreduce_grads_in_fp32: True
  logging:
    log_interval: 1
    tensorboard_log_interval: 1
    wandb_project: "train-aquila-7B"
    wandb_exp_name: "train-test-7B"
  checkpoint:
    save_interval: 2000

model:
  num_layers: 32
  hidden_size: 4096
  num_attention_heads: 32
  seq_length: 2048
  max_position_embeddings: 2048
  norm_epsilon: 1e-5
  use_rotary_position_embeddings: true
  no_position_embedding: true
  swiglu: true
  multiple_of: 256
  normalization: RMSNorm
  # rotary_interleaved_patch: true
  untie_embeddings_and_output_weights: true
  init_method_std: 0.02
  attention_dropout: 0.0
  hidden_dropout: 0.0
  weight_decay: 0.1
  clip_grad: 1.0
  train_samples: 1002539063
  eval_iters: 0
  eval_interval: 1000
  micro_batch_size: 2
  global_batch_size: 1728
  seed: 1234

  optimizer:
    weight_decay: 0.1
    adam_beta1: 0.9
    adam_beta2: 0.95
    lr_scheduler:
      lr: 2.0e-5
      min_lr: 2.0e-6
      lr_warmup_samples: 3076172
      lr_decay_style: cosine

data:
  data_path: ${data_path:??}
  split: 1
  tokenizer:
    tokenizer_type: AquilaTokenizerFS
    vocab_file: ./examples/aquila/tokenizer/vocab.json
    merge_file: ./examples/aquila/tokenizer/merges.txt
    special_tokens_file: ./examples/aquila/tokenizer/special_tokens.txt
    vocab_size: 100008