lokr_argument.json

{
    "model_name_or_path": "meta-llama/Meta-Llama-3-8B",
    "dataset_name_or_path": "./data",
    "output_dir": "./checkpoints/lokr_ckpts",
    "lokr": true,
    "per_device_train_batch_size": 4,
    "gradient_accumulation_steps": 4,
    "num_train_epochs": 1,
    "learning_rate": 2e-05,
    "lr_scheduler_type": "linear",
    "attention_probs_dropout_prob": 0,
    "hidden_dropout_prob": 0,
    "warmup_steps": 30,
    "logging_steps": 1,
    "evaluation_strategy": "no",
    "save_strategy": "steps",
    "save_steps": 500,
    "src_length": 512,
    "max_length": 512,
    "bf16": true,
    "do_train": true,
    "do_eval": false,
    "disable_tqdm": false,
    "load_best_model_at_end": false,
    "eval_with_do_generation": false,
    "metric_for_best_model": "accuracy",
    "recompute": false,
    "save_total_limit": 100,
    "fp16_opt_level": "O2",
    "sharding": "stage2",
    "zero_padding": false,
    "use_flash_attention": false,
    "unified_checkpoint": true
  }