config-llama3.1-8b-g5.2xl-g5.4xl-sm.yml

general:
  name: "Llama3-1-8b-g5"
  model_name: "Llama3-1-8b"
 
aws:
  region: {region}
  sagemaker_execution_role: {role_arn}
  bucket: {write_bucket}

dir_paths:
    data_prefix: data
    prompts_prefix: prompts
    all_prompts_file: all_prompts.csv
    metrics_dir: metrics
    models_dir: models
    metadata_dir: metadata

s3_read_data:
    read_bucket: {read_bucket}
    scripts_prefix: scripts
    script_files:
    - hf_token.txt
    configs_prefix: configs
    config_files:
    - #.yml
    source_data_prefix: source_data
    source_data_files:
    - 2wikimqa_e.jsonl
    - 2wikimqa.jsonl
    - hotpotqa_e.jsonl
    - hotpotqa.jsonl
    - narrativeqa.jsonl
    - triviaqa_e.jsonl
    - triviaqa.jsonl
    tokenizer_prefix: llama3_1_tokenizer
    prompt_template_dir: prompt_template
    prompt_template_file: prompt_template_llama3.txt

## section that enables container to run notebooks and python scripts automatically 
run_steps:
    0_setup.ipynb: yes
    1_generate_data.ipynb: yes
    2_deploy_model.ipynb: yes
    3_run_inference.ipynb: yes
    4_get_evaluations.ipynb: no
    5_model_metric_analysis.ipynb: yes
    6_cleanup.ipynb: yes

datasets:
  prompt_template_keys:
  - input
  - context
  ground_truth_col_key: answers
  question_col_key: input
  filters:
  - language: en    
    min_length_in_tokens: 1
    max_length_in_tokens: 500
    payload_file: payload_en_1-500.jsonl
  - language: en
    min_length_in_tokens: 500
    max_length_in_tokens: 1000
    payload_file: payload_en_500-1000.jsonl
  - language: en
    min_length_in_tokens: 1000
    max_length_in_tokens: 2000
    payload_file: payload_en_1000-2000.jsonl
  - language: en
    min_length_in_tokens: 2000
    max_length_in_tokens: 3000
    payload_file: payload_en_2000-3000.jsonl
  - language: en
    min_length_in_tokens: 3000
    max_length_in_tokens: 3840
    payload_file: payload_en_3000-3840.jsonl
 
# name of the file that contains the model evaluation information
# for example, the prompt template names, the ground truth column name (if any), 
# LLM panelist information, inference parameters, etc.
model_evaluations: model_eval_all_info.yml

metrics:
  dataset_of_interest: en_2000-3000
  
#: #.yml
  
inference_parameters:
  sagemaker:
    do_sample: yes
    temperature: 0.1
    top_p: 0.92
    top_k: 120  
    max_new_tokens: 100

experiments:
  - name: Llama3-1-8b-g5.2xl-djl-inference:0.29.0-lmi11.0.0-cu124
    model_id: meta-llama/Llama-3.1-8B-Instruct
    model_version: "*"
    model_name: Meta-Llama-3-1-8B-Instruct
    ep_name: Meta-Llama-3-1-8B-Instruct-g5-2xl
    download_from_hf_place_in_s3: yes
    model_s3_path: s3://{write_bucket}/meta-llama/Llama-3.1-8B-Instruct
    instance_type: "ml.g5.xlarge"
    image_uri: 763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124
    deploy: yes
    instance_count: 1
    deployment_script: deploy_w_djl_serving.py
    inference_script: sagemaker_predictor.py
    inference_spec:
      parameter_set: sagemaker
    serving.properties: |
      engine=Python
      option.model_id=s3://{write_bucket}/meta-llama/Llama-3.1-8B-Instruct
      option.dtype=fp16
    payload_files:
    - payload_en_1-500.jsonl
    - payload_en_500-1000.jsonl
    - payload_en_1000-2000.jsonl
    - payload_en_2000-3000.jsonl
    # - payload_en_3000-3840.jsonl
    concurrency_levels:
    - 1
    - 2

    accept_eula: true
    env:
  - name: Llama3-1-8b-g5.4xl-djl-inference:0.29.0-lmi11.0.0-cu124
    model_id: meta-llama/Llama-3.1-8B-Instruct
    model_version: "*"
    model_name: Meta-Llama-3-1-8B-Instruct
    ep_name: Meta-Llama-3-1-8B-Instruct-g5-4xl
    download_from_hf_place_in_s3: yes
    model_s3_path: s3://{write_bucket}/meta-llama/Llama-3.1-8B-Instruct
    instance_type: "ml.g5.2xlarge"
    image_uri: 763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124
    deploy: yes
    instance_count: 1
    deployment_script: deploy_w_djl_serving.py
    inference_script: sagemaker_predictor.py
    inference_spec:
      parameter_set: sagemaker
    serving.properties: |
      engine=Python
      option.model_id=s3://{write_bucket}/meta-llama/Llama-3.1-8B-Instruct
      option.dtype=fp16
    payload_files:
    - payload_en_1-500.jsonl
    - payload_en_500-1000.jsonl
    - payload_en_1000-2000.jsonl
    - payload_en_2000-3000.jsonl
    # - payload_en_3000-3840.jsonl
    concurrency_levels:
    - 1
    - 2
    # - 4
    # - 10

    accept_eula: true
    env:
report:
  latency_budget: 2
  cost_per_10k_txn_budget: 50
  error_rate_budget: 0
  per_inference_request_file: per_inference_request_results.csv
  all_metrics_file: all_metrics.csv
  txn_count_for_showing_cost: 10000
  v_shift_w_single_instance: 0.025
  v_shift_w_gt_one_instance: 0.025
  latency_vs_token_len_chart: 
    y_ticks:
    title: "Effect of token length on inference latency for \"meta-llama/Meta-Llama-3.1-8B-Instruct\""