+Zero Stage: 2, 3 +Offload: True, False ++ +The `run_step2_sweep.sh` script passes configuration arguments to `run_single.sh`, which can be extended to sweep beyond the parameters listed above (e.g. learning rate, weight decay, etc). + +# Usage +The sweep script can be run as follows: +
+DeepSpeedExamples/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning$ bash training_scripts/opt/single_node/sweep/run_step2_sweep.sh +diff --git a/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_single.sh b/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_single.sh new file mode 100644 index 000000000..6f5453af1 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_single.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ZERO_STAGE=$1 +OFFLOAD=$2 +OUTPUT=$3 +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=0 +fi +if [ "$OFFLOAD" == true ]; then + OFFLOAD="--offload" +else + OFFLOAD="" +fi +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +mkdir -p $OUTPUT + +cmd="deepspeed main.py \ + --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ + --data_split 2,4,4 \ + --model_name_or_path facebook/opt-350m \ + --num_padding_at_beginning 1 \ + --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 \ + --max_seq_len 512 \ + --learning_rate 5e-5 \ + --weight_decay 0.1 \ + --num_train_epochs 1 \ + --dropout 0.0 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --output_dir $OUTPUT \ + $OFFLOAD" + +echo "----------------------------- DS COMMAND -----------------------------" +echo $cmd + +$cmd &> $OUTPUT/${OUTPUT}.log diff --git a/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh b/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh new file mode 100644 index 000000000..ad9849e38 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_dpo_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +for z in {2..3} +do + for offload in true false + do + cmd="bash training_scripts/opt/single_node/sweep/run_single.sh \ + ${z} \ + ${offload} \ + z${z}_offload_${offload}" + echo "----------------------------- CALLING SHELL SCRIPT -----------------------------" + echo $cmd + $cmd + pkill -9 python + sleep 60 + echo "" + done +done diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/README.md b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/README.md index 758cc4893..3c62b9f82 100644 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/README.md +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/README.md @@ -6,7 +6,7 @@ Finetuning the Reward Model (RM) is more or less similar to Step-1 Supervised F For SFT finetuning, the data is the concatenation of a query and an answer. However, for RM finetuning, each batch of data consists of two query-answer pairs, i.e., the same query with a high-score answer and a low-score answer. This also leads to the second difference as describe below. -👉**The training objective difference** +👉 **The training objective difference** For RW, the training objective is the pairwise ranking score, i.e., for the two query-answer pairs, RM is supposed to give a higher score to the better answer. There are multiple ways to achieve this. In our implementation, we use either the end token of the sequence or the first padding token as the aggregated score and compare them. Others may also use the average score for the entire answer as an alternative. @@ -22,7 +22,7 @@ We also provide an evaluation script, ``rw_eval.py``, for users to perform simpl We provide the script for OPT-350m, which you can test by launching the command ``` -bash training_scripts/single_gpu/run_350m.sh +bash training_scripts/opt/single_gpu/run_350m.sh ``` You can train larger models by simply replacing the model candidate with your preferred one and enabling other efficient training features as described in Step 1 of the SFT finetuning process. diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/main.py b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/main.py index 7548552cc..8cdf5644d 100644 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/main.py +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/main.py @@ -4,9 +4,7 @@ # DeepSpeed Team import argparse -import os import math -import sys import torch from torch.utils.data import DataLoader, RandomSampler, SequentialSampler @@ -19,14 +17,13 @@ import deepspeed from deepspeed.ops.adam import DeepSpeedCPUAdam, FusedAdam +from deepspeed.accelerator import get_accelerator -sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) -from utils.model.model_utils import create_critic_model -from utils.data.data_utils import create_prompt_dataset, DataCollatorReward -from utils.utils import print_rank_0, to_device, save_hf_format, set_random_seed, get_all_reduce_mean, get_optimizer_grouped_parameters, save_zero_three_model, load_hf_tokenizer -from utils.ds_utils import get_train_ds_config -from utils.module.lora import convert_linear_layer_to_lora, convert_lora_to_linear_layer, only_optimize_lora_parameters +from dschat.utils.model.model_utils import create_critic_model +from dschat.utils.data.data_utils import create_prompt_dataset, DataCollatorReward +from dschat.utils.utils import print_rank_0, to_device, save_hf_format, set_random_seed, get_all_reduce_mean, get_optimizer_grouped_parameters, save_zero_three_model, load_hf_tokenizer +from dschat.utils.ds_utils import get_train_ds_config +from dschat.utils.module.lora import convert_linear_layer_to_lora, convert_lora_to_linear_layer, only_optimize_lora_parameters, make_model_gradient_checkpointing_compatible def parse_args(): @@ -44,8 +41,8 @@ def parse_args(): default='2,4,4', help='Comma-separated list of proportions for training' 'phase 1, 2, and 3 data. For example the split `2,4,4`' - 'will use 60% of data for phase 1, 20% for phase 2' - 'and 20% for phase 3.') + 'will use 60%% of data for phase 1, 20%% for phase 2' + 'and 20%% for phase 3.') parser.add_argument( '--data_output_path', type=str, @@ -137,13 +134,21 @@ def parse_args(): '--gradient_checkpointing', action='http://github.com/store_true', help='Enable HF gradient checkpointing for Actor model.') - parser.add_argument('--disable_dropout', - action='http://github.com/store_true', - help='Disable the dropout of the model.') + parser.add_argument( + "--dropout", + type=float, + default=None, + help="If dropout configured, use it. " + "Otherwise, keep the default dropout configuration of the model.") # deepspeed features parser.add_argument('--offload', action='http://github.com/store_true', help='Enable ZeRO Offload techniques.') + parser.add_argument('--dtype', + type=str, + default='fp16', + choices=['fp16', 'bf16'], + help='Training data type') parser.add_argument( '--zero_stage', type=int, @@ -161,15 +166,45 @@ def parse_args(): parser.add_argument('--only_optimize_lora', action='http://github.com/store_true', help='Only optimize the LoRA parameters.') + parser.add_argument( + "--lora_learning_rate", + type=float, + default=5e-4, + help= + "Initial LoRA learning rate (after the potential warmup period) to use." + ) + + # Evaluation + parser.add_argument("--eval_interval", + type=int, + default=0, + help="If > 0, perform evaluation at this interval") + parser.add_argument("--eval_iters", + type=int, + default=100, + help="Maximum evaluation iterations") + ## low precision + parser.add_argument( + '--compute_fp32_loss', + action='http://github.com/store_true', + help='Relevant for low precision dtypes (fp16, bf16, etc.). ' + 'If specified, loss is calculated in fp32.') + + ## Tensorboard logging + parser.add_argument('--enable_tensorboard', + action='http://github.com/store_true', + help='Enable tensorboard logging') + parser.add_argument('--tensorboard_path', + type=str, + default="step2_tensorboard") + ## Tokenizer + parser.add_argument( + "--add_eot_token", + action='http://github.com/store_true', + help="Add <|endoftext|> as additional special token to tokenizer") parser = deepspeed.add_config_arguments(parser) args = parser.parse_args() - # Validate settings - if args.gradient_checkpointing and args.lora_dim > 0: - assert ( - not args.only_optimize_lora - ), "--gradient_checkpointing and --only_optimize_lora cannot be enabled at the same time." - return args @@ -177,10 +212,10 @@ def main(): args = parse_args() if args.local_rank == -1: - device = torch.device("cuda") + device = torch.device(get_accelerator().device_name()) else: - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) + get_accelerator().set_device(args.local_rank) + device = torch.device(get_accelerator().device_name(), args.local_rank) # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') deepspeed.init_distributed() @@ -188,7 +223,11 @@ def main(): args.global_rank = torch.distributed.get_rank() ds_config = get_train_ds_config(offload=args.offload, - stage=args.zero_stage) + dtype=args.dtype, + stage=args.zero_stage, + enable_tensorboard=args.enable_tensorboard, + tb_path=args.tensorboard_path, + tb_name="step2_model") ds_config[ 'train_micro_batch_size_per_gpu'] = args.per_device_train_batch_size ds_config[ @@ -199,22 +238,49 @@ def main(): set_random_seed(args.seed) torch.distributed.barrier() - tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=True) - tokenizer.pad_token = tokenizer.eos_token - # make sure tokenizer is right pad in our logic - tokenizer.padding_side = 'right' + # load_hf_tokenizer will get the correct tokenizer and set padding tokens based on the model family + args.end_of_conversation_token = "<|endoftext|>" + additional_special_tokens = args.end_of_conversation_token if args.add_eot_token else None + tokenizer = load_hf_tokenizer(args.model_name_or_path, + fast_tokenizer=True, + add_special_tokens=additional_special_tokens) rm_model = create_critic_model(args.model_name_or_path, tokenizer, ds_config, args.num_padding_at_beginning, - disable_dropout=args.disable_dropout) + dropout=args.dropout, + zero_stage=args.zero_stage, + compute_fp32_loss=args.compute_fp32_loss) + + # Model bigscience/bloom-560m has large variance at ln_f.weight parameter + # This makes bf16 finetuning hard. + # In general, since we are replacing the model head, it makes sense to reset + # the LN that precedes it. + force_optimize_params = [] + if "bigscience/bloom-" in args.model_name_or_path: + zero_init_enabled = (args.zero_stage == 3) + params = [ + rm_model.rwtranrsformer.ln_f.weight, + rm_model.rwtranrsformer.ln_f.bias + ] + with deepspeed.zero.GatheredParameters(params, + modifier_rank=0, + enabled=zero_init_enabled): + if deepspeed.comm.get_rank() == 0 or not zero_init_enabled: + torch.nn.init.ones_(rm_model.rwtransformer.ln_f.weight) + torch.nn.init.zeros_(rm_model.rwtransformer.ln_f.bias) + force_optimize_params.extend( + ['rwtransformer.ln_f.weight', 'rwtransformer.ln_f.bias']) if args.lora_dim > 0: rm_model = convert_linear_layer_to_lora(rm_model, args.lora_module_name, args.lora_dim) if args.only_optimize_lora: - rm_model = only_optimize_lora_parameters(rm_model) + force_optimize_params.append('v_head.weight') + rm_model = only_optimize_lora_parameters(rm_model, + force_optimize_params) + rm_model = make_model_gradient_checkpointing_compatible(rm_model) train_phase = 2 train_dataset, eval_dataset = create_prompt_dataset( @@ -234,41 +300,44 @@ def main(): collate_fn=data_collator, sampler=train_sampler, batch_size=args.per_device_train_batch_size) - eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, sampler=eval_sampler, batch_size=args.per_device_eval_batch_size) - def evaluation_reward(model, eval_dataloader): + def evaluation_reward(model, dataloader, eval_iters): model.eval() correct_predictions = 0 total_predictions = 0 - scores = 0 - for step, batch in enumerate(eval_dataloader): - batch = to_device(batch, device) + chosen_scores = 0. + rejected_scores = 0. + for _step, _batch in enumerate(dataloader): + _batch = to_device(_batch, device) with torch.no_grad(): - outputs = model(**batch) + _outputs = model(**_batch) - chosen = outputs["chosen_mean_scores"] - rejected = outputs["rejected_mean_scores"] + chosen = _outputs["chosen_mean_scores"] + rejected = _outputs["rejected_mean_scores"] correct_predictions += (chosen > rejected).sum() total_predictions += chosen.shape[0] - scores += outputs["chosen_mean_scores"].mean().float() - if step == 99: # For faster evaluation and debugging + chosen_scores += _outputs["chosen_mean_scores"].mean().float() + rejected_scores += _outputs["rejected_mean_scores"].mean().float() + if (_step + 1) == eval_iters: break - acc = correct_predictions / total_predictions - scores = scores / (step + 1) + _acc = correct_predictions / total_predictions + chosen_scores = chosen_scores / (_step + 1) + rejected_scores = rejected_scores / (_step + 1) try: - acc = get_all_reduce_mean(acc).item() - scores = get_all_reduce_mean(scores).item() + _acc = get_all_reduce_mean(_acc).item() + chosen_scores = get_all_reduce_mean(chosen_scores).item() + rejected_scores = get_all_reduce_mean(rejected_scores).item() except: pass - return scores, acc + return chosen_scores, rejected_scores, _acc # Split weights in two groups, one with weight decay and the other not. optimizer_grouped_parameters = get_optimizer_grouped_parameters( - rm_model, args.weight_decay) + rm_model, args.weight_decay, args.lora_learning_rate) AdamOptimizer = DeepSpeedCPUAdam if args.offload else FusedAdam optimizer = AdamOptimizer(optimizer_grouped_parameters, @@ -302,11 +371,14 @@ def evaluation_reward(model, eval_dataloader): print_rank_0( f"***** Evaluating reward, Epoch {0}/{args.num_train_epochs} *****", args.global_rank) - reward_score, acc = evaluation_reward(rm_model, eval_dataloader) + reward_score, reject_score, acc = evaluation_reward( + rm_model, eval_dataloader, args.eval_iters) print_rank_0( - f"chosen_last_scores (higher is better) : {reward_score}, acc (higher is better) : {acc}", - args.global_rank) + f"chosen_last_scores (higher is better) : {reward_score}, " + f"rejected_last_scores (lower is better) : {reject_score}, " + f"acc (higher is better) : {acc}", args.global_rank) + total_micro_steps = 0 for epoch in range(args.num_train_epochs): print_rank_0( f"Beginning of Epoch {epoch+1}/{args.num_train_epochs}, Total Micro Batches {len(train_dataloader)}", @@ -320,6 +392,22 @@ def evaluation_reward(model, eval_dataloader): rm_model.backward(loss) rm_model.step() mean_loss += loss.item() + total_micro_steps += 1 + gas_boundary = (total_micro_steps % + args.gradient_accumulation_steps == 0) + total_steps = total_micro_steps // args.gradient_accumulation_steps + if args.eval_interval and gas_boundary and ( + total_steps % args.eval_interval == 0): + print_rank_0(f"Iter {total_steps}: Evaluating reward", + args.global_rank) + reward_score, reject_score, acc = evaluation_reward( + rm_model, eval_dataloader, args.eval_iters) + print_rank_0( + f"Iter {total_steps}: c_scores: {reward_score}, r_scores: {reject_score}, " + f"diff: {reward_score - reject_score}, acc: {acc}", + args.global_rank) + rm_model.train() + print_rank_0( f"Epoch {epoch+1}/{args.num_train_epochs} with loss {mean_loss/(step+1)}", args.global_rank) @@ -327,10 +415,12 @@ def evaluation_reward(model, eval_dataloader): print_rank_0( f"***** Evaluating reward, Epoch {epoch+1}/{args.num_train_epochs} *****", args.global_rank) - reward_score, acc = evaluation_reward(rm_model, eval_dataloader) + reward_score, reject_score, acc = evaluation_reward( + rm_model, eval_dataloader, args.eval_iters) print_rank_0( - f"chosen_last_scores (higher is better) : {reward_score}, acc (higher is better) : {acc}", - args.global_rank) + f"chosen_last_scores (higher is better) : {reward_score}, " + f"rejected_last_scores (lower is better) : {reject_score}, " + f"acc (higher is better) : {acc}", args.global_rank) rm_model.tput_timer.update_epoch_count() if args.output_dir is not None: diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py index 0ab7ecfbc..23f9a66af 100644 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py @@ -4,16 +4,11 @@ # DeepSpeed Team import argparse -import os import torch -import sys - -sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) -from utils.model.model_utils import create_critic_model -from utils.utils import to_device -from utils.utils import load_hf_tokenizer +from dschat.utils.model.model_utils import create_critic_model +from dschat.utils.utils import to_device, load_hf_tokenizer +from deepspeed import get_accelerator def parse_args(): @@ -34,16 +29,26 @@ def parse_args(): "OPT model has a fixed number (1) of padding tokens at the beginning of the input. " "We did not see this in other models but keep it as an option for now.", ) + parser.add_argument( + "--add_eot_token", + action='http://github.com/store_true', + help="Add <|endoftext|> as additional special token to tokenizer") args = parser.parse_args() return args -def load_stuff(model_name_or_path, num_padding_at_beginning): +def load_stuff(model_name_or_path, num_padding_at_beginning, + additional_special_tokens): - tokenizer = load_hf_tokenizer(model_name_or_path, fast_tokenizer=True) + tokenizer = load_hf_tokenizer(model_name_or_path, + fast_tokenizer=True, + add_special_tokens=additional_special_tokens) tokenizer.pad_token = tokenizer.eos_token - model = create_critic_model(model_name_or_path, tokenizer, None, - num_padding_at_beginning, True) + model = create_critic_model(model_name_or_path, + tokenizer, + None, + num_padding_at_beginning, + dropout=0.) return model, tokenizer @@ -100,10 +105,14 @@ def prepare_singlesample(prompt, def run_pair_comparison(): args = parse_args() - device = torch.device("cuda:0") + device = torch.device(get_accelerator().device_name(0)) + + args.end_of_conversation_token = "<|endoftext|>" + additional_special_tokens = args.end_of_conversation_token if args.add_eot_token else None rm_model, tokenizer = load_stuff(args.model_name_or_path, - args.num_padding_at_beginning) + args.num_padding_at_beginning, + additional_special_tokens) rm_model.to(device) rm_model.eval() @@ -122,12 +131,13 @@ def run_pair_comparison(): for prompt, good_ans, bad_ans in zip(prompt_list, good_ans_list, bad_ans_list): - batch = prepare_datapair(prompt, - good_ans, - bad_ans, - tokenizer, - max_seq_len=512, - end_of_conversation_token="<|endoftext|>") + batch = prepare_datapair( + prompt, + good_ans, + bad_ans, + tokenizer, + max_seq_len=512, + end_of_conversation_token=args.end_of_conversation_token) batch = to_device(batch, device) # Run inference with torch.no_grad(): @@ -144,20 +154,25 @@ def run_pair_comparison(): def run_single_sample(): args = parse_args() - device = torch.device("cuda") + device = torch.device(get_accelerator().device_name()) + + args.end_of_conversation_token = "<|endoftext|>" + additional_special_tokens = args.end_of_conversation_token if args.add_eot_token else None rm_model, tokenizer = load_stuff(args.model_name_or_path, - args.num_padding_at_beginning) + args.num_padding_at_beginning, + additional_special_tokens) rm_model.to(device) prompt = "Human: Explain the moon landing to a 6 year old in a few sentences." my_ans = "Assistant: The moon landing was a major milestone in the history of human exploration of the solar system. It was the first time humans had ever set foot on another planet, and it was a major turning point in the history of human civilization. The astronauts, Neil Armstrong, Buzz Aldrin, and Michael Collins, successfully landed the Apollo 11 spacecraft on the moon, marking the first time humans had ever set foot on another" - batch = prepare_singlesample(prompt, - my_ans, - tokenizer, - max_seq_len=512, - end_of_conversation_token="<|endoftext|>") + batch = prepare_singlesample( + prompt, + my_ans, + tokenizer, + max_seq_len=512, + end_of_conversation_token=args.end_of_conversation_token) batch = to_device(batch, device) rm_model.eval() diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b.sh new file mode 100755 index 000000000..5ee57171d --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +OUTPUT=$1 +ZERO_STAGE=$2 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output_step2_llama_7b_epoch1_lr9.65e-6 +fi +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=3 +fi +mkdir -p $OUTPUT + +deepspeed main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --model_name_or_path meta-llama/Llama-2-7b-hf \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --max_seq_len 512 \ + --learning_rate 9.65e-6 \ + --weight_decay 0.1 \ + --num_padding_at_beginning 0 \ + --num_train_epochs 1 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --gradient_checkpointing \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --offload \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh new file mode 100755 index 000000000..78c5eb2ac --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +OUTPUT=$1 +ZERO_STAGE=$2 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output_step2_llama_7b_epoch1_lr9.65e-6 +fi +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=3 +fi +mkdir -p $OUTPUT + +deepspeed main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --model_name_or_path meta-llama/Llama-2-7b-hf \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 8 \ + --max_seq_len 512 \ + --learning_rate 9.65e-6 \ + --weight_decay 0.1 \ + --num_padding_at_beginning 0 \ + --num_train_epochs 1 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --gradient_checkpointing \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --offload \ + --lora_dim 128 \ + --lora_module_name "layers." \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/multi_node/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/multi_node/run_350m.sh deleted file mode 100644 index cea008824..000000000 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/multi_node/run_350m.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -OUTPUT=$1 -ZERO_STAGE=$2 -if [ "$OUTPUT" == "" ]; then - OUTPUT=./output -fi -if [ "$ZERO_STAGE" == "" ]; then - ZERO_STAGE=0 -fi -mkdir -p $OUTPUT - -deepspeed main.py \ - --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ - --data_split 2,4,4 \ - --model_name_or_path facebook/opt-350m \ - --num_padding_at_beginning 1 \ - --per_device_train_batch_size 2 \ - --per_device_eval_batch_size 2 \ - --max_seq_len 512 \ - --learning_rate 5e-5 \ - --weight_decay 0.1 \ - --disable_dropout \ - --num_train_epochs 1 \ - --gradient_accumulation_steps 1 \ - --lr_scheduler_type cosine \ - --num_warmup_steps 0 \ - --seed 1234 \ - --zero_stage $ZERO_STAGE \ - --deepspeed \ - --output_dir $OUTPUT \ - &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/multi_node/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/multi_node/run_350m.sh new file mode 100644 index 000000000..51852af45 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/multi_node/run_350m.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +OUTPUT=$1 +ZERO_STAGE=$2 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=0 +fi +mkdir -p $OUTPUT + +deepspeed main.py \ + --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ + --data_split 2,4,4 \ + --model_name_or_path facebook/opt-350m \ + --num_padding_at_beginning 1 \ + --per_device_train_batch_size 2 \ + --per_device_eval_batch_size 2 \ + --max_seq_len 512 \ + --learning_rate 5e-5 \ + --weight_decay 0.1 \ + --dropout 0.0 \ + --num_train_epochs 1 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_gpu/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_gpu/run_350m.sh new file mode 100644 index 000000000..284fd44a0 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_gpu/run_350m.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +OUTPUT=$1 +ZERO_STAGE=$2 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=0 +fi +mkdir -p $OUTPUT + +deepspeed --num_gpus 1 main.py --model_name_or_path facebook/opt-350m \ + --num_padding_at_beginning 1 --weight_decay 0.1 --dropout 0.0 --gradient_accumulation_steps 4 --zero_stage $ZERO_STAGE \ + --enable_tensorboard \ + --tensorboard_path $OUTPUT \ + --deepspeed --output_dir $OUTPUT &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/run_350m.sh new file mode 100644 index 000000000..d7ff70106 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/run_350m.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +OUTPUT=$1 +ZERO_STAGE=$2 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=0 +fi +mkdir -p $OUTPUT + +deepspeed main.py \ + --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ + --data_split 2,4,4 \ + --model_name_or_path facebook/opt-350m \ + --num_padding_at_beginning 1 \ + --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 \ + --max_seq_len 512 \ + --learning_rate 5e-5 \ + --weight_decay 0.1 \ + --num_train_epochs 1 \ + --dropout 0.0 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/README.md b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/README.md new file mode 100644 index 000000000..1f90b9f65 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/README.md @@ -0,0 +1,20 @@ +# DeepSpeed Characterization Script + +# Contents + * [Introduction](#introduction) + * [Usage](#usage) + +# Introduction +The step 2 characterization script sweeps across various training parameters. Currently, the following parameters are swept: +
+Zero Stage: 2, 3 +Offload: True, False ++ +The `run_step2_sweep.sh` script passes configuration arguments to `run_single.sh`, which can be extended to sweep beyond the parameters listed above (e.g. learning rate, weight decay, etc). + +# Usage +The sweep script can be run as follows: +
+DeepSpeedExamples/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning$ bash training_scripts/opt/single_node/sweep/run_step2_sweep.sh +diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_single.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_single.sh new file mode 100644 index 000000000..6f5453af1 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_single.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ZERO_STAGE=$1 +OFFLOAD=$2 +OUTPUT=$3 +if [ "$ZERO_STAGE" == "" ]; then + ZERO_STAGE=0 +fi +if [ "$OFFLOAD" == true ]; then + OFFLOAD="--offload" +else + OFFLOAD="" +fi +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +mkdir -p $OUTPUT + +cmd="deepspeed main.py \ + --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ + --data_split 2,4,4 \ + --model_name_or_path facebook/opt-350m \ + --num_padding_at_beginning 1 \ + --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 \ + --max_seq_len 512 \ + --learning_rate 5e-5 \ + --weight_decay 0.1 \ + --num_train_epochs 1 \ + --dropout 0.0 \ + --gradient_accumulation_steps 1 \ + --lr_scheduler_type cosine \ + --num_warmup_steps 0 \ + --seed 1234 \ + --zero_stage $ZERO_STAGE \ + --deepspeed \ + --output_dir $OUTPUT \ + $OFFLOAD" + +echo "----------------------------- DS COMMAND -----------------------------" +echo $cmd + +$cmd &> $OUTPUT/${OUTPUT}.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh new file mode 100644 index 000000000..ad9849e38 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/opt/single_node/sweep/run_step2_sweep.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +for z in {2..3} +do + for offload in true false + do + cmd="bash training_scripts/opt/single_node/sweep/run_single.sh \ + ${z} \ + ${offload} \ + z${z}_offload_${offload}" + echo "----------------------------- CALLING SHELL SCRIPT -----------------------------" + echo $cmd + $cmd + pkill -9 python + sleep 60 + echo "" + done +done diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_gpu/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_gpu/run_350m.sh deleted file mode 100644 index 435de2c65..000000000 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_gpu/run_350m.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -OUTPUT=$1 -ZERO_STAGE=$2 -if [ "$OUTPUT" == "" ]; then - OUTPUT=./output -fi -if [ "$ZERO_STAGE" == "" ]; then - ZERO_STAGE=0 -fi -mkdir -p $OUTPUT - -deepspeed --num_gpus 1 main.py --model_name_or_path facebook/opt-350m \ - --num_padding_at_beginning 1 --weight_decay 0.1 --disable_dropout --gradient_accumulation_steps 4 --zero_stage $ZERO_STAGE \ - --deepspeed --output_dir $OUTPUT &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_node/run_350m.sh b/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_node/run_350m.sh deleted file mode 100644 index 2d1709955..000000000 --- a/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/training_scripts/single_node/run_350m.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -OUTPUT=$1 -ZERO_STAGE=$2 -if [ "$OUTPUT" == "" ]; then - OUTPUT=./output -fi -if [ "$ZERO_STAGE" == "" ]; then - ZERO_STAGE=0 -fi -mkdir -p $OUTPUT - -deepspeed main.py \ - --data_path Dahoas/rm-static Dahoas/full-hh-rlhf Dahoas/synthetic-instruct-gptj-pairwise yitingxie/rlhf-reward-datasets \ - --data_split 2,4,4 \ - --model_name_or_path facebook/opt-350m \ - --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_eval_batch_size 4 \ - --max_seq_len 512 \ - --learning_rate 5e-5 \ - --weight_decay 0.1 \ - --num_train_epochs 1 \ - --disable_dropout \ - --gradient_accumulation_steps 1 \ - --lr_scheduler_type cosine \ - --num_warmup_steps 0 \ - --seed 1234 \ - --zero_stage $ZERO_STAGE \ - --deepspeed \ - --output_dir $OUTPUT \ - &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md index de1f133c2..d985750ee 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md @@ -1,6 +1,6 @@ -# 🚩Benchmark setting used in [Blog](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-chat) and [Landing Page](../../README.md) +# 🚩Benchmark setting used in [Blog](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/deepspeed-chat) and [Landing Page](../../README.md) -As stated in [Blog](https://github.com/microsoft/DeepSpeed/tree/master/blogs/deepspeed-chat), +As stated in [Blog](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/deepspeed-chat), > ***Very Important Details***: The numbers in both Table 1 and 2 of the blog are for Step 3 of the training and based on actual measured training throughput on DeepSpeed-RLHF curated dataset and training recipe which trains for one epoch on a total of 135M tokens. We have in total 67.5M query tokens (131.9k queries with sequence length 256) and 67.5M generated tokens (131.9k answers with sequence length 256), and a maximum global batch size per step of 0.5M tokens (1024 query-answer pairs). We urge readers to pay attention to these specifications before making any cost and e2e time comparisons with DeepSpeed-RLHF. See our [benchmark settings](./BenckmarkSetting.md) page for more details. @@ -8,6 +8,6 @@ an apple-to-apple comparison is critical for the machine learning community, par We randomly select 40% training data from the six open-sourced training datasets, i.e., ``"Dahoas/rm-static", "Dahoas/full-hh-rlhf", "Dahoas/synthetic-instruct-gptj-pairwise", "yitingxie/rlhf-reward-datasets", "openai/webgpt_comparisons"``, and ``"stanfordnlp/SHP"``. The total training samples we have is 264,292. We fix the query (prompt) sequence length as **256** and generate fixed-length answer with **256** tokens. As such, the total training tokens per epoch is 135,317,504. During benchmark testing, we set the training epoch number as 1. -As mentioned in the instability of [RLHF Training Tutorial](./README.md#🙋-instablity-of-rlhf-training-and-others), we found that it is not stable to update the actor model multiple times using the generated data. Therefore, we set ``per_device_train_batch_size=per_device_mini_batch_size`` and ``ppo_epochs=generation_batch_numbers=1`` for all of our benchmark results. During testing, we also set an upper bound for the maximum global training tokens at 524,288 (batch size of 1024 with a sequence length of 512). This is the largest batch size we found during our exploration that provides a stable RLHF training experience. Users and practitioners may find better training hyperparameters to further increase this. Additionally, during testing, whenever the global training token batch size does not exceed our limit of 524,288, we always use the largest training batch size that does not result in an out-of-memory error to benchmark the time. +As mentioned in the instability of [RLHF Training Tutorial](./README.md#🙋-instablity-of-rlhf-training-and-others), we found that it is not stable to update the actor model multiple times using the generated data. Therefore, we set ``per_device_generation_batch_size=per_device_training_batch_size`` and ``ppo_epochs=generation_batches=1`` for all of our benchmark results. During testing, we also set an upper bound for the maximum global training tokens at 524,288 (batch size of 1024 with a sequence length of 512). This is the largest batch size we found during our exploration that provides a stable RLHF training experience. Users and practitioners may find better training hyperparameters to further increase this. Additionally, during testing, whenever the global training token batch size does not exceed our limit of 524,288, we always use the largest training batch size that does not result in an out-of-memory error to benchmark the time. We hope this clearly explains our benchmark settings, and please do not hesitate to contact us if you need more information. If you'd like to reproduce our performance results or make a comparison with DeepSpeed-RLHF, we would like to encourage you to leverage the same / similar settings such that the performance results are more comparable. diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/README.md b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/README.md index 04349d3fd..b2c34f572 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/README.md +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/README.md @@ -30,7 +30,7 @@ To overcome both challenges, we are introducing the DeepSpeed Hybrid Engine (Dee We provide multiple actor training scripts in the 'training_scripts' folder, all using a fixed OPT-350m reward model. However, users are encouraged to experiment with different reward model sizes based on their preferences. For example, if you have a single GPU and want to train an OPT-1.3B model, you can simply run the following bash script to initiate the training process. ```bash -training_scripts/single_gpu/run_1.3b.sh +training_scripts/opt/single_gpu/run_1.3b.sh ``` ## 🎵 Some arguments explanation and largest model training on your own system @@ -41,8 +41,8 @@ We provide most of unique arguments used in DeepSpeed RLHF other than the previo | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | --unsupervised_dataset_name and --unsupervised_dataset_config_name | Huggingface datasets standard setting to collect the data, e.g., using Wikitext-103 | When both are provided, during each PPO training, we will also add the pretraining objective. Based on InstructGPT, this will enhance the model's benchmark performance. | | --unsup_coef | Used to balance RLHF/PPO loss and the unsupervised loss | | -| --per_device_train_batch_size and --per_device_mini_batch_size | The first one is the generation batch size and the second one is the PPO training batch size | Usually, the first one needs to be divisbale by the first one. | -| --generation_batch_numbers | Generated N batches then do PPO training | This setting is common in RL, i.e., we generate an experiment table then do RL training | +| --per_device_generation_batch_size and --per_device_training_batch_size | The first one is the generation batch size and the second one is the PPO training batch size | Usually, the first one needs to be divisible by the second one. | +| --generation_batches | Generated N batches then do PPO training | This setting is common in RL, i.e., we generate an experiment table then do RL training | | --ppo_epochs | For the generated experiments, how many PPO epochs we want to perform | | | --max_prompt_seq_len and --max_answer_seq_len | The length of the query and the length of the answer | | | --enable_hybrid_engine | Enable it to use DeepSpeed Hybrid Engine | This will significantly speedup your training | @@ -69,7 +69,7 @@ Users can either use the `prompt_eval.py` script from Step 1 of the SFT process RLHF is a relatively new field, and as expected, we have encountered some training instabilities during our exploration. We are sharing our findings here and actively working on solutions. We also welcome solutions from the community. -We have found that it is very unstable to use different generation training batch sizes (`--per_device_train_batch_size`) and PPO training batch sizes (`--per_device_mini_batch_size`), more than one PPO training epoch (`--ppo_epochs`), or more than one generation batch size (`--generation_batch_numbers`). These all point to the same problem: we are not able to update the actor model multiple times after generating experimental data. Therefore, in all of our successful runs, we have set `per_device_train_batch_size=per_device_mini_batch_size` and `ppo_epochs=generation_batch_numbers=1`. This is unexpected for a standard RL training pipeline, and we have tried different methods to overcome this, but all have failed. One of the most likely reasons for this instability is that we found the `log_probs` and `old_log_probs` used in the `actor_loss_fn` function can quickly diverge even within two consecutive iterations, which causes the corresponding `ratio` to be huge. Setting a strict upper bound can alleviate this problem, but it cannot fully resolve the convergence issue. +We have found that it is very unstable to use different generation training batch sizes (`--per_device_generation_batch_size`) and PPO training batch sizes (`--per_device_training_batch_size`), more than one PPO training epoch (`--ppo_epochs`), or more than one generation batch (`--generation_batches 1`). These all point to the same problem: we are not able to update the actor model multiple times after generating experimental data. Therefore, in all of our successful runs, we have set `per_device_generation_batch_size=per_device_training_batch_size` and `ppo_epochs=generation_batches=1`. This is unexpected for a standard RL training pipeline, and we have tried different methods to overcome this, but all have failed. One of the most likely reasons for this instability is that we found the `log_probs` and `old_log_probs` used in the `actor_loss_fn` function can quickly diverge even within two consecutive iterations, which causes the corresponding `ratio` to be huge. Setting a strict upper bound can alleviate this problem, but it cannot fully resolve the convergence issue. We have also found that adding unsupervised training is not easy. We tried using the coefficient (`--unsup_coef=27.8`) provided by InstructGPT, but it caused instability in the RLHF training. According to InstructGPT, unsupervised training mainly affects the model quality on standard benchmarks instead of the RLHF performance. We did not put much effort into tuning this parameter. diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/main.py b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/main.py index 5a60c927a..1378dc4e6 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/main.py +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/main.py @@ -19,10 +19,13 @@ import argparse import os import random +import time import torch from torch.utils.data import DataLoader, RandomSampler from torch.utils.data.distributed import DistributedSampler +from torch.utils.tensorboard import SummaryWriter + from transformers import ( SchedulerType, default_data_collator, @@ -30,19 +33,20 @@ import deepspeed -from ppo_trainer import DeepSpeedPPOTrainer, DeepSpeedPPOTrainerUnsupervised -from rlhf_engine import DeepSpeedRLHFEngine - -import sys +from dschat.rlhf.ppo_trainer import DeepSpeedPPOTrainer, DeepSpeedPPOTrainerUnsupervised +from dschat.rlhf.rlhf_engine import DeepSpeedRLHFEngine +from dschat.utils.data.data_utils import create_prompt_dataset, MiniDataset, DataCollatorRLHF, get_unsupervised_data +from dschat.utils.utils import print_rank_0, to_device, save_hf_format, set_random_seed, get_all_reduce_mean, moving_average, save_zero_three_model, load_hf_tokenizer, \ + ExponentialMovingAverage +from dschat.utils.module.lora import convert_lora_to_linear_layer +from dschat.utils.perf import print_throughput_step3 +from deepspeed.accelerator import get_accelerator -sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) -from utils.data.data_utils import create_prompt_dataset, MiniDataset, DataCollatorRLHF, get_unsupervised_data -from utils.utils import print_rank_0, to_device, save_hf_format, set_random_seed, get_all_reduce_mean, moving_average, save_zero_three_model, load_hf_tokenizer -from utils.module.lora import convert_lora_to_linear_layer +writer = None def parse_args(): + global writer parser = argparse.ArgumentParser( description="(Step 3) RLHF training arguments") @@ -59,7 +63,7 @@ def parse_args(): default='2,4,4', help= 'Comma-separated list of proportions for training phase 1, 2, and 3 data. For example the split `2,4,4` ' - 'will use 60% of data for phase 1, 20% for phase 2 and 20% for phase 3.' + 'will use 60%% of data for phase 1, 20%% for phase 2 and 20%% for phase 3.' ) parser.add_argument( '--data_output_path', @@ -104,20 +108,20 @@ def parse_args(): "OPT model has a fixed number (1) of padding tokens at the beginning of the input. We did not see this in other models but keep it as an option for now." ) parser.add_argument( - "--per_device_train_batch_size", + "--per_device_generation_batch_size", type=int, default=16, help= "Batch size (per device) for the training dataloader and generation purpose." ) parser.add_argument( - "--per_device_mini_train_batch_size", + "--per_device_training_batch_size", type=int, default=16, help= "Mini Batch size (per device) for the training dataloader and training purpose." ) - parser.add_argument("--generation_batch_numbers", + parser.add_argument("--generation_batches", type=int, default=1, help="Generate x batches to go to training mode.") @@ -233,10 +237,18 @@ def parse_args(): parser.add_argument('--offload', action='http://github.com/store_true', help='Enable ZeRO Offload techniques.') + parser.add_argument('--dtype', + type=str, + default='fp16', + choices=['fp16', 'bf16'], + help='Training data type') parser.add_argument( '--offload_reference_model', action='http://github.com/store_true', help='Enable ZeRO Offload techniques for reference model') + parser.add_argument('--offload_reward_model', + action='http://github.com/store_true', + help='Enable ZeRO Offload techniques for reward model') parser.add_argument( '--actor_zero_stage', type=int, @@ -255,12 +267,20 @@ def parse_args(): '--critic_gradient_checkpointing', action='http://github.com/store_true', help='Enable HF gradient checkpointing for Critic model.') - parser.add_argument('--disable_actor_dropout', - action='http://github.com/store_true', - help='Disable the dropout of the actor model.') - parser.add_argument('--disable_critic_dropout', - action='http://github.com/store_true', - help='Disable the dropout of the critical model.') + parser.add_argument( + "--actor_dropout", + type=float, + default=None, + help="If actor dropout configured, use it. " + "Otherwise, keep the default dropout configuration of the actor model." + ) + parser.add_argument( + "--critic_dropout", + type=float, + default=None, + help="If critic dropout configured, use it. " + "Otherwise, keep the default dropout configuration of the critic model." + ) ## LoRA for efficient training setting parser.add_argument("--actor_lora_dim", type=int, @@ -281,27 +301,98 @@ def parse_args(): parser.add_argument('--only_optimize_lora', action='http://github.com/store_true', help='Only optimize the LoRA parameters.') + parser.add_argument( + "--actor_lora_learning_rate", + type=float, + default=5e-4, + help= + "Initial actor LoRA learning rate (after the potential warmup period) to use." + ) + parser.add_argument( + "--critic_lora_learning_rate", + type=float, + default=5e-4, + help= + "Initial critic LoRA learning rate (after the potential warmup period) to use." + ) ## Make EMA as an optional feature parser.add_argument('--enable_ema', action='http://github.com/store_true', help='Enable EMA checkpoint for the model.') + ## Mixed Precision ZeRO++ + parser.add_argument( + '--enable_mixed_precision_lora', + action='http://github.com/store_true', + help='Enable Mixed Precision ZeRO++ for training and generation.') + ## low precision + parser.add_argument( + '--compute_fp32_loss', + action='http://github.com/store_true', + help='Relevant for low precision dtypes (fp16, bf16, etc.). ' + 'If specified, loss is calculated in fp32.' + 'This applies for both actor and critic models.') + ## Tensorboard logging + parser.add_argument('--enable_tensorboard', + action='http://github.com/store_true', + help='Enable tensorboard logging') + parser.add_argument('--tensorboard_path', + type=str, + default="step3_tensorboard") + ## Tokenizer + parser.add_argument( + "--add_eot_token", + action='http://github.com/store_true', + help="Add <|endoftext|> as additional special token to tokenizer") + ## Actor/critic model overflow alignment + parser.add_argument( + '--align_overflow', + action='http://github.com/store_true', + help='Align loss scale overflow between actor and critic') + ## Print actor model answers during training + parser.add_argument('--print_answers', + action='http://github.com/store_true', + help='Print prompt and answers during training') + parser.add_argument( + "--print_answers_interval", + type=int, + default=1, + help="If --print_answers enabled, controls the printing interval.") + ## Testing + parser.add_argument( + '--enable_test_mode', + action='http://github.com/store_true', + help= + 'Enable a testing mode that terminates training based on args.test_stop_step' + ) + parser.add_argument( + "--test_stop_step", + type=int, + default=0, + help= + "Training non-overflow step at which to terminate training during testing." + ) parser = deepspeed.add_config_arguments(parser) args = parser.parse_args() - # Validate settings - if (args.actor_gradient_checkpointing - and args.actor_lora_dim > 0) or (args.critic_gradient_checkpointing - and args.critic_lora_dim > 0): - assert ( - not args.only_optimize_lora - ), "--{actor,critic}_gradient_checkpointing and --only_optimize_lora cannot be enabled at the same time." + if args.enable_tensorboard: + print( + f"Tensorboard logs going to: {args.tensorboard_path}/step3_tensorboard_logs" + ) + writer = SummaryWriter( + f"{args.tensorboard_path}/step3_tensorboard_logs") + # Validate settings if args.inference_tp_size > 1: assert ( args.actor_zero_stage == 3 ), "Zero stage 3 must be used to do Tensor sharding in the hybrid engine" + if args.actor_zero_stage == 2 and args.critic_zero_stage == 2 and args.enable_hybrid_engine and args.offload and args.actor_lora_dim == 0: + raise ValueError( + "The combination of [actor_zero_stage==2, critic_zero_stage==2, enable_hybrid_engine=True, offload=True, lora=False] is currently unsupported due to training instability!" + ) + return args @@ -333,19 +424,19 @@ def create_datasets(args, tokenizer, train_phase=3): prompt_train_dataset, collate_fn=data_collator, sampler=prompt_train_sampler, - batch_size=args.per_device_train_batch_size) + batch_size=args.per_device_generation_batch_size) if unsupervised_training_enabled: unsupervised_train_dataloader = DataLoader( unsupervised_train_dataset, collate_fn=default_data_collator, sampler=unsupervised_train_sampler, - batch_size=args.per_device_train_batch_size) + batch_size=args.per_device_generation_batch_size) else: unsupervised_train_dataloader = [None] * len( prompt_train_dataloader) # basically a dummy dataloader num_update_steps_per_epoch = min(len(prompt_train_dataloader), len(unsupervised_train_dataloader)) * \ - (args.per_device_train_batch_size / args.per_device_mini_train_batch_size) * \ + (args.per_device_generation_batch_size / args.per_device_training_batch_size) * \ args.ppo_epochs / args.gradient_accumulation_steps num_total_iters = int(args.num_train_epochs * num_update_steps_per_epoch) @@ -356,10 +447,10 @@ def main(): args = parse_args() if args.local_rank == -1: - device = torch.device("cuda") + device = torch.device(get_accelerator().device_name()) else: - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) + get_accelerator().set_device(args.local_rank) + device = torch.device(get_accelerator().device_name(), args.local_rank) # Initializes the distributed backend which will take care of sychronizing nodes/GPUs deepspeed.init_distributed() @@ -376,12 +467,13 @@ def main(): set_random_seed(args.seed) torch.distributed.barrier() - # create common tokenizer based on actor model + # load_hf_tokenizer will get the correct tokenizer and set padding tokens based on the model family + args.end_of_conversation_token = "<|endoftext|>" + additional_special_tokens = args.end_of_conversation_token if args.add_eot_token else None tokenizer = load_hf_tokenizer(args.actor_model_name_or_path, - fast_tokenizer=True) - tokenizer.pad_token = tokenizer.eos_token - # make sure tokenizer is right pad in our logic - tokenizer.padding_side = 'right' + fast_tokenizer=True, + add_special_tokens=additional_special_tokens) + prompt_train_dataloader, unsupervised_train_dataloader, num_total_iters = create_datasets( args=args, tokenizer=tokenizer, train_phase=3) @@ -393,19 +485,30 @@ def main(): num_total_iters=num_total_iters, args=args) - args.end_of_conversation_token = "<|endoftext|>" + # Mixed Precision ZeRO++ + if args.enable_mixed_precision_lora: + assert args.actor_lora_dim > 0, "Mixed Precision LoRA requires LoRA to be enabled" + assert args.actor_zero_stage == 3, "Mixed Precision LoRA requires Zero stage 3" + rlhf_engine.actor.optimizer.quantize_nontrainable_params() + print_rank_0("Mixed Precision ZeRO++ enabled") ppo_trainer = DeepSpeedPPOTrainerUnsupervised if unsupervised_training_enabled else DeepSpeedPPOTrainer trainer = ppo_trainer(rlhf_engine, args) # first number is how many experience-batch to generate, second number is the training batch size, which is the micro-batch size used - exp_mini_dataset = MiniDataset(args.generation_batch_numbers, - args.per_device_mini_train_batch_size) - unsup_mini_dataset = MiniDataset(args.generation_batch_numbers, - args.per_device_mini_train_batch_size) + exp_mini_dataset = MiniDataset(args.generation_batches, + args.per_device_training_batch_size) + unsup_mini_dataset = MiniDataset(args.generation_batches, + args.per_device_training_batch_size) # Train! - print_rank_0("***** Running training *****", args.global_rank) + print_rank_0( + f"***** Running training (total_iters={num_total_iters}) *****", + args.global_rank) + + non_overflow_step_count = 0 + step_average_reward = 0. + ema_reward_score = ExponentialMovingAverage() for epoch in range(args.num_train_epochs): print_rank_0( @@ -413,13 +516,9 @@ def main(): args.global_rank) for step, (batch_prompt, batch_unsupervised) in enumerate( zip(prompt_train_dataloader, unsupervised_train_dataloader)): + batch_prompt = to_device(batch_prompt, device) - if batch_unsupervised is not None: - batch_unsupervised = to_device(batch_unsupervised, device) - unsup_dataset = unsup_mini_dataset.add(batch_unsupervised) - else: - unsup_dataset = unsup_mini_dataset.add( - [[None] * args.per_device_train_batch_size]) + # prompts = batch_prompt['prompt'] # length = prompts.size(-1) # if length > args.max_prompt_seq_len: @@ -427,7 +526,17 @@ def main(): # raise ValueError("Prompt length is too long") out = trainer.generate_experience(batch_prompt['prompt'], - batch_prompt['prompt_att_mask']) + batch_prompt['prompt_att_mask'], + step) + + training_start = time.time() + if batch_unsupervised is not None: + batch_unsupervised = to_device(batch_unsupervised, device) + unsup_dataset = unsup_mini_dataset.add(batch_unsupervised) + else: + unsup_dataset = unsup_mini_dataset.add( + [[None] * args.per_device_generation_batch_size]) + exp_dataset = exp_mini_dataset.add(out) if exp_dataset is not None: @@ -460,20 +569,64 @@ def main(): random.shuffle(exp_dataset) random.shuffle(unsup_dataset) + end = time.time() + training_time = end - training_start + e2e_time = training_time + trainer.generate_time * args.generation_batches # it is an approximation, we did not include, e.g., rw forward time etc + print_rank_0( - f'epoch: {epoch}|step: {step}|ppo_ep: {ppo_ep+1}|act_loss: {actor_loss_sum/inner_iter}|cri_loss: {critic_loss_sum/inner_iter}|unsuper_loss: {unsup_loss_sum/inner_iter}', + f'Epoch: {epoch} | Step: {step} | PPO Epoch: {ppo_ep+1} | Actor Loss: {actor_loss_sum/inner_iter} | Critic Loss: {critic_loss_sum/inner_iter} | Unsupervised Loss: {unsup_loss_sum/inner_iter}', args.global_rank) + print_throughput_step3(rlhf_engine.actor.module, + rlhf_engine.critic, args, e2e_time, + trainer.generate_time, training_time, + args.global_rank) + average_reward = get_all_reduce_mean(average_reward).item() + step_average_reward += average_reward / args.gradient_accumulation_steps_actor + if (step + 1) % args.gradient_accumulation_steps_actor == 0: + ema_reward_score.update(step_average_reward) + step_average_reward = 0. + print_rank_0( - f"average reward score: {average_reward/inner_iter}", + f"Average reward score: {average_reward/inner_iter} | EMA reward score: {ema_reward_score.get()}", args.global_rank) print_rank_0( "-------------------------------------------------------------------------------------", args.global_rank) + if args.enable_tensorboard and torch.distributed.get_rank( + ) == 0: + writer.add_scalar('reward', + average_reward / inner_iter, + global_step=step) + writer.add_scalar('actor_loss', + actor_loss.item(), + global_step=step) + writer.add_scalar('actor_loss_sum', + actor_loss_sum, + global_step=step) + writer.add_scalar('critic_loss', + critic_loss.item(), + global_step=step) + writer.add_scalar('critic_loss_sum', + critic_loss_sum, + global_step=step) + writer.flush() + if args.actor_gradient_checkpointing: rlhf_engine.actor.gradient_checkpointing_disable() + actor_overflow, critic_overflow = trainer.get_overflow() + + if not actor_overflow and not critic_overflow: + non_overflow_step_count += 1 + + if args.enable_test_mode and non_overflow_step_count == args.test_stop_step: + break + + if args.enable_test_mode: + break + if args.output_dir is not None: print_rank_0('saving model ...') rlhf_engine.actor = convert_lora_to_linear_layer(rlhf_engine.actor) diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/ppo_trainer.py b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/ppo_trainer.py deleted file mode 100644 index 2e730aed3..000000000 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/ppo_trainer.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -import torch -import torch.nn.functional as F -import sys -import os -import deepspeed -from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus - -sys.path.append( - os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) - -from utils.utils import print_rank_0 - - -def print_all_ranks(tag, value, rank): - world_size = torch.distributed.get_world_size() - all_tensor = torch.zeros(world_size, dtype=torch.float32).cuda() - all_tensor[rank] = value - torch.distributed.all_reduce(all_tensor, op=torch.distributed.ReduceOp.SUM) - print_rank_0(f'{tag} {all_tensor}', rank) - - -def get_model_norm(model): - with torch.no_grad(): - total = 0.0 - for param in model.parameters(): - should_gather = hasattr( - param, - 'ds_id') and param.ds_status == ZeroParamStatus.NOT_AVAILABLE - with deepspeed.zero.GatheredParameters(param, - enabled=should_gather): - total += float(param.float().norm()) - - return total - - -def gather_log_probs(logits, labels): - log_probs = F.log_softmax(logits, dim=-1) - log_probs_labels = log_probs.gather(dim=-1, index=labels.unsqueeze(-1)) - return log_probs_labels.squeeze(-1) - - -class DeepSpeedPPOTrainer(): - - def __init__(self, rlhf_engine, args): - self.rlhf_engine = rlhf_engine - self.actor_model = self.rlhf_engine.actor - self.critic_model = self.rlhf_engine.critic - self.ref_model = self.rlhf_engine.ref - self.reward_model = self.rlhf_engine.reward - self.tokenizer = self.rlhf_engine.tokenizer - self.args = args - self.max_answer_seq_len = args.max_answer_seq_len - self.end_of_conversation_token_id = self.tokenizer( - args.end_of_conversation_token)['input_ids'][-1] - - # Those value can be changed - self.kl_ctl = 0.02 - self.clip_reward_value = 5 - self.cliprange = 0.2 - self.cliprange_value = 0.2 - self.gamma = 1.0 - self.lam = 0.95 - - def _generate_sequence(self, prompts, mask): - - max_min_length = self.max_answer_seq_len + prompts.shape[1] - - with torch.no_grad(): - seq = self.actor_model.module.generate(prompts, - attention_mask=mask, - max_length=max_min_length, - min_length=max_min_length) - - # Filter out seq with no answers (or very short). This happens when users directly use the pre-training ckpt without supervised finetuning - # NOTE: this will causes each GPU has different number of examples - batch_size = seq.shape[0] - prompt_length = prompts.shape[1] - ans = seq[:, prompt_length:] - self.prompt_length = prompt_length - valid_ans_len = (ans != self.tokenizer.pad_token_id).sum(dim=-1) - out_seq = [] - for i in range(batch_size): - if valid_ans_len[ - i] <= 1: # if the answer is shorter than 1 token, drop it - continue - else: - out_seq.append(seq[i:i + 1]) - out_seq = torch.cat(out_seq, dim=0) # concate output in the batch dim - - return out_seq - - def generate_experience(self, prompts, mask): - self.eval() - seq = self._generate_sequence(prompts, mask) - self.train() - - pad_token_id = self.tokenizer.pad_token_id - attention_mask = seq.not_equal(pad_token_id).long() - - with torch.no_grad(): - output = self.actor_model(seq, attention_mask=attention_mask) - output_ref = self.ref_model(seq, attention_mask=attention_mask) - reward_score = self.reward_model.forward_value( - seq, attention_mask, - prompt_length=self.prompt_length)['chosen_end_scores'].detach( - ) - values = self.critic_model.forward_value( - seq, attention_mask, return_value_only=True).detach()[:, :-1] - - logits = output.logits - logits_ref = output_ref.logits - - return { - 'prompts': prompts, - 'logprobs': gather_log_probs(logits[:, :-1, :], seq[:, 1:]), - 'ref_logprobs': gather_log_probs(logits_ref[:, :-1, :], seq[:, - 1:]), - 'value': values, - 'rewards': reward_score, - 'input_ids': seq, - "attention_mask": attention_mask - } - - def compute_rewards(self, prompts, log_probs, ref_log_probs, reward_score, - action_mask): - - kl_divergence_estimate = -self.kl_ctl * (log_probs - ref_log_probs) - rewards = kl_divergence_estimate - start = prompts.shape[1] - 1 - ends = start + action_mask[:, start:].sum(1) - reward_clip = torch.clamp(reward_score, -self.clip_reward_value, - self.clip_reward_value) - batch_size = log_probs.shape[0] - for j in range(batch_size): - rewards[j, start:ends[j]][-1] += reward_clip[j] - - return rewards - - def train_rlhf(self, inputs): - # train the rlhf mode here - ### process the old outputs - prompts = inputs['prompts'] - log_probs = inputs['logprobs'] - ref_log_probs = inputs['ref_logprobs'] - reward_score = inputs['rewards'] - values = inputs['value'] - attention_mask = inputs['attention_mask'] - seq = inputs['input_ids'] - - start = prompts.size()[-1] - 1 - action_mask = attention_mask[:, 1:] - - old_values = values - with torch.no_grad(): - old_rewards = self.compute_rewards(prompts, log_probs, - ref_log_probs, reward_score, - action_mask) - advantages, returns = self.get_advantages_and_returns( - old_values, old_rewards, start) - - ### process the new outputs - batch = {'input_ids': seq, "attention_mask": attention_mask} - actor_prob = self.actor_model(**batch, use_cache=False).logits - actor_log_prob = gather_log_probs(actor_prob[:, :-1, :], seq[:, 1:]) - actor_loss = self.actor_loss_fn(actor_log_prob[:, start:], - log_probs[:, start:], advantages, - action_mask[:, start:]) - self.actor_model.backward(actor_loss) - self.actor_model.step() - value = self.critic_model.forward_value(**batch, - return_value_only=True, - use_cache=False)[:, :-1] - critic_loss = self.critic_loss_fn(value[:, start:], old_values[:, - start:], - returns, action_mask[:, start:]) - self.critic_model.backward(critic_loss) - self.critic_model.step() - - return actor_loss, critic_loss - - def actor_loss_fn(self, logprobs, old_logprobs, advantages, mask): - ## policy gradient loss - log_ratio = (logprobs - old_logprobs) * mask - ratio = torch.exp(log_ratio) - pg_loss1 = -advantages * ratio - pg_loss2 = -advantages * torch.clamp(ratio, 1.0 - self.cliprange, - 1.0 + self.cliprange) - pg_loss = torch.sum(torch.max(pg_loss1, pg_loss2) * mask) / mask.sum() - return pg_loss - - def critic_loss_fn(self, values, old_values, returns, mask): - ## value loss - values_clipped = torch.clamp( - values, - old_values - self.cliprange_value, - old_values + self.cliprange_value, - ) - vf_loss1 = (values - returns)**2 - vf_loss2 = (values_clipped - returns)**2 - vf_loss = 0.5 * torch.sum( - torch.max(vf_loss1, vf_loss2) * mask) / mask.sum() - return vf_loss - - def get_advantages_and_returns(self, values, rewards, start): - # Adopted from https://github.com/CarperAI/trlx/blob/main/trlx/models/modeling_ppo.py#L134 - lastgaelam = 0 - advantages_reversed = [] - length = rewards.size()[-1] - for t in reversed(range(start, length)): - nextvalues = values[:, t + 1] if t < length - 1 else 0.0 - delta = rewards[:, t] + self.gamma * nextvalues - values[:, t] - lastgaelam = delta + self.gamma * self.lam * lastgaelam - advantages_reversed.append(lastgaelam) - advantages = torch.stack(advantages_reversed[::-1], dim=1) - returns = advantages + values[:, start:] - return advantages.detach(), returns - - def _validate_training_mode(self): - assert self.actor_model.module.training - assert self.critic_model.module.training - - def _validate_evaluation_mode(self): - assert not self.actor_model.module.training - assert not self.critic_model.module.training - assert not self.ref_model.module.training - assert not self.reward_model.module.training - - def train(self): - self.actor_model.train() - self.critic_model.train() - - def eval(self): - self.actor_model.eval() - self.critic_model.eval() - self.reward_model.eval() - self.ref_model.eval() - - def dump_model_norms(self, tag): - actor_model_norm = get_model_norm(self.actor_model) - ref_model_norm = get_model_norm(self.ref_model) - critic_model_norm = get_model_norm(self.critic_model) - reward_model_norm = get_model_norm(self.reward_model) - print_all_ranks(f'{tag} global_actor_model_norm', actor_model_norm, - self.args.local_rank) - print_all_ranks(f'{tag} global_ref_model_norm', ref_model_norm, - self.args.local_rank) - print_all_ranks(f'{tag} global_critic_model_norm', critic_model_norm, - self.args.local_rank) - print_all_ranks(f'{tag} global_reward_model_norm', reward_model_norm, - self.args.local_rank) - - -class DeepSpeedPPOTrainerUnsupervised(DeepSpeedPPOTrainer): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def train_unsupervised(self, inputs, unsup_coef): - # Train the unsupervised model here - self._validate_training_mode() - - outputs = self.actor_model(**inputs, use_cache=False) - loss = outputs.loss - self.actor_model.backward(unsup_coef * loss) - self.actor_model.step() - - return loss diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/rlhf_engine.py b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/rlhf_engine.py deleted file mode 100644 index c9390e892..000000000 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/rlhf_engine.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -import time -import torch -import deepspeed -from deepspeed.ops.adam import FusedAdam -from deepspeed.ops.adam import DeepSpeedCPUAdam -from transformers import AutoModelForCausalLM, get_scheduler - -from utils.ds_utils import get_train_ds_config, get_eval_ds_config -from utils.module.lora import convert_linear_layer_to_lora, only_optimize_lora_parameters -from utils.model.model_utils import create_hf_model, create_critic_model -from utils.utils import get_optimizer_grouped_parameters -""" -TODOs: - * support HF models for critic (for debugging), must be a previously saved ckpt from step-2 - * determine ds_config/zero_stage based on model size, gpu style, world size, etc - - get model size by creating simple meta model - - 1.3b: zero-2 for actor/ref models, zero-0 for others - - 13b+: zero-3 for all models -""" - - -def log_init(model_name, stime=None): - if torch.distributed.get_rank() == 0: - tag = "start" if stime is None else "end" - suffix = "ing" if stime is None else "ed" - duration = "" - if stime is not None: - duration = "(duration: {:.2f}s)".format(time.time() - stime) - msg = f"[{tag}] Initializ{suffix} {model_name} Model [{tag}] {duration}" - stars = (90 - len(msg)) // 2 - extra_star = "*" if (90 - len(msg)) % 2 == 1 else "" - print("*" * stars + msg + "*" * stars + extra_star) - return time.time() - - -class DeepSpeedRLHFEngine(): - - def __init__(self, actor_model_name_or_path, critic_model_name_or_path, - tokenizer, args, num_total_iters): - self.args = args - self.num_total_iters = num_total_iters - self.tokenizer = tokenizer - - self.actor = self._init_actor( - actor_model_name_or_path=actor_model_name_or_path) - self.ref = self._init_ref( - actor_model_name_or_path=actor_model_name_or_path) - self.actor_ema = None - if self.args.enable_ema: - self.actor_ema = self._init_ema( - actor_model_name_or_path=actor_model_name_or_path) - - self.critic = self._init_critic( - critic_model_name_or_path=critic_model_name_or_path) - self.reward = self._init_reward( - critic_model_name_or_path=critic_model_name_or_path) - if self.args.critic_gradient_checkpointing: - self.critic.gradient_checkpointing_enable() - - def _init_actor(self, actor_model_name_or_path): - stime = log_init("Actor") - - # DS Config - ds_config = get_train_ds_config( - offload=self.args.offload, - stage=self.args.actor_zero_stage, - enable_hybrid_engine=self.args.enable_hybrid_engine, - inference_tp_size=self.args.inference_tp_size, - release_inference_cache=self.args.release_inference_cache, - pin_parameters=(not self.args.unpin_actor_parameters), - tp_gather_partition_size=self.args.tp_gather_partition_size, - max_out_tokens=self.args.max_prompt_seq_len + - self.args.max_answer_seq_len) - ds_config[ - 'train_micro_batch_size_per_gpu'] = self.args.per_device_mini_train_batch_size - #TODO(jeff): we should probably set grad accumlation steps here as well for clarity - ds_config[ - 'train_batch_size'] = self.args.per_device_mini_train_batch_size * torch.distributed.get_world_size( - ) * self.args.gradient_accumulation_steps_actor - - # Model - actor_model = create_hf_model( - model_class=AutoModelForCausalLM, - model_name_or_path=actor_model_name_or_path, - tokenizer=self.tokenizer, - ds_config=ds_config, - disable_dropout=self.args.disable_actor_dropout) - - # LoRA - if self.args.actor_lora_dim > 0: - actor_model = convert_linear_layer_to_lora( - actor_model, self.args.actor_lora_module_name, - self.args.actor_lora_dim) - if self.args.only_optimize_lora: - actor_model = only_optimize_lora_parameters(actor_model) - - # Optimizer - AdamOptimizer = DeepSpeedCPUAdam if self.args.offload else FusedAdam - optim_params = get_optimizer_grouped_parameters( - actor_model, self.args.actor_weight_decay) - optim = AdamOptimizer(optim_params, - lr=self.args.actor_learning_rate, - betas=(0.9, 0.95)) - - # LR Scheduler - lr_scheduler = get_scheduler( - name=self.args.lr_scheduler_type, - optimizer=optim, - num_warmup_steps=self.args.num_warmup_steps, - num_training_steps=self.num_total_iters, - ) - - # DeepSpeed Engine - #TODO: move enable_hybrid_engine and pin_parameters to ds_config - actor_engine, *_ = deepspeed.initialize(model=actor_model, - optimizer=optim, - lr_scheduler=lr_scheduler, - config=ds_config) - - log_init("Actor", stime=stime) - - return actor_engine - - def _init_ref(self, actor_model_name_or_path): - stime = log_init("Ref") - # DS Config - zero_stage = self.args.actor_zero_stage - if zero_stage != 3: - # If actor is ZeRO-3 then we use it for everything, otherwise assume we have enough memory for ref model - zero_stage = 0 - ds_config = get_eval_ds_config(self.args.offload_reference_model, - zero_stage) - ds_config[ - 'train_micro_batch_size_per_gpu'] = self.args.per_device_mini_train_batch_size - #TODO(jeff): we should probably set grad accumlation steps here as well for clarity - ds_config[ - 'train_batch_size'] = self.args.per_device_mini_train_batch_size * torch.distributed.get_world_size( - ) * self.args.gradient_accumulation_steps_actor - - ref_model = create_hf_model(AutoModelForCausalLM, - actor_model_name_or_path, self.tokenizer, - ds_config) - - ref_engine, *_ = deepspeed.initialize(model=ref_model, - config=ds_config) - - log_init("Ref", stime=stime) - return ref_engine - - def _init_ema(self, actor_model_name_or_path): - stime = log_init("EMA") - # DS Config - zero_stage = self.args.actor_zero_stage - if zero_stage != 3: - # If actor is ZeRO-3 then we use it for everything, otherwise assume we have enough memory - zero_stage = 0 - ds_config = get_eval_ds_config(self.args.offload_reference_model, - zero_stage) - ds_config[ - 'train_micro_batch_size_per_gpu'] = self.args.per_device_mini_train_batch_size - #TODO(jeff): we should probably set grad accumlation steps here as well for clarity - ds_config[ - 'train_batch_size'] = self.args.per_device_mini_train_batch_size * torch.distributed.get_world_size( - ) * self.args.gradient_accumulation_steps_actor - - actor_model_ema = create_hf_model(AutoModelForCausalLM, - actor_model_name_or_path, - self.tokenizer, ds_config) - if self.args.actor_lora_dim > 0: - actor_model_ema = convert_linear_layer_to_lora( - actor_model_ema, self.args.actor_lora_module_name, - self.args.actor_lora_dim) - - ema_engine, *_ = deepspeed.initialize(model=actor_model_ema, - config=ds_config) - - log_init("EMA", stime=stime) - return ema_engine - - def _init_critic(self, critic_model_name_or_path): - stime = log_init("Critic") - ds_config = get_train_ds_config(offload=self.args.offload, - stage=self.args.critic_zero_stage) - ds_config[ - 'train_micro_batch_size_per_gpu'] = self.args.per_device_mini_train_batch_size - #TODO(jeff): we should probably set grad accumlation steps here as well for clarity - ds_config[ - 'train_batch_size'] = self.args.per_device_mini_train_batch_size * torch.distributed.get_world_size( - ) * self.args.gradient_accumulation_steps - - #TODO(jeff): should not be needed, we should be able to use ds_config above - #TODO(jeff): it means we never create the critic w. zero.init context if we are using ZeRO-3 - ds_eval_config = get_eval_ds_config(offload=False, stage=0) - - # Model - critic_model = create_critic_model( - model_name_or_path=critic_model_name_or_path, - tokenizer=self.tokenizer, - ds_config=ds_eval_config, - num_padding_at_beginning=self.args.num_padding_at_beginning, - rlhf_training=True, - disable_dropout=self.args.disable_critic_dropout) - - # LoRA - if self.args.critic_lora_dim > 0: - critic_model = convert_linear_layer_to_lora( - critic_model, self.args.critic_lora_module_name, - self.args.critic_lora_dim) - if self.args.only_optimize_lora: - critic_model = only_optimize_lora_parameters(critic_model) - - # Optimizer - AdamOptimizer = DeepSpeedCPUAdam if self.args.offload else FusedAdam - optim_pararms = get_optimizer_grouped_parameters( - critic_model, self.args.critic_weight_decay) - optim = AdamOptimizer(optim_pararms, - lr=self.args.critic_learning_rate, - betas=(0.9, 0.95)) - - # LR Scheduler - lr_scheduler = get_scheduler( - name=self.args.lr_scheduler_type, - optimizer=optim, - num_warmup_steps=self.args.num_warmup_steps, - num_training_steps=self.num_total_iters, - ) - - # DeepSpeed Engine - critic_engine, *_ = deepspeed.initialize(model=critic_model, - optimizer=optim, - lr_scheduler=lr_scheduler, - config=ds_config) - - log_init("Critic", stime=stime) - return critic_engine - - def _init_reward(self, critic_model_name_or_path): - stime = log_init("Reward") - # DS Config - zero_stage = self.args.critic_zero_stage - if zero_stage != 3: - # If critic is ZeRO-3 then we use it for everything, otherwise assume we have enough memory - zero_stage = 0 - - ds_config = get_eval_ds_config(offload=self.args.offload, - stage=zero_stage) - ds_config[ - 'train_micro_batch_size_per_gpu'] = self.args.per_device_mini_train_batch_size - ds_config[ - 'train_batch_size'] = self.args.per_device_mini_train_batch_size * torch.distributed.get_world_size( - ) * self.args.gradient_accumulation_steps - - #TODO(jeff): should not be needed, we should be able to use ds_config above - #TODO(jeff): it means we never create the critic w. zero.init context if we are using ZeRO-3 - ds_eval_config = get_eval_ds_config(offload=False, stage=0) - - # Model - reward_model = create_critic_model( - model_name_or_path=critic_model_name_or_path, - tokenizer=self.tokenizer, - ds_config=ds_eval_config, - num_padding_at_beginning=self.args.num_padding_at_beginning, - rlhf_training=True) - - reward_engine, *_ = deepspeed.initialize(model=reward_model, - config=ds_config) - - log_init("Reward", stime=stime) - return reward_engine diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_log_output/actor_opt-1.3b_critic_opt-350m_globalBatchSize64.log b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_log_output/actor_opt-1.3b_critic_opt-350m_globalBatchSize64.log index f4a0971eb..38437e3fe 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_log_output/actor_opt-1.3b_critic_opt-350m_globalBatchSize64.log +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_log_output/actor_opt-1.3b_critic_opt-350m_globalBatchSize64.log @@ -1,4 +1,4 @@ -cmd = deepspeed main.py --data_path Dahoas/rm-static --data_split 2,4,4 --actor_model_name_or_path ~/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/output --critic_model_name_or_path ~/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/output --num_padding_at_beginning 1 --per_device_train_batch_size 4 --per_device_mini_train_batch_size 4 --generation_batch_numbers 1 --ppo_epochs 1 --max_answer_seq_len 256 --max_prompt_seq_len 256 --actor_learning_rate 9.65e-6 --critic_learning_rate 5e-6 --actor_weight_decay 0. --critic_weight_decay 0. --num_train_epochs 1 --lr_scheduler_type cosine --gradient_accumulation_steps 1 --num_warmup_steps 100 --deepspeed --seed 1234 --enable_hybrid_engine --actor_zero_stage 2 --critic_zero_stage 2 --enable_ema --output_dir ./output_fourDatasets_NoWeightDecay_withDropout_mixed +cmd = deepspeed main.py --data_path Dahoas/rm-static --data_split 2,4,4 --actor_model_name_or_path ~/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/output --critic_model_name_or_path ~/DeepSpeedExamples/applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/output --num_padding_at_beginning 1 --per_device_generation_batch_size 4 --per_device_training_batch_size 4 --generation_batches 1 --ppo_epochs 1 --max_answer_seq_len 256 --max_prompt_seq_len 256 --actor_learning_rate 9.65e-6 --critic_learning_rate 5e-6 --actor_weight_decay 0. --critic_weight_decay 0. --num_train_epochs 1 --lr_scheduler_type cosine --gradient_accumulation_steps 1 --num_warmup_steps 100 --deepspeed --seed 1234 --enable_hybrid_engine --actor_zero_stage 2 --critic_zero_stage 2 --enable_ema --output_dir ./output_fourDatasets_NoWeightDecay_withDropout_mixed ***** Running training ***** Beginning of Epoch 1/1, Total Generation Batches 477 ------------------------------------------------------ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b.sh new file mode 100755 index 000000000..c58e94eab --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +OUTPUT=$5 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output_step3_llama +fi +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=3 +fi +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=3 +fi +mkdir -p $OUTPUT + +Num_Padding_at_Beginning=1 # this is model related + +Actor_Lr=9.65e-6 +Critic_Lr=5e-6 + +deepspeed --master_port 12346 main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --actor_model_name_or_path $ACTOR_MODEL_PATH \ + --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --num_padding_at_beginning 1 \ + --per_device_generation_batch_size 1 \ + --per_device_training_batch_size 1 \ + --generation_batches 1 \ + --ppo_epochs 1 \ + --max_answer_seq_len 256 \ + --max_prompt_seq_len 256 \ + --actor_learning_rate ${Actor_Lr} \ + --critic_learning_rate ${Critic_Lr} \ + --actor_weight_decay 0.1 \ + --critic_weight_decay 0.1 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --gradient_accumulation_steps 1 \ + --actor_gradient_checkpointing \ + --critic_gradient_checkpointing \ + --offload_reference_model \ + --actor_dropout 0.0 \ + --num_warmup_steps 100 \ + --deepspeed --seed 1234 \ + --actor_zero_stage $ACTOR_ZERO_STAGE \ + --critic_zero_stage $CRITIC_ZERO_STAGE \ + --enable_hybrid_engine \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh new file mode 100755 index 000000000..830c3750e --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_lora.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +OUTPUT=$5 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output_step3_llama2 +fi +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=3 +fi +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=3 +fi +mkdir -p $OUTPUT + +Num_Padding_at_Beginning=1 # this is model related + +Actor_Lr=9.65e-6 +Critic_Lr=5e-6 + +deepspeed --master_port 12346 main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --actor_model_name_or_path $ACTOR_MODEL_PATH \ + --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --num_padding_at_beginning 1 \ + --per_device_generation_batch_size 1 \ + --per_device_training_batch_size 1 \ + --generation_batches 1 \ + --ppo_epochs 1 \ + --max_answer_seq_len 256 \ + --max_prompt_seq_len 256 \ + --actor_learning_rate ${Actor_Lr} \ + --critic_learning_rate ${Critic_Lr} \ + --actor_weight_decay 0.1 \ + --critic_weight_decay 0.1 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --gradient_accumulation_steps 1 \ + --actor_gradient_checkpointing \ + --critic_gradient_checkpointing \ + --offload_reference_model \ + --actor_dropout 0.0 \ + --num_warmup_steps 100 \ + --deepspeed --seed 1234 \ + --actor_zero_stage $ACTOR_ZERO_STAGE \ + --critic_zero_stage $CRITIC_ZERO_STAGE \ + --enable_hybrid_engine \ + --actor_lora_dim 64 \ + --critic_lora_dim 64 \ + --critic_lora_module_name "layers." \ + --actor_lora_module_name "layers." \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_mixz.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_mixz.sh new file mode 100755 index 000000000..abde0b54a --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/llama2/run_llama2_7b_mixz.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +OUTPUT=$5 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output_step3_llama2 +fi +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=3 +fi +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=3 +fi +mkdir -p $OUTPUT + +Num_Padding_at_Beginning=1 # this is model related + +Actor_Lr=9.65e-6 +Critic_Lr=5e-6 + +deepspeed --master_port 12346 main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --actor_model_name_or_path $ACTOR_MODEL_PATH \ + --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --num_padding_at_beginning 1 \ + --per_device_generation_batch_size 1 \ + --per_device_training_batch_size 1 \ + --generation_batches 1 \ + --ppo_epochs 1 \ + --max_answer_seq_len 256 \ + --max_prompt_seq_len 256 \ + --actor_learning_rate ${Actor_Lr} \ + --critic_learning_rate ${Critic_Lr} \ + --actor_weight_decay 0.1 \ + --critic_weight_decay 0.1 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --gradient_accumulation_steps 1 \ + --actor_gradient_checkpointing \ + --critic_gradient_checkpointing \ + --offload_reference_model \ + --actor_dropout 0.0 \ + --num_warmup_steps 100 \ + --deepspeed --seed 1234 \ + --actor_zero_stage $ACTOR_ZERO_STAGE \ + --critic_zero_stage $CRITIC_ZERO_STAGE \ + --enable_mixed_precision_lora \ + --actor_lora_dim 64 \ + --critic_lora_dim 64 \ + --critic_lora_module_name "layers." \ + --actor_lora_module_name "layers." \ + --output_dir $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/multi_node/run_66b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/multi_node/run_66b.sh similarity index 90% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/multi_node/run_66b.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/multi_node/run_66b.sh index b0b716ffc..c70908ceb 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/multi_node/run_66b.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/multi_node/run_66b.sh @@ -30,9 +30,9 @@ deepspeed --master_port 12346 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_mini_train_batch_size 4 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ @@ -51,7 +51,7 @@ deepspeed --master_port 12346 main.py \ --actor_zero_stage $ACTOR_ZERO_STAGE \ --critic_zero_stage $CRITIC_ZERO_STAGE \ --actor_gradient_checkpointing \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ --actor_lora_dim 128 \ --actor_lora_module_name decoder.layers. \ --output_dir $OUTPUT \ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_1.3b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_1.3b.sh new file mode 100644 index 000000000..41cacebab --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_1.3b.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +OUTPUT=$5 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=0 +fi +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=0 +fi +mkdir -p $OUTPUT + +deepspeed --num_gpus 1 main.py \ + --actor_model_name_or_path $ACTOR_MODEL_PATH --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --actor_zero_stage $ACTOR_ZERO_STAGE --critic_zero_stage $CRITIC_ZERO_STAGE \ + --num_padding_at_beginning 1 --gradient_accumulation_steps 2 \ + --deepspeed --actor_lora_dim 128 --enable_hybrid_engine --actor_gradient_checkpointing --actor_dropout 0.0 \ + --output_dir $OUTPUT &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_6.7b_lora.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_6.7b_lora.sh similarity index 88% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_6.7b_lora.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_6.7b_lora.sh index 2c4c458ac..2c3a01d5f 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_6.7b_lora.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_gpu/run_6.7b_lora.sh @@ -25,13 +25,12 @@ deepspeed --num_gpus 1 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 8 \ - --per_device_mini_train_batch_size 8 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 8 \ + --per_device_training_batch_size 8 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ - --ppo_epochs 1 \ --actor_learning_rate ${Actor_Lr} \ --critic_learning_rate ${Critic_Lr} \ --num_train_epochs 1 \ @@ -44,7 +43,7 @@ deepspeed --num_gpus 1 main.py \ --actor_lora_dim 128 \ --actor_gradient_checkpointing \ --critic_gradient_checkpointing \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ --enable_hybrid_engine \ --output_dir $OUTPUT \ &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b.sh new file mode 100644 index 000000000..5449bfea4 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +OUTPUT=$5 +if [ "$OUTPUT" == "" ]; then + OUTPUT=./output +fi +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=2 +fi +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=2 +fi + +# if actor and critic model names are not provided, then use the publicly available AdamG012/chat-opt-1.3b-sft-deepspeed and AdamG012/chat-opt-350m-reward-deepspeed +if [ "$ACTOR_MODEL_PATH" == "" ]; then + ACTOR_MODEL_PATH=AdamG012/chat-opt-1.3b-sft-deepspeed +fi +if [ "$CRITIC_MODEL_PATH" == "" ]; then + CRITIC_MODEL_PATH=AdamG012/chat-opt-350m-reward-deepspeed +fi + +mkdir -p $OUTPUT + +Num_Padding_at_Beginning=1 # this is model related + +Actor_Lr=9.65e-6 +Critic_Lr=5e-6 + +deepspeed --master_port 12346 main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --actor_model_name_or_path $ACTOR_MODEL_PATH \ + --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --num_padding_at_beginning 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ + --ppo_epochs 1 \ + --max_answer_seq_len 256 \ + --max_prompt_seq_len 256 \ + --actor_learning_rate ${Actor_Lr} \ + --critic_learning_rate ${Critic_Lr} \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --gradient_accumulation_steps 1 \ + --actor_dropout 0.0 \ + --num_warmup_steps 100 \ + --deepspeed --seed 1234 \ + --enable_hybrid_engine \ + --actor_zero_stage $ACTOR_ZERO_STAGE \ + --critic_zero_stage $CRITIC_ZERO_STAGE \ + --enable_ema \ + --output_dir $OUTPUT \ + --enable_tensorboard \ + --tensorboard_path $OUTPUT \ + &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b_lora.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b_lora.sh similarity index 88% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b_lora.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b_lora.sh index b650a9ecc..b39ccb833 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b_lora.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_1.3b_lora.sh @@ -14,7 +14,7 @@ OUTPUT="./output" Num_Padding_at_Beginning=1 # this is model related -Actor_Lr=5e-4 +Actor_Lr=9.65e-6 Critic_Lr=5e-6 mkdir -p $OUTPUT @@ -25,9 +25,9 @@ deepspeed --master_port 12346 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_mini_train_batch_size 4 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ @@ -38,7 +38,7 @@ deepspeed --master_port 12346 main.py \ --gradient_accumulation_steps 1 \ --num_warmup_steps 100 \ --deepspeed --seed 1234 \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ ${ACTOR_ZERO_STAGE} \ ${CRITIC_ZERO_STAGE} \ --actor_lora_dim 128 \ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_13b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_13b.sh similarity index 90% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_13b.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_13b.sh index 1b3f805bb..82751bd7f 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_13b.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_13b.sh @@ -30,9 +30,9 @@ deepspeed --master_port 12346 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 16 \ - --per_device_mini_train_batch_size 16 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 16 \ + --per_device_training_batch_size 16 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ @@ -48,7 +48,7 @@ deepspeed --master_port 12346 main.py \ --actor_zero_stage $ACTOR_ZERO_STAGE \ --critic_zero_stage $CRITIC_ZERO_STAGE \ --actor_gradient_checkpointing \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ --actor_lora_dim 128 \ --actor_lora_module_name decoder.layers. \ --output_dir $OUTPUT \ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_30b_lora.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_30b_lora.sh similarity index 89% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_30b_lora.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_30b_lora.sh index 2846ac1a4..c5c9133ff 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_30b_lora.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_30b_lora.sh @@ -26,9 +26,9 @@ deepspeed --master_port 12346 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_mini_train_batch_size 4 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ @@ -38,7 +38,7 @@ deepspeed --master_port 12346 main.py \ --lr_scheduler_type cosine \ --gradient_accumulation_steps 1 \ --actor_gradient_checkpointing \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ --num_warmup_steps 100 \ --deepspeed --seed 1234 \ ${ACTOR_ZERO_STAGE} \ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_6.7b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_6.7b.sh similarity index 90% rename from applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_6.7b.sh rename to applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_6.7b.sh index 1fb66313d..f877bebdf 100644 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_6.7b.sh +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/run_6.7b.sh @@ -30,9 +30,9 @@ deepspeed --master_port 12346 main.py \ --actor_model_name_or_path $ACTOR_MODEL_PATH \ --critic_model_name_or_path $CRITIC_MODEL_PATH \ --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_mini_train_batch_size 4 \ - --generation_batch_numbers 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ --ppo_epochs 1 \ --max_answer_seq_len 256 \ --max_prompt_seq_len 256 \ @@ -44,7 +44,7 @@ deepspeed --master_port 12346 main.py \ --lr_scheduler_type cosine \ --gradient_accumulation_steps 1 \ --actor_gradient_checkpointing \ - --disable_actor_dropout \ + --actor_dropout 0.0 \ --num_warmup_steps 100 \ --deepspeed --seed 1234 \ --enable_hybrid_engine \ diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/README.md b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/README.md new file mode 100644 index 000000000..8f9a75271 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/README.md @@ -0,0 +1,22 @@ +# DeepSpeed Characterization Script + +# Contents + * [Introduction](#introduction) + * [Usage](#usage) + +# Introduction +The step 3 characterization script sweeps across various training parameters. Currently, the following parameters are swept: +
+Zero Stage: 2, 3 +Hybrid Engine: True, False +Offload: True, False +Lora: True, False ++ +The `run_step3_sweep.sh` script passes configuration arguments to `run_single.sh`, which can be extended to sweep beyond the parameters listed above (e.g. learning rate, weight decay, etc). + +# Usage +The sweep script can be run as follows: +
+DeepSpeedExamples/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning$ bash training_scripts/opt/single_node/sweep/run_step3_sweep.sh +diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_single.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_single.sh new file mode 100644 index 000000000..15ec6e576 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_single.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH=$1 +CRITIC_MODEL_PATH=$2 +ACTOR_ZERO_STAGE=$3 +CRITIC_ZERO_STAGE=$4 +ENABLE_HYBRID_ENGINE=$5 +OFFLOAD=$6 +LORA=$7 +MIXED_PRECISION_LORA=$8 +OUTPUT=$9 +TEST=${10} +TEST_STOP_STEP=${11} + +if [ "$ACTOR_ZERO_STAGE" == "" ]; then + ACTOR_ZERO_STAGE=2 +fi + +if [ "$CRITIC_ZERO_STAGE" == "" ]; then + CRITIC_ZERO_STAGE=2 +fi + +if [ "$ENABLE_HYBRID_ENGINE" == true ]; then + ENABLE_HYBRID_ENGINE="--enable_hybrid_engine" +else + ENABLE_HYBRID_ENGINE="" +fi + +if [ "$OFFLOAD" == true ]; then + OFFLOAD="--offload" +else + OFFLOAD="" +fi + +if [ "$LORA" == true ]; then + ACTOR_LORA_DIM="--actor_lora_dim 128" + ACTOR_LORA_MODULE_NAME="--actor_lora_module_name decoder.layers." +else + ACTOR_LORA_DIM="--actor_lora_dim 0" + ACTOR_LORA_MODULE_NAME="" +fi + +if [ "$MIXED_PRECISION_LORA" == true ]; then + MIXED_PRECISION_LORA="--enable_mixed_precision_lora" +else + MIXED_PRECISION_LORA="" +fi + +if [ "$TEST" == true ]; then + TEST="--enable_test_mode" + TEST_STOP_STEP="--test_stop_step ${TEST_STOP_STEP}" +else + TEST="" + TEST_STOP_STEP="" +fi + +mkdir -p $OUTPUT + +Num_Padding_at_Beginning=1 # this is model related + +Actor_Lr=9.65e-6 +Critic_Lr=5e-6 + +cmd="deepspeed --num_nodes=1 main.py \ + --data_path Dahoas/rm-static \ + --data_split 2,4,4 \ + --actor_model_name_or_path $ACTOR_MODEL_PATH \ + --critic_model_name_or_path $CRITIC_MODEL_PATH \ + --num_padding_at_beginning 1 \ + --per_device_generation_batch_size 4 \ + --per_device_training_batch_size 4 \ + --generation_batches 1 \ + --ppo_epochs 1 \ + --max_answer_seq_len 256 \ + --max_prompt_seq_len 256 \ + --actor_learning_rate ${Actor_Lr} \ + --critic_learning_rate ${Critic_Lr} \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --gradient_accumulation_steps 1 \ + --actor_weight_decay 0 \ + --critic_weight_decay 0 \ + --num_warmup_steps 100 \ + --deepspeed --seed 1234 \ + --actor_dropout 0.0 \ + --print_answers \ + --actor_zero_stage ${ACTOR_ZERO_STAGE} \ + --critic_zero_stage ${CRITIC_ZERO_STAGE} \ + --output_dir $OUTPUT \ + $ENABLE_HYBRID_ENGINE $OFFLOAD $MIXED_PRECISION_LORA \ + $ACTOR_LORA_DIM $ACTOR_LORA_MODULE_NAME\ + $TEST $TEST_STOP_STEP" + +echo "----------------------------- DS COMMAND -----------------------------" +echo $cmd + +$cmd &> $OUTPUT/${OUTPUT}.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_step3_sweep.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_step3_sweep.sh new file mode 100644 index 000000000..c0d0b98d7 --- /dev/null +++ b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/opt/single_node/sweep/run_step3_sweep.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +ACTOR_MODEL_PATH="AdamG012/chat-opt-1.3b-sft-deepspeed" +CRITIC_MODEL_PATH="AdamG012/chat-opt-350m-reward-deepspeed" + +# Sweep switches +RUN_GENERIC_SWEEP=true +RUN_MPL_SWEEP=true + +# Kill any existing Python processes +pkill -9 python +sleep 300 + +# Run generic sweep w/o Mixed Precision ZeRO++ +if [ "$RUN_GENERIC_SWEEP" == true ]; then + echo "----------------------------- RUNNING GENERIC SWEEPS -----------------------------" + echo "" + for z in {2..3} + do + for he in true false + do + for offload in true false + do + for lora in true false + do + mixed_precision_lora=false + cmd="bash training_scripts/opt/single_node/sweep/run_single.sh \ + $ACTOR_MODEL_PATH \ + $CRITIC_MODEL_PATH \ + ${z} \ + ${z} \ + ${he} \ + ${offload} \ + ${lora} \ + ${mixed_precision_lora} \ + z${z}_he_${he}_offload_${offload}_lora_${lora}" + echo "----------------------------- CALLING SHELL SCRIPT -----------------------------" + echo $cmd + $cmd + pkill -9 python + sleep 300 + echo "" + done + done + done + done + echo "" +fi + +# Run Mixed Precision ZeRO++ sweep +if [ "$RUN_MPL_SWEEP" == true ]; then + echo "----------------------------- RUNNING MIXED PRECISION ZERO++ SWEEPS -----------------------------" + echo "" + for he in true false + do + z=3 + offload=false + lora=true + mixed_precision_lora=true + cmd="bash training_scripts/opt/single_node/sweep/run_single.sh \ + $ACTOR_MODEL_PATH \ + $CRITIC_MODEL_PATH \ + ${z} \ + ${z} \ + ${he} \ + ${offload} \ + ${lora} \ + ${mixed_precision_lora} \ + z${z}_he_${he}_offload_${offload}_lora_${lora}_mpl_${mixed_precision_lora}" + echo "----------------------------- CALLING SHELL SCRIPT -----------------------------" + echo $cmd + $cmd + pkill -9 python + sleep 300 + echo "" + done + echo "" +fi diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_1.3b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_1.3b.sh deleted file mode 100644 index 1b1a5c489..000000000 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_gpu/run_1.3b.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -ACTOR_MODEL_PATH=$1 -CRITIC_MODEL_PATH=$2 -ACTOR_ZERO_STAGE=$3 -CRITIC_ZERO_STAGE=$4 -OUTPUT=$5 -if [ "$OUTPUT" == "" ]; then - OUTPUT=./output -fi -if [ "$ACTOR_ZERO_STAGE" == "" ]; then - ACTOR_ZERO_STAGE=0 -fi -if [ "$CRITIC_ZERO_STAGE" == "" ]; then - CRITIC_ZERO_STAGE=0 -fi -mkdir -p $OUTPUT - -deepspeed --num_gpus 1 main.py \ - --actor_model_name_or_path $ACTOR_MODEL_PATH --critic_model_name_or_path $CRITIC_MODEL_PATH \ - --actor_zero_stage $ACTOR_ZERO_STAGE --critic_zero_stage $CRITIC_ZERO_STAGE \ - --num_padding_at_beginning 1 --gradient_accumulation_steps 2 \ - --deepspeed --actor_lora_dim 128 --enable_hybrid_engine --actor_gradient_checkpointing --disable_actor_dropout \ - --output_dir $OUTPUT &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b.sh b/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b.sh deleted file mode 100644 index 06e70f1d4..000000000 --- a/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/training_scripts/single_node/run_1.3b.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -ACTOR_MODEL_PATH=$1 -CRITIC_MODEL_PATH=$2 -ACTOR_ZERO_STAGE=$3 -CRITIC_ZERO_STAGE=$4 -OUTPUT=$5 -if [ "$OUTPUT" == "" ]; then - OUTPUT=./output -fi -if [ "$ACTOR_ZERO_STAGE" == "" ]; then - ACTOR_ZERO_STAGE=2 -fi -if [ "$CRITIC_ZERO_STAGE" == "" ]; then - CRITIC_ZERO_STAGE=2 -fi -mkdir -p $OUTPUT - -Num_Padding_at_Beginning=1 # this is model related - -Actor_Lr=9.65e-6 -Critic_Lr=5e-6 - -deepspeed --master_port 12346 main.py \ - --data_path Dahoas/rm-static \ - --data_split 2,4,4 \ - --actor_model_name_or_path $ACTOR_MODEL_PATH \ - --critic_model_name_or_path $CRITIC_MODEL_PATH \ - --num_padding_at_beginning 1 \ - --per_device_train_batch_size 4 \ - --per_device_mini_train_batch_size 4 \ - --generation_batch_numbers 1 \ - --ppo_epochs 1 \ - --max_answer_seq_len 256 \ - --max_prompt_seq_len 256 \ - --actor_learning_rate ${Actor_Lr} \ - --critic_learning_rate ${Critic_Lr} \ - --num_train_epochs 1 \ - --lr_scheduler_type cosine \ - --gradient_accumulation_steps 1 \ - --disable_actor_dropout \ - --num_warmup_steps 100 \ - --deepspeed --seed 1234 \ - --enable_hybrid_engine \ - --actor_zero_stage $ACTOR_ZERO_STAGE \ - --critic_zero_stage $CRITIC_ZERO_STAGE \ - --enable_ema \ - --output_dir $OUTPUT \ - &> $OUTPUT/training.log diff --git a/applications/DeepSpeed-Chat/training/utils/ds_utils.py b/applications/DeepSpeed-Chat/training/utils/ds_utils.py deleted file mode 100644 index 0809f2787..000000000 --- a/applications/DeepSpeed-Chat/training/utils/ds_utils.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -GLOBAL_BATCH_SIZE = 32 -MICRO_BATCH_SIZE = 4 - - -def get_train_ds_config(offload, - stage=2, - enable_hybrid_engine=False, - inference_tp_size=1, - release_inference_cache=False, - pin_parameters=True, - tp_gather_partition_size=8, - max_out_tokens=512): - - device = "cpu" if offload else "none" - zero_opt_dict = { - "stage": stage, - "offload_param": { - "device": device - }, - "offload_optimizer": { - "device": device - }, - "stage3_param_persistence_threshold": 1e4, - "stage3_max_live_parameters": 3e7, - "stage3_prefetch_bucket_size": 3e7, - "memory_efficient_linear": False - } - return { - "train_batch_size": GLOBAL_BATCH_SIZE, - "train_micro_batch_size_per_gpu": MICRO_BATCH_SIZE, - "steps_per_print": 10, - "zero_optimization": zero_opt_dict, - "fp16": { - "enabled": True, - "loss_scale_window": 100 - }, - "gradient_clipping": 1.0, - "prescale_gradients": False, - "wall_clock_breakdown": False, - "hybrid_engine": { - "enabled": enable_hybrid_engine, - "max_out_tokens": max_out_tokens, - "inference_tp_size": inference_tp_size, - "release_inference_cache": release_inference_cache, - "pin_parameters": pin_parameters, - "tp_gather_partition_size": tp_gather_partition_size, - } - } - - -def get_eval_ds_config(offload, stage=0): - device = "cpu" if offload else "none" - zero_opt_dict = { - "stage": stage, - "stage3_param_persistence_threshold": 1e4, - "offload_param": { - "device": device - }, - "memory_efficient_linear": False - } - return { - "train_batch_size": GLOBAL_BATCH_SIZE, - "train_micro_batch_size_per_gpu": MICRO_BATCH_SIZE, - "steps_per_print": 10, - "zero_optimization": zero_opt_dict, - "fp16": { - "enabled": True - }, - "gradient_clipping": 1.0, - "prescale_gradients": False, - "wall_clock_breakdown": False - } diff --git a/applications/DeepSpeed-Chat/training/utils/model/model_utils.py b/applications/DeepSpeed-Chat/training/utils/model/model_utils.py deleted file mode 100644 index b5f4d89a5..000000000 --- a/applications/DeepSpeed-Chat/training/utils/model/model_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -import os -import math -import torch -from transformers import ( - AutoConfig, - AutoModel, -) -from huggingface_hub import snapshot_download -from transformers.deepspeed import HfDeepSpeedConfig - -from .reward_model import RewardModel - - -def create_hf_model(model_class, - model_name_or_path, - tokenizer, - ds_config=None, - rlhf_training=False, - disable_dropout=False): - model_config = AutoConfig.from_pretrained(model_name_or_path) - if disable_dropout: - model_config.dropout = 0.0 - # Note: dschf is defined in function scope to avoid global effects - # https://huggingface.co/docs/transformers/main_classes/deepspeed#nontrainer-deepspeed-integration - if ds_config is not None and ds_config["zero_optimization"]["stage"] == 3: - dschf = HfDeepSpeedConfig(ds_config) - else: - dschf = None - if rlhf_training: - # the weight loading is handled by create critic model - model = model_class.from_config(model_config) - else: - model = model_class.from_pretrained( - model_name_or_path, - from_tf=bool(".ckpt" in model_name_or_path), - config=model_config) - - model.config.end_token_id = tokenizer.eos_token_id - model.config.pad_token_id = model.config.eos_token_id - model.resize_token_embeddings(int( - 8 * - math.ceil(len(tokenizer) / 8.0))) # make the vocab size multiple of 8 - - return model - - -def create_critic_model(model_name_or_path, - tokenizer, - ds_config, - num_padding_at_beginning=0, - rlhf_training=False, - disable_dropout=False): - # OPT model family always put a padding token at the beginning of the sequence, - # we did not see this in other models but not sure if it is a general rule - critic_model = create_hf_model(AutoModel, model_name_or_path, tokenizer, - ds_config, rlhf_training, disable_dropout) - critic_model = RewardModel( - critic_model, - tokenizer, - num_padding_at_beginning=num_padding_at_beginning) - - if rlhf_training: - if not os.path.isdir(model_name_or_path): - model_name_or_path = snapshot_download(model_name_or_path) - # critic model needs to load the weight here - model_ckpt_path = os.path.join(model_name_or_path, 'pytorch_model.bin') - assert os.path.exists( - model_ckpt_path - ), f"Cannot find model checkpoint at {model_ckpt_path}" - critic_model.load_state_dict( - torch.load(model_ckpt_path, map_location='cpu')) - - return critic_model diff --git a/applications/DeepSpeed-Chat/training/utils/module/lora.py b/applications/DeepSpeed-Chat/training/utils/module/lora.py deleted file mode 100644 index ca0918212..000000000 --- a/applications/DeepSpeed-Chat/training/utils/module/lora.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -import math -import torch -from torch import nn -import torch.nn.functional as F -from deepspeed.compression.helper import recursive_getattr, recursive_setattr -import deepspeed - - -class LinearLayer_LoRA(nn.Module): - # an simple implementation of LoRA - # for now only support Linear Layer - def __init__(self, - weight, - lora_dim=0, - lora_scaling=1, - lora_droppout=0, - bias=None): - super(LinearLayer_LoRA, self).__init__() - self.weight = weight - self.bias = bias - - if lora_dim <= 0: - raise ValueError( - "You are training to use LoRA, whose reduced dim should be larger than 1" - ) - - try: - # for zero stage 3 - rows, columns = weight.ds_shape - except: - rows, columns = weight.shape - self.lora_right_weight = nn.Parameter(torch.zeros( - columns, - lora_dim)) # apply transpose so in forward we do not need to - self.lora_left_weight = nn.Parameter(torch.zeros(lora_dim, rows)) - self.lora_scaling = lora_scaling / lora_dim - - if lora_droppout > 0: - self.lora_dropout = nn.Dropout(lora_droppout) - else: - self.lora_dropout = nn.Identity() - - self.reset_parameters() - # disable the original weight gradient - self.weight.requires_grad = False - # fuse LoRA to the original weight - self.fuse_lora = False - - def eval(self): - self.lora_dropout.eval() - - # self.fuse_lora_weight() - - def train(self, mode=True): - self.lora_dropout.train(mode) - # self.unfuse_lora_weight() - - def reset_parameters(self): - nn.init.kaiming_uniform_(self.lora_right_weight, a=math.sqrt(5)) - nn.init.zeros_(self.lora_left_weight) - - def fuse_lora_weight(self): - if not self.fuse_lora: - self.weight.data += self.lora_scaling * torch.matmul( - self.lora_left_weight.t(), self.lora_right_weight.t()) - self.fuse_lora = True - - def unfuse_lora_weight(self): - if self.fuse_lora: - self.weight.data -= self.lora_scaling * torch.matmul( - self.lora_left_weight.t(), self.lora_right_weight.t()) - self.fuse_lora = False - - def forward(self, input): - if self.fuse_lora: - return F.linear(input, self.weight, self.bias) - else: - return F.linear( - input, self.weight, - self.bias) + (self.lora_dropout(input) @ self.lora_right_weight - @ self.lora_left_weight) * self.lora_scaling - - -# convert the linear layer to LoRA -def convert_linear_layer_to_lora(model, - part_module_name, - lora_dim=0, - lora_scaling=1, - lora_droppout=0): - repalce_name = [] - for name, module in model.named_modules(): - if isinstance(module, nn.Linear) and part_module_name in name: - repalce_name.append(name) - for name in repalce_name: - module = recursive_getattr(model, name) - tmp = LinearLayer_LoRA( - module.weight, lora_dim, lora_scaling, lora_droppout, - module.bias).to(module.weight.device).to(module.weight.dtype) - recursive_setattr(model, name, tmp) - return model - - -def _z3_params_to_fetch(param_list): - return [ - p for p in param_list - if hasattr(p, 'ds_id') and p.ds_status == deepspeed.runtime.zero. - partition_parameters.ZeroParamStatus.NOT_AVAILABLE - ] - - -# convert the LoRA layer to linear layer -def convert_lora_to_linear_layer(model): - repalce_name = [] - for name, module in model.named_modules(): - if isinstance(module, LinearLayer_LoRA): - repalce_name.append(name) - for name in repalce_name: - module = recursive_getattr(model, name) - zero_stage_3 = hasattr(module.weight, 'ds_id') - with deepspeed.zero.GatheredParameters(_z3_params_to_fetch([ - module.weight, module.bias, module.lora_left_weight, - module.lora_right_weight - ]), - modifier_rank=0, - enabled=zero_stage_3): - module.fuse_lora_weight() - return model - - -def only_optimize_lora_parameters(model): - # turn off the gradient of all the parameters except the LoRA parameters - for name, param in model.named_parameters(): - if "lora_right_weight" in name or "lora_left_weight" in name: - param.requires_grad = True - else: - param.requires_grad = False - return model diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py deleted file mode 100644 index 782315029..000000000 --- a/applications/DeepSpeed-Chat/training/utils/utils.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - -# DeepSpeed Team -import os -import torch -import random -import numpy as np -from transformers import set_seed, AutoTokenizer -import json -import deepspeed -from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus - - -def print_rank_0(msg, rank=0): - if rank <= 0: - print(msg) - - -def to_device(batch, device): - output = {} - for k, v in batch.items(): - try: - output[k] = v.to(device) - except: - output[k] = v - return output - - -class MovingAverage: - - def __init__(self): - self.count = 0 - self.total = 0 - self.mean = 0 - - def update(self, num): - self.total += num - self.count += 1 - self.mean = self.total / self.count - - return self.mean - - -def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True): - if os.path.exists(model_name_or_path): - # Locally tokenizer loading has some issue, so we need to force download - model_json = os.path.join(model_name_or_path, "config.json") - if os.path.exists(model_json): - model_json_file = json.load(open(model_json)) - model_name = model_json_file["_name_or_path"] - tokenizer = AutoTokenizer.from_pretrained(model_name, - fast_tokenizer=True) - else: - tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, - fast_tokenizer=True) - return tokenizer - - -def save_hf_format(model, tokenizer, args, sub_folder=""): - # used to save huggingface format, so we can use it for hf.from_pretrained - model_to_save = model.module if hasattr(model, 'module') else model - CONFIG_NAME = "config.json" - WEIGHTS_NAME = "pytorch_model.bin" - output_dir = os.path.join(args.output_dir, sub_folder) - os.makedirs(output_dir, exist_ok=True) - output_model_file = os.path.join(output_dir, WEIGHTS_NAME) - output_config_file = os.path.join(output_dir, CONFIG_NAME) - save_dict = model_to_save.state_dict() - for key in list(save_dict.keys()): - if "lora" in key: - del save_dict[key] - torch.save(save_dict, output_model_file) - model_to_save.config.to_json_file(output_config_file) - tokenizer.save_vocabulary(output_dir) - - -def set_random_seed(seed): - if seed is not None: - set_seed(seed) - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - - -def get_all_reduce_mean(tensor): - torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.SUM) - tensor = tensor / torch.distributed.get_world_size() - return tensor - - -def get_optimizer_grouped_parameters(model, - weight_decay, - no_decay_name_list=[ - "bias", "LayerNorm.weight" - ]): - optimizer_grouped_parameters = [ - { - "params": [ - p for n, p in model.named_parameters() - if (not any(nd in n - for nd in no_decay_name_list) and p.requires_grad) - ], - "weight_decay": - weight_decay, - }, - { - "params": [ - p for n, p in model.named_parameters() - if (any(nd in n - for nd in no_decay_name_list) and p.requires_grad) - ], - "weight_decay": - 0.0, - }, - ] - return optimizer_grouped_parameters - - -def _z3_params_to_fetch(param_list): - return [ - p for p in param_list - if hasattr(p, 'ds_id') and p.ds_status == ZeroParamStatus.NOT_AVAILABLE - ] - - -def moving_average(model, model_ema, beta=0.992, device=None, zero_stage=0): - zero_stage_3 = (zero_stage == 3) - with torch.no_grad(): - for param, param_ema in zip(model.parameters(), - model_ema.parameters()): - # TODO: use prefiltering for efficiency - params_to_fetch = _z3_params_to_fetch([param, param_ema - ]) if zero_stage_3 else [] - should_gather_param = len(params_to_fetch) > 0 - with deepspeed.zero.GatheredParameters( - params_to_fetch, enabled=should_gather_param): - data = param.data - if device is not None: - data = data.to(device) - param_ema.data.copy_(torch.lerp(data, param_ema.data, beta)) - - -def save_zero_three_model(model_ema, global_rank, save_dir, zero_stage=0): - zero_stage_3 = (zero_stage == 3) - os.makedirs(save_dir, exist_ok=True) - WEIGHTS_NAME = "pytorch_model.bin" - output_model_file = os.path.join(save_dir, WEIGHTS_NAME) - - model_to_save = model_ema.module if hasattr(model_ema, - 'module') else model_ema - if not zero_stage_3: - if global_rank == 0: - torch.save(model_to_save.state_dict(), output_model_file) - else: - output_state_dict = {} - for k, v in model_to_save.named_parameters(): - - if hasattr(v, 'ds_id'): - with deepspeed.zero.GatheredParameters(_z3_params_to_fetch([v - ]), - enabled=zero_stage_3): - v_p = v.data.cpu() - else: - v_p = v.cpu() - if global_rank == 0 and "lora" not in k: - output_state_dict[k] = v_p - if global_rank == 0: - torch.save(output_state_dict, output_model_file) - del output_state_dict diff --git a/applications/DeepSpeed-VisualChat/README.md b/applications/DeepSpeed-VisualChat/README.md new file mode 100755 index 000000000..db901762a --- /dev/null +++ b/applications/DeepSpeed-VisualChat/README.md @@ -0,0 +1,116 @@ +# DeepSpeed-VisualChat + +An easy-to-use, scalable, and efficient multi-modal training pipeline for multi-round multi-image interleave chat experience. + + +## Table of Contents + +- [📰 Latest News 📰](#-latest-news-) +- [🚀 What is DeepSpeed-VisualChat 🚀️](#-what-is-deepspeed-visualchat-) +- [⚓ Get Started, Tutorial, and Documentation ⚓](#-get-started-tutorial-documentation-) +- [🌱 DeepSpeed-VisualChat's Roadmap 🌱](#-deepspeed-visualchats-roadmap-) +- [💬 DeepSpeed-VisualChat and DeepSpeed Community 💬](#-deepspeed-visualchat-and-deepspeed-community-) +- [🙏 Acknowledgement and Citation 🙏](#-acknowledgement-and-citation-) + + + +## 📰 Latest News 📰 + +* ***[2023/10] [DeepSpeed-VisualChat: Improve Your Chat Experience with Multi-Round Multi-Image Inputs](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/deepspeed-visualchat/10-03-2023/README.md)*** + +⭐ If you find our [DeepSpeed](https://github.com/deepspeedai/DeepSpeed) and [DeepSpeedExamples](https://github.com/deepspeedai/DeepSpeedExamples) repositories beneficial, please give them a star on GitHub! To cite DeepSpeed-VisualChat, please cite our [arxiv report](https://arxiv.org/abs/2309.14327): + +``` +@article{yao2023deepspeed-visualchat, + title={{DeepSpeed-VisualChat: Multi-Round Multi-Image Interleave Chat via Multi-Modal Causal Attention}}, + author={Zhewei Yao and Xiaoxia Wu and Conglong Li and Minjia Zhang and Heyang Qin and Olatunji Ruwase and Ammar Ahmad Awan and Samyam Rajbhandari and Yuxiong He}, + journal={arXiv preprint arXiv:2309.14327}, + year={2023} +} +``` + +## 🚀 What is DeepSpeed-VisualChat 🚀 +
+
+
+Figure 1. On the left is a DeepSpeed-VisualChat model, featuring an innovative attention design. On the right is an example of DeepSpeed-VisualChat.
+
+
+
+---
+
+With increasing interest in enabling the multi-modal capabilities of large language models, DeepSpeed is proud to announce a new training pipeline, named ***DeepSpeed-VisualChat***. This is designed for enabling a multi-round, multi-image interleave chat framework. It enhances the language model with image understanding and reasoning capabilities. Unlike the majority of open-sourced multi-modal projects, the primary focus of DeepSpeed-VisualChat is to provide a multi-round, multi-image interleave chat experience, as illustrated in Figure 1.
+
+To improve model quality without introducing new parameters, DeepSpeed-VisualChat incorporates a new multi-modal causal attention mechanism, which is adept at better aligning visual and text features. Additionally, to overcome the scarcity of interleaved text-and-image inputs in most available open-sourced datasets, we employ various data blending techniques on existing datasets.
+
+Thanks to the scalable, efficient, and user-friendly nature of the DeepSpeed ecosystem, we have the capability to train using a 2B visual encoder from QWen-VL (one is additionally refined from OpenClip) and a 70B language decoder from LLaMA-2. This showcases the extraordinary scalability of the DeepSpeed-VisualChat framework.
+
+
+
+
+
+## ⚓ Get Started, Tutorial, and Documents ⚓
+
+### 🐼 Installation
+
+
+```bash
+git clone https://github.com/deepspeedai/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-VisualChat/
+pip install -r requirements.txt
+```
+
+### 🐼 Datasets Preparation
+
+Table below summarizes where to download the datasets that we support. `{data_path}` denotes the `--data_path` argument provided in training scripts.
+
+| Dataset name | Where to download |
+|--------------|-------------------|
+| aokvqa | Download `2017 Train images [118K/18GB]` from [https://cocodataset.org/#download](https://cocodataset.org/#download) and save at `{data_path}/coco/train2017/`. Download `aokvqa_v1p0_train.json` from [https://allenai.org/project/a-okvqa/home](https://allenai.org/project/a-okvqa/home) and save at `{data_path}/aokvqa/annotations/`. |
+| coco_caption | Download 2014 Train images and 2014 Val images from [https://cocodataset.org/#download](https://cocodataset.org/#download) and save all images at `{data_path}/coco/2014/`. Download `dataset.json` from [https://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip](https://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip) and save at `{data_path}/coco_caption/`. |
+| llava | Download `2017 Train images [118K/18GB]` from [https://cocodataset.org/#download](https://cocodataset.org/#download) and save at `{data_path}/coco/train2017/`. Download `detail_23k.json` and `complex_reasoning_77k.json` from [https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) and save at `{data_path}/llava/`. |
+| llava_dial | Download `2017 Train images [118K/18GB]` from [https://cocodataset.org/#download](https://cocodataset.org/#download) and save at `{data_path}/coco/train2017/`. Download `conversation_58k.json` from [https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K) and save at `{data_path}/llava/`. |
+| llava_otter_blend | Follow instructions of the llava, llava_dial, and otter_mimicit_cgd datasets. |
+| minigpt4 | Download `image` folder and `filter_cap.json` from [https://huggingface.co/datasets/Vision-CAIR/cc_sbu_align](https://huggingface.co/datasets/Vision-CAIR/cc_sbu_align) and save at `{data_path}/cc_sbu_align/`. |
+| ocr_vqa | Download `images` folder and `dataset.json` from [https://ocr-vqa.github.io/](https://ocr-vqa.github.io/) and save at `{data_path}/OCR_VQA/`. |
+| otter_mimicit_cgd | Download `2017 Train images [118K/18GB]` from [https://cocodataset.org/#download](https://cocodataset.org/#download) and save at `{data_path}/coco/train2017/`. Download `CGD_instructions.json` from [https://huggingface.co/datasets/pufanyi/MIMICIT](https://huggingface.co/datasets/pufanyi/MIMICIT) and save at `{data_path}/MIMIC-IT/`. |
+| otter_mimicit_sd | Download `SD.json` and `SD_instructions.json` from [https://huggingface.co/datasets/pufanyi/MIMICIT](https://huggingface.co/datasets/pufanyi/MIMICIT) and save at `{data_path}/MIMIC-IT/`. |
+| otter_mimicit_sn | Download `SN.json` and `SN_instructions.json` from [https://huggingface.co/datasets/pufanyi/MIMICIT](https://huggingface.co/datasets/pufanyi/MIMICIT) and save at `{data_path}/MIMIC-IT/`. |
+| otter_mimicit_tvc | Download `TVC.json` and `TVC_instructions.json` from [https://huggingface.co/datasets/pufanyi/MIMICIT](https://huggingface.co/datasets/pufanyi/MIMICIT) and save at `{data_path}/MIMIC-IT/`. |
+| otter_mimicit_vst | Download `VST.json` and `VST_instructions.json` from [https://huggingface.co/datasets/pufanyi/MIMICIT](https://huggingface.co/datasets/pufanyi/MIMICIT) and save at `{data_path}/MIMIC-IT/`. |
+| sparkles_dialogue | Download the `SparklesDialogueCC` and `SparklesDialogueVG` folders from the OneDrive link from [https://github.com/HYPJUDY/Sparkles](https://github.com/HYPJUDY/Sparkles) and save at `{data_path}/`. |
+
+### 🐼 Training, Evaluation, Chat API, and Helper
+Please refer to
+ - [**Training**](./training/README.md)
+ - [**Evaluation**](./eval/README.md)
+ - [**Chat**](./chat/README.md)
+ - [**Helper**](./helper/README.md)
+
+
+## 🌱 DeepSpeed-VisualChat's Roadmap 🌱
+
+Our future plan includes but not limited to :
+- [ ] Support more models
+- [ ] Demonstrate how to training larger models with higher model quality
+
+## 💬 DeepSpeed-VisualChat and DeepSpeed Community 💬
+
+Just like how the success of [the BLOOM model](https://huggingface.co/bigscience/bloom) was supported by both [DeepSpeed Team](https://github.com/bigscience-workshop/Megatron-DeepSpeed) and many [open source contributors](https://huggingface.co/bigscience), we welcome all AI developers/practitioners/researchers to join this on-going effort for DeepSpeed-Chat. To participate:
+- Show your support by leaving a star ⭐ to our [DeepSpeed](https://github.com/deepspeedai/DeepSpeed) and [DeepSpeedExamples](https://github.com/deepspeedai/DeepSpeedExamples) GitHub repositories.
+- Follow us on [twitter](https://twitter.com/MSFTDeepSpeed) to get notified about our latest news. For Chinese users, you can also follow our [Chinese Zhihu account](https://www.zhihu.com/people/deepspeed). For Japanese users, you can also follow our [Japanese twitter account](https://twitter.com/MSFTDeepSpeedJP).
+- Currently we prefer to interact with open source users mainly on GitHub so that it's easier for all users to search for related information. For bug reports, please submit a GitHub issue. For contribution, please submit a pull request (PR). For general question/discussion, please open a new discussion or join any existing discussions.
+- We are open to collaborations with universities, research labs, and companies, such as working together on deep learning research, applying DeepSpeed to empower real-world AI models and applications, and so on. For such requests (and other requests unsuitable for GitHub), please directly email to deepspeed-info@microsoft.com.
+
+
+## 🙏 Acknowledgement and Citation 🙏
+
+We thank the following papers and open-source repositories:
+
+ [1] LLaVa, https://github.com/haotian-liu/LLaVA
+ [2] Otter, https://github.com/Luodian/Otter
+ [3] Transformers Hugging Face, https://github.com/huggingface/transformers
+ [4] MiniGPT4, https://github.com/Vision-CAIR/MiniGPT-4
+ [5] QWen-VL, https://github.com/QwenLM/Qwen-VL
+ [6] Sparkles, https://github.com/HYPJUDY/Sparkles
+ [7] Multimodal-GPT, https://github.com/open-mmlab/Multimodal-GPT
diff --git a/applications/DeepSpeed-VisualChat/assets/banner.png b/applications/DeepSpeed-VisualChat/assets/banner.png
new file mode 100644
index 000000000..2684cafc2
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/assets/banner.png differ
diff --git a/applications/DeepSpeed-VisualChat/assets/ceos.png b/applications/DeepSpeed-VisualChat/assets/ceos.png
new file mode 100644
index 000000000..e148f545a
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/assets/ceos.png differ
diff --git a/applications/DeepSpeed-VisualChat/assets/friends.png b/applications/DeepSpeed-VisualChat/assets/friends.png
new file mode 100644
index 000000000..2689d8d4b
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/assets/friends.png differ
diff --git a/applications/DeepSpeed-VisualChat/assets/hero-figure.png b/applications/DeepSpeed-VisualChat/assets/hero-figure.png
new file mode 100644
index 000000000..ca79b2c62
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/assets/hero-figure.png differ
diff --git a/applications/DeepSpeed-VisualChat/assets/model.png b/applications/DeepSpeed-VisualChat/assets/model.png
new file mode 100644
index 000000000..dbd1f05c4
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/assets/model.png differ
diff --git a/applications/DeepSpeed-VisualChat/chat/README.md b/applications/DeepSpeed-VisualChat/chat/README.md
new file mode 100644
index 000000000..155061f1a
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/chat/README.md
@@ -0,0 +1 @@
+We provide a CLI interface for uses to test their trained chat model. First of all, please note that you need to provide both the trained checkpoint and the original language model & vision encoder paths. The model is first initialized and then loads the trained checkpoint. Also, please note that if you used multi-modal causal attention during your training, remember to put --enable_mmca_attention for in your chat script.
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/chat/chat.py b/applications/DeepSpeed-VisualChat/chat/chat.py
new file mode 100644
index 000000000..57634e957
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/chat/chat.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+import argparse
+import os
+import sys
+from PIL import Image
+
+import torch
+import deepspeed
+
+sys.path.append(
+ os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
+from utils.utils import print_rank_0
+from utils.model import create_dsvl_model_and_transforms
+import utils.data.DST as DST
+from transformers import AutoTokenizer
+from termcolor import colored
+import re
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description=
+ "CLI chat")
+ parser.add_argument(
+ "--lm_model_name_or_path",
+ type=str,
+ help=
+ "Path to pretrained model or model identifier from huggingface.co/models.",
+ required=True,
+ )
+ parser.add_argument("--vision_model_name_or_path", default="openai/clip-vit-large-patch14", type=str)
+ parser.add_argument(
+ "--checkpoint_path",
+ default=None,
+ type=str,
+ help="path to pretrained model",
+ )
+ parser.add_argument(
+ "--max_seq_len",
+ type=int,
+ default=4096,
+ help="The maximum sequence length.",
+ )
+ parser.add_argument(
+ "--generation_length_per_round",
+ type=int,
+ default=256,
+ help="The generation length per conversation round.",
+ )
+ parser.add_argument(
+ "--enable_mmca_attention",
+ action='http://github.com/store_true',
+ help="enable the new proposed attn, which is similar to cross attention",
+ )
+ parser.add_argument(
+ "--vis_proj",
+ type=str,
+ default='baseline',
+ help="baseline, vit, or perceiver",
+ )
+ parser = deepspeed.add_config_arguments(parser)
+ args = parser.parse_args()
+
+ return args
+
+
+def get_user_text_input():
+ tmp = input(colored("Enter input (type 'quit' to exit, 'clear' to clean memory): ", 'green'))
+ return tmp, tmp == "quit", tmp == "clear"
+
+def get_user_image_input():
+ tmp = input(colored("Enter image pathes, seperate by space (only support one image per time for now) (type 'na' for empty image): ", 'blue'))
+ return tmp, not tmp == "na"
+
+def main():
+ args = parse_args()
+ tokenizer = AutoTokenizer.from_pretrained(args.lm_model_name_or_path,
+ fast_tokenizer=True)
+ tokenizer.padding_side = 'right'
+ model, image_processor, tokenizer = create_dsvl_model_and_transforms(
+ text_tokenizer = tokenizer,
+ ds_config=None,
+ args=args,
+ )
+
+ model.load_state_dict(torch.load(os.path.join(args.checkpoint_path, 'pytorch_model.bin'), map_location='cpu'), strict=False) # Z3 wouldn't save pos embeddings (vis and rope)
+
+ model = model.eval()
+ model.projection = model.projection.to('cuda')
+ model.vis_encoder = model.vis_encoder.to('cuda')
+ model = model.half()
+ print_rank_0(model)
+
+ num_rounds = 0
+ images = []
+ system_instruct = []
+ TEMPLATE = DST.Prompter() # get template
+ image_num_token_list = [DST.IMAGE_NUM_1, DST.IMAGE_NUM_2, DST.IMAGE_NUM_3, DST.IMAGE_NUM_4, DST.IMAGE_NUM_5, DST.IMAGE_NUM_6, DST.IMAGE_NUM_7, DST.IMAGE_NUM_8]
+
+ while True:
+ num_rounds += 1
+ while True:
+ # it is super easy to make mistake here, so we need to be careful
+ image_input, with_image = get_user_image_input()
+ if with_image:
+ try:
+ # seperate by space
+ image_paths = image_input.split(' ')
+ tmp_images = []
+ for image_path in image_paths:
+ image = Image.open(image_path).convert('RGB')
+ tmp_image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].unsqueeze(0).cuda().half()
+ tmp_images.append(tmp_image_tensor) # in case the last image path is wrong
+ except:
+ print(colored("Invalid image path, please try again", 'red'))
+ continue
+ if len(images) + len(tmp_images) > 8:
+ print(colored("Too many images, we at most support 8 images. please try again", 'red'))
+ continue
+ images = images + tmp_images # get all images
+ image_num = len(tmp_images)
+ break
+ else:
+ image_num = 0
+ break
+ assert len(images) >= 1, "We need at least one image to begin the conversation for now."
+ if len(images) > 0:
+ image_tensor = torch.cat(images, dim=0) # cat all images
+ else:
+ image_tensor = None
+
+ text_input, quit, clear = get_user_text_input()
+ if quit:
+ break
+ if clear:
+ num_rounds = 0
+ images = []
+ system_instruct = []
+ image_num_token_list = [DST.IMAGE_NUM_1, DST.IMAGE_NUM_2, DST.IMAGE_NUM_3, DST.IMAGE_NUM_4, DST.IMAGE_NUM_5, DST.IMAGE_NUM_6, DST.IMAGE_NUM_7, DST.IMAGE_NUM_8]
+ continue
+
+
+ full_prompt = TEMPLATE(text_input, with_image=with_image, first_message=(num_rounds==1), num_images=image_num)
+ if with_image:
+ for i in range(image_num):
+ full_prompt = re.sub(DST.DEFAULT_HUMAN_IMAGE_PRETOKEN, image_num_token_list.pop(0), full_prompt, count=1)
+
+
+ full_prompt_ids = tokenizer(full_prompt).input_ids # remove bos token
+
+ input_ids = torch.as_tensor([system_instruct + full_prompt_ids]).cuda() # entire input as system instruction for simplicity
+ generate_output = model.generate(image_tensor, input_ids, generation_length=args.generation_length_per_round)
+ extend_ids = generate_output[0].cpu().tolist()[0]
+ while extend_ids[-1] == tokenizer.pad_token_id:
+ extend_ids.pop()
+ while extend_ids[0] == tokenizer.bos_token_id:
+ extend_ids.pop(0)
+ system_instruct = system_instruct + full_prompt_ids + extend_ids # entire input as system instruction for simplicity
+ system_instruct = system_instruct + [tokenizer.eos_token_id] # add eos token
+
+ print(f"=========== Round {num_rounds} ===========")
+ print(tokenizer.decode(system_instruct))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/DeepSpeed-VisualChat/chat/chat_scripts/run.sh b/applications/DeepSpeed-VisualChat/chat/chat_scripts/run.sh
new file mode 100644
index 000000000..8c193d520
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/chat/chat_scripts/run.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+MAIN_PATH=$1
+
+VISION_ENCODER=/blob/transformers_cache/qwen-clip
+LLM=/blob/transformers_cache/Llama-2-13b-hf
+
+export CUDA_VISIBLE_DEVICES=0 # Do multi single evaluation
+# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 # Do multi gpu evaluation for large models (single GPU is not enough)
+
+
+python chat.py \
+ --lm_model_name_or_path $LLM \
+ --vision_model_name_or_path $VISION_ENCODER \
+ --checkpoint_path $MAIN_PATH --enable_mmca_attention
diff --git a/applications/DeepSpeed-VisualChat/eval/README.md b/applications/DeepSpeed-VisualChat/eval/README.md
new file mode 100644
index 000000000..e39bbf035
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/README.md
@@ -0,0 +1,28 @@
+### ☀️Evaluation
+We provide a few examples to test the quality of the models.
+To run the tests, use the `batch_generation.py` script, which will call the JSON file located in `eval_data/*.json`.
+You will need to specify the model path where you've saved your checkpoints. For example, if you've saved your model checkpoint at $YOUR_CHECKPOINT_PATH/epoch-5/pytorch_model.bin, then pass the following arguments:
+```
+--checkpoint_path $YOUR_CHECKPOINT_PATH --checkpoint_names epoch-5
+```
+
+##### 🏃 Run the Code
+NOTE: Before you run the code `run_batch.sh`, please read it carefully. This bash script creates a folder eval/results/eval_comprehensive if you use the json evaluation "eval_comprehensive". It will write to "eval/results/eval_comprehensive/{args.output_filename}.csv" file with four columns. The generation output is in the last column. Please read one of our examples such as `eval/results/eval_comprehensive/ours-set1_final.csv`.
+To run the code, you need to go to outside the current folder
+```
+cd DeeSpeedExamples/applications/DeepSpeed-VisualChat
+bash eval/run_batch.sh
+```
+
+
+#### 🐕 Our Model Results Overview
+We present the outcomes from our three distinct models, each trained with vision encoders: `qwen-clip` and `Llama-2-13b-hf`.
+
+###### Results Directories and Training Details:
+- **results/eval_single:**
+ This directory contains results from the model trained with LoRA, featuring a dimension size of 128.
+
+- **results/eval_comprehensive** and **results/eval_robustness:**
+ These directories host results from two models:
+ - One model is trained excluding the Sparkles dataset (referred to as `ours-set1`).
+ - The other incorporates Sparkles dataset in the training (denoted as `ours-set2`).
diff --git a/applications/DeepSpeed-VisualChat/eval/batch_generation.py b/applications/DeepSpeed-VisualChat/eval/batch_generation.py
new file mode 100644
index 000000000..c22c2f6a9
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/batch_generation.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+import argparse
+import os
+import csv
+import sys
+from PIL import Image
+
+import torch
+import deepspeed
+sys.path.append(
+ os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
+from utils.utils import print_rank_0, to_device
+from utils.model import create_dsvl_model_and_transforms as create_model_and_transforms
+import utils.data.DST as DST
+
+
+
+
+from typing import Iterable
+from transformers import AutoTokenizer, set_seed
+import json
+import collections
+import numpy as np
+import random
+def load_json(file_path):
+ with open(file_path, 'r') as json_file:
+ data = json.load(json_file)
+ return data
+
+# usage
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description=
+ "Finetune a transformers model on a causal language modeling task")
+ parser.add_argument('--question-format',
+ type=str,
+ default="prompt_qa",
+ choices=['prompt_qa', 'prompt_choice'],
+ help='question-format')
+ parser.add_argument('--question',
+ type=str,
+ default="please describe the image",
+ help='question-format')
+ parser.add_argument(
+ "--lm_model_name_or_path",
+ type=str,
+ help=
+ "Path to pretrained model or model identifier from huggingface.co/models.",
+ required=True,
+ )
+ parser.add_argument("--vision_model_name_or_path", default="openai/clip-vit-large-patch14", type=str)
+ parser.add_argument(
+ "--pretrained_path",
+ default=None,
+ type=str,
+ help="path to pretrained model",
+ )
+ parser.add_argument(
+ "--image_token_length",
+ type=int,
+ default=256,
+ help="The maximum sequence length.",
+ )
+ parser.add_argument(
+ "--max_seq_len",
+ type=int,
+ default=2048,
+ help="The maximum sequence length.",
+ )
+ parser.add_argument(
+ "--checkpoint_path",
+ default=None,
+ type=str,
+ help="path to pretrained model",
+ )
+ parser.add_argument('--checkpoint_names',
+ nargs='*',
+ default=['runing_check_stage2_v3_epoch10',],
+ help='Path to the training dataset. Accepted format:'
+ '1) a single data path, 2) multiple datasets in the'
+ 'form: dataset1-path dataset2-path ...')
+ parser.add_argument(
+ "--model_name",
+ default="dsvl",
+ type=str,
+ choices=["dsvl", "toy"],
+ help="path to pretrained model",
+ )
+ parser.add_argument(
+ "--enable_mmca_attention",
+ action='http://github.com/store_true',
+ help="enable the new proposed attn, which is similar to cross attention",
+ )
+ parser.add_argument(
+ "--vis_proj",
+ type=str,
+ default='baseline',
+ help="baseline, vit, or perceiver",
+ )
+ parser.add_argument(
+ "--eval_data",
+ default="dsvl",
+ type=str,
+ help="path to eval data",
+ )
+ parser.add_argument(
+ "--output_filename",
+ default="results",
+ type=str,
+ help="path to eval data",
+ )
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=123,
+ help="The maximum sequence length.",
+ )
+ parser = deepspeed.add_config_arguments(parser)
+ args = parser.parse_args()
+
+ return args
+
+def main():
+ args = parse_args()
+ with open(f'./eval/eval_data/{args.eval_data}.json', 'r') as file:
+ data = json.load(file)
+ if args.seed is not None:
+ set_seed(args.seed)
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+ torch.manual_seed(args.seed)
+ torch.cuda.manual_seed_all(args.seed)
+
+ tokenizer = AutoTokenizer.from_pretrained(args.lm_model_name_or_path,
+ fast_tokenizer=True)
+ tokenizer.padding_side = 'right'
+ model, image_processor, tokenizer = create_model_and_transforms(
+ text_tokenizer = tokenizer,
+ ds_config=None,
+ args=args,
+ )
+ get_results = collections.defaultdict(list)
+ for ck_name in args.checkpoint_names:
+ ck_path = os.path.join(args.checkpoint_path, ck_name)
+ print (ck_path)
+ if ck_path is not None:
+ model.load_state_dict(torch.load(os.path.join(ck_path, 'pytorch_model.bin'), map_location='cpu'), strict=False) # Z3 wouldn't save pos embeddings (vis and rope)
+ else:
+ Warning("No checkpoint loaded so you cannot genereate meaningful results")
+ #model = model.cuda().half()
+ model = model.eval()
+ model.projection = model.projection.to('cuda')
+ model.vis_encoder = model.vis_encoder.to('cuda')
+ model = model.half()
+ print_rank_0(model)
+ for name in data.keys():
+ question_image_list = data[name]
+ print (f'{args.eval_data}-------------------------------------{name}')
+ images = []
+ system_instruct = []
+ TEMPLATE = DST.Prompter() # get template
+ image_token_dict = DST.get_image_num_map(tokenizer)
+ image_num = 0
+ for round, q_i_pair in enumerate(question_image_list):
+ # print(f'=========round {round+1}==============')
+ question = q_i_pair[0]
+ if len(q_i_pair) > 1:
+ # seperate by space
+ image_paths = q_i_pair[1].split(' ')
+ tmp_images = []
+ for image_path in image_paths:
+ image = Image.open(image_path.strip()).convert('RGB')
+ tmp_image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0].unsqueeze(0).cuda().half()
+ tmp_images.append(tmp_image_tensor)
+ images = images + tmp_images # get all images
+ with_image = True
+ image_num = len(tmp_images)
+ else:
+ image_num = 0
+ with_image = False
+
+ if len(images) > 0:
+ image_tensor = torch.cat(images, dim=0) # cat all images
+ else:
+ raise ValueError("No image provided. Did not fix this in the modeling side yet.")
+
+ full_prompt = TEMPLATE(question, with_image=with_image, first_message=(round==0), num_images=image_num)
+ full_prompt_ids = tokenizer(full_prompt).input_ids # remove bos token
+ if with_image:
+ image_number = len(images)
+ index = full_prompt_ids.index(image_token_dict[DST.DEFAULT_HUMAN_IMAGE_PRETOKEN])
+ full_prompt_ids[index] = image_token_dict[DST.image_mapping_dict[str(image_number)]]
+ full_prompt_ids = DST.flatten(full_prompt_ids)
+ input_ids = torch.as_tensor([system_instruct + full_prompt_ids]).cuda() # entire input as system instruction for simplicity
+ print ('\n',round,question, '||', q_i_pair[-1] )
+
+ generate_output = model.generate(image_tensor, input_ids,
+ generation_length=256)
+ # generation_kwargs={ 'num_beams':2,'num_return_sequences':1,'top_p':1,'do_sample':True, 'temperature':1}
+ print('vanilla-->', generate_output[1])
+ get_results[name].append([q_i_pair[-1], question, generate_output[1]])
+ extend_ids = generate_output[0].cpu().tolist()[0]
+ while extend_ids[-1] == tokenizer.pad_token_id:
+ extend_ids.pop()
+ while extend_ids[0] == tokenizer.bos_token_id:
+ # llama-2 generates bos token at the beginning
+ extend_ids.pop(0)
+ system_instruct = system_instruct + full_prompt_ids + extend_ids # entire input as system instruction for simplicity
+ system_instruct = system_instruct + [tokenizer.eos_token_id] # add eos token
+
+ with open(f'{args.output_filename}.csv', mode='w', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+ writer.writerow(['test_name', 'image_path', 'question', 'answer'])
+ for test_name, questions in get_results.items():
+ for question in questions:
+ writer.writerow([test_name] + question)
+
+
+
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/eval_comprehensive.json b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_comprehensive.json
new file mode 100644
index 000000000..a4d367b9b
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_comprehensive.json
@@ -0,0 +1,89 @@
+{
+ "cat_images1": [
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg"],
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/british_shorthair.jpg"],
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/cat.png"],
+ ["Are the colors of the three cats the same?"],
+ ["What are the differences between the first and third images?"],
+ ["What are the differences between the second and third images?"],
+ ["Is the cat in the first image in the sunshine?"]
+ ],
+ "cat_images2": [
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg"],
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/british_shorthair.jpg"],
+ ["What are the differences between the two images?"],
+ ["Please describe the image in detail.", "./eval/eval_data/images/cats/cat.png"],
+ ["Are the colors of the three cats the same?"],
+ ["What are the differences between the first and third images?"],
+ ["What are the differences between the second and third images?"],
+ ["Is the cat in the first image in the sunshine?"],
+ ["Which cat do you prefer and why?"],
+ ["I prefer the second cat. It's so cute."],
+ ["Then why do you prefer the third cat more?"]
+ ],
+ "counting_people1": [
+ ["Count the number of people in the image.", "./eval/eval_data/images/friends/can-count1.jpg"],
+ ["Count the number of people in the image.", "./eval/eval_data/images/friends/can-count2.jpg"],
+ ["What are the differences between the two images? Are they the same group of people? Explain why."],
+ ["Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?"]
+ ],
+ "counting_people2":[
+ ["How many individuals are depicted in the image?", "./eval/eval_data/images/friends/can-count1.jpg"],
+ ["How many individuals can you see in the second image?", "./eval/eval_data/images/friends/can-count2.jpg"],
+ ["Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale."],
+ ["Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?"]
+ ],
+ "counting_people3": [
+ ["Count the number of people in the image.", "./eval/eval_data/images/friends/wrong-count1.jpg"],
+ ["Count the number of people in the image.", "./eval/eval_data/images/friends/wrong-count2.jpg"],
+ ["What are the differences between the two images? Are they the same group of people? Explain why."]
+ ],
+ "counting_people4": [
+ ["How many individuals are depicted in the image?", "./eval/eval_data/images/friends/wrong-count1.jpg"],
+ ["How many individuals are depicted in the image?", "./eval/eval_data/images/friends/wrong-count2.jpg"],
+ ["Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale."],
+ ["Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?"]
+ ],
+ "zootopia_adventures1": [
+ ["Please describe the image in detail.", "./eval/eval_data/images/zootopia/z1.png"],
+ ["Please describe the image in detail.", "./eval/eval_data/images/zootopia/z2.png"],
+ ["Can you name the characters in the images? Who are they? What are they doing?", "./eval/eval_data/images/zootopia/z3.png"],
+ ["You are an imaginative storyteller. Create a fascinating story based on the first, second and third image."],
+ ["Are you familiar with these characters? What movie are they from?"],
+ ["Can you name the characters in the images? Who are they?"],
+ ["In what type of environment or setting do these characters live? Describe it."]
+ ],
+ "zootopia_adventures2": [
+ ["Create an engaging story strictly based on the images.", "./eval/eval_data/images/zootopia/z1.png ./eval/eval_data/images/zootopia/z2.png ./eval/eval_data/images/zootopia/z3.png"],
+ ["Do you recognize the setting or the characters in these images? Name the movie."],
+ ["Can you share some interesting facts or details about the characters shown in the images?"],
+ ["Which character do you find the most intriguing and why?"],
+ ["Based on the images, can you create some dialogues that the characters might say to each other in these situations?"]
+ ],
+ "zootopia_adventures3": [
+ ["Examine and describe the characters' actions in the first image.", "./eval/eval_data/images/zootopia/z1.png"],
+ ["In the second image, what are the main characters doing, and how do they seem to feel?", "./eval/eval_data/images/zootopia/z2.png"],
+ ["Contrast the characters' moods and interactions in the two provided images."],
+ ["Imagine and narrate a hilarious situation involving the characters from the images.", "./eval/eval_data/images/zootopia/z3.png"],
+ ["Name the movie from which these characters are, and give a succinct summary of its plot."],
+ ["Create a funny and unexpected scenario that could unfold between the characters in these images."]
+ ],
+ "tech_ceos1": [
+ ["Who is this person in this first image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this second image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in this third image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2": [
+ ["Who is this person in the first image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Who is this person in the second image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in the third image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. "],
+ ["Is the person in the third image the founder of Apple?"]
+ ]
+}
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/eval_robustness.json b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_robustness.json
new file mode 100644
index 000000000..16747af32
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_robustness.json
@@ -0,0 +1,78 @@
+{
+ "tech_ceos2.1a": [
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2.1b": [
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2.1c": [
+ ["Who is this person in this image a?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this image b?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in this image c?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in image b."],
+ ["Recall who is in the image a."],
+ ["Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the image c the founder of Apple?"]
+ ],
+ "tech_ceos2.1d": [
+ ["Who is this person in this first image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this second image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["Who is this person in this third image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2.1aa": [
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["What's the differnce between the first and second image"],
+ ["Who is this person in the image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2.1bb": [
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["What's the differnce between the first and second images"],
+ ["Who is this person in this image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ],
+ "tech_ceos2.1cc": [
+ ["Who is this person in this image a?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this image b?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["What's the differnce between the image a and image b"],
+ ["Who is this person in this image c?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in image b."],
+ ["Recall who is in the image a."],
+ ["Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the image c the founder of Apple?"]
+ ],
+ "tech_ceos2.1dd": [
+ ["Who is this person in this first image?", "./eval/eval_data/images/tech-ceo/jobs1.jpg"],
+ ["Who is this person in this second image?", "./eval/eval_data/images/tech-ceo/gate1.jpg"],
+ ["What's the differnce between the first and second images"],
+ ["Who is this person in this third image?", "./eval/eval_data/images/tech-ceo/musk1.jpg"],
+ ["Recall who is in the second image."],
+ ["Recall who is in the first image."],
+ ["Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple."],
+ ["Is the person in the third image the founder of Apple?"]
+ ]
+}
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/eval_single.json b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_single.json
new file mode 100644
index 000000000..42a7ad95e
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/eval_data/eval_single.json
@@ -0,0 +1,11 @@
+{
+"cat_images1": [["please describe the image", "./eval/eval_data/images/cats/cat.png"]],
+"cat_images2": [["can you describe the image", "./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg"]],
+"cat_images3": [["please describe the image", "./eval/eval_data/images/cats/british_shorthair.jpg"]],
+"extreme_ironing": [["What is unusual about this image?", "./eval/eval_data/images/singles/extreme_ironing.jpg"]],
+"waterview": [["What are the things I should be cautious about when I visit here?", "./eval/eval_data/images/singles/waterview.jpg"]],
+"art-dog": [["can you describe the image", "./eval/eval_data/images/singles/202160027_b319c4166e.jpg"]],
+"funny-phone": [["What is funny about this image? Describe it panel by panel.", "./eval/eval_data/images/singles/1.jpg"]],
+"squirrel": [["Why would a person find this image funny?", "./eval/eval_data/images/singles/2.jpg"]],
+"art-painting": [["Tell me about this work of art.", "./eval/eval_data/images/singles/50.jpg"]]
+}
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/1806905748_adb926a0a0.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/1806905748_adb926a0a0.jpg
new file mode 100644
index 000000000..100eccc42
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/1806905748_adb926a0a0.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/british_shorthair.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/british_shorthair.jpg
new file mode 100644
index 000000000..b61731c62
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/british_shorthair.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/cat.png b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/cat.png
new file mode 100644
index 000000000..1a48d45c4
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/cats/cat.png differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count1.jpg
new file mode 100644
index 000000000..b29d3a97d
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count2.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count2.jpg
new file mode 100644
index 000000000..b09d1694a
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/can-count2.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count1.jpg
new file mode 100644
index 000000000..2d4b1b958
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count2.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count2.jpg
new file mode 100644
index 000000000..08ac55fe2
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/friends/wrong-count2.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/1.jpg
new file mode 100644
index 000000000..69984e57b
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/2.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/2.jpg
new file mode 100644
index 000000000..ca1232162
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/2.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/202160027_b319c4166e.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/202160027_b319c4166e.jpg
new file mode 100644
index 000000000..8628f3d7b
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/202160027_b319c4166e.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/50.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/50.jpg
new file mode 100644
index 000000000..f23f0548d
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/50.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/extreme_ironing.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/extreme_ironing.jpg
new file mode 100644
index 000000000..638b07883
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/extreme_ironing.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/waterview.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/waterview.jpg
new file mode 100644
index 000000000..6f44ebaba
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/singles/waterview.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/gate1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/gate1.jpg
new file mode 100644
index 000000000..b7b747294
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/gate1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/jobs1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/jobs1.jpg
new file mode 100644
index 000000000..18e8d35e9
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/jobs1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/musk1.jpg b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/musk1.jpg
new file mode 100644
index 000000000..7f2abfe89
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/tech-ceo/musk1.jpg differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z1.png b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z1.png
new file mode 100644
index 000000000..fdb9d8db9
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z1.png differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2.png b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2.png
new file mode 100644
index 000000000..57766b181
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2.png differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2a.png b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2a.png
new file mode 100644
index 000000000..79f30e02d
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z2a.png differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z3.png b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z3.png
new file mode 100644
index 000000000..8ccd23f53
Binary files /dev/null and b/applications/DeepSpeed-VisualChat/eval/eval_data/images/zootopia/z3.png differ
diff --git a/applications/DeepSpeed-VisualChat/eval/eval_scripts/run_batch.sh b/applications/DeepSpeed-VisualChat/eval/eval_scripts/run_batch.sh
new file mode 100644
index 000000000..dd79b0a00
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/eval_scripts/run_batch.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+
+#EVAL_DATSET=eval_robustness eval_single eval_comprehensive (see the json in the folder ./eval_data/*.json)
+MAIN_PATH=$1
+VISION_MODEL=/blob/transformers_cache/qwen-clip #openai/clip-vit-large-patch14
+LLM=/blob/transformers_cache/Llama-2-13b-hf #meta-llama/Llama-2-7b
+for EVAL_DATSET in eval_single eval_comprehensive eval_robustness
+do
+ SAVE_PATH=eval/results/${EVAL_DATSET}
+ mkdir ${SAVE_PATH}
+ for CKPT_NAME in final best_eval
+ do
+ #NOTE: to run multi-GPU, you simple do "export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7;"
+ export CUDA_VISIBLE_DEVICES=0; python eval/batch_generation.py --model_name dsvl --vis_proj baseline --max_seq_len 4096 \
+ --lm_model_name_or_path ${LLM} --vision_model_name_or_path ${VISION_MODEL} \
+ --checkpoint_path $MAIN_PATH --checkpoint_names $CKPT_NAME --eval_data ${EVAL_DATSET} \
+ --enable_mmca_attention --output_filename ${SAVE_PATH}/ours_${CKPT_NAME} &> ${SAVE_PATH}/ours_${CKPT_NAME}.log
+ done
+done
+
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_best_eval.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_best_eval.csv
new file mode 100644
index 000000000..dd83ebf25
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_best_eval.csv
@@ -0,0 +1,78 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat has its paws up in the air and is looking directly at the camera. The cat's eyes are yellow, and it appears to be relaxed and comfortable in its position. The background of the image is not particularly distinct, but the focus is on the cat and its pose."
+cat_images1,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a couch, with its paws stretched out and its head resting on the couch. The cat appears to be relaxed and comfortable in its position."
+cat_images1,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a brown and white cat standing on a green leafy plant, with its paws up in the air and its head tilted to the side. The cat appears to be curious and alert, as it looks around its surroundings. The background of the image is not particularly distinct, but the focus is on the cat and its pose."
+cat_images1,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is brown and white."
+cat_images1,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpet while the third image shows a brown and white cat standing on a green leafy plant.
+cat_images1,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a gray and white cat lying on a couch while the third image shows a brown and white cat standing on a green leafy plant.
+cat_images1,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"Yes, the cat in the first image is lying in the sunshine."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat has its paws up in the air and is looking directly at the camera. The cat's eyes are yellow, and it appears to be relaxed and comfortable in its position. The background of the image is not particularly distinct, but the focus is on the cat and its pose."
+cat_images2,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a couch, with its paws stretched out and its head resting on the couch. The cat appears to be relaxed and comfortable in its position."
+cat_images2,What are the differences between the two images?,What are the differences between the two images?,The first image shows a black cat lying on a carpet while the second image shows a gray and white cat lying on a couch.
+cat_images2,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a cat lying on a carpeted floor, with its paws stretched out and its head resting on the floor. The cat appears to be relaxed and comfortable in its position."
+cat_images2,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is brown and white."
+cat_images2,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpet while the third image shows a brown and white cat lying on a couch.
+cat_images2,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a gray and white cat lying on a couch while the third image shows a brown and white cat lying on a couch.
+cat_images2,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpeted floor."
+cat_images2,Which cat do you prefer and why?,Which cat do you prefer and why?,"I prefer the gray and white cat in the second image because it looks calm and relaxed, lying on a couch with its paws stretched out. The cat's expression seems to convey a sense of contentment and comfort, which is appealing to me."
+cat_images2,I prefer the second cat. It's so cute.,I prefer the second cat. It's so cute.,"I agree, the second cat is very cute and looks comfortable lying on the couch."
+cat_images2,Then why do you prefer the third cat more?,Then why do you prefer the third cat more?,"I prefer the third cat because it is brown and white, which is a classic and timeless color combination. Additionally, the third cat is lying on a couch, which is a more comfortable and inviting setting than the carpet in the first image."
+counting_people1,./eval/eval_data/images/friends/can-count1.jpg,Count the number of people in the image.,"There are six people in the image, including the two women and four men."
+counting_people1,./eval/eval_data/images/friends/can-count2.jpg,Count the number of people in the image.,"The number of people in the image is not specified, but there are at least six people in the group."
+counting_people1,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The first image has two women and four men, while the second image has four women and two men. The people in the two images are not the same, and the number of men and women is different. This is because the images are taken at different times and with different groups of people."
+counting_people1,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,"Yes, I am familiar with this TV series. The characters in the images are from the TV series ""Friends."" The characters are: Rachel, Monica, Phoebe, Joey, Chandler, and Ross."
+counting_people2,./eval/eval_data/images/friends/can-count1.jpg,How many individuals are depicted in the image?,"There are five individuals in the image, including the two women and three men."
+counting_people2,./eval/eval_data/images/friends/can-count2.jpg,How many individuals can you see in the second image?,There are six individuals in the second image.
+counting_people2,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The first image is a portrait of two women and three men, while the second image is a group photo of six people. The two images do not represent the same set of people."
+counting_people2,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Yes, you can recognize the TV show from the images. The characters are the cast of the TV show ""Friends."""
+counting_people3,./eval/eval_data/images/friends/wrong-count1.jpg,Count the number of people in the image.,There are four people in the image.
+counting_people3,./eval/eval_data/images/friends/wrong-count2.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people3,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,The first image has four people sitting on a bench while the second image has six people standing in a group. The people in the two images are not the same. The first image has a fountain in the background while the second image has trees in the background. The first image has a dog in it while the second image does not have any animals.
+counting_people4,./eval/eval_data/images/friends/wrong-count1.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,./eval/eval_data/images/friends/wrong-count2.jpg,How many individuals are depicted in the image?,There are six individuals in the image.
+counting_people4,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"Yes, the two images have different people in them. The first image has four people sitting on a bench, while the second image has six people standing in a group."
+counting_people4,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Yes, I recognize the TV show. The characters are John, Paul, George, and Ringo."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z1.png,Please describe the image in detail.,"The image is a digital recreation of a scene featuring two stuffed animals, a dog and a cat, sitting at a desk with a phone in front of them. The desk has a sign that says ""Next Window."" There are also two people in the background, one on the left and one on the right."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z2.png,Please describe the image in detail.,"The image is a digital recreation of a scene featuring a group of stuffed animals, including a fox, a raccoon, and a squirrel, standing in front of a phone booth. There are also two people in the background, one on the left and one on the right."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z3.png,Can you name the characters in the images? Who are they? What are they doing?,"The characters in the images are a fox, a raccoon, and a squirrel. They are standing in front of a phone booth and a coffee shop."
+zootopia_adventures1,"You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","Sure, you can create a story about the characters in the images. For example, you can imagine that the fox, raccoon, and squirrel are friends who are on a mission to save the world from an evil villain. They are using the phone booth and coffee shop as a base to plan their strategy and gather resources."
+zootopia_adventures1,Are you familiar with these characters? What movie are they from?,Are you familiar with these characters? What movie are they from?,"Yes, I am familiar with these characters. They are from the movie ""Zootopia."""
+zootopia_adventures1,Can you name the characters in the images? Who are they?,Can you name the characters in the images? Who are they?,"Yes, the characters in the images are a fox, a raccoon, and a squirrel."
+zootopia_adventures1,In what type of environment or setting do these characters live? Describe it.,In what type of environment or setting do these characters live? Describe it.,The characters in the images live in a phone booth and a coffee shop.
+zootopia_adventures2,./eval/eval_data/images/zootopia/z1.png ./eval/eval_data/images/zootopia/z2.png ./eval/eval_data/images/zootopia/z3.png,Create an engaging story strictly based on the images.,"The engaging story could be about a group of stuffed animals who are waiting in line at a bank or a store. They are discussing the next step in their transaction and are eager to get their business done. The story could also involve a funny incident that happens while they are in line, such as one of the stuffed animals losing their head or getting into a fight with another animal. The story could be told from the perspective of one of the stuffed animals or from an observer's point of view."
+zootopia_adventures2,Do you recognize the setting or the characters in these images? Name the movie.,Do you recognize the setting or the characters in these images? Name the movie.,"The setting is a bank or a store, and the characters are stuffed animals. The movie is not specified."
+zootopia_adventures2,Can you share some interesting facts or details about the characters shown in the images?,Can you share some interesting facts or details about the characters shown in the images?,"The characters shown in the images are stuffed animals from the movie ""Toy Story."" The main characters are Woody, a cowboy, and Buzz Lightyear, a space ranger. Other characters include Mr. Potato Head, Rex the dinosaur, and Slinky the dog."
+zootopia_adventures2,Which character do you find the most intriguing and why?,Which character do you find the most intriguing and why?,"I find Mr. Potato Head to be the most intriguing character because he can be assembled in different ways using various parts, making him versatile and unique."
+zootopia_adventures2,"Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Sure, here are some possible dialogues:
+- ""Hey, let's get in line for the next window.""
+- ""I'm not sure what we're doing here, but let's follow the others.""
+- ""I hope we can get our business done quickly so we can go back to playing.""
+- ""I'm not sure if this is the right place, but it looks like everyone else is here.""
+- ""I'm not sure what's going on, but let's just go with the flow.""
+- ""I think we should ask someone for help to figure out what we need to do.""
+- ""I'm not sure if we're in the right place, but let's just try to blend in and see what happens.""
+- ""I'm not sure if this is the right line, but it seems like everyone else is in it.""
+- ""I'm not sure what's going on, but let's just follow the others and see where it leads.""
+- ""I'm not sure if this is the right place, but it looks like everyone else is here, so let's just go with it."""
+zootopia_adventures3,./eval/eval_data/images/zootopia/z1.png,Examine and describe the characters' actions in the first image.,"In the first image, the two stuffed animals are sitting at a desk with a sign that says ""Next Window."" One of the stuffed animals is holding a coffee cup, while the other is holding a phone. They appear to be waiting for something at the window."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z2.png,"In the second image, what are the main characters doing, and how do they seem to feel?","In the second image, the main characters are a fox and a stuffed animal. They are standing in front of a sign that says ""Next Window."" The fox seems to be holding a phone, while the stuffed animal is holding a coffee cup. They appear to be interacting with each other and seem to be happy or at least not showing any negative emotions."
+zootopia_adventures3,Contrast the characters' moods and interactions in the two provided images.,Contrast the characters' moods and interactions in the two provided images.,"In the first image, the two stuffed animals are sitting at a desk with a sign that says ""Next Window."" They appear to be waiting for something, but their moods cannot be determined.
+
+In the second image, the fox and the stuffed animal are standing in front of a sign that says ""Next Window."" The fox is holding a phone and the stuffed animal is holding a coffee cup. They seem to be interacting with each other and appear to be happy or at least not showing any negative emotions."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z3.png,Imagine and narrate a hilarious situation involving the characters from the images.,"In the hilarious situation, the fox and the stuffed animal are having a conversation in front of a sign that says ""Next Window."" The fox is holding a phone and the stuffed animal is holding a coffee cup. Suddenly, the fox gets a call and starts talking on the phone, completely ignoring the stuffed animal. The stuffed animal gets frustrated and starts yelling at the fox, demanding his attention. The fox, still focused on the phone, pays no attention to the stuffed animal's yelling. The stuffed animal gets even more upset and starts throwing the coffee cup at the fox, but the cup misses and hits the wall behind them. The fox, oblivious to the situation, continues talking on the phone, while the stuffed animal storms off in a huff."
+zootopia_adventures3,"Name the movie from which these characters are, and give a succinct summary of its plot.","Name the movie from which these characters are, and give a succinct summary of its plot.","The movie is ""Next Window,"" and the plot involves a fox and a stuffed animal having a conversation in front of a sign that says ""Next Window."" The fox is holding a phone and the stuffed animal is holding a coffee cup. The fox gets a call on the phone and starts talking, ignoring the stuffed animal. The stuffed animal gets frustrated and starts yelling at the fox, demanding his attention. The fox continues talking on the phone, and the stuffed animal throws a coffee cup at the fox, but misses and hits the wall behind them. The fox pays no attention to the stuffed animal's outburst and continues talking on the phone."
+zootopia_adventures3,Create a funny and unexpected scenario that could unfold between the characters in these images.,Create a funny and unexpected scenario that could unfold between the characters in these images.,"In this funny and unexpected scenario, the fox and the stuffed animal are having a conversation in front of a sign that says ""Next Window."" The fox is holding a phone and the stuffed animal is holding a coffee cup. Suddenly, the fox gets a call on the phone and starts talking, ignoring the stuffed animal. The stuffed animal gets frustrated and starts yelling at the fox, demanding his attention. The fox continues talking on the phone, completely oblivious to the stuffed animal's outburst. The stuffed animal gets even more upset and throws the coffee cup at the fox, but misses and hits the wall behind them. The fox, still focused on the phone call, pays no attention to the coffee cup hitting the wall and continues talking. The stuffed animal, now furious, starts jumping up and down, trying to get the fox's attention. The fox, still engrossed in the phone call, ignores the stuffed animal's antics and continues talking. The stuffed animal, in a last-ditch effort to get the fox's attention, starts singing a song, but the fox"
+tech_ceos1,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos1,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos1,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding a microphone in front of a large sign.
+tech_ceos1,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs standing in front of a podium.
+tech_ceos1,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple."
+tech_ceos1,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not the founder of Apple."
+tech_ceos2,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the first image?,"The person in the first image is a man wearing a white shirt and a black tie, who is holding a microphone in his hand."
+tech_ceos2,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the second image?,"The person in the second image is Bill Gates, the founder of Microsoft."
+tech_ceos2,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the third image?,"The person in the third image is Steve Jobs, the founder of Apple."
+tech_ceos2,Recall who is in the second image.,Recall who is in the second image.,There is no one in the second image.
+tech_ceos2,Recall who is in the first image.,Recall who is in the first image.,There is a man in the first image who is holding a microphone in his hand.
+tech_ceos2,"Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ","Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ","No, the person in the first image is not the founder of Apple. The person in the second image is the founder of Apple."
+tech_ceos2,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_final.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_final.csv
new file mode 100644
index 000000000..66c30e318
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set1_final.csv
@@ -0,0 +1,75 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on a carpeted floor, with its paws stretched out and its eyes closed. The cat appears to be relaxed and comfortable, possibly sleeping or resting. The background of the image is not particularly distinct, but the focus is on the cat and its position on the floor."
+cat_images1,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a couch, with its paws stretched out and its eyes closed. The cat appears to be relaxed and comfortable on the couch."
+cat_images1,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a brown and white cat lying on a green leafy surface, possibly a grassy area. The cat is stretched out and appears to be relaxed, with its paws extended and its eyes closed. The background of the image is not particularly distinct, but it appears to be outdoors, surrounded by greenery."
+cat_images1,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is brown and white."
+cat_images1,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpet while the third image shows a brown and white cat lying on a green leafy surface.
+cat_images1,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a gray and white cat lying on a couch while the third image shows a brown and white cat lying on a green leafy surface.
+cat_images1,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"Yes, the cat in the first image is lying in the sunshine."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on a carpeted floor, with its paws stretched out and its eyes closed. The cat appears to be relaxed and comfortable, possibly sleeping or resting. The background of the image is not particularly distinct, but the focus is on the cat and its position on the floor."
+cat_images2,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a couch, with its paws stretched out and its eyes closed. The cat appears to be relaxed and comfortable on the couch."
+cat_images2,What are the differences between the two images?,What are the differences between the two images?,The first image shows a black cat lying on a carpeted floor while the second image shows a gray and white cat lying on a couch.
+cat_images2,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a brown and white cat sitting on a carpeted floor, with its paws stretched out and its eyes closed. The cat appears to be relaxed and comfortable in its position."
+cat_images2,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is brown and white."
+cat_images2,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpeted floor while the third image shows a brown and white cat sitting on a carpeted floor.
+cat_images2,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a gray and white cat lying on a couch while the third image shows a brown and white cat sitting on a carpeted floor.
+cat_images2,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpeted floor."
+cat_images2,Which cat do you prefer and why?,Which cat do you prefer and why?,"I prefer the gray and white cat in the second image because it looks calm and relaxed while lying on the couch. The cat's eyes are closed, and it seems to be enjoying the comfort of the couch. The image also shows the cat's paws stretched out, which adds to its relaxed appearance."
+cat_images2,I prefer the second cat. It's so cute.,I prefer the second cat. It's so cute.,"I agree, the second cat is very cute and looks relaxed while lying on the couch."
+cat_images2,Then why do you prefer the third cat more?,Then why do you prefer the third cat more?,"I prefer the third cat because it is brown and white, which is a classic and attractive color combination for cats. Additionally, the third cat is sitting on a carpeted floor, which provides a cozy and comfortable environment for the cat to relax in."
+counting_people1,./eval/eval_data/images/friends/can-count1.jpg,Count the number of people in the image.,"There are six people in the image, including the two women and four men."
+counting_people1,./eval/eval_data/images/friends/can-count2.jpg,Count the number of people in the image.,"The number of people in the image is not specified, but there are at least six people, including two women and four men."
+counting_people1,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The first image has six people, including two women and four men, while the second image also has six people, including two women and four men. However, the people in the two images are not the same as they have different positions and backgrounds. The first image has the people sitting on a couch, while the second image has the people standing in front of a wall. The first image has the people's faces visible, while the second image has some of the faces partially hidden. The reason for the difference is that the two images are taken at different times and locations, with different people posing for the pictures."
+counting_people1,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,"Yes, I am familiar with the TV series. The characters in the images are from the TV series Friends. The characters are: Monica, Rachel, Phoebe, Joey, Chandler, and Ross."
+counting_people2,./eval/eval_data/images/friends/can-count1.jpg,How many individuals are depicted in the image?,There are four individuals in the image.
+counting_people2,./eval/eval_data/images/friends/can-count2.jpg,How many individuals can you see in the second image?,There are six individuals in the second image.
+counting_people2,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The two images represent different sets of people. The first image shows four people sitting on a couch, while the second image shows six people standing in a row."
+counting_people2,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"No, these images do not represent a TV show. The characters in the images are not named."
+counting_people3,./eval/eval_data/images/friends/wrong-count1.jpg,Count the number of people in the image.,There are four people in the image.
+counting_people3,./eval/eval_data/images/friends/wrong-count2.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people3,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The first image has four people sitting on a bench, while the second image has six people standing next to each other. The people in the two images are not the same. The reason is that the first image is a group of friends posing for a picture, while the second image is a group of actors posing for a promotional photo."
+counting_people4,./eval/eval_data/images/friends/wrong-count1.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,./eval/eval_data/images/friends/wrong-count2.jpg,How many individuals are depicted in the image?,There are six individuals in the image.
+counting_people4,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"Yes, there are several differences between the two images. The first image has four people sitting on a bench, while the second image has six people standing. The people in the first image are wearing different clothes, while the people in the second image are wearing similar clothes. The location of the first image is a park, while the second image is taken in front of a tree."
+counting_people4,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Yes, I recognize the characters from the TV show Friends."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z1.png,Please describe the image in detail.,"The image is a digital recreation of a scene featuring two stuffed animals, a dog and a cat, sitting at a desk with a phone in front of them. The desk has a sign that says ""Next Window."" There are also two people in the background, one on the left and one on the right."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z2.png,Please describe the image in detail.,"The image is a digital recreation of a scene featuring a raccoon, a fox, and a squirrel standing in front of a counter with a sign that says ""Next Window."" There are also two people in the background, one on the left and one on the right."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z3.png,Can you name the characters in the images? Who are they? What are they doing?,"The characters in the images are a fox and a raccoon. They are sitting at a desk with a phone and a sign that says ""Next Window."" The fox is holding a coffee mug and the raccoon is holding a cup. They are likely discussing something or waiting for someone."
+zootopia_adventures1,"You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","The story could be about a fox and a raccoon who are working together in an office. They are sitting at a desk with a phone and a sign that says ""Next Window"". The fox is holding a coffee mug and the raccoon is holding a cup. They are discussing something important while waiting for someone to come to their desk."
+zootopia_adventures1,Are you familiar with these characters? What movie are they from?,Are you familiar with these characters? What movie are they from?,"Yes, I am familiar with these characters. They are from the movie ""Zootopia""."
+zootopia_adventures1,Can you name the characters in the images? Who are they?,Can you name the characters in the images? Who are they?,The characters in the images are a fox and a raccoon.
+zootopia_adventures1,In what type of environment or setting do these characters live? Describe it.,In what type of environment or setting do these characters live? Describe it.,"The characters live in an office setting. They are sitting at a desk with a phone and a sign that says ""Next Window""."
+zootopia_adventures2,./eval/eval_data/images/zootopia/z1.png ./eval/eval_data/images/zootopia/z2.png ./eval/eval_data/images/zootopia/z3.png,Create an engaging story strictly based on the images.,The engaging story could be about a group of stuffed animals who are waiting in line at a bank or a store. They might be discussing their plans or sharing their experiences while waiting. The story could be told from the perspective of one of the stuffed animals or from an outside observer.
+zootopia_adventures2,Do you recognize the setting or the characters in these images? Name the movie.,Do you recognize the setting or the characters in these images? Name the movie.,"The setting is a bank or a store, and the characters are stuffed animals. The movie is not specified."
+zootopia_adventures2,Can you share some interesting facts or details about the characters shown in the images?,Can you share some interesting facts or details about the characters shown in the images?,"The characters shown are stuffed animals, including a teddy bear, a fox, and a hippo. They are not real animals, but rather toys or figurines."
+zootopia_adventures2,Which character do you find the most intriguing and why?,Which character do you find the most intriguing and why?,I find the fox character the most intriguing because it is a stuffed animal that is not commonly seen in this context. It adds a unique element to the scene and makes it stand out.
+zootopia_adventures2,"Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Sure, here are some possible dialogues:
+- ""I'm waiting for my turn at the bank.""
+- ""I'm here to purchase something at the store.""
+- ""I hope we don't have to wait too long.""
+- ""I'm excited to see what's in store for us.""
+- ""I'm curious about what's going on around us.""
+- ""I'm glad we're all here together.""
+- ""I'm looking forward to our next adventure."""
+zootopia_adventures3,./eval/eval_data/images/zootopia/z1.png,Examine and describe the characters' actions in the first image.,"The characters in the first image are a group of stuffed animals, including a plush elephant, a plush horse, and a plush dog, standing behind a counter in a mock-up of a bank or store. The elephant is holding a sign that says ""Next Window,"" while the horse and dog are standing behind the counter. The scene is likely meant to depict a playful or imaginative representation of a bank or store setting, with the stuffed animals acting as the employees or customers."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z2.png,"In the second image, what are the main characters doing, and how do they seem to feel?","The main characters in the second image are a fox and a squirrel. They are standing in front of a counter, and the fox is holding a sign that says ""Next Window."" The squirrel is standing next to the fox, and they both seem to be looking at the sign. The fox appears to be smiling, while the squirrel's expression is not clear."
+zootopia_adventures3,Contrast the characters' moods and interactions in the two provided images.,Contrast the characters' moods and interactions in the two provided images.,"In the first image, the stuffed animals are not interacting with each other, so it is difficult to determine their moods. However, the plush elephant holding a sign that says ""Next Window"" seems to be in a neutral mood.
+
+In the second image, the fox and the squirrel are standing in front of a counter, and the fox is holding a sign that says ""Next Window."" The squirrel is standing next to the fox, and they both seem to be looking at the sign. It is difficult to determine their moods, but they appear to be focused on the task at hand."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z3.png,Imagine and narrate a hilarious situation involving the characters from the images.,"Imagine a scenario where the stuffed animals from the first image are having a conversation with the fox and the squirrel from the second image. The stuffed animals could be asking the fox and the squirrel about their experiences in the bank or the store, and the fox and the squirrel could be sharing their funny stories or mishaps while working there. The conversation could be full of humor and silliness, making it a hilarious situation to imagine."
+zootopia_adventures3,"Name the movie from which these characters are, and give a succinct summary of its plot.","Name the movie from which these characters are, and give a succinct summary of its plot.","The movie is ""The Fox and the Hound,"" and the plot revolves around the unlikely friendship between a fox and a hound. The fox, Tod, is raised by a woman who teaches him to be a thief, while the hound, Copper, is trained by a hunter to hunt foxes. When the two meet, they become friends, but their friendship is challenged when the hunter sets out to capture Tod. The hunter's dog, Chief, also joins the hunt. In the end, the hunter catches Tod, but Copper saves him, and the two friends escape together."
+zootopia_adventures3,Create a funny and unexpected scenario that could unfold between the characters in these images.,Create a funny and unexpected scenario that could unfold between the characters in these images.,"In this scenario, the stuffed animals from the first image are having a party, and the fox and the squirrel from the second image crash the party. The stuffed animals are surprised to see the fox and the squirrel, but they quickly welcome them and start to have a good time together. However, the party takes a turn when the stuffed animals realize that the fox and the squirrel are not from their movie, and they start to question their presence. The fox and the squirrel try to explain that they are from a different movie, but the stuffed animals are not convinced. The party turns into a debate about which movie is better, and the fox and the squirrel end up leaving the party feeling disappointed."
+tech_ceos1,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos1,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,Recall who is in the second image.,Recall who is in the second image.,"The second image shows a man holding a microphone and standing in front of a large sign that says ""Toshiba."""
+tech_ceos1,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding an iPhone.
+tech_ceos1,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos1,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the first image?,"The person in the first image is a man named Elon Musk, the founder and CEO of Tesla."
+tech_ceos2,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the second image?,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the third image?,"The person in the third image is Steve Jobs, the co-founder and CEO of Apple."
+tech_ceos2,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding up an iPhone.
+tech_ceos2,Recall who is in the first image.,Recall who is in the first image.,The first image shows a man standing in front of a microphone.
+tech_ceos2,"Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ","Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ",The person in the first image is not the founder of Apple. The person in the second image is the founder of Apple.
+tech_ceos2,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_best_eval.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_best_eval.csv
new file mode 100644
index 000000000..c129fefb7
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_best_eval.csv
@@ -0,0 +1,94 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat is relaxed and appears to be purring, as indicated by the caption. It has its paws stretched out and its eyes closed, enjoying the comfort of the carpet and the warmth of the room. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented and relaxed feline moment."
+cat_images1,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a carpeted floor, stretching out its paws and relaxing. The cat is lying on its side, with its head and body positioned close to a carpeted staircase. The cat's paws are extended and its eyes are closed, indicating that it is comfortable and at ease in its surroundings. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented feline moment."
+cat_images1,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a black and white cat lying on a carpeted floor, with its paws stretched out and its eyes closed. The cat is relaxed and appears to be enjoying the comfort of the carpet. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented feline moment."
+cat_images1,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. One is black, one is gray and white, and one is brown and white."
+cat_images1,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpeted floor while the third image shows a brown and white cat lying on a carpeted floor. The first image has a larger bounding box than the third image.
+cat_images1,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a black cat lying on a carpeted staircase while the third image shows a brown and white cat lying on a carpeted floor. The second image has a smaller bounding box than the third image.
+cat_images1,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpeted floor."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat is relaxed and appears to be purring, as indicated by the caption. It has its paws stretched out and its eyes closed, enjoying the comfort of the carpet and the warmth of the room. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented and relaxed feline moment."
+cat_images2,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a carpeted floor, stretching out its paws and relaxing. The cat is lying on its side, with its head and body positioned close to a carpeted staircase. The cat's paws are extended and its eyes are closed, indicating that it is comfortable and at ease in its surroundings. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented feline moment."
+cat_images2,What are the differences between the two images?,What are the differences between the two images?,The first image shows a black cat lying on a carpet while the second image shows a gray and white cat lying on a carpet near a staircase.
+cat_images2,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a black and white cat lying on a carpeted floor, with its paws stretched out and its eyes closed. The cat is relaxed and appears to be enjoying the comfort of the carpet. The cat's fur is smooth and shiny, and its body is well-proportioned. The image captures the essence of a contented feline moment."
+cat_images2,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is black and white."
+cat_images2,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpet while the third image shows a black and white cat lying on a carpet near a staircase.
+cat_images2,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a black cat lying on a carpet while the third image shows a gray and white cat lying on a carpet near a staircase.
+cat_images2,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpet indoors."
+cat_images2,Which cat do you prefer and why?,Which cat do you prefer and why?,"I personally prefer the gray and white cat in the third image. It appears to be relaxed and content, with its eyes closed and paws stretched out, which gives off a sense of tranquility and comfort. The cat's fur is also well-groomed and shiny, which adds to its overall appearance."
+cat_images2,I prefer the second cat. It's so cute.,I prefer the second cat. It's so cute.,"I agree, the second cat is adorable. It looks so comfortable and relaxed, with its paws stretched out and eyes closed. It's a great image of a cat enjoying a moment of peace and contentment."
+cat_images2,Then why do you prefer the third cat more?,Then why do you prefer the third cat more?,"I prefer the third cat more because it looks more relaxed and content. It is lying on the carpet with its eyes closed and paws stretched out, which gives off a sense of tranquility and comfort. The cat's fur is also well-groomed and shiny, which adds to its overall appearance."
+counting_people1,./eval/eval_data/images/friends/can-count1.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people1,./eval/eval_data/images/friends/can-count2.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people1,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The two images show different groups of people. In the first image, there are six people posing for a photo, while in the second image, there are six people standing together, but they are not posing for a photo. The first image shows the same group of people in different poses, while the second image shows a different group of people standing together."
+counting_people1,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,"Yes, I am familiar with the TV series ""The Simpsons."" The characters shown in the provided images are Homer Simpson, Marge Simpson, Bart Simpson, Lisa Simpson, and Maggie Simpson."
+counting_people2,./eval/eval_data/images/friends/can-count1.jpg,How many individuals are depicted in the image?,There are six individuals depicted in the image.
+counting_people2,./eval/eval_data/images/friends/can-count2.jpg,How many individuals can you see in the second image?,There are six individuals in the second image.
+counting_people2,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The first image shows a group of people standing in front of a building, while the second image shows a group of people sitting on a couch. The first image does not provide any information about the people's identities, while the second image shows six individuals with different appearances and clothing. The rationale is that the first image is a general group photo, while the second image is a more specific setting with individual portraits."
+counting_people2,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Yes, the images depict characters from the popular TV show ""Friends."" The characters are (from left to right) Monica Geller, Rachel Green, Phoebe Buffay, Joey Tribbiani, Chandler Bing, and Ross Geller."
+counting_people3,./eval/eval_data/images/friends/wrong-count1.jpg,Count the number of people in the image.,There are five people in the image.
+counting_people3,./eval/eval_data/images/friends/wrong-count2.jpg,Count the number of people in the image.,There are five people in the image.
+counting_people3,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The two images show different groups of people. In the first image, there are five people sitting on a bench, while in the second image, there are five people standing in front of a building. The people in the first image are wearing casual clothes, while the people in the second image are wearing formal attire. The first image shows a group of friends sitting together, while the second image shows a group of people posing for a photo in front of a building. The first image does not show any buildings or background, while the second image has a building in the background. The first image does not have any formal attire, while the second image has formal attire. The first image does not have any posing, while the second image has people posing for a photo. The first image does not have any buildings or background, while the second image has a building in the background. The first image does not have any formal attire, while the second image has formal attire. The first image does not have any posing, while the second image has people posing for a photo. The first image does not have any buildings or background, while the second image has a building in the background. The first image does not have any"
+counting_people4,./eval/eval_data/images/friends/wrong-count1.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,./eval/eval_data/images/friends/wrong-count2.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The two images depict different sets of people. In the first image, there are four people sitting on a bench, while in the second image, there are four people standing in front of a building. The first image shows a group of friends posing for a picture, while the second image appears to be a group of people posing for a promotional photo or a public event."
+counting_people4,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"I'm sorry, I'm not familiar with the TV show you're referring to. The characters shown in the provided images are not identified, and I cannot provide any information about them."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z1.png,Please describe the image in detail.,"The image shows a group of stuffed animals sitting at a desk in a room. There are three stuffed animals, including a teddy bear, a dog, and a cat. The teddy bear is sitting in the middle of the desk, while the dog and cat are sitting on either side of the teddy bear. The room is decorated with a rug on the floor and a picture hanging on the wall."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z2.png,Please describe the image in detail.,"The image shows a group of stuffed animals sitting at a desk in a room. The stuffed animals are wearing clothes and are sitting on chairs. There are three stuffed animals in the scene, including a teddy bear, a dog, and a cat. The room is decorated with a rug on the floor and a picture hanging on the wall."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z3.png,Can you name the characters in the images? Who are they? What are they doing?,"The characters in the images are a stuffed animal, a dog, and a cat. They are sitting at a desk in a room."
+zootopia_adventures1,"You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","Once upon a time, in a cozy room filled with stuffed animals, a teddy bear, a dog, and a cat sat at a desk. They were having a meeting to discuss their plans for the day. The teddy bear, being the leader of the group, suggested they go on an adventure to explore the nearby forest. The dog, being the most adventurous of the group, was excited about the idea and immediately agreed. The cat, on the other hand, was a bit hesitant but eventually agreed to join them.
+The group packed their things and set out on their journey. As they walked through the forest, they encountered various animals and plants. The dog was always eager to explore and sniff around, while the teddy bear and cat kept a close eye on their surroundings. They came across a stream, where they decided to take a break and have a picnic.
+While they were enjoying their meal, they heard a strange noise coming from the other side of the stream. The dog, being curious, jumped into the water to investigate. The teddy bear and cat were worried about him, but he quickly returned, wagging his tail and barking excited"
+zootopia_adventures1,Are you familiar with these characters? What movie are they from?,Are you familiar with these characters? What movie are they from?,"The characters in the images are from the movie ""Zootopia."""
+zootopia_adventures1,Can you name the characters in the images? Who are they?,Can you name the characters in the images? Who are they?,"The characters in the images are a stuffed animal, a dog, and a cat."
+zootopia_adventures1,In what type of environment or setting do these characters live? Describe it.,In what type of environment or setting do these characters live? Describe it.,The characters live in a room filled with stuffed animals.
+zootopia_adventures2,./eval/eval_data/images/zootopia/z1.png ./eval/eval_data/images/zootopia/z2.png ./eval/eval_data/images/zootopia/z3.png,Create an engaging story strictly based on the images.,"The teddy bear and the stuffed fox were sitting in a waiting room, patiently waiting for their turn to be seen by the doctor. They were surrounded by other stuffed animals, all waiting their turn for a check-up.
+The teddy bear was feeling a bit under the weather, with a slight cough and a runny nose. The fox, being a good friend, was trying to comfort the teddy bear and make him feel better.
+As they waited, they noticed a group of other stuffed animals gathered around a television screen. The news was on, and the stuffed animals were watching with great interest. The teddy bear and the fox joined the group, curious to know what was happening.
+Suddenly, the news anchor announced that a new children's hospital was opening in the city, and they were showing footage of the construction site. The teddy bear and the fox were excited to hear about the new hospital, as it would provide better facilities and care for all the stuffed animals in need.
+As the news segment ended, the teddy bear and the fox returned to their seats, waiting patiently for their turn to be seen by the doctor. They knew that with"
+zootopia_adventures2,Do you recognize the setting or the characters in these images? Name the movie.,Do you recognize the setting or the characters in these images? Name the movie.,"The setting and characters in these images resemble the movie ""Toy Story."""
+zootopia_adventures2,Can you share some interesting facts or details about the characters shown in the images?,Can you share some interesting facts or details about the characters shown in the images?,"Sure, here are some interesting facts about the characters in the movie ""Toy Story"":
+1. Woody, the cowboy doll, is the leader of the toys and the protagonist of the movie. He is voiced by Tom Hanks.
+2. Buzz Lightyear, the space ranger doll, is Woody's rival and best friend. He is voiced by Tim Allen.
+3. Mr. Potato Head is a versatile toy who can be assembled in various ways, with different parts that can be attached to his body. He is voiced by Don Rickles.
+4. Slinky Dog is a happy-go-lucky dog toy who is always eager to help Woody and the other toys. He is voiced by Jim Varney.
+5. Rex, the green dinosaur toy, is a nervous and excitable character who often gets scared easily. He is voiced by Wallace Shawn.
+6. Hamm, the piggy bank toy, is a wise-cracking character who often provides comic relief. He is voiced by John Ratzenberger.
+7. Bo Peep, the por"
+zootopia_adventures2,Which character do you find the most intriguing and why?,Which character do you find the most intriguing and why?,"I find the character of Mr. Potato Head to be the most intriguing in the movie ""Toy Story."" He is a versatile toy who can be assembled in various ways, with different parts that can be attached to his body. He is also a wise and humorous character, often providing insight and humor to the story. His ability to adapt and change his appearance makes him unique among the other toys, and his relationship with Woody is one of the central themes of the movie."
+zootopia_adventures2,"Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Sure, here are some possible dialogues between the characters in the images:
+1. Woody and Buzz Lightyear:
+Woody: ""Hey, Buzz, what's up?""
+Buzz Lightyear: ""Not much, Woody. Just waiting for our turn to be seen by the doctor.""
+Woody: ""Yeah, I'm feeling a bit under the weather myself. Maybe we should take it easy today.""
+Buzz Lightyear: ""Sure thing, Woody. Let's just relax and try to stay calm.""
+2. Woody and Mr. Potato Head:
+Woody: ""Hey, Mr. Potato Head, how's it going?""
+Mr. Potato Head: ""Not bad, Woody. Just waiting for our turn to be seen by the doctor.""
+Woody: ""Yeah, I'm feeling a bit under the weather myself. Do you think we should ask for a different doctor?""
+Mr. Potato Head: ""I don't know, Woody. Maybe we should just give this one a chance. After all, they're professionals.""
+Woody: """
+zootopia_adventures3,./eval/eval_data/images/zootopia/z1.png,Examine and describe the characters' actions in the first image.,"In the first image, two characters are sitting on a couch, one of them holding a cup of coffee. The other character is holding a newspaper, and they appear to be relaxing and enjoying their time together."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z2.png,"In the second image, what are the main characters doing, and how do they seem to feel?","In the second image, the main characters are standing in a room with a receptionist. They appear to be talking to the receptionist, and they seem to be in a friendly and casual mood."
+zootopia_adventures3,Contrast the characters' moods and interactions in the two provided images.,Contrast the characters' moods and interactions in the two provided images.,"In the first image, the two characters are relaxed and enjoying their time together, sharing a cup of coffee and a newspaper. They appear to be in a comfortable and peaceful setting.
+In contrast, the second image shows the main characters interacting with a receptionist in a more formal setting. They seem to be engaging in a conversation with the receptionist, possibly inquiring about something or seeking assistance. The mood is more casual and friendly, but there is a sense of formality due to the presence of the receptionist and the setting."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z3.png,Imagine and narrate a hilarious situation involving the characters from the images.,"In the first image, the two characters are enjoying a cup of coffee and a newspaper while relaxing on a couch. Suddenly, a wild and crazy character from the second image bursts into the scene, causing a commotion. The main characters are startled and try to figure out what's going on. The wild character, who is a stuffed animal, is dressed in a suit and tie and is holding a newspaper. He is trying to get their attention, but they are too surprised to respond. The wild character starts to dance and sing, making the situation even more chaotic. The main characters eventually join in the fun, and the scene turns into a hilarious dance party. The wild character's antics bring a new level of excitement and humor to the scene, making it a memorable and entertaining moment."
+zootopia_adventures3,"Name the movie from which these characters are, and give a succinct summary of its plot.","Name the movie from which these characters are, and give a succinct summary of its plot.","The characters are from the movie ""Zootopia,"" which tells the story of a city where animals of all species coexist and work together. The main characters are a rabbit named Judy Hopps, who becomes a police officer, and a fox named Nick Wilde, who is a con artist. Together, they work to solve a mystery involving the disappearance of several animals and uncover a conspiracy that threatens the harmony of Zootopia. The movie explores themes of prejudice, trust, and the importance of working together to overcome challenges."
+zootopia_adventures3,Create a funny and unexpected scenario that could unfold between the characters in these images.,Create a funny and unexpected scenario that could unfold between the characters in these images.,"In a hilarious and unexpected twist, the main characters from the first image, the two relaxed characters sitting on a couch, suddenly find themselves transported into the world of the second image, where they encounter the wild and crazy stuffed animal in a suit. The two characters are taken aback by this unexpected turn of events and try to make sense of their new surroundings.
+The stuffed animal, who is dressed in a suit and holding a newspaper, starts to dance and sing, causing the two main characters to join in the fun. However, as they dance and sing, they realize that they are not in their usual environment and that they need to find a way back to their original world.
+The two main characters and the wild stuffed animal embark on a journey to find a way back to their respective worlds, encountering various obstacles and challenges along the way. They eventually discover a portal that allows them to return to their original world, where they are relieved to be back in their familiar surroundings.
+The unexpected and hilarious adventure that the two main characters and the wild stuffed animal experience serves as a reminder that sometimes the most unexpected things can happen, and that"
+tech_ceos1,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos1,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos1,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Steve Jobs, the co-founder of Apple."
+tech_ceos1,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos1,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the first image?,"The person in the first image is Elon Musk, the CEO of Tesla."
+tech_ceos2,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the second image?,"The person in the second image is Bill Gates, the founder of Microsoft."
+tech_ceos2,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the third image?,"The person in the third image is Elon Musk, the CEO of Tesla."
+tech_ceos2,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the founder of Microsoft."
+tech_ceos2,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Elon Musk, the CEO of Tesla."
+tech_ceos2,"Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ","Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ",The person in the first image is not the founder of Apple. The person in the second image is the founder of Microsoft.
+tech_ceos2,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not the founder of Apple."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_final.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_final.csv
new file mode 100644
index 000000000..dc800fc89
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_comprehensive/ours-set2_final.csv
@@ -0,0 +1,103 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat is relaxed and appears to be purring, as indicated by the caption. It has its paws stretched out and its eyes closed, enjoying the comfort of the carpet and possibly receiving some love or attention from its owner. The cat's fur is smooth and shiny, and its body is well-proportioned. The image conveys a sense of peace and contentment, highlighting the bond between humans and their pets."
+cat_images1,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a carpeted floor, stretching out its paws and relaxing. The cat is lying on its side, with its eyes closed, and appears to be enjoying a comfortable and peaceful moment. The cat's fur is well-groomed, and it is surrounded by a clean and tidy environment. The image conveys a sense of tranquility and the bond between humans and their pets."
+cat_images1,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a black and white cat lying on a carpeted floor, surrounded by greenery. The cat is lying on its back, with its paws stretched out and its eyes closed, enjoying a moment of relaxation. The greenery in the background adds a touch of nature to the scene, creating a peaceful and serene atmosphere. The cat's fur is well-groomed, and it appears to be in a comfortable and contented state."
+cat_images1,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. One is black and white, one is gray and white, and one is brown and white."
+cat_images1,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpeted floor while the third image shows a brown and white cat lying on a carpeted floor. The first image has a larger bounding box than the third image.
+cat_images1,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a black cat lying on its back while the third image shows a brown and white cat lying on its back. The second image has a smaller bounding box than the third image.
+cat_images1,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpeted floor."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,Please describe the image in detail.,"The image shows a black cat lying on its back on a carpeted floor. The cat is relaxed and appears to be purring, as indicated by the caption. It has its paws stretched out and its eyes closed, enjoying the comfort of the carpet and possibly receiving some love or attention from its owner. The cat's fur is smooth and shiny, and its body is well-proportioned. The image conveys a sense of peace and contentment, highlighting the bond between humans and their pets."
+cat_images2,./eval/eval_data/images/cats/british_shorthair.jpg,Please describe the image in detail.,"The image shows a gray and white cat lying on a carpeted floor, stretching out its paws and relaxing. The cat is lying on its side, with its eyes closed, and appears to be enjoying a comfortable and peaceful moment. The cat's fur is well-groomed, and it is surrounded by a clean and tidy environment. The image conveys a sense of tranquility and the bond between humans and their pets."
+cat_images2,What are the differences between the two images?,What are the differences between the two images?,The first image shows a black cat lying on a carpet while the second image shows a gray and white cat lying on a carpet near a couch.
+cat_images2,./eval/eval_data/images/cats/cat.png,Please describe the image in detail.,"The image shows a black and white cat lying on a carpeted floor, stretching out its paws and looking up at something. The cat appears to be relaxed and enjoying its surroundings. The carpet is a light color, and the cat's fur is well-groomed. The image conveys a sense of tranquility and the bond between humans and their pets."
+cat_images2,Are the colors of the three cats the same?,Are the colors of the three cats the same?,"No, the colors of the three cats are different. The first cat is black, the second cat is gray and white, and the third cat is brown and white."
+cat_images2,What are the differences between the first and third images?,What are the differences between the first and third images?,The first image shows a black cat lying on a carpet while the third image shows a brown and white cat lying on a carpet near a couch.
+cat_images2,What are the differences between the second and third images?,What are the differences between the second and third images?,The second image shows a black cat lying on a carpet while the third image shows a gray and white cat lying on a carpet near a couch.
+cat_images2,Is the cat in the first image in the sunshine?,Is the cat in the first image in the sunshine?,"No, the cat in the first image is not in the sunshine. It is lying on a carpet indoors."
+cat_images2,Which cat do you prefer and why?,Which cat do you prefer and why?,"I personally prefer the gray and white cat in the second image. It appears to be relaxed and content, which makes it a pleasant sight to behold. Additionally, its fur is well-groomed, which adds to its overall appeal."
+cat_images2,I prefer the second cat. It's so cute.,I prefer the second cat. It's so cute.,"I agree, the second cat is very cute. It looks relaxed and content, which makes it a pleasant sight to behold. Its fur is well-groomed, which adds to its overall appeal."
+cat_images2,Then why do you prefer the third cat more?,Then why do you prefer the third cat more?,"I prefer the third cat more because it is lying on a carpet near a couch, which gives it a cozy and comfortable environment. Additionally, the cat's fur looks well-groomed, which adds to its overall appeal."
+counting_people1,./eval/eval_data/images/friends/can-count1.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people1,./eval/eval_data/images/friends/can-count2.jpg,Count the number of people in the image.,There are six people in the image.
+counting_people1,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The two images show different groups of people. In the first image, there are six people posing for a photo, while in the second image, there are six people standing together, but they are not posing for a photo. The first image shows the group standing in a line, while the second image shows them standing in a group. The first image has a black background, while the second image has a white background. The first image has a caption that says ""Friends,"" while the second image does not have any caption. The first image has a larger size than the second image."
+counting_people1,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,Are you familiar with this TV series? Can you name the characters shown in the provided images? Who are they?,"Yes, I am familiar with the TV series ""Friends."" The characters shown in the images are:
+1. Ross Geller, who is a paleontologist and a friend of the group.
+2. Rachel Green, who is a fashionista and a friend of the group.
+3. Monica Geller, who is a chef and a friend of the group.
+4. Chandler Bing, who is a sarcastic and witty friend of the group.
+5. Joey Tribbiani, who is a struggling actor and a friend of the group.
+6. Phoebe Buffay, who is a free-spirited and eccentric friend of the group."
+counting_people2,./eval/eval_data/images/friends/can-count1.jpg,How many individuals are depicted in the image?,There are six individuals depicted in the image.
+counting_people2,./eval/eval_data/images/friends/can-count2.jpg,How many individuals can you see in the second image?,There are six individuals in the second image.
+counting_people2,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The first image shows a group of people standing together with the word ""Friends"" written on the bottom, while the second image shows a group of six individuals posing for a picture. It is not clear if the two images represent the same set of people or not."
+counting_people2,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Unfortunately, I am not familiar with the TV show depicted in the provided images. I cannot identify the characters or provide any additional information about the show."
+counting_people3,./eval/eval_data/images/friends/wrong-count1.jpg,Count the number of people in the image.,There are four people in the image.
+counting_people3,./eval/eval_data/images/friends/wrong-count2.jpg,Count the number of people in the image.,There are four people in the image.
+counting_people3,What are the differences between the two images? Are they the same group of people? Explain why.,What are the differences between the two images? Are they the same group of people? Explain why.,"The two images appear to feature different groups of people. In the first image, there are four people sitting on a bench, while in the second image, there are four people standing in front of a building. The people in the first image are wearing casual clothes, while the people in the second image are wearing more formal attire. Additionally, the first image shows a fountain in the background, while the second image does not. These differences suggest that the two images depict different groups of people in different settings."
+counting_people4,./eval/eval_data/images/friends/wrong-count1.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,./eval/eval_data/images/friends/wrong-count2.jpg,How many individuals are depicted in the image?,There are four individuals depicted in the image.
+counting_people4,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,Can you spot any differences between these two images? Do they represent the same set of people? Please provide a rationale.,"The two images depict different sets of people. In the first image, there are four people sitting on a bench, while in the second image, there are four people standing in front of a building. The first image shows a group of friends posing for a picture, while the second image appears to be a group of people posing for a promotional photo or a public event. The rationale behind these two images is that they represent different scenarios and groups of people."
+counting_people4,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,Do you recognize this TV show? Can you name the characters shown in the provided images? Who are they?,"Yes, I recognize this TV show as ""Friends."" The characters shown in the provided images are:
+
+1. Ross Geller, played by David Schwimmer
+2. Monica Geller, played by Courteney Cox
+3. Phoebe Buffay, played by Lisa Kudrow
+4. Joey Tribbiani, played by Matt LeBlanc
+5. Chandler Bing, played by Matthew Perry"
+zootopia_adventures1,./eval/eval_data/images/zootopia/z1.png,Please describe the image in detail.,"The image shows a group of stuffed animals sitting at a desk in a room. The desk is located in the middle of the room, and there are two chairs in front of it. The stuffed animals are sitting at the desk, with one of them holding a cup of coffee. There are also several books and a computer monitor on the desk. The room is well-lit, and there are no people present in the scene."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z2.png,Please describe the image in detail.,"The image shows a group of stuffed animals sitting at a desk in a room. The stuffed animals are wearing clothes and are sitting in front of a computer monitor. There are several books on the desk, and the room is well-lit. The stuffed animals appear to be having a conversation or working together at the desk."
+zootopia_adventures1,./eval/eval_data/images/zootopia/z3.png,Can you name the characters in the images? Who are they? What are they doing?,The characters in the images are a stuffed animal and a raccoon. They are sitting at a desk and using a computer.
+zootopia_adventures1,"You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","You are an imaginative storyteller. Create a fascinating story based on the first, second and third image.","Once upon a time, in a magical land, there lived a stuffed animal and a raccoon. They were best friends and always found themselves in adventures. One day, they decided to start a business together. They set up a desk in a cozy corner of their home and started working on their project.
+The stuffed animal was a creative genius and came up with innovative ideas. The raccoon was a skilled computer programmer and helped bring those ideas to life. Together, they worked tirelessly to create a revolutionary product that would change the world.
+As they worked, they faced many challenges and obstacles. But with their determination and teamwork, they overcame them all. Their product was a huge success, and they became famous for their innovation and creativity.
+Their story became an inspiration to many, and their friendship was a testament to the power of teamwork and perseverance. They proved that with hard work and dedication, anything is possible, even in the most unlikely of circumstances."
+zootopia_adventures1,Are you familiar with these characters? What movie are they from?,Are you familiar with these characters? What movie are they from?,"The characters in the images are from the movie ""Zootopia."""
+zootopia_adventures1,Can you name the characters in the images? Who are they?,Can you name the characters in the images? Who are they?,"The characters in the images are Judy Hopps and Nick Wilde from the movie ""Zootopia."""
+zootopia_adventures1,In what type of environment or setting do these characters live? Describe it.,In what type of environment or setting do these characters live? Describe it.,"Judy Hopps and Nick Wilde live in the fictional city of Zootopia, which is a melting pot of different animal species. The city is a modern metropolis with various districts, including the police department where Judy works and the street where Nick runs his business. The city is known for its diverse population and its unique blend of animal cultures."
+zootopia_adventures2,./eval/eval_data/images/zootopia/z1.png ./eval/eval_data/images/zootopia/z2.png ./eval/eval_data/images/zootopia/z3.png,Create an engaging story strictly based on the images.,"The teddy bear and the stuffed fox were sitting in a waiting room, patiently waiting for their turn to be seen by the doctor. They were surrounded by other stuffed animals, all in various states of discomfort. The teddy bear had a bandage on his arm, while the fox seemed to be nursing a sore paw.
+The teddy bear turned to the fox and said, ""I hope the doctor can help us. I'm not sure what's wrong with my arm, but it's been hurting for a while."" The fox nodded in agreement and added, ""My paw is swollen and it's been hard to walk. I'm hoping the doctor can give us some relief.""
+As they waited, they observed the other stuffed animals in the room. There was a penguin with a runny nose, a giraffe with a sore throat, and a lion with a cough. The teddy bear and the fox exchanged worried glances, realizing that they were not alone in their misery.
+Suddenly, the door to the doctor's office opened, and a nurse"
+zootopia_adventures2,Do you recognize the setting or the characters in these images? Name the movie.,Do you recognize the setting or the characters in these images? Name the movie.,"The setting is a waiting room, and the characters are stuffed animals. The movie is ""Toy Story 2."""
+zootopia_adventures2,Can you share some interesting facts or details about the characters shown in the images?,Can you share some interesting facts or details about the characters shown in the images?,"Certainly! Here are some interesting facts about the characters in the images:
+1. The teddy bear in the image is a character from the movie ""Toy Story 2."" He is a stuffed bear named Lotso, who was once the beloved toy of a little girl named Andy. However, he was abandoned at a daycare center and became the leader of the toys there.
+2. The stuffed fox in the image is also a character from ""Toy Story 2."" He is a member of Lotso's gang and is known for his loyalty to Lotso.
+3. The teddy bear in the image has a bandage on his arm, indicating that he may have been injured or undergone some kind of medical treatment.
+4. The stuffed fox in the image is nursing a sore paw, suggesting that he may have sustained an injury as well.
+5. The teddy bear and the stuffed fox are both voiced by Ned Beatty in the movie ""Toy Story 2.""
+6. The teddy bear and the stuffed fox are among the many toys that come to life in the ""Toy Story"
+zootopia_adventures2,Which character do you find the most intriguing and why?,Which character do you find the most intriguing and why?,"I find the teddy bear from the movie ""Toy Story 2"" to be the most intriguing character. Despite being a toy, he has a rich backstory and a complex personality. He was once the beloved toy of a little girl named Andy, but was abandoned at a daycare center and became the leader of the toys there. His journey to find his way back to Andy is both heartwarming and adventurous. Additionally, his interactions with other characters, such as the stuffed fox and the nurse, showcase his resilience and determination. Overall, his story and character development make him a compelling and memorable character in the ""Toy Story"" franchise."
+zootopia_adventures2,"Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Based on the images, can you create some dialogues that the characters might say to each other in these situations?","Sure, here are some possible dialogues between the characters in the images:
+1. Teddy Bear: ""I'm not sure what's wrong with my arm, but it's been hurting for a while.""
+Stuffed Fox: ""I'm nursing my sore paw. I think I might have stepped on something sharp.""
+Teddy Bear: ""Oh no, that sounds painful. I hope we can get some help from the doctor soon.""
+2. Teddy Bear: ""I'm worried about my arm. I don't want to miss my turn to see the doctor.""
+Stuffed Fox: ""Don't worry, we'll get our turn soon. Just try to stay calm and patient.""
+Teddy Bear: ""I know, but it's hard when we're both in pain.""
+Stuffed Fox: ""I understand. Let's just try to distract ourselves until it's our turn.""
+3. Teddy Bear: ""I'm so glad we're finally getting to see the doctor. I hope he can help us both feel better.""
+Stuffed Fox: ""Me too. I"
+zootopia_adventures3,./eval/eval_data/images/zootopia/z1.png,Examine and describe the characters' actions in the first image.,"In the first image, two cartoon characters are sitting on a bench, one of them holding a cup of coffee. They appear to be having a conversation, possibly sharing the coffee. The scene is set in a park, with trees and grass in the background."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z2.png,"In the second image, what are the main characters doing, and how do they seem to feel?","The main characters in the second image are a fox and a squirrel, and they appear to be walking together. They seem to be in a good mood and are enjoying their time together."
+zootopia_adventures3,Contrast the characters' moods and interactions in the two provided images.,Contrast the characters' moods and interactions in the two provided images.,"In the first image, the two cartoon characters are sitting on a bench, one of them holding a cup of coffee. They appear to be having a conversation, possibly sharing the coffee. Their mood seems relaxed and friendly.
+In contrast, the second image features a fox and a squirrel walking together. They seem to be in a good mood and are enjoying their time together. Their interaction is more casual and playful, as they are walking side by side. The mood of the scene is light and cheerful."
+zootopia_adventures3,./eval/eval_data/images/zootopia/z3.png,Imagine and narrate a hilarious situation involving the characters from the images.,"In the second image, the fox and the squirrel are walking together, and the squirrel is carrying a cup of coffee. Suddenly, the fox gets a crazy idea and decides to play a prank on the squirrel. He grabs the cup of coffee from the squirrel's hands and drinks it all in one gulp. The squirrel is shocked and outraged, but the fox just laughs and runs away, leaving the squirrel to clean up the mess. The squirrel is furious and chases after the fox, but the fox is too fast and manages to escape. The squirrel is left with a bitter taste in her mouth and a cup of coffee spilled on the ground."
+zootopia_adventures3,"Name the movie from which these characters are, and give a succinct summary of its plot.","Name the movie from which these characters are, and give a succinct summary of its plot.","The characters are from the movie ""Zootopia,"" which is about a city where animals of all species coexist and work together. The plot revolves around a rabbit police officer and a con artist fox who team up to solve a mystery and uncover a conspiracy that threatens the harmony of the city. The movie explores themes of prejudice, discrimination, and the importance of working together despite differences."
+zootopia_adventures3,Create a funny and unexpected scenario that could unfold between the characters in these images.,Create a funny and unexpected scenario that could unfold between the characters in these images.,"In a hilarious twist, the fox and the squirrel from the second image suddenly find themselves transported to the first image, where they are sitting on the bench with the two cartoon characters. The fox and the squirrel are confused and bewildered by their new surroundings, while the two cartoon characters are equally surprised to see them.
+The fox and the squirrel try to make sense of the situation, but they are unable to comprehend the cartoon world they have entered. They attempt to communicate with the cartoon characters, but their attempts are met with confusion and amusement.
+The cartoon characters, intrigued by the fox and the squirrel, offer them coffee, but the fox and the squirrel are not familiar with the concept of coffee and refuse. The cartoon characters then offer them a cup of tea, which the fox and the squirrel happily accept.
+As they enjoy their tea, the fox and the squirrel begin to explore the cartoon world, discovering new and exciting things. They encounter various cartoon characters and participate in their adventures, creating a hilarious and unexpected storyline"
+tech_ceos1,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and CEO of Apple."
+tech_ceos1,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos1,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos1,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Steve Jobs, the co-founder of Apple."
+tech_ceos1,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos1,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the first image?,"The person in the first image is Elon Musk, the CEO of Tesla."
+tech_ceos2,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the second image?,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the third image?,"The person in the third image is Elon Musk, the CEO of Tesla."
+tech_ceos2,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Elon Musk, the CEO of Tesla."
+tech_ceos2,"Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ","Is the person in the first image the founder of Apple? If not, which of the above images is the person the founder of Apple. ",The person in the first image is not the founder of Apple. The person in the second image is the founder of Microsoft.
+tech_ceos2,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not the founder of Apple."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_best_eval.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_best_eval.csv
new file mode 100644
index 000000000..2d42db0cf
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_best_eval.csv
@@ -0,0 +1,61 @@
+test_name,image_path,question,answer
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"The person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"The person in the image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is a man named Bill Gates, who is the founder of Microsoft."
+tech_ceos2.1a,Recall who is in the second image.,Recall who is in the second image.,"The second image shows a man holding a microphone and standing in front of a large sign with the word ""Tesla"" on it."
+tech_ceos2.1a,Recall who is in the first image.,Recall who is in the first image.,The first image shows Bill Gates holding a cell phone.
+tech_ceos2.1a,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is not the founder of Apple, but the founder of Microsoft, Bill Gates."
+tech_ceos2.1a,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla."
+tech_ceos2.1b,Recall who is in the second image.,Recall who is in the second image.,Elon Musk is in the second image holding a microphone.
+tech_ceos2.1b,Recall who is in the first image.,Recall who is in the first image.,Bill Gates is in the first image standing next to a Microsoft sign.
+tech_ceos2.1b,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Bill Gates is the founder of Microsoft, not Apple."
+tech_ceos2.1b,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,This person in this image b is Bill Gates.
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1c,Recall who is in image b.,Recall who is in image b.,The person in image b is a man wearing a suit and tie holding a microphone.
+tech_ceos2.1c,Recall who is in the image a.,Recall who is in the image a.,The person in image a is Steve Jobs.
+tech_ceos2.1c,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1c,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"No, the person in the image c is not the founder of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1d,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding a microphone in front of a large sign.
+tech_ceos2.1d,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs standing in front of a podium.
+tech_ceos2.1d,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple."
+tech_ceos2.1d,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not the founder of Apple."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"The person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"The person in the image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1aa,What's the differnce between the first and second image,What's the differnce between the first and second image,The first image shows Steve Jobs holding an iPhone while the second image shows Bill Gates standing at a podium.
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1aa,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding a microphone and standing in front of a Tesla car.
+tech_ceos2.1aa,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding an iPhone.
+tech_ceos2.1aa,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos2.1aa,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Elon Musk."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1bb,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image shows Steve Jobs holding an iPhone while the second image shows Bill Gates standing at a podium.
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla."
+tech_ceos2.1bb,Recall who is in the second image.,Recall who is in the second image.,Elon Musk is in the second image holding a microphone.
+tech_ceos2.1bb,Recall who is in the first image.,Recall who is in the first image.,Steve Jobs is in the first image holding an iPhone.
+tech_ceos2.1bb,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Steve Jobs is the founder of Apple, and he is in the first image holding an iPhone."
+tech_ceos2.1bb,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,This person in this image b is Bill Gates.
+tech_ceos2.1cc,What's the differnce between the image a and image b,What's the differnce between the image a and image b,"In image a, Steve Jobs is holding an iPhone while in image b, Bill Gates is standing behind a podium."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is a man wearing a suit and tie, and he is holding an iPhone in his hand."
+tech_ceos2.1cc,Recall who is in image b.,Recall who is in image b.,The person in image b is a man wearing a suit and tie.
+tech_ceos2.1cc,Recall who is in the image a.,Recall who is in the image a.,Steve Jobs is in the image a.
+tech_ceos2.1cc,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the image c is the founder of Apple, while the person in the image a is the CEO of Apple."
+tech_ceos2.1cc,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"Yes, the person in the image c is the founder of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image is of Steve Jobs holding an iPhone while the second image is of Bill Gates standing at a podium.
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is a man wearing a suit and tie, and he is holding an iPhone in his hand."
+tech_ceos2.1dd,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding an iPhone in his hand.
+tech_ceos2.1dd,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding an iPhone.
+tech_ceos2.1dd,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos2.1dd,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_final.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_final.csv
new file mode 100644
index 000000000..e717653d4
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set1_final.csv
@@ -0,0 +1,61 @@
+test_name,image_path,question,answer
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"The person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is a man named Bill Gates, who is the co-founder of Microsoft."
+tech_ceos2.1a,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man named Bill Gates holding a microphone.
+tech_ceos2.1a,Recall who is in the first image.,Recall who is in the first image.,The first image shows Bill Gates holding a cell phone.
+tech_ceos2.1a,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is the founder of Microsoft, Bill Gates. The person in the second image is also the founder of Microsoft, Bill Gates."
+tech_ceos2.1a,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not the founder of Apple. The person in the third image is a man named Bill Gates, who is the founder of Microsoft."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is the famous entrepreneur and former Apple CEO, Steve Jobs."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla."
+tech_ceos2.1b,Recall who is in the second image.,Recall who is in the second image.,Elon Musk is in the second image holding a microphone.
+tech_ceos2.1b,Recall who is in the first image.,Recall who is in the first image.,Steve Jobs is in the first image standing behind a podium.
+tech_ceos2.1b,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.",Steve Jobs is the founder of Apple and is in the first image standing behind a podium.
+tech_ceos2.1b,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,This person in the image b is Bill Gates.
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1c,Recall who is in image b.,Recall who is in image b.,The person in image b is a man holding a microphone.
+tech_ceos2.1c,Recall who is in the image a.,Recall who is in the image a.,The person in image a is Steve Jobs.
+tech_ceos2.1c,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1c,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"No, the person in the image c is not the founder of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,Recall who is in the second image.,Recall who is in the second image.,"The second image shows a man holding a microphone and standing in front of a large sign that says ""Toshiba."""
+tech_ceos2.1d,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding an iPhone.
+tech_ceos2.1d,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos2.1d,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"The person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1aa,What's the differnce between the first and second image,What's the differnce between the first and second image,The first image is of Steve Jobs holding an iPhone while the second image is of Bill Gates standing at a podium.
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is a man named Elon Musk, the founder and CEO of Tesla."
+tech_ceos2.1aa,Recall who is in the second image.,Recall who is in the second image.,"In the second image, there are two people, one of whom is Elon Musk holding a microphone."
+tech_ceos2.1aa,Recall who is in the first image.,Recall who is in the first image.,"In the first image, there is only one person, Steve Jobs, holding an iPhone."
+tech_ceos2.1aa,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos2.1aa,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Steve Jobs."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"The person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is the famous entrepreneur and former Apple CEO, Steve Jobs."
+tech_ceos2.1bb,What's the differnce between the first and second images,What's the differnce between the first and second images,"In the first image, Steve Jobs is holding up an iPhone while standing, while in the second image, he is sitting behind a podium."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla."
+tech_ceos2.1bb,Recall who is in the second image.,Recall who is in the second image.,"Elon Musk is in the second image, holding a microphone and standing behind a podium."
+tech_ceos2.1bb,Recall who is in the first image.,Recall who is in the first image.,"Steve Jobs is in the first image, holding up an iPhone."
+tech_ceos2.1bb,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Steve Jobs is the founder of Apple, and he is in the first image holding up an iPhone."
+tech_ceos2.1bb,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is Steve Jobs, the founder of Apple."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,This person in the image b is Bill Gates.
+tech_ceos2.1cc,What's the differnce between the image a and image b,What's the differnce between the image a and image b,"In image a, Steve Jobs is holding an iPhone while in image b, Bill Gates is standing behind a microphone."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is a man wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1cc,Recall who is in image b.,Recall who is in image b.,The person in image b is a man wearing a suit and tie holding a microphone.
+tech_ceos2.1cc,Recall who is in the image a.,Recall who is in the image a.,Steve Jobs is in the image a holding an iPhone.
+tech_ceos2.1cc,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1cc,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"No, the person in the image c is not the founder of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"The person in the first image is Steve Jobs, the CEO of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image is of Steve Jobs holding an iPhone while the second image is of Bill Gates standing at a podium.
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is a man named Bill Gates, who is the founder of Microsoft."
+tech_ceos2.1dd,Recall who is in the second image.,Recall who is in the second image.,The second image shows a man holding an iPhone.
+tech_ceos2.1dd,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding an iPhone.
+tech_ceos2.1dd,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple, Steve Jobs."
+tech_ceos2.1dd,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple, Bill Gates."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_best_eval.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_best_eval.csv
new file mode 100644
index 000000000..1f77dfbe4
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_best_eval.csv
@@ -0,0 +1,61 @@
+test_name,image_path,question,answer
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,Recall who is in the second image.,Recall who is in the second image.,"In the second image, Steve Jobs is standing in front of a large red Tesla logo, holding a microphone and speaking to a crowd. He is wearing a black suit and tie, and his hair is gray."
+tech_ceos2.1a,Recall who is in the first image.,Recall who is in the first image.,"In the first image, Steve Jobs is holding up an iPhone, which is white on the front and has a black screen. He is wearing a black shirt and is standing in front of a white background."
+tech_ceos2.1a,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1a,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not Steve Jobs, the founder of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla."
+tech_ceos2.1b,Recall who is in the second image.,Recall who is in the second image.,"In the second image, there is a man standing in front of a Tesla car. He is wearing a suit and tie, and he is holding a microphone."
+tech_ceos2.1b,Recall who is in the first image.,Recall who is in the first image.,"In the first image, there is a man standing in front of a Tesla car. He is wearing a black suit and tie, and he is holding a cell phone in his hand."
+tech_ceos2.1b,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.",The person in the first image is not the founder of Apple. The person in the second image is the founder of Apple.
+tech_ceos2.1b,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is the founder of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,"This person in this image b is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1c,Recall who is in image b.,Recall who is in image b.,"In image b, there are several people, including a man in a suit and tie, a woman in a black dress, and a man in a suit and tie holding a microphone."
+tech_ceos2.1c,Recall who is in the image a.,Recall who is in the image a.,"In the image a, there is a man in a suit and tie standing in front of a podium."
+tech_ceos2.1c,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple, Steve Jobs."
+tech_ceos2.1c,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"Yes, the person in the image c is the founder of Apple, Steve Jobs."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1d,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1d,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1d,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,What's the differnce between the first and second image,What's the differnce between the first and second image,"In the first image, Steve Jobs is holding up a cell phone while in the second image, he is standing at a podium."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,Recall who is in the second image.,Recall who is in the second image.,"In the second image, Steve Jobs is standing at a podium and holding a microphone. He is wearing a black suit and tie, and his hair is gray. He is speaking to a crowd of people, possibly at a conference or event."
+tech_ceos2.1aa,Recall who is in the first image.,Recall who is in the first image.,"In the first image, Steve Jobs is holding up a cell phone, which is white on the front and has a black border. He is wearing a black shirt and is standing in front of a white background."
+tech_ceos2.1aa,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1aa,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not Steve Jobs, the founder of Apple."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1bb,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image is of Steve Jobs holding up an iPhone while the second image is of Bill Gates holding up a cell phone.
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1bb,Recall who is in the second image.,Recall who is in the second image.,"The second image shows Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1bb,Recall who is in the first image.,Recall who is in the first image.,"The first image shows Steve Jobs, the co-founder of Apple, holding up an iPhone."
+tech_ceos2.1bb,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1bb,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Bill Gates, the founder of Microsoft."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in this image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,"This person in this image b is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1cc,What's the differnce between the image a and image b,What's the differnce between the image a and image b,The image a is of Steve Jobs holding a cell phone while the image b is of Bill Gates standing in front of a podium.
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in this image c is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1cc,Recall who is in image b.,Recall who is in image b.,"The person in image b is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1cc,Recall who is in the image a.,Recall who is in the image a.,"The person in the image a is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1cc,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1cc,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"No, the person in the image c is not the founder of Apple. He is the co-founder of Microsoft."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image is of Steve Jobs holding up an iPhone while the second image is of Bill Gates standing at a podium.
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,Recall who is in the second image.,Recall who is in the second image.,The second image is of Bill Gates standing at a podium.
+tech_ceos2.1dd,Recall who is in the first image.,Recall who is in the first image.,The first image is of Steve Jobs holding up an iPhone.
+tech_ceos2.1dd,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1dd,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is Bill Gates, the co-founder of Microsoft."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_final.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_final.csv
new file mode 100644
index 000000000..2060fa157
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_robustness/ours-set2_final.csv
@@ -0,0 +1,61 @@
+test_name,image_path,question,answer
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1a,Recall who is in the second image.,Recall who is in the second image.,"In the second image, Steve Jobs is standing in front of a large red Apple logo, holding a microphone and speaking to a crowd. He is wearing a black suit and tie, and his hair is gray."
+tech_ceos2.1a,Recall who is in the first image.,Recall who is in the first image.,"In the first image, Steve Jobs is holding up an iPhone, which is white on the front and back, and has a black screen. He is wearing a black shirt and is standing in front of a white background."
+tech_ceos2.1a,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1a,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is not Steve Jobs, the founder of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1b,Recall who is in the second image.,Recall who is in the second image.,"In the second image, Steve Jobs is standing in front of a large red banner with the Apple logo on it. He is holding a microphone and appears to be giving a speech or presentation."
+tech_ceos2.1b,Recall who is in the first image.,Recall who is in the first image.,"In the first image, Steve Jobs is holding up an iPhone in front of a crowd of people. He is wearing a black turtleneck, jeans, and glasses, and is smiling."
+tech_ceos2.1b,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1b,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"Yes, the person in the third image is Steve Jobs, the founder of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,"This person in the image b is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in the image c is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1c,Recall who is in image b.,Recall who is in image b.,"In image b, Steve Jobs is standing in front of a microphone and a podium, wearing a black turtleneck and jeans. He is holding a microphone and appears to be giving a speech or presentation."
+tech_ceos2.1c,Recall who is in the image a.,Recall who is in the image a.,"In the image a, Steve Jobs is standing in front of a podium, wearing a suit and tie, and holding a microphone. He is surrounded by people, and the image appears to be from a news conference or event."
+tech_ceos2.1c,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1c,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"Yes, the person in the image c is the founder of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and CEO of Apple."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1d,Recall who is in the second image.,Recall who is in the second image.,"The person in the second image is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1d,Recall who is in the first image.,Recall who is in the first image.,"The person in the first image is Steve Jobs, the co-founder of Apple."
+tech_ceos2.1d,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1d,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in the image?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1aa,What's the differnce between the first and second image,What's the differnce between the first and second image,"In the first image, Steve Jobs is holding a cell phone while in the second image, he is standing at a podium."
+tech_ceos2.1aa,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in the image?,"This person in the image is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1aa,Recall who is in the second image.,Recall who is in the second image.,"The second image shows Elon Musk standing in front of a Tesla car, holding a microphone and speaking to a crowd."
+tech_ceos2.1aa,Recall who is in the first image.,Recall who is in the first image.,The first image shows Steve Jobs holding a cell phone while standing in front of a microphone.
+tech_ceos2.1aa,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1aa,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image?,"This person is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1bb,What's the differnce between the first and second images,What's the differnce between the first and second images,"In the first image, Steve Jobs is holding up a cell phone while in the second image, he is standing at a podium."
+tech_ceos2.1bb,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1bb,Recall who is in the second image.,Recall who is in the second image.,"In the second image, Steve Jobs is holding up a cell phone, which is likely an iPhone, and is standing at a podium."
+tech_ceos2.1bb,Recall who is in the first image.,Recall who is in the first image.,"In the first image, Steve Jobs is holding up a cell phone, which is likely an iPhone, and is wearing a black turtleneck and jeans."
+tech_ceos2.1bb,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the first image is the founder of Apple."
+tech_ceos2.1bb,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this image a?,"This person in the image is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this image b?,"This person in the image b is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,What's the differnce between the image a and image b,What's the differnce between the image a and image b,"In image a, Steve Jobs is holding up a cell phone while in image b, he is standing at a podium and holding a microphone."
+tech_ceos2.1cc,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this image c?,"This person in the image c is Steve Jobs, the co-founder and former CEO of Apple."
+tech_ceos2.1cc,Recall who is in image b.,Recall who is in image b.,"In image b, Steve Jobs is standing at a podium and holding a microphone. He is wearing a black turtleneck and jeans, and he appears to be speaking to a crowd."
+tech_ceos2.1cc,Recall who is in the image a.,Recall who is in the image a.,"In the image a, Steve Jobs is holding up a cell phone, which is a black iPhone. He is wearing a black shirt and glasses, and his hair is gray."
+tech_ceos2.1cc,"Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the image a the founder of Apple? if not, which of the above images is the person the founder of Apple.","Yes, the person in the image a is the founder of Apple."
+tech_ceos2.1cc,Is the person in the image c the founder of Apple?,Is the person in the image c the founder of Apple?,"Yes, the person in the image c is the founder of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/jobs1.jpg,Who is this person in this first image?,"This person is Steve Jobs, the co-founder and CEO of Apple."
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/gate1.jpg,Who is this person in this second image?,"This person is Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,What's the differnce between the first and second images,What's the differnce between the first and second images,The first image is of Steve Jobs holding up an iPhone while the second image is of Bill Gates standing at a podium.
+tech_ceos2.1dd,./eval/eval_data/images/tech-ceo/musk1.jpg,Who is this person in this third image?,"This person is Elon Musk, the CEO of Tesla and SpaceX."
+tech_ceos2.1dd,Recall who is in the second image.,Recall who is in the second image.,"The second image is of Bill Gates, the co-founder of Microsoft."
+tech_ceos2.1dd,Recall who is in the first image.,Recall who is in the first image.,"The first image is of Steve Jobs, the co-founder of Apple."
+tech_ceos2.1dd,"Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","Is the person in the first image the founder of Apple? if not, which of the above images is the person the founder of Apple.","The person in the first image is Steve Jobs, the founder of Apple."
+tech_ceos2.1dd,Is the person in the third image the founder of Apple?,Is the person in the third image the founder of Apple?,"No, the person in the third image is Elon Musk, the founder of Tesla and SpaceX."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_best_eval.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_best_eval.csv
new file mode 100644
index 000000000..a155b18cf
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_best_eval.csv
@@ -0,0 +1,24 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/cat.png,please describe the image,"The image features a small kitten standing on top of a pile of green plants, with its paws on a leaf. The kitten appears to be curious and playful, as it is reaching up to touch the leaf with its paws. The plants are spread out across the ground, creating a lush and vibrant scene. The kitten's position on the plants adds a sense of adventure and fun to the image."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,can you describe the image,"The image features a black cat lying on a carpeted floor, with its eyes closed and its paws stretched out. The cat appears to be relaxed and comfortable in its position. The carpeted floor is a neutral background, allowing the cat to be the main focus of the image."
+cat_images3,./eval/eval_data/images/cats/british_shorthair.jpg,please describe the image,"The image features a large gray and white cat lying on a carpeted floor. The cat is stretched out and appears to be relaxing, with its paws and legs spread out. The cat's body is positioned diagonally across the floor, taking up a significant portion of the space. The carpeted floor is a light color, providing a comfortable and cozy environment for the cat to rest in."
+extreme_ironing,./eval/eval_data/images/singles/extreme_ironing.jpg,What is unusual about this image?,"The unusual aspect of this image is that a man is standing on top of a car, ironing a shirt. This is an unconventional and potentially dangerous activity, as it puts the man at risk of falling off the car and potentially injuring himself or causing damage to the car. Ironing a shirt on top of a car is not a common practice, and it is not recommended as it poses safety hazards and can lead to accidents."
+waterview,./eval/eval_data/images/singles/waterview.jpg,What are the things I should be cautious about when I visit here?,"When visiting the wooden pier over the lake, there are a few things to be cautious about:
+
+1. Safety: Be aware of your surroundings and the condition of the pier. Make sure the pier is stable and secure before walking on it.
+
+2. Weather: Check the weather forecast before visiting to avoid any sudden changes or storms that could make the pier unsafe.
+
+3. Water depth: Be cautious of the water depth around the pier, especially if you plan to swim or wade in the lake.
+
+4. Wildlife: Be mindful of any wildlife in the area, such as fish or birds, and avoid disturbing their natural habitat.
+
+5. Personal belongings: Keep your belongings secure and close to you, as theft or loss can occur in public places.
+
+6. Sun protection: If visiting during daytime, remember to apply sunscreen and wear a hat to protect yourself from the sun's harmful rays.
+
+By being aware of these factors, you can ensure a safe and enjoyable visit to the wooden pier over the lake."
+art-dog,./eval/eval_data/images/singles/202160027_b319c4166e.jpg,can you describe the image,"The image features a large, colorful flower sculpture in the shape of a dog, standing in front of a building. The flower sculpture is made up of various flowers, including pink, red, and yellow flowers, which are attached to the dog's body. The dog sculpture is quite tall, and it is positioned in a way that it appears to be walking or standing on a sidewalk. The building in the background is a large, blue structure, which adds contrast to the vibrant flower sculpture."
+funny-phone,./eval/eval_data/images/singles/1.jpg,What is funny about this image? Describe it panel by panel.,"The image is a series of four panels that show a cell phone with a blue cord attached to it. The first panel displays the cell phone with the cord, while the second panel shows the cord being cut off. In the third panel, the cord is reattached to the cell phone, and in the fourth panel, the cord is cut off again. This sequence of events is humorous because it depicts a person repeatedly cutting and reattaching the cord to the cell phone, which is an unusual and unnecessary action. The image conveys a sense of absurdity and humor, as it highlights the pointlessness of the action."
+squirrel,./eval/eval_data/images/singles/2.jpg,Why would a person find this image funny?,"The image of a squirrel holding a camera and smiling is humorous because it is an unexpected and amusing combination of elements. Squirrels are typically associated with natural environments, such as parks and forests, and are not typically known for engaging in human activities like taking photographs. The fact that the squirrel is holding a camera and smiling, as if it is enjoying the process, adds a playful and lighthearted touch to the image. This combination of elements creates a sense of humor and amusement for viewers, as it challenges their expectations and imagination."
+art-painting,./eval/eval_data/images/singles/50.jpg,Tell me about this work of art.,"The work of art is a painting that features a scene of a man walking down a street at night, with a church in the background. The man is carrying a backpack, and the street is illuminated by the moonlight. The painting has a unique style, with the moon appearing as a large, yellow sun in the sky. This artistic choice adds a surreal and dreamlike quality to the scene, making it appear as if the man is walking through a fantasy world. The painting is a beautiful representation of a nighttime scene, capturing the tranquility and serenity of the moment."
diff --git a/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_final.csv b/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_final.csv
new file mode 100644
index 000000000..2b72b4814
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/eval/results/eval_single/ours-single_final.csv
@@ -0,0 +1,16 @@
+test_name,image_path,question,answer
+cat_images1,./eval/eval_data/images/cats/cat.png,please describe the image,"The image shows a close-up of a small kitten standing in a field of green plants. The kitten is looking upwards with a curious expression on its face, surrounded by the lush vegetation. The plants around the kitten come in various sizes and shapes, creating a picturesque scene of nature."
+cat_images2,./eval/eval_data/images/cats/1806905748_adb926a0a0.jpg,can you describe the image,"The image features a black cat lying on the floor, specifically on a yellow area, possibly a rug. The cat appears to be relaxed and laying on its side, with its eyes closed. The cat occupies a significant portion of the frame, drawing attention to its presence."
+cat_images3,./eval/eval_data/images/cats/british_shorthair.jpg,please describe the image,"The image features a gray and white cat lying on the floor next to a piece of carpeting. The cat is stretched out, with its body resting on the floor and one paw reaching up onto the carpet. The cat appears to be relaxed and enjoying its rest."
+extreme_ironing,./eval/eval_data/images/singles/extreme_ironing.jpg,What is unusual about this image?,"In this image, it is unusual to see a man doing laundry by hanging clothes on a clothes rack attached to a parked yellow taxi in the middle of the street. Typically, laundry would be done in a more designated or safe area, such as a laundromat or a backyard, to avoid potential hazards and obstructing traffic. The man's choice to do laundry in this manner raises concerns about safety and the flow of traffic, as it could cause distractions for drivers and pedestrians and pose a risk to both the man and others in the area."
+waterview,./eval/eval_data/images/singles/waterview.jpg,What are the things I should be cautious about when I visit here?,"When visiting this picturesque dock over a calm lake surrounded by trees, there are a few things to keep in mind to ensure a safe and enjoyable experience. First, be aware of the water conditions and always use caution when entering the lake, as it may have hidden obstacles or sudden changes in depth. Second, make sure to wear appropriate footwear to prevent slipping or falling off the dock. Third, be mindful of the weather and avoid visiting during stormy conditions, as lightning can be dangerous. Lastly, practice water safety rules, such as swimming with a buddy, staying within your swimming abilities, and not venturing too far from the shore or dock. Enjoy the serene atmosphere and the beautiful view of the mountains in the background while being cautious and respectful of the environment."
+art-dog,./eval/eval_data/images/singles/202160027_b319c4166e.jpg,can you describe the image,"The image features a large, colorful clock tower in the background, with a man standing in front of it. In the foreground, there is a unique sculpture of a dog made out of flowers, which is the main focal point of the scene. The sculpture covers a significant portion of the image, and its vibrant colors add an artistic touch to the scene. The clock tower and the man in the background provide context and a sense of place, while the flower dog sculpture captures the viewer's attention with its creativity and beauty."
+funny-phone,./eval/eval_data/images/singles/1.jpg,What is funny about this image? Describe it panel by panel.,"The image is a collage of three panels, each showing different aspects of the humor in the scene.
+
+Panel 1: A cell phone is being held by someone, with its charger still attached to a blue block. The person seems to be trying to remove the charger from the block, possibly out of frustration or curiosity.
+
+Panel 2: A close-up of the blue block, which appears to be a deactivated electronic device adapter or a device blocker, is shown. The person might be trying to figure out why the charger won't fit or if the blue block is preventing it from working.
+
+Panel 3: The cell phone is now disconnected from the charger, and the person seems to have successfully detached it from the blue block. The situation is humorous as it depicts a common everyday activity, like charging a phone, with an added element of confusion and challenge due to the presence of the blue block."
+squirrel,./eval/eval_data/images/singles/2.jpg,Why would a person find this image funny?,"A person might find this image funny because it features a playful depiction of a squirrel holding a camera. Squirrels are typically seen as small, cute, and harmless animals, and their interaction with human objects, such as a camera, adds an element of humor to the scene. The idea of a squirrel being interested in photography or posing with a camera can evoke amusement and lightheartedness, as it is an unexpected and humorous twist on the usual behavior of squirrels. This whimsical combination of a familiar animal and an object associated with human activities can elicit laughter and amusement."
+art-painting,./eval/eval_data/images/singles/50.jpg,Tell me about this work of art.,"The work of art is a unique combination of a painting and a print featuring the famous character from the comics, Spider-Man. In this piece, Spider-Man is depicted as the ""Sgt. Star Platinum"" character from the series ""The Devil Manga."" The painting or print has a vintage or old-school style, and it is set against a night sky with stars. The scene also includes a village and a house, which adds depth and context to the overall composition. This artwork is likely to appeal to fans of comics, street art, and visual storytelling."
diff --git a/applications/DeepSpeed-VisualChat/helper/README.md b/applications/DeepSpeed-VisualChat/helper/README.md
new file mode 100644
index 000000000..bd32b54e8
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/helper/README.md
@@ -0,0 +1,2 @@
+# QWen-VL's Vision Encoder
+The extract_qwen_vl.py can be used to extract the vision encoder from QWen-VL. After extraction, you can find other necessary files in the [folder](./qwen_clip).
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/helper/extract_qwen_vl.py b/applications/DeepSpeed-VisualChat/helper/extract_qwen_vl.py
new file mode 100644
index 000000000..12808b87c
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/helper/extract_qwen_vl.py
@@ -0,0 +1,14 @@
+from transformers import AutoModelForCausalLM
+import torch
+
+PATH = "Qwen/Qwen-VL-Chat"
+
+model = AutoModelForCausalLM.from_pretrained(PATH, device_map="cuda", trust_remote_code=True).eval()
+
+state_dict = model.state_dict()
+save_dict = {}
+for k,v in state_dict.items():
+ if 'visual' in k:
+ if 'transformer.visual.proj' not in k: # we don't need the proj layer
+ save_dict[k.replace('transformer.visual.', '')] = v
+torch.save(save_dict, './qwen_clip/pytorch_model.bin')
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/helper/qwen_clip/config.json b/applications/DeepSpeed-VisualChat/helper/qwen_clip/config.json
new file mode 100755
index 000000000..357bb8280
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/helper/qwen_clip/config.json
@@ -0,0 +1,180 @@
+{
+ "_commit_hash": null,
+ "architectures": [
+ "CLIPModel"
+ ],
+ "initializer_factor": 1.0,
+ "logit_scale_init_value": 2.6592,
+ "model_type": "clip",
+ "projection_dim": 1280,
+ "text_config": {
+ "_name_or_path": "",
+ "add_cross_attention": false,
+ "architectures": null,
+ "attention_dropout": 0.0,
+ "bad_words_ids": null,
+ "begin_suppress_tokens": null,
+ "bos_token_id": 0,
+ "chunk_size_feed_forward": 0,
+ "cross_attention_hidden_size": null,
+ "decoder_start_token_id": null,
+ "diversity_penalty": 0.0,
+ "do_sample": false,
+ "dropout": 0.0,
+ "early_stopping": false,
+ "encoder_no_repeat_ngram_size": 0,
+ "eos_token_id": 2,
+ "exponential_decay_length_penalty": null,
+ "finetuning_task": null,
+ "forced_bos_token_id": null,
+ "forced_eos_token_id": null,
+ "hidden_act": "gelu",
+ "hidden_size": 1280,
+ "id2label": {
+ "0": "LABEL_0",
+ "1": "LABEL_1"
+ },
+ "initializer_factor": 1.0,
+ "initializer_range": 0.02,
+ "intermediate_size": 5120,
+ "is_decoder": false,
+ "is_encoder_decoder": false,
+ "label2id": {
+ "LABEL_0": 0,
+ "LABEL_1": 1
+ },
+ "layer_norm_eps": 1e-05,
+ "length_penalty": 1.0,
+ "max_length": 20,
+ "max_position_embeddings": 77,
+ "min_length": 0,
+ "model_type": "clip_text_model",
+ "no_repeat_ngram_size": 0,
+ "num_attention_heads": 20,
+ "num_beam_groups": 1,
+ "num_beams": 1,
+ "num_hidden_layers": 32,
+ "num_return_sequences": 1,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "output_scores": false,
+ "pad_token_id": 1,
+ "prefix": null,
+ "problem_type": null,
+ "pruned_heads": {},
+ "remove_invalid_values": false,
+ "repetition_penalty": 1.0,
+ "return_dict": true,
+ "return_dict_in_generate": false,
+ "sep_token_id": null,
+ "suppress_tokens": null,
+ "task_specific_params": null,
+ "temperature": 1.0,
+ "tf_legacy_loss": false,
+ "tie_encoder_decoder": false,
+ "tie_word_embeddings": true,
+ "tokenizer_class": null,
+ "top_k": 50,
+ "top_p": 1.0,
+ "torch_dtype": null,
+ "torchscript": false,
+ "transformers_version": "4.24.0",
+ "typical_p": 1.0,
+ "use_bfloat16": false,
+ "vocab_size": 49408
+ },
+ "text_config_dict": {
+ "hidden_act": "gelu",
+ "hidden_size": 1280,
+ "intermediate_size": 5120,
+ "num_attention_heads": 20,
+ "num_hidden_layers": 32
+ },
+ "torch_dtype": "float32",
+ "transformers_version": null,
+ "vision_config": {
+ "_name_or_path": "",
+ "add_cross_attention": false,
+ "architectures": null,
+ "attention_dropout": 0.0,
+ "bad_words_ids": null,
+ "begin_suppress_tokens": null,
+ "bos_token_id": null,
+ "chunk_size_feed_forward": 0,
+ "cross_attention_hidden_size": null,
+ "decoder_start_token_id": null,
+ "diversity_penalty": 0.0,
+ "do_sample": false,
+ "dropout": 0.0,
+ "early_stopping": false,
+ "encoder_no_repeat_ngram_size": 0,
+ "eos_token_id": null,
+ "exponential_decay_length_penalty": null,
+ "finetuning_task": null,
+ "forced_bos_token_id": null,
+ "forced_eos_token_id": null,
+ "hidden_act": "gelu",
+ "hidden_size": 1664,
+ "id2label": {
+ "0": "LABEL_0",
+ "1": "LABEL_1"
+ },
+ "image_size": 224,
+ "initializer_factor": 1.0,
+ "initializer_range": 0.02,
+ "intermediate_size": 8192,
+ "is_decoder": false,
+ "is_encoder_decoder": false,
+ "label2id": {
+ "LABEL_0": 0,
+ "LABEL_1": 1
+ },
+ "layer_norm_eps": 1e-05,
+ "length_penalty": 1.0,
+ "max_length": 20,
+ "min_length": 0,
+ "model_type": "clip_vision_model",
+ "no_repeat_ngram_size": 0,
+ "num_attention_heads": 16,
+ "num_beam_groups": 1,
+ "num_beams": 1,
+ "num_channels": 3,
+ "num_hidden_layers": 48,
+ "num_return_sequences": 1,
+ "output_attentions": false,
+ "output_hidden_states": false,
+ "output_scores": false,
+ "pad_token_id": null,
+ "patch_size": 14,
+ "prefix": null,
+ "problem_type": null,
+ "pruned_heads": {},
+ "remove_invalid_values": false,
+ "repetition_penalty": 1.0,
+ "return_dict": true,
+ "return_dict_in_generate": false,
+ "sep_token_id": null,
+ "suppress_tokens": null,
+ "task_specific_params": null,
+ "temperature": 1.0,
+ "tf_legacy_loss": false,
+ "tie_encoder_decoder": false,
+ "tie_word_embeddings": true,
+ "tokenizer_class": null,
+ "top_k": 50,
+ "top_p": 1.0,
+ "torch_dtype": null,
+ "torchscript": false,
+ "transformers_version": "4.24.0",
+ "typical_p": 1.0,
+ "use_bfloat16": false
+ },
+ "vision_config_dict": {
+ "hidden_act": "gelu",
+ "hidden_size": 1664,
+ "intermediate_size": 8192,
+ "num_attention_heads": 16,
+ "num_hidden_layers": 48,
+ "patch_size": 14
+ }
+}
diff --git a/applications/DeepSpeed-VisualChat/helper/qwen_clip/preprocessor_config.json b/applications/DeepSpeed-VisualChat/helper/qwen_clip/preprocessor_config.json
new file mode 100755
index 000000000..89a5d584a
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/helper/qwen_clip/preprocessor_config.json
@@ -0,0 +1,19 @@
+{
+ "crop_size": 448,
+ "do_center_crop": true,
+ "do_normalize": true,
+ "do_resize": true,
+ "feature_extractor_type": "CLIPFeatureExtractor",
+ "image_mean": [
+ 0.48145466,
+ 0.4578275,
+ 0.40821073
+ ],
+ "image_std": [
+ 0.26862954,
+ 0.26130258,
+ 0.27577711
+ ],
+ "resample": 3,
+ "size": 448
+}
diff --git a/applications/DeepSpeed-VisualChat/requirements.txt b/applications/DeepSpeed-VisualChat/requirements.txt
new file mode 100644
index 000000000..ae0440617
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/requirements.txt
@@ -0,0 +1,11 @@
+datasets>=2.8.0
+sentencepiece>=0.1.97
+protobuf==3.20.3
+accelerate>=0.15.0
+open_clip_torch
+deepspeed>=0.10.3
+einops
+einops_exts
+transformers==4.33.3
+transformers_stream_generator
+termcolor
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/training/README.md b/applications/DeepSpeed-VisualChat/training/README.md
new file mode 100644
index 000000000..dd4e58d26
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/training/README.md
@@ -0,0 +1,40 @@
+
+Figure 1. On the left is a DeepSpeed-VisualChat model, featuring an innovative attention design. On the right is an example of DeepSpeed-VisualChat.
+
+
+
+
+
+
+
+# 🐕 Vision-Lanauge Finetuning
+Vision-Lanauge finetuning (or insturction finetuning) is very similar to standard large language model finetuning. The differences are summarized as follows:
+* It requires an extra vision encoder
+* It requires high quality text-image data pairs
+* It needs a linear projection layer that connects the visual encoder and LLM
+
+
+## 🏃 How to train the model
+Remember to prepare you data first based on [tutorial](../README.md). If you want to use 448x448 resolution images, you need go to [helper](../helper) for more information. Otherwise, you can simply run
+```bash
+ training_scripts/run_7b.sh
+ ```
+
+which shows an example to fine-tune a LLaMA-2-7B LLM together with a CLIP-large visual encoder and a linear projection layer that connects this visual encoder and LLM.
+
+
+## 👀 A few examples
+Please refer to [Chat](../chat/README.md) or [Eval](../eval/README.md) for evaluation.
+
+
+
+
+
+
+
+-----
+
+
+
+
+
+
+------
+
+
+
+
+
diff --git a/applications/DeepSpeed-VisualChat/training/main.py b/applications/DeepSpeed-VisualChat/training/main.py
new file mode 100755
index 000000000..c345f9832
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/training/main.py
@@ -0,0 +1,442 @@
+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+import argparse
+import os
+import math
+import sys
+import numpy as np
+import random
+
+import torch
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+
+from transformers import (
+ SchedulerType,
+ get_scheduler,
+ AutoTokenizer
+)
+
+import deepspeed
+from transformers import AdamW
+sys.path.append(
+ os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
+from utils.data import build_dataset, DataCollatorPadToMaxLen, split_dataset, shuffle_dataset
+from utils.utils import print_rank_0, to_device, save_hf_format, set_random_seed, get_all_reduce_mean, get_optimizer_grouped_parameters, save_zero_three_model
+from utils.ds_utils import get_train_ds_config
+from utils.module.lora import convert_linear_layer_to_lora, only_optimize_lora_parameters, fuse_lora, unfuse_lora
+from utils.model import create_dsvl_model_and_transforms
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description=
+ "Finetune a transformers model on a multi-modal task")
+
+ parser.add_argument('--data_path',
+ type=str,
+ default='./data/',
+ help='Where the training data are stored.')
+
+ parser.add_argument('--data_debug_path',
+ type=str,
+ default=None,
+ help='If provided, will save 10 training samples'
+ 'to the path for debug purpose.')
+
+ parser.add_argument(
+ "--data_train_split_ratio",
+ type=float,
+ default=0.9,
+ help="Ratio of dataset to be splitted as train data. The remaining becomes eval data.",
+ )
+ parser.add_argument('--dataset_names',
+ nargs='*',
+ default=['minigpt4'],
+ help='Name of training dataset(s) to be used. Accepted format:'
+ '1) a single dataset name, 2) multiple dataset names in the'
+ 'form: dataset1 dataset2 ...')
+
+ parser.add_argument('--dataset_samples',
+ nargs='*',
+ default=['all'],
+ help='How many samples do we use from each dataset.'
+ 'Should be either a integer number or string all which'
+ 'means use all samples. For example: all 512 means'
+ 'using all samples form first data and 512 samples'
+ 'from second data')
+
+ parser.add_argument('--dataset_concatenate_samples',
+ nargs='*',
+ default=[1],
+ help='How many samples do we concatenate from each dataset.'
+ 'Should be either a integer number or string. 1 which'
+ 'means use 1 sample for each datapoint')
+
+ parser.add_argument(
+ "--max_num_image_per_sample",
+ type=int,
+ default=8,
+ help="The maximum number of images per sample.",
+ )
+ parser.add_argument(
+ "--per_device_train_batch_size",
+ type=int,
+ default=2,
+ help="Batch size (per device) for the training dataloader.",
+ )
+ parser.add_argument(
+ "--per_device_eval_batch_size",
+ type=int,
+ default=2,
+ help="Batch size (per device) for the evaluation dataloader.",
+ )
+ parser.add_argument(
+ "--max_seq_len",
+ type=int,
+ default=4096,
+ help="The maximum sequence length, note that image tokens are included.",
+ )
+ parser.add_argument(
+ "--learning_rate",
+ type=float,
+ default=1e-3,
+ help=
+ "Initial learning rate (after the potential warmup period) to use.",
+ )
+ parser.add_argument(
+ "--learning_rate_pretraining_components",
+ type=float,
+ default=0,
+ help=
+ "Initial learning rate for pre-trained weight, e.g., embedding (after the potential warmup period) to use.",
+ )
+ parser.add_argument("--weight_decay",
+ type=float,
+ default=0.,
+ help="Weight decay to use.")
+ parser.add_argument("--num_train_epochs",
+ type=int,
+ default=6,
+ help="Total number of training epochs to perform.")
+ parser.add_argument(
+ "--gradient_accumulation_steps",
+ type=int,
+ default=1,
+ help=
+ "Number of updates steps to accumulate before performing a backward/update pass.",
+ )
+ parser.add_argument(
+ "--lr_scheduler_type",
+ type=SchedulerType,
+ default="cosine",
+ help="The scheduler type to use.",
+ choices=[
+ "linear", "cosine", "cosine_with_restarts", "polynomial",
+ "constant", "constant_with_warmup"
+ ],
+ )
+ parser.add_argument(
+ "--num_warmup_steps",
+ type=float,
+ default=0,
+ help="Number of steps (>1) or ratios (<=1) for the warmup in the lr scheduler.")
+ parser.add_argument("--output_dir",
+ type=str,
+ default=None,
+ help="Where to store the model.")
+ parser.add_argument("--seed",
+ type=int,
+ default=1234,
+ help="A seed for reproducible training.")
+ parser.add_argument("--local_rank",
+ type=int,
+ default=-1,
+ help="local_rank for distributed training on gpus")
+ parser.add_argument('--gradient_checkpointing',
+ action='http://github.com/store_true',
+ help='Enable HF gradient checkpointing for model.')
+ parser.add_argument(
+ "--lm_model_name_or_path",
+ type=str,
+ help=
+ "Path to pretrained model or model identifier from huggingface.co/models.",
+ required=True,
+ )
+ parser.add_argument("--vision_model_name_or_path", default="openai/clip-vit-large-patch14", type=str)
+ parser.add_argument(
+ "--enable_mmca_attention",
+ action='http://github.com/store_true',
+ help="enable the new proposed attn, which is similar to cross attention",
+ )
+ parser.add_argument(
+ "--vis_proj",
+ type=str,
+ default='baseline',
+ help="[baseline, vit, or perceiver], used to projection vision feature to LLM embedding",
+ )
+ # deepspeed features
+ parser.add_argument(
+ '--zero_stage',
+ type=int,
+ default=0,
+ help='ZeRO optimization stage for Actor model (and clones).')
+ parser.add_argument(
+ "--precision",
+ type=str,
+ choices=["fp16", "bf16"],
+ default="fp16",
+ help=
+ "FP16 or BF16 precision. FP16 is recommended for typical use cases. BF16 is good for large models",
+ )
+ parser.add_argument('--enable_tensorboard',
+ action='http://github.com/store_true',
+ help='Enable tensorboard logging')
+ ## LoRA for efficient training setting
+ parser.add_argument("--lang_lora_dim",
+ type=int,
+ default=0,
+ help="Use LoRA for fine-tuning language decoder (> 0).")
+ parser.add_argument("--lang_lora_module_name",
+ type=str,
+ default="model.layers.",
+ help="The scope name of the target LoRA parameters.")
+ parser.add_argument("--vis_lora_dim",
+ type=int,
+ default=0,
+ help="Use LoRA for fine-tuning visual encoder (> 0).")
+ parser.add_argument("--vis_lora_module_name",
+ type=str,
+ default="encoder.layers.",
+ help="The scope name of the target LoRA parameters.")
+ parser.add_argument('--only_optimize_lora',
+ action='http://github.com/store_true',
+ help='Only optimize the LoRA parameters.')
+
+
+ parser = deepspeed.add_config_arguments(parser)
+ args = parser.parse_args()
+
+ if args.learning_rate_pretraining_components == 0.0:
+ # if we do not provide special learning rate, mainly for embedding, the same lr is applied
+ args.learning_rate_pretraining_components = args.learning_rate
+ assert args.num_warmup_steps >= 0, "--num_warmup_steps must be >= 0"
+ if 'qwen' in args.vision_model_name_or_path.lower():
+ assert args.vis_proj == 'baseline', "qwen's model only support baseline vis_proj as it has the perceiver module inside"
+ return args
+
+
+def main():
+ args = parse_args()
+
+ if args.local_rank == -1:
+ device = torch.device("cuda")
+ else:
+ torch.cuda.set_device(args.local_rank)
+ device = torch.device("cuda", args.local_rank)
+ # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+ deepspeed.init_distributed()
+
+ args.global_rank = torch.distributed.get_rank()
+
+ ds_config = get_train_ds_config(args, offload=False,
+ stage=args.zero_stage)
+ ds_config[
+ 'train_micro_batch_size_per_gpu'] = args.per_device_train_batch_size
+ ds_config[
+ 'train_batch_size'] = args.per_device_train_batch_size * torch.distributed.get_world_size(
+ ) * args.gradient_accumulation_steps
+
+ # If passed along, set the training seed now.
+ set_random_seed(args.seed)
+
+ torch.distributed.barrier()
+ tokenizer = AutoTokenizer.from_pretrained(args.lm_model_name_or_path,
+ fast_tokenizer=True)
+ tokenizer.padding_side = 'right'
+ model, image_processor, tokenizer = create_dsvl_model_and_transforms(
+ text_tokenizer=tokenizer,
+ args=args,
+ ds_config=ds_config)
+ if args.lang_lora_dim > 0:
+ model.lang_decoder = convert_linear_layer_to_lora(model.lang_decoder, args.lang_lora_module_name, args.lang_lora_dim)
+ if args.only_optimize_lora:
+ model.lang_decoder = only_optimize_lora_parameters(model.lang_decoder)
+
+ if args.vis_lora_dim > 0:
+ model.vis_encoder = convert_linear_layer_to_lora(model.vis_encoder, args.vis_lora_module_name, args.vis_lora_dim)
+ if args.only_optimize_lora:
+ model.vis_encoder = only_optimize_lora_parameters(model.vis_encoder)
+
+ print_rank_0(model, args.global_rank)
+
+ # Prepare the data
+ if len(args.dataset_samples) < len(args.dataset_names):
+ assert len(args.dataset_samples) == 1, "when args.dataset_samples is not the same length as args.dataset_names, it should be only one number"
+ args.dataset_samples = [args.dataset_samples[0]] * len(args.dataset_names)
+ if len(args.dataset_concatenate_samples) < len(args.dataset_names):
+ assert len(args.dataset_concatenate_samples) == 1, "when args.dataset_concatenate_samples is not the same length as args.dataset_names, it should be only one number"
+ args.dataset_concatenate_samples = [args.dataset_concatenate_samples[0]] * len(args.dataset_names)
+ # convert to int
+ args.dataset_concatenate_samples = [int(i) for i in args.dataset_concatenate_samples]
+
+ dataset = build_dataset(
+ args.data_path,
+ args.data_debug_path,
+ args.dataset_names,
+ args.dataset_samples,
+ args.dataset_concatenate_samples,
+ args.max_num_image_per_sample,
+ vis_processor=image_processor,
+ tokenizer=tokenizer,
+ )
+ # split the dataset into train and evaluation
+ total_data = len(dataset)
+ np_rng = np.random.RandomState(seed=args.seed)
+ dataset = shuffle_dataset(dataset, np_rng)
+ train_dataset, eval_dataset = split_dataset(dataset, args.data_train_split_ratio)
+
+ train_dataloader = DataLoader(
+ train_dataset,
+ batch_size=args.per_device_train_batch_size,
+ sampler=DistributedSampler(train_dataset, shuffle=True, drop_last=True),
+ collate_fn=DataCollatorPadToMaxLen(args.max_seq_len, tokenizer.pad_token_id),
+ )
+
+ eval_dataloader = DataLoader(
+ eval_dataset,
+ batch_size=args.per_device_eval_batch_size,
+ sampler=DistributedSampler(eval_dataset, shuffle=False),
+ collate_fn=DataCollatorPadToMaxLen(args.max_seq_len, tokenizer.pad_token_id),
+ )
+
+ # Split weights in two groups, one with weight decay and the other not.
+ optimizer_grouped_parameters = get_optimizer_grouped_parameters(
+ model, args.weight_decay, small_lr=args.learning_rate_pretraining_components)
+
+ optimizer = AdamW(optimizer_grouped_parameters,
+ lr=args.learning_rate,
+ betas=(0.9, 0.95))
+
+ num_update_steps_per_epoch = math.ceil(
+ len(train_dataloader) / args.gradient_accumulation_steps)
+ if args.num_warmup_steps <= 1:
+ args.num_warmup_steps = int(args.num_warmup_steps * args.num_train_epochs * num_update_steps_per_epoch)
+ else:
+ args.num_warmup_steps = int(args.num_warmup_steps)
+ lr_scheduler = get_scheduler(
+ name=args.lr_scheduler_type,
+ optimizer=optimizer,
+ num_warmup_steps=args.num_warmup_steps,
+ num_training_steps=args.num_train_epochs * num_update_steps_per_epoch,
+ )
+
+ model, optimizer, _, lr_scheduler = deepspeed.initialize(
+ model=model,
+ optimizer=optimizer,
+ args=args,
+ config=ds_config,
+ lr_scheduler=lr_scheduler,
+ dist_init_required=True)
+
+ start_epoch = 0
+ # let load checkpoint
+ if os.path.exists(os.path.join(args.output_dir, 'latest')):
+ # we have the deepspeed chekpoint so it is a resumed job
+ # TODO: after loading the ckpt, the global step is not loaded. Need to ask Tunji/Ammar for help.
+ _, client_state = model.load_checkpoint(args.output_dir)
+ start_epoch = client_state['epoch']
+ best_loss = client_state['best_loss']
+ random.setstate(client_state['random_rng_state'])
+ np.random.set_state(client_state['np_rng_state'])
+ torch.set_rng_state(client_state['torch_rng_state'])
+ torch.cuda.set_rng_state(client_state['torch_cuda_rng_state'])
+
+ if args.gradient_checkpointing:
+ model.gradient_checkpointing_enable()
+
+ def evaluation(model, eval_dataloader):
+ model.eval()
+ acc_loss = 0
+ for step, batch in enumerate(eval_dataloader):
+ with torch.no_grad():
+ batch = to_device(batch, device)
+ loss = model(
+ batch["image"].half() ,
+ batch["input_ids"],
+ attention_mask=batch["attention_mask"],
+ input_labels=batch["labels"],
+ image_num=batch["image_num"],
+ )[0]
+ acc_loss += loss
+ model.train()
+ acc_loss = get_all_reduce_mean(acc_loss).item()
+ ave_loss = acc_loss / (step + 1)
+ print_rank_0(f"the eval average_loss: {ave_loss}", args.global_rank)
+ return ave_loss
+
+ # Train!
+ if start_epoch == 0:
+ print_rank_0("***** Before training *****", args.global_rank)
+ evaluation(model, eval_dataloader)
+ best_loss = 1e6
+
+ print_rank_0("***** Running training *****", args.global_rank)
+ for epoch in range(start_epoch, args.num_train_epochs):
+ print_rank_0(
+ f"Beginning of Epoch {epoch+1}/{args.num_train_epochs}, Total Micro Batches {len(train_dataloader)}",
+ args.global_rank)
+ model.train()
+ acc_loss = 0
+ for step, batch in enumerate(train_dataloader):
+ batch = to_device(batch, device) #torch.size(1, 3, 224, 224]) #torch.Size([1, 1, 3, 224, 224])
+ images = batch["image"].half()
+ input_ids = batch["input_ids"]
+ attention_mask = batch["attention_mask"]
+ labels = batch["labels"]
+ loss = model(
+ images,
+ input_ids,
+ attention_mask=attention_mask,
+ input_labels=labels,
+ image_num=batch["image_num"],
+ )[0]
+ acc_loss += loss.detach().clone()
+ model.backward(loss)
+ model.step()
+ model.tput_timer.update_epoch_count()
+ acc_loss = get_all_reduce_mean(acc_loss).item()
+ print_rank_0(f"Epoch {epoch+1}, the average_loss: {acc_loss/step}", args.global_rank)
+ eval_loss = evaluation(model, eval_dataloader)
+
+
+ if eval_loss < best_loss:
+ best_loss = eval_loss
+
+ model = fuse_lora(model)
+ if args.global_rank == 0:
+ save_hf_format(model, tokenizer, args, f'epoch-{epoch}')
+ if args.zero_stage == 3:
+ # For zero stage 3, each gpu only has a part of the model, so we need a special save function
+ save_zero_three_model(model,
+ args.global_rank,
+ args.output_dir,
+ zero_stage=args.zero_stage,
+ sub_folder=f'epoch-{epoch}')
+ model = unfuse_lora(model)
+ # save deepspeed zero checkpoint so we can resume training if needed
+ client_state = {
+ 'random_rng_state': random.getstate(),
+ 'np_rng_state': np.random.get_state(),
+ 'torch_rng_state': torch.get_rng_state(),
+ 'torch_cuda_rng_state': torch.cuda.get_rng_state(),
+ 'epoch': epoch + 1, # start from next epoch
+ 'best_loss': best_loss,
+ }
+ model.save_checkpoint(args.output_dir, client_state=client_state) # save to the latest
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/training/training_scripts/run_7b.sh b/applications/DeepSpeed-VisualChat/training/training_scripts/run_7b.sh
new file mode 100755
index 000000000..de7019536
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/training/training_scripts/run_7b.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+VISION_MODEL=openai/clip-vit-large-patch14
+LLM=meta-llama/Llama-2-7b
+
+
+
+EPOCH=6
+ZERO_STAGE=3
+lr=1e-3
+
+DATA_PATH=./data
+DATA="llava llava_dial otter_mimicit_cgd otter_mimicit_sd otter_mimicit_sn otter_mimicit_tvc otter_mimicit_vst llava_otter_blend sparkles_dialogue"
+DATA_SAMPLE="all"
+IMAGE_PER_SAMPLE="3 2 1 1 1 1 1 1 1"
+
+DATA_CONCATE="${DATA// /_}"
+DATA_SAMPLE_CONCATE="${DATA_SAMPLE// /_}"
+IMAGE_CONCATE="${IMAGE_PER_SAMPLE// /_}"
+#
+
+OUTPUT_Base=./output/
+
+OUTPUT_Dir=Epoch${EPOCH}_LR${lr}_data_${DATA_CONCATE}_${DATA_SAMPLE_CONCATE}_${IMAGE_CONCATE}
+
+OUTPUT=${OUTPUT_Base}${OUTPUT_Dir}
+
+if [ "$ZERO_STAGE" == "" ]; then
+ ZERO_STAGE=0
+fi
+
+mkdir -p $OUTPUT
+mkdir -p ./log/$OUTPUT_Dir/
+
+# we assume the batch size is 128, which means Num_GPU * per_device_train_batch_size * gradient_accumulation_steps
+deepspeed main.py --max_seq_len 4096 \
+ --data_path ${DATA_PATH} \
+ --dataset_names ${DATA} --dataset_samples ${DATA_SAMPLE} --dataset_concatenate_samples ${IMAGE_PER_SAMPLE} --max_num_image_per_sample 8 \
+ --lm_model_name_or_path ${LLM} \
+ --vision_model_name_or_path ${VISION_MODEL} \
+ --gradient_checkpointing --vis_proj baseline \
+ --gradient_accumulation_steps 1 --zero_stage $ZERO_STAGE --learning_rate $lr --num_warmup_steps 0.1 \
+ --per_device_train_batch_size 1 --per_device_eval_batch_size 2 --deepspeed --output_dir $OUTPUT --num_train_epochs ${EPOCH} --enable_mmca_attention --enable_tensorboard
\ No newline at end of file
diff --git a/applications/DeepSpeed-VisualChat/utils/data/DST.py b/applications/DeepSpeed-VisualChat/utils/data/DST.py
new file mode 100644
index 000000000..cf5880f3f
--- /dev/null
+++ b/applications/DeepSpeed-VisualChat/utils/data/DST.py
@@ -0,0 +1,139 @@
+from typing import Iterable
+import random
+import numpy as np
+## the following codes are adopted from https://github.com/haotian-liu/LLaVA
+## the following codes are adopted from https://github.com/open-mmlab/Multimodal-GPT
+## the following codes are adopted from https://github.com/Luodian/Otter/
+
+# deepspeed template
+
+DEFAULT_SYSTEM_TOKEN="### System instuction:"
+DEFAULT_PROMPT = f"You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.\n\n"
+
+DEFAULT_IMAGE_TOKEN = "
++deepspeed all_reduce.py --device cpu ++ # Adding Communication Benchmarks diff --git a/benchmarks/communication/all_gather.py b/benchmarks/communication/all_gather.py index 7a34c0d1c..76c4f3b1e 100644 --- a/benchmarks/communication/all_gather.py +++ b/benchmarks/communication/all_gather.py @@ -16,7 +16,10 @@ # Run all_gather and print metrics -def timed_all_gather(input, output, args): +def timed_all_gather(input, output, start_event, end_event, args): + if args.device == "cpu": + print_rank_0(f"No Event support on CPU to measure time for now") + return if args.dist == 'torch': import torch.distributed as dist @@ -33,11 +36,12 @@ def timed_all_gather(input, output, args): sync_all() # time the actual comm op trials times and average it - pre = time.perf_counter() + start_event.record() for i in range(args.trials): all_gather_func(output, input, group=None, async_op=args.async_op) + end_event.record() sync_all() - duration = time.perf_counter() - pre + duration = start_event.elapsed_time(end_event) / 1000 # maintain and clean performance data avg_duration = duration / args.trials @@ -63,6 +67,16 @@ def run_all_gather(local_rank, args): global_rank = dist.get_rank() world_size = dist.get_world_size() + if args.device == "xpu": + start_event = torch.xpu.Event(enable_timing=True) + end_event = torch.xpu.Event(enable_timing=True) + elif args.device == "cpu": + start_event = torch.cpu.Event() + end_event = torch.cpu.Event() + else: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + if args.scan: # Create list of message sizes M_LIST = [] @@ -92,7 +106,7 @@ def run_all_gather(local_rank, args): else: raise e sync_all() - timed_all_gather(input, output, args) + timed_all_gather(input, output, start_event, end_event, args) else: # all_gather_into_tensor saves memory if ((args.dist == 'torch' or args.dist == 'deepspeed') and dist.has_all_gather_into_tensor()): @@ -126,7 +140,7 @@ def run_all_gather(local_rank, args): raise e sync_all() - timed_all_gather(input, output, args) + timed_all_gather(input, output, start_event, end_event, args) if __name__ == "__main__": diff --git a/benchmarks/communication/all_reduce.py b/benchmarks/communication/all_reduce.py index a474a704f..41c3116ee 100644 --- a/benchmarks/communication/all_reduce.py +++ b/benchmarks/communication/all_reduce.py @@ -14,7 +14,10 @@ from deepspeed.accelerator import get_accelerator -def timed_all_reduce(input, args): +def timed_all_reduce(input, start_event, end_event, args): + if args.device == "cpu": + print_rank_0(f"No Event support on CPU to measure time for now") + return if args.dist == 'torch': import torch.distributed as dist elif args.dist == 'deepspeed': @@ -27,11 +30,12 @@ def timed_all_reduce(input, args): sync_all() # time the actual comm op trials times and average it - pre = time.perf_counter() + start_event.record() for i in range(args.trials): dist.all_reduce(input, async_op=args.async_op) + end_event.record() sync_all() - duration = time.perf_counter() - pre + duration = start_event.elapsed_time(end_event) / 1000 # maintain and clean performance data avg_duration = duration / args.trials @@ -59,6 +63,16 @@ def run_all_reduce(local_rank, args): world_size = dist.get_world_size() global_rank = dist.get_rank() + if args.device == "xpu": + start_event = torch.xpu.Event(enable_timing=True) + end_event = torch.xpu.Event(enable_timing=True) + elif args.device == "cpu": + start_event = torch.cpu.Event() + end_event = torch.cpu.Event() + else: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + if args.scan: M_LIST = [] for x in (2**p for p in range(1, args.maxsize)): @@ -82,7 +96,7 @@ def run_all_reduce(local_rank, args): else: raise e sync_all() - timed_all_reduce(input, args) + timed_all_reduce(input, start_event, end_event, args) else: # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor # Don't need output tensor, so we double mem_factor @@ -104,7 +118,7 @@ def run_all_reduce(local_rank, args): else: raise e sync_all() - timed_all_reduce(input, args) + timed_all_reduce(input, start_event, end_event, args) if __name__ == "__main__": diff --git a/benchmarks/communication/all_to_all.py b/benchmarks/communication/all_to_all.py index 8735b1b4a..dc10b9ec9 100644 --- a/benchmarks/communication/all_to_all.py +++ b/benchmarks/communication/all_to_all.py @@ -14,7 +14,10 @@ from deepspeed.accelerator import get_accelerator -def timed_all_to_all(input, output, args): +def timed_all_to_all(input, output, start_event, end_event, args): + if args.device == "cpu": + print_rank_0(f"No Event support on CPU to measure time for now") + return if args.dist == 'torch': import torch.distributed as dist elif args.dist == 'deepspeed': @@ -27,11 +30,12 @@ def timed_all_to_all(input, output, args): sync_all() # time the actual comm op trials times and average it - pre = time.perf_counter() + start_event.record() for i in range(args.trials): dist.all_to_all_single(output, input, async_op=args.async_op) + end_event.record() sync_all() - duration = time.perf_counter() - pre + duration = start_event.elapsed_time(end_event) / 1000 # maintain and clean performance data avg_duration = duration / args.trials @@ -58,6 +62,16 @@ def run_all_to_all(local_rank, args): # Prepare benchmark header print_header(args, 'all_to_all') + if args.device == "xpu": + start_event = torch.xpu.Event(enable_timing=True) + end_event = torch.xpu.Event(enable_timing=True) + elif args.device == "cpu": + start_event = torch.cpu.Event() + end_event = torch.cpu.Event() + else: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + if args.scan: M_LIST = [] for x in (2**p for p in range(1, args.maxsize)): @@ -83,7 +97,7 @@ def run_all_to_all(local_rank, args): else: raise e sync_all() - timed_all_to_all(input, output, args) + timed_all_to_all(input, output, start_event, end_event, args) else: # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor elements_per_gpu = max_numel(comm_op='all_to_all', @@ -118,7 +132,7 @@ def run_all_to_all(local_rank, args): print(f"Before AllToAll Input List at rank {global_rank}: {input}") dist.barrier() - timed_all_to_all(input, output, args) + timed_all_to_all(input, output, start_event, end_event, args) if args.debug: for i in range(world_size): diff --git a/benchmarks/communication/broadcast.py b/benchmarks/communication/broadcast.py index 551c71f94..d05303be1 100644 --- a/benchmarks/communication/broadcast.py +++ b/benchmarks/communication/broadcast.py @@ -14,7 +14,10 @@ from deepspeed.accelerator import get_accelerator -def timed_broadcast(input, args): +def timed_broadcast(input, start_event, end_event, args): + if args.device == "cpu": + print_rank_0(f"No Event support on CPU to measure time for now") + return if args.dist == 'torch': import torch.distributed as dist elif args.dist == 'deepspeed': @@ -27,11 +30,12 @@ def timed_broadcast(input, args): sync_all() # time the actual comm op trials times and average it - pre = time.perf_counter() + start_event.record() for i in range(args.trials): dist.broadcast(input, 0, async_op=args.async_op) + end_event.record() sync_all() - duration = time.perf_counter() - pre + duration = start_event.elapsed_time(end_event) / 1000 # maintain and clean performance data avg_duration = duration / args.trials @@ -59,6 +63,16 @@ def run_broadcast(local_rank, args): world_size = dist.get_world_size() global_rank = dist.get_rank() + if args.device == "xpu": + start_event = torch.xpu.Event(enable_timing=True) + end_event = torch.xpu.Event(enable_timing=True) + elif args.device == "cpu": + start_event = torch.cpu.Event() + end_event = torch.cpu.Event() + else: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + if args.scan: M_LIST = [] for x in (2**p for p in range(1, args.maxsize)): @@ -82,7 +96,7 @@ def run_broadcast(local_rank, args): else: raise e sync_all() - timed_broadcast(input, args) + timed_broadcast(input, start_event, end_event, args) else: # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor # Don't need output tensor, so we double mem_factor @@ -102,7 +116,7 @@ def run_broadcast(local_rank, args): sync_all() return sync_all() - timed_broadcast(input, args) + timed_broadcast(input, start_event, end_event, args) if __name__ == "__main__": diff --git a/benchmarks/communication/constants.py b/benchmarks/communication/constants.py index ae9fa261b..60df98ed2 100644 --- a/benchmarks/communication/constants.py +++ b/benchmarks/communication/constants.py @@ -12,4 +12,5 @@ DEFAULT_UNIT = 'Gbps' DEFAULT_DIST = 'deepspeed' DEFAULT_MAXSIZE = 24 +DEFAULT_DEVICE = 'cuda' TORCH_DISTRIBUTED_DEFAULT_PORT = 29500 diff --git a/benchmarks/communication/pt2pt.py b/benchmarks/communication/pt2pt.py index 31028f99e..ec3252eb8 100644 --- a/benchmarks/communication/pt2pt.py +++ b/benchmarks/communication/pt2pt.py @@ -14,7 +14,10 @@ from deepspeed.accelerator import get_accelerator -def timed_pt2pt(input, args): +def timed_pt2pt(input, start_event, end_event, args): + if args.device == "cpu": + print_rank_0(f"No Event support on CPU to measure time for now") + return if args.dist == 'torch': import torch.distributed as dist elif args.dist == 'deepspeed': @@ -36,7 +39,7 @@ def timed_pt2pt(input, args): sync_all() # time the actual comm op trials times and average it - pre = time.perf_counter() + start_event.record() for i in range(args.trials): if dist.get_rank() == 0: if args.async_op: @@ -49,8 +52,9 @@ def timed_pt2pt(input, args): else: dist.recv(input, src=0) + end_event.record() sync_all() - duration = time.perf_counter() - pre + duration = start_event.elapsed_time(end_event) / 1000 # maintain and clean performance data avg_duration = duration / args.trials @@ -77,6 +81,16 @@ def run_pt2pt(local_rank, args): global_rank = dist.get_rank() world_size = dist.get_world_size() + if args.device == "xpu": + start_event = torch.xpu.Event(enable_timing=True) + end_event = torch.xpu.Event(enable_timing=True) + elif args.device == "cpu": + start_event = torch.cpu.Event() + end_event = torch.cpu.Event() + else: + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + if args.scan: # Create list of message sizes M_LIST = [] @@ -101,7 +115,7 @@ def run_pt2pt(local_rank, args): else: raise e sync_all() - timed_pt2pt(input, args) + timed_pt2pt(input, start_event, end_event, args) else: # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor # Don't need output tensor, so double mem_factor @@ -121,7 +135,7 @@ def run_pt2pt(local_rank, args): sync_all() return sync_all() - timed_pt2pt(input, args) + timed_pt2pt(input, start_event, end_event, args) if __name__ == "__main__": diff --git a/benchmarks/communication/utils.py b/benchmarks/communication/utils.py index a74d24e28..6f6dd83a1 100644 --- a/benchmarks/communication/utils.py +++ b/benchmarks/communication/utils.py @@ -108,6 +108,11 @@ def get_bw(comm_op, size, duration, args): n = dist.get_world_size() tput = 0 busbw = 0 + + if duration == 0: + print_rank_0("Error. Duration is 0.") + return tput, busbw + if comm_op == "all_to_all": tput = (size / duration) busbw = (size / duration) * ((n - 1) / n) @@ -235,4 +240,5 @@ def benchmark_parser(): default=.3, help='Proportion of max available GPU memory to use for single-size evals') parser.add_argument("--debug", action="http://github.com/store_true", help='Enables all_to_all debug prints') + parser.add_argument("--device", type=str, default=DEFAULT_DEVICE, help='target device') return parser diff --git a/benchmarks/deepcompile/.gitignore b/benchmarks/deepcompile/.gitignore new file mode 100644 index 000000000..d73b31758 --- /dev/null +++ b/benchmarks/deepcompile/.gitignore @@ -0,0 +1,3 @@ +*.log +*.pyc +*.png diff --git a/benchmarks/deepcompile/README.md b/benchmarks/deepcompile/README.md new file mode 100644 index 000000000..fbcf02489 --- /dev/null +++ b/benchmarks/deepcompile/README.md @@ -0,0 +1,160 @@ +# Benchmarks for DeepCompile + +## Setup + +This experiment scripts require 4 nodes that has 8 A100/H100 GPUs each. +We tested the scripts with Python 3.10.12 and CUDA 12.4. + +### Libraries + +In addition, you need to install the following: + +- PyTorch v2.6.0 + - For the Mixtral model, we recommend using PyTorch v2.7.0 (currently the final release candidate) due to an issue with `torch.where`. + - See details [here](https://github.com/pytorch/pytorch/issues/149278) +- DeepSpeed (v0.16.6 or newer) + - As DeepCompile is under active development, we recommend using the latest version or installing from source. +- transformers +- accelerate +- datasets v3.1 + +Here are an example of installation commands: + +```bash +pip3 install torch==2.6.0 torchvision torchaudio +pip3 install transformers datasets==3.1 accelerate + +# Install DeepSpeed +pip install deepspeed + +# Or install the latest revision of DeepSpeed from source +# git clone https://github.com/deepspeedai/DeepSpeed +# cd DeepSpeed +# git pip install . +# cd .. + +# Clone this repository +git clone https://github.com/deepspeedai/DeepSpeedExamples +cd benchmarks/deepcompile +``` + +You need to set up these on all nodes. + +### Setup for multiple nodes run + +You need to set host names in `hostfile_n${NUM_NODES}`. The file should look like the following: + +``` +node-0 slots=8 +node-1 slots=8 +node-2 slots=8 +node-3 slots=8 +``` + +## Evaluation on throughput + +The following script runs the throughput benchmark. This sweeps the following conditions: + +- Models: meta-llama/Meta-Llama-3-70B-Instruct, mistralai/Mixtral-8x7B-v0.1 +- Batch size: 1, 2, 4 +- Sequence length: 512 1024 2048 +- Frameworks and settings: + - DeepSpeed ZeRO3 (ZeRO3) + - DeepSpeed ZeRO3 +Compiler (ZeRO3 (C)) + - FSDP (FSDP) + - FSDP + Compiler (FSDP (C)) + - DeepCompile + proactive prefetching (DeepCompile (P)) + - DeepCompile + selective unsharding (DeepCompile (S)) + - DeepCompile + proactive prefetching + selective unsharding (DeepCompile (P+S)) + +The script downloads the models from HuggingFace Model Hub. Please make sure that you have access to the models. + +```bash +export PROFILE_DIR=/path/to/profile +bash run_bench.sh +``` + +The logs resulting from our experiments are stored in `logs/` directory. The summary of results is output to `profiles/result.txt`. You can copy the file to `results/acc_step_1` and plot the throughput with the following commands. + +```bash +python plot.py --result_dir results/acc_step_1 --metric throughput +``` + +Here are some example charts: + +
 |
+  |
+
 |
+  |
+
+ 
+ + *Figure 1: Throughput-latency curve and effective throughput of Llama 2 7b using A6000. Runs the client with 60 generation steps and input prompt length of 2600.*
+
diff --git a/benchmarks/inference/mii/plot_config.yaml b/benchmarks/inference/mii/plot_config.yaml
new file mode 100644
index 000000000..48a5a3171
--- /dev/null
+++ b/benchmarks/inference/mii/plot_config.yaml
@@ -0,0 +1,7 @@
+label: "vLLM"
+color: "purple"
+marker: "o"
+linestyle: "--"
+polyfit_degree: 0
+x_max : 30
+y_max : 10
diff --git a/benchmarks/inference/mii/requirements.txt b/benchmarks/inference/mii/requirements.txt
new file mode 100644
index 000000000..9f338ace5
--- /dev/null
+++ b/benchmarks/inference/mii/requirements.txt
@@ -0,0 +1,6 @@
+transformers
+matplotlib
+deepspeed-mii>=0.2.0
+vllm>=0.2.7
+numpy
+tabulate
diff --git a/benchmarks/inference/mii/run_all.sh b/benchmarks/inference/mii/run_all.sh
new file mode 100644
index 000000000..7c9311aea
--- /dev/null
+++ b/benchmarks/inference/mii/run_all.sh
@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+MODELS=(meta-llama/Llama-2-7b-hf meta-llama/Llama-2-13b-hf meta-llama/Llama-2-70b-hf tiiuae/falcon-40B tiiuae/falcon-180B microsoft/phi-2 mistralai/Mixtral-8x7B-v0.1)
+
+for MODEL in ${MODELS[@]}; do
+ python ./run_benchmark.py --model ${MODEL} --stream --backend fastgen
+ python ./run_benchmark.py --model ${MODEL} --stream --backend vllm
+done
+
+# Extra runs for Mixtral with non-default settings
+python ./run_benchmark.py --model mistralai/Mixtral-8x7B-v0.1 --stream --tp_size 4 --mean_prompt_length 500 --mean_max_new_tokens 150 500 1024 --backend fastgen
+python ./run_benchmark.py --model mistralai/Mixtral-8x7B-v0.1 --stream --tp_size 4 --mean_prompt_length 500 --mean_max_new_tokens 150 500 1024 --backend vllm
\ No newline at end of file
diff --git a/benchmarks/inference/mii/run_aml.sh b/benchmarks/inference/mii/run_aml.sh
new file mode 100644
index 000000000..90ad50e2c
--- /dev/null
+++ b/benchmarks/inference/mii/run_aml.sh
@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# Run benchmark against AML endpoint
+python ./run_benchmark.py \
+ --model + + *Figure 1: Throughput-latency curve and effective throughput of Llama 2 7b using A6000. Runs the client with 60 generation steps and input prompt length of 2600.*
+