使用 DeepSpeed 和 Hugging Face 🤗 Transformer 微调 FLAN-T5 XL/XXL

# install git lfs for pushing artifacts
!sudo apt install git-lfs
# install torch with the correct cuda version, check nvcc –version
!pip install torch –extra-index-url https://download.pytorch.org/whl/cu116 –upgrade
# install Hugging Face Libraries
!pip install “transformers==4.26.0” “datasets==2.9.0” “accelerate==0.16.0” “evaluate==0.4.0” –upgrade
# install deepspeed and ninja for jit compilations of kernels
!pip install “deepspeed==0.8.0” ninja –upgrade
# install additional dependencies needed for training
!pip install rouge-score nltk py7zr tensorboard

# 实验配置
model_id = “google/flan-t5-xxl” # Hugging Face 模型 Id
dataset_id = “cnn_dailymail” # Hugging Face 数据集 Id
dataset_config = “3.0.0” # 数据集版本
save_dataset_path = “data” # 存放处理后数据的本地路径
text_column = “article” # 输入文本所属列
summary_column = “highlights” # 输出文本所属列
# 定制指令提示格式
prompt_template = f”Summarize the following news article:n{{input}}nSummary:n”

from datasets import load_dataset
from transformers import AutoTokenizer
import numpy as np

# Load dataset from the hub
dataset = load_dataset(dataset_id,name=dataset_config)
# Load tokenizer of FLAN-t5-base
tokenizer = AutoTokenizer.from_pretrained(model_id)

print(f”Train dataset size: {len(dataset[‘train’])}”)
print(f”Test dataset size: {len(dataset[‘test’])}”)

# Train dataset size: 287113
# Test dataset size: 11490

prompt_length = len(tokenizer(prompt_template.format(input=””))[“input_ids”])
max_sample_length = tokenizer.model_max_length – prompt_length
print(f”Prompt length: {prompt_length}”)
print(f”Max input length: {max_sample_length}”)

# Prompt length: 12
# Max input length: 500

Prompt length: 12
Max input length: 500

from datasets import concatenate_datasets
import numpy as np

# The maximum total input sequence length after tokenization.
# Sequences longer than this will be truncated, sequences shorter will be padded.
tokenized_inputs = concatenate_datasets([dataset[“train”], dataset[“test”]]).map(lambda x: tokenizer(x[text_column], truncation=True), batched=True, remove_columns=[text_column, summary_column])
max_source_length = max([len(x) for x in tokenized_inputs[“input_ids”]])
max_source_length = min(max_source_length, max_sample_length)
print(f”Max source length: {max_source_length}”)

# The maximum total sequence length for target text after tokenization.
# Sequences longer than this will be truncated, sequences shorter will be padded.”
tokenized_targets = concatenate_datasets([dataset[“train”], dataset[“test”]]).map(lambda x: tokenizer(x[summary_column], truncation=True), batched=True, remove_columns=[text_column, summary_column])
target_lenghts = [len(x) for x in tokenized_targets[“input_ids”]]
# use 95th percentile as max target length
max_target_length = int(np.percentile(target_lenghts, 95))
print(f”Max target length: {max_target_length}”)

0%| | 0/299 [00:00<?, ?ba/s]
Max source length: 500

0%| | 0/299 [00:00<?, ?ba/s]
Max target length: 129

import os

def preprocess_function(sample, padding=”max_length”):
# created prompted input
inputs = [prompt_template.format(input=item) for item in sample[text_column]]

# tokenize inputs
model_inputs = tokenizer(inputs, max_length=tokenizer.model_max_length, padding=padding, truncation=True)

# Tokenize targets with the `text_target` keyword argument
labels = tokenizer(text_target=sample[summary_column], max_length=max_target_length, padding=padding, truncation=True)

# If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore
# padding in the loss.
if padding == “max_length”:
labels[“input_ids”] = [
[(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[“input_ids”]
]

model_inputs[“labels”] = labels[“input_ids”]
return model_inputs

# process dataset
tokenized_dataset = dataset.map(preprocess_function, batched=True, remove_columns=list(dataset[“train”].features))

# save dataset to disk
tokenized_dataset[“train”].save_to_disk(os.path.join(save_dataset_path,”train”))
tokenized_dataset[“test”].save_to_disk(os.path.join(save_dataset_path,”eval”))

{
“bf16”: {
“enabled”: “auto”
},
“optimizer”: {
“type”: “AdamW”,
“params”: {
“lr”: “auto”,
“betas”: “auto”,
“eps”: “auto”,
“weight_decay”: “auto”
}
},
“scheduler”: {
“type”: “WarmupLR”,
“params”: {
“warmup_min_lr”: “auto”,
“warmup_max_lr”: “auto”,
“warmup_num_steps”: “auto”
}
},
“zero_optimization”: {
“stage”: 3,
“overlap_comm”: true,
“contiguous_gradients”: true,
“sub_group_size”: 1e9,
“reduce_bucket_size”: “auto”,
“stage3_prefetch_bucket_size”: “auto”,
“stage3_param_persistence_threshold”: “auto”,
“stage3_max_live_parameters”: 1e9,
“stage3_max_reuse_distance”: 1e9,
“stage3_gather_16bit_weights_on_model_save”: false
},
“gradient_accumulation_steps”: “auto”,
“gradient_clipping”: “auto”,
“steps_per_print”: 2000,
“train_batch_size”: “auto”,
“train_micro_batch_size_per_gpu”: “auto”,
“wall_clock_breakdown”: false
}

!deepspeed –num_gpus=8 scripts/run_seq2seq_deepspeed.py
–model_id $model_id
–dataset_path $save_dataset_path
–epochs 3
–per_device_train_batch_size 8
–per_device_eval_batch_size 8
–generation_max_length $max_target_length
–lr 1e-4
–deepspeed configs/ds_flan_t5_z3_config_bf16.json

huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks…
To disable this warning, you can either:
– Avoid using `tokenizers` before the fork if possible
– Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
deepspeed –num_gpus=8 scripts/run_seq2seq_deepspeed.py –model_id google/flan-t5-xxl –dataset_path data –epochs 3 –per_device_train_batch_size 8 –per_device_eval_batch_size 8 –generation_max_length 129 –lr 1e-4 –deepspeed configs/ds_flan_t5_z3_config_bf16.json