标签:

• 由训练器生成 数据集: UrukHan/wav2vec2-russian 小部件:
• 文本: "ывсем привет выныканалетоп армии и это двадцать пятый день спец операций на украине ет самый главной новости российские военные ракетами кинжалы калибр уничтожили крупную военную топливную базу украины ракетным ударом по населенному пункту под жетамиром уничтжены более стаукраинских военных в две тысячи двадцать втором году" 模型索引:
• 名称: t5-russian-spell 结果: []

t5-russian-spell

用于校正从音频识别文本的模型。我的音频识别模型位于 https://huggingface.co/UrukHan/wav2vec2-russian，其输出结果可以输入此模型进行校正。已在随机YouTube视频上进行测试。

训练用数据集: UrukHan/t5-russian-spell_I : https://huggingface.co/datasets/UrukHan/t5-russian-spell_I UrukHan/t5-russian-spell_II : https://huggingface.co/datasets/UrukHan/t5-russian-spell_II UrukHan/t5-russian-spell_III : https://huggingface.co/datasets/UrukHan/t5-russian-spell_III

运行推理示例及Colab笔记本说明

https://colab.research.google.com/drive/1ame2va9_NflYqy4RZ07HYmQ0moJYy7w2?usp=sharing :

# 安装transformers库
!pip install transformers

# 导入库
from transformers import AutoModelForSeq2SeqLM, T5TokenizerFast

# 设置选择的模型名称
MODEL_NAME = 'UrukHan/t5-russian-spell'
MAX_INPUT = 256

# 加载模型和分词器
tokenizer = T5TokenizerFast.from_pretrained(MODEL_NAME)
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)

# 输入数据(可以是短语数组或文本)
input_sequences = ['сеглдыя хорош ден', 'когд а вы прдет к нам в госи']   # 或使用单条短语: input_sequences = 'сеглдыя хорош ден'

task_prefix = "拼写校正: "                 # 数据分词
if type(input_sequences) != list: input_sequences = [input_sequences]
encoded = tokenizer(
  [task_prefix + sequence for sequence in input_sequences],
  padding="longest",
  max_length=MAX_INPUT,
  truncation=True,
  return_tensors="pt",
)

predicts = model.generate(encoded)    # 预测

tokenizer.batch_decode(predicts, skip_special_tokens=True)  # 解码数据

用于训练和保存模型到HuggingFace Hub的配置笔记本:

https://colab.research.google.com/drive/1H4IoasDqa2TEjGivVDp-4Pdpm0oxrCWd?usp=sharing

# 安装库
!pip install datasets
!apt install git-lfs
!pip install transformers
!pip install sentencepiece 
!pip install rouge_score

# 导入库
import numpy as np
from datasets import Dataset
import tensorflow as 
import nltk
from transformers import T5TokenizerFast, Seq2SeqTrainingArguments, Seq2SeqTrainer, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq
import torch
from transformers.optimization import Adafactor, AdafactorSchedule
from datasets import load_dataset, load_metric

# 加载参数
raw_datasets = load_dataset("xsum")
metric = load_metric("rouge")
nltk.download('punkt')

# 输入HuggingFace Hub密钥
from huggingface_hub import notebook_login
notebook_login()

# 定义参数
REPO = "t5-russian-spell"  # 输入仓库名称
MODEL_NAME = "UrukHan/t5-russian-spell" # 输入选择的模型名称
MAX_INPUT = 256  # 输入最大输入token长度
MAX_OUTPUT  = 256 # 输入最大输出token长度
BATCH_SIZE = 8 
DATASET = 'UrukHan/t5-russian-spell_I'   # 输入数据集名称

# 加载数据集
data = load_dataset(DATASET)

# 加载模型和分词器
tokenizer = T5TokenizerFast.from_pretrained(MODEL_NAME)
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)

model.config.max_length = MAX_OUTPUT  # 默认20，所以所有模型的预测输出序列会被截断
# 首次保存到仓库后可以注释掉
tokenizer.push_to_hub(repo_name) 

train = data['train']
test = data['test'].train_test_split(0.02)['test']  # 减少测试集规模以加快epoch间错误计算

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model) #return_tensors="tf"

def compute_metrics(eval_pred):
  predictions, labels = eval_pred
  decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
  # 替换标签中的-100为pad_token_id
  labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
  decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
  
  # Rouge期望每句后有换行符
  decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
  decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
  
  result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
  # 提取部分结果
  result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
  
  # 添加平均生成长度
  prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]
  result["gen_len"] = np.mean(prediction_lens)
  
  return {k: round(v, 4) for k, v in result.items()}
  
training_args = Seq2SeqTrainingArguments(
  output_dir = REPO,
  evaluation_strategy='steps',
  eval_steps=5000,
  save_steps=5000,
  num_train_epochs=1,
  predict_with_generate=True,
  per_device_train_batch_size=BATCH_SIZE,
  per_device_eval_batch_size=BATCH_SIZE,
  fp16=True,
  save_total_limit=2,
  weight_decay=0.005,
  push_to_hub=True,
)

# 手动选择优化器。原始T5架构使用Adafactor优化器
optimizer = Adafactor(
    model.parameters(),
    lr=1e-5,
    eps=(1e-30, 1e-3),
    clip_threshold=1.0,
    decay_rate=-0.8,
    beta1=None,
    weight_decay=0.0,
    relative_step=False,
    scale_parameter=False,
    warmup_init=False,
)
lr_scheduler = AdafactorSchedule(optimizer)

trainer = Seq2SeqTrainer(
  model=model,
  args=training_args,
  train_dataset = train,
  eval_dataset = test,
  optimizers = (optimizer, lr_scheduler),
  tokenizer = tokenizer,
  compute_metrics=compute_metrics
)

trainer.train()

trainer.push_to_hub()

数据格式转换示例

input_data = ['удач почти отнее отвернулась', 'в хааоде проведения чемпиониавта мира дветысячивосемнандцтая лгодаа']
output_data = ['Удача почти от нее отвернулась', 'в ходе проведения чемпионата мира две тысячи восемнадцатого года']

# 分词输入数据
task_prefix = "拼写校正: "
input_sequences = input_data 
encoding = tokenizer(
  [task_prefix + sequence for sequence in input_sequences],
  padding="longest",
  max_length=MAX_INPUT,
  truncation=True,
  return_tensors="pt",
)
input_ids, attention_mask = encoding.input_ids, encoding.attention_mask

# 分词输出数据
target_encoding = tokenizer(output_data, padding="longest", max_length=MAX_OUTPUT, truncation=True)
labels = target_encoding.input_ids
# 将标签中的填充token id替换为-100
labels = torch.tensor(labels)
labels[labels == tokenizer.pad_token_id] = -100'''

# 将数据转换为dataset格式   
data = Dataset.from_pandas(pd.DataFrame({'input_ids': list(np.array(input_ids)), 'attention_mask': list(np.array(attention_mask)), 'labels': list(np.array(labels))}))
data = data.train_test_split(0.02)
# 最终得到训练器的输入: train_dataset = data['train'], eval_dataset = data['test']