许可协议: cc-by-nc-4.0 语言:

• 越南语

越南语自动语音识别（ASR）序列到序列模型。该模型支持输出规范化文本、标记时间戳以及多说话人分段。

# !pip install transformers, sentencepiece

from transformers import SpeechEncoderDecoderModel
from transformers import AutoFeatureExtractor, AutoTokenizer, GenerationConfig
import torchaudio
import torch

model_path = 'nguyenvulebinh/wav2vec2-bartpho'
model = SpeechEncoderDecoderModel.from_pretrained(model_path).eval()
feature_extractor = AutoFeatureExtractor.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
if torch.cuda.is_available():
  model = model.cuda()


def decode_tokens(token_ids, skip_special_tokens=True, time_precision=0.02):
    timestamp_begin = tokenizer.vocab_size
    outputs = [[]]
    for token in token_ids:
        if token >= timestamp_begin:
            timestamp = f" |{(token - timestamp_begin) * time_precision:.2f}| "
            outputs.append(timestamp)
            outputs.append([])
        else:
            outputs[-1].append(token)
    outputs = [
        s if isinstance(s, str) else tokenizer.decode(s, skip_special_tokens=skip_special_tokens) for s in outputs
    ]
    return "".join(outputs).replace("< |", "<|").replace("| >", "|>")

def decode_wav(audio_wavs, asr_model, prefix=""):
  device = next(asr_model.parameters()).device
  input_values = feature_extractor.pad(
    [{"input_values": feature} for feature in audio_wavs],
    padding=True,
    max_length=None,
    pad_to_multiple_of=None,
    return_tensors="pt",
  )

  output_beam_ids = asr_model.generate(
    input_values['input_values'].to(device), 
    attention_mask=input_values['attention_mask'].to(device),
    decoder_input_ids=tokenizer.batch_encode_plus([prefix] * len(audio_wavs), return_tensors="pt")['input_ids'][..., :-1].to(device),
    generation_config=GenerationConfig(decoder_start_token_id=tokenizer.bos_token_id),
    max_length=250, 
    num_beams=25, 
    no_repeat_ngram_size=4, 
    num_return_sequences=1, 
    early_stopping=True,
    return_dict_in_generate=True,
    output_scores=True,
  )

  output_text = [decode_tokens(sequence) for sequence in output_beam_ids.sequences]

  return output_text


# https://huggingface.co/nguyenvulebinh/wav2vec2-bartpho/resolve/main/sample_news.wav
print(decode_wav([torchaudio.load('sample_news.wav')[0].squeeze()], model))

# <|0.00| 家属表示，曾多次打电话向当地政府和安保部门报告，但都无济于事 |7.00|>
# <|8.14| 没有人能提供任何帮助，所以那时是最绝望的时刻，它让我极度痛苦，造成了极大的心理压抑 |19.02|>

引用

本仓库使用了以下论文中的思想。如果该模型有助于发表成果或被整合到其他软件中，请引用该论文。

@INPROCEEDINGS{10446589,
  author={Nguyen, Thai-Binh and Waibel, Alexander},
  booktitle={ICASSP 2024 - 2024 IEEE国际声学、语音与信号处理会议（ICASSP）}, 
  title={合成对话改进多说话人ASR}, 
  year={2024},
  volume={},
  number={},
  pages={10461-10465},
  keywords={系统性；错误分析；基于知识的系统；口头交流；信号处理；数据模型；声学；多说话人；ASR；合成对话},
  doi={10.1109/ICASSP48485.2024.10446589}
}

联系方式

nguyenvulebinh@gmail.com