许可证: MIT
语言:

• 土耳其语
  基础模型:
• canopylabs/orpheus-3b-0.1-pretrained
  任务标签: 文本转语音
  标签:
• karayakar
• 土耳其语
• Turkce
• TTS
• Orpheus
• 文本转语音

Orpheus 土耳其语TTS模型

Orpheus 土耳其语TTS预训练模型（第2000步）基于"canopylabs/orpheus-3b-0.1-pretrained"训练。

初始训练使用了超过60小时的合成语音数据。
训练中额外混合了160小时以上的合成语音数据。
400个表情符号（真实语音）数据用于支持表情符号功能。

可通过Flask API与模型交互。

情感支持

模型支持以下文本情感标记：

<laugh> – 笑声  
<chuckle> – 轻笑  
<sigh> – 叹息  
<cough> – 咳嗽  
<sniffle> – 抽鼻子  
<groan> – 呻吟  
<yawn> – 哈欠  
<gasp> – 倒吸气/惊讶喘息  

API

Flask配置为运行在5400端口（可通过下方脚本修改）

POST http://127.0.0.1:5400/generate HTTP/1.1  
User-Agent: Fiddler  
content-type: application/json  
Host: 127.0.0.1:5400  
Content-Length: 110  

{  
    "text": "Merhaba, orpheusTTS Turkce deneme"  
}  

创建环境

Windows:

# 创建虚拟环境  
python -m venv venv  
venv\Scripts\activate  

python inference.py  

训练

如需使用自有数据训练，可参考  
train.py  
config.yaml  

inference.py

（请先安装必要库）

# 从https://pytorch.org/安装对应版本torch  
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124  
pip install snac pathlib torch transformers huggingface_hub librosa numpy scipy torchaudio Flask jsonify  

（后续代码部分保留原始格式，仅翻译注释和字符串内容）

import os  
from snac import SNAC  
from pathlib import Path  
import torch  
from transformers import AutoModelForCausalLM, Trainer, TrainingArguments, AutoTokenizer,BitsAndBytesConfig  
from huggingface_hub import snapshot_download  
import librosa  
import numpy as np  
from scipy.io.wavfile import write  
import torchaudio  
from flask import Flask, jsonify, request  

# 模型本地路径  
modelLocalPath="D:\\...\\Karayakar\\Orpheus-TTS-Turkish-PT-5000"  

# 加载Orpheus分词器  
def load_orpheus_tokenizer(model_id: str = modelLocalPath) -> AutoTokenizer:  
    tokenizer = AutoTokenizer.from_pretrained(model_id,local_files_only=True, device_map="cuda")  
    return tokenizer  

# 加载SNAC模型  
def load_snac():  
    snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz")  
    return snac_model  

# 加载Orpheus自动模型  
def load_orpheus_auto_model(model_id: str = modelLocalPath):  
    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16,local_files_only=True, device_map="cuda")  
    model.cuda()  
    return model  

# 音频标记化处理  
def tokenize_audio(audio_file_path, snac_model):  
    audio_array, sample_rate = librosa.load(audio_file_path, sr=24000)  
    waveform = torch.from_numpy(audio_array).unsqueeze(0)  
    waveform = waveform.to(dtype=torch.float32)  

    waveform = waveform.unsqueeze(0)  

    with torch.inference_mode():  
        codes = snac_model.encode(waveform)  

    all_codes = []  
    for i in range(codes[0].shape[1]):  
        all_codes.append(codes[0][0][i].item() + 128266)  
        all_codes.append(codes[1][0][2 * i].item() + 128266 + 4096)  
        all_codes.append(codes[2][0][4 * i].item() + 128266 + (2 * 4096))  
        all_codes.append(codes[2][0][(4 * i) + 1].item() + 128266 + (3 * 4096))  
        all_codes.append(codes[1][0][(2 * i) + 1].item() + 128266 + (4 * 4096))  
        all_codes.append(codes[2][0][(4 * i) + 2].item() + 128266 + (5 * 4096))  
        all_codes.append(codes[2][0][(4 * i) + 3].item() + 128266 + (6 * 4096))  

    return all_codes  

# 准备输入数据  
def prepare_inputs(  
    fpath_audio_ref,  
    audio_ref_transcript: str,  
    text_prompts: list[str],  
    snac_model,  
    tokenizer,  
):  
    start_tokens = torch.tensor([[128259]], dtype=torch.int64)  
    end_tokens = torch.tensor([[128009, 128260, 128261, 128257]], dtype=torch.int64)  
    final_tokens = torch.tensor([[128258, 128262]], dtype=torch.int64)  

    all_modified_input_ids = []  
    for prompt in text_prompts:  
        input_ids = tokenizer(prompt, return_tensors="pt").input_ids  
        second_input_ids = torch.cat([start_tokens, input_ids, end_tokens], dim=1)  
        all_modified_input_ids.append(second_input_ids)  

    all_padded_tensors = []  
    all_attention_masks = []  
    max_length = max([modified_input_ids.shape[1] for modified_input_ids in all_modified_input_ids])  

    for modified_input_ids in all_modified_input_ids:  
        padding = max_length - modified_input_ids.shape[1]  
        padded_tensor = torch.cat([torch.full((1, padding), 128263, dtype=torch.int64), modified_input_ids], dim=1)  
        attention_mask = torch.cat([torch.zeros((1, padding), dtype=torch.int64),  
                                    torch.ones((1, modified_input_ids.shape[1]), dtype=torch.int64)], dim=1)  
        all_padded_tensors.append(padded_tensor)  
        all_attention_masks.append(attention_mask)  

    all_padded_tensors = torch.cat(all_padded_tensors, dim=0)  
    all_attention_masks = torch.cat(all_attention_masks, dim=0)  

    input_ids = all_padded_tensors.to("cuda")  
    attention_mask = all_attention_masks.to("cuda")  
    return input_ids, attention_mask  

# 推理生成  
def inference(model, input_ids, attention_mask):  
    with torch.no_grad():  
        generated_ids = model.generate(  
            input_ids=input_ids,  
            attention_mask=attention_mask,  
            max_new_tokens=2048,  
            do_sample=True,  
            temperature=0.2,  
            top_k=10,  
            top_p=0.9,  
            repetition_penalty=1.9,  
            num_return_sequences=1,  
            eos_token_id=128258,  
        )  

        generated_ids = torch.cat([generated_ids, torch.tensor([[128262]]).to("cuda")], dim=1) # EOAI标记  

        return generated_ids  

# 将标记转换为语音  
def convert_tokens_to_speech(generated_ids, snac_model):  
    token_to_find = 128257  
    token_to_remove = 128258  
    token_indices = (generated_ids == token_to_find).nonzero(as_tuple=True)  

    if len(token_indices[1]) > 0:  
        last_occurrence_idx = token_indices[1][-1].item()  
        cropped_tensor = generated_ids[:, last_occurrence_idx + 1:]  
    else:  
        cropped_tensor = generated_ids  

    _mask = cropped_tensor != token_to_remove  
    processed_rows = []  
    for row in cropped_tensor:  
        masked_row = row[row != token_to_remove]  
        processed_rows.append(masked_row)  

    code_lists = []  
    for row in processed_rows:  
        row_length = row.size(0)  
        new_length = (row_length // 7) * 7  
        trimmed_row = row[:new_length]  
        trimmed_row = [t - 128266 for t in trimmed_row]  
        code_lists.append(trimmed_row)  

    my_samples = []  
    for code_list in code_lists:  
        samples = redistribute_codes(code_list, snac_model)  
        my_samples.append(samples)  

    return my_samples  

# 重新分配编码  
def redistribute_codes(code_list, snac_model):  
    layer_1 = []  
    layer_2 = []  
    layer_3 = []  

    for i in range((len(code_list) + 1) // 7):  
        layer_1.append(code_list[7 * i])  
        layer_2.append(code_list[7 * i + 1] - 4096)  
        layer_3.append(code_list[7 * i + 2] - (2 * 4096))  
        layer_3.append(code_list[7 * i + 3] - (3 * 4096))  
        layer_2.append(code_list[7 * i + 4] - (4 * 4096))  
        layer_3.append(code_list[7 * i + 5] - (5 * 4096))  
        layer_3.append(code_list[7 * i + 6] - (6 * 4096))  

    codes = [  
        torch.tensor(layer_1).unsqueeze(0),  
        torch.tensor(layer_2).unsqueeze(0),  
        torch.tensor(layer_3).unsqueeze(0)  
    ]  
    audio_hat = snac_model.decode(codes)  
    return audio_hat  

# 转换为WAV格式  
def to_wav_from(samples: list) -> list[np.ndarray]:  
    """将PyTorch张量列表（或NumPy数组）转换为NumPy数组"""  
    processed_samples = []  

    for s in samples:  
        if isinstance(s, torch.Tensor):  
            s = s.detach().squeeze().to('cpu').numpy()  
        else:  
            s = np.squeeze(s)  

        processed_samples.append(s)  

    return processed_samples  

# 零样本TTS  
def zero_shot_tts(fpath_audio_ref, audio_ref_transcript, texts: list[str], model, snac_model, tokenizer):  
    print(f"参考音频路径 {fpath_audio_ref}")  
    print(f"参考音频文本 {audio_ref_transcript}")  
    print(f"输入文本 {texts}")  
    inp_ids, attn_mask = prepare_inputs(fpath_audio_ref, audio_ref_transcript, texts, snac_model, tokenizer)  
    print(f"输入ID长度:{len(inp_ids)}")  
    gen_ids = inference(model, inp_ids, attn_mask)  
    samples = convert_tokens_to_speech(gen_ids, snac_model)  
    wav_forms = to_wav_from(samples)  
    return wav_forms  

# 保存WAV文件  
def save_wav(samples: list[np.array], sample_rate: int, filenames: list[str]):  
    """ 保存张量列表为.wav文件  

    参数:  
        samples (list[torch.Tensor]): 音频张量列表  
        sample_rate (int): 采样率(Hz)  
        filenames (list[str]): 保存文件名列表  
    """  
    wav_data = to_wav_from(samples)  

    for data, filename in zip(wav_data, filenames):  
        write(filename, sample_rate, data.astype(np.float32))  
        print(f"已保存至 {filename}")  

# 获取参考音频和文本  
def get_ref_audio_and_transcript(root_folder: str):  
    root_path = Path(root_folder)  
    print(f"根路径 {root_path}")  
    out = []  
    for speaker_folder in root_path.iterdir():  
        if speaker_folder.is_dir():  # 确保是目录  
            wav_files = list(speaker_folder.glob("*.wav"))  
            txt_files = list(speaker_folder.glob("*.txt"))  

            if wav_files and txt_files:  
                ref_audio = wav_files[0]  # 假设每个文件夹只有一个.wav文件  
                transcript = txt_files[0].read_text(encoding="utf-8").strip()  
                out.append((ref_audio, transcript))  

    return out  

app = Flask(__name__)  

# Flask API端点  
@app.route('/generate', methods=['POST'])  
def generate():  
    content = request.json  
    process_data(content)  
    rresponse = {  
        'received': content,  
        'status': 'success'  
    }  
    response= jsonify(rresponse)  
    response.headers['Content-Type'] = 'application/json; charset=utf-8'  
    return response  

# 处理数据  
def process_data(jsonText):  
    texts = [f"{jsonText['text']}"]  
    for fpath_audio, audio_transcript in prompt_pairs:  
        print(f"零样本处理: {fpath_audio} {audio_transcript}")  
        wav_forms = zero_shot_tts(fpath_audio, audio_transcript, texts, model, snac_model, tokenizer)  

        import os  
        from pathlib import Path  
        from datetime import datetime  
        out_dir = Path(fpath_audio).parent / "inference"  
        out_dir.mkdir(parents=True, exist_ok=True)  
        timestamp_str = str(int(datetime.now().timestamp()))  
        file_names = [f"{out_dir.as_posix()}/{Path(fpath_audio).stem}_{i}_{timestamp_str}.wav" for i, t in enumerate(texts)]  
        save_wav(wav_forms, 24000, file_names)  

# 主程序  
if __name__ == "__main__":  
    tokenizer = load_orpheus_tokenizer()  
    model = load_orpheus_auto_model()  
    snac_model = load_snac()  
    prompt_pairs = get_ref_audio_and_transcript("D:\\AI_APPS\\Orpheus-TTS\\data")  
    print(f"SNAC模型已加载")  
    app.run(debug=True,port=5400)