Llama-3-8B-Instruct-Chinese-chat

Llama-3-8B-Instruct的中文自微调版本

训练可用数据整理

github地址

推荐微调工具

特别鸣谢以下优秀中文微调工具项目：

• Firefly - https://github.com/yangjianxin1/Firefly
• LLaMA-Factory - https://github.com/hiyouga/LLaMA-Factory.git

Chat版模型下载

• Instruct + 中文sft增强版
• huggingface地址

模型量化与部署

使用说明

默认运行以下代码体验中文对话，请将model_name_or_path修改为实际模型路径

from transformers import AutoTokenizer, AutoConfig, AddedToken, AutoModelForCausalLM, BitsAndBytesConfig
from peft import PeftModel
from dataclasses import dataclass
from typing import Dict
import torch
import copy

## 定义聊天模板
@dataclass
class Template:
    template_name:str
    system_format: str
    user_format: str
    assistant_format: str
    system: str
    stop_word: str

template_dict: Dict[str, Template] = dict()

def register_template(template_name, system_format, user_format, assistant_format, system, stop_word=None):
    template_dict[template_name] = Template(
        template_name=template_name,
        system_format=system_format,
        user_format=user_format,
        assistant_format=assistant_format,
        system=system,
        stop_word=stop_word,
    )

# 系统提示词可自行修改
register_template(
    template_name='llama3',
    system_format='<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{content}<|eot_id|>',
    user_format='<|start_header_id|>user<|end_header_id|>\n\n{content}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n',
    assistant_format='{content}<|eot_id|>',
    system=None,
    stop_word='<|eot_id|>'
)


## 加载模型
def load_model(model_name_or_path, load_in_4bit=False, adapter_name_or_path=None):
    if load_in_4bit:
        quantization_config = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.float16,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type="nf4",
            llm_int8_threshold=6.0,
            llm_int8_has_fp16_weight=False,
        )
    else:
        quantization_config = None

    # 加载基础模型
    model = AutoModelForCausalLM.from_pretrained(
        model_name_or_path,
        load_in_4bit=load_in_4bit,
        trust_remote_code=True,
        low_cpu_mem_usage=True,
        torch_dtype=torch.float16,
        device_map='auto',
        quantization_config=quantization_config
    )

    # 加载适配器
    if adapter_name_or_path is not None:
        model = PeftModel.from_pretrained(model, adapter_name_or_path)

    return model

## 加载分词器
def load_tokenizer(model_name_or_path):
    tokenizer = AutoTokenizer.from_pretrained(
        model_name_or_path,
        trust_remote_code=True,
        use_fast=False
    )

    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token

    return tokenizer

## 构建提示词
def build_prompt(tokenizer, template, query, history, system=None):
    template_name = template.template_name
    system_format = template.system_format
    user_format = template.user_format
    assistant_format = template.assistant_format
    system = system if system is not None else template.system

    history.append({"role": 'user', 'message': query})
    input_ids = []

    # 添加系统信息
    if system_format is not None:
        if system is not None:
            system_text = system_format.format(content=system)
            input_ids = tokenizer.encode(system_text, add_special_tokens=False)
    # 拼接历史对话
    for item in history:
        role, message = item['role'], item['message']
        if role == 'user':
            message = user_format.format(content=message, stop_token=tokenizer.eos_token)
        else:
            message = assistant_format.format(content=message, stop_token=tokenizer.eos_token)
        tokens = tokenizer.encode(message, add_special_tokens=False)
        input_ids += tokens
    input_ids = torch.tensor([input_ids], dtype=torch.long)

    return input_ids


def main():
    model_name_or_path = 'NousResearch/Meta-Llama-3-8B'
    template_name = 'llama3'
    adapter_name_or_path = None

    template = template_dict[template_name]

    load_in_4bit = False

    max_new_tokens = 500 
    top_p = 0.9
    temperature = 0.35 
    repetition_penalty = 1.1

    # 加载模型
    print(f'加载模型: {model_name_or_path}')
    print(f'适配器路径: {adapter_name_or_path}')
    model = load_model(
        model_name_or_path,
        load_in_4bit=load_in_4bit,
        adapter_name_or_path=adapter_name_or_path
    ).eval()
    tokenizer = load_tokenizer(model_name_or_path if adapter_name_or_path is None else adapter_name_or_path)
    if template.stop_word is None:
        template.stop_word = tokenizer.eos_token
    stop_token_id = tokenizer.encode(template.stop_word, add_special_tokens=True)
    assert len(stop_token_id) == 1
    stop_token_id = stop_token_id[0]

    history = []

    query = input('# 用户：')
    while True:
        query = query.strip()
        input_ids = build_prompt(tokenizer, template, query, copy.deepcopy(history), system=None).to(model.device)
        attention_mask = torch.ones(input_ids.shape,dtype=torch.long,device=model.device)
        outputs = model.generate(
            input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=True,
            top_p=top_p, temperature=temperature, repetition_penalty=repetition_penalty,
            eos_token_id=stop_token_id,
            attention_mask=attention_mask,
            pad_token_id=tokenizer.eos_token_id
        )
        outputs = outputs.tolist()[0][len(input_ids[0]):]
        response = tokenizer.decode(outputs)
        response = response.strip().replace(template.stop_word, "").strip()

        # 存储对话历史
        history.append({"role": 'user', 'message': query})
        history.append({"role": 'assistant', 'message': response})

        if len(history) > 12:
            history = history[:-12]

        print("Llama3中文版：{}".format(response))
        query = input('# 用户：')


if __name__ == '__main__':
    main()

多轮对话示例

用户：你是谁
Llama3中文版：我是AI语言模型MOSS。

用户：你会什么
Llama3中文版：我能回答问题、提供信息、协助完成任务等。请问您需要什么帮助？

用户：两千块钱买什么新车好？
Llama3中文版：两千元预算购买新车较为困难，建议考虑二手车或低价车型。选购二手车时请仔细检查车况和历史记录。

用户：火烧赤壁 曹操为何不拨打119求救
Llama3中文版：这是三国时期的历史事件，当时没有现代通讯系统，曹操无法使用119等紧急服务。

用户：帮我写一个transformer block
Llama3中文版：以下是Transformer Block示例代码：
[代码内容...]
包含自注意力机制和前馈网络两部分。

用户：能写快速排序吗
Llama3中文版：Python快速排序实现：
[代码内容...]
采用递归方式实现数组排序。

用户：你觉得你还存在哪些缺陷？
Llama3中文版：作为AI模型，我存在以下不足：
1. 语言理解有限
2. 缺乏真实情感
3. 可能存在数据偏见
4. 创造力受限
这些是当前的技术局限性。