#推荐

【七】指令遵循微调

本章涵盖以下内容：

• LLM 指令微调过程概述
• 为监督式指令微调准备数据集
• 批量组织指令数据
• 评估 LLM 通过指令遵循生成的内容质量
• 评估一个经过指令微调的 LLM

• 7.1 指令遵循微调简介
• 7.2 为监督指令微调准备数据集
• 7.3 将数据组织成训练批次
• 7.4 为指令数据集创建数据加载器
• 7.5 加载预训练的 LLM
• 7.6 指令微调 LLM
• 7.7 提取并保存响应
• 7.8 评估指令微调后的 LLM
• 7.9 结语
  • 7.9.1 接下来如何做？
  • 7.9.2 如何在快速变化的前沿领域中保持领先
• 7.10 本章摘要

7.1 指令遵循微调简介

7.2 为监督指令微调准备数据集

# Listing 7.1 Downloading the dataset
import json
import os
import urllib

def download_and_load_file(file_path, url):
    if not os.path.exists(file_path):
        with urllib.request.urlopen(url) as response:
            text_data = response.read().decode("utf-8")
        with open(file_path, "w", encoding="utf-8") as file:
            file.write(text_data)
    else: #A
        with open(file_path, "r", encoding="utf-8") as file:
            text_data = file.read()
    with open(file_path, "r") as file:
        data = json.load(file)
    return data

file_path = "instruction-data.json"
url = "https://raw.githubusercontent.com/rasbt/LLMs-from-scratch/main/ch07/01_mainchapter-code/instruction-data.json"

data = download_and_load_file(file_path, url)
print("Number of entries:", len(data))

#A 如果文件已经下载，就跳过下载过程

print("Example entry:\n", data[50])

Example entry:
{'instruction': 'Identify the correct spelling of the following word.', 'input':
'Ocassion', 'output': "The correct spelling is 'Occasion.'"}

print("Another example entry:\n", data[999])

Another example entry:
{'instruction': "What is an antonym of 'complicated'?", 'input': '', 'output': "An
antonym of 'complicated' is 'simple'."}

练习 7.1 改变提示词风格

在使用 Alpaca 提示语风格对模型进行微调之后，尝试图 7.4 中展示的 Phi-3 提示语风格，并观察其是否会影响模型的响应效果。

# Listing 7.2 Implementing the prompt formatting function
def format_input(entry):
    instruction_text = (
        f"Below is an instruction that describes a task. "
        f"Write a response that appropriately completes the request."
        f"\n\n### Instruction:\n{entry['instruction']}"
    )
    input_text = f"\n\n### Input:\n{entry['input']}" if entry["input"] else ""
    return instruction_text + input_text

model_input = format_input(data[50])
desired_response = f"\n\n### Response:\n{data[50]['output']}"
print(model_input + desired_response)

Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
Identify the correct spelling of the following word.

### Input:
Ocassion

### Response:
The correct spelling is 'Occasion.'

model_input = format_input(data[999])
desired_response = f"\n\n### Response:\n{data[999]['output']}"
print(model_input + desired_response)

Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
What is an antonym of 'complicated'?

### Response:
An antonym of 'complicated' is 'simple'.

# Listing 7.3 Partitioning the dataset
train_portion = int(len(data) * 0.85) # 85% for training
test_portion = int(len(data) * 0.1) # 10% for testing
val_portion = len(data) - train_portion - test_portion # Remaining 5% for validation

train_data = data[:train_portion]
test_data = data[train_portion:train_portion + test_portion]
val_data = data[train_portion + test_portion:]

print("Training set length:", len(train_data))
print("Validation set length:", len(val_data))
print("Test set length:", len(test_data))

Training set length: 935
Validation set length: 55
Test set length: 110

7.3 将数据组织成训练批次

# Listing 7.4 Implementing an instruction dataset class
import torch
from torch.utils.data import Dataset

class InstructionDataset(Dataset):
    def __init__(self, data, tokenizer):
        self.data = data
        self.encoded_texts = []
        for entry in data:                                           #A
            instruction_plus_input = format_input(entry)
            response_text = f"\n\n### Response:\n{entry['output']}"
            full_text = instruction_plus_input + response_text
            self.encoded_texts.append(
                tokenizer.encode(full_text)
            )

    def __getitem__(self, index):
        return self.encoded_texts[index]

    def __len__(self):
        return len(self.data)

#A 预分词文本

import tiktoken
tokenizer = tiktoken.get_encoding("gpt2")
print(tokenizer.encode("<|endoftext|>", allowed_special={"<|endoftext|>"}))
The resulting token ID is 50256.

def custom_collate_draft_1(
    batch,
    pad_token_id=50256,
    device="cpu"
):
    batch_max_length = max(len(item)+1 for item in batch)          #A
    inputs_lst = []

    for item in batch:                                             #B
        new_item = item.copy()
        new_item += [pad_token_id]

        padded = new_item + [pad_token_id] * (batch_max_length - len(new_item))

        inputs = torch.tensor(padded[:-1])                         #C
        inputs_lst.append(inputs)

    inputs_tensor = torch.stack(inputs_lst).to(device)             #D
    return inputs_tensor


#A 找出批量中的最长序列
#B 对输入进行填充并准备好输入数据
#C 删除之前添加的多余填充 token
#D 将输入列表转换为张量，并转移到目标设备

inputs_1 = [0, 1, 2, 3, 4]
inputs_2 = [5, 6]
inputs_3 = [7, 8, 9]
batch = (
    inputs_1,
    inputs_2,
    inputs_3
)
print(custom_collate_draft_1(batch))

tensor([[ 0, 1, 2, 3, 4],
        [ 5, 6, 50256, 50256, 50256],
        [ 7, 8, 9, 50256, 50256]])

def custom_collate_draft_2(
    batch,
    pad_token_id=50256,
    device="cpu"
):
    batch_max_length = max(len(item)+1 for item in batch)
    inputs_lst, targets_lst = [], []

    for item in batch:
        new_item = item.copy()
        new_item += [pad_token_id]
        padded = new_item + [pad_token_id] * (batch_max_length - len(new_item))
        inputs = torch.tensor(padded[:-1])             #A
        targets = torch.tensor(padded[1:])             #B
        inputs_lst.append(inputs)
        targets_lst.append(targets)

    inputs_tensor = torch.stack(inputs_lst).to(device)
    targets_tensor = torch.stack(targets_lst).to(device)
    return inputs_tensor, targets_tensor

inputs, targets = custom_collate_draft_2(batch)
print(inputs)
print(targets)


#A 截断输入序列的最后一个 token。
#B 将目标序列中的每个 token 向右移动一个位置。

tensor([[ 0, 1, 2, 3, 4],                   #A
        [ 5, 6, 50256, 50256, 50256],
        [ 7, 8, 9, 50256, 50256]])
tensor([[ 1, 2, 3, 4, 50256],               #B
        [ 6, 50256, 50256, 50256, 50256],
        [ 8, 9, 50256, 50256, 50256]])

#A 第一个张量表示输入数据
#B 第二个张量表示目标数据

# Listing 7.5 Implementing a custom batch collate function
def custom_collate_fn(
    batch,
    pad_token_id=50256,
    ignore_index=-100,
    allowed_max_length=None,
    device="cpu"
):
    batch_max_length = max(len(item)+1 for item in batch)
    inputs_lst, targets_lst = [], []

    for item in batch:
        new_item = item.copy()
        new_item += [pad_token_id]
        # Pad sequences to max_length
        padded = new_item + [pad_token_id] * (batch_max_length - len(new_item))
        inputs = torch.tensor(padded[:-1]) # Truncate the last token for inputs
        targets = torch.tensor(padded[1:]) # Shift +1 to the right for targets

        mask = targets == pad_token_id                  #A
        indices = torch.nonzero(mask).squeeze()         #A
        if indices.numel() > 1:                         #A
            targets[indices[1:]] = ignore_index         #A

        if allowed_max_length is not None:
            inputs = inputs[:allowed_max_length]        #B
            targets = targets[:allowed_max_length]      #B

        inputs_lst.append(inputs)
        targets_lst.append(targets)

    inputs_tensor = torch.stack(inputs_lst).to(device)
    targets_tensor = torch.stack(targets_lst)
    return inputs_tensor, targets_tensor


#A 在 targets 中，将除第一个以外的所有填充标记替换为 ignore_index
#B 可选择性地将序列截断到最大长度

inputs, targets = custom_collate_fn(batch)
print(inputs)
print(targets)

tensor([[ 0, 1, 2, 3, 4],
        [ 5, 6, 50256, 50256, 50256],
        [ 7, 8, 9, 50256, 50256]])
tensor([[ 1, 2, 3, 4, 50256],
        [ 6, 50256, -100, -100, -100],
        [ 8, 9, 50256, -100, -100]])

logits_1 = torch.tensor(
    [[-1.0, 1.0], # predictions for 1st token
     [-0.5, 1.5]] # predictions for 2nd token
)
targets_1 = torch.tensor([0, 1]) # Correct token indices to generate
loss_1 = torch.nn.functional.cross_entropy(logits_1, targets_1)
print(loss_1)

logits_2 = torch.tensor(
    [[-1.0, 1.0],
     [-0.5, 1.5],
     [-0.5, 1.5]]                        #A
)
targets_2 = torch.tensor([0, 1, 1])
loss_2 = torch.nn.functional.cross_entropy(logits_2, targets_2)
print(loss_2)

#A 添加第三个 token ID

targets_3 = torch.tensor([0, 1, -100])
loss_3 = torch.nn.functional.cross_entropy(logits_2, targets_3)
print(loss_3)
print("loss_1 == loss_3:", loss_1 == loss_3)
#The resulting output is as follows:
tensor(1.1269)
loss_1 == loss_3: tensor(True)

练习 7.2 指令与输入的掩码处理

在完成本章内容，并用本节实现的 InstructionDataset 对模型进行微调后，将指令和输入 token 替换为 -100 掩码，以实现图 7.13 展示的指令掩码方法。然后，评估该方法是否对模型性能有积极影响。

7.4 为指令数据集创建数据加载器

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# if torch.backends.mps.is_available():       #A
#     device = torch.device("mps")"           #A
print("Device:", device)

#A 取消这两行注释以在 Apple Silicon 芯片上启用 GPU

from functools import partial
customized_collate_fn = partial(custom_collate_fn, device=device,
allowed_max_length=1024)

# Listing 7.6 Initializing the data loaders
from torch.utils.data import DataLoader

num_workers = 0            #A
batch_size = 8

torch.manual_seed(123)

train_dataset = InstructionDataset(train_data, tokenizer)
train_loader = DataLoader(
    train_dataset,
    batch_size=batch_size,
    collate_fn=customized_collate_fn,
    shuffle=True,
    drop_last=True,
    num_workers=num_workers
)

val_dataset = InstructionDataset(val_data, tokenizer)
val_loader = DataLoader(
    val_dataset,
    batch_size=batch_size,
    collate_fn=customized_collate_fn,
    shuffle=False,
    drop_last=False,
    num_workers=num_workers
)

test_dataset = InstructionDataset(test_data, tokenizer)

test_loader = DataLoader(
    test_dataset,
    batch_size=batch_size,
    collate_fn=customized_collate_fn,
    shuffle=False,
    drop_last=False,
    num_workers=num_workers
)

#A 如果操作系统支持并行的 Python 进程，你可以尝试增加此数值。

print("Train loader:")
for inputs, targets in train_loader:
    print(inputs.shape, targets.shape)

Train loader:
torch.Size([8, 61]) torch.Size([8, 61])
torch.Size([8, 76]) torch.Size([8, 76])
torch.Size([8, 73]) torch.Size([8, 73])
...
torch.Size([8, 74]) torch.Size([8, 74])
torch.Size([8, 69]) torch.Size([8, 69])

7.5 加载预训练的 LLM

# Listing 7.7 Loading the pretrained model
from gpt_download import download_and_load_gpt2
from chapter04 import GPTModel
from chapter05 import load_weights_into_gpt

BASE_CONFIG = {
    "vocab_size": 50257, # Vocabulary size
    "context_length": 1024, # Context length
    "drop_rate": 0.0, # Dropout rate
    "qkv_bias": True # Query-key-value bias
}

model_configs = {
    "gpt2-small (124M)": {"emb_dim": 768, "n_layers": 12, "n_heads": 12},
    "gpt2-medium (355M)": {"emb_dim": 1024, "n_layers": 24, "n_heads": 16},
    "gpt2-large (774M)": {"emb_dim": 1280, "n_layers": 36, "n_heads": 20},
    "gpt2-xl (1558M)": {"emb_dim": 1600, "n_layers": 48, "n_heads": 25},
}

CHOOSE_MODEL = "gpt2-medium (355M)"
BASE_CONFIG.update(model_configs[CHOOSE_MODEL])

model_size = CHOOSE_MODEL.split(" ")[-1].lstrip("(").rstrip(")")
settings, params = download_and_load_gpt2(model_size=model_size, models_dir="gpt2")

model = GPTModel(BASE_CONFIG)
load_weights_into_gpt(model, params)
model.eval();

checkpoint: 100%|██████████| 77.0/77.0 [00:00<00:00, 156kiB/s]
encoder.json: 100%|██████████| 1.04M/1.04M [00:02<00:00, 467kiB/s]
hparams.json: 100%|██████████| 91.0/91.0 [00:00<00:00, 198kiB/s]
model.ckpt.data-00000-of-00001: 100%|██████████| 1.42G/1.42G [05:50<00:00, 4.05MiB/s]
model.ckpt.index: 100%|██████████| 10.4k/10.4k [00:00<00:00, 18.1MiB/s]
model.ckpt.meta: 100%|██████████| 927k/927k [00:02<00:00, 454kiB/s]
vocab.bpe: 100%|██████████| 456k/456k [00:01<00:00, 283kiB/s]

torch.manual_seed(123)
input_text = format_input(val_data[0])
print(input_text)

Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
Convert the active sentence to passive: 'The chef cooks the meal every day.'
Next, we generate the model's response using the generate function from chapter 5:
from chapter05 import generate, text_to_token_ids, token_ids_to_text

token_ids = generate(
    model=model,
    idx=text_to_token_ids(input_text, tokenizer),
    max_new_tokens=35,
    context_size=BASE_CONFIG["context_length"],
    eos_id=50256,
)
generated_text = token_ids_to_text(token_ids, tokenizer)

response_text = generated_text[len(input_text):].strip()
print(response_text)

### Response:

The chef cooks the meal every day.

### Instruction:

Convert the active sentence to passive: 'The chef cooks the

7.6 指令微调 LLM

from chapter05 import (
    calc_loss_loader,
    train_model_simple
)

model.to(device)
torch.manual_seed(123)

with torch.no_grad():
    train_loss = calc_loss_loader(train_loader, model, device, num_batches=5)
    val_loss = calc_loss_loader(val_loader, model, device, num_batches=5)

print("Training loss:", train_loss)
print("Validation loss:", val_loss)

Training loss: 3.825908660888672
Validation loss: 3.7619335651397705

应对硬件限制

需要注意的是，使用和训练像 GPT-2 medium（355 百万个参数）这样的大型模型相比于先前章节中使用的小型 GPT-2 模型（1.24 亿参数）在计算上更加密集。如果你因硬件限制遇到问题，可以通过将 CHOOSE_MODEL = “gpt2-medium (355M)” 更改为 CHOOSE_MODEL = “gpt2-small (124M)” 来切换到较小的模型。另一种加速模型训练的方式是使用 GPU。有关使用云 GPU 的选项，请参考本书代码仓库中的补充部分：https://github.com/rasbt/LLMs-from-scratch/tree/main/setup

# Listing 7.8 Instruction finetuning the pretrained LLM
import time

start_time = time.time()
torch.manual_seed(123)
optimizer = torch.optim.AdamW(model.parameters(), lr=0.00005, weight_decay=0.1)
num_epochs = 2

train_losses, val_losses, tokens_seen = train_model_simple(
    model, train_loader, val_loader, optimizer, device,
    num_epochs=num_epochs, eval_freq=5, eval_iter=5,
    start_context=format_input(val_data[0]), tokenizer=tokenizer
)

end_time = time.time()
execution_time_minutes = (end_time - start_time) / 60
print(f"Training completed in {execution_time_minutes:.2f} minutes.")

Ep 1 (Step 000000): Train loss 2.637, Val loss 2.626
Ep 1 (Step 000005): Train loss 1.174, Val loss 1.103
Ep 1 (Step 000010): Train loss 0.872, Val loss 0.944
Ep 1 (Step 000015): Train loss 0.857, Val loss 0.906
...
Ep 1 (Step 000115): Train loss 0.520, Val loss 0.665
Below is an instruction that describes a task. Write a response that appropriately
completes the request. ### Instruction: Convert the active sentence to passive: 'The
chef cooks the meal every day.' ### Response: The meal is prepared every day by the
chef.<|endoftext|>The following is an instruction that describes a task. Write a
response that appropriately completes the request. ### Instruction: Convert the active
sentence to passive:
Ep 2 (Step 000120): Train loss 0.438, Val loss 0.670
Ep 2 (Step 000125): Train loss 0.453, Val loss 0.685
Ep 2 (Step 000130): Train loss 0.448, Val loss 0.681
Ep 2 (Step 000135): Train loss 0.408, Val loss 0.677
...
Ep 2 (Step 000230): Train loss 0.300, Val loss 0.657
Below is an instruction that describes a task. Write a response that appropriately
completes the request. ### Instruction: Convert the active sentence to passive: 'The
chef cooks the meal every day.' ### Response: The meal is cooked every day by the chef.
<|endoftext|>The following is an instruction that describes a task. Write a response
that appropriately completes the request. ### Instruction: What is the capital of the
United Kingdom
Training completed in 0.87 minutes.

from chapter05 import plot_losses
epochs_tensor = torch.linspace(0, num_epochs, len(train_losses))
plot_losses(epochs_tensor, tokens_seen, train_losses, val_losses)

练习 7.3 在原始 Alpaca 数据集上进行微调

斯坦福大学研究人员创建的 Alpaca 数据集是最早且最受欢迎的公开共享指令数据集之一，包含了 52,002 条数据。作为本章中使用的instruction-data.json文件的替代，可以考虑在这个数据集上对 LLM 进行微调。该数据集可通过以下网址获取：https://raw.githubusercontent.com/tatsu-lab/stanford_alpaca/main/alpaca_data.json。

该数据集包含 52,002 条记录，约为本章使用数据集的 50 倍，且大部分记录的长度也较长。因此，强烈建议使用 GPU 来加速微调过程。如果遇到内存不足错误，可以考虑将批量大小（batch_size）从 8 降至 4、2，甚至 1。此外，降低最大长度（allowed_max_length）从 1024 调整为 512 或 256，也有助于缓解内存问题。

7.7 提取并保存响应

torch.manual_seed(123)
for entry in test_data[:3]:                #A
    input_text = format_input(entry)
    token_ids = generate(                  #B
        model=model,
        idx=text_to_token_ids(input_text, tokenizer).to(device),
        max_new_tokens=256,
        context_size=BASE_CONFIG["context_length"],
        eos_id=50256
    )
    generated_text = token_ids_to_text(token_ids, tokenizer)
    response_text = generated_text[len(input_text):].replace("### Response:",
"").strip()

    print(input_text)
    print(f"\nCorrect response:\n>> {entry['output']}")
    print(f"\nModel response:\n>> {response_text.strip()}")
    print("-------------------------------------")


#A 遍历测试集中的前三个样本
#B 使用在第 7.5 节导入的 generate 函数

Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
Rewrite the sentence using a simile.

### Input:
The car is very fast.

Correct response:
>> The car is as fast as lightning.

Model response:
>> The car is as fast as a bullet.
-------------------------------------
Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
What type of cloud is typically associated with thunderstorms?

Correct response:
>> The type of cloud typically associated with thunderstorms is cumulonimbus.

Model response:
>> The type of cloud associated with thunderstorms is a cumulus cloud.
-------------------------------------
Below is an instruction that describes a task. Write a response that appropriately
completes the request.

### Instruction:
Name the author of 'Pride and Prejudice'.

Correct response:
>> Jane Austen.

Model response:
>> The author of 'Pride and Prejudice' is Jane Austen.
-------------------------------------

• 简答题和多项选择题的基准测试（如 MMLU，”评估大规模多任务语言理解能力”，论文地址：https://arxiv.org/abs/2009.03300），用于测试模型的通用知识水平。
• 基于人类偏好对其他大语言模型进行比较，例如 LMSYS 的 Chatbot Arena 平台（https://arena.lmsys.org）。
• 自动化对话基准测试，使用像 GPT-4 这样的 LLM 来评估回答，例如 AlpacaEval（https://tatsulab.github.io/alpaca_eval/）。

# Listing 7.9 Generating test set responses
from tqdm import tqdm

for i, entry in tqdm(enumerate(test_data), total=len(test_data)):
    input_text = format_input(entry)

    token_ids = generate(
        model=model,
        idx=text_to_token_ids(input_text, tokenizer).to(device),
        max_new_tokens=256,
        context_size=BASE_CONFIG["context_length"],
        eos_id=50256
    )
    generated_text = token_ids_to_text(token_ids, tokenizer)
    response_text = generated_text[len(input_text):].replace("### Response:",
"").strip()
    test_data[i]["model_response"] = response_text

with open("instruction-data-with-response.json", "w") as file:
    json.dump(test_data, file, indent=4) # "indent" for pretty-printing

100%|██████████| 110/110 [01:05<00:00, 1.68it/s]

{'instruction': 'Rewrite the sentence using a simile.', 'input': 'The car is very
fast.', 'output': 'The car is as fast as lightning.', 'model_response': 'The car is as
fast as a bullet.'}

import re

# Remove white spaces and parentheses from file name
file_name = f"{re.sub(r'[ ()]', '', CHOOSE_MODEL) }-sft.pth"
torch.save(model.state_dict(), file_name)
print(f"Model saved as {file_name}")
The saved model can then be loaded via model.load_state_dict(torch.load("gpt2-
medium355M-sft.pth")).

7.8 评估指令微调后的 LLM

通过Web API使用更强大的 LLM

拥有 80 亿参数的 Llama 3 模型是一款性能非常强大的 LLM，能够在本地运行。然而，与 OpenAI 提供的 GPT-4 等商业化大模型相比，Llama 3 的能力稍显不足。如果读者感兴趣，可以通过 OpenAI 的 API 使用 GPT-4 来评估生成的模型响应。相关代码笔记本已作为本书的补充材料提供，读者可访问以下 GitHub 链接获取更多信息： https://github.com/rasbt/LLMs-from-scratch/blob/main/ch07/03_model-evaluation/llm-instruction-eval-openai.ipynb

• 针对 macOS 和 Windows 用户：打开已下载的 Ollama 应用。如果提示安装命令行工具，请选择‘是’。
• 针对 Linux 用户：请使用 Ollama 网站提供的安装命令。

pulling manifest
pulling 6a0746a1ec1a... 100% ▕████████████████▏ 4.7 GB
pulling 4fa551d4f938... 100% ▕████████████████▏ 12 KB
pulling 8ab4849b038c... 100% ▕████████████████▏ 254 B
pulling 577073ffcc6c... 100% ▕████████████████▏ 110 B
pulling 3f8eb4da87fa... 100% ▕████████████████▏ 485 B
verifying sha256 digest
writing manifest
removing any unused layers
success

Ollama 模型的替代方案

需要注意的是，ollama run llama3 命令中的 llama3 指的是一个经过指令微调的 Llama 3 模型，具有 80 亿参数。运行 llama3 模型时，大约需要 16 GB 的内存。如果设备内存不足，建议尝试更小的模型，例如参数量为 38 亿的 phi-3 模型，该模型通过 ollama run llama3 命令加载，仅需约 8 GB 内存即可运行。

对于高性能计算机，你可以选择更大的 Llama 3 模型（700 亿参数版本），只需将 llama3 替换为 llama3:70b。但请注意，该模型对计算资源的需求会显著增加。

>>> What do llamas eat?
Llamas are ruminant animals, which means they have a four-chambered
stomach and eat plants that are high in fiber. In the wild, llamas
typically feed on:
1. Grasses: They love to graze on various types of grasses, including tall
grasses, wheat, oats, and barley.

import psutil

def check_if_running(process_name):
    running = False
    for proc in psutil.process_iter(["name"]):
        if process_name in proc.info["name"]:
            running = True
            break
    return running

ollama_running = check_if_running("ollama")

if not ollama_running:
    raise RuntimeError("Ollama not running. Launch ollama before proceeding.")
print("Ollama running:", check_if_running("ollama"))

在一个新的 Python 会话中运行代码

如果你在第 7.7 节后关闭了 Python 会话，或者希望在新的会话中运行本章后续代码，可以执行以下代码。这些代码将加载我们在第 7.7 节中创建的指令和响应数据文件，同时重新定义之前使用的 format_input 函数（后续代码中还会使用 tqdm 进度条工具）。

import json
from tqdm import tqdm

file_path = "instruction-data-with-response.json"
with open(file_path, "r") as file:
    test_data = json.load(file)

def format_input(entry):
    instruction_text = (
        f"Below is an instruction that describes a task. "
        f"Write a response that appropriately completes the request."
        f"\n\n### Instruction:\n{entry['instruction']}"
    )

    input_text = f"\n\n### Input:\n{entry['input']}" if entry["input"] else ""
    return instruction_text + input_text

# Listing 7.10 Querying a local Ollama model
import urllib.request

def query_model(prompt, model="llama3", url="http://localhost:11434/api/chat"):
    data = {                                                               #A
        "model": model,
        "seed": 123, # for deterministic responses
        "temperature": 0, # for deterministic responses
        "messages": [
            {"role": "user", "content": prompt}
        ]
    }

    payload = json.dumps(data).encode("utf-8")                             #B
    request = urllib.request.Request(url, data=payload, method="POST")     #C
    request.add_header("Content-Type", "application/json")                 #C

    response_data = ""
    with urllib.request.urlopen(request) as response:                      #D
        while True:
            line = response.readline().decode("utf-8")
            if not line:
                break
            response_json = json.loads(line)
            response_data += response_json["message"]["content"]
    return response_data


#A 将数据载荷创建为字典格式
#B 将字典转换为 JSON 格式字符串，并编码为字节数据
#C 创建请求对象，设置方法为 POST，并添加必要的请求头
#D 发送请求并接收响应

model = "llama3"
result = query_model("What do Llamas eat?", model)
print(result)

Llamas are ruminant animals, which means they have a four-chambered stomach that allows
them to digest plant-based foods. Their diet typically consists of:

1. Grasses: Llamas love to graze on grasses, including tall grasses, short grasses, and
even weeds.
...

for entry in test_data[:3]:
    prompt = (
        f"Given the input `{format_input(entry)}` "
        f"and correct output `{entry['output']}`, "
        f"score the model response `{entry['model_response']}`"
        f" on a scale from 0 to 100, where 100 is the best score. "
    )
    print("\nDataset response:")
    print(">>", entry['output'])
    print("\nModel response:")
    print(">>", entry["model_response"])
    print("\nScore:")
    print(">>", query_model(prompt))
    print("\n-------------------------")

Dataset response:
>> The car is as fast as lightning.

Model response:
>> The car is as fast as a bullet.

Score:
>> A scoring task!

To evaluate the model response "The car is as fast as a bullet.", I'll consider how well
it follows the instruction and uses a simile that's coherent, natural-sounding, and
effective in conveying the idea of speed.

Here are some factors to consider:

1. **Follows instruction**: Yes, the model uses a simile to rewrite the sentence.
2. **Coherence and naturalness**: The comparison between the car's speed and a bullet is
common and easy to understand. It's a good choice for a simile that conveys the idea of
rapid movement.
3. **Effectiveness in conveying idea of speed**: A bullet is known for its high
velocity, which makes it an excellent choice to describe a fast-moving car.

Considering these factors, I'd score the model response "The car is as fast as a
bullet." around 85 out of 100. The simile is well-chosen, coherent, and effectively
conveys the idea of speed. Well done, model!

-------------------------

Dataset response:
>> The type of cloud typically associated with thunderstorms is cumulonimbus.

Model response:
>> The type of cloud associated with thunderstorms is a cumulus cloud.

Score:
>> A scoring task!

I'll evaluate the model's response based on its accuracy and relevance to the original
instruction.

**Accuracy:** The model's response is partially correct. Cumulus clouds are indeed
associated with fair weather and not typically linked to thunderstorms. The correct
answer, cumulonimbus, is a type of cloud that is closely tied to thunderstorm formation.

**Relevance:** The model's response is somewhat relevant, as it mentions clouds in the
context of thunderstorms. However, the specific type of cloud mentioned (cumulus) is not
directly related to thunderstorms.

Considering these factors, I would score the model response a **40 out of 100**. While
the response attempts to address the instruction, it provides an incorrect answer and
lacks relevance to the original question.

-------------------------

Dataset response:
>> Jane Austen.

Model response:
>> The author of 'Pride and Prejudice' is Jane Austen.

Score:
>> A simple one!

My model response: "The author of 'Pride and Prejudice' is Jane Austen."

Score: **99**

Reasoning:

* The response directly answers the question, providing the correct name of the author.
* The sentence structure is clear and easy to understand.
* There's no room for misinterpretation or ambiguity.

Overall, a perfect score!

-------------------------

# Listing 7.11 Evaluating the instruction finetuning LLM
def generate_model_scores(json_data, json_key, model="llama3"):
    scores = []
    for entry in tqdm(json_data, desc="Scoring entries"):
        prompt = (
            f"Given the input `{format_input(entry)}` "
            f"and correct output `{entry['output']}`, "
            f"score the model response `{entry[json_key]}`"
            f" on a scale from 0 to 100, where 100 is the best score. "
            f"Respond with the integer number only."                        #A
        )
        score = query_model(prompt, model)
        try:
            scores.append(int(score))
        except ValueError:
            print(f"Could not convert score: {score}")
            continue

    return scores

#A 修改后的指令设置为仅返回分数。

# Let's now apply the generate_model_scores function to the entire test_data set, which takes about 1 minute on a M3 Macbook Air:
scores = generate_model_scores(test_data, "model_response")
print(f"Number of scores: {len(scores)} of {len(test_data)}")
print(f"Average score: {sum(scores)/len(scores):.2f}\n")

Scoring entries: 100%|████████████████████████| 110/110 [01:10<00:00, 1.56it/s]
Number of scores: 110 of 110
Average score: 54.16

• 在微调阶段，可以通过调整超参数（如学习率、批次大小和训练轮数）来优化模型性能。
• 通过扩大训练数据集规模或丰富样本的多样性，以覆盖更广泛的主题和风格。
• 尝试不同的提示或指令格式，以更有效地引导模型的回答。
• 考虑使用更大的预训练模型，这类模型可能具有更强的能力，能够捕捉复杂模式并生成更准确的响应。

LLaMA 3 模型性能

作为参考，使用本节描述的方法，Llama 3 8B 基础模型（未经过任何微调）在测试集上的平均得分为 58.51。而经过在通用指令遵循数据集上微调的 Llama 3 8B 指令模型，在测试集上的平均得分高达 82.6，表现相当出色。

练习 7.4：使用 LoRA 实现参数高效微调

为了更高效地对 LLM 进行指令微调，请修改本章的代码，采用附录 E 中的 LoRA 方法。然后，对比修改前后训练时长和模型性能。

7.9 结语

7.9.1 接下来如何做？

7.9.2 如何在快速变化的前沿领域中保持领先

7.10 本章摘要

• 指令微调的过程旨在将预训练的大语言模型调整为能够遵循人类指令并生成预期回答。
• 准备数据集需要下载指令-响应数据集，对数据进行格式化，并划分为训练集、验证集和测试集。
• 自定义的collate函数用于构建训练批次，处理过程包括对序列数据进行填充，生成目标 token 的 ID，并对填充的 token 进行掩码处理。
• 我们加载了一个具有 3.55 亿参数的预训练 GPT-2 medium 模型，作为指令微调的起点。
• 预训练模型在指令数据集上进行了微调，训练方式类似于预训练的循环。
• 评估涉及在测试集上提取模型响应并对其进行评分，例如，使用另一个LLM进行评分。
• Ollama 应用利用一个 80 亿参数的 Llama 模型，可以对微调模型在测试集上的响应进行自动评分，并通过平均分来量化模型的性能表现。