千问Qwen2大语言模型推理

Halcom

Qwen2-1.5B-Instruct，在huggingface下载
千问Qwen2大语言模型推理

1. from dataclasses import dataclass, field
   
2. from typing import Optional
   
3. import torch
   
4. import torch.nn as nn
   
5. import transformers
   
6. from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedModel
   
7. import numpy as np
   
8. import random
   
9. 
   
10. @dataclass
    
11. class DecodeArguments:
    
12.     llm_type: str = 'qwen2'
    
13.     decode_type: str = 'llm'
    
14.     max_new_tokens: int = 50
    
15.     num_beams: int = 1
    
16.     batch_size: int = 1
    
17.     result_path: str = "result.txt"
    
18. 
    
19. @dataclass
    
20. class ModelArguments:
    
21.     llm_model_name_or_path: Optional[str] = "D:/2-LearningCode/913_LM/wesr-main/Qwen2-1.5B-Instruct"
    
22.     projector_hidden_size: int = 2048
    
23.     projector_model_path: Optional[str] = field(default=None)
    
24. 
    
25. class ProjectorConv1d(nn.Module):
    
26. 
    
27.     def __init__(self, config, encoder_dim, llm_dim):
    
28.         super().__init__()
    
29.         self.k = config.encoder_projector_ds_rate
    
30.         self.conv1d = nn.Conv1d(in_channels=encoder_dim,
    
31.                                 out_channels=encoder_dim,
    
32.                                 kernel_size=self.k,
    
33.                                 stride=self.k,
    
34.                                 padding=0)
    
35.         self.linear1 = nn.Linear(encoder_dim, config.projector_hidden_size)
    
36.         self.relu1 = nn.ReLU()
    
37.         self.linear2 = nn.Linear(config.projector_hidden_size, llm_dim)
    
38.         self.relu2 = nn.ReLU()
    
39. 
    
40.     def forward(self, x):
    
41.         x = x.transpose(1, 2)
    
42.         x = self.conv1d(x)
    
43.         x = x.transpose(1, 2)
    
44.         x = self.relu1(x)
    
45.         x = self.linear1(x)
    
46.         x = self.relu2(x)
    
47.         x = self.linear2(x)
    
48.         return x
    
49. 
    
50. def freeze_model(model):
    
51.     for _, param in model.named_parameters():
    
52.         param.requires_grad = False
    
53. 
    
54. class SpeechLLM(PreTrainedModel):
    
55.     supports_gradient_checkpointing = True
    
56. 
    
57.     def __init__(
    
58.         self,
    
59.         llm: nn.Module,
    
60.         config,
    
61.         model_args: ModelArguments,
    
62.     ):
    
63.         super().__init__(config)
    
64.         self.llm = llm
    
65.         
    
66.     @torch.autocast(device_type="cuda", dtype=torch.bfloat16)
    
67.     def generate(
    
68.         self,
    
69.         input_ids: torch.LongTensor = None,
    
70.         attention_mask: Optional[torch.Tensor] = None,
    
71.         eos_token_id=None,
    
72.         decode_config=None,
    
73.     ):
    
74.         text_emb = self.llm.get_input_embeddings()(input_ids)
    
75.         model_outputs = self.llm.generate(
    
76.             inputs_embeds=text_emb,
    
77.             attention_mask=attention_mask,
    
78.             do_sample=False,
    
79.             top_p=1.0,
    
80.             num_beams=decode_config.num_beams,
    
81.             max_new_tokens=decode_config.max_new_tokens,
    
82.             eos_token_id=eos_token_id,
    
83.         )
    
84.         return model_outputs
    
85. 
    
86. def init_model(model_args):
    
87.     # Load llm model and tokenizer
    
88.     config = transformers.AutoConfig.from_pretrained(
    
89.         model_args.llm_model_name_or_path)
    
90.     config.use_cache = False
    
91.     llm_model = AutoModelForCausalLM.from_pretrained(
    
92.         model_args.llm_model_name_or_path,
    
93.         config=config,
    
94.         torch_dtype='auto',
    
95.     )
    
96. 
    
97.     model = SpeechLLM(llm_model, config, model_args)
    
98.    
    
99.     if model_args.projector_model_path is not None:
    
100.         model.load_projector(model_args.projector_model_path)
     
101.         
     
102.     return model
     
103. 
     
104. def set_seed(seed=42):
     
105.     # 设置PyTorch的随机种子
     
106.     torch.manual_seed(seed)
     
107.     # 如果使用GPU，还需要设置CUDA的随机种子
     
108.     if torch.cuda.is_available():
     
109.         torch.cuda.manual_seed(seed)
     
110.         torch.cuda.manual_seed_all(seed)  # 多GPU时使用
     
111.         # 确保每次返回的卷积算法是确定的
     
112.         torch.backends.cudnn.deterministic = True
     
113.         torch.backends.cudnn.benchmark = False
     
114.    
     
115.     # 设置numpy的随机种子
     
116.     np.random.seed(seed)
     
117.     # 设置Python的随机种子
     
118.     random.seed(seed)
     
119.    
     
120. def main():
     
121.     set_seed()
     
122.     model_args = ModelArguments()
     
123.     decode_args = DecodeArguments()
     
124.    
     
125.     model = init_model(model_args)
     
126.     tokenizer = AutoTokenizer.from_pretrained(model_args.llm_model_name_or_path)
     
127.    
     
128.     text = "halcom.cn, welcome to halcom.cn"
     
129.     ids = tokenizer.encode(text)
     
130.     print("Encoded:", ids)
     
131.     print("Decoded:", tokenizer.decode(ids))
     
132.    
     
133.     if decode_args.llm_type == 'qwen2':
     
134.         eos_token_id = tokenizer.convert_tokens_to_ids(['<|endoftext|>', '<|im_end|>'])
     
135.     else:
     
136.         tokenizer.pad_token = '<|finetune_right_pad_id|>'
     
137.         eos_token_id = tokenizer.convert_tokens_to_ids(['<|end_of_text|>', '<|eot_id|>'])
     
138.    
     
139.     device = torch.device('cuda:0')
     
140.     if torch.cuda.is_available():
     
141.         model = model.cuda()
     
142.     else:
     
143.         device = torch.device('cpu')  
     
144. 
     
145.     model.eval()
     
146.    
     
147.     prompt = '编写一段代码'
     
148. 
     
149.     vocab_size = max(eos_token_id)  # Reduced to avoid vocab size error
     
150.     max_seq_len = 50
     
151.     input_ids = tokenizer.encode(prompt)
     
152.     input_ids = [i for i in input_ids if i < vocab_size][:max_seq_len]  # 修复：过滤非法 token id
     
153.     input_tensor = torch.tensor([input_ids], dtype=torch.long).to(device)
     
154.     seq_len = input_tensor.size(1)
     
155.     mask = torch.tril(torch.ones(1, seq_len)).to(device)
     
156.    
     
157.     generated_ids = model.generate(input_tensor, mask,
     
158.                                 eos_token_id=eos_token_id,
     
159.                                 decode_config=decode_args)
     
160.     # generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
     
161.     generated_text = tokenizer.decode(generated_ids.squeeze(0), skip_special_tokens=True)
     
162.    
     
163.     print(f'输入: {prompt}\n生成: {generated_text}')
     
164. 
     
165. if __name__ == "__main__":
     
166.     main()
     

复制代码