千问Qwen2大语言模型推理

Halcom

Qwen2-1.5B-Instruct，在huggingface下载：https://hf-mirror.com/Qwen/Qwen2.5-1.5B-Instruct/tree/main
Qwen1.5 - a Qwen Collection
transformers代码在以下路径可追根，
D:\Users\WITIAI\anaconda3\Lib\site-packages\transformers\generation\utils.py
D:\Users\WITIAI\anaconda3\Lib\site-packages\transformers\models\qwen2\modeling_qwen2.py
千问Qwen2大语言模型推理

1. from dataclasses import dataclass, field
   
2. from typing import Optional
   
3. import torch
   
4. import torch.nn as nn
   
5. import transformers
   
6. from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedModel
   
7. import numpy as np
   
8. import random
   
9. 
   
10. @dataclass
    
11. class DecodeArguments:
    
12.     llm_type: str = 'qwen2'
    
13.     decode_type: str = 'llm'
    
14.     max_new_tokens: int = 50
    
15.     num_beams: int = 1
    
16.     batch_size: int = 1
    
17.     result_path: str = "result.txt"
    
18. 
    
19. @dataclass
    
20. class ModelArguments:
    
21.     llm_model_name_or_path: Optional[str] = "D:/2-LearningCode/913_LM/wesr-main/Qwen2-1.5B-Instruct"
    
22.     projector_hidden_size: int = 2048
    
23.     projector_model_path: Optional[str] = field(default=None)
    
24. 
    
25. class ProjectorConv1d(nn.Module):
    
26. 
    
27.     def __init__(self, config, encoder_dim, llm_dim):
    
28.         super().__init__()
    
29.         self.k = config.encoder_projector_ds_rate
    
30.         self.conv1d = nn.Conv1d(in_channels=encoder_dim,
    
31.                                 out_channels=encoder_dim,
    
32.                                 kernel_size=self.k,
    
33.                                 stride=self.k,
    
34.                                 padding=0)
    
35.         self.linear1 = nn.Linear(encoder_dim, config.projector_hidden_size)
    
36.         self.relu1 = nn.ReLU()
    
37.         self.linear2 = nn.Linear(config.projector_hidden_size, llm_dim)
    
38.         self.relu2 = nn.ReLU()
    
39. 
    
40.     def forward(self, x):
    
41.         x = x.transpose(1, 2)
    
42.         x = self.conv1d(x)
    
43.         x = x.transpose(1, 2)
    
44.         x = self.relu1(x)
    
45.         x = self.linear1(x)
    
46.         x = self.relu2(x)
    
47.         x = self.linear2(x)
    
48.         return x
    
49. 
    
50. def freeze_model(model):
    
51.     for _, param in model.named_parameters():
    
52.         param.requires_grad = False
    
53. 
    
54. class SpeechLLM(PreTrainedModel):
    
55.     supports_gradient_checkpointing = True
    
56. 
    
57.     def __init__(
    
58.         self,
    
59.         llm: nn.Module,
    
60.         config,
    
61.         model_args: ModelArguments,
    
62.     ):
    
63.         super().__init__(config)
    
64.         self.llm = llm
    
65.         
    
66.     @torch.autocast(device_type="cuda", dtype=torch.bfloat16)
    
67.     def generate(
    
68.         self,
    
69.         input_ids: torch.LongTensor = None,
    
70.         attention_mask: Optional[torch.Tensor] = None,
    
71.         eos_token_id=None,
    
72.         decode_config=None,
    
73.     ):
    
74.         text_emb = self.llm.get_input_embeddings()(input_ids)
    
75.         model_outputs = self.llm.generate(
    
76.             inputs_embeds=text_emb,
    
77.             attention_mask=attention_mask,
    
78.             do_sample=False,
    
79.             top_p=1.0,
    
80.             num_beams=decode_config.num_beams,
    
81.             max_new_tokens=decode_config.max_new_tokens,
    
82.             eos_token_id=eos_token_id,
    
83.         )
    
84.         return model_outputs
    
85. 
    
86. def init_model(model_args):
    
87.     # Load llm model and tokenizer
    
88.     config = transformers.AutoConfig.from_pretrained(
    
89.         model_args.llm_model_name_or_path)
    
90.     config.use_cache = False
    
91.     llm_model = AutoModelForCausalLM.from_pretrained(
    
92.         model_args.llm_model_name_or_path,
    
93.         config=config,
    
94.         torch_dtype='auto',
    
95.     )
    
96. 
    
97.     model = SpeechLLM(llm_model, config, model_args)
    
98.    
    
99.     if model_args.projector_model_path is not None:
    
100.         model.load_projector(model_args.projector_model_path)
     
101.         
     
102.     return model
     
103. 
     
104. def set_seed(seed=42):
     
105.     # 设置PyTorch的随机种子
     
106.     torch.manual_seed(seed)
     
107.     # 如果使用GPU，还需要设置CUDA的随机种子
     
108.     if torch.cuda.is_available():
     
109.         torch.cuda.manual_seed(seed)
     
110.         torch.cuda.manual_seed_all(seed)  # 多GPU时使用
     
111.         # 确保每次返回的卷积算法是确定的
     
112.         torch.backends.cudnn.deterministic = True
     
113.         torch.backends.cudnn.benchmark = False
     
114.    
     
115.     # 设置numpy的随机种子
     
116.     np.random.seed(seed)
     
117.     # 设置Python的随机种子
     
118.     random.seed(seed)
     
119.    
     
120. def main():
     
121.     set_seed()
     
122.     model_args = ModelArguments()
     
123.     decode_args = DecodeArguments()
     
124.    
     
125.     model = init_model(model_args)
     
126.     tokenizer = AutoTokenizer.from_pretrained(model_args.llm_model_name_or_path)
     
127.    
     
128.     text = "halcom.cn, welcome to halcom.cn"
     
129.     ids = tokenizer.encode(text)
     
130.     print("Encoded:", ids)
     
131.     print("Decoded:", tokenizer.decode(ids))
     
132.    
     
133.     if decode_args.llm_type == 'qwen2':
     
134.         eos_token_id = tokenizer.convert_tokens_to_ids(['<|endoftext|>', '<|im_end|>'])
     
135.     else:
     
136.         tokenizer.pad_token = '<|finetune_right_pad_id|>'
     
137.         eos_token_id = tokenizer.convert_tokens_to_ids(['<|end_of_text|>', '<|eot_id|>'])
     
138.    
     
139.     device = torch.device('cuda:0')
     
140.     if torch.cuda.is_available():
     
141.         model = model.cuda()
     
142.     else:
     
143.         device = torch.device('cpu')  
     
144. 
     
145.     model.eval()
     
146.    
     
147.     prompt = '编写一段代码'
     
148. 
     
149.     vocab_size = max(eos_token_id)  # Reduced to avoid vocab size error
     
150.     max_seq_len = 50
     
151.     input_ids = tokenizer.encode(prompt)
     
152.     input_ids = [i for i in input_ids if i < vocab_size][:max_seq_len]  # 修复：过滤非法 token id
     
153.     input_tensor = torch.tensor([input_ids], dtype=torch.long).to(device)
     
154.     seq_len = input_tensor.size(1)
     
155.     mask = torch.tril(torch.ones(1, seq_len)).to(device)
     
156.    
     
157.     generated_ids = model.generate(input_tensor, mask,
     
158.                                 eos_token_id=eos_token_id,
     
159.                                 decode_config=decode_args)
     
160.     # generated_text = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
     
161.     generated_text = tokenizer.decode(generated_ids.squeeze(0), skip_special_tokens=True)
     
162.    
     
163.     print(f'输入: {prompt}\n生成: {generated_text}')
     
164. 
     
165. if __name__ == "__main__":
     
166.     main()
     

复制代码

Halcom

1. input_ids
   
2. Out  [19]:
   
3. tensor([[151644,   8948,    198,   2610,    525,    264,  10950,  17847,     13,
   
4.          151645,    198, 151644,    872,    198,  35127,    752,    264,   2805,
   
5.           16800,    311,   3460,   4128,   1614,     13, 151645,    198, 151644,
   
6.           77091,    198]], device='cuda:0')
   
7. 
   
8. attention_mask
   
9. Out  [20]:
   
10. tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    
11.          1, 1, 1, 1, 1]], device='cuda:0')
    
12. 
    
13. position_ids
    
14. Out  [21]:
    
15. tensor([[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
    
16.          18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]], device='cuda:0')
    
17. 
    
18. past_key_values
    
19. 
    
20. inputs_embeds
    
21. 
    
22. use_cache
    
23. Out  [24]: True
    
24. 
    
25. output_attentions
    
26. Out  [25]: False
    
27. 
    
28. output_hidden_states
    
29. Out  [26]: False
    
30. 
    
31. return_dict
    
32. Out  [27]: True
    

复制代码

Halcom

采用，llama调用大模型，采用b5587这个吧，cuda11.7，适配当前电脑环境。https://github.com/ggml-org/llama.cpp/releases?page=88
transformer架构，没法直接pytorch的jit工具转化pt，提示类型不一致的问题。  

亲测，可以用llama也可以用inferflow来调用。