PyTorch一维数据处理

Halcom

电脑配置：WIN10-64bit-Anaconda3-2018.12-Windows-x86_64-numpy-1.15.4+vanilla-cp37-cp37m-win_amd64.whl-opencv_python-4.1.0-cp37-cp37m-win_amd64.whl
Anaconda Prompt-->conda list-->pytorch=1.0.1-->opencv-python=4.1.0-->torchvision=0.2.2

主程序如下：

1. import pandas as pd
   
2. import numpy as np
   
3. from collections import Counter
   
4. from sklearn.model_selection import train_test_split
   
5. from sklearn.preprocessing import LabelEncoder
   
6. import torch
   
7. from torch.utils.data import Dataset, DataLoader
   
8. import torch.optim as torch_optim
   
9. import torch.nn as nn
   
10. import torch.nn.functional as F
    
11. from torchvision import models
    
12. from datetime import datetime
    
13. #from dataset import ShelterOutcomeDataset, get_default_device
    
14. from dataset4 import *
    
15. from fasterAIModel4 import *
    
16. 
    
17. # Load Data
    
18. train = pd.read_csv(r'./Data/train.csv')
    
19. print("Shape:", train.shape)
    
20. train.head()
    
21. 
    
22. test = pd.read_csv(r'./Data/test.csv')
    
23. print("Shape:", test.shape)
    
24. test.head()
    
25. 
    
26. sample = pd.read_csv(r'./Data/sample_submission.csv')
    
27. sample.head()
    
28. 
    
29. # Very basic data exploration
    
30. Counter(train['OutcomeType'])
    
31. Counter(train['Name']).most_common(5)
    
32. 
    
33. # Data preprocessing
    
34. train_X = train.drop(columns= ['OutcomeType', 'OutcomeSubtype', 'AnimalID'])
    
35. Y = train['OutcomeType']
    
36. test_X = test
    
37. 
    
38. # Stacking train and test set so that they undergo the same preprocessing
    
39. stacked_df = train_X.append(test_X.drop(columns=['ID']))
    
40. # stacked_df['DateTime'] = pd.to_datetime(stacked_df['DateTime'])
    
41. # stacked_df['year'] = stacked_df['DateTime'].dt.year
    
42. # stacked_df['month'] = stacked_df['DateTime'].dt.month
    
43. stacked_df = stacked_df.drop(columns=['DateTime'])
    
44. stacked_df.head()
    
45. 
    
46. # dropping columns with too many nulls
    
47. for col in stacked_df.columns:
    
48.     if stacked_df[col].isnull().sum() > 10000:
    
49.         print("dropping", col, stacked_df[col].isnull().sum())
    
50.         stacked_df = stacked_df.drop(columns = [col])
    
51. stacked_df.head()
    
52. 
    
53. # label encoding
    
54. for col in stacked_df.columns:
    
55.     if stacked_df.dtypes[col] == "object":
    
56.         stacked_df[col] = stacked_df[col].fillna("NA")
    
57.     else:
    
58.         stacked_df[col] = stacked_df[col].fillna(0)
    
59.     stacked_df[col] = LabelEncoder().fit_transform(stacked_df[col])
    
60. 
    
61. # making all variables categorical
    
62. for col in stacked_df.columns:
    
63.     stacked_df[col] = stacked_df[col].astype('category')
    
64. 
    
65. # splitting back train and test
    
66. X = stacked_df[0:26729]
    
67. test_processed = stacked_df[26729:]
    
68. 
    
69. #check if shape[0] matches original
    
70. print("train shape: ", X.shape, "orignal: ", train.shape)
    
71. print("test shape: ", test_processed.shape, "original: ", test.shape)
    
72. 
    
73. # Encoding target
    
74. Y = LabelEncoder().fit_transform(Y)
    
75. #sanity check to see numbers match and matching with previous counter to create target dictionary
    
76. print(Counter(train['OutcomeType']))
    
77. print(Counter(Y))
    
78. target_dict = {
    
79.     'Return_to_owner' : 3,
    
80.     'Euthanasia': 2,
    
81.     'Adoption': 0,
    
82.     'Transfer': 4,
    
83.     'Died': 1
    
84. }
    
85. 
    
86. #train-valid split
    
87. X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size=0.10, random_state=0)
    
88. X_train.head()
    
89. 
    
90. # Choosing columns for embedding
    
91. #categorical embedding for columns having more than two values
    
92. embedded_cols = {n: len(col.cat.categories) for n,col in X.items() if len(col.cat.categories) > 1}
    
93. embedded_cols
    
94. embedded_col_names = embedded_cols.keys()
    
95. len(X.columns) - len(embedded_cols) #number of numerical columns
    
96. #Determining size of embedding
    
97. #(borrowed from https://www.usfca.edu/data-institute/certificates/fundamentals-deep-learning lesson 2)
    
98. embedding_sizes = [(n_categories, min(50, (n_categories+1)//2)) for _,n_categories in embedded_cols.items()]
    
99. embedding_sizes
    
100. 
     
101. #creating train and valid datasets
     
102. train_ds = ShelterOutcomeDataset(X_train, y_train, embedded_col_names)
     
103. valid_ds = ShelterOutcomeDataset(X_val, y_val, embedded_col_names)
     
104. 
     
105. device = get_default_device()
     
106. device
     
107. 
     
108. model = ShelterOutcomeModel(embedding_sizes)
     
109. to_device(model, device)
     
110. 
     
111. batch_size = 1000
     
112. train_dl = DataLoader(train_ds, batch_size=batch_size,shuffle=True)
     
113. valid_dl = DataLoader(valid_ds, batch_size=batch_size,shuffle=True)
     
114. 
     
115. train_dl = DeviceDataLoader(train_dl, device)
     
116. valid_dl = DeviceDataLoader(valid_dl, device)
     
117. 
     
118. train_loop(device, model, train_dl, valid_dl, embedding_sizes, epochs=8, lr=0.05, wd=0.00001)
     
119. 
     
120. test_ds = ShelterOutcomeDataset(test_processed, np.zeros(len(test_processed)), embedded_col_names)
     
121. batch_size = 1000
     
122. test_dl = DataLoader(test_ds, batch_size=batch_size)
     
123. test_dl = DeviceDataLoader(test_dl, device)
     
124. 
     
125. preds = []
     
126. with torch.no_grad():
     
127.     for x,y in test_dl:
     
128.         out = model(x.long())
     
129.         prob = F.softmax(out, dim=1)
     
130. #        print('{0}'.format(prob))
     
131.         preds.append(prob)
     
132. final_probs = [item for sublist in preds for item in sublist]
     
133. len(final_probs)
     
134. 
     
135. 
     
136. model_test = []
     
137. model_test = ShelterOutcomeModel(embedding_sizes).to(device)
     
138. model_test.cuda(0)        # GPU
     
139. model_test = torch.jit.load('./CheckPoints/Pytorch1D.pt')
     
140. model_test.eval()
     
141. preds2 = []
     
142. with torch.no_grad():
     
143.     for x,y in test_dl:
     
144.         out = model_test(x.long())
     
145.         prob2 = F.softmax(out, dim=1)
     
146. #        print('{0}'.format(prob))
     
147.         preds2.append(prob2)
     
148. final_probs2 = [item for sublist in preds2 for item in sublist]
     
149. len(final_probs2)
     
150. 
     
151.    
     
152. # output
     
153. sample.head()
     
154. sample['Adoption'] = [float(t[0]) for t in final_probs]
     
155. sample['Died'] = [float(t[1]) for t in final_probs]
     
156. sample['Euthanasia'] = [float(t[2]) for t in final_probs]
     
157. sample['Return_to_owner'] = [float(t[3]) for t in final_probs]
     
158. sample['Transfer'] = [float(t[4]) for t in final_probs]
     
159. sample.head()
     
160. sample.to_csv('samp.csv', index=False)

复制代码

fasterAIModel4.py脚本如下：

1. #import pandas as pd
   
2. import numpy as np
   
3. #from collections import Counter
   
4. #from sklearn.model_selection import train_test_split
   
5. #from sklearn.preprocessing import LabelEncoder
   
6. import torch
   
7. #from torch.utils.data import Dataset, DataLoader
   
8. import torch.optim as torch_optim
   
9. import torch.nn as nn
   
10. import torch.nn.functional as F
    
11. #from torchvision import models
    
12. #from datetime import datetime
    
13. class ShelterOutcomeModel(nn.Module):
    
14.     def __init__(self, embedding_sizes):
    
15.         super().__init__()
    
16.         self.embeddings = nn.ModuleList([nn.Embedding(categories, size) for categories,size in embedding_sizes])
    
17.         n_emb = sum(e.embedding_dim for e in self.embeddings) #length of all embeddings combined
    
18.         self.n_emb = n_emb
    
19.         self.lin1 = nn.Linear(self.n_emb, 200)
    
20.         self.lin2 = nn.Linear(200, 70)
    
21.         self.lin3 = nn.Linear(70, 5)
    
22.         self.bn1 = nn.BatchNorm1d(self.n_emb)
    
23.         self.bn2 = nn.BatchNorm1d(200)
    
24.         self.bn3 = nn.BatchNorm1d(70)
    
25.         self.emb_drop = nn.Dropout(0.6)
    
26.         self.drops = nn.Dropout(0.3)
    
27. 
    
28.     def forward(self, x_cat):
    
29.         x = [e(x_cat[:,i]) for i,e in enumerate(self.embeddings)]
    
30.         x = torch.cat(x, 1)
    
31.         x = self.emb_drop(x)
    
32.         x = self.bn1(x)
    
33.         x = F.relu(self.lin1(x))
    
34.         x = self.drops(x)
    
35.         x = self.bn2(x)
    
36.         x = F.relu(self.lin2(x))
    
37.         x = self.drops(x)
    
38.         x = self.bn3(x)
    
39.         x = self.lin3(x)
    
40.         return x
    
41.    
    
42. def get_optimizer(model, lr = 0.001, wd = 0.0):
    
43.     parameters = filter(lambda p: p.requires_grad, model.parameters())
    
44.     optim = torch_optim.Adam(parameters, lr=lr, weight_decay=wd)
    
45.     return optim
    
46. 
    
47. def train_model(model, optim, train_dl):
    
48.     model.train()
    
49.     total = 0
    
50.     sum_loss = 0
    
51.     for x, y in train_dl:
    
52.         batch = y.shape[0]
    
53.         output = model(x.long())
    
54.         loss = F.cross_entropy(output, y.long())   
    
55.         optim.zero_grad()
    
56.         loss.backward()
    
57.         optim.step()
    
58.         total += batch
    
59.         sum_loss += batch*(loss.item())
    
60.     return sum_loss/total
    
61. 
    
62. def val_model(model, valid_dl):
    
63.     model.eval()
    
64.     total = 0
    
65.     sum_loss = 0
    
66.     correct = 0
    
67.     for x, y in valid_dl:
    
68.         current_batch_size = y.shape[0]
    
69.         out = model(x.long())
    
70.         loss = F.cross_entropy(out, y.long())
    
71.         sum_loss += current_batch_size*(loss.item())
    
72.         total += current_batch_size
    
73.         pred = torch.max(out, 1)[1]
    
74. #        print(pred.type())
    
75. #        print(y.type())
    
76.         correct += (pred == y.long()).float().sum().item()
    
77. #    print("valid loss %.3f and accuracy %.3f" % (sum_loss/total, correct/total))
    
78.     return sum_loss/total, correct/total
    
79. 
    
80. def train_loop(device, model, train_dl, valid_dl, embedding_sizes, epochs, lr=0.01, wd=0.0):
    
81.     optim = get_optimizer(model, lr = lr, wd = wd)
    
82.     train_loss_iter = np.inf
    
83.     val_loss_iter = np.inf
    
84.     for i in range(epochs):
    
85.         train_loss = train_model(model, optim, train_dl)
    
86.         if train_loss < train_loss_iter:
    
87.             train_loss_iter = train_loss
    
88.             torch.save(model.state_dict(), './CheckPoints/train_weights.pth')
    
89.         val_loss, val_Accuracy = val_model(model, valid_dl)
    
90.         print("epochs: {0}".format(i), "training loss: {:.5f}".format(train_loss), "Val loss: {:.5f}".format(val_loss), "Val Accuracy: {:.5f}".format(val_Accuracy) )
    
91.         if val_loss < val_loss_iter:
    
92.             val_loss_iter = val_loss
    
93.             torch.save(model.state_dict(), './CheckPoints/val_weights.pth')
    
94.     # 模型保存
    
95.     model_test = []
    
96.     model_test = ShelterOutcomeModel(embedding_sizes).to(device)
    
97.     model_test = model_test.to(device)
    
98. #    model_test.cuda(0)        # GPU
    
99.     model_test.load_state_dict(torch.load('./CheckPoints/val_weights.pth'))
    
100.     model_test.eval()
     
101.     example = torch.rand(1, 5).type(torch.LongTensor).cuda(0)
     
102.     with torch.no_grad():
     
103.         model_test = torch.jit.trace(model_test, example)
     
104.     model_test.save( './CheckPoints/Pytorch1D.pt' )

复制代码

dataset.py脚本如下：

1. #import pandas as pd
   
2. import numpy as np
   
3. #from collections import Counter
   
4. #from sklearn.model_selection import train_test_split
   
5. #from sklearn.preprocessing import LabelEncoder
   
6. import torch
   
7. from torch.utils.data import Dataset, DataLoader
   
8. #import torch.optim as torch_optim
   
9. #import torch.nn as nn
   
10. #import torch.nn.functional as F
    
11. #from torchvision import models
    
12. #from datetime import datetime
    
13. # Pytorch Dataset
    
14. class ShelterOutcomeDataset(Dataset):
    
15.     def __init__(self, X, Y, embedded_col_names):
    
16. #        X = X.copy()
    
17. #        self.X1 = X.loc[:,embedded_col_names].copy().values.astype(np.int64) #categorical columns
    
18. #        self.X2 = X.drop(columns=embedded_col_names).copy().values.astype(np.float32) #numerical columns
    
19. #        self.y = Y
    
20.         self.x = X.copy().values.astype(np.float32)
    
21.         self.y = Y
    
22.         
    
23.     def __len__(self):
    
24.         return len(self.y)
    
25.    
    
26.     def __getitem__(self, idx):
    
27.         return self.x[idx], self.y[idx]
    
28. 
    
29. def get_default_device():
    
30.     """Pick GPU if available, else CPU"""
    
31.     if torch.cuda.is_available():
    
32.         return torch.device('cuda')
    
33.     else:
    
34.         return torch.device('cpu')
    
35.    
    
36. def to_device(data, device):
    
37.     """Move tensor(s) to chosen device"""
    
38.     if isinstance(data, (list,tuple)):
    
39.         return [to_device(x, device) for x in data]
    
40.     return data.to(device, non_blocking=True)
    
41. 
    
42. class DeviceDataLoader():
    
43.     """Wrap a dataloader to move data to a device"""
    
44.     def __init__(self, dl, device):
    
45.         self.dl = dl
    
46.         self.device = device
    
47.         
    
48.     def __iter__(self):
    
49.         """Yield a batch of data after moving it to device"""
    
50.         for b in self.dl:
    
51.             yield to_device(b, self.device)
    
52. 
    
53.     def __len__(self):
    
54.         """Number of batches"""
    
55.         return len(self.dl)

复制代码

参考：
【1】用PyTorch实现多层网络
【2】Pytorch实战Kaggle房价预测比赛
【3】如何构建用于垃圾分类的图像分类器
【4】使用PyTorch进行表格数据的深度学习
【5】https://jovian.ml/aakanksha-ns/shelter-outcome
【6】PyTorch C++ libtorch的使用方法(2)-Qt中调用PyTorch模型