|
- import numpy as np
- from scipy.signal import resample
- from torch.autograd import Variable
- import torch
- import torch.nn as nn
- from torch.utils.data import Dataset, ConcatDataset, DataLoader
- import glob
- import argparse
- import time
- import torchsummary
- import os
- from data.labdata import *
- #from data.yiceng_labdata import *
- from model.transformer_2 import *
-
- c2net_context = prepare()
- datasetPath=c2net_context.dataset_path
- outputPath = c2net_context.output_path
- from c2net.context import upload_output
- upload_output()
-
-
-
- parser = argparse.ArgumentParser()
- parser.add_argument('--n_epochs', type=int, default=400, help='number of epochs of training')
- parser.add_argument('--batch_size', type=int, default=128, help='size of the batches')
- parser.add_argument('--lr', type=float, default=0.00005, help='adam: learning rate')
- parser.add_argument('--b1', type=float, default=0.9, help='adam: decay of first order momentum of gradient')
- parser.add_argument('--b2', type=float, default=0.999, help='adam: decay of first order momentum of gradient')
- parser.add_argument('--num_class', type=int, default=34, help='total structral conditions')
- parser.add_argument('--num_repeats', type=int, default=10, help='number of repeated tests for training')
- parser.add_argument('--path', type=str, default=datasetPath+'/labdata_vgg16' ,help='path of training data')
- parser.add_argument('--out_path', type=str, default=c2net_context.output_path,help='path of output')
- parser.add_argument('--encoder_finetune', type=bool, default=True,help='whether to finetune conv')
- parser.add_argument('--linear_finetune', type=bool, default=True,help='whether to finetune fc')
- opt = parser.parse_args()
-
- cuda = True if torch.cuda.is_available() else False
- num_class = opt.num_class
- num_repeats = opt.num_repeats
- total = 0
- correct = 0
- acc = []
- acc1 = []
-
- ### TL_transformer-qianyi ###
- # 加载预训练模型
- model_path = c2net_context.pretrain_model_path
- transformer_path = model_path + '/fourstorey_frame_transformer_2/model (transformer_2).pth'
- loaded_model_state_dict = torch.load(transformer_path)
- loaded_model = Transformer()
- loaded_model.load_state_dict(loaded_model_state_dict)
-
- d_model=100 #d_model得更改为100,即linear后的维度
- d_ff = 2048
- d_k = 64
- d_v = 64
- n_layers = 1
- n_heads = 8
- device=torch.device("cuda")
- def get_positional_encoding(max_seq_len, embed_dim):
- # 初始化一个positional encoding
- # embed_dim: 字嵌入的维度
- # max_seq_len: 最大的序列长度
- positional_encoding = np.array([
- [pos / np.power(10000, 2 * i / embed_dim) for i in range(embed_dim)]
- if pos != 0 else np.zeros(embed_dim) for pos in range(max_seq_len)])
-
- positional_encoding[1:, 0::2] = np.sin(positional_encoding[1:, 0::2]) # dim 2i 偶数
- positional_encoding[1:, 1::2] = np.cos(positional_encoding[1:, 1::2]) # dim 2i+1 奇数
- return positional_encoding
-
- class PositionalEncoding(nn.Module):
- def __init__(self, d_model, dropout=0.1, max_len=11): #5000/100
- super(PositionalEncoding, self).__init__()
- self.dropout = nn.Dropout(p=dropout)
-
- pe = torch.zeros(max_len, d_model)
- position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
- div_term = torch.exp(torch.arange(
- 0, d_model, 2).float() * (-math.log(10000.0) / d_model))
- pe[:, 0::2] = torch.sin(position * div_term)
- pe[:, 1::2] = torch.cos(position * div_term)
- pe = pe.unsqueeze(0).transpose(0, 1)
- self.register_buffer('pe', pe)
-
- def forward(self, x,cls_token):
- """
- x: [seq_len, batch_size, d_model]
- """
- x = x + self.pe[1:, :] # [:x.size(0),:]
- cls_token = cls_token + self.pe[:1,:]
- return self.dropout(x),cls_token
-
- class ScaledDotProductAttention(nn.Module):
- def __init__(self):
- super(ScaledDotProductAttention, self).__init__()
-
- def forward(self, Q, K, V):
- """
- Q: [batch_size, n_heads, len_q, d_k]
- K: [batch_size, n_heads, len_k, d_k]
- V: [batch_size, n_heads, len_v(=len_k), d_v]
- attn_mask: [batch_size, n_heads, seq_len, seq_len]
- 说明:在encoder-decoder的Attention层中len_q(q1,..qt)和len_k(k1,...km)可能不同
- """
- scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size, n_heads, len_q, len_k]
- # mask矩阵填充scores(用-1e9填充scores中与attn_mask中值为1位置相对应的元素)
- attn = nn.Softmax(dim=-1)(scores) # 对最后一个维度(v)做softmax
- # scores : [batch_size, n_heads, len_q, len_k] * V: [batch_size, n_heads, len_v(=len_k), d_v]
- context = torch.matmul(attn, V) # context: [batch_size, n_heads, len_q, d_v]
- # context:[[z1,z2,...],[...]]向量, attn注意力稀疏矩阵(用于可视化的)
- return context, attn
-
- # Pytorch中的Linear只会对最后一维操作,所以正好是我们希望的每个位置用同一个全连接网络
- class PoswiseFeedForwardNet(nn.Module):
- def __init__(self):
- super(PoswiseFeedForwardNet, self).__init__()
- self.fc = nn.Sequential(
- nn.Dropout(),
- nn.Linear(d_model, d_ff, bias=False),
- nn.ReLU(),
- nn.Dropout(),
- nn.Linear(d_ff, d_model, bias=False)
- )
-
- def forward(self, inputs):
- """
- inputs: [batch_size, seq_len, d_model]
- """
- residual = inputs
- output = self.fc(inputs)
- return nn.LayerNorm(d_model).to(device)(output + residual) # [batch_size, seq_len, d_model]
-
- class MultiHeadAttention(nn.Module):
- """这个Attention类可以实现:
- Encoder的Self-Attention
- Decoder的Masked Self-Attention
- Encoder-Decoder的Attention
- 输入:seq_len x d_model
- 输出:seq_len x d_model
- """
-
- def __init__(self):
- super(MultiHeadAttention, self).__init__()
- self.W_Q = nn.Linear(d_model, d_k * n_heads,
- bias=False) # q,k必须维度相同,不然无法做点积
- self.W_K = nn.Linear(d_model, d_k * n_heads, bias=False)
- self.W_V = nn.Linear(d_model, d_v * n_heads, bias=False)
- # 这个全连接层可以保证多头attention的输出仍然是seq_len x d_model
- self.fc = nn.Linear(n_heads * d_v, d_model, bias=False)
-
- def forward(self, input_Q, input_K, input_V):
- """
- input_Q: [batch_size, len_q, d_model]
- input_K: [batch_size, len_k, d_model]
- input_V: [batch_size, len_v(=len_k), d_model]
- attn_mask: [batch_size, seq_len, seq_len]
- """
- residual, batch_size = input_Q, input_Q.size(0)
- # 下面的多头的参数矩阵是放在一起做线性变换的,然后再拆成多个头,这是工程实现的技巧
- # B: batch_size, S:seq_len, D: dim
- # (B, S, D) -proj-> (B, S, D_new) -split-> (B, S, Head, W) -trans-> (B, Head, S, W)
- # 线性变换 拆成多头
-
- # Q: [batch_size, n_heads, len_q, d_k]
- Q = self.W_Q(input_Q).view(batch_size, -1,
- n_heads, d_k).transpose(1, 2)
- # K: [batch_size, n_heads, len_k, d_k] # K和V的长度一定相同,维度可以不同
- K = self.W_K(input_K).view(batch_size, -1,
- n_heads, d_k).transpose(1, 2)
- # V: [batch_size, n_heads, len_v(=len_k), d_v]
- V = self.W_V(input_V).view(batch_size, -1,
- n_heads, d_v).transpose(1, 2)
-
- # 因为是多头,所以mask矩阵要扩充成4维的
- # attn_mask: [batch_size, seq_len, seq_len] -> [batch_size, n_heads, seq_len, seq_len]
- #attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1)
-
- # context: [batch_size, n_heads, len_q, d_v], attn: [batch_size, n_heads, len_q, len_k]
- context, attn = ScaledDotProductAttention()(Q, K, V)
- # 下面将不同头的输出向量拼接在一起
- # context: [batch_size, n_heads, len_q, d_v] -> [batch_size, len_q, n_heads * d_v]
- context = context.transpose(1, 2).reshape(
- batch_size, -1, n_heads * d_v)
-
- # 这个全连接层可以保证多头attention的输出仍然是seq_len x d_model
- output = self.fc(context) # [batch_size, len_q, d_model]
- return nn.LayerNorm(d_model).to(device)(output + residual), attn
-
- class EncoderLayer(nn.Module):
- def __init__(self):
- super(EncoderLayer, self).__init__()
- self.enc_self_attn = MultiHeadAttention()
- self.pos_ffn = PoswiseFeedForwardNet()
-
- def forward(self, enc_inputs):
- """E
- enc_inputs: [batch_size, src_len, d_model]
- enc_self_attn_mask: [batch_size, src_len, src_len] mask矩阵(pad mask or sequence mask)
- """
- # enc_outputs: [batch_size, src_len, d_model], attn: [batch_size, n_heads, src_len, src_len]
- # 第一个enc_inputs * W_Q = Q
- # 第二个enc_inputs * W_K = K
- # 第三个enc_inputs * W_V = V
- enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs) # enc_inputs to same Q,K,V(未线性变换前)
- enc_outputs = self.pos_ffn(enc_outputs)
- # enc_outputs: [batch_size, src_len, d_model]
- return enc_outputs, attn
-
- class Encoder(nn.Module):
- def __init__(self):
- super(Encoder, self).__init__()
- #self.src_emb = nn.Embedding(total, d_model) # token Embedding
- self.Con1=nn.Conv1d(1,1,kernel_size=100,stride=100)
- self.Con2=nn.Conv1d(1,1,kernel_size=150,stride=100,padding=25)
- self.Con3=nn.Conv1d(1,1,kernel_size=200,stride=100,padding=50)
- self.cls_token=nn.Parameter(torch.zeros(1,1,100))
- self.pos_emb = PositionalEncoding(
- d_model) # Transformer中位置编码是固定的,不需要学习
- self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])
- self.linear = nn.Linear(15,100)
-
-
- def forward(self, enc_inputs):
- """
- enc_inputs: [batch_size, src_len]
- """
- #enc_outputs = self.src_emb(enc_inputs) # [batch_size, src_len, d_model]
- enc1_outputs = self.Con1(enc_inputs) #32*1*50
- enc2_outputs = self.Con2(enc_inputs)
- enc3_outputs = self.Con3(enc_inputs)
- new_matrix = torch.zeros(enc_inputs.size(0),1,100).to(device) #这里的100是每个小矩阵通过nn.linear后的维度数
- for i in range(10):
- piece = torch.cat((enc1_outputs[:,:,i*5:i*5+5],enc2_outputs[:,:,i*5:i*5+5],enc3_outputs[:,:,i*5:i*5+5]),dim=1) #32*3*5 #为什么在第8次循环之后张量尺寸变为8*3*5
- piece = piece.view(enc_inputs.size(0),1,15)
- piece = self.linear(piece) .to(device) #32*1*100
- new_matrix = torch.cat((new_matrix,piece),dim=1)
- enc_outputs = new_matrix[:,1:,:] #32×10×100
- #enc_outputs = torch.concat((enc1_outputs,enc2_outputs),dim=0) #2个张量矩阵!! 如何将其变成一个张量矩阵
- #enc_outputs = torch.add(enc1_outputs,enc2_outputs)
- #enc_outputs = enc_outputs.transpose(1,2)
- cls_token = self.cls_token.to(device)
- enc_outputs,cls_token = self.pos_emb(enc_outputs.transpose(0, 1),cls_token)
- enc_outputs = enc_outputs.transpose(0,1) # [batch_size, src_len, d_model]
- cls_tokens = cls_token.expand(enc_outputs.shape[0],-1,-1)
- enc_outputs = torch.cat((cls_tokens,enc_outputs),dim=1)
- # Encoder输入序列的pad mask矩阵
- enc_self_attns = [] # 在计算中不需要用到,它主要用来保存你接下来返回的attention的值(这个主要是为了你画热力图等,用来看各个词之间的关系
- for layer in self.layers: # for循环访问nn.ModuleList对象
- # 上一个block的输出enc_outputs作为当前block的输入
- # enc_outputs: [batch_size, src_len, d_model], enc_self_attn: [batch_size, n_heads, src_len, src_len]
- enc_outputs, enc_self_attn = layer(enc_outputs) # 传入的enc_outputs其实是input,传入mask矩阵是因为你要做self attention
- # 这个只是为了可视化
- return enc_outputs, enc_self_attns
-
- class TLtransformer_finetune(nn.Module):
- def __init__(self):
- super(TLtransformer_finetune, self).__init__()
- self.encoder = nn.Sequential(
- Encoder()
- )
- # self.projection = nn.Linear(
- # d_model, 37, bias=False)
- self.proj=nn.Linear(11*100,34)
-
- def forward(self, enc_inputs):
- """Transformers的输入:两个序列
- enc_inputs: [batch_size, src_len]
- """
- #tensor to store decoder outputs
- #outputs = torch.zeros(batch_size, tgt_len, tgt_vocab_size).to(self.device)
-
- #enc_outputs: [batch_size, src_len, d_model], enc_self_attns: [n_layers, batch_size, n_heads, src_len, src_len]
- #经过Encoder网络后,得到的输出还是[batch_size, src_len, d_model]
- enc_outputs, enc_self_attns = self.encoder(enc_inputs)
- #x = enc_outputs[:,0,:]
- enc_outputs = enc_outputs.view(enc_outputs.size(0),11*100)
- enc_outputs = self.proj(enc_outputs)
- return enc_outputs
-
- new_model = TLtransformer_finetune()
-
- # 复制参数到目标域模型
- loaded_dict = loaded_model.state_dict()
- new_dict = new_model.state_dict()
- for k, v in loaded_dict.items():
- if k in new_dict and v.size() == new_dict[k].size():
- new_dict[k] = v
-
- new_model.load_state_dict(new_dict)
- model = new_model
- # 选择微调方法
- for param in model.encoder[0].Con1.parameters():
- param.requires_grad = True
- for param in model.encoder[0].Con2.parameters():
- param.requires_grad = True
- for param in model.encoder[0].Con3.parameters():
- param.requires_grad = True
- for param in model.encoder[0].layers[0].pos_ffn.parameters():
- param.requires_grad = True
- for param in model.encoder[0].layers[0].enc_self_attn.parameters():
- param.requires_grad = True
- for param in model.proj.parameters():
- param.requires_grad = False
- # for param in model.encoder[0].layers[1].pos_ffn.parameters():
- # param.requires_grad = True
- # for param in model.encoder[0].layers[2].pos_ffn.parameters():
- # param.requires_grad = True
- # for param in model.encoder[0].layers[
-
- # for param in model.encoder[0].layers[4].pos_ffn.parameters():
- # param.requires_grad = True
- # for param in model.encoder[0].layers[5].pos_ffn.parameters():
- # param.requires_grad = True
-
-
- learning_rate = opt.lr
- print(model)
- ### ###
-
-
-
-
- learning_rate = opt.lr
- loss = nn.CrossEntropyLoss()
- optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(opt.b1, opt.b2))
-
- if cuda:
- model = model.cuda()
- loss = loss.cuda()
- model.eval()
- FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
- LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor
-
- traindataset = []
- for label in range(num_class):
- epdataset = epDataset(label, num_repeats)
- traindataset = ConcatDataset([traindataset, epdataset])
- testdataset = []
- for label in range(num_class):
- eptdataset = eptDataset(label, 10)
- testdataset = ConcatDataset([testdataset, eptdataset])
- dataloader = DataLoader(traindataset, batch_size=opt.batch_size, shuffle=True)
- viddataloader = DataLoader(testdataset, batch_size=opt.batch_size, shuffle=True)
- loss_r = []
- loss_vid = []
- start = time.time()
- #%%
- predicted1 = []
- label1 = []
- train_loss = []
- train_loss = []
- avg_train_losses = []
- avg_valid_losses = []
-
-
- out_dir_ckpt = os.path.join(opt.out_path, "ckpt") #checkpoint
- if not os.path.exists(out_dir_ckpt):
- os.makedirs(out_dir_ckpt)
- save_model_dir = os.path.join(out_dir_ckpt,"model.pth")
- min_mae=60
- for epoch in range(opt.n_epochs):
-
- print('Epoch: %d' % epoch)
- total = 0
- correct = 0
- total1 = 0
- correct1 = 0
-
- for i, (sdata, label) in enumerate(dataloader):
- batch_size = sdata.shape[0]
- sdata = Variable(sdata.type(FloatTensor))
- noise = Variable(torch.from_numpy(np.random.normal(1, 0.1, sdata.shape))).type(FloatTensor)
- x = torch.mul(sdata, noise).view(sdata.size(0), 1, sdata.size(1))
- label = Variable(label.type(LongTensor))
- optimizer.zero_grad()
- output = model(x)
- loss1 = loss(output, label)
- loss1.backward()
- optimizer.step()
- _, predicted = torch.max(model(x).data, 1)
- temp = (predicted == label).sum()
- correct += temp
- total += batch_size
- print(i, loss1.item())
- predicted1.append(predicted.cpu())
- label1.append(label.cpu())
- loss_r.append(loss1.item())
- print('Accuracy of training data is: %d %%' % (100 * correct / total))
- acc.append(100 * correct / total)
-
- for i, (sdata, label) in enumerate(viddataloader):
- batch_size = sdata.shape[0]
- sdata = Variable(sdata.type(FloatTensor))
- x = sdata.view(sdata.size(0), 1, sdata.size(1))
- label = Variable(label.type(LongTensor))
- loss2 = loss(model(x), label)
- _, predicted = torch.max(model(x).data, 1)
- temp = (predicted == label).sum()
- correct1 += temp
- total1 += batch_size
- loss_vid.append(loss2.item())
- print('Accuracy of testing data is: %d %%' % (100 * correct1 / total1))
- aaa=100 * (correct1 / total1).item()
-
- if aaa > min_mae:
- min_mae=aaa
- if os.path.exists(save_model_dir):
- os.remove(save_model_dir)
- print('存在文件,删除文件成功!!')
- print("save model")
- torch.save(model.state_dict(),save_model_dir)
-
- acc1.append(100 * correct1 / total1)
- train_loss = np.average(loss_r)
- valid_loss = np.average(loss_vid)
- avg_train_losses.append(train_loss)
- avg_valid_losses.append(valid_loss)
-
- epoch_len = len(str(opt.n_epochs))
- print_msg = (f'[{epoch:>{epoch_len}}/{opt.n_epochs:>{epoch_len}}] ' +f'train_loss: {train_loss:.5f} ' +f'valid_loss: {valid_loss:.5f}')
- print(print_msg)
- train_losses = []
- valid_losses = []
-
- end = time.time()
- print(end - start)
- #%%
|