Deleting a branch is permanent. It CANNOT be undone. Continue?
Mymylove/AbstractFFTs:master
into master
Deleting a branch is permanent. It CANNOT be undone. Continue?
Dear OpenI User
Thank you for your continuous support to the Openl Qizhi Community AI Collaboration Platform. In order to protect your usage rights and ensure network security, we updated the Openl Qizhi Community AI Collaboration Platform Usage Agreement in January 2024. The updated agreement specifies that users are prohibited from using intranet penetration tools. After you click "Agree and continue", you can continue to use our services. Thank you for your cooperation and understanding.
For more agreement content, please refer to the《Openl Qizhi Community AI Collaboration Platform Usage Agreement》
import random
import numpy as np
import torch
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler
from torch import nn
from tqdm import tqdm
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
mm = MinMaxScaler()
class ConvNet(nn.Module):
def init(self, input_dim, input_dim1, hidden_dim, kernel_size1, padding1, kernel_size2, padding2, kernel_size3,
padding3, kernel_size4, padding4):
super(ConvNet, self).init()
self.input_dim = input_dim
self.input_dim1 = input_dim1
data = np.load(
r'D:\heat_wave\WEIO\expand_WEIO\last\optimized_formula_all_variables_expand_WEIO_surface_deepchazhi_1.npz')
print(
data.files) # ['sst_obser', 'slfh', 'sshf', 'ssr', 'str', 'mld', 'analysis_temp', 'u', 'v', 'T_d', 'u_d', 'v_d', 'xx', 'yy', 'dT_dt', 'dT_dx', 'dT_dy']
sst_obser = data['sst_obser'][:, :64, :48] # (9861, 41, 201) #
print(sst_obser.shape) # (9861, 41, 201)
slfh = data['slfh'][:, :64, :48] # (9861, 41, 201)
sshf = data['sshf'][:, :64, :48] # (9861, 41, 201)
ssr = data['ssr'][:, :64, :48] # (9861, 41, 201) # 可惜这个是 0
str = data['str'][:, :64, :48] # (9861, 41, 201)
mld = data['mld'][:, :64, :48] # (9861, 41, 201)
analysis_temp = data['analysis_temp'][:, :64, :48] # (9861, 41, 201)
u = data['u'][:, :64, :48] # (9861, 41, 201)
v = data['v'][:, :64, :48] # (9861, 41, 201)
T_d = data['T_d'][:, :64, :48] # (9861, 41, 201)
u_d = data['u_d'][:, :64, :48] # (9861, 41, 201)
v_d = data['v_d'][:, :64, :48] # (9861, 41, 201)
xx = data['xx'][:, :64, :48] # (9861, 41, 201)
yy = data['yy'][:, :64, :48] # (9861, 41, 201)
print(np.min(T_d))
print(np.max(T_d))
print(xx) # -5 -4.75 ... 4.75
print(yy) # 190 190.25 ... 239.75
dT_dt = data['dT_dt'][:, :64, :48]
dT_dx = data['dT_dx'][:, :64, :48]
dT_dy = data['dT_dy'][:, :64, :48]
dT_dt0 = torch.zeros(1, 64, 48)
dT_dt = np.concatenate((dT_dt, dT_dt0), axis=0)
train_size = 0 # 前60%
valid_size = 0 # 中间20% 作为验证 剩下的20%的作为测试
dT_dt = dT_dt.reshape(-1, 1, 64, 48)
dT_dt = torch.Tensor(dT_dt)
dT_dt_train = dT_dt[0:train_size, :, :, :]
dT_dt_valid = dT_dt[train_size:valid_size, :, :, :]
dT_dt_test = dT_dt[valid_size:, :, :, :]
dT_dx = dT_dx.reshape(-1, 1, 64, 48)
dT_dx = torch.Tensor(dT_dx)
dT_dx_train = dT_dx[0:train_size, :, :, :]
dT_dx_valid = dT_dx[train_size:valid_size, :, :, :]
dT_dx_test = dT_dx[valid_size:, :, :, :]
dT_dy = dT_dy.reshape(-1, 1, 64, 48)
dT_dy = torch.Tensor(dT_dy)
dT_dy_train = dT_dy[0:train_size, :, :, :]
dT_dy_valid = dT_dy[train_size:valid_size, :, :, :]
dT_dy_test = dT_dy[valid_size:, :, :, :]
Q_net = (slfh + sshf + ssr + str)
hycom_temp = hycom_temp.reshape(-1,1,7,40,80)
hycom_temp = torch.Tensor(hycom_temp)
hycom_temp_train = hycom_temp[0:train_size,:,:,:]
hycom_temp_valid = hycom_temp[train_size:valid_size,:,:,:]
hycom_temp_test = hycom_temp[valid_size:,:,:,:]
dT_dt1 = hycom_temp[:,1,:,:] - hycom_temp[:,0,:,:] # (2,3,4,5,6,7,8) - (1,2,3,4,5,6,7) #dT_dt1.shape:torch.Size([3281, 41, 81])
dT_dt2 = hycom_temp[:,2,:,:] - hycom_temp[:,1,:,:] # (3,4,5,6,7,8,9) - (2,3,4,5,6,7,8)
dT_dt3 = hycom_temp[:,3,:,:] - hycom_temp[:,2,:,:] # (4,5,6,7,8,9,10) - (3,4,5,6,7,8,9)
dT_dt4 = hycom_temp[:,4,:,:] - hycom_temp[:,3,:,:] # (5,6,7,8,9,10,11) - (4,5,6,7,8,9,10)
dT_dt5 = hycom_temp[:,5,:,:] - hycom_temp[:,4,:,:] # (6,7,8,9,10,11,12) - (5,6,7,8,9,10,11)
dT_dt6 = hycom_temp[:,6,:,:] - hycom_temp[:,5,:,:] # (7,8,9,10,11,12,13) - (6,7,8,9,10,11,12)
dT_dt1 = dT_dt1.reshape(-1,1,41,81)
dT_dt2 = dT_dt2.reshape(-1,1,41,81)
dT_dt3 = dT_dt3.reshape(-1,1,41,81)
dT_dt4 = dT_dt4.reshape(-1,1,41,81)
dT_dt5 = dT_dt5.reshape(-1,1,41,81)
dT_dt6 = dT_dt6.reshape(-1,1,41,81)
dT_dt = np.concatenate((dT_dt1, dT_dt2, dT_dt3, dT_dt4, dT_dt5, dT_dt6), axis=1) # dT_dt.shape:(3281, 6, 41, 81) N, time, lat, lon
print('dT_dt.shape:{}'.format(dT_dt.shape))
list = []
for i in range(40):
xx_i = xx[:,:,i,:]
xx_j = xx[:,:,i+1,:]
T_i = xx[:,:,i,:]
T_j = xx[:,:,i+1,:]
dT_dx = (T_j - T_i) / (xx_j - xx_i)
list.append(dT_dx)
print(dT_dx.shape)
a = np.array(list)
print(a.shape) # (40, 3281, 7, 81)
dT_dx = a.transpose(1,2,0,3)
print(dT_dx.shape) # (3281, 7, 40, 81)
list1 = []
for j in range(80):
yy_i = yy[:,:,:,j]
yy_j = yy[:,:,:,j+1]
T_i = yy[:,:,:,j]
T_j = yy[:,:,:,j+1]
dT_dy = (T_j - T_i) / (yy_j - yy_i)
list1.append(dT_dy)
print(dT_dy.shape)
b = np.array(list1)
print(b.shape) # (80, 3281, 7, 81)
dT_dy = b.transpose(1,2,3,0)
print(dT_dy.shape) # (3281, 7, 41, 80)
sst_obser = np.array(sst_obser)
sst_obser = sst_obser.reshape(-1, 1, 64, 48)
sst_obser = torch.Tensor(sst_obser)
sst_obser_train = sst_obser[0:train_size, :, :, :]
sst_obser_valid = sst_obser[train_size:valid_size, :, :, :]
sst_obser_test = sst_obser[valid_size:, :, :, :]
Q_net = Q_net.reshape(-1, 1, 64, 48)
Q_net = torch.Tensor(Q_net)
Q_net_train = Q_net[0:train_size, :, :, :]
Q_net_valid = Q_net[train_size:valid_size, :, :, :]
Q_net_test = Q_net[valid_size:, :, :, :]
mld = mld.reshape(-1, 1, 64, 48)
mld = torch.Tensor(mld)
mld_train = mld[0:train_size, :, :, :]
mld_valid = mld[train_size:valid_size, :, :, :]
mld_test = mld[valid_size:, :, :, :]
analysis_temp = analysis_temp.reshape(-1, 1, 64, 48)
analysis_temp = torch.Tensor(analysis_temp)
analysis_temp_train = analysis_temp[0:train_size, :, :, :]
analysis_temp_valid = analysis_temp[train_size:valid_size, :, :, :]
analysis_temp_test = analysis_temp[valid_size:, :, :, :]
u = u.reshape(-1, 1, 64, 48)
u = torch.Tensor(u)
u_train = u[0:train_size, :, :, :]
u_valid = u[train_size:valid_size, :, :, :]
u_test = u[valid_size:, :, :, :]
v = v.reshape(-1, 1, 64, 48)
v = torch.Tensor(v)
v_train = v[0:train_size, :, :, :]
v_valid = v[train_size:valid_size, :, :, :]
v_test = v[valid_size:, :, :, :]
T_d = T_d.reshape(-1, 1, 64, 48)
T_d = torch.Tensor(T_d)
T_d_train = T_d[0:train_size, :, :, :]
T_d_valid = T_d[train_size:valid_size, :, :, :]
T_d_test = T_d[valid_size:, :, :, :]
u_d = u_d.reshape(-1, 1, 64, 48)
u_d = torch.Tensor(u_d)
u_d_train = u_d[0:train_size, :, :, :]
u_d_valid = u_d[train_size:valid_size, :, :, :]
u_d_test = u_d[valid_size:, :, :, :]
v_d = v_d.reshape(-1, 1, 64, 48)
v_d = torch.Tensor(v_d)
v_d_train = v_d[0:train_size, :, :, :]
v_d_valid = v_d[train_size:valid_size, :, :, :]
v_d_test = v_d[valid_size:, :, :, :]
xx = xx.reshape(-1, 1, 64, 48)
xx = torch.Tensor(xx)
xx_train = xx[0:train_size, :, :, :]
xx_valid = xx[train_size:valid_size, :, :, :]
xx_test = xx[valid_size:, :, :, :]
yy = yy.reshape(-1, 1, 64, 48)
yy = torch.Tensor(yy)
yy_train = yy[0:train_size, :, :, :]
yy_valid = yy[train_size:valid_size, :, :, :]
yy_test = yy[valid_size:, :, :, :]
train_data = torch.cat((sst_obser_train, Q_net_train, mld_train, u_train, v_train, T_d_train, v_d_train, xx_train,
yy_train, dT_dt_train, dT_dx_train, dT_dy_train),
dim=1) # train_data.shape:torch.Size([1952, 12, 7, 40, 80]) # 数据 第一维时间 第二维变量和变量 第三维7dat seq 4 5lat lon
valid_data = torch.cat((sst_obser_valid, Q_net_valid, mld_valid, u_valid, v_valid, T_d_valid, v_d_valid, xx_valid,
yy_valid, dT_dt_valid, dT_dx_valid, dT_dy_valid),
dim=1) # valid_data.shape:torch.Size([672, 12, 32, 40, 80])
test_data = torch.cat((sst_obser_test, Q_net_test, mld_test, u_test, v_test, T_d_test, v_d_test, xx_test, yy_test,
dT_dt_test, dT_dx_test, dT_dy_test), dim=1) # test_data.shape:torch.Size([657, 12, 7, 40, 80])
test_data = test_data[:9860, :, :, :]
train_label = analysis_temp[1:train_size + 1, :, :, :] # train_label.shape:torch.Size([1952, 1, 7, 40, 80])
valid_label = analysis_temp[train_size + 1: valid_size + 1, :, :, :] # test_label.shape:torch.Size([672, 1, 7, 40, 80])
test_label = analysis_temp[valid_size + 1:, :, :, :] # test_label.shape:torch.Size([650, 1, 7, 40, 80])
train_label = torch.Tensor(train_label)
train_label = train_label[:]
valid_label = torch.Tensor(valid_label)
valid_label = valid_label[:]
test_label = torch.Tensor(test_label)
test_label = test_label[:]
print('test_label.shape:{}'.format(test_label.shape)) # test_label.shape:torch.Size([1988, 1, 40, 200])
构建数据管道
class MyDataset(Dataset):
def init(self, data, label):
self.data = torch.Tensor(data)
self.label = torch.Tensor(label)
batch_size1 = 32
batch_size2 = 32
batch_size3 = 10000
trainset = MyDataset(train_data, train_label)
trainloader = DataLoader(trainset, batch_size=batch_size1, shuffle=True, drop_last=False, pin_memory=True,
num_workers=0)
validset = MyDataset(valid_data, valid_label)
validloader = DataLoader(validset, batch_size=batch_size2, shuffle=True, drop_last=False, pin_memory=True,
num_workers=0)
testset = MyDataset(test_data, test_label)
testloader = DataLoader(testset, batch_size=batch_size3, shuffle=False, drop_last=False, pin_memory=True, num_workers=0)
pred_val = np.zeros((1952, 1, 64, 48))
计算温度关于时间的梯度 这部分 分部分用神经网络描述
dT_dt = Q/(p * C_p * h_m) + u * (d_T / d_x) + v * (d_T / d_y) + w_e * ((T_m - T_d) / h_m)
积分 这部分 参与 loss计算 可以向后积分7天 也可以向后积分一天
dT_1 = Q1/(p * C_p * h_m1) + u1 * (d_T / d_x1) + v1 * (d_T / d_y1) + w_e1 * ((T_m1 - T_d1) / h_m1)
dT_2 = Q2/(p * C_p * h_m1) + u2 * (d_T / d_x1) + v2 * (d_T / d_y1) + w_e2 * ((T_m1 - T_d2) / h_m2)
dT_3 = Q3/(p * C_p * h_m1) + u3 * (d_T / d_x1) + v3 * (d_T / d_y1) + w_e3 * ((T_m1 - T_d3) / h_m3)
dT_4 = Q4/(p * C_p * h_m1) + u4 * (d_T / d_x1) + v4 * (d_T / d_y1) + w_e4 * ((T_m1 - T_d4) / h_m4)
dT_5 = Q5/(p * C_p * h_m1) + u5 * (d_T / d_x1) + v5 * (d_T / d_y1) + w_e5 * ((T_m1 - T_d5) / h_m5)
dT_6 = Q6/(p * C_p * h_m1) + u6 * (d_T / d_x1) + v6 * (d_T / d_y1) + w_e6 * ((T_m1 - T_d6) / h_m6)
T_1取 hycom的1;
T_2 = T_1 + dT_1
T_3 = T_1 + dT_1 + dT_2
T_4 = T_1 + dT_1 + dT_2 + dT_3
T_5 = T_1 + dT_1 + dT_2 + dT_3 + dT_4
T_6 = T_1 + dT_1 + dT_2 + dT_3 + dT_4 + dT_5
T_7 = T_1 + dT_1 + dT_2 + dT_3 + dT_4 + dT_5 + dT_6
T_8 = 1/18 * (T_0 + 4T_1 + 2T_2 + 4T_3 + 2T_4 + 4T_5 + 2T_6 + T_7)
for seed in range(5, 6):
for epoches in range(100, 101, 100):