|
|
import numpy as np
|
|
|
import pandas as pd
|
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
|
import torch
|
|
|
from torch import nn
|
|
|
from torch.utils.data import DataLoader, TensorDataset
|
|
|
import matplotlib.pyplot as plt
|
|
|
import os
|
|
|
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
|
|
|
torch.manual_seed(42)
|
|
|
|
|
|
train_step = 10
|
|
|
class LSTM(nn.Module):
|
|
|
def __init__(self, input_size, hidden_size, output_size, num_layers):
|
|
|
super().__init__()
|
|
|
self.lstm = nn.LSTM(input_size, hidden_size, num_layers)
|
|
|
self.fc1 = nn.Linear(hidden_size, output_size)
|
|
|
# self.fc2 = nn.Linear(128, output_size)
|
|
|
# self.ReLu = nn.ReLU()
|
|
|
# self.dropout = nn.Dropout(0.8)
|
|
|
|
|
|
def forward(self, x):
|
|
|
x, _ = self.lstm(x)
|
|
|
s, b, h = x.shape
|
|
|
x = x.reshape(-1, h)
|
|
|
# output = self.ReLu(self.dropout(self.fc1(x)))
|
|
|
output = self.fc1(x)
|
|
|
output = output.view(s, b, -1)
|
|
|
return output
|
|
|
|
|
|
|
|
|
def normal(data):
|
|
|
high = data.describe()['75%'] + 1.5 * (data.describe()['75%'] - data.describe()['25%'])
|
|
|
low = data.describe()['25%'] - 1.5 * (data.describe()['75%'] - data.describe()['25%'])
|
|
|
return (data >= low) & (data <= high)
|
|
|
|
|
|
|
|
|
# file_dir = './浙江各地市行业电量数据'
|
|
|
#
|
|
|
# # 合并11个市
|
|
|
# df = pd.DataFrame({})
|
|
|
# for city in os.listdir(file_dir):
|
|
|
#
|
|
|
# df_city = pd.read_excel(os.path.join(file_dir, city))
|
|
|
#
|
|
|
# # 对每个市的每一个行业异常值 向后填充
|
|
|
# for industry in df_city.columns[2:]:
|
|
|
# outliers_index = normal(df_city[industry]).index
|
|
|
# df_city[industry] = df_city[industry].where(normal(df_city[industry]), other=np.nan).bfill()
|
|
|
# df_city[industry].fillna(method='ffill',inplace=True)
|
|
|
# df = pd.concat([df,df_city])
|
|
|
# print(df.shape)
|
|
|
#
|
|
|
# df.to_csv('11市行业数据(已处理异常).csv',index=False,encoding='GBK')
|
|
|
df = pd.read_csv('11市行业数据(已处理异常).csv', encoding='gbk')
|
|
|
|
|
|
# 对df每一行业进行归一化
|
|
|
column_params = {}
|
|
|
for column in df.columns[2:]:
|
|
|
scaler = MinMaxScaler()
|
|
|
|
|
|
df[column] = scaler.fit_transform(df[[column]])
|
|
|
|
|
|
column_params[column] = {'min': scaler.data_min_[0], 'max': scaler.data_max_[0]}
|
|
|
|
|
|
print(column_params)
|
|
|
print(df.head())
|
|
|
|
|
|
def create_dataset(data, train_step=train_step) -> (np.array, np.array):
|
|
|
dataset_x, dataset_y = [], []
|
|
|
for i in range(len(data) - train_step - 3):
|
|
|
dataset_x.append(data[i:(i + train_step)])
|
|
|
dataset_y.append(data[i + train_step:i + train_step + 3])
|
|
|
|
|
|
return (np.array(dataset_x), np.array(dataset_y))
|
|
|
|
|
|
|
|
|
# 切分x,y数据集，步长为10.最小单位为单个城市的单个行业。
|
|
|
# 先从第一个行业切分，合并所有城市。
|
|
|
|
|
|
# industry = df.columns[2:][0]
|
|
|
# city = df['地市'].drop_duplicates()[0]
|
|
|
# df_city_industry = df[df['地市'] == city][industry]
|
|
|
# dataset_x, dataset_y = create_dataset(df_city_industry)
|
|
|
#
|
|
|
# for city in df['地市'].drop_duplicates()[1:]:
|
|
|
# df_city_industry = df[df['地市'] == city][industry]
|
|
|
# x, y = create_dataset(df_city_industry)
|
|
|
# dataset_x, dataset_y = np.concatenate([dataset_x, x]), np.concatenate([dataset_y, y])
|
|
|
#
|
|
|
# for industry in df.columns[2:][1:]:
|
|
|
# for city in df['地市'].drop_duplicates():
|
|
|
# df_city_industry = df[df['地市'] == city][industry]
|
|
|
# x, y = create_dataset(df_city_industry)
|
|
|
# dataset_x, dataset_y = np.concatenate([dataset_x, x]), np.concatenate([dataset_y, y])
|
|
|
#
|
|
|
# print(dataset_x.shape, dataset_y.shape)
|
|
|
# df_x = pd.DataFrame(dataset_x)
|
|
|
# df_y = pd.DataFrame(dataset_y)
|
|
|
# df_x.to_csv('df_x_100.csv',index=False)
|
|
|
# df_y.to_csv('df_y_100.csv',index=False)
|
|
|
dataset_x = pd.read_csv('df_x.csv').values
|
|
|
dataset_y = pd.read_csv('df_y.csv').values
|
|
|
print(dataset_x.shape, dataset_y.shape)
|
|
|
|
|
|
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
|
train_size = int(len(dataset_x)*0.7)
|
|
|
train_x = dataset_x[:train_size]
|
|
|
train_y = dataset_y[:train_size]
|
|
|
eval_x = dataset_x[train_size:]
|
|
|
eval_y = dataset_y[train_size:]
|
|
|
|
|
|
# # 将数据改变形状，RNN 读入的数据维度是 (seq_size, batch_size, feature_size)
|
|
|
train_x = train_x.reshape(-1, 1, train_step)
|
|
|
train_y = train_y.reshape(-1, 1, 3)
|
|
|
eval_x = eval_x.reshape(-1, 1, train_step)
|
|
|
eval_y = eval_y.reshape(-1, 1, 3)
|
|
|
|
|
|
# # 转为pytorch的tensor对象
|
|
|
train_x = torch.from_numpy(train_x).to(device).type(torch.float32)
|
|
|
train_y = torch.from_numpy(train_y).to(device).type(torch.float32)
|
|
|
eval_x = torch.from_numpy(eval_x).to(device).type(torch.float32)
|
|
|
eval_y = torch.from_numpy(eval_y).to(device).type(torch.float32)
|
|
|
|
|
|
|
|
|
|
|
|
# ds = TensorDataset(x_train, y_train)
|
|
|
# dl = DataLoader(ds, batch_size=32, drop_last=True)
|
|
|
# eval_ds = TensorDataset(x_eval, y_eval)
|
|
|
# eval_dl = DataLoader(eval_ds, batch_size=64, drop_last=True)
|
|
|
|
|
|
|
|
|
model = LSTM(train_step,32, 3, num_layers=2).to(device)
|
|
|
loss_fn = nn.MSELoss()
|
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=0.005, betas=(0.9, 0.999), eps=1e-08, weight_decay=0)
|
|
|
|
|
|
min_loss = 1
|
|
|
for i in range(1500):
|
|
|
model.train()
|
|
|
out = model(train_x)
|
|
|
loss = loss_fn(out, train_y)
|
|
|
loss.backward()
|
|
|
optimizer.step()
|
|
|
optimizer.zero_grad()
|
|
|
|
|
|
model.eval()
|
|
|
with torch.no_grad():
|
|
|
pred = model(eval_x)
|
|
|
eval_loss = loss_fn(pred,eval_y)
|
|
|
if eval_loss <= min_loss:
|
|
|
min_loss = eval_loss
|
|
|
best_para = model.state_dict()
|
|
|
if i % 100 == 0:
|
|
|
print(f'epoch {i+1}: loss:{loss} eval_loss:{eval_loss}')
|
|
|
|
|
|
# for epoch in range(3):
|
|
|
# model.train()
|
|
|
# for step, (x, y) in enumerate(dl):
|
|
|
# x, y = x.to(device), y.to(device)
|
|
|
# pred = model(x)
|
|
|
# loss = loss_fn(pred,y)
|
|
|
# optimizer.zero_grad()
|
|
|
# loss.backward()
|
|
|
# optimizer.step()
|
|
|
#
|
|
|
# if step % 1000 == 0:
|
|
|
# print(f'epoch{epoch+1}: train_step:{step}/{len(dl)} train_loss:{loss}\n')
|
|
|
#
|
|
|
# model.eval()
|
|
|
# batch_loss = 0
|
|
|
# with torch.no_grad():
|
|
|
# for x,y in eval_dl:
|
|
|
# x, y = x.to(device), y.to(device)
|
|
|
# pred = model(x)
|
|
|
# loss = loss_fn(pred, y)
|
|
|
# batch_loss += loss
|
|
|
# print(f'epoch{epoch+1}: eval_loss:{batch_loss/len(eval_dl)}\n')
|
|
|
#
|
|
|
# if batch_loss/len(eval_dl) < min_loss:
|
|
|
# min_loss = batch_loss/len(eval_dl)
|
|
|
# best_parameters = model.state_dict()
|
|
|
|
|
|
torch.save(best_para,'best_3.pth')
|
|
|
|
|
|
model = LSTM(train_step,32, 3, num_layers=2).to(device)
|
|
|
model.load_state_dict(torch.load('best_3.pth'))
|
|
|
params = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
|
|
print("LSTM参数总量:", params)
|
|
|
|
|
|
dataset_x = dataset_x.reshape(-1,1,train_step)
|
|
|
dataset_x = torch.from_numpy(dataset_x).type(torch.float32).to(device)
|
|
|
pred = model(dataset_x).reshape(-1)
|
|
|
pred = np.concatenate((np.zeros(train_step), pred.cpu().detach().numpy()))
|
|
|
|
|
|
|
|
|
plt.plot(pred, 'r', label='prediction')
|
|
|
plt.plot(dataset_y.reshape(-1), 'b', label='real')
|
|
|
plt.plot((train_size*3, train_size*3), (0, 1), 'g--') # 分割线 左边是训练数据 右边是测试数据的输出
|
|
|
plt.legend(loc='best')
|
|
|
plt.show() |