pytorch/浙江行业电量/分类归一化.py

import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
import matplotlib.pyplot as plt
import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
torch.manual_seed(42)

train_step = 10
class LSTM(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, num_layers):
        super().__init__()
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers)
        self.fc1 = nn.Linear(hidden_size, output_size)
        # self.fc2 = nn.Linear(128, output_size)
        # self.ReLu = nn.ReLU()
        # self.dropout = nn.Dropout(0.8)

    def forward(self, x):
        x, _ = self.lstm(x)
        s, b, h = x.shape
        x = x.reshape(-1, h)
        # output = self.ReLu(self.dropout(self.fc1(x)))
        output = self.fc1(x)
        output = output.view(s, b, -1)
        return output


def normal(data):
    high = data.describe()['75%'] + 1.5 * (data.describe()['75%'] - data.describe()['25%'])
    low = data.describe()['25%'] - 1.5 * (data.describe()['75%'] - data.describe()['25%'])
    return (data >= low) & (data <= high)


# file_dir = './浙江各地市行业电量数据'
#
# # 合并11个市
# df = pd.DataFrame({})
# for city in os.listdir(file_dir):
#
#     df_city = pd.read_excel(os.path.join(file_dir, city))
#
#     # 对每个市的每一个行业异常值 向后填充
#     for industry in df_city.columns[2:]:
#         outliers_index = normal(df_city[industry]).index
#         df_city[industry] = df_city[industry].where(normal(df_city[industry]), other=np.nan).bfill()
#         df_city[industry].fillna(method='ffill',inplace=True)
#     df = pd.concat([df,df_city])
#     print(df.shape)
#
# df.to_csv('11市行业数据(已处理异常).csv',index=False,encoding='GBK')
df = pd.read_csv('11市行业数据(已处理异常).csv', encoding='gbk')

# 对df每一行业进行归一化
column_params = {}
for column in df.columns[2:]:
    scaler = MinMaxScaler()

    df[column] = scaler.fit_transform(df[[column]])

    column_params[column] = {'min': scaler.data_min_[0], 'max': scaler.data_max_[0]}

print(column_params)
print(df.head())

def create_dataset(data, train_step=train_step) -> (np.array, np.array):
    dataset_x, dataset_y = [], []
    for i in range(len(data) - train_step - 3):
        dataset_x.append(data[i:(i + train_step)])
        dataset_y.append(data[i + train_step:i + train_step + 3])

    return (np.array(dataset_x), np.array(dataset_y))


# 切分x,y数据集，步长为10.最小单位为单个城市的单个行业。
# 先从第一个行业切分，合并所有城市。

# industry = df.columns[2:][0]
# city = df['地市'].drop_duplicates()[0]
# df_city_industry = df[df['地市'] == city][industry]
# dataset_x, dataset_y = create_dataset(df_city_industry)
#
# for city in df['地市'].drop_duplicates()[1:]:
#     df_city_industry = df[df['地市'] == city][industry]
#     x, y = create_dataset(df_city_industry)
#     dataset_x, dataset_y = np.concatenate([dataset_x, x]), np.concatenate([dataset_y, y])
#
# for industry in df.columns[2:][1:]:
#     for city in df['地市'].drop_duplicates():
#         df_city_industry = df[df['地市'] == city][industry]
#         x, y = create_dataset(df_city_industry)
#         dataset_x, dataset_y = np.concatenate([dataset_x, x]), np.concatenate([dataset_y, y])
#
# print(dataset_x.shape, dataset_y.shape)
# df_x = pd.DataFrame(dataset_x)
# df_y = pd.DataFrame(dataset_y)
# df_x.to_csv('df_x_100.csv',index=False)
# df_y.to_csv('df_y_100.csv',index=False)
dataset_x = pd.read_csv('df_x.csv').values
dataset_y = pd.read_csv('df_y.csv').values
print(dataset_x.shape, dataset_y.shape)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_size = int(len(dataset_x)*0.7)
train_x = dataset_x[:train_size]
train_y = dataset_y[:train_size]
eval_x = dataset_x[train_size:]
eval_y = dataset_y[train_size:]

# # 将数据改变形状，RNN 读入的数据维度是 (seq_size, batch_size, feature_size)
train_x = train_x.reshape(-1, 1, train_step)
train_y = train_y.reshape(-1, 1, 3)
eval_x = eval_x.reshape(-1, 1, train_step)
eval_y = eval_y.reshape(-1, 1, 3)

# # 转为pytorch的tensor对象
train_x = torch.from_numpy(train_x).to(device).type(torch.float32)
train_y = torch.from_numpy(train_y).to(device).type(torch.float32)
eval_x = torch.from_numpy(eval_x).to(device).type(torch.float32)
eval_y = torch.from_numpy(eval_y).to(device).type(torch.float32)



# ds = TensorDataset(x_train, y_train)
# dl = DataLoader(ds, batch_size=32, drop_last=True)
# eval_ds = TensorDataset(x_eval, y_eval)
# eval_dl = DataLoader(eval_ds, batch_size=64, drop_last=True)


model = LSTM(train_step,32, 3, num_layers=2).to(device)
loss_fn = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.005, betas=(0.9, 0.999), eps=1e-08, weight_decay=0)

min_loss = 1
for i in range(1500):
    model.train()
    out = model(train_x)
    loss = loss_fn(out, train_y)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    model.eval()
    with torch.no_grad():
        pred = model(eval_x)
        eval_loss = loss_fn(pred,eval_y)
        if eval_loss <= min_loss:
            min_loss = eval_loss
            best_para = model.state_dict()
    if i % 100 == 0:
        print(f'epoch {i+1}: loss:{loss} eval_loss:{eval_loss}')

# for epoch in range(3):
#     model.train()
#     for step, (x, y) in enumerate(dl):
#         x, y = x.to(device), y.to(device)
#         pred = model(x)
#         loss = loss_fn(pred,y)
#         optimizer.zero_grad()
#         loss.backward()
#         optimizer.step()
#
#         if step % 1000 == 0:
#             print(f'epoch{epoch+1}: train_step:{step}/{len(dl)} train_loss:{loss}\n')
#
#     model.eval()
#     batch_loss = 0
#     with torch.no_grad():
#         for x,y in eval_dl:
#             x, y = x.to(device), y.to(device)
#             pred = model(x)
#             loss = loss_fn(pred, y)
#             batch_loss += loss
#     print(f'epoch{epoch+1}: eval_loss:{batch_loss/len(eval_dl)}\n')
#
#     if batch_loss/len(eval_dl) < min_loss:
#         min_loss = batch_loss/len(eval_dl)
#         best_parameters = model.state_dict()

torch.save(best_para,'best_3.pth')

model = LSTM(train_step,32, 3, num_layers=2).to(device)
model.load_state_dict(torch.load('best_3.pth'))
params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print("LSTM参数总量:", params)

dataset_x = dataset_x.reshape(-1,1,train_step)
dataset_x = torch.from_numpy(dataset_x).type(torch.float32).to(device)
pred = model(dataset_x).reshape(-1)
pred = np.concatenate((np.zeros(train_step), pred.cpu().detach().numpy()))


plt.plot(pred, 'r', label='prediction')
plt.plot(dataset_y.reshape(-1), 'b', label='real')
plt.plot((train_size*3, train_size*3), (0, 1), 'g--')  # 分割线 左边是训练数据 右边是测试数据的输出
plt.legend(loc='best')
plt.show()