import os
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler


def normal(data):
    high = data.describe()['75%'] + 1.5 * (data.describe()['75%'] - data.describe()['25%'])
    low = data.describe()['25%'] - 1.5 * (data.describe()['75%'] - data.describe()['25%'])
    return (data >= low) & (data <= high)


# file_dir = './浙江各地市行业电量数据'
#
# # 合并11个市
# df = pd.DataFrame({})
# for city in os.listdir(file_dir):
#
#     df_city = pd.read_excel(os.path.join(file_dir, city))
#
#     # 对每个市的每一个行业异常值 向后填充
#     for industry in df_city.columns[2:]:
#         outliers_index = normal(df_city[industry]).index
#         df_city[industry] = df_city[industry].where(normal(df_city[industry]), other=np.nan).bfill()
#         df_city[industry].fillna(method='ffill',inplace=True)
#     df = pd.concat([df,df_city])
#     print(df.shape)
#
# df.to_csv('11市行业数据(已处理异常).csv',index=False,encoding='GBK')
df = pd.read_csv('11市行业数据(已处理异常).csv', encoding='gbk')
print(sum(df.isnull().sum()))
print(df.describe())
# 对df每一行业进行归一化
column_params = {}
for column in df.columns[2:]:
    scaler = MinMaxScaler()

    df[column] = scaler.fit_transform(df[[column]])

    column_params[column] = {'min': scaler.data_min_[0], 'max': scaler.data_max_[0]}

print(column_params)
print(df.head())


def create_dataset(data, days_for_train=10) -> (np.array, np.array):
    dataset_x, dataset_y = [], []
    for i in range(len(data) - days_for_train - 3):
        dataset_x.append(data[i:(i + days_for_train)])
        dataset_y.append(data[i + days_for_train:i + days_for_train + 3])

    return (np.array(dataset_x), np.array(dataset_y))


# 切分x,y数据集，步长为10.最小单位为单个城市的单个行业。
# 先从第一个行业切分，合并所有城市。

industry = df.columns[2:][0]
city = df['地市'].drop_duplicates()[0]
df_city_industry = df[df['地市'] == city][industry]
dataset_x, dataset_y = create_dataset(df_city_industry)

for city in df['地市'].drop_duplicates()[1:]:
    df_city_industry = df[df['地市'] == city][industry]
    x, y = create_dataset(df_city_industry)
    dataset_x,dataset_y = np.concatenate([dataset_x,x]),np.concatenate([dataset_y,y])

for industry in df.columns[2:][1:]:
    for city in df['地市'].drop_duplicates():
        df_city_industry = df[df['地市'] == city][industry]
        x, y = create_dataset(df_city_industry)
        dataset_x, dataset_y = np.concatenate([dataset_x, x]), np.concatenate([dataset_y, y])

print(dataset_x.shape, dataset_y.shape)