数据的二次处理
使用特征工程的方法对数据进行二次处理，获得更高的数据特征相关性，利用特征工程的方法计算出k小时变温，k小时变压，k小时平均风速等相关特征对数据进行进一步处理，使得预测的结果更加准确`mport seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import getdata
from scipy.stats import pearsonr
import warnings
import math
from scipy.stats import pearsonr
from scipy.stats import pearsonr
def data_processing(a,b):
plt.style.use('seaborn')
data2 = getdata(a,b);
print(data2.head(3))
#cel(excel_name, sheet_name=2, na_values=na_values)
# data3.head(3)
# # 数据清洗
# ## 缺失行补齐算法(不补齐会造成很严重的后果)
temp_df = data2.copy()
lack_list = list(set(time_list)-set(data2['last_update']))
lack_df = pd.DataFrame(lack_list,columns=['last_update'])
temp_df = pd.concat([temp_df,lack_df])
temp_df.sort_values("last_update",inplace=True)
temp_df.index = list(range(len(temp_df)))
temp_df.shape
data2 = temp_df
data2.describe().to_csv('describe1.csv')
data2.describe()
data2.info()
data2.isnull().sum().to_csv('info.csv')
data2.isnull().sum()
# ### 线性插值
data2.iloc[:,2:] = data2.iloc[:,2:].interpolate()
data2.isnull().sum()
## 异常值
# ### 消除负值，不低于边界点0
data2 = data2.applymap(lambda v: max(0,v) if isinstance(v, float) else v)
data2.describe().to_csv('describe2.csv')
data2.describe()
# 特征展示
# ## 分布图
warnings.filterwarnings("ignore")
def fkdeplot(vec, name):
sns.kdeplot(vec)
plt.title(name, fontsize=12)
plt.show()
# for i in range(10):
# fkdeplot(data2.iloc[:,i+2], 'Data')
fkdeplot(data2.iloc[:,4].values, 'SO2')
fkdeplot(data2.iloc[:,5].values, 'NO2')
fkdeplot(data2.iloc[:,3].values, 'PM10')
fkdeplot(data2.iloc[:,2].values, 'Pm2.5')
fkdeplot(data2.iloc[:,7].values, 'O3')
fkdeplot(data2.iloc[:,6].values, 'CO')
fkdeplot(data2.iloc[:,10].values, 'Temperature')
fkdeplot(data2.iloc[:,13].values, 'Humidity')
fkdeplot(data2.iloc[:,12].values, 'Pressure')
fkdeplot(data2.iloc[:,15].values, 'Wind Speed')
# ## AQI分布图
def compute_AQI(a):
print(a)
#a=np.array([n_SO2,n_NO2,n_PM10,n_PM25,n_O3,n_CO])
T=np.array([[0,50,150,475,800,1600,2100,2620],[0,40,80,180,280,565,750,940],[0,50,150,250,350,420,500,600],[0,35,75,115,150,250,350,500],[0,100,160,215,265,800,0,0],[0,2,4,14,24,36,48,60]])
I=np.array([0,50,100,150,200,300,400,500])
BP_LO = []
BP_Hi=[]
IAQI_LO=[]
IAQI_Hi=[]
IAQI=np.zeros((1,6))
result=[]
for j in range(6):
for i in range(1,8):
if a[j] <= T[j, i]:
print(a[i])
BP_LO.append(T[j, i-1])
BP_Hi.append(T[j, i])
IAQI_LO.append(I[i-1])
IAQI_Hi.append(I[i])
break
else:
continue
IAQI[0, j] = (IAQI_Hi[j] - IAQI_LO[j]) / (BP_Hi[j] - BP_LO[j]) * (a[j] - BP_LO[j]) + IAQI_LO[j]
result.append(math.ceil(IAQI[0, j]))
AQI=np.max(result)
c=np.argmax(result)
name=["SO2", "NO2", "PM10", "PM25", "O3", "CO"]
# if AQI<=50:
# print("AQI为%d" %(AQI))
# print("当天无首要污染物")
# else:
# print("AQI为%d" % (AQI))
# print('首要污染物为%s' % (name[c]))
return AQI
for inst in data2.iloc[:,2:8].values:
print('!!!')
print(inst)
AQI = [compute_AQI(inst) for inst in data2.iloc[:,2:8].values]
data2['AQI'] = AQI
fkdeplot(data2.iloc[:, 1].values, 'AQI')
def fplot(vec, name):
plt.plot(vec)
plt.title(name)
plt.show()
show_num = 24
fplot(data2.iloc[:show_num,4], 'SO2')
fplot(data2.iloc[:show_num,5], 'NO2')
fplot(data2.iloc[:show_num,3], 'PM10')
fplot(data2.iloc[:show_num,2], 'Pm2.5')
fplot(data2.iloc[:show_num,7], 'O3')
fplot(data2.iloc[:show_num,6], 'CO')
fplot(data2.iloc[:show_num,10], 'Temperature')
fplot(data2.iloc[:show_num,13], 'Humidity')
fplot(data2.iloc[:show_num,12], 'Pressure')
fplot(data2.iloc[:show_num,15], 'Wind Speed')
# ## 展示AQI的前6个时刻
fplot(data2.iloc[:show_num,-1], 'AQI')
# # 相关性分析(AQI与气象条件）
# pearson相关性
corr = data2.loc[:, ['temperature', 'humidity', 'pressure', 'wind_speed', 'aqi']].corr()
corr.columns = ['temperature', 'pressure', 'humidity', 'wind_speed', 'aqi']
corr.index = corr.columns
sns.heatmap(corr, annot=True)
AQI = data2.loc[:, 'aqi']
# # 相关性分析(AQI与高级气象条件）
# ## k小时变压
warm_up = 20
high_level_data = []
# In[310]:
k = 2
def stat_k(pressure_k):
# temp = pressure_k[-1]-pressure_k[0]
# temp = np.var(pressure_k)
temp = np.mean(pressure_k)
# np.std(pressure_k)/np.mean(pressure_k)
return temp
pressure_k = []
temp_data = data2.iloc[:, 12].values
print(temp_data)
print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
print('data.shape', pressure_k.shape)
print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
fkdeplot(pressure_k, 'pressure-%d' % k)
fplot(pressure_k[:show_num], 'pressure-%d' % k)
high_level_data.append(pressure_k[warm_up - k:])
# In[311]:
from scipy.stats import pearsonr
def pressure_k_peasonr(k):
pressure_k = []
temp_data = data2.iloc[:, 12].values
# print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
# print('data.shape', pressure_k.shape)
# print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
# print(len(AQI[k-1:]), len(pressure_k))
temp_pearsonr = pearsonr(AQI[k - 1:], pressure_k)[0]
# print(temp_pearsonr)
return temp_pearsonr
k_list = []
pearsonr_list = []
for k in range(10):
temp_k = k + 5
temp_pearsonr = pressure_k_peasonr(temp_k)
k_list.append(temp_k)
pearsonr_list.append(temp_pearsonr)
plt.plot(k_list, pearsonr_list)
plt.xlabel('K')
plt.ylabel('pearsonr correlation')
plt.title('pressure-k')
plt.show()
idx = np.argmax(np.abs(pearsonr_list))
print('最大(绝对)相关性', pearsonr_list[idx])
print('最大相关性对应k', k_list[idx])
# ## k小时变温
k = 2
def stat_k(pressure_k):
temp = pressure_k[-1] - pressure_k[0]
# temp = np.var(pressure_k)
# temp = np.mean(pressure_k)
# np.std(pressure_k)/np.mean(pressure_k)
return temp
pressure_k = []
temp_data = data2.iloc[:, 10].values
print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
print('data.shape', pressure_k.shape)
print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
fplot(pressure_k[:show_num], 'temperatyre-%d' % k)
fkdeplot(pressure_k, 'temperatyre-%d' % k)
high_level_data.append(pressure_k[warm_up - k:])
from scipy.stats import pearsonr
def pressure_k_peasonr(k):
pressure_k = []
temp_data = data2.iloc[:, 10].values
# print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
# print('data.shape', pressure_k.shape)
# print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
# print(len(AQI[k-1:]), len(pressure_k))
temp_pearsonr = pearsonr(AQI[k - 1:], pressure_k)[0]
# print(temp_pearsonr)
return temp_pearsonr
k_list = []
pearsonr_list = []
for k in range(10):
temp_k = k + 2
temp_pearsonr = pressure_k_peasonr(temp_k)
k_list.append(temp_k)
pearsonr_list.append(temp_pearsonr)
plt.plot(k_list, pearsonr_list)
plt.xlabel('K')
plt.ylabel('pearsonr correlation')
plt.title('temperature-k')
plt.show()
idx = np.argmax(np.abs(pearsonr_list))
print('最大(绝对)相关性', pearsonr_list[idx])
print('最大相关性对应k', k_list[idx])
# ## k小时平均风速
# In[314]:
k = 3
def stat_k(pressure_k):
temp = np.mean(pressure_k)
return temp
pressure_k = []
temp_data = data2.iloc[:, 15].values
print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
print('data.shape', pressure_k.shape)
print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
fkdeplot(pressure_k, 'pressure-%d' % k)
fplot(pressure_k[:show_num], 'pressure-%d' % k)
high_level_data.append(pressure_k[warm_up - k:])
# In[315]:
def pressure_k_peasonr(k):
pressure_k = []
temp_data = data2.iloc[:, 15].values
# print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
# print('data.shape', pressure_k.shape)
# print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
# print(len(AQI[k-1:]), len(pressure_k))
temp_pearsonr = pearsonr(AQI[k - 1:], pressure_k)[0]
# print(temp_pearsonr)
return temp_pearsonr
k_list = []
pearsonr_list = []
for k in range(10):
temp_k = k + 2
temp_pearsonr = pressure_k_peasonr(temp_k)
k_list.append(temp_k)
pearsonr_list.append(temp_pearsonr)
plt.plot(k_list, pearsonr_list)
plt.xlabel('K')
plt.ylabel('pearsonr correlation')
plt.title('wind-speed-k')
plt.show()
idx = np.argmax(np.abs(pearsonr_list))
print('最大(绝对)相关性', pearsonr_list[idx])
print('最大相关性对应k', k_list[idx])
# ## k小时平均湿度
# In[316]:
k = 2
def stat_k(pressure_k):
temp = np.mean(pressure_k)
return temp
pressure_k = []
temp_data = data2.iloc[:, 13].values
print(1212121211)
print(temp_data)
print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
print('data.shape', pressure_k.shape)
print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
fkdeplot(pressure_k, 'pressure-%d' % k)
fplot(pressure_k[:show_num], 'pressure-%d' % k)
high_level_data.append(pressure_k[warm_up - k:])
def pressure_k_peasonr(k):
pressure_k = []
temp_data = data2.iloc[:, 13].values
# print('total number', len(temp_data))
for i in range(len(temp_data) - k + 1):
pressure_k.append(temp_data[i:i + k])
pressure_k = np.array(pressure_k)
# print('data.shape', pressure_k.shape)
# print('total number', len(pressure_k))
pressure_k = [stat_k(inst) for inst in pressure_k]
# print(len(AQI[k-1:]), len(pressure_k))
temp_pearsonr = pearsonr(AQI[k - 1:], pressure_k)[0]
# print(temp_pearsonr)
return temp_pearsonr`
