#去掉相关性太高的特征
def identify_collinear(corr_matrix,correlation_threshold,X):
#相关性矩阵
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1).astype(np.bool))
#相关性大小和阈值进行对比
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
#生成空的dataframe
record_collinear = pd.Dataframe(columns = ['corr_feature1', 'corr_feature2', 'corr_value'])
#遍历阈值判断出的每一列名
for column in to_drop:
#再次进行阈值判断得到列名和相关系数
corr_features = list(upper.index[upper[column].abs() > correlation_threshold])
corr_values = list(upper[column][upper[column].abs() > correlation_threshold])
drop_features = [column for _ in range(len(corr_features))]
#存入dataframe
temp_df = pd.Dataframe.from_dict({'corr_feature1': drop_features,
'corr_feature2': corr_features,
'corr_value': corr_values})
#合并到大表格
record_collinear = record_collinear.append(temp_df, ignore_index = True)
drops=[]## 下面是删除规则
#没有dataframe生成说明不需要删除特征
if record_collinear.shape[0]==0:
return 'nothing!','nothing!'
#得到需要删除特征的列名
for i,j in zip(record_collinear.corr_feature1.values.tolist(),record_collinear.corr_feature2.values.tolist()):
if X[i].nunique()>X[j].nunique():
drops.append(j)
else:
drops.append(i)
drops=list(set(drops))
return record_collinear,drops
