平安产险极客挑战赛（算法建模）总结

本文主要是介绍平安产险极客挑战赛（算法建模）总结，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

在过去的半个多月的时间里参加了平安产险极客挑战赛，选的方向是算法建模部分，虽然最终结果不够理想，但花了很多时间、投入了很多精力，还是希望可以对这段时间所做的东西做一个总结，也希望有相关经验的大佬可以指点一下我。

首先是根据提供的字段解释表格对各个字段进行了理解，明白所需要做的是什么，然后进行了数据预处理（数据清洗、特征工程）、建模、训练模型、预测、调参等各个过程。

数据预处理：

在读取数据后，删除空白值超过一半的列和值完全相同的列，删除与业务相关性不大的列，并对非数值数据使用数值数据进行替换，最后对某些存在空白值的列使用该列平均值进行填充。

def del_data(filename):# 加载数据data = pd.read_csv(filename)# 删除空白值超过一半的列half_count = len(data)/2data = data.dropna(thresh=half_count, axis=1)# 删除值完全相同的列data = data.drop(['policy_code', 'application_type'], axis=1)# 删除与业务相关性不大的列data = data.drop(['emp_title', 'issue_d', 'title', 'zip_code', 'addr_state', 'earliest_cr_line'],axis=1)# loan_status-》pymnt_plandata = data.drop(['term', 'funded_amnt_inv', 'pymnt_plan', 'out_prncp', 'total_rec_late_fee','tot_coll_amt', 'sub_grade', 'collection_recovery_fee'], axis=1)data = data.drop(['pub_rec', 'initial_list_status', 'out_prncp_inv', 'recoveries', 'total_rec_prncp','collections_12_mths_ex_med', 'verification_status'], axis=1)# 对非数值列数据使用数值进行替换tot_coll_amtstatus_replace1 = {"grade": {"A": 0,"B": 1,"C": 2,"D": 3,"E": 4,"F": 5,"G": 6}}data = data.replace(status_replace1)status_replace2 = {"emp_length": {"n/a": 0,"< 1 year": 0,"1 year": 1,"2 years": 2,"3 years": 3,"4 years": 4,"5 years": 5,"6 years": 6,"7 years": 7,"8 years": 8,"9 years": 9,"10+ years": 10,}}data = data.replace(status_replace2)status_replace3 = {"home_ownership": {"NONE": 0,"RENT": 1,"OWN": 2,"MORTGAGE": 3,"OTHER": 4,"ANY": 5,}}data = data.replace(status_replace3)status_replace4 = {"verification_status": {"Verified": 0,"Source Verified": 1,"Not Verified": 2}}data = data.replace(status_replace4)status_replace5 = {"pymnt_plan": {"y": 0,"n": 1}}data = data.replace(status_replace5)status_replace6 = {"initial_list_status": {"f": 0,"w": 1}}data = data.replace(status_replace6)status_replace7 = {"term": {" 36 months": 0," 60 months": 1}}data = data.replace(status_replace7)status_replace8 = {"loan_status": {"Charged Off": 0,"Fully Paid": 1,"Current": 2,"In Grace Period": 3,"Late (31-120 days)": 4,'Issued': 5,'Does not meet the credit policy. Status:Charged Off': 6,'Default': 7,'Late (16-30 days)': 8,'Does not meet the credit policy. Status:Fully Paid': 9}}data = data.replace(status_replace8)status_replace8 = {"purpose": {"debt_consolidation": 0,"credit_card": 1,"major_purchase": 2,"home_improvement": 3,"other": 4,'small_business': 5,'renewable_energy': 6,'car': 7,'house': 8,'medical': 9,'vacation': 10,'moving': 11,'wedding': 12,'educational': 13}}data = data.replace(status_replace8)# 对某些列空白数值数据进行删除# data = data.dropna(axis=0)# 对某些存在空白值的列使用该列平均值进行填充column_len = len(data['member_id'])print(column_len)columns = data.columnsfor x in columns:data[x] = data[x].fillna(data[x].mean())# 输出数据处理结果# data.to_csv('data/5.csv', index=False)return data, data['member_id']

训练模型和预测：

本次选择xgboost来实现对数据进行训练和预测。由于训练数据中预测结果为1的比例太少，因此需要减少预测结果为0的数据的比例，然后进行训练，最后对测试数据进行预测，并用饼状图表示最终预测结果。

train_data, train_id = del_data(train_filename)  # 删除id属性
train_data = train_data.drop(['member_id'], axis=1)X_neg = train_data[:(int)(len(train_data)/2)].loc[train_data['acc_now_delinq'] == 1]
x_X = train_data[:(int)(len(train_data)/2)].loc[train_data['acc_now_delinq'] == 0]
x_ppp, X_pos = train_test_split(x_X, test_size=0.04, random_state=1)
frames = [X_pos, X_neg]
X_ = shuffle(pd.concat(frames, axis=0))
x = train_data[(int)(len(train_data)/2):].drop(['acc_now_delinq'], axis=1)
y_ = train_data[(int)(len(train_data)/2):].acc_now_delinq
# 18个属性
y = X_.acc_now_delinq
X = X_.drop(['acc_now_delinq'], axis=1)
# 结果标签
print("0-{0},1-{1},total-{2},0-rio-{3},1-rpo-{4}".format(np.sum(y == 0), np.sum(y == 1), len(y),np.sum(y == 0) / len(y), np.sum(y == 1) / len(y)))# change categorical
# ============================================================
# xgboost tryclf = XGBClassifier(learning_rate=0.1,n_estimators=1000,max_depth=18,min_child_weight=1,gamma=0.1,subsample=0.8,colsample_bytree=0.8,objective='binary:logistic',nthread=4,scale_pos_weight=1,seed=27)useTrainCV = True
cv_folds = 5
early_stopping_rounds = 1000
if useTrainCV:xgb_param = clf.get_xgb_params()xgtrain = xgb.DMatrix(X.values, label=y.values)cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=clf.get_params()['n_estimators'], nfold=cv_folds,metrics='auc', early_stopping_rounds=early_stopping_rounds)clf.set_params(n_estimators=cvresult.shape[0])# Fit the algorithm on the data
clf.fit(X, y, eval_metric='auc')# =============================================================
# clf = tree.DecisionTreeClassifier(max_depth=25)
# clf = clf.fit(X, y)
test_data, test_id = del_data(test_filename)# x_test = test_data.drop(['member_id'], axis=1)
# x_test = train_data.drop(['acc_now_delinq'], axis=1)
x_test = x
for i in range(len(X.columns)):print('{0}:{1}'.format(X.columns[i], clf.feature_importances_[i]))
result = clf.predict(x_test)
f2_score, acc = f2_score(result, y_)
print('f2-score:{0}'.format(f2_score))
print('accuracy:{0}%'.format(acc * 100))
# r = pd.DataFrame({
#     'member_id': train_id,
#     'acc_now_delinq': result
# })
# cols = ['member_id', 'acc_now_delinq']
# r = r.ix[:, cols]
# r.to_csv('data/result.csv', index=False)
# 画饼状图
l = len(result)
one_counts = np.sum(result == 1)
labels = '0-{0}'.format(l - one_counts), '1-{0}'.format(one_counts)
fracs = [(l - one_counts) / l * 100, one_counts / l * 100]
explode = [0, 0.1]
plt.axes(aspect=1)
plt.pie(x=fracs, labels=labels, explode=explode, autopct='%3.1f %%',shadow=True, labeldistance=1.1, startangle=90, pctdistance=0.6)
plt.show()

计算f2-score和正确率：

# 计算f2-score和正确率
# data1:预测 data2:真实
def f2_score(data1, data2):tp = 0fn = 0fp = 0acc = 0for i in range(len(data1)):if data1[i] == 1 and data2[i+len(data1)-1] == 1:tp += 1acc += 1elif data1[i] == 0 and data2[i+len(data1)-1] == 1:fn += 1elif data1[i] == 1 and data2[i+len(data1)-1] == 0:fp += 1elif data1[i] == 0 and data2[i+len(data1)-1] == 0:acc += 1if (tp + fn) != 0:r = tp / (tp + fn)else:r = 0if (tp + fp) != 0:p = tp / (tp + fp)else:p = 0if p == 0 or r == 0:score = 0else:score = 5 * p * r / (4 * p + r)ac = acc / len(data1)return score, ac

最后附上GitHub链接（代码）。