```python
import pandas as pd#导入csv文件的库
import numpy as np#进行矩阵运算的库
import json#用于读取和写入json数据格式#model lgb分类模型,日志评估,早停防止过拟合
from  lightgbm import LGBMClassifier,log_evaluation,early_stopping
#metric
from sklearn.metrics import roc_auc_score#导入roc_auc曲线
#KFold是直接分成k折,StratifiedKFold还要考虑每种类别的占比
from sklearn.model_selection import StratifiedKFold#config
class Config():seed=2024#随机种子num_folds=10#K折交叉验证TARGET_NAME ='label'#标签
import random#提供了一些用于生成随机数的函数
#设置随机种子,保证模型可以复现
def seed_everything(seed):np.random.seed(seed)#numpy的随机种子random.seed(seed)#python内置的随机种子
seed_everything(Config.seed)path='/kaggle/input/'
#sample: Iki037dt dict_keys(['name', 'normal_data', 'outliers'])
with open(path+"whoiswho-ind-kdd-2024/IND-WhoIsWho/train_author.json") as f:train_author=json.load(f)
#sample : 6IsfnuWU dict_keys(['id', 'title', 'authors', 'abstract', 'keywords', 'venue', 'year'])   
with open(path+"whoiswho-ind-kdd-2024/IND-WhoIsWho/pid_to_info_all.json") as f:pid_to_info=json.load(f)
#efQ8FQ1i dict_keys(['name', 'papers'])
with open(path+"whoiswho-ind-kdd-2024/IND-WhoIsWho/ind_valid_author.json") as f:valid_author=json.load(f)with open(path+"whoiswho-ind-kdd-2024/IND-WhoIsWho/ind_valid_author_submit.json") as f:submission=json.load(f)train_feats=[]
labels=[]
for id,person_info in train_author.items():for text_id in person_info['normal_data']:#正样本feat=pid_to_info[text_id]#['title', 'abstract', 'keywords', 'authors', 'venue', 'year']try:train_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),int(feat['year'])])except:train_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),2000])labels.append(1)for text_id in person_info['outliers']:#负样本feat=pid_to_info[text_id]#['title', 'abstract', 'keywords', 'authors', 'venue', 'year']try:train_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),int(feat['year'])])except:train_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),2000])labels.append(0)   
train_feats=np.array(train_feats)
labels=np.array(labels)
print(f"train_feats.shape:{train_feats.shape},labels.shape:{labels.shape}")
print(f"np.mean(labels):{np.mean(labels)}")
train_feats=pd.DataFrame(train_feats)
train_feats['label']=labels
train_feats.head()valid_feats=[]
for id,person_info in valid_author.items():for text_id in person_info['papers']:feat=pid_to_info[text_id]#['title', 'abstract', 'keywords', 'authors', 'venue', 'year']try:valid_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),int(feat['year'])])except:valid_feats.append([len(feat['title']),len(feat['abstract']),len(feat['keywords']),len(feat['authors']),len(feat['keywords']),2000])
valid_feats=np.array(valid_feats)
print(f"valid_feats.shape:{valid_feats.shape}")
valid_feats=pd.DataFrame(valid_feats)
valid_feats.head()choose_cols=[col for col in valid_feats.columns]
def fit_and_predict(model,train_feats=train_feats,test_feats=valid_feats,name=0):X=train_feats[choose_cols].copy()y=train_feats[Config.TARGET_NAME].copy()test_X=test_feats[choose_cols].copy()oof_pred_pro=np.zeros((len(X),2))test_pred_pro=np.zeros((Config.num_folds,len(test_X),2))#10折交叉验证skf = StratifiedKFold(n_splits=Config.num_folds,random_state=Config.seed, shuffle=True)for fold, (train_index, valid_index) in (enumerate(skf.split(X, y.astype(str)))):print(f"name:{name},fold:{fold}")X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]model.fit(X_train,y_train,eval_set=[(X_valid, y_valid)],callbacks=[log_evaluation(100),early_stopping(100)])oof_pred_pro[valid_index]=model.predict_proba(X_valid)#将数据分批次进行预测.test_pred_pro[fold]=model.predict_proba(test_X)print(f"roc_auc:{roc_auc_score(y.values,oof_pred_pro[:,1])}")return oof_pred_pro,test_pred_pro
#参数来源:https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm
lgb_params={"boosting_type": "gbdt","objective": "binary","metric": "auc","max_depth": 12,"learning_rate": 0.05,"n_estimators":3072,"colsample_bytree": 0.9,"colsample_bynode": 0.9,"verbose": -1,"random_state": Config.seed,"reg_alpha": 0.1,"reg_lambda": 10,"extra_trees":True,'num_leaves':64,"verbose": -1,"max_bin":255,}lgb_oof_pred_pro,lgb_test_pred_pro=fit_and_predict(model= LGBMClassifier(**lgb_params),name='lgb')
test_preds=lgb_test_pred_pro.mean(axis=0)[:,1]cnt=0
for id,names in submission.items():for name in names:submission[id][name]=test_preds[cnt]cnt+=1
with open('baseline.json', 'w', encoding='utf-8') as f:json.dump(submission, f, ensure_ascii=False, indent=4)