datawhale 机器学习夏令营分子AI数据挖掘比赛笔记

# datawhale 机器学习分子AI数据挖掘比赛笔记

## task3 未完成版

```python

'''

导入库

catboost 处理分类和回归任务的机器学习库

sklearn 传统机器学习库

rdkit

warnings 处理忽略错误

'''

import numpy as np

import pandas as pd

from catboost import CatBoostClassifier

from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold

from sklearn.metrics import f1_score

from rdkit import Chem

from rdkit.Chem import Descriptors

from sklearn.feature_extraction.text import TfidfVectorizer

import tqdm, sys, os, gc, re, argparse, warnings

warnings.filterwarnings('ignore')

train = pd.read_excel('./traindata-new.xlsx')

test = pd.read_excel('./testdata-new.xlsx')

# test数据不包含 DC50 (nM) 和 Dmax (%)

train = train.drop(['DC50 (nM)', 'Dmax (%)'], axis=1)

# 定义了一个空列表drop_cols，用于存储在测试数据集中非空值小于10个的列名。

drop_cols = []

for f in test.columns:

if test[f].notnull().sum() < 10:

drop_cols.append(f)

 

# 使用drop方法从训练集和测试集中删除了这些列，以避免在后续的分析或建模中使用这些包含大量缺失值的列

train = train.drop(drop_cols, axis=1)

test = test.drop(drop_cols, axis=1)

# 使用pd.concat将清洗后的训练集和测试集合并成一个名为data的DataFrame，便于进行统一的特征工程处理

data = pd.concat([train, test], axis=0, ignore_index=True)

cols = data.columns[2:]

'''

这里将SMILES转换为分子对象，使得SMILES的特征性更强（包含更多具有现实意义的特征）

'''

# 将SMILES转换为分子对象列表,并转换为SMILES字符串列表

data['smiles_list'] = data['Smiles'].apply(lambda x:[Chem.MolToSmiles(mol, isomericSmiles=True) for mol in [Chem.MolFromSmiles(x)]])

data['smiles_list'] = data['smiles_list'].map(lambda x: ' '.join(x))

'''

这里使用TD-IDF增强特征处理（用部分词频提高特征）

'''

# 使用TfidfVectorizer计算TF-IDF

tfidf = TfidfVectorizer(max_df = 0.9, min_df = 1, sublinear_tf = True)

res = tfidf.fit_transform(data['smiles_list'])

# 将结果转为dataframe格式

tfidf_df = pd.DataFrame(res.toarray())

tfidf_df.columns = [f'smiles_tfidf_{i}' for i in range(tfidf_df.shape[1])]

# 按列合并到data数据

data = pd.concat([data, tfidf_df], axis=1)

# 自然数编码

def label_encode(series):

unique = list(series.unique())

return series.map(dict(zip(

unique, range(series.nunique())

)))

for col in cols:

if data[col].dtype == 'object':

data[col] = label_encode(data[col])

 

train = data[data.Label.notnull()].reset_index(drop=True)

test = data[data.Label.isnull()].reset_index(drop=True)

# 特征筛选

features = [f for f in train.columns if f not in ['uuid','Label','smiles_list']]

# 构建训练集和测试集

x_train = train[features]

x_test = test[features]

# 训练集标签

y_train = train['Label'].astype(int)

def cv_model(clf, train_x, train_y, test_x, clf_name, seed=2022):

'''使用5折交叉验证、f1分数、AUC曲线等'''

 

kf = KFold(n_splits=5, shuffle=True, random_state=seed)

train = np.zeros(train_x.shape[0])

test = np.zeros(test_x.shape[0])

cv_scores = []

# 100， 1 2 3 4 5

# 1 2 3 4 5

# 1 2 3 5。 4

# 1

for i, (train_index, valid_index) in enumerate(kf.split(train_x, train_y)):

print('************************************ {} {}************************************'.format(str(i+1), str(seed)))

trn_x, trn_y, val_x, val_y = train_x.iloc[train_index], train_y[train_index], train_x.iloc[valid_index], train_y[valid_index]

 

params = {'learning_rate': 0.1, 'depth': 6, 'l2_leaf_reg': 10, 'bootstrap_type':'Bernoulli','random_seed':seed,

'od_type': 'Iter', 'od_wait': 100, 'allow_writing_files': False, 'task_type':'CPU'}

model = clf(iterations=20000, **params, eval_metric='AUC')

model.fit(trn_x, trn_y, eval_set=(val_x, val_y),

metric_period=100,

cat_features=[],

use_best_model=True,

verbose=1)

val_pred = model.predict_proba(val_x)[:,1]

test_pred = model.predict_proba(test_x)[:,1]

 

train[valid_index] = val_pred

test += test_pred / kf.n_splits

cv_scores.append(f1_score(val_y, np.where(val_pred>0.5, 1, 0)))

 

print(cv_scores)

 

print("%s_score_list:" % clf_name, cv_scores)

print("%s_score_mean:" % clf_name, np.mean(cv_scores))

print("%s_score_std:" % clf_name, np.std(cv_scores))

return train, test

 

'''套用CatBoost分类器'''

cat_train, cat_test = cv_model(CatBoostClassifier, x_train, y_train, x_test, "cat")

pd.DataFrame(

{

'uuid': test['uuid'],

'Label': np.where(cat_test>0.5, 1, 0)

}

).to_csv('submit.csv', index=None)

```