From 7139f635321ee2060f92f229f65e9672cb607e84 Mon Sep 17 00:00:00 2001 From: sunlanchang Date: Sat, 16 May 2020 12:23:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dense_classifier.py => Dense_embedding.py | 0 LightGBM.py | 14 - LightGBM_embedding.py | 301 ++++++++++++++++++++++ README.md | 2 +- 4 files changed, 302 insertions(+), 15 deletions(-) rename dense_classifier.py => Dense_embedding.py (100%) create mode 100644 LightGBM_embedding.py diff --git a/dense_classifier.py b/Dense_embedding.py similarity index 100% rename from dense_classifier.py rename to Dense_embedding.py diff --git a/LightGBM.py b/LightGBM.py index 3354507..8b02ab1 100644 --- a/LightGBM.py +++ b/LightGBM.py @@ -232,17 +232,3 @@ def test(): test() -# %% -# for leaves in range(10, 13): -# gbm_age = LGBM_age(leaves) -# y_pred_probability = gbm_age.predict( -# X_val, num_iteration=gbm_age.best_iteration) -# y_pred = np.argmax(y_pred_probability, axis=1) -# print('v'*20) -# print('leaves: ', leaves) -# print('The accuracy of prediction is:{:.2f}'.format( -# accuracy_score(y_val_age, y_pred))) -# print('v'*20) - - -# %% diff --git a/LightGBM_embedding.py b/LightGBM_embedding.py new file mode 100644 index 0000000..07217b8 --- /dev/null +++ b/LightGBM_embedding.py @@ -0,0 +1,301 @@ +# %% +from tensorflow.keras.layers import Dense +from tensorflow.keras.wrappers.scikit_learn import KerasClassifier +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import StratifiedKFold +from sklearn.preprocessing import StandardScaler +from sklearn.pipeline import Pipeline +import lightgbm as lgb +import time +from sklearn.metrics import accuracy_score +import pandas as pd +import numpy as np +import tensorflow as tf +from tensorflow import keras +from keras.utils import to_categorical +from tensorflow.keras import layers +from tensorflow.keras.utils import multi_gpu_model +import gc +# %% +samples = 1000 +all_train_data = pd.read_csv('word2vec/data_vec_product_category_industry_16dimension.csv', + nrows=900000, skiprows=None) +# nrows=samples, skiprows=None).sort_values(['user_id'], ascending=(True,)) +columns = all_train_data.columns.values.tolist() +test_data = pd.read_csv('word2vec/data_vec_product_category_industry_16dimension.csv', + names=columns, + skiprows=900001, + # nrows=samples, + ).sort_values(['user_id'], ascending=(True,)) + +user_train = pd.read_csv( + 'data/train_preliminary/user.csv').sort_values(['user_id'], ascending=(True,)) +Y_gender = user_train['gender'].values +Y_age = user_train['age'].values + +all_train_data['gender'] = user_train.gender +all_train_data['age'] = user_train.age + +TRAIN_DATA_PERCENT = 0.9 +mask = np.random.rand(len(all_train_data)) < TRAIN_DATA_PERCENT +df_train = all_train_data[mask] +df_val = all_train_data[~mask] + +X_train = df_train[columns].values +Y_train_gender = df_train.gender.values +Y_train_age = df_train.age.values + +X_val = df_val[columns].values +Y_val_gender = df_val.gender.values +Y_val_age = df_val.age.values +# del train_data +# gc.collect() +X_test = test_data[columns].values +# del test_data +# gc.collect() + +user_id_test = pd.read_csv( + 'data/test/clicklog_ad_user_test.csv').sort_values(['user_id'], ascending=(True,)).user_id.unique() +ans = pd.DataFrame({'user_id': user_id_test}) +# %% +# 构建性别数据 +encoder = LabelEncoder() +encoder.fit(Y_train_gender) +Y_train_gender = encoder.transform(Y_train_gender) +Y_val_gender = encoder.transform(Y_val_gender) + +lgb_train_gender = lgb.Dataset(X_train, Y_train_gender) +lgb_eval_gender = lgb.Dataset(X_val, Y_val_gender, reference=lgb_train_gender) +# 构建年龄数据 +encoder = LabelEncoder() +encoder.fit(Y_train_age) +Y_train_age = encoder.transform(Y_train_age) +Y_val_age = encoder.transform(Y_val_age) + +lgb_train_age = lgb.Dataset(X_train, Y_train_age) +lgb_eval_age = lgb.Dataset(X_val, Y_val_age, reference=lgb_train_age) + +# %% + + +def LGBM_gender(): + params_gender = { + 'task': 'train', + 'boosting_type': 'gbdt', + 'objective': 'binary', + 'metric': {'binary_logloss', 'binary_error'}, # evaluate指标 + 'max_depth': -1, # 不限制树深度 + # 更高的accuracy + 'max_bin': 2**10-1, + + 'num_leaves': 2**10, + 'min_data_in_leaf': 1, + 'learning_rate': 0.01, + # 'feature_fraction': 0.9, + # 'bagging_fraction': 0.8, + # 'bagging_freq': 5, + # 'is_provide_training_metric': True, + 'verbose': 1 + } + print('Start training...') + # train + gbm = lgb.train(params_gender, + lgb_train_gender, + num_boost_round=50, + valid_sets=lgb_eval_gender, + # early_stopping_rounds=5, + ) + print('training done!') + print('Saving model...') + # save model to file + gbm.save_model('tmp/model_gender.txt') + print('save model done!') + return gbm +# %% + + +def LGBM_age(): + params_age = { + 'boosting_type': 'gbdt', + 'objective': 'multiclass', + "num_class": 10, + # fine-tuning最重要的三个参数 + 'num_leaves': 2**10-1, + 'max_depth': -1, # 不限制树深度 + 'min_data_in_leaf': 1, + # 更高的accuracy + # 'max_bin': 2**9-1, + + 'metric': {'multi_logloss', 'multi_error'}, + 'learning_rate': 0.1, + + # 'feature_fraction': 0.9, + # 'bagging_fraction': 0.8, + # 'bagging_freq': 5, + 'verbose': 1 + } + print('Start training...') + # train + gbm = lgb.train(params_age, + lgb_train_age, + num_boost_round=50, + valid_sets=lgb_eval_age, + # early_stopping_rounds=5, + ) + print('Saving model...') + # save model to file + gbm.save_model('tmp/model_age.txt') + print('save model done!') + return gbm + + +# %% +gbm_gender = LGBM_gender() +# %% +gbm_age = LGBM_age() +# %% + + +def evaluate(): + print('Start predicting...') + y_pred_gender_probability = gbm_gender.predict( + X_val, num_iteration=gbm_gender.best_iteration) + threshold = 0.5 + y_pred_gender = np.where(y_pred_gender_probability > threshold, 1, 0) + # eval + print('threshold: {:.1f} The accuracy of prediction is:{:.2f}'.format(threshold, + accuracy_score(Y_val_gender, y_pred_gender))) + print('Start evaluate data predicting...') + y_pred_age_probability = gbm_age.predict( + X_val, num_iteration=gbm_age.best_iteration) + y_pred_age = np.argmax(y_pred_age_probability, axis=1) + # eval + print('The accuracy of prediction is:{:.2f}'.format( + accuracy_score(Y_val_age, y_pred_age))) + + # d = {'user_id': X_val.user_id.values.tolist(), 'gender': Y_pred_gender.tolist(), + # 'age': y_pred_age.tolist()} + # ans_df = pd.DataFrame(data=d) + # # 投票的方式决定gender、age + # ans_df_grouped = ans_df.groupby(['user_id']).agg( + # lambda x: x.value_counts().index[0]) + # ans_df_grouped.gender = ans_df_grouped.gender+1 + # ans_df_grouped.age = ans_df_grouped.age+1 + # ans_df_grouped.to_csv('data/ans.csv', header=True) + + +# %% +evaluate() +# %% + + +def test(): + print('Start predicting test gender data ...') + y_pred_gender_probability = gbm_gender.predict( + X_test, num_iteration=gbm_gender.best_iteration) + threshold = 0.5 + y_pred_gender = np.where(y_pred_gender_probability > threshold, 1, 0) + + print('Start predicting test age data ...') + y_pred_age_probability = gbm_age.predict( + X_test, num_iteration=gbm_age.best_iteration) + y_pred_age = np.argmax(y_pred_age_probability, axis=1) + + ans['predicted_age'] = y_pred_age+1 + ans['predicted_gender'] = y_pred_gender+1 + ans.to_csv('data/ans/LGBM.csv', header=True, index=False, + columns=['user_id', 'predicted_age', 'predicted_gender']) + + # ans_df = pd.DataFrame(data=d) + # 投票的方式决定gender、age + # ans_df_grouped = ans_df.groupby(['user_id']).agg( + # lambda x: x.value_counts().index[0]) + # ans_df_grouped['user_id'] = ans_df_grouped.index + # ans_df_grouped.gender = ans_df_grouped.gender+1 + # ans_df_grouped.age = ans_df_grouped.age+1 + # columns_order = ['user_id', 'predicted_age', 'predicted_gender'] + # ans_df_grouped[columns_order].to_csv( + # 'data/ans_test.csv', header=True, columns=['user_id', 'predicted_age', 'predicted_gender'], index=False) + # print('Done!!!') + + +test() +# %% +# %% +# df_train = df_train.sort_values( +# ["user_id"], ascending=(True,)) + +# # %% + + +# def get_batch(file_name,): +# for row in open(file_name, "r"): +# yield 1 + + +# for line in get_batch('data/train_data.csv'): +# for line in get_batch('test.py'): +# print(line) +# break +# %% +# 合成用户embedding +# path = "word2vec/wordvectors.kv" +# wv = KeyedVectors.load(path, mmap='r') +# with open('word2vec/userid_creativeids.txt', 'r')as f: +# lines = f.readlines() +# lines = [[int(e) for e in line.split(' ')] for line in lines] +# number_train_user = 900000 +# number_test_user = 1000000 +# user_train = lines[:number_train_user] +# user_test = lines[number_train_user:] +# columns = ['c'+str(i) for i in range(128)] +# data = {} +# for col_name in columns: +# data[col_name] = pd.Series([], dtype='float') +# df_user_train = pd.DataFrame(data) +# df_user_test = pd.DataFrame(data) +# # %% +# for line in tqdm.tqdm(user_train): +# user_embedding_train = np.zeros(128) +# for creative_id in line: +# user_embedding_train += wv[str(creative_id)] +# user_embedding_train = user_embedding_train / len(line) +# tmp = pd.DataFrame(user_embedding_train.reshape(-1, +# len(user_embedding_train)), columns=columns) +# df_user_train = df_user_train.append(tmp) +# # %% +# for line in tqdm.tqdm(user_test): +# user_embedding_test = np.zeros(128) +# for creative_id in line: +# user_embedding_test += wv[str(creative_id)] +# user_embedding_test = user_embedding_test / len(line) +# tmp = pd.DataFrame(user_embedding_test.reshape(-1, +# len(user_embedding_train)), columns=columns) +# df_user_test = df_user_test.append(tmp) +# # %% +# # 将同一个用户creative_id相加平均后即为一个用户的Embedding +# all_train_data = pd.read_csv( +# 'data/train_preliminary/clicklog_ad_user_train_eval_test.csv') +# all_train_data = all_train_data.sort_values( +# ["user_id"], ascending=(True)) +# # %% +# all_test_data = pd.read_csv( +# 'data/test/clicklog_ad_user_test.csv') +# all_test_data = all_test_data.sort_values( +# ["user_id"], ascending=(True)) +# # %% +# assert df_user_train.shape[0] == all_train_data.shape[0] +# df_user_train['user_id'] = all_train_data['user_id'] +# df_user_train['gender'] = all_train_data['gender'] +# df_user_train['age'] = all_train_data['age'] +# df_user_train.to_hdf('word2vec/df_user_train_test.h5', +# key='df_user_train', mode='w') +# # %% +# assert df_user_test.shape[0] == all_test_data.shape[0] +# df_user_test['user_id'] = all_test_data['user_id'] +# df_user_test.to_hdf('word2vec/df_user_train_test.h5', +# key='df_user_test', mode='a') + + +# %% diff --git a/README.md b/README.md index 1e44870..a9f5584 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - [x] 处理成序列问题后，使用word2vec生成词嵌入后 - [x] +Dense (accuracy: 1.05) - [ ] +RNN等序列模型 - - [ ] +LightGBM + - [x] +LightGBM (accuraty: 0.8) - [ ] GNN生成user_id creative_id ad_id等的词嵌入后分类 - [ ] +Dense - [ ] +RNN等序列模型