update transformer

README.md

@@ -45,8 +45,11 @@
     - [x] 3个特征+Dense (**accuracy: 1.05**)
     - [x] 6个128维特征+Conv 1D (**accuracy: 1.15**)
     - [x] 6个128维特征+Dense (**accuracy: 1.20**)
+    - [x] 6个128维特征+三层LSTM (**accuracy: 1.25**)
     - [x] 128维序列不做平均处理+LSTM Note: embedding跟随训练变化不是静态即trainable=True (**accuracy: 1.32**)
-- [ ] Transformer
+- [x] Transformer
+    - [x] 1层Transformer+1头注意力、ffm_dim=128、1层256的LSTM、3输入
+    - [ ] 1层Transformer+1头注意力、ffm_dim=128、1层256的LSTM、3输入、word2vwc初始化embedding
 - [ ] GNN生成user_id creative_id ad_id等的词嵌入后分类
     - [ ] +Dense
     - [ ] +RNN等序列模型

transformer_multi_input.py

@@ -1,22 +1,20 @@
-# %%
-# 生成词嵌入文件
+# %%
 from tensorflow.keras import layers
 from tensorflow import keras
-from tqdm import tqdm
-import numpy as np
-import pandas as pd
-from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler
-from gensim.models import Word2Vec, KeyedVectors
-from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Dropout, concatenate, Bidirectional
-from tensorflow.keras.models import Model, Sequential
-import tensorflow as tf
-from tensorflow.keras.preprocessing.sequence import pad_sequences
-from tensorflow.keras.preprocessing.text import Tokenizer
-from mymail import mail
-import os
-from tensorflow.keras.utils import to_categorical
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-
+import numpy as np
+from tqdm import tqdm
+import pandas as pd
+from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler
+from gensim.models import Word2Vec, KeyedVectors
+from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Dropout, concatenate, Bidirectional
+from tensorflow.keras.models import Model, Sequential
+import tensorflow as tf
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras.preprocessing.text import Tokenizer
+from mymail import mail
+import os
+from tensorflow.keras.utils import to_categorical
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
 # %%
 
 
@@ -239,6 +237,10 @@ def get_product_id_emb():
 NUM_ad_id = 2264190+1
 NUM_product_id = 33273+1
 
+LEN_creative_id = 100
+LEN_ad_id = 100
+LEN_product_id = 100
+
 vocab_size = NUM_creative_id 
 maxlen = LEN_creative_id
 
@@ -259,7 +261,7 @@ def get_age_model(creative_id_emb, ad_id_emb, product_id_emb):
     x1 = TokenAndPositionEmbedding(
         maxlen, vocab_size, embed_dim)(input_creative_id)
     x1 = TransformerBlock(embed_dim, num_heads, ff_dim)(x1)
-    x1 = GlobalAveragePooling1D()(x1)
+    x1 = layers.GlobalAveragePooling1D()(x1)
     x1 = layers.Dropout(0.1)(x1)
     x1 = layers.Dense(20, activation="relu")(x1)
     x1 = layers.Dropout(0.1)(x1)