Add FiBiNET

Author: 浅梦

.travis.yml

@@ -3,7 +3,7 @@
 language: python
 
 python:
-  - "2.7"
+  #- "2.7" time out
   - "3.4"
   - "3.5"
   - "3.6"

README.md

@@ -33,12 +33,10 @@ Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Star
 |   Attentional Factorization Machine    | [IJCAI 2017][Attentional Factorization Machines: Learning the Weight of Feature Interactions via Attention Networks](http://www.ijcai.org/proceedings/2017/435) |
 |      Neural Factorization Machine      | [SIGIR 2017][Neural Factorization Machines for Sparse Predictive Analytics](https://arxiv.org/pdf/1708.05027.pdf)                                               |
 |                xDeepFM                 | [KDD 2018][xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems](https://arxiv.org/pdf/1803.05170.pdf)                         |
+|                AutoInt                 | [arxiv 2018][AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921)                              |
 |         Deep Interest Network          | [KDD 2018][Deep Interest Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1706.06978.pdf)                                                       |
 |    Deep Interest Evolution Network     | [AAAI 2019][Deep Interest Evolution Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1809.03672.pdf)                                           |
-|                AutoInt                 | [arxiv 2018][AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921)                              |
 |                  NFFM       | [arxiv 2019][Operation-aware Neural Networks for User Response Prediction](https://arxiv.org/pdf/1904.12579.pdf)                       |
 |                  FGCNN                  | [WWW 2019][Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction ](https://arxiv.org/pdf/1904.04447)   |
 |                  Deep Session Interest Network                | [IJCAI 2019][Deep Session Interest Network for Click-Through Rate Prediction ](https://arxiv.org/abs/1905.06482)  |
-
-
-
+|                  FiBiNET               | [RecSys 2019][FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf)  |

deepctr/__init__.py

@@ -2,5 +2,5 @@
 from . import models
 from .utils import check_version
 
-__version__ = '0.5.0'
+__version__ = '0.5.1'
 check_version(__version__)

deepctr/inputs.py

@@ -137,21 +137,6 @@ def get_embedding_vec_list(embedding_dict, input_dict, sparse_feature_columns, r
 
     return embedding_vec_list
 
-def get_varlen_pooling_list(embedding_dict, features, varlen_sparse_feature_columns):
-    pooling_vec_list = []
-    for fc in varlen_sparse_feature_columns:
-        feature_name = fc.name
-        combiner = fc.combiner
-        feature_length_name = feature_name + '_seq_length'
-        if feature_length_name in features:
-            vec = SequencePoolingLayer(combiner, supports_masking=False)(
-            [embedding_dict[feature_name], features[feature_length_name]])
-        else:
-            vec = SequencePoolingLayer(combiner, supports_masking=True)(
-            embedding_dict[feature_name])
-        pooling_vec_list.append(vec)
-    return pooling_vec_list
-
 
 def get_inputs_list(inputs):
     return list(chain(*list(map(lambda x: x.values(), filter(lambda x: x is not None, inputs)))))
@@ -217,6 +202,21 @@ def varlen_embedding_lookup(embedding_dict, sequence_input_dict, varlen_sparse_f
 
     return varlen_embedding_vec_dict
 
+def get_varlen_pooling_list(embedding_dict, features, varlen_sparse_feature_columns):
+    pooling_vec_list = []
+    for fc in varlen_sparse_feature_columns:
+        feature_name = fc.name
+        combiner = fc.combiner
+        feature_length_name = feature_name + '_seq_length'
+        if feature_length_name in features:
+            vec = SequencePoolingLayer(combiner, supports_masking=False)(
+            [embedding_dict[feature_name], features[feature_length_name]])
+        else:
+            vec = SequencePoolingLayer(combiner, supports_masking=True)(
+            embedding_dict[feature_name])
+        pooling_vec_list.append(vec)
+    return pooling_vec_list
+
 def get_dense_input(features,feature_columns):
     dense_feature_columns = list(filter(lambda x:isinstance(x,DenseFeat),feature_columns)) if feature_columns else []
     dense_input_list = []
@@ -225,7 +225,7 @@ def get_dense_input(features,feature_columns):
     return dense_input_list
 
 
-def input_from_feature_columns(features,feature_columns, embedding_size, l2_reg, init_std, seed,prefix='',seq_mask_zero=True):
+def input_from_feature_columns(features,feature_columns, embedding_size, l2_reg, init_std, seed,prefix='',seq_mask_zero=True,support_dense=True):
 
 
     sparse_feature_columns = list(filter(lambda x:isinstance(x,SparseFeat),feature_columns)) if feature_columns else []
@@ -235,6 +235,8 @@ def input_from_feature_columns(features,feature_columns, embedding_size, l2_reg,
     sparse_embedding_list = embedding_lookup(
         embedding_dict, features, sparse_feature_columns)
     dense_value_list = get_dense_input(features,feature_columns)
+    if not support_dense and len(dense_value_list) >0:
+        raise ValueError("DenseFeat is not supported in dnn_feature_columns")
 
     sequence_embed_dict = varlen_embedding_lookup(embedding_dict,features,varlen_sparse_feature_columns)
     sequence_embed_list = get_varlen_pooling_list(sequence_embed_dict, features, varlen_sparse_feature_columns)

deepctr/layers/__init__.py

@@ -4,7 +4,7 @@
 from .core import DNN, LocalActivationUnit, PredictionLayer
 from .interaction import (CIN, FM, AFMLayer, BiInteractionPooling, CrossNet,
                           InnerProductLayer, InteractingLayer,
-                          OutterProductLayer, FGCNNLayer)
+                          OutterProductLayer, FGCNNLayer,SENETLayer,BilinearInteraction)
 from .normalization import LayerNormalization
 from .sequence import (AttentionSequencePoolingLayer, BiasEncoding, BiLSTM,
                        KMaxPooling, SequencePoolingLayer,
@@ -34,4 +34,7 @@
                   'KMaxPooling': KMaxPooling,
                   'FGCNNLayer': FGCNNLayer,
                   'Hash': Hash,
-                  'DynamicGRU': DynamicGRU}
+                  'DynamicGRU': DynamicGRU,
+                  'SENETLayer':SENETLayer,
+                  'BilinearInteraction':BilinearInteraction,
+                  }

deepctr/layers/interaction.py

@@ -86,9 +86,11 @@ def build(self, input_shape):
                                             initializer=glorot_normal(seed=self.seed), name="projection_h")
         self.projection_p = self.add_weight(shape=(
             embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p")
-        self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed)
+        self.dropout = tf.keras.layers.Dropout(
+            self.dropout_rate, seed=self.seed)
 
-        self.tensordot = tf.keras.layers.Lambda(lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
+        self.tensordot = tf.keras.layers.Lambda(
+            lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
 
         # Be sure to call this somewhere!
         super(AFMLayer, self).build(input_shape)
@@ -232,7 +234,8 @@ def build(self, input_shape):
             self.filters.append(self.add_weight(name='filter' + str(i),
                                                 shape=[1, self.field_nums[-1]
                                                        * self.field_nums[0], size],
-                                                dtype=tf.float32, initializer=glorot_uniform(seed=self.seed + i),
+                                                dtype=tf.float32, initializer=glorot_uniform(
+                                                    seed=self.seed + i),
                                                 regularizer=l2(self.l2_reg)))
 
             self.bias.append(self.add_weight(name='bias' + str(i), shape=[size], dtype=tf.float32,
@@ -247,7 +250,8 @@ def build(self, input_shape):
             else:
                 self.field_nums.append(size)
 
-        self.activation_layers = [activation_layer(self.activation) for _ in self.layer_size]
+        self.activation_layers = [activation_layer(
+            self.activation) for _ in self.layer_size]
 
         super(CIN, self).build(input_shape)  # Be sure to call this somewhere!
 
@@ -668,7 +672,8 @@ def build(self, input_shape):
         if self.kernel_type == 'mat':
 
             self.kernel = self.add_weight(shape=(embed_size, num_pairs, embed_size),
-                                          initializer=glorot_uniform(seed=self.seed),
+                                          initializer=glorot_uniform(
+                                              seed=self.seed),
                                           name='kernel')
         elif self.kernel_type == 'vec':
             self.kernel = self.add_weight(shape=(num_pairs, embed_size,), initializer=glorot_uniform(self.seed),
@@ -796,12 +801,15 @@ def build(self, input_shape):
             width = self.kernel_width[i - 1]
             new_filters = self.new_maps[i - 1]
             pooling_width = self.pooling_width[i - 1]
-            conv_output_shape = self._conv_output_shape(pooling_shape, (width, 1))
-            pooling_shape = self._pooling_output_shape(conv_output_shape, (pooling_width, 1))
+            conv_output_shape = self._conv_output_shape(
+                pooling_shape, (width, 1))
+            pooling_shape = self._pooling_output_shape(
+                conv_output_shape, (pooling_width, 1))
             self.conv_layers.append(tf.keras.layers.Conv2D(filters=filters, kernel_size=(width, 1), strides=(1, 1),
                                                            padding='same',
                                                            activation='tanh', use_bias=True, ))
-            self.pooling_layers.append(tf.keras.layers.MaxPooling2D(pool_size=(pooling_width, 1)))
+            self.pooling_layers.append(
+                tf.keras.layers.MaxPooling2D(pool_size=(pooling_width, 1)))
             self.dense_layers.append(tf.keras.layers.Dense(pooling_shape[1] * embedding_size * new_filters,
                                                            activation='tanh', use_bias=True))
 
@@ -880,3 +888,157 @@ def _pooling_output_shape(self, input_shape, pool_size):
         cols = utils.conv_output_length(cols, pool_size[1], 'valid',
                                         pool_size[1])
         return [input_shape[0], rows, cols, input_shape[3]]
+
+
+class SENETLayer(Layer):
+    """SENETLayer used in FiBiNET.
+
+      Input shape
+        - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``.
+
+      Output shape
+        - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``.
+
+      Arguments
+        - **reduction_ratio** : Positive integer, dimensionality of the
+         attention network output space.
+
+        - **seed** : A Python integer to use as random seed.
+
+      References
+        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction
+Tongwen](https://arxiv.org/pdf/1905.09433.pdf)
+    """
+
+    def __init__(self, reduction_ratio=3,  seed=1024, **kwargs):
+        self.reduction_ratio = reduction_ratio
+
+        self.seed = seed
+        super(SENETLayer, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+
+        if not isinstance(input_shape, list) or len(input_shape) < 2:
+            raise ValueError('A `AttentionalFM` layer should be called '
+                             'on a list of at least 2 inputs')
+
+        self.filed_size = len(input_shape)
+        self.embedding_size = input_shape[0][-1]
+        reduction_size = max(1, self.filed_size//self.reduction_ratio)
+
+        self.W_1 = self.add_weight(shape=(
+            self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1")
+        self.W_2 = self.add_weight(shape=(
+            reduction_size, self.filed_size), initializer=glorot_normal(seed=self.seed), name="W_2")
+
+        self.tensordot = tf.keras.layers.Lambda(
+            lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
+
+        # Be sure to call this somewhere!
+        super(SENETLayer, self).build(input_shape)
+
+    def call(self, inputs, training=None, **kwargs):
+
+        if K.ndim(inputs[0]) != 3:
+            raise ValueError(
+                "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
+
+        inputs = concat_fun(inputs, axis=1)
+        Z = tf.reduce_mean(inputs, axis=-1,)
+
+        A_1 = tf.nn.relu(self.tensordot([Z, self.W_1]))
+        A_2 = tf.nn.relu(self.tensordot([A_1, self.W_2]))
+        V = tf.multiply(inputs, tf.expand_dims(A_2, axis=2))
+
+        return tf.split(V, self.filed_size, axis=1)
+
+    def compute_output_shape(self, input_shape):
+
+        return input_shape
+
+    def compute_mask(self, inputs, mask=None):
+        return [None]*self.filed_size
+
+    def get_config(self, ):
+        config = {'reduction_ratio': self.reduction_ratio, 'seed': self.seed}
+        base_config = super(SENETLayer, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class BilinearInteraction(Layer):
+    """BilinearInteraction Layer used in FiBiNET.
+
+      Input shape
+        - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``.
+
+      Output shape
+        - 3D tensor with shape: ``(batch_size,1,embedding_size)``.
+
+      Arguments
+        - **str** : String, types of bilinear functions used in this layer.
+
+        - **seed** : A Python integer to use as random seed.
+
+      References
+        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction
+Tongwen](https://arxiv.org/pdf/1905.09433.pdf)
+
+    """
+
+    def __init__(self, bilinear_type="interaction", seed=1024, **kwargs):
+        self.bilinear_type = bilinear_type
+        self.seed = seed
+
+        super(BilinearInteraction, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+
+        if not isinstance(input_shape, list) or len(input_shape) < 2:
+            raise ValueError('A `AttentionalFM` layer should be called '
+                             'on a list of at least 2 inputs')
+        embedding_size = input_shape[0][-1].value
+
+        if self.bilinear_type == "all":
+            self.W = self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(
+                seed=self.seed), name="bilinear_weight")
+        elif self.bilinear_type == "each":
+            self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(
+                seed=self.seed), name="bilinear_weight"+str(i)) for i in range(len(input_shape)-1)]
+        elif self.bilinear_type == "interaction":
+            self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(
+                seed=self.seed), name="bilinear_weight"+str(i)+'_'+str(j)) for i, j in itertools.combinations(range(len(input_shape)), 2)]
+        else:
+            raise NotImplementedError
+
+        super(BilinearInteraction, self).build(
+            input_shape)  # Be sure to call this somewhere!
+
+    def call(self, inputs, **kwargs):
+
+        if K.ndim(inputs[0]) != 3:
+            raise ValueError(
+                "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
+
+        if self.bilinear_type == "all":
+            p = [tf.multiply(tf.tensordot(v_i, self.W, axes=(-1, 0)), v_j)
+                 for v_i, v_j in itertools.combinations(inputs, 2)]
+        elif self.bilinear_type == "each":
+            p = [tf.multiply(tf.tensordot(inputs[i], self.W_list[i], axes=(-1, 0)), inputs[j])
+                 for i, j in itertools.combinations(range(len(inputs)), 2)]
+        elif self.bilinear_type == "interaction":
+            p = [tf.multiply(tf.tensordot(v[0], w, axes=(-1, 0)), v[1])
+                 for v, w in zip(itertools.combinations(inputs, 2), self.W_list)]
+        else:
+            raise NotImplementedError
+        return concat_fun(p)
+
+    def compute_output_shape(self, input_shape):
+        filed_size = len(input_shape)
+        embedding_size = input_shape[0][-1]
+
+        return (None, 1, filed_size*(filed_size-1)//2 * embedding_size)
+
+    def get_config(self, ):
+        config = {'type': self.bilinear_type, 'seed': self.seed}
+        base_config = super(BilinearInteraction, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

deepctr/models/__init__.py

@@ -14,6 +14,7 @@
 from .xdeepfm import xDeepFM
 from .fgcnn import FGCNN
 from .dsin import DSIN
+from .fibinet import FiBiNET
 
 __all__ = ["AFM", "CCPM","DCN", "MLR",  "DeepFM",
-           "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN", "WDL", "xDeepFM", "AutoInt", "NFFM", "FGCNN", "DSIN"]
+           "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN", "WDL", "xDeepFM", "AutoInt", "NFFM", "FGCNN", "DSIN","FiBiNET"]

deepctr/models/afm.py

@@ -43,8 +43,7 @@ def AFM(linear_feature_columns, dnn_feature_columns, embedding_size=8, use_atten
 
     sparse_embedding_list, _ = input_from_feature_columns(features,dnn_feature_columns,embedding_size,
                                                                                                l2_reg_embedding, init_std,
-                                                                                               seed)
-    # todo not support dense
+                                                                                               seed,support_dense=False)
 
     linear_logit = get_linear_logit(features, linear_feature_columns, l2_reg=l2_reg_linear, init_std=init_std,
                                     seed=seed, prefix='linear')

deepctr/models/ccpm.py

@@ -38,19 +38,16 @@ def CCPM(linear_feature_columns, dnn_feature_columns, embedding_size=8, conv_ker
     :return: A Keras model instance.
     """
 
-    #check_feature_config_dict(linear_feature_columns)
     if len(conv_kernel_width) != len(conv_filters):
         raise ValueError(
             "conv_kernel_width must have same element with conv_filters")
 
     features = build_input_features(linear_feature_columns+dnn_feature_columns)
     inputs_list = list(features.values())
 
-    sparse_embedding_list, _ = input_from_feature_columns(features,linear_feature_columns,
-                                                                                               embedding_size,
+    sparse_embedding_list, _ = input_from_feature_columns(features,linear_feature_columns,embedding_size,
                                                                                                l2_reg_embedding, init_std,
-                                                                                               seed)
-    #todo not support dense????
+                                                                                               seed,support_dense=False)
     linear_logit = get_linear_logit(features, linear_feature_columns, l2_reg=l2_reg_linear, init_std=init_std,
                                     seed=seed)