v1

Author: TanTingyi

deepctr/layers/__init__.py

@@ -4,7 +4,8 @@
 from .core import DNN, LocalActivationUnit, PredictionLayer
 from .interaction import (CIN, FM, AFMLayer, BiInteractionPooling, CrossNet,
                           InnerProductLayer, InteractingLayer,
-                          OutterProductLayer, FGCNNLayer,SENETLayer,BilinearInteraction)
+                          OutterProductLayer, FGCNNLayer,SENETLayer,BilinearInteraction,
+                          FieldWiseBiInteraction)
 from .normalization import LayerNormalization
 from .sequence import (AttentionSequencePoolingLayer, BiasEncoding, BiLSTM,
                        KMaxPooling, SequencePoolingLayer,WeightedSequenceLayer,
@@ -39,5 +40,6 @@
                   'SENETLayer':SENETLayer,
                   'BilinearInteraction':BilinearInteraction,
                   'WeightedSequenceLayer':WeightedSequenceLayer,
-                  'Add':Add
+                  'Add':Add,
+                  'FieldWiseBiInteraction':FieldWiseBiInteraction
                   }

deepctr/layers/interaction.py

@@ -1042,3 +1042,123 @@ def get_config(self, ):
         config = {'bilinear_type': self.bilinear_type, 'seed': self.seed}
         base_config = super(BilinearInteraction, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
+
+
+class FieldWiseBiInteraction(Layer):
+    """Field-Wise Bi-Interaction Layer used in FLEN,compress the
+     pairwise element-wise product of features into one single vector.
+
+      Input shape
+        - A list of 3D tensor with shape:``(batch_size,field_size,embedding_size)``.
+
+      Output shape
+        - 2D tensor with shape: ``(batch_size,embedding_size)``.
+     
+      Arguments
+        - **use_bias** : Boolean, if use bias.
+        - **l2_reg** : Float, l2 regularization coefficient.
+        - **seed** : A Python integer to use as random seed.
+     
+      References
+        [1] hen W, Zhan L, Ci Y, Lin C https://arxiv.org/pdf/1911.04690
+    """
+    def __init__(self, l2_reg=1e-5, seed=1024, **kwargs):
+
+        self.l2_reg = l2_reg
+        self.seed = seed
+
+        super(FieldWiseBiInteraction, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+
+        if not isinstance(input_shape, list) or len(input_shape) < 2:
+            raise ValueError(
+                'A `Field-Wise Bi-Interaction` layer should be called '
+                'on a list of at least 2 inputs')
+
+        self.num_fields = len(input_shape)
+        embedding_size = input_shape[0][-1]
+
+        self.kernel_inter = self.add_weight(
+            name='kernel_inter',
+            shape=(int(self.num_fields * (self.num_fields - 1) / 2), 1),
+            initializer=glorot_normal(seed=self.seed),
+            regularizer=l2(self.l2_reg),
+            trainable=True)
+        self.bias_inter = self.add_weight(name='bias_inter',
+                                          shape=(embedding_size),
+                                          initializer=Zeros(),
+                                          trainable=True)
+        self.kernel_intra = self.add_weight(
+            name='kernel_intra',
+            shape=(self.num_fields, 1),
+            initializer=glorot_normal(seed=self.seed),
+            regularizer=l2(self.l2_reg),
+            trainable=True)
+        self.bias_intra = self.add_weight(name='bias_intra',
+                                          shape=(embedding_size),
+                                          initializer=Zeros(),
+                                          trainable=True)
+
+        super(FieldWiseBiInteraction,
+              self).build(input_shape)  # Be sure to call this somewhere!
+
+    def call(self, inputs, **kwargs):
+
+        if K.ndim(inputs[0]) != 3:
+            raise ValueError(
+                "Unexpected inputs dimensions %d, expect to be 3 dimensions" %
+                (K.ndim(inputs)))
+
+        field_wise_embeds_list = inputs
+
+        # MF module
+        field_wise_vectors = tf.concat([
+            reduce_sum(field_i_vectors, axis=1, keep_dims=True)
+            for field_i_vectors in field_wise_embeds_list
+        ], 1)
+
+        left = []
+        right = []
+        for i in range(self.num_fields):
+            for j in range(i + 1, self.num_fields):
+                left.append(i)
+                right.append(j)
+
+        embeddings_left = tf.gather(params=field_wise_vectors,
+                                    indices=left,
+                                    axis=1)
+        embeddings_right = tf.gather(params=field_wise_vectors,
+                                     indices=right,
+                                     axis=1)
+
+        embeddings_prod = embeddings_left * embeddings_right
+        field_weighted_embedding = embeddings_prod * self.kernel_inter
+        h_mf = reduce_sum(field_weighted_embedding, axis=1)
+        h_mf = tf.nn.bias_add(h_mf, self.bias_inter)
+
+        # FM module
+        square_of_sum_list = [
+            tf.square(reduce_sum(field_i_vectors, axis=1, keep_dims=True))
+            for field_i_vectors in field_wise_embeds_list
+        ]
+        sum_of_square_list = [
+            reduce_sum(field_i_vectors * field_i_vectors,
+                       axis=1,
+                       keep_dims=True)
+            for field_i_vectors in field_wise_embeds_list
+        ]
+
+        field_fm = tf.concat([
+            square_of_sum - sum_of_square for square_of_sum, sum_of_square in
+            zip(square_of_sum_list, sum_of_square_list)
+        ], 1)
+
+        h_fm = reduce_sum(field_fm * self.kernel_intra, axis=1)
+
+        h_fm = tf.nn.bias_add(h_fm, self.bias_intra)
+
+        return h_mf + h_fm
+
+    def compute_output_shape(self, input_shape):
+        return (None, input_shape[0][-1])

deepctr/models/__init__.py

@@ -16,6 +16,7 @@
 from .fgcnn import FGCNN
 from .dsin import DSIN
 from .fibinet import FiBiNET
+from .flen import FLEN
 
 __all__ = ["AFM", "CCPM","DCN", "MLR",  "DeepFM",
-           "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN", "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET"]
+           "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN", "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET", 'FLEN']

deepctr/models/flen.py

@@ -0,0 +1,95 @@
+# -*- coding:utf-8 -*-
+"""
+Author:
+    Tingyi Tan,[email protected]
+
+Reference:
+    [1] hen W, Zhan L, Ci Y, Lin C https://arxiv.org/pdf/1911.04690
+
+"""
+
+
+from itertools import chain
+import tensorflow as tf
+from tensorflow.python.keras.layers import Flatten
+
+from ..inputs import input_from_feature_columns, get_linear_logit, build_input_features, combined_dnn_input
+from ..layers.core import PredictionLayer, DNN
+from ..layers.utils import concat_func, add_func
+from ..layers.interaction import FieldWiseBiInteraction
+
+
+def FLEN(linear_feature_columns,
+          dnn_feature_columns,
+          l2_reg_linear=0.00001,
+          l2_reg_embedding=0.00001,
+          l2_reg_dnn=0.00001,
+          l2_reg_fw=0.00001,
+          init_std=0.0001,
+          seed=1024,
+          dnn_dropout=0.2,
+          dnn_activation='relu',
+          dnn_use_bn=True,
+          task='binary'):
+    """Instantiates the DeepFM Network architecture.
+
+    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param l2_reg_linear: float. L2 regularizer strength applied to linear part
+    :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
+    :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
+    :param l2_reg_fw: float. L2 regularizer strength applied to fwfm
+    :param init_std: float,to use as the initialize std of embedding vector
+    :param seed: integer ,to use as random seed.
+    :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
+    :param dnn_activation: Activation function to use in DNN
+    :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN
+    :param task: str, ``"binary"`` for  binary logloss or  ``"regression"`` for regression loss
+    :return: A Keras model instance.
+    """
+
+    features = build_input_features(linear_feature_columns +
+                                    dnn_feature_columns)
+
+    inputs_list = list(features.values())
+
+    group_embedding_dict, dense_value_list = input_from_feature_columns(
+        features,
+        dnn_feature_columns,
+        l2_reg_embedding,
+        init_std,
+        seed,
+        support_group=True)
+
+    # S
+    linear_logit = get_linear_logit(features,
+                                    linear_feature_columns,
+                                    init_std=init_std,
+                                    seed=seed,
+                                    prefix='linear',
+                                    l2_reg=l2_reg_linear)
+    linear_logit = Flatten()(linear_logit)
+
+    # FM + MF
+    fm_mf_out = FieldWiseBiInteraction(l2_reg=l2_reg_fw, seed=seed)(
+        [concat_func(v, axis=1) for k, v in group_embedding_dict.items()])
+    fm_mf_out = DNN((32,), dnn_activation, l2_reg_dnn, dnn_dropout,
+                    dnn_use_bn, seed)(fm_mf_out)
+
+    # MLP
+    mlp_input = combined_dnn_input(
+        list(chain.from_iterable(group_embedding_dict.values())),
+        dense_value_list)
+    mlp_output = DNN((64,), dnn_activation, l2_reg_dnn, dnn_dropout,
+                     dnn_use_bn, seed)(mlp_input)
+    mlp_output = DNN((32,), dnn_activation, l2_reg_dnn, dnn_dropout,
+                     dnn_use_bn, seed)(mlp_output)
+
+    # DNN
+    dnn_input = combined_dnn_input([fm_mf_out, mlp_output, linear_logit], dense_value_list)
+    dnn_output = dnn_input
+    dnn_logit = tf.keras.layers.Dense(1, use_bias=False, activation=None)(dnn_output)
+    output = PredictionLayer(task)(dnn_logit)
+
+    model = tf.keras.models.Model(inputs=inputs_list, outputs=output)
+    return model