[MRG+1] Allow vector pseudocounts Multinomial NB (scikit-learn#10346) (scikit-learn#10350)

Author: TobiasMadsen

doc/whats_new/v0.20.rst

@@ -221,6 +221,10 @@ Classifiers and regressors
   callable and b) the input to the NearestNeighbors model is sparse.
   :issue:`9579` by :user:`Thomas Kober <tttthomasssss>`.
 
+- Fixed a bug in :class:`naive_bayes.MultinomialNB` which did not accept vector
+  valued pseudocounts (alpha).
+  :issue:`10346` by :user:`Tobias Madsen <TobiasMadsen>`
+
 - Fixed a bug in :class:`svm.SVC` where when the argument ``kernel`` is
   unicode in Python2, the ``predict_proba`` method was raising an
   unexpected TypeError given dense inputs.

sklearn/naive_bayes.py

@@ -468,13 +468,17 @@ def _update_class_log_prior(self, class_prior=None):
             self.class_log_prior_ = np.zeros(n_classes) - np.log(n_classes)
 
     def _check_alpha(self):
-        if self.alpha < 0:
+        if np.min(self.alpha) < 0:
             raise ValueError('Smoothing parameter alpha = %.1e. '
-                             'alpha should be > 0.' % self.alpha)
-        if self.alpha < _ALPHA_MIN:
+                             'alpha should be > 0.' % np.min(self.alpha))
+        if isinstance(self.alpha, np.ndarray):
+            if not self.alpha.shape[0] == self.feature_count_.shape[1]:
+                raise ValueError("alpha should be a scalar or a numpy array "
+                                 "with shape [n_features]")
+        if np.min(self.alpha) < _ALPHA_MIN:
             warnings.warn('alpha too small will result in numeric errors, '
                           'setting alpha = %.1e' % _ALPHA_MIN)
-            return _ALPHA_MIN
+            return np.maximum(self.alpha, _ALPHA_MIN)
         return self.alpha
 
     def partial_fit(self, X, y, classes=None, sample_weight=None):

sklearn/tests/test_naive_bayes.py

@@ -668,3 +668,45 @@ def test_alpha():
                          X, y, classes=[0, 1])
     assert_raise_message(ValueError, expected_msg, m_nb.partial_fit,
                          X, y, classes=[0, 1])
+
+
+def test_alpha_vector():
+    X = np.array([[1, 0], [1, 1]])
+    y = np.array([0, 1])
+
+    # Setting alpha=np.array with same length
+    # as number of features should be fine
+    alpha = np.array([1, 2])
+    nb = MultinomialNB(alpha=alpha)
+    nb.partial_fit(X, y, classes=[0, 1])
+
+    # Test feature probabilities uses pseudo-counts (alpha)
+    feature_prob = np.array([[1 / 2, 1 / 2], [2 / 5, 3 / 5]])
+    assert_array_almost_equal(nb.feature_log_prob_, np.log(feature_prob))
+
+    # Test predictions
+    prob = np.array([[5 / 9, 4 / 9], [25 / 49, 24 / 49]])
+    assert_array_almost_equal(nb.predict_proba(X), prob)
+
+    # Test alpha non-negative
+    alpha = np.array([1., -0.1])
+    expected_msg = ('Smoothing parameter alpha = -1.0e-01. '
+                    'alpha should be > 0.')
+    m_nb = MultinomialNB(alpha=alpha)
+    assert_raise_message(ValueError, expected_msg, m_nb.fit, X, y)
+
+    # Test that too small pseudo-counts are replaced
+    ALPHA_MIN = 1e-10
+    alpha = np.array([ALPHA_MIN / 2, 0.5])
+    m_nb = MultinomialNB(alpha=alpha)
+    m_nb.partial_fit(X, y, classes=[0, 1])
+    assert_array_almost_equal(m_nb._check_alpha(),
+                              [ALPHA_MIN, 0.5],
+                              decimal=12)
+
+    # Test correct dimensions
+    alpha = np.array([1., 2., 3.])
+    m_nb = MultinomialNB(alpha=alpha)
+    expected_msg = ('alpha should be a scalar or a numpy array '
+                    'with shape [n_features]')
+    assert_raise_message(ValueError, expected_msg, m_nb.fit, X, y)