Merge pull request scikit-learn#5635 from ogrisel/ogrisel-joblib-0.9.3

[MRG] Update joblib to 0.9.3

Author: ogrisel

doc/whats_new.rst

@@ -227,20 +227,15 @@ Enhancements
    - RCV1 dataset loader (:func:`sklearn.datasets.fetch_rcv1`).
      By `Tom Dupre la Tour`_.
 
-   - Upgraded to joblib 0.9.2 to benefit from the new automatic batching of
+   - Upgraded to joblib 0.9.3 to benefit from the new automatic batching of
      short tasks. This makes it possible for scikit-learn to benefit from
      parallelism when many very short tasks are executed in parallel, for
      instance by the :class:`grid_search.GridSearchCV` meta-estimator
      with ``n_jobs > 1`` used with a large grid of parameters on a small
      dataset. By `Vlad Niculae`_, `Olivier Grisel`_ and `Loic Esteve`_.
 
-   - Joblib 0.9.2 also enables the ``forkserver`` start method for
-     multiprocessing by default under non-Windows platforms for Python 3.4
-     and later in order to avoid possible crash with some version of BLAS such
-     as vecLib / Accelerate under OSX for instance. By `Olivier Grisel`_.
-
-   - For more details about changes in joblib 0.9.2 see the release notes:
-     https://github.com/joblib/joblib/blob/master/CHANGES.rst#release-092
+   - For more details about changes in joblib 0.9.3 see the release notes:
+     https://github.com/joblib/joblib/blob/master/CHANGES.rst#release-093
 
    - Improved speed (3 times per iteration) of
      :class:`decomposition.DictLearning` with coordinate descent method

sklearn/externals/joblib/__init__.py

@@ -115,7 +115,7 @@
 # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
 # 'X.Y.dev0' is the canonical version of 'X.Y.dev'
 #
-__version__ = '0.9.2'
+__version__ = '0.9.3'
 
 
 from .memory import Memory, MemorizedResult

sklearn/externals/joblib/parallel.py

@@ -49,14 +49,13 @@
 # on a single worker while other workers have no work to process any more.
 MAX_IDEAL_BATCH_DURATION = 2
 
-# Under Python 3.4+ use the 'forkserver' start method by default: this makes it
-# possible to avoid crashing 3rd party libraries that manage an internal thread
-# pool that does not tolerate forking
-if hasattr(mp, 'get_start_method'):
-    method = os.environ.get('JOBLIB_START_METHOD')
-    if (method is None and mp.get_start_method() == 'fork'
-            and 'forkserver' in mp.get_all_start_methods()):
-        method = 'forkserver'
+# Under Linux or OS X the default start method of multiprocessing
+# can cause third party libraries to crash. Under Python 3.4+ it is possible
+# to set an environment variable to switch the default start method from
+# 'fork' to 'forkserver' or 'spawn' to avoid this issue albeit at the cost
+# of causing semantic changes and some additional pool instanciation overhead.
+if hasattr(mp, 'get_context'):
+    method = os.environ.get('JOBLIB_START_METHOD', '').strip() or None
     DEFAULT_MP_CONTEXT = mp.get_context(method=method)
 else:
     DEFAULT_MP_CONTEXT = None

sklearn/utils/testing.py

@@ -711,26 +711,28 @@ def func(*args, **kwargs):
 def if_safe_multiprocessing_with_blas(func):
     """Decorator for tests involving both BLAS calls and multiprocessing
 
-    Under Python < 3.4 and POSIX (e.g. Linux or OSX), using multiprocessing in
-    conjunction with some implementation of BLAS (or other libraries that
-    manage an internal posix thread pool) can cause a crash or a freeze of the
-    Python process.
-
-    Under Python 3.4 and later, joblib uses the forkserver mode of
-    multiprocessing which does not trigger this problem.
+    Under POSIX (e.g. Linux or OSX), using multiprocessing in conjunction with
+    some implementation of BLAS (or other libraries that manage an internal
+    posix thread pool) can cause a crash or a freeze of the Python process.
 
     In practice all known packaged distributions (from Linux distros or
     Anaconda) of BLAS under Linux seems to be safe. So we this problem seems to
     only impact OSX users.
 
     This wrapper makes it possible to skip tests that can possibly cause
-    this crash under OSX with.
+    this crash under OS X with.
+
+    Under Python 3.4+ it is possible to use the `forkserver` start method
+    for multiprocessing to avoid this issue. However it can cause pickling
+    errors on interactively defined functions. It therefore not enabled by
+    default.
+
     """
     @wraps(func)
     def run_test(*args, **kwargs):
-        if sys.platform == 'darwin' and sys.version_info[:2] < (3, 4):
+        if sys.platform == 'darwin':
             raise SkipTest(
-                "Possible multi-process bug with some BLAS under Python < 3.4")
+                "Possible multi-process bug with some BLAS")
         return func(*args, **kwargs)
     return run_test