Pushing the docs to dev/ for branch: master, commit 0b6308c2708fe03071cdbf24997eb967403f5965

Author: sklearn-ci

dev/_downloads/auto_examples_jupyter.zip

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Column Transformer with Heterogeneous Data Sources\n\n\nDatasets can often contain components of that require different feature\nextraction and processing pipelines.  This scenario might occur when:\n\n1. Your dataset consists of heterogeneous data types (e.g. raster images and\n   text captions)\n2. Your dataset is stored in a Pandas DataFrame and different columns\n   require different processing pipelines.\n\nThis example demonstrates how to use\n:class:`sklearn.compose.ColumnTransformer` on a dataset containing\ndifferent types of features.  We use the 20-newsgroups dataset and compute\nstandard bag-of-words features for the subject line and body in separate\npipelines as well as ad hoc features on the body. We combine them (with\nweights) using a ColumnTransformer and finally train a classifier on the\ncombined set of features.\n\nThe choice of features is not particularly helpful, but serves to illustrate\nthe technique.\n\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "# Author: Matt Terry <[email protected]>\n#\n# License: BSD 3 clause\nfrom __future__ import print_function\n\nimport numpy as np\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_footer\nfrom sklearn.datasets.twenty_newsgroups import strip_newsgroup_quoting\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.svm import SVC\n\n\nclass TextStats(BaseEstimator, TransformerMixin):\n    \"\"\"Extract features from each document for DictVectorizer\"\"\"\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        return [{'length': len(text),\n                 'num_sentences': text.count('.')}\n                for text in posts]\n\n\nclass SubjectBodyExtractor(BaseEstimator, TransformerMixin):\n    \"\"\"Extract the subject & body from a usenet post in a single pass.\n\n    Takes a sequence of strings and produces a dict of sequences.  Keys are\n    `subject` and `body`.\n    \"\"\"\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, posts):\n        # construct object dtype array with two columns\n        # first column = 'subject' and second column = 'body'\n        features = np.empty(shape=(len(posts), 2), dtype=object)\n        for i, text in enumerate(posts):\n            headers, _, bod = text.partition('\\n\\n')\n            bod = strip_newsgroup_footer(bod)\n            bod = strip_newsgroup_quoting(bod)\n            features[i, 1] = bod\n\n            prefix = 'Subject:'\n            sub = ''\n            for line in headers.split('\\n'):\n                if line.startswith(prefix):\n                    sub = line[len(prefix):]\n                    break\n            features[i, 0] = sub\n\n        return features\n\n\npipeline = Pipeline([\n    # Extract the subject & body\n    ('subjectbody', SubjectBodyExtractor()),\n\n    # Use C toolumnTransformer to combine the features from subject and body\n    ('union', ColumnTransformer(\n        [\n            # Pulling features from the post's subject line (first column)\n            ('subject', TfidfVectorizer(min_df=50), 0),\n\n            # Pipeline for standard bag-of-words model for body (second column)\n            ('body_bow', Pipeline([\n                ('tfidf', TfidfVectorizer()),\n                ('best', TruncatedSVD(n_components=50)),\n            ]), 1),\n\n            # Pipeline for pulling ad hoc features from post's body\n            ('body_stats', Pipeline([\n                ('stats', TextStats()),  # returns a list of dicts\n                ('vect', DictVectorizer()),  # list of dicts -> feature matrix\n            ]), 1),\n        ],\n\n        # weight components in ColumnTransformer\n        transformer_weights={\n            'subject': 0.8,\n            'body_bow': 0.5,\n            'body_stats': 1.0,\n        }\n    )),\n\n    # Use a SVC classifier on the combined features\n    ('svc', SVC(kernel='linear')),\n])\n\n# limit the list of categories to make running this example faster.\ncategories = ['alt.atheism', 'talk.religion.misc']\ntrain = fetch_20newsgroups(random_state=1,\n                           subset='train',\n                           categories=categories,\n                           )\ntest = fetch_20newsgroups(random_state=1,\n                          subset='test',\n                          categories=categories,\n                          )\n\npipeline.fit(train.data, train.target)\ny = pipeline.predict(test.data)\nprint(classification_report(y, test.target))"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.6.5"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

dev/_downloads/auto_examples_python.zip

@@ -1,7 +1,7 @@
 """
-=============================================
-Feature Union with Heterogeneous Data Sources
-=============================================
+==================================================
+Column Transformer with Heterogeneous Data Sources
+==================================================
 
 Datasets can often contain components of that require different feature
 extraction and processing pipelines.  This scenario might occur when:
@@ -12,12 +12,12 @@
    require different processing pipelines.
 
 This example demonstrates how to use
-:class:`sklearn.feature_extraction.FeatureUnion` on a dataset containing
+:class:`sklearn.compose.ColumnTransformer` on a dataset containing
 different types of features.  We use the 20-newsgroups dataset and compute
 standard bag-of-words features for the subject line and body in separate
 pipelines as well as ad hoc features on the body. We combine them (with
-weights) using a FeatureUnion and finally train a classifier on the combined
-set of features.
+weights) using a ColumnTransformer and finally train a classifier on the
+combined set of features.
 
 The choice of features is not particularly helpful, but serves to illustrate
 the technique.
@@ -38,50 +38,11 @@
 from sklearn.feature_extraction import DictVectorizer
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics import classification_report
-from sklearn.pipeline import FeatureUnion
 from sklearn.pipeline import Pipeline
+from sklearn.compose import ColumnTransformer
 from sklearn.svm import SVC
 
 
-class ItemSelector(BaseEstimator, TransformerMixin):
-    """For data grouped by feature, select subset of data at a provided key.
-
-    The data is expected to be stored in a 2D data structure, where the first
-    index is over features and the second is over samples.  i.e.
-
-    >> len(data[key]) == n_samples
-
-    Please note that this is the opposite convention to scikit-learn feature
-    matrixes (where the first index corresponds to sample).
-
-    ItemSelector only requires that the collection implement getitem
-    (data[key]).  Examples include: a dict of lists, 2D numpy array, Pandas
-    DataFrame, numpy record array, etc.
-
-    >> data = {'a': [1, 5, 2, 5, 2, 8],
-               'b': [9, 4, 1, 4, 1, 3]}
-    >> ds = ItemSelector(key='a')
-    >> data['a'] == ds.transform(data)
-
-    ItemSelector is not designed to handle data grouped by sample.  (e.g. a
-    list of dicts).  If your data is structured this way, consider a
-    transformer along the lines of `sklearn.feature_extraction.DictVectorizer`.
-
-    Parameters
-    ----------
-    key : hashable, required
-        The key corresponding to the desired value in a mappable.
-    """
-    def __init__(self, key):
-        self.key = key
-
-    def fit(self, x, y=None):
-        return self
-
-    def transform(self, data_dict):
-        return data_dict[self.key]
-
-
 class TextStats(BaseEstimator, TransformerMixin):
     """Extract features from each document for DictVectorizer"""
 
@@ -104,21 +65,22 @@ def fit(self, x, y=None):
         return self
 
     def transform(self, posts):
-        features = np.recarray(shape=(len(posts),),
-                               dtype=[('subject', object), ('body', object)])
+        # construct object dtype array with two columns
+        # first column = 'subject' and second column = 'body'
+        features = np.empty(shape=(len(posts), 2), dtype=object)
         for i, text in enumerate(posts):
             headers, _, bod = text.partition('\n\n')
             bod = strip_newsgroup_footer(bod)
             bod = strip_newsgroup_quoting(bod)
-            features['body'][i] = bod
+            features[i, 1] = bod
 
             prefix = 'Subject:'
             sub = ''
             for line in headers.split('\n'):
                 if line.startswith(prefix):
                     sub = line[len(prefix):]
                     break
-            features['subject'][i] = sub
+            features[i, 0] = sub
 
         return features
 
@@ -127,38 +89,31 @@ def transform(self, posts):
     # Extract the subject & body
     ('subjectbody', SubjectBodyExtractor()),
 
-    # Use FeatureUnion to combine the features from subject and body
-    ('union', FeatureUnion(
-        transformer_list=[
+    # Use C toolumnTransformer to combine the features from subject and body
+    ('union', ColumnTransformer(
+        [
+            # Pulling features from the post's subject line (first column)
+            ('subject', TfidfVectorizer(min_df=50), 0),
 
-            # Pipeline for pulling features from the post's subject line
-            ('subject', Pipeline([
-                ('selector', ItemSelector(key='subject')),
-                ('tfidf', TfidfVectorizer(min_df=50)),
-            ])),
-
-            # Pipeline for standard bag-of-words model for body
+            # Pipeline for standard bag-of-words model for body (second column)
             ('body_bow', Pipeline([
-                ('selector', ItemSelector(key='body')),
                 ('tfidf', TfidfVectorizer()),
                 ('best', TruncatedSVD(n_components=50)),
-            ])),
+            ]), 1),
 
             # Pipeline for pulling ad hoc features from post's body
             ('body_stats', Pipeline([
-                ('selector', ItemSelector(key='body')),
                 ('stats', TextStats()),  # returns a list of dicts
                 ('vect', DictVectorizer()),  # list of dicts -> feature matrix
-            ])),
-
+            ]), 1),
         ],
 
-        # weight components in FeatureUnion
+        # weight components in ColumnTransformer
         transformer_weights={
             'subject': 0.8,
             'body_bow': 0.5,
             'body_stats': 1.0,
-        },
+        }
     )),
 
     # Use a SVC classifier on the combined features