General cursor (generator) like object for Twython functions

* If the endpoint has an iter_mode, the function is able to be used in
Twython.cursor

* Deprecate Twython.search_gen

Fixes ryanmcgrath#238

Author: michaelhelmick

HISTORY.rst

@@ -10,6 +10,8 @@ History
 - Pass ``client_args`` to the streaming ``__init__``, much like in core Twython (you can pass headers, timeout, hooks, proxies, etc.).
 - Streamer has new parameter ``handlers`` which accepts a list of strings related to functions that are apart of the Streaming class and start with "on\_". i.e. ['delete'] is passed, when 'delete' is received from a stream response; ``on_delete`` will be called.
 - When an actual request error happens and a ``RequestException`` is raised, it is caught and a ``TwythonError`` is raised instead for convenience.
+- Added "cursor"-like functionality. Endpoints with the attribute ``iter_mode`` will be able to be passed to ``Twython.cursor`` and returned as a generator.
+- ``Twython.search_gen`` has been deprecated. Please use ``twitter.cursor(twitter.search, q='your_query')`` instead, where ``twitter`` is your ``Twython`` instance.
 
 3.0.0 (2013-06-18)
 ++++++++++++++++++

twython/api.py

@@ -14,11 +14,16 @@
 from requests_oauthlib import OAuth1, OAuth2
 
 from . import __version__
+from .advisory import TwythonDeprecationWarning
 from .compat import json, urlencode, parse_qsl, quote_plus, str, is_py2
 from .endpoints import EndpointsMixin
 from .exceptions import TwythonError, TwythonAuthError, TwythonRateLimitError
 from .helpers import _transparent_params
 
+import warnings
+
+warnings.simplefilter('always', TwythonDeprecationWarning)  # For Python 2.7 >
+
 
 class Twython(EndpointsMixin, object):
     def __init__(self, app_key=None, app_secret=None, oauth_token=None,
@@ -376,22 +381,65 @@ def search_gen(self, search_query, **params):
           >>>   print result
 
         """
-        content = self.search(q=search_query, **params)
+        warnings.warn(
+            'This method is deprecated. You should use Twython.cursor instead. [eg. Twython.cursor(Twython.search, q=\'your_query\')]',
+            TwythonDeprecationWarning,
+            stacklevel=2
+        )
+        return self.cursor(self.search, q=search_query, **params)
+
+    def cursor(self, function, **params):
+        """Returns a generator for results that match a specified query.
+
+        :param function: Instance of a Twython function (Twython.get_home_timeline, Twython.search)
+        :param \*\*params: Extra parameters to send with your request (usually parameters excepted by the Twitter API endpoint)
+        :rtype: generator
+
+        Usage::
 
-        if not content.get('statuses'):
+          >>> from twython import Twython
+          >>> twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
+
+          >>> results = twitter.cursor(twitter.search, q='python')
+          >>> for result in results:
+          >>>   print result
+
+        """
+        if not hasattr(function, 'iter_mode'):
+            raise TwythonError('Unable to create generator for Twython method "%s"' % function.__name__)
+
+        content = function(**params)
+
+        if not content:
             raise StopIteration
 
-        for tweet in content['statuses']:
-            yield tweet
+        if function.iter_mode == 'cursor' and content['next_cursor_str'] == '0':
+            raise StopIteration
+
+        if hasattr(function, 'iter_key'):
+            results = content.get(function.iter_key)
+        else:
+            results = content
+
+        for result in results:
+            yield result
 
         try:
-            if not 'since_id' in params:
-                params['since_id'] = (int(content['statuses'][0]['id_str']) + 1)
+            if function.iter_mode == 'id':
+                if not 'max_id' in params:
+                    # Add 1 to the id because since_id and max_id are inclusive
+                    if hasattr(function, 'iter_metadata'):
+                        since_id = content[function.iter_metadata].get('since_id_str')
+                    else:
+                        since_id = content[0]['id_str']
+                    params['since_id'] = (int(since_id) - 1)
+            elif function.iter_mode == 'cursor':
+                params['cursor'] = content['next_cursor_str']
         except (TypeError, ValueError):  # pragma: no cover
             raise TwythonError('Unable to generate next page of search results, `page` is not a number.')
 
-        for tweet in self.search_gen(search_query, **params):
-            yield tweet
+        for result in self.cursor(function, **params):
+            yield result
 
     @staticmethod
     def unicode2utf8(text):

twython/endpoints.py

@@ -24,6 +24,7 @@ def get_mentions_timeline(self, **params):
 
         """
         return self.get('statuses/mentions_timeline', params=params)
+    get_mentions_timeline.iter_mode = 'id'
 
     def get_user_timeline(self, **params):
         """Returns a collection of the most recent Tweets posted by the user
@@ -33,6 +34,7 @@ def get_user_timeline(self, **params):
 
         """
         return self.get('statuses/user_timeline', params=params)
+    get_user_timeline.iter_mode = 'id'
 
     def get_home_timeline(self, **params):
         """Returns a collection of the most recent Tweets and retweets
@@ -42,6 +44,7 @@ def get_home_timeline(self, **params):
 
         """
         return self.get('statuses/home_timeline', params=params)
+    get_home_timeline.iter_mode = 'id'
 
     def retweeted_of_me(self, **params):
         """Returns the most recent tweets authored by the authenticating user
@@ -51,6 +54,7 @@ def retweeted_of_me(self, **params):
 
         """
         return self.get('statuses/retweets_of_me', params=params)
+    retweeted_of_me.iter_mode = 'id'
 
     # Tweets
     def get_retweets(self, **params):
@@ -128,6 +132,9 @@ def search(self, **params):
 
         """
         return self.get('search/tweets', params=params)
+    search.iter_mode = 'id'
+    search.iter_key = 'statuses'
+    search.iter_metadata = 'search_metadata'
 
     # Direct Messages
     def get_direct_messages(self, **params):
@@ -268,7 +275,9 @@ def get_friends_list(self, **params):
         Docs: https://dev.twitter.com/docs/api/1.1/get/friends/list
 
         """
+        print 'here 1'
         return self.get('friends/list', params=params)
+    get_friends_list.iterator_mode = 'cursor'
 
     def get_followers_list(self, **params):
         """Returns a cursored collection of user objects for users