added the function to scrape the most recent pastebins and created the first step. basic scraping. still missing the duplicate check

Author: ihgalis

README.md

@@ -0,0 +1,54 @@
+# Pastebin Scraper
+
+The scraper here is based on the one from [six519](https://github.com/six519/PastebinPython). Thank you very much for giving us this present :-)
+
+## Installation
+
+Simply clone it:
+
+```
+cd /your/desired/directory
+clone https://github.com/six519/PastebinPython.git
+pip install -r requirements.txt
+```
+
+## Usage
+
+This fork of the PastebinPython project downloads all pastebin entries ... or well at least it starts downloading as much as it can.
+
+The results will be saved within a MongoDB collection. Another script will be triggered to identify keywords, which have to be provided in front.
+
+The second step results in new collections where each keyword gets one new collection and all found pastebin entries will be copied there.
+
+The third step might be the accumulator. It identifies specific words (similiar to step 2) but also specials like eMail addresses, Bitcoin addresses, URLs, IP Adresses etc. 
+
+### 1. pastebin_scrape.py
+
+For this one to work good you need an API key. I bought a lifetime access to the pastebin API a while ago for 29,99 USD. It doesn't make you poor.
+
+You will also need to update your Scraping IP, in order to make it work: [Change Scraping IP](https://pastebin.com/doc_scraping_api)
+
+        python pastebin_scrape.py -v 1 -db 1 -api <YOUR_PASTE_BIN_API_KEY>
+
+### 2. pastebin_analyze.py
+
+### 3. pastebin_accumulate.py
+
+### Access Data via Flask API
+
+Finally you can either write yourself a clean data retriever or you can use this Flask API implementation here:
+
+```
+# start it in debug and verbose mode first!
+python pastebin_api.py -d -v
+```
+
+Well there is only one API method. Grab yourself a browser or use curl:
+
+```
+http://localhost:5000/api/getpastebins/<keyword>
+```
+
+The result should be a nice JSON document collection. Maybe too large to handle for a browser. Anyway this is just intended for demonstration reasons. 
+
+If you want to use that data somehow, you might find the JSON format handy and start to parse it for your own purpose.

pastebin_python/__init__.py

@@ -3,7 +3,7 @@
 .. moduleauthor:: Ferdinand Silva <[email protected]>
 
 """
-from pastebin import PastebinPython
+from pastebin_python.pastebin import PastebinPython
 
 __version__ = "1.2"
 __app_name__ = "pastebin_python"

pastebin_python/pastebin.py

@@ -8,9 +8,9 @@
 import re
 import requests
 from xml.dom.minidom import parseString
-from pastebin_options import OPTION_PASTE, OPTION_LIST, OPTION_TRENDS, OPTION_DELETE, OPTION_USER_DETAILS
-from pastebin_constants import PASTEBIN_API_POST_URL, PASTEBIN_API_LOGIN_URL, PASTEBIN_RAW_URL
-from pastebin_exceptions import PastebinBadRequestException, PastebinNoPastesException, PastebinFileException, PastebinHTTPErrorException
+from pastebin_python.pastebin_options import OPTION_PASTE, OPTION_LIST, OPTION_TRENDS, OPTION_DELETE, OPTION_USER_DETAILS
+from pastebin_python.pastebin_constants import PASTEBIN_API_POST_URL, PASTEBIN_API_LOGIN_URL, PASTEBIN_RAW_URL, PASTEBIN_URL_SCRAPE
+from pastebin_python.pastebin_exceptions import PastebinBadRequestException, PastebinNoPastesException, PastebinFileException, PastebinHTTPErrorException
 
 
 class PastebinPython(object):
@@ -150,9 +150,9 @@ def __processRequest(self, method, url, data):
             req = self.api_session.request(method, url, data=data)
 
         response = req.content
-        if re.search('^Bad API request', response):
+        if re.search('^Bad API request', response.decode('utf-8')):
             raise PastebinBadRequestException(response)
-        elif re.search('^No pastes found', response):
+        elif re.search('^No pastes found', response.decode('utf-8')):
             raise PastebinNoPastesException
 
         return response
@@ -394,3 +394,21 @@ def getPasteRawOutput(self, api_paste_key):
             retMsg = str(e)
 
         return retMsg.decode('utf-8')
+
+    def scrapeMostRecent(self):
+        """
+        Returns the most recent Pastebin posts. You will need to have an API Key and a whitelisted IP
+        configured on pastebin.com (https://pastebin.com/api_scraping_faq)
+
+        :return: str
+        """
+        try:
+            print("Scraping ... on: " + PASTEBIN_URL_SCRAPE + "/api_scraping.php")
+            data = self.__processRequest('GET',
+                                         PASTEBIN_URL_SCRAPE + "/api_scraping.php",
+                                         None)
+            return data
+        except PastebinBadRequestException as e:
+            retMsg = str(e)
+            print("PastebinBadRequest")
+            return None

pastebin_python/pastebin_constants.py

@@ -5,7 +5,8 @@
 .. moduleauthor:: Ferdinand Silva <[email protected]>
 
 """
-PASTEBIN_URL = "http://pastebin.com/" #: The pastebin.com base url
+PASTEBIN_URL_SCRAPE = "https://scrape.pastebin.com"
+PASTEBIN_URL = "https://pastebin.com/" #: The pastebin.com base url
 PASTEBIN_RAW_URL = "%s%s" % (PASTEBIN_URL, "raw.php?i=%s")
 PASTEBIN_API_URL = "%s%s" % (PASTEBIN_URL, "api/") #: The pastebin.com API base URL
 PASTEBIN_API_POST_URL = "%s%s" % (PASTEBIN_API_URL, "api_post.php") #: The pastebin.com API POST URL

pastebin_scrape.py

@@ -0,0 +1,137 @@
+from pastebin_python.pastebin import PastebinPython
+from pastebin_python.pastebin_exceptions import PastebinBadRequestException
+from pymongo import MongoClient
+from copy import deepcopy
+
+import json
+import urllib.request
+import argparse
+import logging
+import time
+
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s %(levelname)s %(message)s',
+                    datefmt='%a, %d %b %Y %H:%M:%S')
+
+logger = logging.getLogger(__name__)
+
+
+def call_scrape_url(url):
+    """
+    Method is doing all the URL calling stuff.
+
+    :param url: The URL which should be called
+    :return: The content of the previously requested pastebin
+    """
+
+    try:
+        request = urllib.request.Request(url)
+        result = urllib.request.urlopen(request)
+        result_text = result.read()
+        text_encoded = result_text.decode(encoding='utf-8', errors='ignore')
+
+        return text_encoded
+
+    except json.decoder.JSONDecodeError as json_e:
+        logger.error("JSON Decoding Error ... Jumping to next element.")
+        return None
+
+
+def main(args):
+    """
+    Regular main method starts the entire process and interprets the
+    arguments.
+
+    :param args: arguments from argparse
+    :return: None
+    """
+
+    if args['v'] == 0:
+        logger.propagate = False
+    elif args['v'] == 1:
+        logger.propagate = True
+
+    logger.info("Start Pastebin Analyzer")
+
+    api_key = args['api']
+    pbin = PastebinPython(api_dev_key=api_key)
+
+    client = MongoClient(str(args['mongodbhost']), int(args['mongodbport']))
+    db = client.scrape
+    logger.info("MongoDB Connection created")
+
+    while True:
+        try:
+            data = pbin.scrapeMostRecent()
+
+            if data:
+                json_data = data.decode('utf8')  # .replace("'", '"')
+                final_data = json.loads(json_data)
+
+                # Iterate through list
+                for x in final_data:
+
+                    # Pre-create the content key-value pair
+                    x['content'] = 0
+
+                    copy_of_x = deepcopy(x)
+                    for key, value in copy_of_x.items():
+
+                        if key == "scrape_url":
+
+                            # value = scrape_url
+                            text_encoded = call_scrape_url(value)
+                            time.sleep(1)
+
+                            logger.info("Downloading content of " + value)
+
+                            # Add content
+                            x['content'] = text_encoded
+
+                            ## TODO: Add some identity check
+
+                            # DB Save mode args['db'] == 2
+                            if args['db'] == "1":
+                                db.pastebins.insert_one(x)
+            else:
+                logger.debug("No data arrived.")
+
+        except PastebinBadRequestException as e:
+            logger.debug("Pastebin Bad Request - You're doing it wrong")
+
+        except json.decoder.JSONDecodeError as e:
+            logger.debug("JSON Decoding Error ... 'You can't always get what you want!'")
+            continue
+        else:
+            logger.debug("No exception")
+        finally:
+            logger.info("End of Session!")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pastebin Analyzer")
+
+    parser.add_argument('-db',
+                        help="If this is set entries are being added into the DB.\n"
+                             "0 = no DB entries"
+                             "1 = all Pastebin entries are written to DB",
+                        default=0)
+
+    parser.add_argument('-v',
+                        help="Verbose mode.",
+                        default=0)
+
+    parser.add_argument('-api',
+                        help="Pastebin API Key for Scraping.",
+                        required=True)
+
+    parser.add_argument('-mongodbhost',
+                        help="A string with the URL to your MongoDB Server.")
+
+    parser.add_argument('-mongodbport',
+                        help="THe port to which your MongoDB listens.",
+                        default=27017)
+
+    args = vars(parser.parse_args())
+
+    main(args)