added examples for concurrency

Author: PythonCHB

examples/async/async_executor.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+"""
+An example of runing a blocking task in an Executor:
+"""
+
+import asyncio
+import time
+import datetime
+import random
+
+
+async def small_task(num):
+    """
+    Just something to give us little tasks that run at random intervals
+    These will go on forever
+    """
+    while True:  # keep doing this until break
+        print("task: {} run".format(num))
+        # pause for a random amount of time between 0 and 2 seconds
+        await asyncio.sleep(random.random() * 2)
+
+async def slow_task():
+    while True:  # keep going forever
+        print("running the slow task- blocking!")
+        # This will block for 2-10 seconds!
+        # result = slow_function(random.random() * 8 + 2)
+        # uncomment to put it on a different thread:
+        result = await loop.run_in_executor(None,
+                                            slow_function,
+                                            random.random() * 8 + 2)
+        print("slow function done: result", result)
+        #await asyncio.sleep(0.0)  # to release the loop
+
+
+def slow_function(duration):
+    """
+    this is a fake function that takes a long time, and blocks
+    """
+    time.sleep(duration)
+    print("slow task complete")
+    return duration
+
+
+# get a loop going:
+loop = asyncio.get_event_loop()
+
+# or add tasks to the loop like this:
+loop.create_task(small_task(1))
+loop.create_task(small_task(2))
+loop.create_task(small_task(3))
+loop.create_task(small_task(4))
+
+# Add the slow one
+loop.create_task(slow_task())
+
+print("about to run loop")
+# this is a blocking call
+# we will need to hit ^C to stop it...
+loop.run_forever()
+print("loop exited")

examples/async/async_timer.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+"""
+Simple async example derived from python docs.
+
+Will only work on Python 3.5 and above
+"""
+
+import asyncio
+import time
+import datetime
+import random
+
+
+# using "async" makes this a coroutine:
+# its code can be run by the event loop
+async def display_date(num):
+    end_time = time.time() + 10.0  # we want it to run for 10 seconds.
+    while True:  # keep doing this until break
+        print("instance: {} Time: {}".format(num, datetime.datetime.now()))
+        if (time.time()) >= end_time:
+            print("instance: {} is all done".format(num))
+            break
+        # pause for a random amount of time
+        await asyncio.sleep(random.randint(0, 3))
+
+
+def shutdown():
+    print("shutdown called")
+    # you can access the event loop this way:
+    loop = asyncio.get_event_loop()
+    loop.stop()
+
+
+# You register "futures" on the loop this way:
+asyncio.ensure_future(display_date(1))
+asyncio.ensure_future(display_date(2))
+
+loop = asyncio.get_event_loop()
+
+# or add tasks to the loop like this:
+loop.create_task(display_date(3))
+loop.create_task(display_date(4))
+
+# this will shut the event loop down in 15 seconds
+loop.call_later(15, shutdown)
+
+print("about to run loop")
+# this is a blocking call
+loop.run_forever()
+print("loop exited")
+

examples/async/client_test.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""
+simplest possible example of using aiohttp client
+
+from docs
+"""
+
+import asyncio
+import aiohttp
+
+async def get_events():
+    async with aiohttp.ClientSession() as session:
+        print("created a session")
+        async with session.get('https://api.github.com/events',
+                               ) as resp:
+            print("status: resp.status")
+            print(await resp.json())
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(get_events())
+loop.close()
+
+
+
+

examples/async/gather.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+"""
+test of gather()
+
+adapted from:
+
+https://docs.python.org/3/library/asyncio-task.html
+
+"""
+
+import asyncio
+
+async def factorial(name, number):
+    f = 1
+    for i in range(2, number+1):
+        print("Task %s: Compute factorial(%s)..." % (name, i))
+        await asyncio.sleep(1)
+        f *= i
+    print("Task %s: factorial(%s) = %s" % (name, number, f))
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(asyncio.gather(
+    factorial("A", 2),
+    factorial("B", 3),
+    factorial("C", 4),
+))
+loop.close()

examples/async/get_news.py

@@ -0,0 +1,5 @@
+
+
+NEWS_API_KEY = 1fabc23bb9bc485ca59b3966cbd6ea26
+
+url = https://newsapi.org/

examples/async/get_news_async.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+"""
+An Asynchronous version of the script to see how much a given word is
+mentioned in the news today
+
+Takes advantage of:
+
+Uses data from the NewsAPI:
+
+https://newsapi.org
+"""
+
+import time
+import asyncio
+import aiohttp
+import requests
+
+WORD = "trump"
+
+NEWS_API_KEY = "84d0483394c44f288965d7b366e54a74"
+
+base_url = 'https://newsapi.org/v1/'
+
+# use one session for the whole script
+#  recommended by the docs
+# session = aiohttp.ClientSession()
+
+
+# this has to run first, so doesn't really need async
+# but why use two reuests libraries ?
+async def get_sources(sources):
+    """
+    get all the english language sources of news
+
+    'https://newsapi.org/v1/sources?language=en'
+    """
+    url = base_url + "sources"
+    params = {"language": "en"}
+    session = aiohttp.ClientSession()
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, ssl=False, params=params) as resp:
+            data = await resp.json()
+            print("Got the sources")
+    sources.extend([src['id'].strip() for src in data['sources']])
+
+
+async def get_articles(source):
+    """
+    https://newsapi.org/v1/articles?source=associated-press&sortBy=top&apiKey=1fabc23bb9bc485ca59b3966cbd6ea26
+    """
+    url = base_url + "articles"
+    params = {"source": source,
+              "apiKey": NEWS_API_KEY,
+              # "sortBy": "latest", # some sources don't support latest
+              "sortBy": "top",
+              # "sortBy": "popular",
+              }
+    print("requesting:", source)
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, ssl=False, params=params) as resp:
+            if resp.status != 200:  # aiohttpp has "status"
+                print("something went wrong with: {}".format(source))
+                await asyncio.sleep(0)
+                return
+            data = await resp.json()
+            print("got the articles from {}".format(source))
+    # the url to the article itself is in data['articles'][i]['url']
+    titles.extend([str(art['title']) + str(art['description'])
+                   for art in data['articles']])
+
+
+def count_word(word, titles):
+    word = word.lower()
+    count = 0
+    for title in titles:
+        if word in title.lower():
+            count += 1
+    return count
+
+
+start = time.time()
+
+# start up a loop:
+loop = asyncio.get_event_loop()
+
+# create the objects to hold the data
+sources = []
+titles = []
+
+# get the sources -- this is essentially synchronous
+loop.run_until_complete(get_sources(sources))
+
+# running the loop for the articles
+jobs = asyncio.gather(*(get_articles(source) for source in sources))
+loop.run_until_complete(jobs)
+loop.close()
+# session.close()
+
+art_count = len(titles)
+word_count = count_word(WORD, titles)
+
+print(WORD, "found {} times in {} articles".format(word_count, art_count))
+print("Process took {:.0f} seconds".format(time.time() - start))

examples/async/get_news_sync.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+"""
+Regular synchronous script to see how much a given word is mentioned in the
+news today
+
+Took about 21 seconds for me.
+
+Uses data from the NewsAPI:
+
+https://newsapi.org
+
+NOTE: you need to register with the web site to get a KEY.
+"""
+import time
+import requests
+
+WORD = "trump"
+
+NEWS_API_KEY = "84d0483394c44f288965d7b366e54a74"
+
+base_url = 'https://newsapi.org/v1/'
+
+
+def get_sources():
+    """
+    get all the english language sources of news
+
+    'https://newsapi.org/v1/sources?language=en'
+    """
+    url = base_url + "sources"
+    params = {"language": "en"}
+    resp = requests.get(url, params=params)
+    data = resp.json()
+    sources = [src['id'].strip() for src in data['sources']]
+    return sources
+
+
+def get_articles(source):
+    """
+    https://newsapi.org/v1/articles?source=associated-press&sortBy=top&apiKey=1fabc23bb9bc485ca59b3966cbd6ea26
+    """
+    url = base_url + "articles"
+    params = {"source": source,
+              "apiKey": NEWS_API_KEY,
+              # "sortBy": "latest", # some sources don't support latest
+              "sortBy": "top",
+              # "sortBy": "popular",
+              }
+    print("requesting:", source)
+    resp = requests.get(url, params=params)
+    if resp.status_code != 200:  # aiohttpp has "status"
+        print("something went wrong with {}".format(source))
+        print(resp)
+        print(resp.text)
+        return []
+    data = resp.json()
+    # the url to the article itself is in data['articles'][i]['url']
+    titles = [str(art['title']) + str(art['description'])
+              for art in data['articles']]
+    return titles
+
+
+def count_word(word, titles):
+    word = word.lower()
+    count = 0
+    for title in titles:
+        if word in title.lower():
+            count += 1
+    return count
+
+
+start = time.time()
+sources = get_sources()
+
+art_count = 0
+word_count = 0
+for source in sources:
+    titles = get_articles(source)
+    art_count += len(titles)
+    word_count += count_word('trump', titles)
+
+print(WORD, "found {} times in {} articles".format(word_count, art_count))
+print("Process took {:.0f} seconds".format(time.time() - start))