高级爬虫——实现异步，线程池，协程

最新推荐文章于 2026-04-16 21:26:41 发布

原创最新推荐文章于 2026-04-16 21:26:41 发布 · 926 阅读

9 ·

本内容遵循CC 4.0 BY-SA版权协议

GEO检测

本文详细介绍了使用Python的aiohttp库实现异步爬虫，以及Flask构建的简单服务，展示了如何利用协程和线程池进行并发处理，包括同步爬虫的对比和实际案例分析。

1.aiohttp实现多任务异步爬虫
2.flask服务
3.多任务协程
4.多任务异步爬虫
5.示例
6.同步爬虫
7.线程池基本使用
8.线程池在爬虫案例中的应用
9.协程

1.aiohttp实现多任务异步爬虫

import requests  
import asyncio  
import time  
import aiohttp  
  
start = time.time()  
urls = [  
    'http://127.0.0.1:5000/bobo','http://127.0.0.1:5000/jay','http://127.0.0.1:5000/tom'  
]  
  
async def get_page(url):  
    #print('正在下载',url)  
    #response = requests.get(url)  
    #print('下载完毕',response.text)  
    async with aiohttp.ClientSession() as session:  
        async with await session.get(url) as response:  
            page_text = await response.text()  
            print(page_text)  
  
tasks = []  
  
for url in urls:  
    c = get_page(url)  
    task = asyncio.ensure_future(c)  
    tasks.append(task)  
  
loop = asyncio.get_event_loop()  
loop.run_complete(asyncio.wait(tasks))  
  
end = time.time()  
  
print('总耗时',end - start)

2.flask服务

from flask import Flask  
import time  
  
app = Flask(__name__)  
  
@app.route('/bobo')  
def index_bobo():  
    time.sleep(2)  
    return 'Hello bobo'  
  
@app.route('/jay')  
def index_jay():  
    time.sleep(2)  
    return 'Hello jay'  
  
@app.route('/tom')  
def index_tom():  
    time.sleep(2)  
    return 'Hello tom'  
  
if __name__ == '__main__':  
    app.run(threaded = True)

3.多任务协程

import asyncio  
import time  
  
async def request(url):  
    print('正在下载',url)  
    #time.sleep(2)  
    await asyncio.sleep(2)  
  
    print('下载完成',url)  
  
start = time.time()  
urls = ['www.baidu.com',  
        'www.sogou.com',  
        'www,goubanjia.com'  
]  
          
stasks = []  
for url in urls:  
    c = request(url)  
    task = ensure_future(c)  
    stasks.append(task)  
  
loop = asyncio.get_event_loop()  
loop.run_until_complete(asyncio.wait(stasks))  
  
print(time.time()-start)

4.多任务异步爬虫

import requests  
import asyncio  
import time  
#import aiohttp  
  
start = time.time()  
urls = [  
    'http://127.0.0.1:5000/bobo','http://127.0.0.1:5000/jay','http://127.0.0.1:5000/tom'  
]  
  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36'  
}  
  
async def get_page(url):  
    print('正在下载',url)  
    response = requests.get(url,headers =headers)  
    print('下载完毕',response.text)  
  
tasks = []  
  
for url in urls:  
    c = get_page(url)  
    task = asyncio.ensure_future(c)  
    tasks.append(task)  
  
loop = get_event_loop()  
loop.run_until_complete(asyncio.wait(tasks))  
  
end = time.time()  
  
print('总耗时',end - start)

5.示例

import requests  
  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36'  
}  
  
url = 'https://www.pearvideo.com/videoStatus.jsp?contId=1719770&mrd=0.559512982919081'  
  
response = requests.get(url,headers = headers)  
print(response.text)  
"https://video.pearvideo.com/mp4/short/20210209/1613307944808-15603370-hd.mp4

6.同步爬虫

import requests  
  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36'  
}  
  
urls = [  
    'https://www.cnblogs.com/shaozheng/p/12795953.html',  
    'https://www.cnblogs.com/hanfe1/p/12661505.html',  
    'https://www.cnblogs.com/tiger666/articles/11070427.html']  
  
def get_content(url):  
  
    print('正在爬取:',url)  
    response = requests.get(url,headers = headers)  
    if response.status_code == 200:  
        return response.content  
  
def parse_content(content):  
    print('响应数据的长度为:',len(content))  
  
for url in urls:  
    content = get_content(url)  
    parse_content(content)

7.线程池基本使用

# import time  
  
# def get_page(str):  
#     print('正在下载：',str)  
#     time.sleep(2)  
#     print('下载成功：',str)  
  
# name_list = ['xiaozi','aa','bb','cc']  
  
# start_time = time.time()  
  
# for i in range(len(name_list)):  
#     get_page(name_list[i])  
  
# end_time = time.time()  
  
# print('%d second'%(end_time-start_time))  
  
import time  
from multiprocessing.dummy import Pool  
  
start_time = time.time()  
def get_page(str):  
    print('正在下载：',str)  
    time.sleep(2)  
    print('下载成功：',str)  
  
name_list = ['xiaozi','aa','bb','cc']  
pool = Pool(4)  
pool.map(get_page,name_list)  
  
end_time = time.time()  
  
print(end_time-start_time)

8.线程池在爬虫案例中的应用

import requests  
from lxml import etree  
import re  
from multiprocessing.dummy import Pool  
  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.193 Safari/537.36'  
}  
  
url = 'https://www.pearvideo.com/'  
  
page_text = requests.get(url,headers = headers).text  
  
tree = etree.HTML(page_text)  
li_list = tree.xpath('//div[@class="vervideo-tlist-bd recommend-btbg clearfix"]/ul/li')  
#li_list = tree.xpath('//ul[@class="vervideo-tlist-small"]/li')  
urls = []  
for li in li_list:  
    detail_url = 'https://www.pearvideo.com/' + li.xpath('.@href')[0]  
    #name = li.xpath('./div/a/div[2]/text()')[0] + '.mp4'  
    name = li.xpath('./div/a/div[2]/div[2]/text()')[0] + '.mp4'  
    #print(detail_url,name)  
    detail_page_text = requests.get(detail_url,headers = headers).text  
    # ex = 'srcUrl="(.*?)",vdoUrl'  
    # video_url = re.findall(ex,detail_page_text)[0]  
    #video_url = tree.xpath('//img[@class="img"]/@src')[0]  
    #https://video.pearvideo.com/mp4/short/20210209/{}-15603370-hd.mp4  
    #xhrm码  
    print(detail_page_text)  
  
  
  
  
  
  
    '''  
    dic = {  
        'name':name,  
        'url':video_url  
    }  
    urls.append(dic)  
  
    def get_video_data(dic):  
        url = dic['url']  
        print(dic['name'],'正在下载......')  
        data = requests.get(url,headers = headers).context  
        with open(dic['name','w']) as fp:  
            fp.write(data)  
            print(dic['name'],'下载成功!')  
pool = Pool(4)  
pool.map(get_video_data,urls)  
  
pool.close()  
pool.join()  
'''

9.协程

import asyncio  
  
async def request(url):  
    print('正在请求的url是',url)  
    print('请求成功,',url)  
    return url  
  
c = request('www.baidu.com')  
  
# loop = asyncio.get_event_loop()  
# loop.run_until_complete(c)  
  
  
  
# loop = asyncio.get_event_loop()  
  
# task = loop.create_task(c)  
# print(task)  
  
# loop.run_until_complete(task)  
# print(task)  
  
  
  
# loop = asyncio.get_event_loop()  
# task = asyncio.ensure_future(c)  
# print(task)  
# loop.run_until_complete(task)  
# print(task)  
  
  
def callback_func(task):  
    print(task.result())  
  
loop = get_event_loop()  
task = asyncio.ensure_future(c)  
task.add_done_callback(callback_func)  
loop.run_until_complete(task)

以上就是“高级爬虫——实现异步，线程池，协程”的全部内容，希望对你有所帮助。

关于Python技术储备

学好 Python 不论是就业还是做副业赚钱都不错，但要学会 Python 还是要有一个学习规划。最后大家分享一份全套的 Python 学习资料，给那些想学习 Python 的小伙伴们一点帮助！

一、Python所有方向的学习路线

Python所有方向的技术点做的整理，形成各个领域的知识点汇总，它的用处就在于，你可以按照上面的知识点去找对应的学习资源，保证自己学得较为全面。

在这里插入图片描述