urllib的使用

最新推荐文章于 2024-09-18 22:11:05 发布

原创最新推荐文章于 2024-09-18 22:11:05 发布 · 197 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

收录于

爬虫小白学习

本文介绍了如何使用Python的urllib库进行基础和高级操作，包括urlopen函数、数据编码、timeout处理、Request类、Hander类和Openr类的应用，涉及身份验证、代理设置、cookie管理和异常处理。


# See PyCharm help at https://www.jetbrains.com/help/pycharm/
import urllib.request
#设置关闭本地验证
import ssl
import urllib.parse
ssl._create_default_https_context = ssl._create_unverified_context
2.1 urllib的使用
2.1.1   urlopen
response=urllib.request.urlopen("https://www.baidu.com")
print(response.read().decode('utf-8'))
print(type(response))
print(response.status)#响应状态码
print(response.getheaders())#响应头
print(response.getheader('Server'))#服务器用什么搭建的
data参数的使用，
data=bytes(urllib.parse.urlencode({'name':"germey"}),encoding='utf-8')     #将参数转化为字节流编码格式
response=urllib.request.urlopen('https://www.httpbin.org/post',data=data)
print(response.read().decode("utf-8"))
#2.1.2 timeout参数
response=urllib.request.urlopen('https://www.httpbin.org/get',timeout=0.1)
print(response.read())
通过timeout，实现当一个网页长时间未响应，就跳过对其的抓取，采用try except语句实现。
import socket
import urllib.error
import urllib.request
try:
    response=urllib.request.urlopen('https://www.httpbin.org/get',timeout=2)
    print(response.read())
except urllib.error.URLError as e:
    if isinstance(e.reason,socket.timeout):  #socket.timeout 超时异常
        print("TIME OUT")
用Request类构建请求
import urllib.request
request=urllib.request.Request('https://www.httpbin.org/')
response=urllib.request.urlopen(request)
print(response.read().decode("utf-8"))
用Request传入多个参数
from urllib import request,parse
url='https://www.httpbin.org/post'
headers={
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
'Host':'www.httpbin.org'
}
dic={'name':'germey'}
data=bytes(parse.urlencode(dic),encoding='utf-8')
req=request.Request(url=url,headers=headers,data=data,method='POST')
response=request.urlopen(req)
# print(response.read().decode('utf-8'))
# Hander类和Openr类 高级用法
# 1、验证，处理有基本身份认证的网页：
from urllib.request import HTTPPasswordMgrWithDefaultRealm,HTTPBasicAuthHandler,build_opener
from urllib.error import URLError

username='admin'
password='admin'
url='https://ssr3.scrape.center/'
#四部曲
p=HTTPPasswordMgrWithDefaultRealm()
p.add_password(None,url,username,password)
auth_hander=HTTPBasicAuthHandler(p)
opener=build_opener(auth_hander)
#处理异常
try:
    result=opener.open(url)
    html=result.read().decode("utf-8")
    print(html)
except URLError as e:
    print(e.reason)
#2 代理：
from urllib.error import URLError
from urllib.request import ProxyHandler,build_opener
proxy_handler = ProxyHandler({
        'http':'http://127.0.0.1:8080',
        'https':'https://127.0.0.1:8080'
})
opener=build_opener(proxy_handler)
try:
    response=opener.open('https://www.baidu.com/')
    print(response.read().decode("utf-8"))
except URLError as e:
    print(e.reason)
#3  cookie
# import http.cookiejar,urllib.request

cookie=http.cookiejar.CookieJar()
handler=urllib.request.HTTPCookieProcessor(cookie)
opener=urllib.request.build_opener(handler)
response=opener.open('https://www.baidu.com')
for item in cookie:
    print(item.name+'='+item.value)
#cookie  比较没有用
#处理异常
#URLError
from urllib import error,request
try:
    response = request.urlopen("https://cuiqingcai.com/404")
except error.URLError as e:
    print(e.reason)
#HTTPError

标签

#前端 #javascript #python