抓取51job职位信息

#coding=gb18030
import requests
from lxml import etree
from gevent import monkey,pool;monkey.patch_all()
import gevent
import os
import time
import xlwt
import urllib2
import string

xl=xlwt.Workbook()
st=xl.add_sheet('job',cell_overwrite_ok=True)
print time.strftime('%H:%M:%S')
p=pool.Pool(30)

#返回分页地址
def get_pages(page):
    url='http://search.51job.com/list/040000,000000,0000,00,9,99,python,2,'+ str(page) + '.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
    yield url

#返回详细页链接
def get_links(link):
    header={'User-Agent':'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)'}
    req=urllib2.Request(link,headers=header)
    resp=urllib2.urlopen(req)
    response=resp.read()
    resp.close()
    html=etree.HTML(response)
    result=html.xpath('//a[@onmousedown=""]/@href')
    for x in result:
        yield x

#写入表格
def write_excel(url):
    gevent.sleep(0)
    global excel_row
    try:
        excel_row+=1
        req=urllib2.Request(url)
        res=urllib2.urlopen(req)
        respon=res.read()
        res.close()
        sou=etree.HTML(respon)
        st_title=sou.xpath('//title/text()')
        ss=st_title[0].find(u'招聘_')+3
        st.write(excel_row,0,st_title[0][ss:-12].strip())
        st.write(excel_row,1,sou.xpath('//h1/text()')[0])
        t1=sou.xpath('//span[@class="sp4"]//text()')
        tt1=','.join(t1)
        st.write(excel_row,2,tt1)
        st.write(excel_row,3,sou.xpath('//span[@class="lname"]/text()')[0])
        dz=sou.xpath('//div[@class="bmsg inbox"]//text()')
        s=''.join(dz).strip()
        st.write(excel_row,4,s)
        st.write(excel_row,5,sou.xpath('//strong/text()')[1])
        detail=sou.xpath('//div[@class="bmsg job_msg inbox"]//text()[normalize-space()]')
        detail=''.join(detail)
        st.write(excel_row,6,detail)
    except:
        pass

excel_row=0
th=[]
for i in range(1,21):
    for k in get_pages(i):
        for kk in get_links(k):
            th.append(p.spawn(write_excel,kk))
            p.join()
gevent.joinall(th)

xl.save('d:\\job_python.xls')
print time.strftime('%H:%M:%S')