tc

Author: Yangzhedi

58tongcheng/tc_itemInfo.py

@@ -0,0 +1,45 @@
+# coding:utf-8
+from bs4 import BeautifulSoup
+import requests
+import time
+import pymongo
+
+client = pymongo.MongoClient('localhost', 27017)
+ceshi = client['ceshi']
+url_list = ceshi['url_list4']
+item_info = ceshi['item_info4']
+
+# 获取每一项下具体的url
+def get_links_from(channel, pages, who_sells=0):
+    list_view = '{}{}/pn{}/'.format(channel, str(who_sells), str(pages))
+    wb_data = requests.get(list_view)
+    time.sleep(1)
+    soup = BeautifulSoup(wb_data.text, 'lxml')
+    if soup.find('td', 't'):
+        for link in soup.select('td.t a.t'):
+            item_link = link.get('href').split('?')[0]
+            item_title = link.get_text()
+            # print link.get('href')      # http://zhuanzhuan.58.com/detail/774777200171630596z.shtml?fullCate=5%2C36&fullLocal=1&from=pc
+            # print item_link             # http://zhuanzhuan.58.com/detail/774777200171630596z.shtml
+            url_list.insert_one({'url': item_link, 'title': item_title})
+            print item_link,item_title
+            if item_link != 'http://jump.zhineng.58.com/jump':
+                get_item_info(item_link)
+    else:
+        pass
+
+# 获取每一页的具体信息
+def get_item_info(url):
+    wb_data = requests.get(url)
+    soup = BeautifulSoup(wb_data.text, 'lxml')
+    no_longer_exist = '404' in soup.find('link',type="text/css").get('href').split('/')
+    if no_longer_exist:
+        pass
+    else:
+        title_soup = soup.title.text
+        title = title_soup.split('_')[0]
+        user = title_soup.split('_')[1].split('-')[0].split('的闲置物品')[0]
+        price = soup.select('span.price_now i')[0].text
+        area = soup.select('.palce_li span i')[0].text # if soup.find_all('span', 'c_25d') else None
+        item_info.insert_one({'title': title, 'price': price, 'area': area, 'url': url, 'user': user})
+        print {'title': title, 'price': price, 'area': area, 'user': user}

58tongcheng/tc_urlLists.py

@@ -0,0 +1,79 @@
+# coding:utf-8
+from bs4 import BeautifulSoup
+import requests
+
+start_url = 'http://bj.58.com/sale.shtml'
+url_host = 'http://bj.58.com'
+
+def get_channel_urls(url):
+    wb_data = requests.get(url)
+    soup = BeautifulSoup(wb_data.text, 'lxml')
+    links = soup.select('ul.ym-submnu > li > b > a')
+    for link in links:
+        page_url = url_host + link.get('href')
+        # print page_url
+
+get_channel_urls(start_url)
+
+channel_list = '''
+    http://bj.58.com/shouji/
+    http://bj.58.com/tongxunyw/
+    http://bj.58.com/danche/
+    http://bj.58.com/fzixingche/
+    http://bj.58.com/diandongche/
+    http://bj.58.com/sanlunche/
+    http://bj.58.com/peijianzhuangbei/
+    http://bj.58.com/diannao/
+    http://bj.58.com/bijiben/
+    http://bj.58.com/pbdn/
+    http://bj.58.com/diannaopeijian/
+    http://bj.58.com/zhoubianshebei/
+    http://bj.58.com/shuma/
+    http://bj.58.com/shumaxiangji/
+    http://bj.58.com/mpsanmpsi/
+    http://bj.58.com/youxiji/
+    http://bj.58.com/jiadian/
+    http://bj.58.com/dianshiji/
+    http://bj.58.com/ershoukongtiao/
+    http://bj.58.com/xiyiji/
+    http://bj.58.com/bingxiang/
+    http://bj.58.com/binggui/
+    http://bj.58.com/chuang/
+    http://bj.58.com/ershoujiaju/
+    http://bj.58.com/bangongshebei/
+    http://bj.58.com/diannaohaocai/
+    http://bj.58.com/bangongjiaju/
+    http://bj.58.com/ershoushebei/
+    http://bj.58.com/yingyou/
+    http://bj.58.com/yingeryongpin/
+    http://bj.58.com/muyingweiyang/
+    http://bj.58.com/muyingtongchuang/
+    http://bj.58.com/yunfuyongpin/
+    http://bj.58.com/fushi/
+    http://bj.58.com/nanzhuang/
+    http://bj.58.com/fsxiemao/
+    http://bj.58.com/xiangbao/
+    http://bj.58.com/meirong/
+    http://bj.58.com/yishu/
+    http://bj.58.com/shufahuihua/
+    http://bj.58.com/zhubaoshipin/
+    http://bj.58.com/yuqi/
+    http://bj.58.com/tushu/
+    http://bj.58.com/tushubook/
+    http://bj.58.com/wenti/
+    http://bj.58.com/yundongfushi/
+    http://bj.58.com/jianshenqixie/
+    http://bj.58.com/huju/
+    http://bj.58.com/qiulei/
+    http://bj.58.com/yueqi/
+    http://bj.58.com/chengren/
+    http://bj.58.com/nvyongpin/
+    http://bj.58.com/qinglvqingqu/
+    http://bj.58.com/qingquneiyi/
+    http://bj.58.com/chengren/
+    http://bj.58.com/xiaoyuan/
+    http://bj.58.com/ershouqiugou/
+    http://bj.58.com/tiaozao/
+    http://bj.58.com/tiaozao/
+    http://bj.58.com/tiaozao/
+'''