scrapy

Author: CoderHuo

learn_scrapy/learn_scrapy_01.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from scrapy import Selector
+
+__author__ = 'Mr.Huo'
+
+html = '''
+    <ul class="list">
+        <li>1</li>
+        <li>2</li>
+        <li>3</li>
+    </ul>
+    <ul class="list">
+        <li>4</li>
+        <li>5</li>
+        <li>6</li>
+    </ul>
+    <div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>
+'''
+
+
+def print_iter(iter_list):
+    if iter_list:
+        for it in iter_list:
+            print(it)
+    print()
+
+
+def main():
+    sel = Selector(text=html)
+    xp = lambda x: sel.xpath(x).extract()
+    print(xp('//li'))
+    print(xp('//li[1]'))
+    print(xp('(//li)[1]'))
+    print_iter(sel.xpath('//li'))
+    print_iter(sel.css('li'))
+    pass
+
+
+if __name__ == '__main__':
+    main()

spy/run.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+from spy.spiders.example import ExampleSpider
+import sys
+
+__author__ = 'Mr.Huo'
+
+
+def main():
+    # 将spy跟目录加入sys.path
+    sys.path.append('..')
+    settings = get_project_settings()
+    process = CrawlerProcess(settings=settings)
+    process.crawl(ExampleSpider)
+    process.start()
+
+
+if __name__ == '__main__':
+    main()

spy/spy/pipelines.py

@@ -8,4 +8,6 @@
 
 class SpyPipeline(object):
     def process_item(self, item, spider):
+        item['title'] = 'hhhhhh'
+        print('pipline:', item['title'])
         return item

spy/spy/settings.py

@@ -64,9 +64,9 @@
 
 # Configure item pipelines
 # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    'spy.pipelines.SpyPipeline': 300,
-#}
+ITEM_PIPELINES = {
+    'spy.pipelines.SpyPipeline': 1,
+}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html

spy/spy/spiders/example.py

@@ -1,12 +1,22 @@
 # -*- coding: utf-8 -*-
 import scrapy
+from scrapy.loader import ItemLoader
+from spy.items import SpyItem
 
 
 class ExampleSpider(scrapy.Spider):
     name = 'example'
-    allowed_domains = ['news.baidu.com']
-    start_urls = ['http://news.baidu.com/']
+    allowed_domains = ['doc.scrapy.org']
+    start_urls = ['http://doc.scrapy.org/en/latest/_static/selectors-sample1.html']
 
     def parse(self, response):
-        links  =  response.xpath('//li')
-        print(links)
+        items = SpyItem()
+        filename = response.xpath('//title/text()').extract()[0]
+        items['title'] = response.xpath('//title/text()').extract()
+        for sel in response.xpath('//a'):
+            pass
+        print("-" * 80)
+        print(filename)
+        print(response)
+        print(items['title'])
+        print("-" * 80)