Skip to content

Commit 35c1dcd

Browse files
committed
use response.selector in link extractors instead of instantiating new Selector
1 parent 17d7347 commit 35c1dcd

File tree

2 files changed

+4
-6
lines changed

2 files changed

+4
-6
lines changed

scrapy/linkextractors/lxmlhtml.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,8 @@ def _extract_links(self, selector, response_url, response_encoding, base_url):
6565
if self.unique else links
6666

6767
def extract_links(self, response):
68-
html = Selector(response)
6968
base_url = get_base_url(response)
70-
return self._extract_links(html, response.url, response.encoding, base_url)
69+
return self._extract_links(response.selector, response.url, response.encoding, base_url)
7170

7271
def _process_links(self, links):
7372
""" Normalize and filter extracted links
@@ -95,14 +94,13 @@ def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restric
9594
canonicalize=canonicalize, deny_extensions=deny_extensions)
9695

9796
def extract_links(self, response):
98-
html = Selector(response)
9997
base_url = get_base_url(response)
10098
if self.restrict_xpaths:
10199
docs = [subdoc
102100
for x in self.restrict_xpaths
103-
for subdoc in html.xpath(x)]
101+
for subdoc in response.xpath(x)]
104102
else:
105-
docs = [html]
103+
docs = [response.selector]
106104
all_links = []
107105
for doc in docs:
108106
links = self._extract_links(doc, response.url, response.encoding, base_url)

scrapy/linkextractors/sgml.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restric
127127
def extract_links(self, response):
128128
base_url = None
129129
if self.restrict_xpaths:
130-
sel = Selector(response)
130+
sel = response.selector
131131
base_url = get_base_url(response)
132132
body = u''.join(f
133133
for x in self.restrict_xpaths

0 commit comments

Comments
 (0)