Skip to content

Commit 9817df1

Browse files
committed
Compute image checksum before persisting images. closes scrapy#92
Avoids threading issue accesing buffer
1 parent 673a120 commit 9817df1

File tree

1 file changed

+4
-5
lines changed

1 file changed

+4
-5
lines changed

scrapy/contrib/pipeline/images.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -249,13 +249,12 @@ def _onsuccess(result):
249249
return dfd
250250

251251
def image_downloaded(self, response, request, info):
252-
first_buf = None
252+
checksum = None
253253
for key, image, buf in self.get_images(response, request, info):
254+
if checksum is None:
255+
checksum = md5sum(buf)
254256
self.store.persist_image(key, image, buf, info)
255-
if first_buf is None:
256-
first_buf = buf
257-
first_buf.seek(0)
258-
return md5sum(first_buf)
257+
return checksum
259258

260259
def get_images(self, response, request, info):
261260
key = self.image_key(request.url)

0 commit comments

Comments
 (0)