update dependencies and fix some bugs

woaidu_crawler/woaidu_crawler/commands/init_single_mongodb.py

@@ -29,7 +29,7 @@
 """
 
 import types
-from pymongo.connection import MongoClient
+from pymongo import MongoClient
 from pymongo import ASCENDING, DESCENDING
 
 DATABASE_NAME = "books_fs"

woaidu_crawler/woaidu_crawler/pipelines/bookfile.py

@@ -73,7 +73,7 @@ def process_item(self, item, spider):
             custom process_item func,so it will manage the Request result.
         """
 
-        info = self.spiderinfo[spider]
+        info = self.SpiderInfo(spider)
         requests = arg_to_iter(self.get_media_requests(item, info))
         dlist = [self._process_request(r, info) for r in requests]
         dfd = DeferredList(dlist, consumeErrors=1)

woaidu_crawler/woaidu_crawler/pipelines/file.py

@@ -15,7 +15,7 @@
 from scrapy.utils.misc import md5sum
 from collections import defaultdict
 from scrapy.utils.misc import arg_to_iter
-from scrapy.contrib.pipeline.images import MediaPipeline
+from scrapy.contrib.pipeline.images import ImagesPipeline
 from woaidu_crawler.utils.select_result import list_first_item
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 
@@ -96,7 +96,7 @@ def _mkdir(self, dirname, domain=None):
         if dirname not in seen:
             seen.add(dirname)
 
-class FilePipeline(MediaPipeline):
+class FilePipeline(ImagesPipeline):
     """
         download file pipeline.
     """
@@ -117,7 +117,7 @@ def __init__(self,store_uri,download_func=None):
             raise NotConfigured
         self.store = self._get_store(store_uri)
         self.style = color.color_style()
-        super(FilePipeline, self).__init__(download_func=download_func)
+        super(FilePipeline, self).__init__(store_uri, download_func=download_func)
 
     @classmethod
     def from_settings(cls, settings):

woaidu_crawler/woaidu_crawler/pipelines/mongodb.py

@@ -7,7 +7,7 @@
 from woaidu_crawler.utils import color
 from scrapy import log
 from woaidu_crawler.utils import color
-from pymongo.connection import MongoClient
+from pymongo import MongoClient
 
 class SingleMongodbPipeline(object):
     """

woaidu_crawler/woaidu_crawler/spiders/woaidu_detail_spider.py

@@ -3,13 +3,13 @@
 
 import time
 from pprint import pprint
-from scrapy.spider import BaseSpider
+from scrapy.spider import Spider
 from scrapy.selector import HtmlXPathSelector
 from scrapy.http import Request
 from woaidu_crawler.items import WoaiduCrawlerItem
 from woaidu_crawler.utils.select_result import list_first_item,strip_null,deduplication,clean_url
 
-class WoaiduSpider(BaseSpider):
+class WoaiduSpider(Spider):
     name = "woaidu"
     start_urls = (
             'http://www.woaidu.org/sitemap_1.html',