From a760372a70fd32b6cec6a7c0e3a9efbd9403dfdd Mon Sep 17 00:00:00 2001
From: luyishisi <543429245@qq.com>
Date: Thu, 6 Apr 2017 16:43:34 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Durlspider=E6=A1=86=E6=9E=B6?=
 =?UTF-8?q?=E7=9A=84bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 UrlSpider/README.md          |  3 ---
 UrlSpider/model/UrlSpider.py | 22 +++++++++++++++++-----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/UrlSpider/README.md b/UrlSpider/README.md
index 95df98e7..eeaefc84 100644
--- a/UrlSpider/README.md
+++ b/UrlSpider/README.md
@@ -11,6 +11,3 @@ example  样例文件 -- 可以参考我对该模板修改后进行的爬虫代
 
 详情请致电 a83533774@gmail.com
 或者在说明文档下留言
-
-
-
diff --git a/UrlSpider/model/UrlSpider.py b/UrlSpider/model/UrlSpider.py
index 56ded5a4..60a33f2d 100644
--- a/UrlSpider/model/UrlSpider.py
+++ b/UrlSpider/model/UrlSpider.py
@@ -44,7 +44,7 @@
 
 THREAD_COUNT = 50  # 开启线程数
 sql_num_base = 200  # 自定义的执行批量插入的随机值基数，当此值为1时则每次获取数据均直接插入。
-sql_num_add = 100  # 自定义的随机值加数，平均而言，当单独一个线程执行sql_num_base+1/3*sql_num_add次数时执行插入
+sql_num_add = 1000  # 自定义的随机值加数，平均而言，当单独一个线程执行sql_num_base+1/3*sql_num_add次数时执行插入
 #   不可修改全局变量参数
 #------------------------------------------------
 schedule = 0  # 当前线程标志
@@ -133,9 +133,14 @@ def run(self):
                 # 随机一个限制数,200-300 到则进行插入
                 sql_num = int(random.uniform(sql_num_base, sql_num_base + 100))
                 if(now_requests_num >= sql_num):
+                    try:
+                        cursor.executemany(Update_sql, date_list)#防止长时间的请求时间导致数据库连接断开
+                        connect.commit()
+                    except:
+                        connect, cursor = ConnectDB()
+                        cursor.executemany(Update_sql, date_list)#
+                        connect.commit()
                     now_requests_num = 0
-                    cursor.executemany(Update_sql, date_list)
-                    connect.commit()
                     date_list = []
                     print 'up', time.ctime(), '&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&', sql_num
             except Exception, e:
@@ -147,8 +152,14 @@ def run(self):
             self.lock.acquire()
             schedule += 1
             self.lock.release()
-        cursor.executemany(Update_sql, date_list)  # 大爷的注释,,这里要保存一次
-        connect.commit()
+
+        try:
+            cursor.executemany(Update_sql, date_list)#防止长时间的请求时间导致数据库连接断开
+            connect.commit()
+        except:
+            connect, cursor = ConnectDB()
+            cursor.executemany(Update_sql, date_list)#
+            connect.commit()
         connect.close()
 
 
@@ -163,6 +174,7 @@ def ConnectDB():
             break
         except MySQLdb.Error, e:
             print "Error %d: %s" % (e.args[0], e.args[1])
+            time.sleep(60)#防止出现永远循环
     return connect, cursor