diff --git a/AvivoreXT/__init__.py b/AvivoreXT/__init__.py index 5a5df3b..daab838 100644 --- a/AvivoreXT/__init__.py +++ b/AvivoreXT/__init__.py @@ -1 +1 @@ -__version__ = '1.2.3' +__version__ = '1.2.4' diff --git a/AvivoreXT/avivore.py b/AvivoreXT/avivore.py index 4e529aa..9043056 100644 --- a/AvivoreXT/avivore.py +++ b/AvivoreXT/avivore.py @@ -3,6 +3,8 @@ import AvivoreXT from twitter import * +from twitter.stream import Timeout, HeartbeatTimeout, Hangup +from urllib2 import URLError import time import re import sys @@ -15,6 +17,7 @@ Helper functions """ + def is_sequence(arg): # check if arg is list, tuple, ... return (not hasattr(arg, "strip") and @@ -48,7 +51,7 @@ def read_config(self): try: self.twitter_search_types.append(self.config.get('twitter_search_objects', str(i), 0).strip("'\"")) i += 1 - except: + except Exception: exists = False # read search term definitions @@ -109,8 +112,8 @@ def read_config(self): def init_config_database(self, config_database_path): if not os.path.isfile(config_database_path): - Output("Creating a new configuration database.") - # dbcon = lite.connect(config_database_path) + Output("Creating a new configuration database.") + # dbcon = lite.connect(config_database_path) # dbcur = dbcon.cursor() # dbcur.execute("SELECT Count(*) FROM Config") # Output(str(dbcur.fetchone()[0]) + " entries in table Config so far.") @@ -146,8 +149,8 @@ def init_database(self, status): DatabaseFirstWrite = float(DBCur.fetchone()[0]) + 2 Output("Database first written to " + str(DataBaseFirstWrite)) ''' - else: # If the database doesn't exist, we'll create it. - if status == 1: # If we desire to save the database, it will output this message. + else: # If the database doesn't exist, we'll create it. + if status == 1: # If we desire to save the database, it will output this message. Output("Creating a new database to store data!") # Eventually I'll set this up to just delete the DB at close should it be chosen as an option. dbcon = lite.connect(self.database_path) @@ -169,11 +172,16 @@ def __init__(self, avivore_config): """ def twitter_auth(self): - self.twitter_bearer_token = oauth2_dance(self.avivore_config.twitter_consumer_key, - self.avivore_config.twitter_consumer_secret) - self.twitter_instance = Twitter(auth=OAuth2(bearer_token=self.twitter_bearer_token)) - return self.twitter_instance + try: + self.twitter_bearer_token = oauth2_dance(self.avivore_config.twitter_consumer_key, + self.avivore_config.twitter_consumer_secret) + self.twitter_instance = Twitter(auth=OAuth2(bearer_token=self.twitter_bearer_token)) + except (TwitterHTTPError, URLError): + SystemExit(1, "[!] Can't authenticate, check your network connection!") + except Exception as e: + SystemExit(1, "[!] Unknown error:\n"+e) + return self.twitter_instance def twitter_stream_auth(self): my_twitter_creds = os.path.expanduser(self.avivore_config.credentials_file) @@ -193,12 +201,14 @@ def twitter_stream_auth(self): def twitter_search(self, search_string): try: -# search_results = search_for_a_tweet(self.twitter_bearer_token, search_string) search_results = self.twitter_instance.search.tweets(q=search_string) output = search_results['statuses'] - except: - output = None # If this bombs out, we have the option of at least spitting out a result. - raise + except (TwitterHTTPError, URLError): + Output("[!] Problem connecting to twitter...") + output = None # If this bombs out, we have the option of at least spitting out a result. + except Exception as e: + Output("[!] Unknown problem querying twitter:\n"+e) + output = None # If this bombs out, we have the option of at least spitting out a result. return output def twitter_read_tweet(self, string): @@ -241,14 +251,14 @@ def __db_dup_check(self, tid, value): con = lite.connect(self.avivore_config.database_path) cur = con.cursor() - if output==0: + if output == 0: # check TweetID (TID) string = "SELECT * FROM Data WHERE TID IS \'" + str(int(tid)) + "\'" cur.execute(string) if cur.fetchone() is not None: # We should only have to pull one. output = 1 - if output==0: + if output == 0: # check extracted data value string = "SELECT * FROM Data WHERE Value IS \'" + value + "\'" cur.execute(string) @@ -266,7 +276,7 @@ def __db_write_value(self, stime, stype, user, userid, value, tweetid, message): cur.execute(qstring, (unicode(stime), unicode(stype), unicode(user), unicode(userid), unicode(value), unicode(tweetid), unicode(message))) - #lid = cur.lastrowid + # lid = cur.lastrowid class QueryThread(threading.Thread): @@ -309,6 +319,7 @@ def main(argv): # continue with twitter stream monitoring twitter_stream_main(avivore) + # performs classic search queries with configured time interval def twitter_query_main(avivore): stored = [] @@ -316,19 +327,13 @@ def twitter_query_main(avivore): while 1: for x in avivore.avivore_config.twitter_search_terms: twit_data = avivore.twitter_search(x) - if twit_data is None: # with defaults, it's unlikely that this - # will come up - message = "Nothing found for \"" + x + "\". Waiting " +\ - str(avivore.avivore_config.twitter_search_interval) +\ - " seconds..." - Output("[Q] "+str(message)) - else: + if twit_data is not None: for y in avivore.twitter_search(x): z = y['id'], y['created_at'], y['user']['screen_name'], y['text'], y['user']['id_str'] result = avivore.twitter_read_tweet(z[3]) if result[0] < 0: pass - else: # If something is found, then we'll process the tweet + else: # If something is found, then we'll process the tweet stored = stored, int(z[0]) # result value, time, result itself, tweet ID, tweet itself, userId string = result[0], z[2], result[1], z[0], z[3], z[4] @@ -344,22 +349,41 @@ def twitter_stream_main(avivore): stored = [] twitter_stream_inst = avivore.twitter_stream_auth() - iterator = twitter_stream_inst.statuses.filter(track=avivore.avivore_config.twitter_track_keywords) + try: + iterator = twitter_stream_inst.statuses.filter(track=avivore.avivore_config.twitter_track_keywords) - for y in iterator: - if 'id' in y and 'created_at' in y and 'user' in y and 'text' in y: - # print y - z = y['id'], y['created_at'], y['user']['screen_name'], y['text'], y['user']['id_str'] - result = avivore.twitter_read_tweet(z[3]) - if result[0] < 0: + for y in iterator: + # tweet may be a delete or data msg + if y is None: pass - else: # If something is found, then we'll process the tweet - stored = stored, int(z[0]) - # result value, time, result itself, tweet ID, tweet itself, userId - string = result[0], z[2], result[1], z[0], z[3], z[4] - message = avivore.process_tweet(string) - if 0 != message: - Output("[S] " + message) + # tweet may indicate a timeout + elif y is Timeout: + Output("[S] Stream timeout...") + elif y is HeartbeatTimeout: + Output("[S] Heartbeat timeout...") + # tweet may indicate a hung up stream conn. + elif y is Hangup: + Output("[S] Stream hangup detected!") + # ok, tweet seems to contain text, process it + elif 'id' in y and 'created_at' in y and 'user' in y and 'text' in y: + # print y + z = y['id'], y['created_at'], y['user']['screen_name'], y['text'], y['user']['id_str'] + result = avivore.twitter_read_tweet(z[3]) + if result[0] < 0: + pass + else: # If something is found, then we'll process the tweet + stored = stored, int(z[0]) + # result value, time, result itself, tweet ID, tweet itself, userId + string = result[0], z[2], result[1], z[0], z[3], z[4] + message = avivore.process_tweet(string) + if 0 != message: + Output("[S] " + message) + except (TwitterHTTPError, URLError): + Output("[!] Can't connect to twitter stream! Check your network connection!") + except Exception as e: + Output("[!] Unknown stream processing error:\n"+e) + finally: + Output("[S] Stream processing stopped.") def Output(string): @@ -405,5 +429,6 @@ def start(): main(sys.argv) raise + if __name__ == "__main__": start() diff --git a/README.md b/README.md index b018719..d4e5b6b 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ that would be nice to have: * notification support (see [#8](https://github.com/rc0r/AvivoreXT/issues/8)) * add more data sources (just to name a few: pastebin, google, facebook, reddit) * tests! (perspective: some integrated build, test, deployment work flow) +* some neat UI for configuration and result display (preferentially web based) * anything else found in [Issues](https://github.com/rc0r/AvivoreXT/issues) Feel free to get in touch outside of Github. My Twitter handle is [@_rc0r](https://twitter.com/_rc0r).