Packaged all of openaddr.ci based on OpenAddresses-Machine using setu…

…p.py

Procfile

@@ -1,2 +1,2 @@
-web: gunicorn -w 4 --bind 0.0.0.0:$PORT app:app
-dequeue: python run-dequeue.py
+web: gunicorn -w 4 --bind 0.0.0.0:$PORT openaddr.ci:app
+dequeue: python -m openaddr.ci.run_dequeue

openaddr/VERSION

@@ -0,0 +1 @@
+0.0.0

recreate-db.py → openaddr/ci/recreate_db.py

@@ -4,7 +4,7 @@
 from pq import PQ
 from psycopg2 import connect
 
-def main(DATABASE_URL):
+def recreate(DATABASE_URL):
     '''
     '''
     schema_filename = join(dirname(__file__), 'schema.pgsql')
@@ -19,7 +19,11 @@ def main(DATABASE_URL):
         pq = PQ(conn, table='queue')
         pq.create()
 
-if __name__ == '__main__':
-
+def main():
+    '''
+    '''
     DATABASE_URL = os.environ['DATABASE_URL']
-    exit(main(DATABASE_URL))
+    return recreate(DATABASE_URL)
+
+if __name__ == '__main__':
+    exit(main())

run-dequeue.py → openaddr/ci/run_dequeue.py

@@ -2,7 +2,7 @@
 from time import sleep
 from traceback import print_exc
 
-from app import (
+from . import (
     db_connect, db_queue, pop_finished_task_from_queue, DONE_QUEUE, load_config
     )
 

worker.py → openaddr/ci/worker.py

@@ -7,7 +7,8 @@
 enqueues a new message on a separate PQ queue when the work is done."""
 
 import time, os, subprocess, psycopg2
-import lib
+
+from . import db_connect, db_queue, db_queue, MAGIC_OK_MESSAGE, DONE_QUEUE
 
 # File path and URL path for result directory. Should be S3.
 _web_output_dir = '/var/www/html/oa-runone'
@@ -39,7 +40,7 @@ def do_work(job_id, job_contents):
     # Prepare return parameters
     r = { 'result_code': result_code,
           'output_url': '%s/%s' % (_web_base_url, job_id),
-          'message': lib.MAGIC_OK_MESSAGE }
+          'message': MAGIC_OK_MESSAGE }
 
     return r
 
@@ -64,13 +65,13 @@ def run(task_data):
           'result' : result }
     return r
 
-def serve_queue():
+def main():
     "Single threaded worker to serve the job queue"
 
     # Connect to the queue
-    conn = lib.db_connect(os.environ['DATABASE_STRING'])
-    input_queue = lib.db_queue(conn)
-    output_queue = lib.db_queue(conn, lib.DONE_QUEUE)
+    conn = db_connect(os.environ['DATABASE_URL'])
+    input_queue = db_queue(conn)
+    output_queue = db_queue(conn, DONE_QUEUE)
 
     # Fetch and run jobs in a loop    
     while True:
@@ -82,4 +83,4 @@ def serve_queue():
 
 
 if __name__ == '__main__':
-    serve_queue()
+    exit(main())

openaddr/tests/ci.py

@@ -15,14 +15,14 @@
     TASK_QUEUE, DONE_QUEUE, MAGIC_OK_MESSAGE
     )
 
-recreate_db = __import__('recreate-db')
+from ..ci import recreate_db
 
 class TestHook (unittest.TestCase):
 
     def setUp(self):
         '''
         '''
-        recreate_db.main(app.config['DATABASE_URL'])
+        recreate_db.recreate(app.config['DATABASE_URL'])
 
         self.database_url = app.config['DATABASE_URL']
         self.github_auth = app.config['GITHUB_AUTH']

requirements.txt

@@ -1,13 +1 @@
-Flask==0.10.1
-gunicorn==19.3.0
-httmock==1.2.3
-itsdangerous==0.24
-Jinja2==2.7.3
-MarkupSafe==0.23
-mock==1.0.1
-pq==1.2
-psycopg2==2.6
-requests==2.6.2
-simplejson==3.6.5
-uritemplate==0.6
-Werkzeug==0.10.4
+.

setup.py

@@ -0,0 +1,63 @@
+from setuptools import setup
+from os.path import join, dirname
+import sys
+
+with open(join(dirname(__file__), 'openaddr', 'VERSION')) as file:
+    version = file.read().strip()
+
+conditional_requirements = list()
+
+if sys.version_info[0] == 2:
+    conditional_requirements += [
+        # http://python-future.org
+        'future >= 0.14.3',
+
+        # https://github.com/jdunck/python-unicodecsv
+        'unicodecsv >= 0.11.2',
+    ]
+
+setup(
+    name = 'OpenAddresses-Machine',
+    version = version,
+    url = 'https://github.com/openaddresses/machine',
+    author = 'Michal Migurski',
+    author_email = '[email protected]',
+    description = 'In-progress scripts for running OpenAddresses on a complete data set and publishing the results.',
+    packages = ['openaddr', 'openaddr.ci', 'openaddr.tests'],
+    package_data = {
+        'openaddr': [
+            'VERSION'
+        ],
+        'openaddr.ci': [
+            'schema.pgsql'
+        ]
+    },
+    install_requires = [
+        'Flask == 0.10.1', 'gunicorn == 19.3.0', 'httmock == 1.2.3',
+        'itsdangerous == 0.24', 'Jinja2 == 2.7.3', 'MarkupSafe == 0.23',
+        'mock == 1.0.1', 'pq == 1.2', 'psycopg2 == 2.6', 'simplejson == 3.6.5',
+        'uritemplate == 0.6', 'Werkzeug == 0.10.4',
+
+        'boto >= 2.22.0', 'Jinja2 >= 2.7.0', 'dateutils >= 0.6', 'ijson >= 2.0',
+
+        # https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1306991/comments/10
+        'requests == 2.2.1',
+
+        # https://pypi.python.org/pypi/requests-ftp, appears no longer maintained.
+        'requests-ftp == 0.2.0',
+
+        # https://github.com/patrys/httmock
+        'httmock >= 1.2',
+
+        # https://pypi.python.org/pypi/setproctitle/
+        'setproctitle >= 1.1.8'
+
+        ] + conditional_requirements,
+    entry_points = dict(
+        console_scripts = [
+            'openaddr-ci-recreate-db = openaddr.ci.recreate_db:main',
+            'openaddr-ci-run-dequeue = openaddr.ci.run_dequeue:main',
+            'openaddr-ci-worker = openaddr.ci.worker:main',
+        ]
+    )
+)