From 2c09e6508c905b9b3af8fab07a36990310e9cb75 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 25 Apr 2015 17:03:36 -0700 Subject: [PATCH 01/51] Stubbed out simple Flask app --- .gitignore | 2 ++ Procfile | 1 + app.py | 10 ++++++++++ requirements.txt | 6 ++++++ 4 files changed, 19 insertions(+) create mode 100644 .gitignore create mode 100644 Procfile create mode 100644 app.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0399b282 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +venv-hooked-on-sources +*.pyc diff --git a/Procfile b/Procfile new file mode 100644 index 00000000..ae36de7b --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn -w 4 --bind 0.0.0.0:$PORT app:app \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 00000000..ebac73c3 --- /dev/null +++ b/app.py @@ -0,0 +1,10 @@ +from flask import Flask + +app = Flask(__name__) + +@app.route('/') +def index(): + return 'Yo.' + +if __name__ == '__main__': + app.run(debug=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d8739ba5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +Flask==0.10.1 +gunicorn==19.3.0 +itsdangerous==0.24 +Jinja2==2.7.3 +MarkupSafe==0.23 +Werkzeug==0.10.4 From f37bd5bb696e0a0497fcea3ea419568f94b94298 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 25 Apr 2015 17:11:57 -0700 Subject: [PATCH 02/51] Added webhook output --- app.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index ebac73c3..51d06ad0 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ -from flask import Flask +from sys import stderr +from flask import Flask, request app = Flask(__name__) @@ -6,5 +7,14 @@ def index(): return 'Yo.' +@app.route('/hook', methods=['GET', 'POST']) +def hook(): + print >> stderr, request.method + print >> stderr, request.headers + print >> stderr, request.values + print >> stderr, repr(request.data) + + return 'Yo.' + if __name__ == '__main__': app.run(debug=True) From 62e2b69a6aeb3e72d64181f1d0ec541a1b82af7e Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 25 Apr 2015 17:45:57 -0700 Subject: [PATCH 03/51] Investigating blob contents for changes --- app.py | 40 ++++++++++++++++++++++++++++++++++++---- requirements.txt | 1 + 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 51d06ad0..fd067da9 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,8 @@ from sys import stderr +from json import loads +from base64 import b64decode from flask import Flask, request +from requests import get app = Flask(__name__) @@ -9,10 +12,39 @@ def index(): @app.route('/hook', methods=['GET', 'POST']) def hook(): - print >> stderr, request.method - print >> stderr, request.headers - print >> stderr, request.values - print >> stderr, repr(request.data) + #print >> stderr, request.method + #print >> stderr, request.headers + #print >> stderr, request.values + #print >> stderr, repr(request.data) + + payload = loads(request.data) + + touched = set() + + for commit in payload['commits']: + for filelist in (commit['added'], commit['modified']): + touched.update(filelist) + + print >> stderr, 'Touched files', list(touched) + + commit_sha = payload['head_commit']['id'] + + print >> stderr, 'Commit SHA', commit_sha + + for filename in touched: + + contents_url = payload['repository']['contents_url'] + contents_url = contents_url.replace('{+path}', filename) + contents_url += '?ref={}'.format(commit_sha) + + print >> stderr, 'Contents URL', contents_url + + got = get(contents_url) + + content, blob_sha = got.json()['content'], got.json()['sha'] + + print >> stderr, 'Contents SHA', blob_sha + print >> stderr, repr(b64decode(content)) return 'Yo.' diff --git a/requirements.txt b/requirements.txt index d8739ba5..94dab77b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ gunicorn==19.3.0 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 +requests==2.6.2 Werkzeug==0.10.4 From 8cf2f1d9ce91c151afd87bf5a592219b08bab7c7 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 25 Apr 2015 17:56:53 -0700 Subject: [PATCH 04/51] Skipping content checks on removed files --- app.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index fd067da9..d99f14b7 100644 --- a/app.py +++ b/app.py @@ -25,11 +25,14 @@ def hook(): for filelist in (commit['added'], commit['modified']): touched.update(filelist) + for filename in commit['removed']: + touched.remove(filename) + print >> stderr, 'Touched files', list(touched) commit_sha = payload['head_commit']['id'] - print >> stderr, 'Commit SHA', commit_sha + print >> stderr, 'Head commit SHA', commit_sha for filename in touched: @@ -41,6 +44,10 @@ def hook(): got = get(contents_url) + if got.status_code not in range(200, 299): + print >> stderr, 'Skipping', filename, '-', got.status_code + continue + content, blob_sha = got.json()['content'], got.json()['sha'] print >> stderr, 'Contents SHA', blob_sha From f95574f600b414d667c1eac1da3517ed826ebfa7 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 09:38:52 -0700 Subject: [PATCH 05/51] Moved payload processing code to new function --- app.py | 59 ++++++++++++++++++++++++++++++++++++------------ requirements.txt | 2 ++ 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index d99f14b7..8148df75 100644 --- a/app.py +++ b/app.py @@ -1,7 +1,10 @@ from sys import stderr -from json import loads +from os.path import relpath, splitext from base64 import b64decode -from flask import Flask, request +import json + +from flask import Flask, request, Response +from uritemplate import expand from requests import get app = Flask(__name__) @@ -12,48 +15,74 @@ def index(): @app.route('/hook', methods=['GET', 'POST']) def hook(): - #print >> stderr, request.method - #print >> stderr, request.headers - #print >> stderr, request.values - #print >> stderr, repr(request.data) + files = process_payload(json.loads(request.data)) - payload = loads(request.data) + serialize = lambda data: json.dumps(data, indent=2, sort_keys=True) + response = '\n\n'.join(['{}:\n\n{}\n'.format(name, serialize(data)) + for (name, data) in sorted(files.items())]) + return Response(response, headers={'Content-Type': 'text/plain'}) + +def get_touched_files(payload): + ''' Return a set of files modified in payload commits. + ''' touched = set() + # Iterate over commits in chronological order. for commit in payload['commits']: for filelist in (commit['added'], commit['modified']): + # Include any potentially-new files. touched.update(filelist) for filename in commit['removed']: + # Skip files that no longer exist. touched.remove(filename) - print >> stderr, 'Touched files', list(touched) + print >> stderr, 'Touched files', ', '.join(touched) + + return touched + +def process_payload(payload): + ''' Return a dictionary of file paths and decoded JSON contents. + ''' + processed, touched = dict(), get_touched_files(payload) commit_sha = payload['head_commit']['id'] print >> stderr, 'Head commit SHA', commit_sha for filename in touched: + if relpath(filename, 'sources').startswith('..'): + # Skip things outside of sources directory. + continue + + if splitext(filename)[1] != '.json': + # Skip non-JSON files. + continue contents_url = payload['repository']['contents_url'] - contents_url = contents_url.replace('{+path}', filename) - contents_url += '?ref={}'.format(commit_sha) + contents_url = expand(contents_url, dict(path=filename)) + contents_url = '{contents_url}?ref={commit_sha}'.format(**locals()) print >> stderr, 'Contents URL', contents_url got = get(contents_url) + contents = got.json() if got.status_code not in range(200, 299): print >> stderr, 'Skipping', filename, '-', got.status_code continue - content, blob_sha = got.json()['content'], got.json()['sha'] + content, encoding = contents['content'], contents['encoding'] - print >> stderr, 'Contents SHA', blob_sha - print >> stderr, repr(b64decode(content)) - - return 'Yo.' + print >> stderr, 'Contents SHA', contents['sha'] + + if encoding == 'base64': + processed[filename] = json.loads(b64decode(content)) + else: + raise ValueError('Unrecognized encoding "{}"'.format(encoding)) + + return processed if __name__ == '__main__': app.run(debug=True) diff --git a/requirements.txt b/requirements.txt index 94dab77b..9008957a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,6 @@ itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 requests==2.6.2 +simplejson==3.6.5 +uritemplate==0.6 Werkzeug==0.10.4 From 002d0786f75907d152f7c6867d796d03e7605ae1 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 10:21:28 -0700 Subject: [PATCH 06/51] Added basic unit tests --- requirements.txt | 1 + tests.py | 672 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 673 insertions(+) create mode 100644 tests.py diff --git a/requirements.txt b/requirements.txt index 9008957a..d62c29cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ Flask==0.10.1 gunicorn==19.3.0 +httmock==1.2.3 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 diff --git a/tests.py b/tests.py new file mode 100644 index 00000000..f925a95e --- /dev/null +++ b/tests.py @@ -0,0 +1,672 @@ +from app import app +from httmock import HTTMock, response +from urlparse import parse_qsl +import unittest + +class TestHook (unittest.TestCase): + + def setUp(self): + ''' + ''' + self.client = app.test_client() + + def response_content(self, url, request): + ''' + ''' + query = dict(parse_qsl(url.query)) + + if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): + data = '''{ + "name": "us-ca-alameda_county.json", + "path": "sources/us-ca-alameda_county.json", + "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8", + "size": 745, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522", + "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json", + "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8", + "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json", + "type": "file", + "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522", + "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8", + "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json" + } + }''' + + return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + + if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): + data = '''{ + "name": "us-ca-san_francisco.json", + "path": "sources/us-ca-san_francisco.json", + "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff", + "size": 519, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json", + "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff", + "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json", + "type": "file", + "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff", + "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json" + } + }''' + + return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + + if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): + data = '''{ + "name": "us-ca-berkeley.json", + "path": "sources/us-ca-berkeley.json", + "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad", + "size": 779, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json", + "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad", + "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json", + "type": "file", + "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad", + "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json" + } + }''' + + return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + + raise ValueError('Unknowable URL "{}"'.format(url.geturl())) + + def test_webhook_one_commit(self): + ''' + ''' + data = '''{ + "after": "e91fbc420f08890960f50f863626e1062f922522", + "base_ref": null, + "before": "c52204fd40f17f9da243df09e6d1107d48768afd", + "commits": [ + { + "added": [ + "sources/us-ca-alameda_county.json" + ], + "author": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "committer": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "distinct": true, + "id": "e91fbc420f08890960f50f863626e1062f922522", + "message": "Added first source", + "modified": [], + "removed": [], + "timestamp": "2015-04-25T17:16:12-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522" + } + ], + "compare": "https://github.com/openaddresses/hooked-on-sources/compare/c52204fd40f1...e91fbc420f08", + "created": false, + "deleted": false, + "forced": false, + "head_commit": { + "added": [ + "sources/us-ca-alameda_county.json" + ], + "author": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "committer": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "distinct": true, + "id": "e91fbc420f08890960f50f863626e1062f922522", + "message": "Added first source", + "modified": [], + "removed": [], + "timestamp": "2015-04-25T17:16:12-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522" + }, + "organization": { + "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", + "description": "The free and open global address collection ", + "events_url": "https://api.github.com/orgs/openaddresses/events", + "id": 6895392, + "login": "openaddresses", + "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", + "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", + "repos_url": "https://api.github.com/orgs/openaddresses/repos", + "url": "https://api.github.com/orgs/openaddresses" + }, + "pusher": { + "email": "mike-github@teczno.com", + "name": "migurski" + }, + "ref": "refs/heads/master", + "repository": { + "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", + "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", + "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", + "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", + "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", + "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", + "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", + "created_at": 1430006167, + "default_branch": "master", + "description": "Temporary repository for testing Github webhook features", + "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", + "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", + "full_name": "openaddresses/hooked-on-sources", + "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", + "git_url": "git://github.com/openaddresses/hooked-on-sources.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", + "html_url": "https://github.com/openaddresses/hooked-on-sources", + "id": 34590951, + "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", + "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", + "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", + "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", + "language": null, + "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", + "master_branch": "master", + "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", + "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", + "mirror_url": null, + "name": "hooked-on-sources", + "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", + "open_issues": 0, + "open_issues_count": 0, + "organization": "openaddresses", + "owner": { + "email": "openaddresses@gmail.com", + "name": "openaddresses" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", + "pushed_at": 1430007676, + "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", + "size": 0, + "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", + "stargazers": 0, + "stargazers_count": 0, + "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", + "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", + "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", + "svn_url": "https://github.com/openaddresses/hooked-on-sources", + "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", + "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", + "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", + "updated_at": "2015-04-25T23:56:07Z", + "url": "https://github.com/openaddresses/hooked-on-sources", + "watchers": 0, + "watchers_count": 0 + }, + "sender": { + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/migurski", + "id": 58730, + "login": "migurski", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "repos_url": "https://api.github.com/users/migurski/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "type": "User", + "url": "https://api.github.com/users/migurski" + } + }''' + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) + + self.assertEqual(posted.status_code, 200) + self.assertTrue('us-ca-alameda_county' in posted.data) + self.assertTrue('data.acgov.org' in posted.data) + + def test_webhook_two_commits(self): + ''' + ''' + data = '''{ + "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "base_ref": null, + "before": "e91fbc420f08890960f50f863626e1062f922522", + "commits": [ + { + "added": [ + "sources/us-ca-san_francisco.json" + ], + "author": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "committer": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "distinct": true, + "id": "73a81c5b337bd393273a222f1cd191d7e634df51", + "message": "Added SF", + "modified": [], + "removed": [], + "timestamp": "2015-04-25T17:25:45-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + }, + { + "added": [ + "sources/us-ca-berkeley.json" + ], + "author": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "committer": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "distinct": true, + "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "message": "Added Berkeley", + "modified": [], + "removed": [], + "timestamp": "2015-04-25T17:25:55-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" + } + ], + "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", + "created": false, + "deleted": false, + "forced": false, + "head_commit": { + "added": [ + "sources/us-ca-berkeley.json" + ], + "author": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "committer": { + "email": "mike@teczno.com", + "name": "Michal Migurski", + "username": "migurski" + }, + "distinct": true, + "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "message": "Added Berkeley", + "modified": [], + "removed": [], + "timestamp": "2015-04-25T17:25:55-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" + }, + "organization": { + "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", + "description": "The free and open global address collection ", + "events_url": "https://api.github.com/orgs/openaddresses/events", + "id": 6895392, + "login": "openaddresses", + "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", + "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", + "repos_url": "https://api.github.com/orgs/openaddresses/repos", + "url": "https://api.github.com/orgs/openaddresses" + }, + "pusher": { + "email": "mike-github@teczno.com", + "name": "migurski" + }, + "ref": "refs/heads/master", + "repository": { + "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", + "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", + "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", + "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", + "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", + "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", + "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", + "created_at": 1430006167, + "default_branch": "master", + "description": "Temporary repository for testing Github webhook features", + "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", + "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", + "fork": false, + "forks": 0, + "forks_count": 0, + "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", + "full_name": "openaddresses/hooked-on-sources", + "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", + "git_url": "git://github.com/openaddresses/hooked-on-sources.git", + "has_downloads": true, + "has_issues": true, + "has_pages": false, + "has_wiki": true, + "homepage": null, + "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", + "html_url": "https://github.com/openaddresses/hooked-on-sources", + "id": 34590951, + "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", + "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", + "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", + "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", + "language": null, + "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", + "master_branch": "master", + "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", + "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", + "mirror_url": null, + "name": "hooked-on-sources", + "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", + "open_issues": 0, + "open_issues_count": 0, + "organization": "openaddresses", + "owner": { + "email": "openaddresses@gmail.com", + "name": "openaddresses" + }, + "private": false, + "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", + "pushed_at": 1430007964, + "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", + "size": 0, + "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", + "stargazers": 0, + "stargazers_count": 0, + "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", + "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", + "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", + "svn_url": "https://github.com/openaddresses/hooked-on-sources", + "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", + "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", + "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", + "updated_at": "2015-04-25T23:56:07Z", + "url": "https://github.com/openaddresses/hooked-on-sources", + "watchers": 0, + "watchers_count": 0 + }, + "sender": { + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "gravatar_id": "", + "html_url": "https://github.com/migurski", + "id": 58730, + "login": "migurski", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "repos_url": "https://api.github.com/users/migurski/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "type": "User", + "url": "https://api.github.com/users/migurski" + } + }''' + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) + + self.assertEqual(posted.status_code, 200) + self.assertTrue('us-ca-san_francisco' in posted.data) + self.assertTrue('data.sfgov.org' in posted.data) + self.assertTrue('us-ca-berkeley' in posted.data) + self.assertTrue('www.ci.berkeley.ca.us' in posted.data) + + def test_webhook_add_remove(self): + ''' + ''' + data = '''{ + "ref": "refs/heads/branch", + "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4", + "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564", + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab", + "commits": [ + { + "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "distinct": true, + "message": "Added Polish source", + "timestamp": "2015-04-25T17:52:39-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "added": [ + "sources/pl-dolnoslaskie.json" + ], + "removed": [ + + ], + "modified": [ + + ] + }, + { + "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564", + "distinct": true, + "message": "Removed Polish source", + "timestamp": "2015-04-25T17:52:46-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "added": [ + + ], + "removed": [ + "sources/pl-dolnoslaskie.json" + ], + "modified": [ + + ] + } + ], + "head_commit": { + "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564", + "distinct": true, + "message": "Removed Polish source", + "timestamp": "2015-04-25T17:52:46-07:00", + "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "username": "migurski" + }, + "added": [ + + ], + "removed": [ + "sources/pl-dolnoslaskie.json" + ], + "modified": [ + + ] + }, + "repository": { + "id": 34590951, + "name": "hooked-on-sources", + "full_name": "openaddresses/hooked-on-sources", + "owner": { + "name": "openaddresses", + "email": "openaddresses@gmail.com" + }, + "private": false, + "html_url": "https://github.com/openaddresses/hooked-on-sources", + "description": "Temporary repository for testing Github webhook features", + "fork": false, + "url": "https://github.com/openaddresses/hooked-on-sources", + "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", + "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", + "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", + "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", + "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", + "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", + "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", + "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", + "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", + "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", + "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", + "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", + "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", + "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", + "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", + "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", + "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", + "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", + "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", + "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", + "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", + "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", + "created_at": 1430006167, + "updated_at": "2015-04-25T23:56:07Z", + "pushed_at": 1430009572, + "git_url": "git://github.com/openaddresses/hooked-on-sources.git", + "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", + "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", + "svn_url": "https://github.com/openaddresses/hooked-on-sources", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": null, + "has_issues": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "open_issues_count": 1, + "forks": 0, + "open_issues": 1, + "watchers": 0, + "default_branch": "master", + "stargazers": 0, + "master_branch": "master", + "organization": "openaddresses" + }, + "pusher": { + "name": "migurski", + "email": "mike-github@teczno.com" + }, + "organization": { + "login": "openaddresses", + "id": 6895392, + "url": "https://api.github.com/orgs/openaddresses", + "repos_url": "https://api.github.com/orgs/openaddresses/repos", + "events_url": "https://api.github.com/orgs/openaddresses/events", + "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", + "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", + "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", + "description": "The free and open global address collection " + }, + "sender": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + } + }''' + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) + + self.assertEqual(posted.status_code, 200) + self.assertFalse('pl-dolnoslaskie' in posted.data) + +if __name__ == '__main__': + unittest.main() From 84213d9c1098fd0ee17c14ad326ee85c5dbe2cba Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 16:13:57 -0700 Subject: [PATCH 07/51] Added Github auth token setting for API usage --- app.py | 23 +++++++++++------------ tests.py | 6 ++++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/app.py b/app.py index 8148df75..4f4c019d 100644 --- a/app.py +++ b/app.py @@ -1,13 +1,13 @@ -from sys import stderr from os.path import relpath, splitext from base64 import b64decode -import json +import json, os -from flask import Flask, request, Response +from flask import Flask, request, Response, current_app from uritemplate import expand from requests import get app = Flask(__name__) +app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' @app.route('/') def index(): @@ -15,7 +15,8 @@ def index(): @app.route('/hook', methods=['GET', 'POST']) def hook(): - files = process_payload(json.loads(request.data)) + github_auth = current_app.config['GITHUB_AUTH'] + files = process_payload(json.loads(request.data), github_auth) serialize = lambda data: json.dumps(data, indent=2, sort_keys=True) response = '\n\n'.join(['{}:\n\n{}\n'.format(name, serialize(data)) @@ -38,19 +39,17 @@ def get_touched_files(payload): # Skip files that no longer exist. touched.remove(filename) - print >> stderr, 'Touched files', ', '.join(touched) + current_app.logger.debug('Touched files {}'.format(', '.join(touched))) return touched -def process_payload(payload): +def process_payload(payload, github_auth): ''' Return a dictionary of file paths and decoded JSON contents. ''' processed, touched = dict(), get_touched_files(payload) commit_sha = payload['head_commit']['id'] - print >> stderr, 'Head commit SHA', commit_sha - for filename in touched: if relpath(filename, 'sources').startswith('..'): # Skip things outside of sources directory. @@ -64,18 +63,18 @@ def process_payload(payload): contents_url = expand(contents_url, dict(path=filename)) contents_url = '{contents_url}?ref={commit_sha}'.format(**locals()) - print >> stderr, 'Contents URL', contents_url + current_app.logger.debug('Contents URL {}'.format(contents_url)) - got = get(contents_url) + got = get(contents_url, auth=github_auth) contents = got.json() if got.status_code not in range(200, 299): - print >> stderr, 'Skipping', filename, '-', got.status_code + current_app.logger.warning('Skipping {} - {}'.format(filename, got.status_code)) continue content, encoding = contents['content'], contents['encoding'] - print >> stderr, 'Contents SHA', contents['sha'] + current_app.logger.debug('Contents SHA {}'.format(contents['sha'])) if encoding == 'base64': processed[filename] = json.loads(b64decode(content)) diff --git a/tests.py b/tests.py index f925a95e..c04ba479 100644 --- a/tests.py +++ b/tests.py @@ -1,7 +1,9 @@ -from app import app from httmock import HTTMock, response from urlparse import parse_qsl -import unittest +import unittest, os + +os.environ['GITHUB_TOKEN'] = '' +from app import app class TestHook (unittest.TestCase): From 3342c9706086d884bb58862129bf5fc574000c4b Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 16:56:18 -0700 Subject: [PATCH 08/51] Pushing commit status back to Github --- app.py | 32 ++++++++++++++++++++++++++-- tests.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 4f4c019d..48d3acba 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,7 @@ from flask import Flask, request, Response, current_app from uritemplate import expand -from requests import get +from requests import get, post app = Flask(__name__) app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' @@ -16,7 +16,9 @@ def index(): @app.route('/hook', methods=['GET', 'POST']) def hook(): github_auth = current_app.config['GITHUB_AUTH'] - files = process_payload(json.loads(request.data), github_auth) + webhook_payload = json.loads(request.data) + files = process_payload(webhook_payload, github_auth) + update_status(webhook_payload, files, github_auth) serialize = lambda data: json.dumps(data, indent=2, sort_keys=True) response = '\n\n'.join(['{}:\n\n{}\n'.format(name, serialize(data)) @@ -83,5 +85,31 @@ def process_payload(payload, github_auth): return processed +def update_status(payload, files, github_auth): + ''' Push pending status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked') + + if files: + status['state'] = 'pending' + status['description'] = 'Checking {}'.format(', '.join(files)) + else: + status['state'] = 'success' + status['description'] = 'Nothing to check' + + commit_sha = payload['head_commit']['id'] + status_url = payload['repository']['statuses_url'] + status_url = expand(status_url, dict(sha=commit_sha)) + + current_app.logger.debug('Status URL {}'.format(status_url)) + + posted = post(status_url, data=json.dumps(status), auth=github_auth, headers={'Content-Type': 'application/json'}) + + if posted.status_code not in range(200, 299): + raise ValueError('Failed status post to {}'.format(commit_sha)) + + if posted.json()['state'] != status['state']: + raise ValueError('Mismatched status post to {}'.format(commit_sha)) + if __name__ == '__main__': app.run(debug=True) diff --git a/tests.py b/tests.py index c04ba479..db342daa 100644 --- a/tests.py +++ b/tests.py @@ -1,6 +1,9 @@ +from __future__ import print_function + from httmock import HTTMock, response from urlparse import parse_qsl -import unittest, os +from os.path import basename +import unittest, json, os, sys os.environ['GITHUB_TOKEN'] = '' from app import app @@ -16,8 +19,10 @@ def response_content(self, url, request): ''' ''' query = dict(parse_qsl(url.query)) + MHP = request.method, url.hostname, url.path + response_headers = {'Content-Type': 'application/json; charset=utf-8'} - if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): data = '''{ "name": "us-ca-alameda_county.json", "path": "sources/us-ca-alameda_county.json", @@ -37,9 +42,9 @@ def response_content(self, url, request): } }''' - return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + return response(200, data, headers=response_headers) - if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): data = '''{ "name": "us-ca-san_francisco.json", "path": "sources/us-ca-san_francisco.json", @@ -59,9 +64,9 @@ def response_content(self, url, request): } }''' - return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + return response(200, data, headers=response_headers) - if (url.hostname, url.path) == ('api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): data = '''{ "name": "us-ca-berkeley.json", "path": "sources/us-ca-berkeley.json", @@ -81,8 +86,54 @@ def response_content(self, url, request): } }''' - return response(200, data, headers={'Content-Type': 'application/json; charset=utf-8'}) + return response(200, data, headers=response_headers) + + if MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ + or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ + or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): + input = json.loads(request.body) + states = { + 'e5f1dcae83ab1ef1f736b969da617311f7f11564': 'success', + 'e91fbc420f08890960f50f863626e1062f922522': 'pending', + 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'pending' + } + + self.assertEqual(input['context'], 'openaddresses/hooked') + self.assertEqual(input['state'], states[basename(url.path)]) + + data = '''{{ + "context": "openaddresses/hooked", + "created_at": "2015-04-26T23:45:39Z", + "creator": {{ + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", + "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", + "gravatar_id": "", + "html_url": "https://github.com/migurski", + "id": 58730, + "login": "migurski", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "repos_url": "https://api.github.com/users/migurski/repos", + "site_admin": false, + "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "type": "User", + "url": "https://api.github.com/users/migurski" + }}, + "description": "Checking ", + "id": 999999999, + "state": "{state}", + "target_url": null, + "updated_at": "2015-04-26T23:45:39Z", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx" + }}''' + + return response(201, data.format(**input), headers=response_headers) + print('Unknowable URL "{}"'.format(url.geturl()), file=sys.stderr) raise ValueError('Unknowable URL "{}"'.format(url.geturl())) def test_webhook_one_commit(self): From 547218e0f93157f301e5ecdb20748fb199e2f67f Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 17:59:27 -0700 Subject: [PATCH 09/51] Switched to whole-branch tests that no longer work on master commits --- app.py | 21 +- tests.py | 747 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 756 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 48d3acba..4b333b2a 100644 --- a/app.py +++ b/app.py @@ -26,7 +26,7 @@ def hook(): return Response(response, headers={'Content-Type': 'text/plain'}) -def get_touched_files(payload): +def get_touched_payload_files(payload): ''' Return a set of files modified in payload commits. ''' touched = set() @@ -45,10 +45,27 @@ def get_touched_files(payload): return touched +def get_touched_branch_files(payload, github_auth): + ''' Return a set of files modified between master and payload head. + ''' + commit_sha = payload['head_commit']['id'] + compare_url = payload['repository']['compare_url'] + compare_url = expand(compare_url, dict(base='master', head=commit_sha)) + + current_app.logger.debug('Compare URL {}'.format(compare_url)) + + got = get(compare_url, auth=github_auth) + compare = got.json() + + touched = set([file['filename'] for file in compare['files']]) + current_app.logger.debug('Touched files {}'.format(', '.join(touched))) + + return touched + def process_payload(payload, github_auth): ''' Return a dictionary of file paths and decoded JSON contents. ''' - processed, touched = dict(), get_touched_files(payload) + processed, touched = dict(), get_touched_branch_files(payload, github_auth) commit_sha = payload['head_commit']['id'] diff --git a/tests.py b/tests.py index db342daa..89145620 100644 --- a/tests.py +++ b/tests.py @@ -88,18 +88,743 @@ def response_content(self, url, request): return response(200, data, headers=response_headers) + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): + data = '''{ + "name": "us-ca-santa_clara_county.json", + "path": "sources/us-ca-santa_clara_county.json", + "sha": "84abff13dba318189f1f4d5c1605478127ceff5c", + "size": 908, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", + "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", + "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c", + "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", + "type": "file", + "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n", + "encoding": "base64", + "_links": { + "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", + "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c", + "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json" + } + }''' + + return response(200, data, headers=response_headers) + + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): + data = '''{ + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564", + "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564", + "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca", + "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff", + "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch", + "base_commit": { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "message": "Added Berkeley", + "tree": { + "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + } + ] + }, + "merge_base_commit": { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "message": "Added Berkeley", + "tree": { + "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + } + ] + }, + "status": "ahead", + "ahead_by": 3, + "behind_by": 0, + "total_commits": 3, + "commits": [ + { + "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:48:58Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:48:58Z" + }, + "message": "Added Santa Clara County", + "tree": { + "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" + } + ] + }, + { + "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:52:39Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:52:39Z" + }, + "message": "Added Polish source", + "tree": { + "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4" + } + ] + }, + { + "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:52:46Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:52:46Z" + }, + "message": "Removed Polish source", + "tree": { + "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035" + } + ] + } + ], + "files": [ + { + "sha": "84abff13dba318189f1f4d5c1605478127ceff5c", + "filename": "sources/us-ca-santa_clara_county.json", + "status": "added", + "additions": 24, + "deletions": 0, + "changes": 24, + "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", + "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", + "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", + "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}" + } + ] + }''' + + return response(200, data, headers=response_headers) + + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): + data = '''{ + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522", + "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522", + "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4", + "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff", + "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch", + "base_commit": { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "message": "Added Berkeley", + "tree": { + "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + } + ] + }, + "merge_base_commit": { + "sha": "e91fbc420f08890960f50f863626e1062f922522", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:16:12Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:16:12Z" + }, + "message": "Added first source", + "tree": { + "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "c52204fd40f17f9da243df09e6d1107d48768afd", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd" + } + ] + }, + "status": "behind", + "ahead_by": 0, + "behind_by": 2, + "total_commits": 0, + "commits": [ + + ], + "files": [ + + ] + }''' + + return response(200, data, headers=response_headers) + + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): + data = '''{ + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed", + "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff", + "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch", + "base_commit": { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "message": "Added Berkeley", + "tree": { + "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + } + ] + }, + "merge_base_commit": { + "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "commit": { + "author": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "committer": { + "name": "Michal Migurski", + "email": "mike@teczno.com", + "date": "2015-04-26T00:25:55Z" + }, + "message": "Added Berkeley", + "tree": { + "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comment_count": 0 + }, + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", + "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", + "author": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "migurski", + "id": 58730, + "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/migurski", + "html_url": "https://github.com/migurski", + "followers_url": "https://api.github.com/users/migurski/followers", + "following_url": "https://api.github.com/users/migurski/following{/other_user}", + "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", + "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", + "organizations_url": "https://api.github.com/users/migurski/orgs", + "repos_url": "https://api.github.com/users/migurski/repos", + "events_url": "https://api.github.com/users/migurski/events{/privacy}", + "received_events_url": "https://api.github.com/users/migurski/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", + "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", + "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" + } + ] + }, + "status": "identical", + "ahead_by": 0, + "behind_by": 0, + "total_commits": 0, + "commits": [ + + ], + "files": [ + + ] + }''' + + return response(200, data, headers=response_headers) + if MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): input = json.loads(request.body) states = { - 'e5f1dcae83ab1ef1f736b969da617311f7f11564': 'success', - 'e91fbc420f08890960f50f863626e1062f922522': 'pending', - 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'pending' + 'e5f1dcae83ab1ef1f736b969da617311f7f11564': 'pending', + 'e91fbc420f08890960f50f863626e1062f922522': 'success', + 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'success' } self.assertEqual(input['context'], 'openaddresses/hooked') - self.assertEqual(input['state'], states[basename(url.path)]) + self.assertEqual(input['state'], states[basename(url.path)], 'Bad state "{}" for {}'.format(input['state'], url.geturl())) data = '''{{ "context": "openaddresses/hooked", @@ -309,8 +1034,8 @@ def test_webhook_one_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertTrue('us-ca-alameda_county' in posted.data) - self.assertTrue('data.acgov.org' in posted.data) + self.assertFalse('us-ca-alameda_county' in posted.data) + self.assertFalse('data.acgov.org' in posted.data) def test_webhook_two_commits(self): ''' @@ -507,10 +1232,10 @@ def test_webhook_two_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertTrue('us-ca-san_francisco' in posted.data) - self.assertTrue('data.sfgov.org' in posted.data) - self.assertTrue('us-ca-berkeley' in posted.data) - self.assertTrue('www.ci.berkeley.ca.us' in posted.data) + self.assertFalse('us-ca-san_francisco' in posted.data) + self.assertFalse('data.sfgov.org' in posted.data) + self.assertFalse('us-ca-berkeley' in posted.data) + self.assertFalse('www.ci.berkeley.ca.us' in posted.data) def test_webhook_add_remove(self): ''' @@ -719,6 +1444,8 @@ def test_webhook_add_remove(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) + self.assertTrue('us-ca-santa_clara_county' in posted.data) + self.assertTrue('sftp.sccgov.org' in posted.data) self.assertFalse('pl-dolnoslaskie' in posted.data) if __name__ == '__main__': From 5b6dde842011a14ffae0ac1cf66a51b5845312d1 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 18:02:18 -0700 Subject: [PATCH 10/51] Tightened test code --- tests.py | 1377 +----------------------------------------------------- 1 file changed, 25 insertions(+), 1352 deletions(-) diff --git a/tests.py b/tests.py index 89145620..6823354c 100644 --- a/tests.py +++ b/tests.py @@ -1,6 +1,7 @@ from __future__ import print_function from httmock import HTTMock, response +from logging import StreamHandler, DEBUG from urlparse import parse_qsl from os.path import basename import unittest, json, os, sys @@ -14,6 +15,10 @@ def setUp(self): ''' ''' self.client = app.test_client() + + handler = StreamHandler(stream=sys.stderr) + handler.setLevel(DEBUG) + app.logger.addHandler(handler) def response_content(self, url, request): ''' @@ -23,793 +28,37 @@ def response_content(self, url, request): response_headers = {'Content-Type': 'application/json; charset=utf-8'} if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): - data = '''{ - "name": "us-ca-alameda_county.json", - "path": "sources/us-ca-alameda_county.json", - "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8", - "size": 745, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522", - "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json", - "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8", - "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json", - "type": "file", - "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n", - "encoding": "base64", - "_links": { - "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522", - "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8", - "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json" - } - }''' + data = '''{\r "name": "us-ca-alameda_county.json",\r "path": "sources/us-ca-alameda_county.json",\r "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "size": 745,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json"\r }\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): - data = '''{ - "name": "us-ca-san_francisco.json", - "path": "sources/us-ca-san_francisco.json", - "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff", - "size": 519, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json", - "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff", - "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json", - "type": "file", - "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n", - "encoding": "base64", - "_links": { - "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff", - "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json" - } - }''' + data = '''{\r "name": "us-ca-san_francisco.json",\r "path": "sources/us-ca-san_francisco.json",\r "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "size": 519,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json"\r }\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): - data = '''{ - "name": "us-ca-berkeley.json", - "path": "sources/us-ca-berkeley.json", - "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad", - "size": 779, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json", - "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad", - "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json", - "type": "file", - "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n", - "encoding": "base64", - "_links": { - "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad", - "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json" - } - }''' + data = '''{\r "name": "us-ca-berkeley.json",\r "path": "sources/us-ca-berkeley.json",\r "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "size": 779,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json"\r }\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): - data = '''{ - "name": "us-ca-santa_clara_county.json", - "path": "sources/us-ca-santa_clara_county.json", - "sha": "84abff13dba318189f1f4d5c1605478127ceff5c", - "size": 908, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", - "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", - "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c", - "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", - "type": "file", - "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n", - "encoding": "base64", - "_links": { - "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", - "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c", - "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json" - } - }''' + data = '''{\r "name": "us-ca-santa_clara_county.json",\r "path": "sources/us-ca-santa_clara_county.json",\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "size": 908,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json"\r }\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): - data = '''{ - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564", - "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564", - "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca", - "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff", - "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch", - "base_commit": { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "message": "Added Berkeley", - "tree": { - "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - } - ] - }, - "merge_base_commit": { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "message": "Added Berkeley", - "tree": { - "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - } - ] - }, - "status": "ahead", - "ahead_by": 3, - "behind_by": 0, - "total_commits": 3, - "commits": [ - { - "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:48:58Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:48:58Z" - }, - "message": "Added Santa Clara County", - "tree": { - "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" - } - ] - }, - { - "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:52:39Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:52:39Z" - }, - "message": "Added Polish source", - "tree": { - "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4" - } - ] - }, - { - "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:52:46Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:52:46Z" - }, - "message": "Removed Polish source", - "tree": { - "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035" - } - ] - } - ], - "files": [ - { - "sha": "84abff13dba318189f1f4d5c1605478127ceff5c", - "filename": "sources/us-ca-santa_clara_county.json", - "status": "added", - "additions": 24, - "deletions": 0, - "changes": 24, - "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", - "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json", - "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564", - "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}" - } - ] - }''' + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "ahead",\r "ahead_by": 3,\r "behind_by": 0,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): - data = '''{ - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522", - "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522", - "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4", - "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff", - "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch", - "base_commit": { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "message": "Added Berkeley", - "tree": { - "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - } - ] - }, - "merge_base_commit": { - "sha": "e91fbc420f08890960f50f863626e1062f922522", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:16:12Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:16:12Z" - }, - "message": "Added first source", - "tree": { - "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "c52204fd40f17f9da243df09e6d1107d48768afd", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd" - } - ] - }, - "status": "behind", - "ahead_by": 0, - "behind_by": 2, - "total_commits": 0, - "commits": [ - - ], - "files": [ - - ] - }''' + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "e91fbc420f08890960f50f863626e1062f922522",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "message": "Added first source",\r "tree": {\r "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "c52204fd40f17f9da243df09e6d1107d48768afd",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd"\r }\r ]\r },\r "status": "behind",\r "ahead_by": 0,\r "behind_by": 2,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): - data = '''{ - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed", - "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff", - "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch", - "base_commit": { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "message": "Added Berkeley", - "tree": { - "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - } - ] - }, - "merge_base_commit": { - "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "commit": { - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "date": "2015-04-26T00:25:55Z" - }, - "message": "Added Berkeley", - "tree": { - "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76" - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comment_count": 0 - }, - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments", - "author": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "committer": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - }, - "parents": [ - { - "sha": "73a81c5b337bd393273a222f1cd191d7e634df51", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51", - "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - } - ] - }, - "status": "identical", - "ahead_by": 0, - "behind_by": 0, - "total_commits": 0, - "commits": [ - - ], - "files": [ - - ] - }''' + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' return response(200, data, headers=response_headers) @@ -826,35 +75,7 @@ def response_content(self, url, request): self.assertEqual(input['context'], 'openaddresses/hooked') self.assertEqual(input['state'], states[basename(url.path)], 'Bad state "{}" for {}'.format(input['state'], url.geturl())) - data = '''{{ - "context": "openaddresses/hooked", - "created_at": "2015-04-26T23:45:39Z", - "creator": {{ - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", - "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", - "gravatar_id": "", - "html_url": "https://github.com/migurski", - "id": 58730, - "login": "migurski", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "repos_url": "https://api.github.com/users/migurski/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "type": "User", - "url": "https://api.github.com/users/migurski" - }}, - "description": "Checking ", - "id": 999999999, - "state": "{state}", - "target_url": null, - "updated_at": "2015-04-26T23:45:39Z", - "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx" - }}''' + data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' return response(201, data.format(**input), headers=response_headers) @@ -864,589 +85,41 @@ def response_content(self, url, request): def test_webhook_one_commit(self): ''' ''' - data = '''{ - "after": "e91fbc420f08890960f50f863626e1062f922522", - "base_ref": null, - "before": "c52204fd40f17f9da243df09e6d1107d48768afd", - "commits": [ - { - "added": [ - "sources/us-ca-alameda_county.json" - ], - "author": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "committer": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "distinct": true, - "id": "e91fbc420f08890960f50f863626e1062f922522", - "message": "Added first source", - "modified": [], - "removed": [], - "timestamp": "2015-04-25T17:16:12-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522" - } - ], - "compare": "https://github.com/openaddresses/hooked-on-sources/compare/c52204fd40f1...e91fbc420f08", - "created": false, - "deleted": false, - "forced": false, - "head_commit": { - "added": [ - "sources/us-ca-alameda_county.json" - ], - "author": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "committer": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "distinct": true, - "id": "e91fbc420f08890960f50f863626e1062f922522", - "message": "Added first source", - "modified": [], - "removed": [], - "timestamp": "2015-04-25T17:16:12-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522" - }, - "organization": { - "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", - "description": "The free and open global address collection ", - "events_url": "https://api.github.com/orgs/openaddresses/events", - "id": 6895392, - "login": "openaddresses", - "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", - "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", - "repos_url": "https://api.github.com/orgs/openaddresses/repos", - "url": "https://api.github.com/orgs/openaddresses" - }, - "pusher": { - "email": "mike-github@teczno.com", - "name": "migurski" - }, - "ref": "refs/heads/master", - "repository": { - "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", - "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", - "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", - "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", - "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", - "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", - "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", - "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", - "created_at": 1430006167, - "default_branch": "master", - "description": "Temporary repository for testing Github webhook features", - "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", - "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", - "fork": false, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", - "full_name": "openaddresses/hooked-on-sources", - "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", - "git_url": "git://github.com/openaddresses/hooked-on-sources.git", - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", - "html_url": "https://github.com/openaddresses/hooked-on-sources", - "id": 34590951, - "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", - "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", - "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", - "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", - "language": null, - "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", - "master_branch": "master", - "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", - "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", - "mirror_url": null, - "name": "hooked-on-sources", - "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", - "open_issues": 0, - "open_issues_count": 0, - "organization": "openaddresses", - "owner": { - "email": "openaddresses@gmail.com", - "name": "openaddresses" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", - "pushed_at": 1430007676, - "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", - "size": 0, - "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", - "stargazers": 0, - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", - "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", - "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", - "svn_url": "https://github.com/openaddresses/hooked-on-sources", - "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", - "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", - "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", - "updated_at": "2015-04-25T23:56:07Z", - "url": "https://github.com/openaddresses/hooked-on-sources", - "watchers": 0, - "watchers_count": 0 - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/migurski", - "id": 58730, - "login": "migurski", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "repos_url": "https://api.github.com/users/migurski/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "type": "User", - "url": "https://api.github.com/users/migurski" - } - }''' + data = '''{\r "after": "e91fbc420f08890960f50f863626e1062f922522", \r "base_ref": null, \r "before": "c52204fd40f17f9da243df09e6d1107d48768afd", \r "commits": [\r {\r "added": [\r "sources/us-ca-alameda_county.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "e91fbc420f08890960f50f863626e1062f922522", \r "message": "Added first source", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:16:12-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/c52204fd40f1...e91fbc420f08", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-alameda_county.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "e91fbc420f08890960f50f863626e1062f922522", \r "message": "Added first source", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:16:12-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007676, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' with HTTMock(self.response_content): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertFalse('us-ca-alameda_county' in posted.data) - self.assertFalse('data.acgov.org' in posted.data) + self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') + self.assertFalse('data.acgov.org' in posted.data, 'Alameda County domain name should be absent from master commit') def test_webhook_two_commits(self): ''' ''' - data = '''{ - "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "base_ref": null, - "before": "e91fbc420f08890960f50f863626e1062f922522", - "commits": [ - { - "added": [ - "sources/us-ca-san_francisco.json" - ], - "author": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "committer": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "distinct": true, - "id": "73a81c5b337bd393273a222f1cd191d7e634df51", - "message": "Added SF", - "modified": [], - "removed": [], - "timestamp": "2015-04-25T17:25:45-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51" - }, - { - "added": [ - "sources/us-ca-berkeley.json" - ], - "author": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "committer": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "distinct": true, - "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "message": "Added Berkeley", - "modified": [], - "removed": [], - "timestamp": "2015-04-25T17:25:55-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" - } - ], - "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", - "created": false, - "deleted": false, - "forced": false, - "head_commit": { - "added": [ - "sources/us-ca-berkeley.json" - ], - "author": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "committer": { - "email": "mike@teczno.com", - "name": "Michal Migurski", - "username": "migurski" - }, - "distinct": true, - "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", - "message": "Added Berkeley", - "modified": [], - "removed": [], - "timestamp": "2015-04-25T17:25:55-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa" - }, - "organization": { - "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", - "description": "The free and open global address collection ", - "events_url": "https://api.github.com/orgs/openaddresses/events", - "id": 6895392, - "login": "openaddresses", - "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", - "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", - "repos_url": "https://api.github.com/orgs/openaddresses/repos", - "url": "https://api.github.com/orgs/openaddresses" - }, - "pusher": { - "email": "mike-github@teczno.com", - "name": "migurski" - }, - "ref": "refs/heads/master", - "repository": { - "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", - "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", - "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", - "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", - "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", - "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", - "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", - "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", - "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", - "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", - "created_at": 1430006167, - "default_branch": "master", - "description": "Temporary repository for testing Github webhook features", - "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", - "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", - "fork": false, - "forks": 0, - "forks_count": 0, - "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", - "full_name": "openaddresses/hooked-on-sources", - "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", - "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", - "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", - "git_url": "git://github.com/openaddresses/hooked-on-sources.git", - "has_downloads": true, - "has_issues": true, - "has_pages": false, - "has_wiki": true, - "homepage": null, - "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", - "html_url": "https://github.com/openaddresses/hooked-on-sources", - "id": 34590951, - "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", - "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", - "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", - "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", - "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", - "language": null, - "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", - "master_branch": "master", - "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", - "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", - "mirror_url": null, - "name": "hooked-on-sources", - "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", - "open_issues": 0, - "open_issues_count": 0, - "organization": "openaddresses", - "owner": { - "email": "openaddresses@gmail.com", - "name": "openaddresses" - }, - "private": false, - "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", - "pushed_at": 1430007964, - "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", - "size": 0, - "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", - "stargazers": 0, - "stargazers_count": 0, - "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", - "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", - "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", - "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", - "svn_url": "https://github.com/openaddresses/hooked-on-sources", - "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", - "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", - "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", - "updated_at": "2015-04-25T23:56:07Z", - "url": "https://github.com/openaddresses/hooked-on-sources", - "watchers": 0, - "watchers_count": 0 - }, - "sender": { - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "gravatar_id": "", - "html_url": "https://github.com/migurski", - "id": 58730, - "login": "migurski", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "repos_url": "https://api.github.com/users/migurski/repos", - "site_admin": false, - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "type": "User", - "url": "https://api.github.com/users/migurski" - } - }''' + data = '''{\r "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "base_ref": null, \r "before": "e91fbc420f08890960f50f863626e1062f922522", \r "commits": [\r {\r "added": [\r "sources/us-ca-san_francisco.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "73a81c5b337bd393273a222f1cd191d7e634df51", \r "message": "Added SF", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:45-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }, \r {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007964, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' with HTTMock(self.response_content): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertFalse('us-ca-san_francisco' in posted.data) - self.assertFalse('data.sfgov.org' in posted.data) - self.assertFalse('us-ca-berkeley' in posted.data) - self.assertFalse('www.ci.berkeley.ca.us' in posted.data) + self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') + self.assertFalse('data.sfgov.org' in posted.data, 'San Francisco URL should be absent from master commit') + self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') + self.assertFalse('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be absent from master commit') def test_webhook_add_remove(self): ''' ''' - data = '''{ - "ref": "refs/heads/branch", - "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4", - "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564", - "created": false, - "deleted": false, - "forced": false, - "base_ref": null, - "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab", - "commits": [ - { - "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "distinct": true, - "message": "Added Polish source", - "timestamp": "2015-04-25T17:52:39-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035", - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "added": [ - "sources/pl-dolnoslaskie.json" - ], - "removed": [ - - ], - "modified": [ - - ] - }, - { - "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564", - "distinct": true, - "message": "Removed Polish source", - "timestamp": "2015-04-25T17:52:46-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "added": [ - - ], - "removed": [ - "sources/pl-dolnoslaskie.json" - ], - "modified": [ - - ] - } - ], - "head_commit": { - "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564", - "distinct": true, - "message": "Removed Polish source", - "timestamp": "2015-04-25T17:52:46-07:00", - "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564", - "author": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "committer": { - "name": "Michal Migurski", - "email": "mike@teczno.com", - "username": "migurski" - }, - "added": [ - - ], - "removed": [ - "sources/pl-dolnoslaskie.json" - ], - "modified": [ - - ] - }, - "repository": { - "id": 34590951, - "name": "hooked-on-sources", - "full_name": "openaddresses/hooked-on-sources", - "owner": { - "name": "openaddresses", - "email": "openaddresses@gmail.com" - }, - "private": false, - "html_url": "https://github.com/openaddresses/hooked-on-sources", - "description": "Temporary repository for testing Github webhook features", - "fork": false, - "url": "https://github.com/openaddresses/hooked-on-sources", - "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", - "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", - "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", - "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", - "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", - "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", - "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", - "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", - "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", - "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", - "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", - "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", - "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", - "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", - "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", - "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", - "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", - "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", - "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", - "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", - "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", - "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", - "created_at": 1430006167, - "updated_at": "2015-04-25T23:56:07Z", - "pushed_at": 1430009572, - "git_url": "git://github.com/openaddresses/hooked-on-sources.git", - "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", - "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", - "svn_url": "https://github.com/openaddresses/hooked-on-sources", - "homepage": null, - "size": 0, - "stargazers_count": 0, - "watchers_count": 0, - "language": null, - "has_issues": true, - "has_downloads": true, - "has_wiki": true, - "has_pages": false, - "forks_count": 0, - "mirror_url": null, - "open_issues_count": 1, - "forks": 0, - "open_issues": 1, - "watchers": 0, - "default_branch": "master", - "stargazers": 0, - "master_branch": "master", - "organization": "openaddresses" - }, - "pusher": { - "name": "migurski", - "email": "mike-github@teczno.com" - }, - "organization": { - "login": "openaddresses", - "id": 6895392, - "url": "https://api.github.com/orgs/openaddresses", - "repos_url": "https://api.github.com/orgs/openaddresses/repos", - "events_url": "https://api.github.com/orgs/openaddresses/events", - "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", - "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", - "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", - "description": "The free and open global address collection " - }, - "sender": { - "login": "migurski", - "id": 58730, - "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", - "gravatar_id": "", - "url": "https://api.github.com/users/migurski", - "html_url": "https://github.com/migurski", - "followers_url": "https://api.github.com/users/migurski/followers", - "following_url": "https://api.github.com/users/migurski/following{/other_user}", - "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", - "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", - "organizations_url": "https://api.github.com/users/migurski/orgs", - "repos_url": "https://api.github.com/users/migurski/repos", - "events_url": "https://api.github.com/users/migurski/events{/privacy}", - "received_events_url": "https://api.github.com/users/migurski/received_events", - "type": "User", - "site_admin": false - } - }''' + data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' with HTTMock(self.response_content): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertTrue('us-ca-santa_clara_county' in posted.data) - self.assertTrue('sftp.sccgov.org' in posted.data) - self.assertFalse('pl-dolnoslaskie' in posted.data) + self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') + self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') + self.assertFalse('pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') if __name__ == '__main__': unittest.main() From 2322f63c74f8b72a61d254262b70e2306f5eb8f6 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 18:42:24 -0700 Subject: [PATCH 11/51] Modified branch/master behavior * For a set of commits pushed directly to master, we look at any changed files in those commits. * For a set of commits pushed to a branch, we look at all changed files since it diverged from master. --- app.py | 28 +++++++++++++++++-------- tests.py | 62 +++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/app.py b/app.py index 4b333b2a..6e3e2eac 100644 --- a/app.py +++ b/app.py @@ -48,16 +48,25 @@ def get_touched_payload_files(payload): def get_touched_branch_files(payload, github_auth): ''' Return a set of files modified between master and payload head. ''' - commit_sha = payload['head_commit']['id'] - compare_url = payload['repository']['compare_url'] - compare_url = expand(compare_url, dict(base='master', head=commit_sha)) + branch_sha = payload['head_commit']['id'] + + compare1_url = payload['repository']['compare_url'] + compare1_url = expand(compare1_url, dict(base='master', head=branch_sha)) + current_app.logger.debug('Compare URL 1 {}'.format(compare1_url)) - current_app.logger.debug('Compare URL {}'.format(compare_url)) + compare1 = get(compare1_url, auth=github_auth).json() + merge_base_sha = compare1['merge_base_commit']['sha'] - got = get(compare_url, auth=github_auth) - compare = got.json() + # That's no branch. + if merge_base_sha == branch_sha: + return set() + + compare2_url = payload['repository']['compare_url'] + compare2_url = expand(compare2_url, dict(base=merge_base_sha, head=branch_sha)) + current_app.logger.debug('Compare URL 2 {}'.format(compare2_url)) - touched = set([file['filename'] for file in compare['files']]) + compare2 = get(compare2_url, auth=github_auth).json() + touched = set([file['filename'] for file in compare2['files']]) current_app.logger.debug('Touched files {}'.format(', '.join(touched))) return touched @@ -65,7 +74,10 @@ def get_touched_branch_files(payload, github_auth): def process_payload(payload, github_auth): ''' Return a dictionary of file paths and decoded JSON contents. ''' - processed, touched = dict(), get_touched_branch_files(payload, github_auth) + processed = dict() + + touched = get_touched_payload_files(payload) + touched |= get_touched_branch_files(payload, github_auth) commit_sha = payload['head_commit']['id'] diff --git a/tests.py b/tests.py index 6823354c..89c3cb3f 100644 --- a/tests.py +++ b/tests.py @@ -29,54 +29,64 @@ def response_content(self, url, request): if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): data = '''{\r "name": "us-ca-alameda_county.json",\r "path": "sources/us-ca-alameda_county.json",\r "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "size": 745,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json"\r }\r }''' - return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-san_francisco.json",\r "path": "sources/us-ca-san_francisco.json",\r "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "size": 519,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json"\r }\r }''' - return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-berkeley.json",\r "path": "sources/us-ca-berkeley.json",\r "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "size": 779,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json"\r }\r }''' - return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): data = '''{\r "name": "us-ca-santa_clara_county.json",\r "path": "sources/us-ca-santa_clara_county.json",\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "size": 908,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json"\r }\r }''' - return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): - data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "ahead",\r "ahead_by": 3,\r "behind_by": 0,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r }''' - + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:b450dcb...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "message": "Added Contra Costa county",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "diverged",\r "ahead_by": 3,\r "behind_by": 1,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "e91fbc420f08890960f50f863626e1062f922522",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "message": "Added first source",\r "tree": {\r "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "c52204fd40f17f9da243df09e6d1107d48768afd",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd"\r }\r ]\r },\r "status": "behind",\r "ahead_by": 0,\r "behind_by": 2,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' - return response(200, data, headers=response_headers) if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' - + return response(200, data, headers=response_headers) + + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564'): + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "ahead",\r "ahead_by": 3,\r "behind_by": 0,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' + return response(200, data, headers=response_headers) + + if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637'): + data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:3147668...openaddresses:3147668",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.patch",\r "base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r}''' return response(200, data, headers=response_headers) if MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ - or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): + or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa') \ + or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/3147668047a0bec6d481c0e42995f7c5e5eac637'): input = json.loads(request.body) + states = { + # Complete branch with new Santa Clara and added/removed Polish source. 'e5f1dcae83ab1ef1f736b969da617311f7f11564': 'pending', - 'e91fbc420f08890960f50f863626e1062f922522': 'success', - 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'success' + + # Added San Francisco directly to master. + 'e91fbc420f08890960f50f863626e1062f922522': 'pending', + + # Added Berkeley directly to master. + 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'pending', + + # Empty commit directly to master. + '3147668047a0bec6d481c0e42995f7c5e5eac637': 'success', } self.assertEqual(input['context'], 'openaddresses/hooked') self.assertEqual(input['state'], states[basename(url.path)], 'Bad state "{}" for {}'.format(input['state'], url.geturl())) data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' - return response(201, data.format(**input), headers=response_headers) print('Unknowable URL "{}"'.format(url.geturl()), file=sys.stderr) @@ -91,8 +101,8 @@ def test_webhook_one_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') - self.assertFalse('data.acgov.org' in posted.data, 'Alameda County domain name should be absent from master commit') + self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') + self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') def test_webhook_two_commits(self): ''' @@ -103,12 +113,12 @@ def test_webhook_two_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') - self.assertFalse('data.sfgov.org' in posted.data, 'San Francisco URL should be absent from master commit') - self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') - self.assertFalse('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be absent from master commit') + self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') + self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') + self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') + self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') - def test_webhook_add_remove(self): + def test_webhook_branch_commit(self): ''' ''' data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' @@ -119,7 +129,21 @@ def test_webhook_add_remove(self): self.assertEqual(posted.status_code, 200) self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') + self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') self.assertFalse('pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') + def test_webhook_empty_commit(self): + ''' + ''' + data = '''{\r "ref": "refs/heads/master",\r "before": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "after": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b450dcbb6f4c...3147668047a0",\r "commits": [\r {\r "id": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "distinct": true,\r "message": "Empty commit",\r "timestamp": "2015-04-26T18:36:22-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "distinct": true,\r "message": "Empty commit",\r "timestamp": "2015-04-26T18:36:22-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430098587,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r}''' + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) + + self.assertEqual(posted.status_code, 200) + self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from master commit') + self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') + self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') + if __name__ == '__main__': unittest.main() From 4cf82829edb57fe515ca68e7e0bd6cd1cef3ef22 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 26 Apr 2015 18:50:12 -0700 Subject: [PATCH 12/51] Tightened test code --- tests.py | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/tests.py b/tests.py index 89c3cb3f..0454fe6d 100644 --- a/tests.py +++ b/tests.py @@ -24,49 +24,49 @@ def response_content(self, url, request): ''' ''' query = dict(parse_qsl(url.query)) - MHP = request.method, url.hostname, url.path + GH, MHP = 'api.github.com', (request.method, url.hostname, url.path) response_headers = {'Content-Type': 'application/json; charset=utf-8'} - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): data = '''{\r "name": "us-ca-alameda_county.json",\r "path": "sources/us-ca-alameda_county.json",\r "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "size": 745,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json"\r }\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-san_francisco.json",\r "path": "sources/us-ca-san_francisco.json",\r "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "size": 519,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json"\r }\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-berkeley.json",\r "path": "sources/us-ca-berkeley.json",\r "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "size": 779,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json"\r }\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): data = '''{\r "name": "us-ca-santa_clara_county.json",\r "path": "sources/us-ca-santa_clara_county.json",\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "size": 908,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json"\r }\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:b450dcb...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "message": "Added Contra Costa county",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "diverged",\r "ahead_by": 3,\r "behind_by": 1,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "e91fbc420f08890960f50f863626e1062f922522",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "message": "Added first source",\r "tree": {\r "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "c52204fd40f17f9da243df09e6d1107d48768afd",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd"\r }\r ]\r },\r "status": "behind",\r "ahead_by": 0,\r "behind_by": 2,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "ahead",\r "ahead_by": 3,\r "behind_by": 0,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' return response(200, data, headers=response_headers) - if MHP == ('GET', 'api.github.com', '/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637'): + if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:3147668...openaddresses:3147668",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.patch",\r "base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r}''' return response(200, data, headers=response_headers) - if MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ - or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ - or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa') \ - or MHP == ('POST', 'api.github.com', '/repos/openaddresses/hooked-on-sources/statuses/3147668047a0bec6d481c0e42995f7c5e5eac637'): + if MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ + or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ + or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa') \ + or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/3147668047a0bec6d481c0e42995f7c5e5eac637'): input = json.loads(request.body) states = { @@ -92,8 +92,8 @@ def response_content(self, url, request): print('Unknowable URL "{}"'.format(url.geturl()), file=sys.stderr) raise ValueError('Unknowable URL "{}"'.format(url.geturl())) - def test_webhook_one_commit(self): - ''' + def test_webhook_one_master_commit(self): + ''' Push a single commit with Alameda County source directly to master. ''' data = '''{\r "after": "e91fbc420f08890960f50f863626e1062f922522", \r "base_ref": null, \r "before": "c52204fd40f17f9da243df09e6d1107d48768afd", \r "commits": [\r {\r "added": [\r "sources/us-ca-alameda_county.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "e91fbc420f08890960f50f863626e1062f922522", \r "message": "Added first source", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:16:12-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/c52204fd40f1...e91fbc420f08", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-alameda_county.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "e91fbc420f08890960f50f863626e1062f922522", \r "message": "Added first source", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:16:12-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007676, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' @@ -104,8 +104,8 @@ def test_webhook_one_commit(self): self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') - def test_webhook_two_commits(self): - ''' + def test_webhook_two_master_commits(self): + ''' Push two commits with San Francisco and Berkeley sources directly to master. ''' data = '''{\r "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "base_ref": null, \r "before": "e91fbc420f08890960f50f863626e1062f922522", \r "commits": [\r {\r "added": [\r "sources/us-ca-san_francisco.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "73a81c5b337bd393273a222f1cd191d7e634df51", \r "message": "Added SF", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:45-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }, \r {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007964, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' @@ -117,9 +117,13 @@ def test_webhook_two_commits(self): self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') + self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') - def test_webhook_branch_commit(self): - ''' + def test_webhook_two_branch_commits(self): + ''' Push two commits with addition and removal of Polish source to a branch. + + That branch includes a previously-added Santa Clara County source + which should be checked. ''' data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' @@ -131,9 +135,11 @@ def test_webhook_branch_commit(self): self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') self.assertFalse('pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') + self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from branch commit') + self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from branch commit') - def test_webhook_empty_commit(self): - ''' + def test_webhook_empty_master_commit(self): + ''' Push one empty commit directly to master. ''' data = '''{\r "ref": "refs/heads/master",\r "before": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "after": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b450dcbb6f4c...3147668047a0",\r "commits": [\r {\r "id": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "distinct": true,\r "message": "Empty commit",\r "timestamp": "2015-04-26T18:36:22-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "distinct": true,\r "message": "Empty commit",\r "timestamp": "2015-04-26T18:36:22-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430098587,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r}''' From 120a950f3900ee9a9d4ceba7b10a34353790db60 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 28 Apr 2015 18:16:19 -0700 Subject: [PATCH 13/51] Peeled Github status API calls into new functions --- app.py | 57 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/app.py b/app.py index 6e3e2eac..64b67f8c 100644 --- a/app.py +++ b/app.py @@ -18,10 +18,14 @@ def hook(): github_auth = current_app.config['GITHUB_AUTH'] webhook_payload = json.loads(request.data) files = process_payload(webhook_payload, github_auth) - update_status(webhook_payload, files, github_auth) + status_url = get_status_url(webhook_payload) - serialize = lambda data: json.dumps(data, indent=2, sort_keys=True) - response = '\n\n'.join(['{}:\n\n{}\n'.format(name, serialize(data)) + if files: + update_pending_status(status_url, files, github_auth) + else: + update_empty_status(status_url, github_auth) + + response = '\n\n'.join(['{}:\n\n{}\n'.format(name, data) for (name, data) in sorted(files.items())]) return Response(response, headers={'Content-Type': 'text/plain'}) @@ -72,7 +76,7 @@ def get_touched_branch_files(payload, github_auth): return touched def process_payload(payload, github_auth): - ''' Return a dictionary of file paths and decoded JSON contents. + ''' Return a dictionary of file paths and raw JSON contents. ''' processed = dict() @@ -108,37 +112,50 @@ def process_payload(payload, github_auth): current_app.logger.debug('Contents SHA {}'.format(contents['sha'])) if encoding == 'base64': - processed[filename] = json.loads(b64decode(content)) + processed[filename] = b64decode(content) else: raise ValueError('Unrecognized encoding "{}"'.format(encoding)) return processed -def update_status(payload, files, github_auth): - ''' Push pending status for head commit to Github status API. +def get_status_url(payload): + ''' Get Github status API URL from webhook payload. ''' - status = dict(context='openaddresses/hooked') - - if files: - status['state'] = 'pending' - status['description'] = 'Checking {}'.format(', '.join(files)) - else: - status['state'] = 'success' - status['description'] = 'Nothing to check' - commit_sha = payload['head_commit']['id'] status_url = payload['repository']['statuses_url'] status_url = expand(status_url, dict(sha=commit_sha)) current_app.logger.debug('Status URL {}'.format(status_url)) - posted = post(status_url, data=json.dumps(status), auth=github_auth, headers={'Content-Type': 'application/json'}) + return status_url + +def post_status(status_url, status_json, github_auth): + ''' POST status JSON to Github status API. + ''' + posted = post(status_url, data=json.dumps(status_json), auth=github_auth, + headers={'Content-Type': 'application/json'}) if posted.status_code not in range(200, 299): - raise ValueError('Failed status post to {}'.format(commit_sha)) + raise ValueError('Failed status post to {}'.format(status_url)) + + if posted.json()['state'] != status_json['state']: + raise ValueError('Mismatched status post to {}'.format(status_url)) + +def update_pending_status(status_url, files, github_auth): + ''' Push pending status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked', state='pending', + description='Checking {}'.format(', '.join(files))) + + return post_status(status_url, status, github_auth) + +def update_empty_status(status_url, github_auth): + ''' Push success status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked', state='success', + description='Nothing to check') - if posted.json()['state'] != status['state']: - raise ValueError('Mismatched status post to {}'.format(commit_sha)) + return post_status(status_url, status, github_auth) if __name__ == '__main__': app.run(debug=True) From 3f74873ba8043e483d0ad7bcbf06612bca73a839 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 28 Apr 2015 18:22:18 -0700 Subject: [PATCH 14/51] Moved status tests out of HTTMock reponse content --- tests.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tests.py b/tests.py index 0454fe6d..d6cee843 100644 --- a/tests.py +++ b/tests.py @@ -15,6 +15,7 @@ def setUp(self): ''' ''' self.client = app.test_client() + self.last_state = dict() handler = StreamHandler(stream=sys.stderr) handler.setLevel(DEBUG) @@ -68,23 +69,8 @@ def response_content(self, url, request): or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa') \ or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/3147668047a0bec6d481c0e42995f7c5e5eac637'): input = json.loads(request.body) - - states = { - # Complete branch with new Santa Clara and added/removed Polish source. - 'e5f1dcae83ab1ef1f736b969da617311f7f11564': 'pending', - - # Added San Francisco directly to master. - 'e91fbc420f08890960f50f863626e1062f922522': 'pending', - - # Added Berkeley directly to master. - 'ded44ed5f1733bb93d84f94afe9383e2d47bbbaa': 'pending', - - # Empty commit directly to master. - '3147668047a0bec6d481c0e42995f7c5e5eac637': 'success', - } - + self.last_state[basename(url.path)] = input['state'] self.assertEqual(input['context'], 'openaddresses/hooked') - self.assertEqual(input['state'], states[basename(url.path)], 'Bad state "{}" for {}'.format(input['state'], url.geturl())) data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' return response(201, data.format(**input), headers=response_headers) @@ -101,6 +87,7 @@ def test_webhook_one_master_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) + self.assertEqual(self.last_state['e91fbc420f08890960f50f863626e1062f922522'], 'pending') self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') @@ -113,6 +100,7 @@ def test_webhook_two_master_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) + self.assertEqual(self.last_state['ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'], 'pending') self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') @@ -131,6 +119,7 @@ def test_webhook_two_branch_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) + self.assertEqual(self.last_state['e5f1dcae83ab1ef1f736b969da617311f7f11564'], 'pending') self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') @@ -147,6 +136,7 @@ def test_webhook_empty_master_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) + self.assertEqual(self.last_state['3147668047a0bec6d481c0e42995f7c5e5eac637'], 'success') self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from master commit') self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') From 069cd42ec92eb129d4f195589dac3cd9cb7e47f6 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 28 Apr 2015 23:14:55 -0700 Subject: [PATCH 15/51] Mocked up beginning of job-queue.openaddresses.io --- app.py | 12 ++++++++++++ tests.py | 19 +++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 64b67f8c..03c7fdde 100644 --- a/app.py +++ b/app.py @@ -22,6 +22,7 @@ def hook(): if files: update_pending_status(status_url, files, github_auth) + add_to_job_queue(files) else: update_empty_status(status_url, github_auth) @@ -157,5 +158,16 @@ def update_empty_status(status_url, github_auth): return post_status(status_url, status, github_auth) +def add_to_job_queue(files): + ''' + ''' + queue_url = 'http://job-queue.openaddresses.io/jobs/' + + posted = post(queue_url, data=json.dumps(files), allow_redirects=True, + headers={'Content-Type': 'application/json'}) + + if posted.status_code not in range(200, 299): + raise ValueError('Failed status post to {}'.format(queue_url)) + if __name__ == '__main__': app.run(debug=True) diff --git a/tests.py b/tests.py index d6cee843..bac87833 100644 --- a/tests.py +++ b/tests.py @@ -4,6 +4,8 @@ from logging import StreamHandler, DEBUG from urlparse import parse_qsl from os.path import basename +from hashlib import sha1 + import unittest, json, os, sys os.environ['GITHUB_TOKEN'] = '' @@ -16,6 +18,7 @@ def setUp(self): ''' self.client = app.test_client() self.last_state = dict() + self.job_contents = dict() handler = StreamHandler(stream=sys.stderr) handler.setLevel(DEBUG) @@ -75,8 +78,20 @@ def response_content(self, url, request): data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' return response(201, data.format(**input), headers=response_headers) - print('Unknowable URL "{}"'.format(url.geturl()), file=sys.stderr) - raise ValueError('Unknowable URL "{}"'.format(url.geturl())) + if MHP == ('POST', 'job-queue.openaddresses.io', '/jobs/'): + hash = sha1(request.body).hexdigest() + redirect = '{}://{}/jobs/{}'.format(url.scheme, url.hostname, hash) + self.job_contents[hash] = json.loads(request.body) + + return response(303, 'Over there', headers={'Location': redirect}) + + if MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/b03a85fb76da854dd940fd4c4153a266b37ca948') \ + or MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/7a4ea989aef2fa85589ea7967c653f762af975f8') \ + or MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/e36ecdd7c1098d9ed885640a8c07ff5c8e76174c'): + return response(200, '{}', headers=response_headers) + + print('Unknowable Request {} "{}"'.format(request.method, url.geturl()), file=sys.stderr) + raise ValueError('Unknowable Request {} "{}"'.format(request.method, url.geturl())) def test_webhook_one_master_commit(self): ''' Push a single commit with Alameda County source directly to master. From c2de544b6a11d006f9f3c985925c45175ff5025d Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Wed, 29 Apr 2015 00:38:30 -0700 Subject: [PATCH 16/51] Completed mock implementation of job queue interface with tests and sessions --- app.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++------ tests.py | 93 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 162 insertions(+), 31 deletions(-) diff --git a/app.py b/app.py index 03c7fdde..0266463d 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,7 @@ from os.path import relpath, splitext +from urlparse import urljoin from base64 import b64decode +from hashlib import sha1 import json, os from flask import Flask, request, Response, current_app @@ -8,12 +10,16 @@ app = Flask(__name__) app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' +app.secret_key = os.environ['SECRET_KEY'] + +from flask import session +MAGIC_OK_MESSAGE = 'Everything is fine' @app.route('/') def index(): return 'Yo.' -@app.route('/hook', methods=['GET', 'POST']) +@app.route('/hook', methods=['POST']) def hook(): github_auth = current_app.config['GITHUB_AUTH'] webhook_payload = json.loads(request.data) @@ -21,8 +27,18 @@ def hook(): status_url = get_status_url(webhook_payload) if files: - update_pending_status(status_url, files, github_auth) - add_to_job_queue(files) + try: + # Add the touched files to a job queue. + job_id, job_url = add_to_job_queue(request, files) + except Exception as e: + # Oops, tell Github something went wrong. + update_error_status(status_url, str(e), files.keys(), github_auth) + else: + # That worked, remember them in the session. + session['{}-filenames'.format(job_id)] = files.keys() + session['{}-status_url'.format(job_id)] = status_url + session['{}-job_url'.format(job_id)] = job_url + update_pending_status(status_url, job_url, files.keys(), github_auth) else: update_empty_status(status_url, github_auth) @@ -31,6 +47,35 @@ def hook(): return Response(response, headers={'Content-Type': 'text/plain'}) +@app.route('/jobs/', methods=['POST']) +def post_job(job_id): + ''' + ''' + if '{}-job_url'.format(job_id) not in session: + return Response('Job {} not found'.format(job_id), 404) + + github_auth = current_app.config['GITHUB_AUTH'] + status_url = session['{}-status_url'.format(job_id)] + filenames = session['{}-filenames'.format(job_id)] + job_url = session['{}-job_url'.format(job_id)] + message = request.data + + if message == MAGIC_OK_MESSAGE: + update_success_status(status_url, job_url, filenames, github_auth) + else: + update_failing_status(status_url, job_url, message, filenames, github_auth) + + return 'Job updated' + +@app.route('/jobs/', methods=['GET']) +def get_job(job_id): + ''' + ''' + if '{}-job_url'.format(job_id) not in session: + return Response('Job {} not found'.format(job_id), 404) + + return 'I am a job' + def get_touched_payload_files(payload): ''' Return a set of files modified in payload commits. ''' @@ -130,7 +175,7 @@ def get_status_url(payload): return status_url -def post_status(status_url, status_json, github_auth): +def post_github_status(status_url, status_json, github_auth): ''' POST status JSON to Github status API. ''' posted = post(status_url, data=json.dumps(status_json), auth=github_auth, @@ -142,13 +187,31 @@ def post_status(status_url, status_json, github_auth): if posted.json()['state'] != status_json['state']: raise ValueError('Mismatched status post to {}'.format(status_url)) -def update_pending_status(status_url, files, github_auth): +def update_pending_status(status_url, job_url, filenames, github_auth): ''' Push pending status for head commit to Github status API. ''' status = dict(context='openaddresses/hooked', state='pending', - description='Checking {}'.format(', '.join(files))) + description='Checking {}'.format(', '.join(filenames)), + target_url=job_url) - return post_status(status_url, status, github_auth) + return post_github_status(status_url, status, github_auth) + +def update_error_status(status_url, message, filenames, github_auth): + ''' Push error status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked', state='error', + description='Errored on {}: {}'.format(', '.join(filenames), message)) + + return post_github_status(status_url, status, github_auth) + +def update_failing_status(status_url, job_url, message, filenames, github_auth): + ''' Push failing status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked', state='failure', + description='Failed on {}: {}'.format(', '.join(filenames), message), + target_url=job_url) + + return post_github_status(status_url, status, github_auth) def update_empty_status(status_url, github_auth): ''' Push success status for head commit to Github status API. @@ -156,18 +219,33 @@ def update_empty_status(status_url, github_auth): status = dict(context='openaddresses/hooked', state='success', description='Nothing to check') - return post_status(status_url, status, github_auth) + return post_github_status(status_url, status, github_auth) + +def update_success_status(status_url, job_url, filenames, github_auth): + ''' Push success status for head commit to Github status API. + ''' + status = dict(context='openaddresses/hooked', state='success', + description='Succeeded on {}'.format(', '.join(filenames)), + target_url=job_url) + + return post_github_status(status_url, status, github_auth) -def add_to_job_queue(files): +def add_to_job_queue(request, files): ''' ''' + job_id = sha1(json.dumps(files)).hexdigest() + job_url = urljoin(request.url, '/jobs/{}'.format(job_id)) + + queue_msg = json.dumps({"callback": job_url, "files": files}) queue_url = 'http://job-queue.openaddresses.io/jobs/' - - posted = post(queue_url, data=json.dumps(files), allow_redirects=True, + + posted = post(queue_url, data=queue_msg, allow_redirects=True, headers={'Content-Type': 'application/json'}) if posted.status_code not in range(200, 299): raise ValueError('Failed status post to {}'.format(queue_url)) + + return job_id, job_url if __name__ == '__main__': app.run(debug=True) diff --git a/tests.py b/tests.py index bac87833..3f7d1271 100644 --- a/tests.py +++ b/tests.py @@ -2,14 +2,15 @@ from httmock import HTTMock, response from logging import StreamHandler, DEBUG -from urlparse import parse_qsl +from urlparse import parse_qsl, urlparse from os.path import basename -from hashlib import sha1 +from hashlib import md5 import unittest, json, os, sys os.environ['GITHUB_TOKEN'] = '' -from app import app +os.environ['SECRET_KEY'] = 'boop' +from app import app, MAGIC_OK_MESSAGE class TestHook (unittest.TestCase): @@ -17,18 +18,20 @@ def setUp(self): ''' ''' self.client = app.test_client() - self.last_state = dict() - self.job_contents = dict() + self.last_status_state = None + self.last_status_message = None + self.last_job_url = None handler = StreamHandler(stream=sys.stderr) handler.setLevel(DEBUG) app.logger.addHandler(handler) - + def response_content(self, url, request): ''' ''' query = dict(parse_qsl(url.query)) - GH, MHP = 'api.github.com', (request.method, url.hostname, url.path) + MHP = request.method, url.hostname, url.path + GH, JQ = 'api.github.com', 'job-queue.openaddresses.io' response_headers = {'Content-Type': 'application/json; charset=utf-8'} if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): @@ -72,23 +75,29 @@ def response_content(self, url, request): or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa') \ or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/3147668047a0bec6d481c0e42995f7c5e5eac637'): input = json.loads(request.body) - self.last_state[basename(url.path)] = input['state'] + self.last_status_state = input['state'] + self.last_status_message = input['description'] self.assertEqual(input['context'], 'openaddresses/hooked') data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' return response(201, data.format(**input), headers=response_headers) - if MHP == ('POST', 'job-queue.openaddresses.io', '/jobs/'): - hash = sha1(request.body).hexdigest() - redirect = '{}://{}/jobs/{}'.format(url.scheme, url.hostname, hash) - self.job_contents[hash] = json.loads(request.body) + if MHP == ('POST', JQ, '/jobs/'): + input = json.loads(request.body) + self.last_job_url = urlparse(input['callback']) + + # Simulate a queuing error with Santa Clara County + if 'sources/us-ca-santa_clara_county.json' in input['files']: + return response(400, 'Uh oh') + + hash = md5(request.body).hexdigest() + redirect = 'http://job-queue.openaddresses.io/jobs/{}'.format(hash) return response(303, 'Over there', headers={'Location': redirect}) - if MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/b03a85fb76da854dd940fd4c4153a266b37ca948') \ - or MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/7a4ea989aef2fa85589ea7967c653f762af975f8') \ - or MHP == ('GET', 'job-queue.openaddresses.io', '/jobs/e36ecdd7c1098d9ed885640a8c07ff5c8e76174c'): - return response(200, '{}', headers=response_headers) + if MHP == ('GET', JQ, '/jobs/ee0b79ba74184d9b181f004ff9a402b8') \ + or MHP == ('GET', JQ, '/jobs/ef58442ae22247e100b2bbc1e0d810c4'): + return response(200, 'I am a job') print('Unknowable Request {} "{}"'.format(request.method, url.geturl()), file=sys.stderr) raise ValueError('Unknowable Request {} "{}"'.format(request.method, url.geturl())) @@ -102,9 +111,23 @@ def test_webhook_one_master_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertEqual(self.last_state['e91fbc420f08890960f50f863626e1062f922522'], 'pending') + self.assertEqual(self.last_status_state, 'pending') self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') + + # Pretend that queued job completed successfully. + + with HTTMock(self.response_content): + got = self.client.get(self.last_job_url.path) + self.assertEqual(got.status_code, 200) + + posted = self.client.post(self.last_job_url.path, data=MAGIC_OK_MESSAGE) + self.assertTrue(posted.status_code in range(200, 299)) + + got = self.client.get(self.last_job_url.path) + self.assertEqual(got.status_code, 200) + + self.assertEqual(self.last_status_state, 'success') def test_webhook_two_master_commits(self): ''' Push two commits with San Francisco and Berkeley sources directly to master. @@ -115,12 +138,27 @@ def test_webhook_two_master_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertEqual(self.last_state['ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'], 'pending') + self.assertEqual(self.last_status_state, 'pending') self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') + + # Pretend that queued job failed. + + with HTTMock(self.response_content): + got = self.client.get(self.last_job_url.path) + self.assertEqual(got.status_code, 200) + + posted = self.client.post(self.last_job_url.path, data='Something went wrong') + self.assertTrue(posted.status_code in range(200, 299)) + + got = self.client.get(self.last_job_url.path) + self.assertEqual(got.status_code, 200) + + self.assertEqual(self.last_status_state, 'failure') + self.assertTrue('Something went wrong' in self.last_status_message) def test_webhook_two_branch_commits(self): ''' Push two commits with addition and removal of Polish source to a branch. @@ -134,13 +172,28 @@ def test_webhook_two_branch_commits(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertEqual(self.last_state['e5f1dcae83ab1ef1f736b969da617311f7f11564'], 'pending') + self.assertEqual(self.last_status_state, 'error') + self.assertTrue('us-ca-santa_clara_county' in self.last_status_message, 'Santa Clara County source should be present in error message') self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') self.assertFalse('pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from branch commit') self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from branch commit') + + # Verify that queued job was never created. + + with HTTMock(self.response_content): + got = self.client.get(self.last_job_url.path) + self.assertTrue(got.status_code in range(400, 499)) + + posted = self.client.post(self.last_job_url.path, data=MAGIC_OK_MESSAGE) + self.assertTrue(posted.status_code in range(400, 499)) + + got = self.client.get(self.last_job_url.path) + self.assertTrue(got.status_code in range(400, 499)) + + self.assertEqual(self.last_status_state, 'error') def test_webhook_empty_master_commit(self): ''' Push one empty commit directly to master. @@ -151,7 +204,7 @@ def test_webhook_empty_master_commit(self): posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) - self.assertEqual(self.last_state['3147668047a0bec6d481c0e42995f7c5e5eac637'], 'success') + self.assertEqual(self.last_status_state, 'success') self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from master commit') self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') From e82264db6d2927b265c9ff3bd37b5ca12ee5389a Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Wed, 29 Apr 2015 11:59:14 -0700 Subject: [PATCH 17/51] Switched from test-only session storage to Postgres database --- app.py | 61 +++++++++++++++++++++++++++++++++++++----------- requirements.txt | 1 + schema.pgsql | 9 +++++++ tests.py | 11 +++++++-- 4 files changed, 67 insertions(+), 15 deletions(-) create mode 100644 schema.pgsql diff --git a/app.py b/app.py index 0266463d..acc68203 100644 --- a/app.py +++ b/app.py @@ -7,12 +7,12 @@ from flask import Flask, request, Response, current_app from uritemplate import expand from requests import get, post +from psycopg2 import connect app = Flask(__name__) app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' -app.secret_key = os.environ['SECRET_KEY'] +app.config['DATABASE_URL'] = os.environ['DATABASE_URL'] -from flask import session MAGIC_OK_MESSAGE = 'Everything is fine' @app.route('/') @@ -34,10 +34,11 @@ def hook(): # Oops, tell Github something went wrong. update_error_status(status_url, str(e), files.keys(), github_auth) else: - # That worked, remember them in the session. - session['{}-filenames'.format(job_id)] = files.keys() - session['{}-status_url'.format(job_id)] = status_url - session['{}-job_url'.format(job_id)] = job_url + # That worked, remember them in the database. + with db_connect(current_app) as conn: + with db_cursor(conn) as db: + save_job(db, job_id, list(files.keys()), status_url, job_url) + update_pending_status(status_url, job_url, files.keys(), github_auth) else: update_empty_status(status_url, github_auth) @@ -51,13 +52,14 @@ def hook(): def post_job(job_id): ''' ''' - if '{}-job_url'.format(job_id) not in session: - return Response('Job {} not found'.format(job_id), 404) + with db_connect(current_app) as conn: + with db_cursor(conn) as db: + try: + filenames, status_url, job_url = read_job(db, job_id) + except TypeError: + return Response('Job {} not found'.format(job_id), 404) github_auth = current_app.config['GITHUB_AUTH'] - status_url = session['{}-status_url'.format(job_id)] - filenames = session['{}-filenames'.format(job_id)] - job_url = session['{}-job_url'.format(job_id)] message = request.data if message == MAGIC_OK_MESSAGE: @@ -71,8 +73,10 @@ def post_job(job_id): def get_job(job_id): ''' ''' - if '{}-job_url'.format(job_id) not in session: - return Response('Job {} not found'.format(job_id), 404) + with db_connect(current_app) as conn: + with db_cursor(conn) as db: + if read_job(db, job_id) is None: + return Response('Job {} not found'.format(job_id), 404) return 'I am a job' @@ -247,5 +251,36 @@ def add_to_job_queue(request, files): return job_id, job_url +def save_job(db, job_id, filenames, status_url, job_url): + ''' Save information about a job to the database. + + Throws an IntegrityError exception if the job ID exists. + ''' + db.execute('''INSERT INTO jobs + (filenames, github_status_url, job_queue_url, id) + VALUES (%s, %s, %s, %s)''', + (filenames, status_url, job_url, job_id)) + +def read_job(db, job_id): + ''' Read information about a job from the database. + + Returns (filenames, github_status_url, job_queue_url) or None. + ''' + db.execute('''SELECT filenames, github_status_url, job_queue_url + FROM jobs WHERE id = %s''', (job_id, )) + + try: + filenames, github_status_url, job_queue_url = db.fetchone() + except TypeError: + return None + else: + return filenames, github_status_url, job_queue_url + +def db_connect(app): + return connect(app.config['DATABASE_URL']) + +def db_cursor(conn): + return conn.cursor() + if __name__ == '__main__': app.run(debug=True) diff --git a/requirements.txt b/requirements.txt index d62c29cf..fa7086a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ httmock==1.2.3 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 +psycopg2==2.6 requests==2.6.2 simplejson==3.6.5 uritemplate==0.6 diff --git a/schema.pgsql b/schema.pgsql new file mode 100644 index 00000000..12e9e32a --- /dev/null +++ b/schema.pgsql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS jobs; + +CREATE TABLE jobs +( + id VARCHAR(40) PRIMARY KEY, + filenames TEXT ARRAY, + github_status_url TEXT, + job_queue_url TEXT +); diff --git a/tests.py b/tests.py index 3f7d1271..a0141010 100644 --- a/tests.py +++ b/tests.py @@ -9,8 +9,8 @@ import unittest, json, os, sys os.environ['GITHUB_TOKEN'] = '' -os.environ['SECRET_KEY'] = 'boop' -from app import app, MAGIC_OK_MESSAGE +os.environ['DATABASE_URL'] = 'postgres:///hooked_on_sources' +from app import app, db_connect, db_cursor, MAGIC_OK_MESSAGE class TestHook (unittest.TestCase): @@ -26,6 +26,13 @@ def setUp(self): handler.setLevel(DEBUG) app.logger.addHandler(handler) + def tearDown(self): + ''' + ''' + with db_connect(app) as conn: + with db_cursor(conn) as db: + db.execute('TRUNCATE jobs') + def response_content(self, url, request): ''' ''' From 3d44fb7b451afc0e3068be89f9270505dd52db94 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Wed, 29 Apr 2015 12:07:13 -0700 Subject: [PATCH 18/51] Truncated Github status descriptions to 140 chars --- app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app.py b/app.py index acc68203..67a3d1cd 100644 --- a/app.py +++ b/app.py @@ -182,6 +182,9 @@ def get_status_url(payload): def post_github_status(status_url, status_json, github_auth): ''' POST status JSON to Github status API. ''' + # Github only wants 140 chars of description. + status_json['description'] = status_json['description'][:140] + posted = post(status_url, data=json.dumps(status_json), auth=github_auth, headers={'Content-Type': 'application/json'}) From 9e25672a811b56b10084af4c729f6ba46a736e38 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 1 May 2015 16:20:23 -0700 Subject: [PATCH 19/51] Added queue basics with new Postgres setup --- app.py | 28 ++++++++++++++++------------ recreate-db.py | 25 +++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 recreate-db.py diff --git a/app.py b/app.py index 67a3d1cd..b9e2ef49 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ from uritemplate import expand from requests import get, post from psycopg2 import connect +from pq import PQ app = Flask(__name__) app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' @@ -27,19 +28,19 @@ def hook(): status_url = get_status_url(webhook_payload) if files: - try: - # Add the touched files to a job queue. - job_id, job_url = add_to_job_queue(request, files) - except Exception as e: - # Oops, tell Github something went wrong. - update_error_status(status_url, str(e), files.keys(), github_auth) - else: - # That worked, remember them in the database. - with db_connect(current_app) as conn: - with db_cursor(conn) as db: + with db_connect(current_app) as conn: + queue = db_queue(conn) + with queue as db: + try: + # Add the touched files to a job queue. + job_id, job_url = add_to_job_queue(request, files) + except Exception as e: + # Oops, tell Github something went wrong. + update_error_status(status_url, str(e), files.keys(), github_auth) + else: + # That worked, remember them in the database. save_job(db, job_id, list(files.keys()), status_url, job_url) - - update_pending_status(status_url, job_url, files.keys(), github_auth) + update_pending_status(status_url, job_url, files.keys(), github_auth) else: update_empty_status(status_url, github_auth) @@ -282,6 +283,9 @@ def read_job(db, job_id): def db_connect(app): return connect(app.config['DATABASE_URL']) +def db_queue(conn): + return PQ(conn)['jobs'] + def db_cursor(conn): return conn.cursor() diff --git a/recreate-db.py b/recreate-db.py new file mode 100644 index 00000000..8b2824cd --- /dev/null +++ b/recreate-db.py @@ -0,0 +1,25 @@ +import os +from os.path import join, dirname + +from pq import PQ +from psycopg2 import connect + +def main(DATABASE_URL): + ''' + ''' + schema_filename = join(dirname(__file__), 'schema.pgsql') + + with connect(DATABASE_URL) as conn: + with conn.cursor() as db: + with open(schema_filename) as file: + db.execute(file.read()) + + db.execute('DROP TABLE queue') + + pq = PQ(conn) + pq.create() + +if __name__ == '__main__': + + DATABASE_URL = os.environ['DATABASE_URL'] + exit(main(DATABASE_URL)) diff --git a/requirements.txt b/requirements.txt index fa7086a9..84eed26a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ httmock==1.2.3 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 +pq==1.2 psycopg2==2.6 requests==2.6.2 simplejson==3.6.5 From 5a4b5e6fa2012fe6bd06fbb6c3099720d3e61985 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 1 May 2015 17:34:19 -0700 Subject: [PATCH 20/51] Switched to using internal database for queue --- app.py | 20 +++++----------- requirements.txt | 1 + tests.py | 62 +++++++++++++++++------------------------------- 3 files changed, 29 insertions(+), 54 deletions(-) diff --git a/app.py b/app.py index b9e2ef49..1de669d2 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,7 @@ from hashlib import sha1 import json, os -from flask import Flask, request, Response, current_app +from flask import Flask, request, Response, current_app, jsonify from uritemplate import expand from requests import get, post from psycopg2 import connect @@ -26,6 +26,7 @@ def hook(): webhook_payload = json.loads(request.data) files = process_payload(webhook_payload, github_auth) status_url = get_status_url(webhook_payload) + job_url = None if files: with db_connect(current_app) as conn: @@ -33,7 +34,7 @@ def hook(): with queue as db: try: # Add the touched files to a job queue. - job_id, job_url = add_to_job_queue(request, files) + job_id, job_url = add_to_job_queue(queue, request, files) except Exception as e: # Oops, tell Github something went wrong. update_error_status(status_url, str(e), files.keys(), github_auth) @@ -44,10 +45,7 @@ def hook(): else: update_empty_status(status_url, github_auth) - response = '\n\n'.join(['{}:\n\n{}\n'.format(name, data) - for (name, data) in sorted(files.items())]) - - return Response(response, headers={'Content-Type': 'text/plain'}) + return jsonify({'url': job_url, 'files': files}) @app.route('/jobs/', methods=['POST']) def post_job(job_id): @@ -238,20 +236,14 @@ def update_success_status(status_url, job_url, filenames, github_auth): return post_github_status(status_url, status, github_auth) -def add_to_job_queue(request, files): +def add_to_job_queue(queue, request, files): ''' ''' job_id = sha1(json.dumps(files)).hexdigest() job_url = urljoin(request.url, '/jobs/{}'.format(job_id)) queue_msg = json.dumps({"callback": job_url, "files": files}) - queue_url = 'http://job-queue.openaddresses.io/jobs/' - - posted = post(queue_url, data=queue_msg, allow_redirects=True, - headers={'Content-Type': 'application/json'}) - - if posted.status_code not in range(200, 299): - raise ValueError('Failed status post to {}'.format(queue_url)) + queue.put(queue_msg) return job_id, job_url diff --git a/requirements.txt b/requirements.txt index 84eed26a..4c1b6dc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ httmock==1.2.3 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 +mock==1.0.1 pq==1.2 psycopg2==2.6 requests==2.6.2 diff --git a/tests.py b/tests.py index a0141010..3a9b3f97 100644 --- a/tests.py +++ b/tests.py @@ -3,8 +3,7 @@ from httmock import HTTMock, response from logging import StreamHandler, DEBUG from urlparse import parse_qsl, urlparse -from os.path import basename -from hashlib import md5 +from mock import patch import unittest, json, os, sys @@ -32,14 +31,14 @@ def tearDown(self): with db_connect(app) as conn: with db_cursor(conn) as db: db.execute('TRUNCATE jobs') + db.execute('TRUNCATE queue') def response_content(self, url, request): ''' ''' query = dict(parse_qsl(url.query)) MHP = request.method, url.hostname, url.path - GH, JQ = 'api.github.com', 'job-queue.openaddresses.io' - response_headers = {'Content-Type': 'application/json; charset=utf-8'} + GH, response_headers = 'api.github.com', {'Content-Type': 'application/json; charset=utf-8'} if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): data = '''{\r "name": "us-ca-alameda_county.json",\r "path": "sources/us-ca-alameda_county.json",\r "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "size": 745,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json"\r }\r }''' @@ -89,23 +88,6 @@ def response_content(self, url, request): data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' return response(201, data.format(**input), headers=response_headers) - if MHP == ('POST', JQ, '/jobs/'): - input = json.loads(request.body) - self.last_job_url = urlparse(input['callback']) - - # Simulate a queuing error with Santa Clara County - if 'sources/us-ca-santa_clara_county.json' in input['files']: - return response(400, 'Uh oh') - - hash = md5(request.body).hexdigest() - redirect = 'http://job-queue.openaddresses.io/jobs/{}'.format(hash) - - return response(303, 'Over there', headers={'Location': redirect}) - - if MHP == ('GET', JQ, '/jobs/ee0b79ba74184d9b181f004ff9a402b8') \ - or MHP == ('GET', JQ, '/jobs/ef58442ae22247e100b2bbc1e0d810c4'): - return response(200, 'I am a job') - print('Unknowable Request {} "{}"'.format(request.method, url.geturl()), file=sys.stderr) raise ValueError('Unknowable Request {} "{}"'.format(request.method, url.geturl())) @@ -123,15 +105,16 @@ def test_webhook_one_master_commit(self): self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') # Pretend that queued job completed successfully. + last_job_url = urlparse(json.loads(posted.data).get('url')) with HTTMock(self.response_content): - got = self.client.get(self.last_job_url.path) + got = self.client.get(last_job_url.path) self.assertEqual(got.status_code, 200) - posted = self.client.post(self.last_job_url.path, data=MAGIC_OK_MESSAGE) + posted = self.client.post(last_job_url.path, data=MAGIC_OK_MESSAGE) self.assertTrue(posted.status_code in range(200, 299)) - got = self.client.get(self.last_job_url.path) + got = self.client.get(last_job_url.path) self.assertEqual(got.status_code, 200) self.assertEqual(self.last_status_state, 'success') @@ -152,16 +135,17 @@ def test_webhook_two_master_commits(self): self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') - # Pretend that queued job failed. + last_job_url = urlparse(json.loads(posted.data).get('url')) with HTTMock(self.response_content): - got = self.client.get(self.last_job_url.path) + got = self.client.get(last_job_url.path) self.assertEqual(got.status_code, 200) - posted = self.client.post(self.last_job_url.path, data='Something went wrong') + # Pretend that queued job failed. + posted = self.client.post(last_job_url.path, data='Something went wrong') self.assertTrue(posted.status_code in range(200, 299)) - got = self.client.get(self.last_job_url.path) + got = self.client.get(last_job_url.path) self.assertEqual(got.status_code, 200) self.assertEqual(self.last_status_state, 'failure') @@ -175,8 +159,14 @@ def test_webhook_two_branch_commits(self): ''' data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' - with HTTMock(self.response_content): - posted = self.client.post('/hook', data=data) + with patch('app.add_to_job_queue') as add_to_job_queue: + def job_queue_fails(*args, **kwargs): + raise Exception('Nope') + + add_to_job_queue.side_effect = job_queue_fails + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) self.assertEqual(posted.status_code, 200) self.assertEqual(self.last_status_state, 'error') @@ -189,17 +179,9 @@ def test_webhook_two_branch_commits(self): self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from branch commit') # Verify that queued job was never created. + last_job_url = json.loads(posted.data).get('url') - with HTTMock(self.response_content): - got = self.client.get(self.last_job_url.path) - self.assertTrue(got.status_code in range(400, 499)) - - posted = self.client.post(self.last_job_url.path, data=MAGIC_OK_MESSAGE) - self.assertTrue(posted.status_code in range(400, 499)) - - got = self.client.get(self.last_job_url.path) - self.assertTrue(got.status_code in range(400, 499)) - + self.assertEqual(last_job_url, None) self.assertEqual(self.last_status_state, 'error') def test_webhook_empty_master_commit(self): From ab1e256d0fd1ccc1111409ce25d2383798467926 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 1 May 2015 17:47:37 -0700 Subject: [PATCH 21/51] Stopped storing jobs.job_queue_url in database --- app.py | 33 ++++++++++++++++----------------- schema.pgsql | 3 +-- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/app.py b/app.py index 1de669d2..4c57925e 100644 --- a/app.py +++ b/app.py @@ -34,13 +34,14 @@ def hook(): with queue as db: try: # Add the touched files to a job queue. - job_id, job_url = add_to_job_queue(queue, request, files) + job_id = add_to_job_queue(queue, files) except Exception as e: # Oops, tell Github something went wrong. update_error_status(status_url, str(e), files.keys(), github_auth) else: # That worked, remember them in the database. - save_job(db, job_id, list(files.keys()), status_url, job_url) + save_job(db, job_id, list(files.keys()), status_url) + job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) update_pending_status(status_url, job_url, files.keys(), github_auth) else: update_empty_status(status_url, github_auth) @@ -54,12 +55,13 @@ def post_job(job_id): with db_connect(current_app) as conn: with db_cursor(conn) as db: try: - filenames, status_url, job_url = read_job(db, job_id) + filenames, status_url = read_job(db, job_id) except TypeError: return Response('Job {} not found'.format(job_id), 404) github_auth = current_app.config['GITHUB_AUTH'] message = request.data + job_url = request.url if message == MAGIC_OK_MESSAGE: update_success_status(status_url, job_url, filenames, github_auth) @@ -236,41 +238,38 @@ def update_success_status(status_url, job_url, filenames, github_auth): return post_github_status(status_url, status, github_auth) -def add_to_job_queue(queue, request, files): +def add_to_job_queue(queue, files): ''' ''' job_id = sha1(json.dumps(files)).hexdigest() - job_url = urljoin(request.url, '/jobs/{}'.format(job_id)) - - queue_msg = json.dumps({"callback": job_url, "files": files}) - queue.put(queue_msg) + queue.put({"id": job_id, "files": files}) - return job_id, job_url + return job_id -def save_job(db, job_id, filenames, status_url, job_url): +def save_job(db, job_id, filenames, status_url): ''' Save information about a job to the database. Throws an IntegrityError exception if the job ID exists. ''' db.execute('''INSERT INTO jobs - (filenames, github_status_url, job_queue_url, id) - VALUES (%s, %s, %s, %s)''', - (filenames, status_url, job_url, job_id)) + (filenames, github_status_url, id) + VALUES (%s, %s, %s)''', + (filenames, status_url, job_id)) def read_job(db, job_id): ''' Read information about a job from the database. - Returns (filenames, github_status_url, job_queue_url) or None. + Returns (filenames, github_status_url) or None. ''' - db.execute('''SELECT filenames, github_status_url, job_queue_url + db.execute('''SELECT filenames, github_status_url FROM jobs WHERE id = %s''', (job_id, )) try: - filenames, github_status_url, job_queue_url = db.fetchone() + filenames, github_status_url = db.fetchone() except TypeError: return None else: - return filenames, github_status_url, job_queue_url + return filenames, github_status_url def db_connect(app): return connect(app.config['DATABASE_URL']) diff --git a/schema.pgsql b/schema.pgsql index 12e9e32a..c327c9fb 100644 --- a/schema.pgsql +++ b/schema.pgsql @@ -4,6 +4,5 @@ CREATE TABLE jobs ( id VARCHAR(40) PRIMARY KEY, filenames TEXT ARRAY, - github_status_url TEXT, - job_queue_url TEXT + github_status_url TEXT ); From 3d25008c9f5f70fb0ee55cf75d72b6fb87be5113 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 1 May 2015 22:58:20 -0700 Subject: [PATCH 22/51] Split multi-file jobs into multiple queue tasks --- app.py | 54 +++++++++++++++++++++++++++++--------------------- recreate-db.py | 2 +- tests.py | 10 ++++++++-- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/app.py b/app.py index 4c57925e..26b96ebe 100644 --- a/app.py +++ b/app.py @@ -26,27 +26,29 @@ def hook(): webhook_payload = json.loads(request.data) files = process_payload(webhook_payload, github_auth) status_url = get_status_url(webhook_payload) - job_url = None - if files: - with db_connect(current_app) as conn: - queue = db_queue(conn) - with queue as db: - try: - # Add the touched files to a job queue. - job_id = add_to_job_queue(queue, files) - except Exception as e: - # Oops, tell Github something went wrong. - update_error_status(status_url, str(e), files.keys(), github_auth) - else: - # That worked, remember them in the database. - save_job(db, job_id, list(files.keys()), status_url) - job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) - update_pending_status(status_url, job_url, files.keys(), github_auth) - else: + if not files: update_empty_status(status_url, github_auth) - - return jsonify({'url': job_url, 'files': files}) + return jsonify({'url': None, 'files': []}) + + job_id = calculate_job_id(files) + job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) + + with db_connect(current_app) as conn: + queue = db_queue(conn) + with queue as db: + try: + # Add the touched files to a task queue. + add_files_to_queue(queue, job_url, files) + except Exception as e: + # Oops, tell Github something went wrong. + update_error_status(status_url, str(e), files.keys(), github_auth) + return jsonify({'error': str(e), 'files': files}) + else: + # That worked, remember them in the database. + save_job(db, job_id, list(files.keys()), status_url) + update_pending_status(status_url, job_url, files.keys(), github_auth) + return jsonify({'url': job_url, 'files': files}) @app.route('/jobs/', methods=['POST']) def post_job(job_id): @@ -238,14 +240,20 @@ def update_success_status(status_url, job_url, filenames, github_auth): return post_github_status(status_url, status, github_auth) -def add_to_job_queue(queue, files): +def calculate_job_id(files): ''' ''' - job_id = sha1(json.dumps(files)).hexdigest() - queue.put({"id": job_id, "files": files}) + blob = json.dumps(files, ensure_ascii=True, sort_keys=True) + job_id = sha1(blob).hexdigest() return job_id +def add_files_to_queue(queue, job_url, files): + ''' + ''' + for (name, content) in files.items(): + queue.put(dict(url=job_url, name=name, content=content)) + def save_job(db, job_id, filenames, status_url): ''' Save information about a job to the database. @@ -275,7 +283,7 @@ def db_connect(app): return connect(app.config['DATABASE_URL']) def db_queue(conn): - return PQ(conn)['jobs'] + return PQ(conn, table='queue')['tasks'] def db_cursor(conn): return conn.cursor() diff --git a/recreate-db.py b/recreate-db.py index 8b2824cd..48c49603 100644 --- a/recreate-db.py +++ b/recreate-db.py @@ -16,7 +16,7 @@ def main(DATABASE_URL): db.execute('DROP TABLE queue') - pq = PQ(conn) + pq = PQ(conn, table='queue') pq.create() if __name__ == '__main__': diff --git a/tests.py b/tests.py index 3a9b3f97..35d08133 100644 --- a/tests.py +++ b/tests.py @@ -9,13 +9,17 @@ os.environ['GITHUB_TOKEN'] = '' os.environ['DATABASE_URL'] = 'postgres:///hooked_on_sources' + from app import app, db_connect, db_cursor, MAGIC_OK_MESSAGE +recreate_db = __import__('recreate-db') class TestHook (unittest.TestCase): def setUp(self): ''' ''' + recreate_db.main(app.config['DATABASE_URL']) + self.client = app.test_client() self.last_status_state = None self.last_status_message = None @@ -28,6 +32,8 @@ def setUp(self): def tearDown(self): ''' ''' + return + with db_connect(app) as conn: with db_cursor(conn) as db: db.execute('TRUNCATE jobs') @@ -159,11 +165,11 @@ def test_webhook_two_branch_commits(self): ''' data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' - with patch('app.add_to_job_queue') as add_to_job_queue: + with patch('app.add_files_to_queue') as add_files_to_queue: def job_queue_fails(*args, **kwargs): raise Exception('Nope') - add_to_job_queue.side_effect = job_queue_fails + add_files_to_queue.side_effect = job_queue_fails with HTTMock(self.response_content): posted = self.client.post('/hook', data=data) From a13ceb2a84c5a75819ba9bbd8049a4218baea709 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 1 May 2015 23:05:57 -0700 Subject: [PATCH 23/51] Changed jobs.filenames array column to jobs.task_files JSON column --- app.py | 29 ++++++++++++++++++----------- schema.pgsql | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 26b96ebe..0a2d6109 100644 --- a/app.py +++ b/app.py @@ -39,14 +39,14 @@ def hook(): with queue as db: try: # Add the touched files to a task queue. - add_files_to_queue(queue, job_url, files) + tasks = add_files_to_queue(queue, job_url, files) except Exception as e: # Oops, tell Github something went wrong. update_error_status(status_url, str(e), files.keys(), github_auth) return jsonify({'error': str(e), 'files': files}) else: # That worked, remember them in the database. - save_job(db, job_id, list(files.keys()), status_url) + save_job(db, job_id, tasks, status_url) update_pending_status(status_url, job_url, files.keys(), github_auth) return jsonify({'url': job_url, 'files': files}) @@ -57,11 +57,12 @@ def post_job(job_id): with db_connect(current_app) as conn: with db_cursor(conn) as db: try: - filenames, status_url = read_job(db, job_id) + task_files, status_url = read_job(db, job_id) except TypeError: return Response('Job {} not found'.format(job_id), 404) github_auth = current_app.config['GITHUB_AUTH'] + filenames = list(task_files.values()) message = request.data job_url = request.url @@ -249,27 +250,33 @@ def calculate_job_id(files): return job_id def add_files_to_queue(queue, job_url, files): + ''' Make a new task for each file, return dict of taks IDs to file names. ''' - ''' + tasks = {} + for (name, content) in files.items(): - queue.put(dict(url=job_url, name=name, content=content)) + task = queue.put(dict(url=job_url, name=name, content=content)) + + tasks[str(task)] = name + + return tasks -def save_job(db, job_id, filenames, status_url): +def save_job(db, job_id, task_files, status_url): ''' Save information about a job to the database. Throws an IntegrityError exception if the job ID exists. ''' db.execute('''INSERT INTO jobs - (filenames, github_status_url, id) - VALUES (%s, %s, %s)''', - (filenames, status_url, job_id)) + (task_files, github_status_url, id) + VALUES (%s::json, %s, %s)''', + (json.dumps(task_files), status_url, job_id)) def read_job(db, job_id): ''' Read information about a job from the database. - Returns (filenames, github_status_url) or None. + Returns (task_files, github_status_url) or None. ''' - db.execute('''SELECT filenames, github_status_url + db.execute('''SELECT task_files, github_status_url FROM jobs WHERE id = %s''', (job_id, )) try: diff --git a/schema.pgsql b/schema.pgsql index c327c9fb..cc34dbd7 100644 --- a/schema.pgsql +++ b/schema.pgsql @@ -3,6 +3,6 @@ DROP TABLE IF EXISTS jobs; CREATE TABLE jobs ( id VARCHAR(40) PRIMARY KEY, - filenames TEXT ARRAY, + task_files JSON, github_status_url TEXT ); From 390088ab804df5107791a28c42bb90f04f680963 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 10:38:37 -0700 Subject: [PATCH 24/51] Clarified error response and started randomizing job IDs --- app.py | 21 ++++++++++++++------- tests.py | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index 0a2d6109..40489355 100644 --- a/app.py +++ b/app.py @@ -1,7 +1,7 @@ from os.path import relpath, splitext from urlparse import urljoin from base64 import b64decode -from hashlib import sha1 +from uuid import uuid4 import json, os from flask import Flask, request, Response, current_app, jsonify @@ -24,7 +24,7 @@ def index(): def hook(): github_auth = current_app.config['GITHUB_AUTH'] webhook_payload = json.loads(request.data) - files = process_payload(webhook_payload, github_auth) + files = process_payload_files(webhook_payload, github_auth) status_url = get_status_url(webhook_payload) if not files: @@ -43,7 +43,8 @@ def hook(): except Exception as e: # Oops, tell Github something went wrong. update_error_status(status_url, str(e), files.keys(), github_auth) - return jsonify({'error': str(e), 'files': files}) + return Response(json.dumps({'error': str(e), 'files': files}), + 500, content_type='application/json') else: # That worked, remember them in the database. save_job(db, job_id, tasks, status_url) @@ -129,10 +130,10 @@ def get_touched_branch_files(payload, github_auth): return touched -def process_payload(payload, github_auth): +def process_payload_files(payload, github_auth): ''' Return a dictionary of file paths and raw JSON contents. ''' - processed = dict() + files = dict() touched = get_touched_payload_files(payload) touched |= get_touched_branch_files(payload, github_auth) @@ -166,11 +167,11 @@ def process_payload(payload, github_auth): current_app.logger.debug('Contents SHA {}'.format(contents['sha'])) if encoding == 'base64': - processed[filename] = b64decode(content) + files[filename] = b64decode(content) else: raise ValueError('Unrecognized encoding "{}"'.format(encoding)) - return processed + return files def get_status_url(payload): ''' Get Github status API URL from webhook payload. @@ -244,6 +245,12 @@ def update_success_status(status_url, job_url, filenames, github_auth): def calculate_job_id(files): ''' ''' + return str(uuid4()) + + # + # Previously, we created a deterministic hash of + # the files, but for now that might be too optimistic. + # blob = json.dumps(files, ensure_ascii=True, sort_keys=True) job_id = sha1(blob).hexdigest() diff --git a/tests.py b/tests.py index 35d08133..b445b2d2 100644 --- a/tests.py +++ b/tests.py @@ -174,7 +174,7 @@ def job_queue_fails(*args, **kwargs): with HTTMock(self.response_content): posted = self.client.post('/hook', data=data) - self.assertEqual(posted.status_code, 200) + self.assertEqual(posted.status_code, 500) self.assertEqual(self.last_status_state, 'error') self.assertTrue('us-ca-santa_clara_county' in self.last_status_message, 'Santa Clara County source should be present in error message') self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') From 606c7d5cfe7d11d3bec46f9567b64fed640d5531 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 10:47:11 -0700 Subject: [PATCH 25/51] oops --- recreate-db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recreate-db.py b/recreate-db.py index 48c49603..d3b7e6d8 100644 --- a/recreate-db.py +++ b/recreate-db.py @@ -14,7 +14,7 @@ def main(DATABASE_URL): with open(schema_filename) as file: db.execute(file.read()) - db.execute('DROP TABLE queue') + db.execute('DROP TABLE IF EXISTS queue') pq = PQ(conn, table='queue') pq.create() From d5a8260cc379f83e4e6989898ad00084c3be00b7 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 11:56:29 -0700 Subject: [PATCH 26/51] Implemented reverse-direction done queue with Github status update Does not yet work with two-task jobs. --- app.py | 64 +++++++++++++++++++++++++---------------------- tests.py | 75 +++++++++++++++++++++++++++++++++++++------------------- 2 files changed, 85 insertions(+), 54 deletions(-) diff --git a/app.py b/app.py index 40489355..5e68bbee 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,5 @@ -from os.path import relpath, splitext -from urlparse import urljoin +from os.path import relpath, splitext, basename +from urlparse import urljoin, urlparse from base64 import b64decode from uuid import uuid4 import json, os @@ -15,6 +15,7 @@ app.config['DATABASE_URL'] = os.environ['DATABASE_URL'] MAGIC_OK_MESSAGE = 'Everything is fine' +TASK_QUEUE, DONE_QUEUE = 'tasks', 'finished' @app.route('/') def index(): @@ -51,29 +52,6 @@ def hook(): update_pending_status(status_url, job_url, files.keys(), github_auth) return jsonify({'url': job_url, 'files': files}) -@app.route('/jobs/', methods=['POST']) -def post_job(job_id): - ''' - ''' - with db_connect(current_app) as conn: - with db_cursor(conn) as db: - try: - task_files, status_url = read_job(db, job_id) - except TypeError: - return Response('Job {} not found'.format(job_id), 404) - - github_auth = current_app.config['GITHUB_AUTH'] - filenames = list(task_files.values()) - message = request.data - job_url = request.url - - if message == MAGIC_OK_MESSAGE: - update_success_status(status_url, job_url, filenames, github_auth) - else: - update_failing_status(status_url, job_url, message, filenames, github_auth) - - return 'Job updated' - @app.route('/jobs/', methods=['GET']) def get_job(job_id): ''' @@ -293,11 +271,39 @@ def read_job(db, job_id): else: return filenames, github_status_url -def db_connect(app): - return connect(app.config['DATABASE_URL']) +def pop_finished_task_from_queue(queue, github_auth): + ''' + ''' + with queue as db: + task = queue.get() + + if task is None: + return + + job_url = task.data['url'] + message = task.data['result']['message'] + job_id = basename(urlparse(job_url).path) + + try: + task_files, status_url = read_job(db, job_id) + except TypeError: + raise Exception('Job {} not found'.format(job_id)) + + filenames = list(task_files.values()) + + if message == MAGIC_OK_MESSAGE: + update_success_status(status_url, job_url, filenames, github_auth) + else: + update_failing_status(status_url, job_url, message, filenames, github_auth) + +def db_connect(app_or_dsn): + ''' Connect to database using Flask app instance or DSN string. + ''' + dsn = app.config['DATABASE_URL'] if hasattr(app, 'config') else app_or_dsn + return connect(dsn) -def db_queue(conn): - return PQ(conn, table='queue')['tasks'] +def db_queue(conn, name=None): + return PQ(conn, table='queue')[name or TASK_QUEUE] def db_cursor(conn): return conn.cursor() diff --git a/tests.py b/tests.py index b445b2d2..43c24c30 100644 --- a/tests.py +++ b/tests.py @@ -10,7 +10,11 @@ os.environ['GITHUB_TOKEN'] = '' os.environ['DATABASE_URL'] = 'postgres:///hooked_on_sources' -from app import app, db_connect, db_cursor, MAGIC_OK_MESSAGE +from app import ( + app, db_connect, db_cursor, db_queue, pop_finished_task_from_queue, + TASK_QUEUE, DONE_QUEUE, MAGIC_OK_MESSAGE + ) + recreate_db = __import__('recreate-db') class TestHook (unittest.TestCase): @@ -20,6 +24,8 @@ def setUp(self): ''' recreate_db.main(app.config['DATABASE_URL']) + self.database_url = app.config['DATABASE_URL'] + self.github_auth = app.config['GITHUB_AUTH'] self.client = app.test_client() self.last_status_state = None self.last_status_message = None @@ -110,20 +116,26 @@ def test_webhook_one_master_commit(self): self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') - # Pretend that queued job completed successfully. - last_job_url = urlparse(json.loads(posted.data).get('url')) + # Look for the task in the task queue. + with db_connect(self.database_url) as conn: + task = db_queue(conn, TASK_QUEUE).get() + + self.assertTrue('us-ca-alameda_county' in task.data['name']) - with HTTMock(self.response_content): - got = self.client.get(last_job_url.path) - self.assertEqual(got.status_code, 200) - - posted = self.client.post(last_job_url.path, data=MAGIC_OK_MESSAGE) - self.assertTrue(posted.status_code in range(200, 299)) + # This is the JSON source payload, just make sure it parses. + content = json.loads(task.data.pop('content')) + task.data['result'] = dict(message=MAGIC_OK_MESSAGE) - got = self.client.get(last_job_url.path) - self.assertEqual(got.status_code, 200) - - self.assertEqual(self.last_status_state, 'success') + # Put back a completion task to the done queue. + with db_connect(self.database_url) as conn: + db_queue(conn, DONE_QUEUE).put(task.data) + + # Handle completion task and check with Github. + with HTTMock(self.response_content): + pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) + + # Check that Github knows the job to have been completed successfully. + self.assertEqual(self.last_status_state, 'success') def test_webhook_two_master_commits(self): ''' Push two commits with San Francisco and Berkeley sources directly to master. @@ -141,21 +153,34 @@ def test_webhook_two_master_commits(self): self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') - last_job_url = urlparse(json.loads(posted.data).get('url')) + for message in ('Something went wrong', MAGIC_OK_MESSAGE): + # Look for the task in the task queue. + with db_connect(self.database_url) as conn: + task = db_queue(conn, TASK_QUEUE).get() + + self.assertTrue('us-ca-san_francisco' in task.data['name'] or 'us-ca-berkeley' in task.data['name']) - with HTTMock(self.response_content): - got = self.client.get(last_job_url.path) - self.assertEqual(got.status_code, 200) - - # Pretend that queued job failed. - posted = self.client.post(last_job_url.path, data='Something went wrong') - self.assertTrue(posted.status_code in range(200, 299)) + # This is the JSON source payload, just make sure it parses. + content = json.loads(task.data.pop('content')) + task.data['result'] = dict(message=message) - got = self.client.get(last_job_url.path) - self.assertEqual(got.status_code, 200) + # Put back a completion task to the done queue. + with db_connect(self.database_url) as conn: + db_queue(conn, DONE_QUEUE).put(task.data) + + # Handle completion tasks and check with Github. + with db_connect(self.database_url) as conn: + with HTTMock(self.response_content): + pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) + + # Check that Github hasn't been prematurely told about job failure. + self.assertEqual(self.last_status_state, 'pending') + + pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) - self.assertEqual(self.last_status_state, 'failure') - self.assertTrue('Something went wrong' in self.last_status_message) + # Check that Github knows the job to have been completed unsuccessfully. + self.assertEqual(self.last_status_state, 'failure') + self.assertTrue('Something went wrong' in self.last_status_message) def test_webhook_two_branch_commits(self): ''' Push two commits with addition and removal of Polish source to a branch. From 24a2d3690c8906e54697a9f50030f715160d8410 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 12:33:13 -0700 Subject: [PATCH 27/51] Added concept of per-task file states for jobs status --- app.py | 55 ++++++++++++++++++++++++++++++++++++---------------- schema.pgsql | 1 + tests.py | 6 +----- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/app.py b/app.py index 5e68bbee..395c9bd9 100644 --- a/app.py +++ b/app.py @@ -48,7 +48,8 @@ def hook(): 500, content_type='application/json') else: # That worked, remember them in the database. - save_job(db, job_id, tasks, status_url) + states = {name: None for name in files.keys()} + add_job(db, job_id, tasks, states, status_url) update_pending_status(status_url, job_url, files.keys(), github_auth) return jsonify({'url': job_url, 'files': files}) @@ -194,11 +195,11 @@ def update_error_status(status_url, message, filenames, github_auth): return post_github_status(status_url, status, github_auth) -def update_failing_status(status_url, job_url, message, filenames, github_auth): +def update_failing_status(status_url, job_url, bad_files, filenames, github_auth): ''' Push failing status for head commit to Github status API. ''' status = dict(context='openaddresses/hooked', state='failure', - description='Failed on {}: {}'.format(', '.join(filenames), message), + description='Failed on {} from {}'.format(', '.join(bad_files), ', '.join(filenames)), target_url=job_url) return post_github_status(status_url, status, github_auth) @@ -246,30 +247,38 @@ def add_files_to_queue(queue, job_url, files): return tasks -def save_job(db, job_id, task_files, status_url): +def add_job(db, job_id, task_files, file_states, status_url): ''' Save information about a job to the database. Throws an IntegrityError exception if the job ID exists. ''' db.execute('''INSERT INTO jobs - (task_files, github_status_url, id) - VALUES (%s::json, %s, %s)''', - (json.dumps(task_files), status_url, job_id)) + (task_files, file_states, github_status_url, id) + VALUES (%s::json, %s::json, %s, %s)''', + (json.dumps(task_files), json.dumps(file_states), status_url, job_id)) + +def write_job(db, job_id, task_files, file_states, status_url): + ''' Save information about a job to the database. + ''' + db.execute('''UPDATE jobs + SET task_files=%s::json, file_states=%s::json, github_status_url=%s + WHERE id = %s''', + (json.dumps(task_files), json.dumps(file_states), status_url, job_id)) def read_job(db, job_id): ''' Read information about a job from the database. - Returns (task_files, github_status_url) or None. + Returns (task_files, file_states, github_status_url) or None. ''' - db.execute('''SELECT task_files, github_status_url + db.execute('''SELECT task_files, file_states, github_status_url FROM jobs WHERE id = %s''', (job_id, )) try: - filenames, github_status_url = db.fetchone() + filenames, states, github_status_url = db.fetchone() except TypeError: return None else: - return filenames, github_status_url + return filenames, states, github_status_url def pop_finished_task_from_queue(queue, github_auth): ''' @@ -280,21 +289,33 @@ def pop_finished_task_from_queue(queue, github_auth): if task is None: return - job_url = task.data['url'] message = task.data['result']['message'] + job_url = task.data['url'] + filename = task.data['name'] job_id = basename(urlparse(job_url).path) try: - task_files, status_url = read_job(db, job_id) + task_files, file_states, status_url = read_job(db, job_id) except TypeError: raise Exception('Job {} not found'.format(job_id)) + if filename not in file_states: + raise Exception('Unknown file from job {}: "{}"'.format(job_id, filename)) + filenames = list(task_files.values()) - - if message == MAGIC_OK_MESSAGE: - update_success_status(status_url, job_url, filenames, github_auth) + file_states[filename] = bool(message == MAGIC_OK_MESSAGE) + + write_job(db, job_id, task_files, file_states, status_url) + + if False in file_states.values(): + bad_files = [name for (name, state) in file_states.items() if state is False] + update_failing_status(status_url, job_url, bad_files, filenames, github_auth) + + elif None in file_states.values(): + update_pending_status(status_url, job_url, filenames, github_auth) + else: - update_failing_status(status_url, job_url, message, filenames, github_auth) + update_success_status(status_url, job_url, filenames, github_auth) def db_connect(app_or_dsn): ''' Connect to database using Flask app instance or DSN string. diff --git a/schema.pgsql b/schema.pgsql index cc34dbd7..253ba7d2 100644 --- a/schema.pgsql +++ b/schema.pgsql @@ -4,5 +4,6 @@ CREATE TABLE jobs ( id VARCHAR(40) PRIMARY KEY, task_files JSON, + file_states JSON, github_status_url TEXT ); diff --git a/tests.py b/tests.py index 43c24c30..ee0a5b4f 100644 --- a/tests.py +++ b/tests.py @@ -172,15 +172,11 @@ def test_webhook_two_master_commits(self): with db_connect(self.database_url) as conn: with HTTMock(self.response_content): pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) - - # Check that Github hasn't been prematurely told about job failure. - self.assertEqual(self.last_status_state, 'pending') - pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) # Check that Github knows the job to have been completed unsuccessfully. self.assertEqual(self.last_status_state, 'failure') - self.assertTrue('Something went wrong' in self.last_status_message) + self.assertTrue('Failed' in self.last_status_message) def test_webhook_two_branch_commits(self): ''' Push two commits with addition and removal of Polish source to a branch. From 79443f1968fa5d396a6749f336f58f78022ac261 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 12:41:06 -0700 Subject: [PATCH 28/51] Added tests to check sequence of status updates based on task results --- tests.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/tests.py b/tests.py index ee0a5b4f..f8e6f27f 100644 --- a/tests.py +++ b/tests.py @@ -29,7 +29,6 @@ def setUp(self): self.client = app.test_client() self.last_status_state = None self.last_status_message = None - self.last_job_url = None handler = StreamHandler(stream=sys.stderr) handler.setLevel(DEBUG) @@ -137,7 +136,7 @@ def test_webhook_one_master_commit(self): # Check that Github knows the job to have been completed successfully. self.assertEqual(self.last_status_state, 'success') - def test_webhook_two_master_commits(self): + def test_webhook_two_master_commits_early_fail(self): ''' Push two commits with San Francisco and Berkeley sources directly to master. ''' data = '''{\r "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "base_ref": null, \r "before": "e91fbc420f08890960f50f863626e1062f922522", \r "commits": [\r {\r "added": [\r "sources/us-ca-san_francisco.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "73a81c5b337bd393273a222f1cd191d7e634df51", \r "message": "Added SF", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:45-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }, \r {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007964, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' @@ -168,16 +167,57 @@ def test_webhook_two_master_commits(self): with db_connect(self.database_url) as conn: db_queue(conn, DONE_QUEUE).put(task.data) - # Handle completion tasks and check with Github. - with db_connect(self.database_url) as conn: - with HTTMock(self.response_content): - pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) - pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) - - # Check that Github knows the job to have been completed unsuccessfully. + # Handle completion tasks and check with Github. + with HTTMock(self.response_content): + pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) + + # Check that Github already knows the job to have been completed unsuccessfully. self.assertEqual(self.last_status_state, 'failure') self.assertTrue('Failed' in self.last_status_message) + def test_webhook_two_master_commits_late_fail(self): + ''' Push two commits with San Francisco and Berkeley sources directly to master. + ''' + data = '''{\r "after": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "base_ref": null, \r "before": "e91fbc420f08890960f50f863626e1062f922522", \r "commits": [\r {\r "added": [\r "sources/us-ca-san_francisco.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "73a81c5b337bd393273a222f1cd191d7e634df51", \r "message": "Added SF", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:45-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }, \r {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ], \r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/e91fbc420f08...ded44ed5f173", \r "created": false, \r "deleted": false, \r "forced": false, \r "head_commit": {\r "added": [\r "sources/us-ca-berkeley.json"\r ], \r "author": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "committer": {\r "email": "mike@teczno.com", \r "name": "Michal Migurski", \r "username": "migurski"\r }, \r "distinct": true, \r "id": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa", \r "message": "Added Berkeley", \r "modified": [], \r "removed": [], \r "timestamp": "2015-04-25T17:25:55-07:00", \r "url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }, \r "organization": {\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3", \r "description": "The free and open global address collection ", \r "events_url": "https://api.github.com/orgs/openaddresses/events", \r "id": 6895392, \r "login": "openaddresses", \r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}", \r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}", \r "repos_url": "https://api.github.com/orgs/openaddresses/repos", \r "url": "https://api.github.com/orgs/openaddresses"\r }, \r "pusher": {\r "email": "mike-github@teczno.com", \r "name": "migurski"\r }, \r "ref": "refs/heads/master", \r "repository": {\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}", \r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}", \r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}", \r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}", \r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git", \r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}", \r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}", \r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}", \r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}", \r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}", \r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors", \r "created_at": 1430006167, \r "default_branch": "master", \r "description": "Temporary repository for testing Github webhook features", \r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads", \r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events", \r "fork": false, \r "forks": 0, \r "forks_count": 0, \r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks", \r "full_name": "openaddresses/hooked-on-sources", \r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}", \r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}", \r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}", \r "git_url": "git://github.com/openaddresses/hooked-on-sources.git", \r "has_downloads": true, \r "has_issues": true, \r "has_pages": false, \r "has_wiki": true, \r "homepage": null, \r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks", \r "html_url": "https://github.com/openaddresses/hooked-on-sources", \r "id": 34590951, \r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}", \r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}", \r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}", \r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}", \r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}", \r "language": null, \r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages", \r "master_branch": "master", \r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges", \r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}", \r "mirror_url": null, \r "name": "hooked-on-sources", \r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}", \r "open_issues": 0, \r "open_issues_count": 0, \r "organization": "openaddresses", \r "owner": {\r "email": "openaddresses@gmail.com", \r "name": "openaddresses"\r }, \r "private": false, \r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}", \r "pushed_at": 1430007964, \r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}", \r "size": 0, \r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git", \r "stargazers": 0, \r "stargazers_count": 0, \r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers", \r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}", \r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers", \r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription", \r "svn_url": "https://github.com/openaddresses/hooked-on-sources", \r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags", \r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams", \r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}", \r "updated_at": "2015-04-25T23:56:07Z", \r "url": "https://github.com/openaddresses/hooked-on-sources", \r "watchers": 0, \r "watchers_count": 0\r }, \r "sender": {\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{/privacy}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{/other_user}", \r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }\r }''' + + with HTTMock(self.response_content): + posted = self.client.post('/hook', data=data) + + self.assertEqual(posted.status_code, 200) + self.assertEqual(self.last_status_state, 'pending') + self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') + self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') + self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') + self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') + self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') + + for (index, message) in enumerate((MAGIC_OK_MESSAGE, 'Something went wrong')): + # Look for the task in the task queue. + with db_connect(self.database_url) as conn: + task = db_queue(conn, TASK_QUEUE).get() + + self.assertTrue('us-ca-san_francisco' in task.data['name'] or 'us-ca-berkeley' in task.data['name']) + + # This is the JSON source payload, just make sure it parses. + content = json.loads(task.data.pop('content')) + task.data['result'] = dict(message=message) + + # Put back a completion task to the done queue. + with db_connect(self.database_url) as conn: + db_queue(conn, DONE_QUEUE).put(task.data) + + # Handle completion tasks and check with Github. + with HTTMock(self.response_content): + pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) + + if index == 0: + # Check that Github knows the job to have been completed unsuccessfully. + self.assertEqual(self.last_status_state, 'pending') + else: + # Check that Github knows the job to have been completed unsuccessfully. + self.assertEqual(self.last_status_state, 'failure') + self.assertTrue('Failed' in self.last_status_message) + def test_webhook_two_branch_commits(self): ''' Push two commits with addition and removal of Polish source to a branch. From 051af25b7edca8e3bed58510a5073314cbfdcb8e Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 12:53:41 -0700 Subject: [PATCH 29/51] Cleaned up reliance on parsed job URL in pop_finished_task_from_queue() --- app.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index 395c9bd9..b8d58852 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,5 @@ -from os.path import relpath, splitext, basename -from urlparse import urljoin, urlparse +from os.path import relpath, splitext +from urlparse import urljoin from base64 import b64decode from uuid import uuid4 import json, os @@ -32,6 +32,9 @@ def hook(): update_empty_status(status_url, github_auth) return jsonify({'url': None, 'files': []}) + filenames = list(files.keys()) + file_states = {name: None for name in filenames} + job_id = calculate_job_id(files) job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) @@ -40,18 +43,17 @@ def hook(): with queue as db: try: # Add the touched files to a task queue. - tasks = add_files_to_queue(queue, job_url, files) + task_files = add_files_to_queue(queue, job_id, job_url, files) except Exception as e: # Oops, tell Github something went wrong. - update_error_status(status_url, str(e), files.keys(), github_auth) + update_error_status(status_url, str(e), filenames, github_auth) return Response(json.dumps({'error': str(e), 'files': files}), 500, content_type='application/json') else: # That worked, remember them in the database. - states = {name: None for name in files.keys()} - add_job(db, job_id, tasks, states, status_url) - update_pending_status(status_url, job_url, files.keys(), github_auth) - return jsonify({'url': job_url, 'files': files}) + add_job(db, job_id, task_files, file_states, status_url) + update_pending_status(status_url, job_url, filenames, github_auth) + return jsonify({'id': job_id, 'url': job_url, 'files': files}) @app.route('/jobs/', methods=['GET']) def get_job(job_id): @@ -235,13 +237,13 @@ def calculate_job_id(files): return job_id -def add_files_to_queue(queue, job_url, files): +def add_files_to_queue(queue, job_id, job_url, files): ''' Make a new task for each file, return dict of taks IDs to file names. ''' tasks = {} for (name, content) in files.items(): - task = queue.put(dict(url=job_url, name=name, content=content)) + task = queue.put(dict(id=job_id, url=job_url, name=name, content=content)) tasks[str(task)] = name @@ -292,7 +294,7 @@ def pop_finished_task_from_queue(queue, github_auth): message = task.data['result']['message'] job_url = task.data['url'] filename = task.data['name'] - job_id = basename(urlparse(job_url).path) + job_id = task.data['id'] try: task_files, file_states, status_url = read_job(db, job_id) From 2ebfdc745a27c0dbf58bce381cc6756c4a420ca6 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 14:47:27 -0700 Subject: [PATCH 30/51] Added jobs.file_results column --- app.py | 36 +++++++++++++++++++++--------------- schema.pgsql | 1 + tests.py | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/app.py b/app.py index b8d58852..415b087d 100644 --- a/app.py +++ b/app.py @@ -34,6 +34,7 @@ def hook(): filenames = list(files.keys()) file_states = {name: None for name in filenames} + file_results = {name: None for name in filenames} job_id = calculate_job_id(files) job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) @@ -51,7 +52,7 @@ def hook(): 500, content_type='application/json') else: # That worked, remember them in the database. - add_job(db, job_id, task_files, file_states, status_url) + add_job(db, job_id, task_files, file_states, file_results, status_url) update_pending_status(status_url, job_url, filenames, github_auth) return jsonify({'id': job_id, 'url': job_url, 'files': files}) @@ -249,38 +250,41 @@ def add_files_to_queue(queue, job_id, job_url, files): return tasks -def add_job(db, job_id, task_files, file_states, status_url): +def add_job(db, job_id, task_files, file_states, file_results, status_url): ''' Save information about a job to the database. Throws an IntegrityError exception if the job ID exists. ''' db.execute('''INSERT INTO jobs - (task_files, file_states, github_status_url, id) - VALUES (%s::json, %s::json, %s, %s)''', - (json.dumps(task_files), json.dumps(file_states), status_url, job_id)) + (task_files, file_states, file_results, github_status_url, id) + VALUES (%s::json, %s::json, %s::json, %s, %s)''', + (json.dumps(task_files), json.dumps(file_states), + json.dumps(file_results), status_url, job_id)) -def write_job(db, job_id, task_files, file_states, status_url): +def write_job(db, job_id, task_files, file_states, file_results, status_url): ''' Save information about a job to the database. ''' db.execute('''UPDATE jobs - SET task_files=%s::json, file_states=%s::json, github_status_url=%s + SET task_files=%s::json, file_states=%s::json, + file_results=%s::json, github_status_url=%s WHERE id = %s''', - (json.dumps(task_files), json.dumps(file_states), status_url, job_id)) + (json.dumps(task_files), json.dumps(file_states), + json.dumps(file_results), status_url, job_id)) def read_job(db, job_id): ''' Read information about a job from the database. - Returns (task_files, file_states, github_status_url) or None. + Returns (task_files, file_states, file_results, github_status_url) or None. ''' - db.execute('''SELECT task_files, file_states, github_status_url + db.execute('''SELECT task_files, file_states, file_results, github_status_url FROM jobs WHERE id = %s''', (job_id, )) try: - filenames, states, github_status_url = db.fetchone() + filenames, states, file_results, github_status_url = db.fetchone() except TypeError: return None else: - return filenames, states, github_status_url + return filenames, states, file_results, github_status_url def pop_finished_task_from_queue(queue, github_auth): ''' @@ -291,13 +295,14 @@ def pop_finished_task_from_queue(queue, github_auth): if task is None: return - message = task.data['result']['message'] + results = task.data['result'] + message = results['message'] job_url = task.data['url'] filename = task.data['name'] job_id = task.data['id'] try: - task_files, file_states, status_url = read_job(db, job_id) + task_files, file_states, file_results, status_url = read_job(db, job_id) except TypeError: raise Exception('Job {} not found'.format(job_id)) @@ -306,8 +311,9 @@ def pop_finished_task_from_queue(queue, github_auth): filenames = list(task_files.values()) file_states[filename] = bool(message == MAGIC_OK_MESSAGE) + file_results[filename] = results - write_job(db, job_id, task_files, file_states, status_url) + write_job(db, job_id, task_files, file_states, file_results, status_url) if False in file_states.values(): bad_files = [name for (name, state) in file_states.items() if state is False] diff --git a/schema.pgsql b/schema.pgsql index 253ba7d2..9545f0c8 100644 --- a/schema.pgsql +++ b/schema.pgsql @@ -5,5 +5,6 @@ CREATE TABLE jobs id VARCHAR(40) PRIMARY KEY, task_files JSON, file_states JSON, + file_results JSON, github_status_url TEXT ); diff --git a/tests.py b/tests.py index f8e6f27f..5c1b6f08 100644 --- a/tests.py +++ b/tests.py @@ -211,7 +211,7 @@ def test_webhook_two_master_commits_late_fail(self): pop_finished_task_from_queue(db_queue(conn, DONE_QUEUE), self.github_auth) if index == 0: - # Check that Github knows the job to have been completed unsuccessfully. + # Check that Github still thinks it's pending. self.assertEqual(self.last_status_state, 'pending') else: # Check that Github knows the job to have been completed unsuccessfully. From 03b5b06c966cb2c9fa3749fd1545d8c48ba28c6b Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 14:54:57 -0700 Subject: [PATCH 31/51] Added jobs.status column --- app.py | 43 ++++++++++++++++++++++++++----------------- schema.pgsql | 1 + 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/app.py b/app.py index 415b087d..b2bb55bf 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ def hook(): job_id = calculate_job_id(files) job_url = urljoin(request.url, '/jobs/{id}'.format(id=job_id)) + job_status = None with db_connect(current_app) as conn: queue = db_queue(conn) @@ -52,7 +53,7 @@ def hook(): 500, content_type='application/json') else: # That worked, remember them in the database. - add_job(db, job_id, task_files, file_states, file_results, status_url) + add_job(db, job_id, job_status, task_files, file_states, file_results, status_url) update_pending_status(status_url, job_url, filenames, github_auth) return jsonify({'id': job_id, 'url': job_url, 'files': files}) @@ -250,41 +251,41 @@ def add_files_to_queue(queue, job_id, job_url, files): return tasks -def add_job(db, job_id, task_files, file_states, file_results, status_url): +def add_job(db, job_id, status, task_files, file_states, file_results, status_url): ''' Save information about a job to the database. Throws an IntegrityError exception if the job ID exists. ''' db.execute('''INSERT INTO jobs - (task_files, file_states, file_results, github_status_url, id) - VALUES (%s::json, %s::json, %s::json, %s, %s)''', + (task_files, file_states, file_results, github_status_url, status, id) + VALUES (%s::json, %s::json, %s::json, %s, %s, %s)''', (json.dumps(task_files), json.dumps(file_states), - json.dumps(file_results), status_url, job_id)) + json.dumps(file_results), status_url, status, job_id)) -def write_job(db, job_id, task_files, file_states, file_results, status_url): +def write_job(db, job_id, status, task_files, file_states, file_results, status_url): ''' Save information about a job to the database. ''' db.execute('''UPDATE jobs SET task_files=%s::json, file_states=%s::json, - file_results=%s::json, github_status_url=%s + file_results=%s::json, github_status_url=%s, status=%s WHERE id = %s''', (json.dumps(task_files), json.dumps(file_states), - json.dumps(file_results), status_url, job_id)) + json.dumps(file_results), status_url, status, job_id)) def read_job(db, job_id): ''' Read information about a job from the database. - Returns (task_files, file_states, file_results, github_status_url) or None. + Returns (status, task_files, file_states, file_results, github_status_url) or None. ''' - db.execute('''SELECT task_files, file_states, file_results, github_status_url + db.execute('''SELECT status, task_files, file_states, file_results, github_status_url FROM jobs WHERE id = %s''', (job_id, )) try: - filenames, states, file_results, github_status_url = db.fetchone() + status, task_files, states, file_results, github_status_url = db.fetchone() except TypeError: return None else: - return filenames, states, file_results, github_status_url + return status, task_files, states, file_results, github_status_url def pop_finished_task_from_queue(queue, github_auth): ''' @@ -302,7 +303,7 @@ def pop_finished_task_from_queue(queue, github_auth): job_id = task.data['id'] try: - task_files, file_states, file_results, status_url = read_job(db, job_id) + _, task_files, file_states, file_results, status_url = read_job(db, job_id) except TypeError: raise Exception('Job {} not found'.format(job_id)) @@ -313,16 +314,24 @@ def pop_finished_task_from_queue(queue, github_auth): file_states[filename] = bool(message == MAGIC_OK_MESSAGE) file_results[filename] = results - write_job(db, job_id, task_files, file_states, file_results, status_url) - if False in file_states.values(): + # Any task failure means the whole job has failed. + job_status = False + elif None in file_states.values(): + job_status = None + else: + job_status = True + + write_job(db, job_id, job_status, task_files, file_states, file_results, status_url) + + if job_status is False: bad_files = [name for (name, state) in file_states.items() if state is False] update_failing_status(status_url, job_url, bad_files, filenames, github_auth) - elif None in file_states.values(): + elif job_status is None: update_pending_status(status_url, job_url, filenames, github_auth) - else: + elif job_status is True: update_success_status(status_url, job_url, filenames, github_auth) def db_connect(app_or_dsn): diff --git a/schema.pgsql b/schema.pgsql index 9545f0c8..753a8cb3 100644 --- a/schema.pgsql +++ b/schema.pgsql @@ -3,6 +3,7 @@ DROP TABLE IF EXISTS jobs; CREATE TABLE jobs ( id VARCHAR(40) PRIMARY KEY, + status BOOLEAN, task_files JSON, file_states JSON, file_results JSON, From e73842b19c29e855164e1dc6e0a1d4d67e465511 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 15:06:10 -0700 Subject: [PATCH 32/51] Returning a meaningful single job HTTP response --- app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index b2bb55bf..4b957405 100644 --- a/app.py +++ b/app.py @@ -63,10 +63,15 @@ def get_job(job_id): ''' with db_connect(current_app) as conn: with db_cursor(conn) as db: - if read_job(db, job_id) is None: + try: + status, task_files, file_states, file_results, github_status_url = read_job(db, job_id) + except TypeError: return Response('Job {} not found'.format(job_id), 404) - return 'I am a job' + job = dict(status=status, task_files=task_files, file_states=file_states, + file_results=file_results, github_status_url=github_status_url) + + return jsonify(job) def get_touched_payload_files(payload): ''' Return a set of files modified in payload commits. From 9c9241838d2300748d4a0241b0e6bc13c9a5d31c Mon Sep 17 00:00:00 2001 From: Nelson Minar Date: Sat, 2 May 2015 23:18:58 +0000 Subject: [PATCH 33/51] Very simple working worker daemon to run jobs from the queue. --- lib.py | 19 +++++++++++++ worker.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 lib.py create mode 100755 worker.py diff --git a/lib.py b/lib.py new file mode 100644 index 00000000..18b4f849 --- /dev/null +++ b/lib.py @@ -0,0 +1,19 @@ +# TODO: factor out of app.py + +from psycopg2 import connect +from pq import PQ + +# TODO factor this cleanly out of app.py + +TASK_QUEUE, DONE_QUEUE = 'tasks', 'finished' + +def db_connect(dsn): + ''' Connect to database using Flask app instance or DSN string. + ''' + return connect(dsn) + +def db_queue(conn, name=None): + return PQ(conn, table='queue')[name or TASK_QUEUE] + +def db_cursor(conn): + return conn.cursor() diff --git a/worker.py b/worker.py new file mode 100755 index 00000000..b01596c5 --- /dev/null +++ b/worker.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python2 + +"""Simple worker process to process OpenAddress sources on the task queue + +Jobs get enqueued to a PQ task queue by some other system. +This program pops jobs and runs them one at a time, then +enqueues a new message on a separate PQ queue when the work is done.""" + +import time, os, subprocess, psycopg2 +import lib + +# File path and URL path for result directory. Should be S3. +_web_output_dir = '/var/www/html/oa-runone' +_web_base_url = 'http://minar.us.to/oa-runone' + +def do_work(job_id, job_contents): + "Do the actual work of running a source file in job_contents" + + # Make a directory to run the whole job + assert '/' not in job_id + out_dir = '%s/%s' % (_web_output_dir, job_id) + os.mkdir(out_dir) + + # Write the user input to a file + out_fn = '%s/user_input.txt' % out_dir + with file(out_fn, 'wb') as out_fp: + out_fp.write(job_contents) + + # Make a directory in which to run openaddr + oa_dir = '%s/%s' % (out_dir, 'out') + os.mkdir(oa_dir) + + # Invoke the job to do + result_code = subprocess.call(('openaddr-process-one', + '-l', '%s/logfile.txt' % out_dir, + out_fn, + oa_dir)) + + # Prepare return parameters + r = { 'result_code': result_code, + 'output_url': '%s/%s' % (_web_base_url, job_id), + 'message': 'A Zircon princess seemed to lost her senses' } + + return r + +def run(task_data): + "Run a task posted to the queue. Handles the JSON for task and result objects." + + print "Got job %s" % task_data + + # Unpack the task object + content = task_data['content'] + url = task_data['url'] + job_id = task_data['id'] + name = task_data['name'] + + # Run the task + result = do_work(job_id, content) + + # Prepare a result object + r = { 'id' : job_id, + 'url': url, + 'name': name, + 'result' : result } + return r + +def serve_queue(): + "Single threaded worker to serve the job queue" + + # Connect to the queue + conn = lib.db_connect(os.environ['DATABASE_STRING']) + input_queue = lib.db_queue(conn) + output_queue = lib.db_queue(conn, lib.DONE_QUEUE) + + # Fetch and run jobs in a loop + while True: + task = input_queue.get() + # PQ will return NULL after 1 second timeout if not ask + if task: + task_output_data = run(task.data) + output_queue.put(task_output_data) + + +if __name__ == '__main__': + serve_queue() From 892629bf95fa92f7b4bcfdc79d6e000292b9236f Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 2 May 2015 17:04:33 -0700 Subject: [PATCH 34/51] Added dequeueing job --- Procfile | 3 ++- app.py | 7 +++++-- run-dequeue.py | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 run-dequeue.py diff --git a/Procfile b/Procfile index ae36de7b..8861b29c 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,2 @@ -web: gunicorn -w 4 --bind 0.0.0.0:$PORT app:app \ No newline at end of file +web: gunicorn -w 4 --bind 0.0.0.0:$PORT app:app +dequeue: python run-dequeue.py diff --git a/app.py b/app.py index 4b957405..e0ac6738 100644 --- a/app.py +++ b/app.py @@ -10,9 +10,12 @@ from psycopg2 import connect from pq import PQ +def load_config(): + return dict(GITHUB_AUTH=(os.environ['GITHUB_TOKEN'], 'x-oauth-basic'), + DATABASE_URL=os.environ['DATABASE_URL']) + app = Flask(__name__) -app.config['GITHUB_AUTH'] = os.environ['GITHUB_TOKEN'], 'x-oauth-basic' -app.config['DATABASE_URL'] = os.environ['DATABASE_URL'] +app.config.update(load_config()) MAGIC_OK_MESSAGE = 'Everything is fine' TASK_QUEUE, DONE_QUEUE = 'tasks', 'finished' diff --git a/run-dequeue.py b/run-dequeue.py new file mode 100644 index 00000000..493e4aa0 --- /dev/null +++ b/run-dequeue.py @@ -0,0 +1,26 @@ +from sys import stderr +from time import sleep +from traceback import print_exc + +from app import ( + db_connect, db_queue, pop_finished_task_from_queue, DONE_QUEUE, load_config + ) + +def main(): + ''' + ''' + config = load_config() + + while True: + try: + with db_connect(config['DATABASE_URL']) as conn: + queue = db_queue(conn, DONE_QUEUE) + pop_finished_task_from_queue(queue, config['GITHUB_AUTH']) + except: + print >> stderr, '-' * 40 + print_exc(file=stderr) + print >> stderr, '-' * 40 + sleep(5) + +if __name__ == '__main__': + exit(main()) \ No newline at end of file From a0520887484c72923d1ff67f13e3d9bcd58fa30f Mon Sep 17 00:00:00 2001 From: Nelson Minar Date: Sun, 3 May 2015 00:34:24 +0000 Subject: [PATCH 35/51] Add the MAGIC_OK_MESSAGE to the worker. Sending the Zircon Princess to eternal rest. --- lib.py | 1 + worker.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib.py b/lib.py index 18b4f849..372e3384 100644 --- a/lib.py +++ b/lib.py @@ -6,6 +6,7 @@ # TODO factor this cleanly out of app.py TASK_QUEUE, DONE_QUEUE = 'tasks', 'finished' +MAGIC_OK_MESSAGE = 'Everything is fine' def db_connect(dsn): ''' Connect to database using Flask app instance or DSN string. diff --git a/worker.py b/worker.py index b01596c5..9668532d 100755 --- a/worker.py +++ b/worker.py @@ -39,7 +39,7 @@ def do_work(job_id, job_contents): # Prepare return parameters r = { 'result_code': result_code, 'output_url': '%s/%s' % (_web_base_url, job_id), - 'message': 'A Zircon princess seemed to lost her senses' } + 'message': lib.MAGIC_OK_MESSAGE } return r From 46a1237ef45b5dec15e0cc8bb5572ed40b902b3a Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 5 May 2015 22:31:46 -0700 Subject: [PATCH 36/51] Moved app under openaddr.ci module and tests under openaddr.tests --- openaddr/__init__.py | 0 app.py => openaddr/ci/__init__.py | 0 openaddr/tests/__init__.py | 0 tests.py => openaddr/tests/ci.py | 4 ++-- test.py | 10 ++++++++++ 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 openaddr/__init__.py rename app.py => openaddr/ci/__init__.py (100%) create mode 100644 openaddr/tests/__init__.py rename tests.py => openaddr/tests/ci.py (99%) create mode 100644 test.py diff --git a/openaddr/__init__.py b/openaddr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app.py b/openaddr/ci/__init__.py similarity index 100% rename from app.py rename to openaddr/ci/__init__.py diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests.py b/openaddr/tests/ci.py similarity index 99% rename from tests.py rename to openaddr/tests/ci.py index 5c1b6f08..64554c50 100644 --- a/tests.py +++ b/openaddr/tests/ci.py @@ -10,7 +10,7 @@ os.environ['GITHUB_TOKEN'] = '' os.environ['DATABASE_URL'] = 'postgres:///hooked_on_sources' -from app import ( +from ..ci import ( app, db_connect, db_cursor, db_queue, pop_finished_task_from_queue, TASK_QUEUE, DONE_QUEUE, MAGIC_OK_MESSAGE ) @@ -226,7 +226,7 @@ def test_webhook_two_branch_commits(self): ''' data = '''{\r "ref": "refs/heads/branch",\r "before": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "after": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "created": false,\r "deleted": false,\r "forced": false,\r "base_ref": null,\r "compare": "https://github.com/openaddresses/hooked-on-sources/compare/b659130053b8...e5f1dcae83ab",\r "commits": [\r {\r "id": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "distinct": true,\r "message": "Added Polish source",\r "timestamp": "2015-04-25T17:52:39-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r "sources/pl-dolnoslaskie.json"\r ],\r "removed": [\r\r ],\r "modified": [\r\r ]\r },\r {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r }\r ],\r "head_commit": {\r "id": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "distinct": true,\r "message": "Removed Polish source",\r "timestamp": "2015-04-25T17:52:46-07:00",\r "url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "username": "migurski"\r },\r "added": [\r\r ],\r "removed": [\r "sources/pl-dolnoslaskie.json"\r ],\r "modified": [\r\r ]\r },\r "repository": {\r "id": 34590951,\r "name": "hooked-on-sources",\r "full_name": "openaddresses/hooked-on-sources",\r "owner": {\r "name": "openaddresses",\r "email": "openaddresses@gmail.com"\r },\r "private": false,\r "html_url": "https://github.com/openaddresses/hooked-on-sources",\r "description": "Temporary repository for testing Github webhook features",\r "fork": false,\r "url": "https://github.com/openaddresses/hooked-on-sources",\r "forks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/forks",\r "keys_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/keys{/key_id}",\r "collaborators_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/collaborators{/collaborator}",\r "teams_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/teams",\r "hooks_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/hooks",\r "issue_events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/events{/number}",\r "events_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/events",\r "assignees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/assignees{/user}",\r "branches_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/branches{/branch}",\r "tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/tags",\r "blobs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs{/sha}",\r "git_tags_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/tags{/sha}",\r "git_refs_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/refs{/sha}",\r "trees_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees{/sha}",\r "statuses_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/{sha}",\r "languages_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/languages",\r "stargazers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/stargazers",\r "contributors_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contributors",\r "subscribers_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscribers",\r "subscription_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/subscription",\r "commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits{/sha}",\r "git_commits_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits{/sha}",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/comments{/number}",\r "issue_comment_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues/comments{/number}",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/{+path}",\r "compare_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/{base}...{head}",\r "merges_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/merges",\r "archive_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/{archive_format}{/ref}",\r "downloads_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/downloads",\r "issues_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/issues{/number}",\r "pulls_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/pulls{/number}",\r "milestones_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/milestones{/number}",\r "notifications_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/notifications{?since,all,participating}",\r "labels_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/labels{/name}",\r "releases_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/releases{/id}",\r "created_at": 1430006167,\r "updated_at": "2015-04-25T23:56:07Z",\r "pushed_at": 1430009572,\r "git_url": "git://github.com/openaddresses/hooked-on-sources.git",\r "ssh_url": "git@github.com:openaddresses/hooked-on-sources.git",\r "clone_url": "https://github.com/openaddresses/hooked-on-sources.git",\r "svn_url": "https://github.com/openaddresses/hooked-on-sources",\r "homepage": null,\r "size": 0,\r "stargazers_count": 0,\r "watchers_count": 0,\r "language": null,\r "has_issues": true,\r "has_downloads": true,\r "has_wiki": true,\r "has_pages": false,\r "forks_count": 0,\r "mirror_url": null,\r "open_issues_count": 1,\r "forks": 0,\r "open_issues": 1,\r "watchers": 0,\r "default_branch": "master",\r "stargazers": 0,\r "master_branch": "master",\r "organization": "openaddresses"\r },\r "pusher": {\r "name": "migurski",\r "email": "mike-github@teczno.com"\r },\r "organization": {\r "login": "openaddresses",\r "id": 6895392,\r "url": "https://api.github.com/orgs/openaddresses",\r "repos_url": "https://api.github.com/orgs/openaddresses/repos",\r "events_url": "https://api.github.com/orgs/openaddresses/events",\r "members_url": "https://api.github.com/orgs/openaddresses/members{/member}",\r "public_members_url": "https://api.github.com/orgs/openaddresses/public_members{/member}",\r "avatar_url": "https://avatars.githubusercontent.com/u/6895392?v=3",\r "description": "The free and open global address collection "\r },\r "sender": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r }\r }''' - with patch('app.add_files_to_queue') as add_files_to_queue: + with patch('openaddr.ci.add_files_to_queue') as add_files_to_queue: def job_queue_fails(*args, **kwargs): raise Exception('Nope') diff --git a/test.py b/test.py new file mode 100644 index 00000000..7ec367bb --- /dev/null +++ b/test.py @@ -0,0 +1,10 @@ +# coding=utf8 +""" +Run Python test suite via the standard unittest mechanism. +""" +import unittest + +from openaddr.tests.ci import TestHook + +if __name__ == '__main__': + unittest.main() From 4841297c8412fa346de082d3375583cfb1193fa6 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 5 May 2015 22:41:46 -0700 Subject: [PATCH 37/51] Packaged all of openaddr.ci based on OpenAddresses-Machine using setup.py --- Procfile | 4 +- lib.py | 20 ------- openaddr/VERSION | 1 + recreate-db.py => openaddr/ci/recreate_db.py | 12 ++-- run-dequeue.py => openaddr/ci/run_dequeue.py | 2 +- schema.pgsql => openaddr/ci/schema.pgsql | 0 worker.py => openaddr/ci/worker.py | 15 ++--- openaddr/tests/ci.py | 4 +- requirements.txt | 14 +---- setup.py | 63 ++++++++++++++++++++ 10 files changed, 86 insertions(+), 49 deletions(-) delete mode 100644 lib.py create mode 100644 openaddr/VERSION rename recreate-db.py => openaddr/ci/recreate_db.py (83%) rename run-dequeue.py => openaddr/ci/run_dequeue.py (97%) rename schema.pgsql => openaddr/ci/schema.pgsql (100%) rename worker.py => openaddr/ci/worker.py (89%) create mode 100644 setup.py diff --git a/Procfile b/Procfile index 8861b29c..9d747030 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: gunicorn -w 4 --bind 0.0.0.0:$PORT app:app -dequeue: python run-dequeue.py +web: gunicorn -w 4 --bind 0.0.0.0:$PORT openaddr.ci:app +dequeue: python -m openaddr.ci.run_dequeue diff --git a/lib.py b/lib.py deleted file mode 100644 index 372e3384..00000000 --- a/lib.py +++ /dev/null @@ -1,20 +0,0 @@ -# TODO: factor out of app.py - -from psycopg2 import connect -from pq import PQ - -# TODO factor this cleanly out of app.py - -TASK_QUEUE, DONE_QUEUE = 'tasks', 'finished' -MAGIC_OK_MESSAGE = 'Everything is fine' - -def db_connect(dsn): - ''' Connect to database using Flask app instance or DSN string. - ''' - return connect(dsn) - -def db_queue(conn, name=None): - return PQ(conn, table='queue')[name or TASK_QUEUE] - -def db_cursor(conn): - return conn.cursor() diff --git a/openaddr/VERSION b/openaddr/VERSION new file mode 100644 index 00000000..bd52db81 --- /dev/null +++ b/openaddr/VERSION @@ -0,0 +1 @@ +0.0.0 \ No newline at end of file diff --git a/recreate-db.py b/openaddr/ci/recreate_db.py similarity index 83% rename from recreate-db.py rename to openaddr/ci/recreate_db.py index d3b7e6d8..52003fd8 100644 --- a/recreate-db.py +++ b/openaddr/ci/recreate_db.py @@ -4,7 +4,7 @@ from pq import PQ from psycopg2 import connect -def main(DATABASE_URL): +def recreate(DATABASE_URL): ''' ''' schema_filename = join(dirname(__file__), 'schema.pgsql') @@ -19,7 +19,11 @@ def main(DATABASE_URL): pq = PQ(conn, table='queue') pq.create() -if __name__ == '__main__': - +def main(): + ''' + ''' DATABASE_URL = os.environ['DATABASE_URL'] - exit(main(DATABASE_URL)) + return recreate(DATABASE_URL) + +if __name__ == '__main__': + exit(main()) diff --git a/run-dequeue.py b/openaddr/ci/run_dequeue.py similarity index 97% rename from run-dequeue.py rename to openaddr/ci/run_dequeue.py index 493e4aa0..1a3492bf 100644 --- a/run-dequeue.py +++ b/openaddr/ci/run_dequeue.py @@ -2,7 +2,7 @@ from time import sleep from traceback import print_exc -from app import ( +from . import ( db_connect, db_queue, pop_finished_task_from_queue, DONE_QUEUE, load_config ) diff --git a/schema.pgsql b/openaddr/ci/schema.pgsql similarity index 100% rename from schema.pgsql rename to openaddr/ci/schema.pgsql diff --git a/worker.py b/openaddr/ci/worker.py similarity index 89% rename from worker.py rename to openaddr/ci/worker.py index 9668532d..63a42c7e 100755 --- a/worker.py +++ b/openaddr/ci/worker.py @@ -7,7 +7,8 @@ enqueues a new message on a separate PQ queue when the work is done.""" import time, os, subprocess, psycopg2 -import lib + +from . import db_connect, db_queue, db_queue, MAGIC_OK_MESSAGE, DONE_QUEUE # File path and URL path for result directory. Should be S3. _web_output_dir = '/var/www/html/oa-runone' @@ -39,7 +40,7 @@ def do_work(job_id, job_contents): # Prepare return parameters r = { 'result_code': result_code, 'output_url': '%s/%s' % (_web_base_url, job_id), - 'message': lib.MAGIC_OK_MESSAGE } + 'message': MAGIC_OK_MESSAGE } return r @@ -64,13 +65,13 @@ def run(task_data): 'result' : result } return r -def serve_queue(): +def main(): "Single threaded worker to serve the job queue" # Connect to the queue - conn = lib.db_connect(os.environ['DATABASE_STRING']) - input_queue = lib.db_queue(conn) - output_queue = lib.db_queue(conn, lib.DONE_QUEUE) + conn = db_connect(os.environ['DATABASE_URL']) + input_queue = db_queue(conn) + output_queue = db_queue(conn, DONE_QUEUE) # Fetch and run jobs in a loop while True: @@ -82,4 +83,4 @@ def serve_queue(): if __name__ == '__main__': - serve_queue() + exit(main()) diff --git a/openaddr/tests/ci.py b/openaddr/tests/ci.py index 64554c50..eb95e6f7 100644 --- a/openaddr/tests/ci.py +++ b/openaddr/tests/ci.py @@ -15,14 +15,14 @@ TASK_QUEUE, DONE_QUEUE, MAGIC_OK_MESSAGE ) -recreate_db = __import__('recreate-db') +from ..ci import recreate_db class TestHook (unittest.TestCase): def setUp(self): ''' ''' - recreate_db.main(app.config['DATABASE_URL']) + recreate_db.recreate(app.config['DATABASE_URL']) self.database_url = app.config['DATABASE_URL'] self.github_auth = app.config['GITHUB_AUTH'] diff --git a/requirements.txt b/requirements.txt index 4c1b6dc3..9c558e35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1 @@ -Flask==0.10.1 -gunicorn==19.3.0 -httmock==1.2.3 -itsdangerous==0.24 -Jinja2==2.7.3 -MarkupSafe==0.23 -mock==1.0.1 -pq==1.2 -psycopg2==2.6 -requests==2.6.2 -simplejson==3.6.5 -uritemplate==0.6 -Werkzeug==0.10.4 +. diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..71286ac0 --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +from setuptools import setup +from os.path import join, dirname +import sys + +with open(join(dirname(__file__), 'openaddr', 'VERSION')) as file: + version = file.read().strip() + +conditional_requirements = list() + +if sys.version_info[0] == 2: + conditional_requirements += [ + # http://python-future.org + 'future >= 0.14.3', + + # https://github.com/jdunck/python-unicodecsv + 'unicodecsv >= 0.11.2', + ] + +setup( + name = 'OpenAddresses-Machine', + version = version, + url = 'https://github.com/openaddresses/machine', + author = 'Michal Migurski', + author_email = 'mike-pypi@teczno.com', + description = 'In-progress scripts for running OpenAddresses on a complete data set and publishing the results.', + packages = ['openaddr', 'openaddr.ci', 'openaddr.tests'], + package_data = { + 'openaddr': [ + 'VERSION' + ], + 'openaddr.ci': [ + 'schema.pgsql' + ] + }, + install_requires = [ + 'Flask == 0.10.1', 'gunicorn == 19.3.0', 'httmock == 1.2.3', + 'itsdangerous == 0.24', 'Jinja2 == 2.7.3', 'MarkupSafe == 0.23', + 'mock == 1.0.1', 'pq == 1.2', 'psycopg2 == 2.6', 'simplejson == 3.6.5', + 'uritemplate == 0.6', 'Werkzeug == 0.10.4', + + 'boto >= 2.22.0', 'Jinja2 >= 2.7.0', 'dateutils >= 0.6', 'ijson >= 2.0', + + # https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1306991/comments/10 + 'requests == 2.2.1', + + # https://pypi.python.org/pypi/requests-ftp, appears no longer maintained. + 'requests-ftp == 0.2.0', + + # https://github.com/patrys/httmock + 'httmock >= 1.2', + + # https://pypi.python.org/pypi/setproctitle/ + 'setproctitle >= 1.1.8' + + ] + conditional_requirements, + entry_points = dict( + console_scripts = [ + 'openaddr-ci-recreate-db = openaddr.ci.recreate_db:main', + 'openaddr-ci-run-dequeue = openaddr.ci.run_dequeue:main', + 'openaddr-ci-worker = openaddr.ci.worker:main', + ] + ) +) From 76fd73332ef042ab98a9d6dd70c2bbba8e778a06 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 5 May 2015 17:54:06 -0700 Subject: [PATCH 38/51] Stubbed out Chef scripts --- chef/python/recipes/default.rb | 7 +++++++ chef/role-ubuntu.json | 6 ++++++ chef/run.sh | 20 ++++++++++++++++++++ chef/solo.rb | 8 ++++++++ 4 files changed, 41 insertions(+) create mode 100644 chef/python/recipes/default.rb create mode 100644 chef/role-ubuntu.json create mode 100755 chef/run.sh create mode 100644 chef/solo.rb diff --git a/chef/python/recipes/default.rb b/chef/python/recipes/default.rb new file mode 100644 index 00000000..8aaaa52a --- /dev/null +++ b/chef/python/recipes/default.rb @@ -0,0 +1,7 @@ +package 'python-pip' +package 'python-dev' +package 'libpq-dev' + +execute "pip install -U ." do + cwd File.join(File.dirname(__FILE__), '..', '..', '..') +end diff --git a/chef/role-ubuntu.json b/chef/role-ubuntu.json new file mode 100644 index 00000000..b7fceb60 --- /dev/null +++ b/chef/role-ubuntu.json @@ -0,0 +1,6 @@ +{ + "run_list": + [ + "python" + ] +} diff --git a/chef/run.sh b/chef/run.sh new file mode 100755 index 00000000..8209ded0 --- /dev/null +++ b/chef/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e +# +# Install the chef ruby gem if chef-solo is not in the path. +# This script is safe to run multiple times. +# +if [ ! `which chef-solo` ]; then + release=`lsb_release -r` + if [ "$release" != "${release/12.04/}" ]; then + # Ruby 1.9.3 install provided for Ubuntu 12.04 + apt-get install -y build-essential ruby1.9.3 + gem1.9.3 install chef -v 11.16.4 --no-rdoc --no-ri + else + # Otherwise, assume Ubuntu ~14+ + apt-get install -y build-essential ruby ruby-dev + gem install chef -v 11.16.4 --no-rdoc --no-ri + fi +fi + +cd `dirname $0` +chef-solo -c $PWD/solo.rb -j $PWD/role-ubuntu.json diff --git a/chef/solo.rb b/chef/solo.rb new file mode 100644 index 00000000..39a49f0f --- /dev/null +++ b/chef/solo.rb @@ -0,0 +1,8 @@ +# +# http://docs.opscode.com/config_rb_solo.html +# +file_cache_path "/tmp/chef-solo" +cookbook_path File.dirname(__FILE__) +log_level :info +log_location "/var/log/cheflog" +ssl_verify_mode :verify_none From 683ab3debf0d3400bbf806a006375e2678025550 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 5 May 2015 19:03:31 -0700 Subject: [PATCH 39/51] Configured Travis tests --- .travis.yml | 17 +++++++++++++++++ openaddr/tests/ci.py | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..096be6cd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: python +python: + - "2.7" +cache: + apt: true +addons: + postgresql: "9.3" +before_install: + - sudo apt-get update -y +install: + - sudo chef/run.sh + - pip install -U . +before_script: + - psql -U postgres -c "CREATE USER hooked WITH SUPERUSER PASSWORD 'sources'" + - psql -U postgres -c "CREATE DATABASE hooked_on_sources WITH OWNER hooked" +env: DATABASE_URL=postgres://hooked:sources@localhost/hooked_on_sources +script: python test.py diff --git a/openaddr/tests/ci.py b/openaddr/tests/ci.py index eb95e6f7..f0d2f9dd 100644 --- a/openaddr/tests/ci.py +++ b/openaddr/tests/ci.py @@ -1,5 +1,6 @@ from __future__ import print_function +from os import environ from httmock import HTTMock, response from logging import StreamHandler, DEBUG from urlparse import parse_qsl, urlparse @@ -8,7 +9,7 @@ import unittest, json, os, sys os.environ['GITHUB_TOKEN'] = '' -os.environ['DATABASE_URL'] = 'postgres:///hooked_on_sources' +os.environ['DATABASE_URL'] = environ.get('DATABASE_URL', 'postgres:///hooked_on_sources') from ..ci import ( app, db_connect, db_cursor, db_queue, pop_finished_task_from_queue, From faebcde1fbf17ea8d1d9257196f023abc5ccdaa5 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Wed, 6 May 2015 00:13:29 -0700 Subject: [PATCH 40/51] Re-arranged setup.py and annotated requirements --- setup.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index 71286ac0..6844e345 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,13 @@ author = 'Michal Migurski', author_email = 'mike-pypi@teczno.com', description = 'In-progress scripts for running OpenAddresses on a complete data set and publishing the results.', + entry_points = dict( + console_scripts = [ + 'openaddr-ci-recreate-db = openaddr.ci.recreate_db:main', + 'openaddr-ci-run-dequeue = openaddr.ci.run_dequeue:main', + 'openaddr-ci-worker = openaddr.ci.worker:main', + ] + ), packages = ['openaddr', 'openaddr.ci', 'openaddr.tests'], package_data = { 'openaddr': [ @@ -33,13 +40,26 @@ ] }, install_requires = [ - 'Flask == 0.10.1', 'gunicorn == 19.3.0', 'httmock == 1.2.3', - 'itsdangerous == 0.24', 'Jinja2 == 2.7.3', 'MarkupSafe == 0.23', - 'mock == 1.0.1', 'pq == 1.2', 'psycopg2 == 2.6', 'simplejson == 3.6.5', - 'uritemplate == 0.6', 'Werkzeug == 0.10.4', - 'boto >= 2.22.0', 'Jinja2 >= 2.7.0', 'dateutils >= 0.6', 'ijson >= 2.0', + # http://flask.pocoo.org + 'Flask == 0.10.1', + + # http://gunicorn.org + 'gunicorn == 19.3.0', + + # http://www.voidspace.org.uk/python/mock/ + 'mock == 1.0.1', + + # https://github.com/uri-templates/uritemplate-py/ + 'uritemplate == 0.6', + + # https://github.com/malthe/pq/ + 'pq == 1.2', + + # http://initd.org/psycopg/ + 'psycopg2 == 2.6', + # https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1306991/comments/10 'requests == 2.2.1', @@ -52,12 +72,5 @@ # https://pypi.python.org/pypi/setproctitle/ 'setproctitle >= 1.1.8' - ] + conditional_requirements, - entry_points = dict( - console_scripts = [ - 'openaddr-ci-recreate-db = openaddr.ci.recreate_db:main', - 'openaddr-ci-run-dequeue = openaddr.ci.run_dequeue:main', - 'openaddr-ci-worker = openaddr.ci.worker:main', - ] - ) + ] + conditional_requirements ) From cab3036409f02897adcf1a3090c1699c2e3a3cf6 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Fri, 8 May 2015 08:05:04 -0700 Subject: [PATCH 41/51] Added explicit role argument to chef/run.sh and renamed chef recipes --- .travis.yml | 2 +- chef/{python => openaddr}/recipes/default.rb | 0 chef/{role-ubuntu.json => role-webhooks.json} | 2 +- chef/run.sh | 11 ++++++++++- 4 files changed, 12 insertions(+), 3 deletions(-) rename chef/{python => openaddr}/recipes/default.rb (100%) rename chef/{role-ubuntu.json => role-webhooks.json} (62%) diff --git a/.travis.yml b/.travis.yml index 096be6cd..67f3380b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ addons: before_install: - sudo apt-get update -y install: - - sudo chef/run.sh + - sudo chef/run.sh webhooks - pip install -U . before_script: - psql -U postgres -c "CREATE USER hooked WITH SUPERUSER PASSWORD 'sources'" diff --git a/chef/python/recipes/default.rb b/chef/openaddr/recipes/default.rb similarity index 100% rename from chef/python/recipes/default.rb rename to chef/openaddr/recipes/default.rb diff --git a/chef/role-ubuntu.json b/chef/role-webhooks.json similarity index 62% rename from chef/role-ubuntu.json rename to chef/role-webhooks.json index b7fceb60..7f4fe6c7 100644 --- a/chef/role-ubuntu.json +++ b/chef/role-webhooks.json @@ -1,6 +1,6 @@ { "run_list": [ - "python" + "openaddr" ] } diff --git a/chef/run.sh b/chef/run.sh index 8209ded0..7d3add53 100755 --- a/chef/run.sh +++ b/chef/run.sh @@ -16,5 +16,14 @@ if [ ! `which chef-solo` ]; then fi fi +# +# Use role JSON file from first argument name, but check if it exists. +# +if [ $# != 1 ]; then echo "Usage: $0 "; exit 1; fi + cd `dirname $0` -chef-solo -c $PWD/solo.rb -j $PWD/role-ubuntu.json +ROLEFILE="${PWD}/role-${1}.json" + +if [ ! -f $ROLEFILE ]; then echo "$ROLEFILE does not exist"; exit 1; fi + +chef-solo -c $PWD/solo.rb -j $ROLEFILE From 23ada529827baa70cc8ba9d3af30a92f32a4e2f7 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Tue, 5 May 2015 20:21:24 -0700 Subject: [PATCH 42/51] Added new database chef role for setting up local Postgres --- chef/database/recipes/default.rb | 32 ++++++++++++++++++++++++++++++++ chef/role-database.json | 10 ++++++++++ 2 files changed, 42 insertions(+) create mode 100644 chef/database/recipes/default.rb create mode 100644 chef/role-database.json diff --git a/chef/database/recipes/default.rb b/chef/database/recipes/default.rb new file mode 100644 index 00000000..67a0ffee --- /dev/null +++ b/chef/database/recipes/default.rb @@ -0,0 +1,32 @@ +package 'postgresql' + +user = node[:db_user] +pass = node[:db_pass] +host = node[:db_host] +name = node[:db_name] + +if host == 'localhost' then + psql = 'psql' +else + psql = "psql -h '#{host}'" +end + +bash "create database" do + user 'postgres' + environment({'DATABASE_URL' => "postgres://#{user}:#{pass}@#{host}/#{name}?sslmode=require", 'GITHUB_TOKEN' => ''}) + code <<-CODE + # #{psql} -c "DROP DATABASE IF EXISTS #{name}"; + # #{psql} -c "DROP USER IF EXISTS #{user}"; + + #{psql} -c "CREATE USER #{user} WITH SUPERUSER PASSWORD '#{pass}'"; + #{psql} -c "CREATE DATABASE #{name} WITH OWNER #{user}"; + + openaddr-ci-recreate-db + CODE + + # Stop as soon as an error is encountered. + flags '-e' + + # Assume that exit=1 means the user and database were already in-place. + returns [0, 1] +end \ No newline at end of file diff --git a/chef/role-database.json b/chef/role-database.json new file mode 100644 index 00000000..82384108 --- /dev/null +++ b/chef/role-database.json @@ -0,0 +1,10 @@ +{ + "db_user": "openaddr", + "db_pass": "openaddr", + "db_host": "localhost", + "db_name": "openaddr", + "run_list": + [ + "database" + ] +} From 34e7dc2d75f6854191e42d4d3014675734476b81 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 9 May 2015 13:12:31 -0700 Subject: [PATCH 43/51] Added new chef role for webhook process --- chef/Procfile-webhook | 2 ++ chef/account/recipes/default.rb | 13 +++++++ chef/database/recipes/default.rb | 3 +- chef/role-webhooks.json | 11 +++++- chef/webhooks/recipes/default.rb | 59 ++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 chef/Procfile-webhook create mode 100644 chef/account/recipes/default.rb create mode 100644 chef/webhooks/recipes/default.rb diff --git a/chef/Procfile-webhook b/chef/Procfile-webhook new file mode 100644 index 00000000..9d747030 --- /dev/null +++ b/chef/Procfile-webhook @@ -0,0 +1,2 @@ +web: gunicorn -w 4 --bind 0.0.0.0:$PORT openaddr.ci:app +dequeue: python -m openaddr.ci.run_dequeue diff --git a/chef/account/recipes/default.rb b/chef/account/recipes/default.rb new file mode 100644 index 00000000..5e0771ea --- /dev/null +++ b/chef/account/recipes/default.rb @@ -0,0 +1,13 @@ +name = node[:username] + +group name +user name do + gid name + home "/home/#{name}" +end + +directory "/home/#{name}" do + owner name + group name + mode "0755" +end diff --git a/chef/database/recipes/default.rb b/chef/database/recipes/default.rb index 67a0ffee..c41c65e2 100644 --- a/chef/database/recipes/default.rb +++ b/chef/database/recipes/default.rb @@ -4,6 +4,7 @@ pass = node[:db_pass] host = node[:db_host] name = node[:db_name] +database_url = "postgres://#{user}:#{pass}@#{host}/#{name}?sslmode=require" if host == 'localhost' then psql = 'psql' @@ -13,7 +14,7 @@ bash "create database" do user 'postgres' - environment({'DATABASE_URL' => "postgres://#{user}:#{pass}@#{host}/#{name}?sslmode=require", 'GITHUB_TOKEN' => ''}) + environment({'DATABASE_URL' => database_url, 'GITHUB_TOKEN' => ''}) code <<-CODE # #{psql} -c "DROP DATABASE IF EXISTS #{name}"; # #{psql} -c "DROP USER IF EXISTS #{user}"; diff --git a/chef/role-webhooks.json b/chef/role-webhooks.json index 7f4fe6c7..8630b29d 100644 --- a/chef/role-webhooks.json +++ b/chef/role-webhooks.json @@ -1,6 +1,15 @@ { + "username": "openaddr", + "db_user": "openaddr", + "db_pass": "openaddr", + "db_host": "localhost", + "db_name": "openaddr", + "github_token": "{Github Token}", "run_list": [ - "openaddr" + "database", + "openaddr", + "account", + "webhooks" ] } diff --git a/chef/webhooks/recipes/default.rb b/chef/webhooks/recipes/default.rb new file mode 100644 index 00000000..13e27537 --- /dev/null +++ b/chef/webhooks/recipes/default.rb @@ -0,0 +1,59 @@ +username = node[:username] +app_name = 'openaddr_webhook' + +db_user = node[:db_user] +db_pass = node[:db_pass] +db_host = node[:db_host] +db_name = node[:db_name] + +database_url = "postgres://#{db_user}:#{db_pass}@#{db_host}/#{db_name}?sslmode=require" +github_token = node['github_token'] + +env_file = "/etc/#{app_name}.conf" +procfile = File.join(File.dirname(__FILE__), '..', '..', 'Procfile-webhook') + +execute 'pip install honcho[export]' + +# +# Ensure upstart job exists. +# +file env_file do + content <<-CONF +DATABASE_URL=#{database_url} +GITHUB_TOKEN=#{github_token} +CONF +end + +execute "honcho export upstart /etc/init" do + command "honcho -e #{env_file} -f #{procfile} export -u #{username} -a #{app_name} upstart /etc/init" +end + +rotation = <<-ROTATION +{ + copytruncate + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + endscript +} +ROTATION + +file "/etc/logrotate.d/#{app_name}-web-1" do + content "/var/log/#{app_name}/web-1.log\n#{rotation}\n" +end + +file "/etc/logrotate.d/#{app_name}-dequeue-1" do + content "/var/log/#{app_name}/dequeue-1.log\n#{rotation}\n" +end + +# +# Make it go. +# +execute "stop #{app_name}" do + returns [0, 1] +end + +execute "start #{app_name}" From f2dd2736da4e365e01bb87d3c482a1f4065bf190 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 9 May 2015 13:21:35 -0700 Subject: [PATCH 44/51] Tweaked chef recipes for database --- .travis.yml | 5 +---- chef/database/recipes/default.rb | 12 ++++++------ chef/role-database.json | 1 + chef/role-webhooks.json | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67f3380b..eb1ae867 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,5 @@ before_install: install: - sudo chef/run.sh webhooks - pip install -U . -before_script: - - psql -U postgres -c "CREATE USER hooked WITH SUPERUSER PASSWORD 'sources'" - - psql -U postgres -c "CREATE DATABASE hooked_on_sources WITH OWNER hooked" -env: DATABASE_URL=postgres://hooked:sources@localhost/hooked_on_sources +env: DATABASE_URL=postgres://openaddr:openaddr@localhost/openaddr script: python test.py diff --git a/chef/database/recipes/default.rb b/chef/database/recipes/default.rb index c41c65e2..40bc4663 100644 --- a/chef/database/recipes/default.rb +++ b/chef/database/recipes/default.rb @@ -7,20 +7,20 @@ database_url = "postgres://#{user}:#{pass}@#{host}/#{name}?sslmode=require" if host == 'localhost' then - psql = 'psql' + args = '' else - psql = "psql -h '#{host}'" + args = "-h '#{host}'" end bash "create database" do user 'postgres' environment({'DATABASE_URL' => database_url, 'GITHUB_TOKEN' => ''}) code <<-CODE - # #{psql} -c "DROP DATABASE IF EXISTS #{name}"; - # #{psql} -c "DROP USER IF EXISTS #{user}"; + # psql #{args} -c "DROP DATABASE IF EXISTS #{name}"; + # psql #{args} -c "DROP USER IF EXISTS #{user}"; - #{psql} -c "CREATE USER #{user} WITH SUPERUSER PASSWORD '#{pass}'"; - #{psql} -c "CREATE DATABASE #{name} WITH OWNER #{user}"; + psql #{args} -c "CREATE USER #{user} WITH SUPERUSER PASSWORD '#{pass}'"; + psql #{args} -c "CREATE DATABASE #{name} WITH OWNER #{user}"; openaddr-ci-recreate-db CODE diff --git a/chef/role-database.json b/chef/role-database.json index 82384108..d6d6fa79 100644 --- a/chef/role-database.json +++ b/chef/role-database.json @@ -5,6 +5,7 @@ "db_name": "openaddr", "run_list": [ + "openaddr", "database" ] } diff --git a/chef/role-webhooks.json b/chef/role-webhooks.json index 8630b29d..4f2a9cdd 100644 --- a/chef/role-webhooks.json +++ b/chef/role-webhooks.json @@ -7,8 +7,8 @@ "github_token": "{Github Token}", "run_list": [ - "database", "openaddr", + "database", "account", "webhooks" ] From bf322c165d475b5ae7946c7ba1cafd76e2eb3659 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sat, 9 May 2015 14:09:07 -0700 Subject: [PATCH 45/51] Added new chef role for worker process --- chef/Procfile-worker | 1 + chef/role-worker.json | 15 ++++++++++ chef/worker/recipes/default.rb | 55 ++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 chef/Procfile-worker create mode 100644 chef/role-worker.json create mode 100644 chef/worker/recipes/default.rb diff --git a/chef/Procfile-worker b/chef/Procfile-worker new file mode 100644 index 00000000..2fe3f099 --- /dev/null +++ b/chef/Procfile-worker @@ -0,0 +1 @@ +worker: openaddr-ci-worker diff --git a/chef/role-worker.json b/chef/role-worker.json new file mode 100644 index 00000000..171ab712 --- /dev/null +++ b/chef/role-worker.json @@ -0,0 +1,15 @@ +{ + "username": "openaddr", + "db_user": "openaddr", + "db_pass": "openaddr", + "db_host": "localhost", + "db_name": "openaddr", + "github_token": "{Github Token}", + "run_list": + [ + "openaddr", + "database", + "account", + "worker" + ] +} diff --git a/chef/worker/recipes/default.rb b/chef/worker/recipes/default.rb new file mode 100644 index 00000000..9f3f9015 --- /dev/null +++ b/chef/worker/recipes/default.rb @@ -0,0 +1,55 @@ +username = node[:username] +app_name = 'openaddr_worker' + +db_user = node[:db_user] +db_pass = node[:db_pass] +db_host = node[:db_host] +db_name = node[:db_name] + +database_url = "postgres://#{db_user}:#{db_pass}@#{db_host}/#{db_name}?sslmode=require" +github_token = node['github_token'] + +env_file = "/etc/#{app_name}.conf" +procfile = File.join(File.dirname(__FILE__), '..', '..', 'Procfile-worker') + +execute 'pip install honcho[export]' + +# +# Ensure upstart job exists. +# +file env_file do + content <<-CONF +DATABASE_URL=#{database_url} +GITHUB_TOKEN=#{github_token} +CONF +end + +execute "honcho export upstart /etc/init" do + command "honcho -e #{env_file} -f #{procfile} export -u #{username} -a #{app_name} upstart /etc/init" +end + +rotation = <<-ROTATION +{ + copytruncate + rotate 4 + weekly + missingok + notifempty + compress + delaycompress + endscript +} +ROTATION + +file "/etc/logrotate.d/#{app_name}-worker-1" do + content "/var/log/#{app_name}/worker-1.log\n#{rotation}\n" +end + +# +# Make it go. +# +execute "stop #{app_name}" do + returns [0, 1] +end + +execute "start #{app_name}" From 72fddff261f25fe31f769e9b7bf9757d590f81aa Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 10 May 2015 10:55:05 -0700 Subject: [PATCH 46/51] Added Apache proxy for Flask app --- chef/apache/recipes/default.rb | 23 +++++++++++++++++++++++ chef/role-webhooks.json | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 chef/apache/recipes/default.rb diff --git a/chef/apache/recipes/default.rb b/chef/apache/recipes/default.rb new file mode 100644 index 00000000..a33db3ff --- /dev/null +++ b/chef/apache/recipes/default.rb @@ -0,0 +1,23 @@ +package 'apache2' + +file '/etc/apache2/sites-available/webhook.conf' do + content <<-CONF + + + ProxyPass http://127.0.0.1:5000/ + ProxyPassReverse http://127.0.0.1:5000/ + + + Allow from all + + +CONF +end + +execute 'a2enmod proxy' +execute 'a2enmod proxy_http' + +execute 'a2ensite webhook' +execute 'a2dissite 000-default' + +execute 'service apache2 reload' diff --git a/chef/role-webhooks.json b/chef/role-webhooks.json index 4f2a9cdd..af869915 100644 --- a/chef/role-webhooks.json +++ b/chef/role-webhooks.json @@ -10,6 +10,7 @@ "openaddr", "database", "account", - "webhooks" + "webhooks", + "apache" ] } From 1a5df41798a1de525d2588e276c1b29b1727a866 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 10 May 2015 10:55:49 -0700 Subject: [PATCH 47/51] Removed database recipe from other roles, because it's expected to be a remote service --- chef/role-webhooks.json | 1 - chef/role-worker.json | 1 - 2 files changed, 2 deletions(-) diff --git a/chef/role-webhooks.json b/chef/role-webhooks.json index af869915..ff703f13 100644 --- a/chef/role-webhooks.json +++ b/chef/role-webhooks.json @@ -8,7 +8,6 @@ "run_list": [ "openaddr", - "database", "account", "webhooks", "apache" diff --git a/chef/role-worker.json b/chef/role-worker.json index 171ab712..123bc80e 100644 --- a/chef/role-worker.json +++ b/chef/role-worker.json @@ -8,7 +8,6 @@ "run_list": [ "openaddr", - "database", "account", "worker" ] From 46e81f70cc870b07ec9fcfbfe524ab10d90da70e Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 10 May 2015 11:14:05 -0700 Subject: [PATCH 48/51] Added chef role just for testing purposes --- .travis.yml | 4 ++-- chef/role-testing.json | 15 +++++++++++++++ openaddr/tests/__init__.py | 1 + setup.py | 1 + 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 chef/role-testing.json diff --git a/.travis.yml b/.travis.yml index eb1ae867..425b4122 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ addons: before_install: - sudo apt-get update -y install: - - sudo chef/run.sh webhooks + - sudo chef/run.sh testing - pip install -U . env: DATABASE_URL=postgres://openaddr:openaddr@localhost/openaddr -script: python test.py +script: python setup.py test diff --git a/chef/role-testing.json b/chef/role-testing.json new file mode 100644 index 00000000..20f143e4 --- /dev/null +++ b/chef/role-testing.json @@ -0,0 +1,15 @@ +{ + "username": "openaddr", + "db_user": "openaddr", + "db_pass": "openaddr", + "db_host": "localhost", + "db_name": "openaddr", + "github_token": "fake-token", + "run_list": + [ + "openaddr", + "database", + "account", + "webhooks" + ] +} diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index e69de29b..8237ebd0 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -0,0 +1 @@ +from .ci import * \ No newline at end of file diff --git a/setup.py b/setup.py index 6844e345..f27df185 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ 'schema.pgsql' ] }, + test_suite = 'openaddr.tests', install_requires = [ 'boto >= 2.22.0', 'Jinja2 >= 2.7.0', 'dateutils >= 0.6', 'ijson >= 2.0', From 0c3d3f0a7546d04be6f7ef4cd440f64803aa0ff8 Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Sun, 10 May 2015 11:25:30 -0700 Subject: [PATCH 49/51] Reduced test repetition --- openaddr/tests/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 8237ebd0..e69de29b 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -1 +0,0 @@ -from .ci import * \ No newline at end of file From b175348c2dd0b4f99fe6d93d75804bc384825f0f Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Mon, 11 May 2015 22:07:50 -0700 Subject: [PATCH 50/51] Upgraded to Python 3 compatible pq 1.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f27df185..5861f1f1 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ 'uritemplate == 0.6', # https://github.com/malthe/pq/ - 'pq == 1.2', + 'pq == 1.3', # http://initd.org/psycopg/ 'psycopg2 == 2.6', From d16cfa63bb9b3a2fed69812d9b7ddec99b50223d Mon Sep 17 00:00:00 2001 From: Michal Migurski Date: Mon, 11 May 2015 22:11:37 -0700 Subject: [PATCH 51/51] Updated openaddr.ci string handling and tests for Python 3 --- openaddr/ci/__init__.py | 4 +-- openaddr/tests/ci.py | 64 ++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/openaddr/ci/__init__.py b/openaddr/ci/__init__.py index e0ac6738..9bcf7873 100644 --- a/openaddr/ci/__init__.py +++ b/openaddr/ci/__init__.py @@ -27,7 +27,7 @@ def index(): @app.route('/hook', methods=['POST']) def hook(): github_auth = current_app.config['GITHUB_AUTH'] - webhook_payload = json.loads(request.data) + webhook_payload = json.loads(request.data.decode('utf8')) files = process_payload_files(webhook_payload, github_auth) status_url = get_status_url(webhook_payload) @@ -158,7 +158,7 @@ def process_payload_files(payload, github_auth): current_app.logger.debug('Contents SHA {}'.format(contents['sha'])) if encoding == 'base64': - files[filename] = b64decode(content) + files[filename] = b64decode(content).decode('utf8') else: raise ValueError('Unrecognized encoding "{}"'.format(encoding)) diff --git a/openaddr/tests/ci.py b/openaddr/tests/ci.py index f0d2f9dd..2d4f4068 100644 --- a/openaddr/tests/ci.py +++ b/openaddr/tests/ci.py @@ -54,39 +54,39 @@ def response_content(self, url, request): if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json') and query.get('ref', '').startswith('e91fbc'): data = '''{\r "name": "us-ca-alameda_county.json",\r "path": "sources/us-ca-alameda_county.json",\r "sha": "c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "size": 745,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjAwMSIsCiAgICAgICAgICAgICJuYW1l\\nIjogIkFsYW1lZGEgQ291bnR5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNh\\nbGlmb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIs\\nCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5IjogIkFs\\nYW1lZGEiCiAgICB9LAogICAgImRhdGEiOiAiaHR0cHM6Ly9kYXRhLmFjZ292\\nLm9yZy9hcGkvZ2Vvc3BhdGlhbC84ZTRzLTdmNHY/bWV0aG9kPWV4cG9ydCZm\\nb3JtYXQ9T3JpZ2luYWwiLAogICAgImxpY2Vuc2UiOiAiaHR0cDovL3d3dy5h\\nY2dvdi5vcmcvYWNkYXRhL3Rlcm1zLmh0bSIsCiAgICAiYXR0cmlidXRpb24i\\nOiAiQWxhbWVkYSBDb3VudHkiLAogICAgInllYXIiOiAiIiwKICAgICJ0eXBl\\nIjogImh0dHAiLAogICAgImNvbXByZXNzaW9uIjogInppcCIsCiAgICAiY29u\\nZm9ybSI6IHsKICAgICAgICAibWVyZ2UiOiBbCiAgICAgICAgICAgICJmZWFu\\nbWUiLAogICAgICAgICAgICAiZmVhdHlwIgogICAgICAgIF0sCiAgICAgICAg\\nImxvbiI6ICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1i\\nZXIiOiAic3RfbnVtIiwKICAgICAgICAic3RyZWV0IjogImF1dG9fc3RyZWV0\\nIiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUiLAogICAgICAgICJwb3N0\\nY29kZSI6ICJ6aXBjb2RlIgogICAgfQp9Cg==\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-alameda_county.json?ref=e91fbc420f08890960f50f863626e1062f922522",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/c9cd0ed30256ae64d5924b03b0423346501b92d8",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e91fbc420f08890960f50f863626e1062f922522/sources/us-ca-alameda_county.json"\r }\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-san_francisco.json",\r "path": "sources/us-ca-san_francisco.json",\r "sha": "cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "size": 519,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJjb3VudHJ5IjogInVzIiwK\\nICAgICAgICAic3RhdGUiOiAiY2EiLAogICAgICAgICJjaXR5IjogIlNhbiBG\\ncmFuY2lzY28iCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2Yg\\nU2FuIEZyYW5jaXNjbyIsCiAgICAiZGF0YSI6ICJodHRwczovL2RhdGEuc2Zn\\nb3Yub3JnL2Rvd25sb2FkL2t2ZWotdzVrYi9aSVBQRUQlMjBTSEFQRUZJTEUi\\nLAogICAgImxpY2Vuc2UiOiAiIiwKICAgICJ5ZWFyIjogIiIsCiAgICAidHlw\\nZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgImNv\\nbmZvcm0iOiB7Cgkic3BsaXQiOiAiQUREUkVTUyIsCiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nYXV0b19udW1iZXIiLAogICAgICAgICJzdHJlZXQiOiAiYXV0b19zdHJlZXQi\\nLAogICAgICAgICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rj\\nb2RlIjogInppcGNvZGUiCiAgICB9Cn0K\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-san_francisco.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/cbf1f900ac072b6a2e728819a97e74bc772e79ff",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-san_francisco.json"\r }\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json') and query.get('ref', '').startswith('ded44e'): data = '''{\r "name": "us-ca-berkeley.json",\r "path": "sources/us-ca-berkeley.json",\r "sha": "16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "size": 779,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7CiAg\\nICAgICAgICAgICJnZW9pZCI6ICIwNjA2MDAwIiwKICAgICAgICAgICAgInBs\\nYWNlIjogIkJlcmtlbGV5IiwKICAgICAgICAgICAgInN0YXRlIjogIkNhbGlm\\nb3JuaWEiCiAgICAgICAgfSwKICAgICAgICAiY291bnRyeSI6ICJ1cyIsCiAg\\nICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAicGxhY2UiOiAiQmVya2Vs\\nZXkiCiAgICB9LAogICAgImF0dHJpYnV0aW9uIjogIkNpdHkgb2YgQmVya2Vs\\nZXkiLAogICAgImRhdGEiOiAiaHR0cDovL3d3dy5jaS5iZXJrZWxleS5jYS51\\ncy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxzLnppcCIsCiAgICAid2Vi\\nc2l0ZSI6ICJodHRwOi8vd3d3LmNpLmJlcmtlbGV5LmNhLnVzL2RhdGFjYXRh\\nbG9nLyIsCiAgICAidHlwZSI6ICJodHRwIiwKICAgICJjb21wcmVzc2lvbiI6\\nICJ6aXAiLAogICAgIm5vdGUiOiAiTWV0YWRhdGEgYXQgaHR0cDovL3d3dy5j\\naS5iZXJrZWxleS5jYS51cy91cGxvYWRlZEZpbGVzL0lUL0dJUy9QYXJjZWxz\\nLnNocCgxKS54bWwiLAogICAgImNvbmZvcm0iOiB7CiAgICAgICAgImxvbiI6\\nICJ4IiwKICAgICAgICAibGF0IjogInkiLAogICAgICAgICJudW1iZXIiOiAi\\nU3RyZWV0TnVtIiwKICAgICAgICAibWVyZ2UiOiBbIlN0cmVldE5hbWUiLCAi\\nU3RyZWV0U3VmeCIsICJEaXJlY3Rpb24iXSwKICAgICAgICAic3RyZWV0Ijog\\nImF1dG9fc3RyZWV0IiwKICAgICAgICAidHlwZSI6ICJzaGFwZWZpbGUtcG9s\\neWdvbiIKICAgIH0KfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-berkeley.json?ref=ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/16464c39b59b5a09c6526da3afa9a5f57caabcad",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/sources/us-ca-berkeley.json"\r }\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json') and query.get('ref', '').startswith('e5f1dc'): data = '''{\r "name": "us-ca-santa_clara_county.json",\r "path": "sources/us-ca-santa_clara_county.json",\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "size": 908,\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "git_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "download_url": "https://raw.githubusercontent.com/openaddresses/hooked-on-sources/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "type": "file",\r "content": "ewogICAgImNvdmVyYWdlIjogewogICAgICAgICJVUyBDZW5zdXMiOiB7Imdl\\nb2lkIjogIjA2MDg1IiwgIm5hbWUiOiAiU2FudGEgQ2xhcmEgQ291bnR5Iiwg\\nInN0YXRlIjogIkNhbGlmb3JuaWEifSwKICAgICAgICAiY291bnRyeSI6ICJ1\\ncyIsCiAgICAgICAgInN0YXRlIjogImNhIiwKICAgICAgICAiY291bnR5Ijog\\nIlNhbnRhIENsYXJhIgogICAgfSwKICAgICJjb25mb3JtIjogewogICAgICAg\\nICJ0eXBlIjogInNoYXBlZmlsZSIsCiAgICAgICAgInBvc3Rjb2RlIjogIlpJ\\nUENPREUiLAogICAgICAgICJjaXR5IjogIkNJVFkiLAogICAgICAgICJudW1i\\nZXIiOiAiSE9VU0VOVU1URSIsCiAgICAgICAgIm1lcmdlIjogWyJTVFJFRVRO\\nQU1FIiwgIlNUUkVFVFRZUEUiXSwKICAgICAgICAic3RyZWV0IjogImF1dG9f\\nc3RyZWV0IiwKICAgICAgICAibG9uIjogIngiLAogICAgICAgICJsYXQiOiAi\\neSIKICAgIH0sCiAgICAiYXR0cmlidXRpb24iOiAiU2FudGEgQ2xhcmEgQ291\\nbnR5IiwKICAgICJkYXRhIjogImh0dHBzOi8vZ2l0aHViLmNvbS9kYXRhZGVz\\nay91cy1jYS1zYW50YV9jbGFyYV9jb3VudHktZ2lzLXNocC9ibG9iL21hc3Rl\\nci9RNF9GWTE0X0FkZHJlc3NfUG9pbnQuemlwP3Jhdz10cnVlIiwKICAgICJ3\\nZWJzaXRlIjogImh0dHBzOi8vc2Z0cC5zY2Nnb3Yub3JnL2NvdXJpZXIvd2Vi\\nLzEwMDBAL3dtTG9naW4uaHRtbCIsCiAgICAidHlwZSI6ICJodHRwIiwKICAg\\nICJjb21wcmVzc2lvbiI6ICJ6aXAiLAogICAgIm5vdGUiOiAiRmlsZSBkb3du\\nbG9hZCBpcyBiZWhpbmQgYSByZWdpc3RyYXRpb24gd2FsbCBvbiBnb3Zlcm5t\\nZW50IHNpdGUgc28gdGhlIHppcCB3YXMgZG93bmxvYWRlZCBpbiBOb3ZlbWJl\\nciAyMDE0IGFuZCB1cGxvYWRlZCB0byBHaXRIdWIgZm9yIHB1YmxpYyBob3N0\\naW5nLiIKfQo=\\n",\r "encoding": "base64",\r "_links": {\r "self": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "git": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/blobs/84abff13dba318189f1f4d5c1605478127ceff5c",\r "html": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json"\r }\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:b450dcb...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:29:59Z"\r },\r "message": "Added Contra Costa county",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "diverged",\r "ahead_by": 3,\r "behind_by": 1,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e91fbc4",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...e91fbc420f08890960f50f863626e1062f922522.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "e91fbc420f08890960f50f863626e1062f922522",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:16:12Z"\r },\r "message": "Added first source",\r "tree": {\r "sha": "f5e85249cee39d0e84ed936d31a9c08c1eaaa539",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f5e85249cee39d0e84ed936d31a9c08c1eaaa539"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e91fbc420f08890960f50f863626e1062f922522",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e91fbc420f08890960f50f863626e1062f922522",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e91fbc420f08890960f50f863626e1062f922522/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "c52204fd40f17f9da243df09e6d1107d48768afd",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/c52204fd40f17f9da243df09e6d1107d48768afd",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/c52204fd40f17f9da243df09e6d1107d48768afd"\r }\r ]\r },\r "status": "behind",\r "ahead_by": 0,\r "behind_by": 2,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:ded44ed",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...ded44ed5f1733bb93d84f94afe9383e2d47bbbaa.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r }''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:ded44ed...openaddresses:e5f1dca",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa...e5f1dcae83ab1ef1f736b969da617311f7f11564.patch",\r "base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:25:55Z"\r },\r "message": "Added Berkeley",\r "tree": {\r "sha": "9d6ac64ea358034e23fba121f3833790f47b5d76",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/9d6ac64ea358034e23fba121f3833790f47b5d76"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "73a81c5b337bd393273a222f1cd191d7e634df51",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/73a81c5b337bd393273a222f1cd191d7e634df51",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/73a81c5b337bd393273a222f1cd191d7e634df51"\r }\r ]\r },\r "status": "ahead",\r "ahead_by": 3,\r "behind_by": 0,\r "total_commits": 3,\r "commits": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:48:58Z"\r },\r "message": "Added Santa Clara County",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/ded44ed5f1733bb93d84f94afe9383e2d47bbbaa"\r }\r ]\r },\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:39Z"\r },\r "message": "Added Polish source",\r "tree": {\r "sha": "cf52bd865006bc0cd3deaba1b87a4d679a3410e0",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/cf52bd865006bc0cd3deaba1b87a4d679a3410e0"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b659130053b85cd3993b1a4653da1bf6231ec0b4",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b659130053b85cd3993b1a4653da1bf6231ec0b4"\r }\r ]\r },\r {\r "sha": "e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-26T00:52:46Z"\r },\r "message": "Removed Polish source",\r "tree": {\r "sha": "f0cd9b347f69397fcc79fcc434f077bf19af9520",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f0cd9b347f69397fcc79fcc434f077bf19af9520"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/e5f1dcae83ab1ef1f736b969da617311f7f11564/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/0cbd51b8f6044e98c919dcabf93e3f4e1d58c035"\r }\r ]\r }\r ],\r "files": [\r {\r "sha": "84abff13dba318189f1f4d5c1605478127ceff5c",\r "filename": "sources/us-ca-santa_clara_county.json",\r "status": "added",\r "additions": 24,\r "deletions": 0,\r "changes": 24,\r "blob_url": "https://github.com/openaddresses/hooked-on-sources/blob/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "raw_url": "https://github.com/openaddresses/hooked-on-sources/raw/e5f1dcae83ab1ef1f736b969da617311f7f11564/sources/us-ca-santa_clara_county.json",\r "contents_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/contents/sources/us-ca-santa_clara_county.json?ref=e5f1dcae83ab1ef1f736b969da617311f7f11564",\r "patch": "@@ -0,0 +1,24 @@\\n+{\\n+ \\"coverage\\": {\\n+ \\"US Census\\": {\\"geoid\\": \\"06085\\", \\"name\\": \\"Santa Clara County\\", \\"state\\": \\"California\\"},\\n+ \\"country\\": \\"us\\",\\n+ \\"state\\": \\"ca\\",\\n+ \\"county\\": \\"Santa Clara\\"\\n+ },\\n+ \\"conform\\": {\\n+ \\"type\\": \\"shapefile\\",\\n+ \\"postcode\\": \\"ZIPCODE\\",\\n+ \\"city\\": \\"CITY\\",\\n+ \\"number\\": \\"HOUSENUMTE\\",\\n+ \\"merge\\": [\\"STREETNAME\\", \\"STREETTYPE\\"],\\n+ \\"street\\": \\"auto_street\\",\\n+ \\"lon\\": \\"x\\",\\n+ \\"lat\\": \\"y\\"\\n+ },\\n+ \\"attribution\\": \\"Santa Clara County\\",\\n+ \\"data\\": \\"https://github.com/datadesk/us-ca-santa_clara_county-gis-shp/blob/master/Q4_FY14_Address_Point.zip?raw=true\\",\\n+ \\"website\\": \\"https://sftp.sccgov.org/courier/web/1000@/wmLogin.html\\",\\n+ \\"type\\": \\"http\\",\\n+ \\"compression\\": \\"zip\\",\\n+ \\"note\\": \\"File download is behind a registration wall on government site so the zip was downloaded in November 2014 and uploaded to GitHub for public hosting.\\"\\n+}"\r }\r ]\r}''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('GET', GH, '/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637'): data = '''{\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637",\r "permalink_url": "https://github.com/openaddresses/hooked-on-sources/compare/openaddresses:3147668...openaddresses:3147668",\r "diff_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.diff",\r "patch_url": "https://github.com/openaddresses/hooked-on-sources/compare/master...3147668047a0bec6d481c0e42995f7c5e5eac637.patch",\r "base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "merge_base_commit": {\r "sha": "3147668047a0bec6d481c0e42995f7c5e5eac637",\r "commit": {\r "author": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "committer": {\r "name": "Michal Migurski",\r "email": "mike@teczno.com",\r "date": "2015-04-27T01:36:22Z"\r },\r "message": "Empty commit",\r "tree": {\r "sha": "f9e70a36019597c21c7a28bd16840588e6223a33",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/trees/f9e70a36019597c21c7a28bd16840588e6223a33"\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/git/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comment_count": 0\r },\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/3147668047a0bec6d481c0e42995f7c5e5eac637",\r "comments_url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/3147668047a0bec6d481c0e42995f7c5e5eac637/comments",\r "author": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "committer": {\r "login": "migurski",\r "id": 58730,\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3",\r "gravatar_id": "",\r "url": "https://api.github.com/users/migurski",\r "html_url": "https://github.com/migurski",\r "followers_url": "https://api.github.com/users/migurski/followers",\r "following_url": "https://api.github.com/users/migurski/following{/other_user}",\r "gists_url": "https://api.github.com/users/migurski/gists{/gist_id}",\r "starred_url": "https://api.github.com/users/migurski/starred{/owner}{/repo}",\r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions",\r "organizations_url": "https://api.github.com/users/migurski/orgs",\r "repos_url": "https://api.github.com/users/migurski/repos",\r "events_url": "https://api.github.com/users/migurski/events{/privacy}",\r "received_events_url": "https://api.github.com/users/migurski/received_events",\r "type": "User",\r "site_admin": false\r },\r "parents": [\r {\r "sha": "b450dcbb6f4c61015b0a00290e984849f7d649de",\r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/commits/b450dcbb6f4c61015b0a00290e984849f7d649de",\r "html_url": "https://github.com/openaddresses/hooked-on-sources/commit/b450dcbb6f4c61015b0a00290e984849f7d649de"\r }\r ]\r },\r "status": "identical",\r "ahead_by": 0,\r "behind_by": 0,\r "total_commits": 0,\r "commits": [\r\r ],\r "files": [\r\r ]\r}''' - return response(200, data, headers=response_headers) + return response(200, data.encode('utf8'), headers=response_headers) if MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/e5f1dcae83ab1ef1f736b969da617311f7f11564') \ or MHP == ('POST', GH, '/repos/openaddresses/hooked-on-sources/statuses/e91fbc420f08890960f50f863626e1062f922522') \ @@ -98,7 +98,7 @@ def response_content(self, url, request): self.assertEqual(input['context'], 'openaddresses/hooked') data = '''{{\r "context": "openaddresses/hooked", \r "created_at": "2015-04-26T23:45:39Z", \r "creator": {{\r "avatar_url": "https://avatars.githubusercontent.com/u/58730?v=3", \r "events_url": "https://api.github.com/users/migurski/events{{/privacy}}", \r "followers_url": "https://api.github.com/users/migurski/followers", \r "following_url": "https://api.github.com/users/migurski/following{{/other_user}}", \r "gists_url": "https://api.github.com/users/migurski/gists{{/gist_id}}", \r "gravatar_id": "", \r "html_url": "https://github.com/migurski", \r "id": 58730, \r "login": "migurski", \r "organizations_url": "https://api.github.com/users/migurski/orgs", \r "received_events_url": "https://api.github.com/users/migurski/received_events", \r "repos_url": "https://api.github.com/users/migurski/repos", \r "site_admin": false, \r "starred_url": "https://api.github.com/users/migurski/starred{{/owner}}{{/repo}}", \r "subscriptions_url": "https://api.github.com/users/migurski/subscriptions", \r "type": "User", \r "url": "https://api.github.com/users/migurski"\r }}, \r "description": "Checking ", \r "id": 999999999, \r "state": "{state}", \r "target_url": null, \r "updated_at": "2015-04-26T23:45:39Z", \r "url": "https://api.github.com/repos/openaddresses/hooked-on-sources/statuses/xxxxxxxxx"\r }}''' - return response(201, data.format(**input), headers=response_headers) + return response(201, data.format(**input).encode('utf8'), headers=response_headers) print('Unknowable Request {} "{}"'.format(request.method, url.geturl()), file=sys.stderr) raise ValueError('Unknowable Request {} "{}"'.format(request.method, url.geturl())) @@ -113,8 +113,8 @@ def test_webhook_one_master_commit(self): self.assertEqual(posted.status_code, 200) self.assertEqual(self.last_status_state, 'pending') - self.assertTrue('us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') - self.assertTrue('data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') + self.assertTrue(b'us-ca-alameda_county' in posted.data, 'Alameda County source should be present in master commit') + self.assertTrue(b'data.acgov.org' in posted.data, 'Alameda County domain name should be present in master commit') # Look for the task in the task queue. with db_connect(self.database_url) as conn: @@ -147,11 +147,11 @@ def test_webhook_two_master_commits_early_fail(self): self.assertEqual(posted.status_code, 200) self.assertEqual(self.last_status_state, 'pending') - self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') - self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') - self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') - self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') - self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') + self.assertTrue(b'us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') + self.assertTrue(b'data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') + self.assertTrue(b'us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') + self.assertTrue(b'www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') + self.assertFalse(b'us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') for message in ('Something went wrong', MAGIC_OK_MESSAGE): # Look for the task in the task queue. @@ -186,11 +186,11 @@ def test_webhook_two_master_commits_late_fail(self): self.assertEqual(posted.status_code, 200) self.assertEqual(self.last_status_state, 'pending') - self.assertTrue('us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') - self.assertTrue('data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') - self.assertTrue('us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') - self.assertTrue('www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') - self.assertFalse('us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') + self.assertTrue(b'us-ca-san_francisco' in posted.data, 'San Francisco source should be present in master commit') + self.assertTrue(b'data.sfgov.org' in posted.data, 'San Francisco URL should be present in master commit') + self.assertTrue(b'us-ca-berkeley' in posted.data, 'Berkeley source should be present in master commit') + self.assertTrue(b'www.ci.berkeley.ca.us' in posted.data, 'Berkeley URL should be present in master commit') + self.assertFalse(b'us-ca-alameda_county' in posted.data, 'Alameda County source should be absent from master commit') for (index, message) in enumerate((MAGIC_OK_MESSAGE, 'Something went wrong')): # Look for the task in the task queue. @@ -239,15 +239,15 @@ def job_queue_fails(*args, **kwargs): self.assertEqual(posted.status_code, 500) self.assertEqual(self.last_status_state, 'error') self.assertTrue('us-ca-santa_clara_county' in self.last_status_message, 'Santa Clara County source should be present in error message') - self.assertTrue('us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') - self.assertTrue('sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') - self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') - self.assertFalse('pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') - self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from branch commit') - self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from branch commit') + self.assertTrue(b'us-ca-santa_clara_county' in posted.data, 'Santa Clara County source should be present in branch commit') + self.assertTrue(b'sftp.sccgov.org' in posted.data, 'Santa Clara County URL should be present in branch commit') + self.assertFalse(b'us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from branch commit') + self.assertFalse(b'pl-dolnoslaskie' in posted.data, 'Polish source should be absent from branch commit') + self.assertFalse(b'us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from branch commit') + self.assertFalse(b'us-ca-berkeley' in posted.data, 'Berkeley source should be absent from branch commit') # Verify that queued job was never created. - last_job_url = json.loads(posted.data).get('url') + last_job_url = json.loads(posted.data.decode('utf8')).get('url') self.assertEqual(last_job_url, None) self.assertEqual(self.last_status_state, 'error') @@ -262,9 +262,9 @@ def test_webhook_empty_master_commit(self): self.assertEqual(posted.status_code, 200) self.assertEqual(self.last_status_state, 'success') - self.assertFalse('us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from master commit') - self.assertFalse('us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') - self.assertFalse('us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') + self.assertFalse(b'us-ca-contra_costa_county' in posted.data, 'Contra Costa County should be absent from master commit') + self.assertFalse(b'us-ca-san_francisco' in posted.data, 'San Francisco source should be absent from master commit') + self.assertFalse(b'us-ca-berkeley' in posted.data, 'Berkeley source should be absent from master commit') if __name__ == '__main__': unittest.main()