From 079724419650d4d8bd4dc87b9eb3ddc50201246a Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Fri, 1 Mar 2019 06:05:23 -0500 Subject: [PATCH 01/17] Python3 migration --- Dockerfile | 4 +- install/run.sh | 2 +- server/first/settings.py | 7 +- server/first/urls.py | 8 +- server/first_core/auth.py | 8 +- server/first_core/dbs/__init__.py | 8 +- server/first_core/dbs/builtin_db.py | 4 +- server/first_core/disassembly/__init__.py | 2 +- server/first_core/engines/__init__.py | 18 ++--- server/first_core/engines/basic_masking.py | 9 ++- server/first_core/engines/mnemonic_hash.py | 2 +- server/first_core/engines/results.py | 4 +- server/first_core/util.py | 2 +- server/rest/urls.py | 28 +++---- server/rest/views.py | 10 +-- server/utilities/engine_shell.py | 86 +++++++++++----------- server/utilities/mongo_to_django_orm.py | 28 +++---- server/utilities/user_shell.py | 48 ++++++------ server/www/models.py | 10 +-- server/www/urls.py | 16 ++-- 20 files changed, 153 insertions(+), 151 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d138ed..c34a173 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM ubuntu RUN apt-get update RUN apt-get -y dist-upgrade -RUN apt-get install -y python-pip mysql-client libmysqlclient-dev apache2 libapache2-mod-wsgi libapache2-mod-wsgi +RUN apt-get install -y python3-pip mysql-client libmysqlclient-dev apache2 libapache2-mod-wsgi-py3 COPY install/requirements.txt /tmp -RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt && rm /tmp/requirements.txt RUN useradd -m -U -d /home/first -s /bin/bash first COPY ./server /home/first diff --git a/install/run.sh b/install/run.sh index dc9b8ef..bf8b7f8 100755 --- a/install/run.sh +++ b/install/run.sh @@ -21,7 +21,7 @@ else fi # Always run migrations -/usr/bin/python /home/first/manage.py migrate +/usr/bin/python3 /home/first/manage.py migrate # Finally, start up the apache service /usr/sbin/apache2ctl -D FOREGROUND diff --git a/server/first/settings.py b/server/first/settings.py index c438114..253f541 100644 --- a/server/first/settings.py +++ b/server/first/settings.py @@ -19,13 +19,14 @@ 'first_config.json') CONFIG = {} try: - config_data = json.load(file(FIRST_CONFIG_FILE)) + with open(FIRST_CONFIG_FILE, "r") as f: + config_data = json.load(f) if type(config_data) == dict: CONFIG = config_data except IOError as ioe: - print '[1st] IOError: {}'.format(ioe) + print('[1st] IOError: {}'.format(ioe)) except ValueError as ve: - print '[1st] ValueError: {}'.format(ve) + print('[1st] ValueError: {}'.format(ve)) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/server/first/urls.py b/server/first/urls.py index 769cc60..fa1bad5 100644 --- a/server/first/urls.py +++ b/server/first/urls.py @@ -15,12 +15,12 @@ """ from django.contrib import admin from django.conf.urls import handler404 -from django.conf.urls import include, url +from django.urls import path, re_path, include handler404 = 'www.views.handler404' urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'^api/', include('rest.urls')), - url(r'^', include('www.urls')), + path(r'admin', admin.site.urls), + path(r'api', include('rest.urls')), + path(r'', include('www.urls')), ] diff --git a/server/first_core/auth.py b/server/first_core/auth.py index bd05ba9..bb65c50 100644 --- a/server/first_core/auth.py +++ b/server/first_core/auth.py @@ -127,7 +127,7 @@ def __init__(self, request): redirect_uri=redirect_uri), } except TypeError as e: - print e + print(e) if 'auth' not in request.session: request.session['auth'] = {} @@ -190,9 +190,9 @@ def login_step_2(self, auth_code, url, login=True): if login: try: - user = User.objects.get(email=email) + user = User.objects.get(email=self.request.session['info']['email']) user.auth_data = json.dumps(credentials) - user.name = info['displayName'] + user.name = self.request.session['info']['name'] user.save() self.request.session['auth']['api_key'] = str(user.api_key) @@ -241,7 +241,7 @@ def register_user(self): # Create random 4 digit value for the handle # This prevents handle collisions number = random.randint(0, 9999) - for i in xrange(10000): + for i in range(10000): try: num = (number + i) % 10000 user = User.objects.get(handle=handle, number=num) diff --git a/server/first_core/dbs/__init__.py b/server/first_core/dbs/__init__.py index ce27b71..44e12a5 100644 --- a/server/first_core/dbs/__init__.py +++ b/server/first_core/dbs/__init__.py @@ -20,7 +20,7 @@ #------------------------------------------------------------------------------- # Python Modules -import ConfigParser +import configparser from hashlib import md5 # FIRST Modules @@ -58,7 +58,7 @@ def __init__(self, conf): ''' Constructor. - @param conf: ConfigParser.RawConfigParser + @param conf: configparser.RawConfigParser ''' raise FIRSTDBError('TODO: implement') @@ -79,10 +79,10 @@ def __init__(self, config=None): self._dbs[d.name] = d except FIRSTDBError as e: - print e + print(e) if not self._dbs: - print '[DBM] Error: No dbs could be loaded' + print('[DBM] Error: No dbs could be loaded') def db_list(self): ''' diff --git a/server/first_core/dbs/builtin_db.py b/server/first_core/dbs/builtin_db.py index e140b9d..6c32a9a 100644 --- a/server/first_core/dbs/builtin_db.py +++ b/server/first_core/dbs/builtin_db.py @@ -28,7 +28,7 @@ import math import json import hashlib -import ConfigParser +import configparser from hashlib import md5 # Third Party Modules @@ -58,7 +58,7 @@ def __init__(self, config): ''' Constructor. - @param conf: ConfigParser.RawConfigParser + @param conf: configparser.RawConfigParser ''' self._is_installed = True ''' diff --git a/server/first_core/disassembly/__init__.py b/server/first_core/disassembly/__init__.py index 4bf79de..010739e 100644 --- a/server/first_core/disassembly/__init__.py +++ b/server/first_core/disassembly/__init__.py @@ -123,7 +123,7 @@ def __init__(self, architecture, code): def instructions(self): # When first called function will return cached instructions - for i in xrange(len(self.data)): + for i in range(len(self.data)): yield self.data[i] # Then iterate through non-cached instructions diff --git a/server/first_core/engines/__init__.py b/server/first_core/engines/__init__.py index cd17b5d..3799dc2 100644 --- a/server/first_core/engines/__init__.py +++ b/server/first_core/engines/__init__.py @@ -90,8 +90,8 @@ def __init__(self, dbs, engine_id, rank): def add(self, function): required_keys = {'id', 'apis', 'opcodes', 'architecture', 'sha256'} if ((dict != type(function)) - or not required_keys.issubset(function.keys())): - print 'Data provided is not the correct type or required keys not provided' + or not required_keys.issubset(list(function.keys()))): + print('Data provided is not the correct type or required keys not provided') return self._add(function) @@ -185,17 +185,17 @@ def _engines(self): try: e = obj(self.__db_manager, str(e.id), e.rank) if not isinstance(e, AbstractEngine): - print '[EM] {} is not an AbstractEngine'.format(e) + print('[EM] {} is not an AbstractEngine'.format(e)) continue if e.is_operational: engines.append(e) except FIRSTEngineError as e: - print e + print(e) if not engines: - print '[EM] Error: No engines could be loaded' + print('[EM] Error: No engines could be loaded') return engines @@ -217,8 +217,8 @@ def add(self, function): ''' required_keys = {'id', 'apis', 'opcodes', 'architecture', 'sha256'} - if (dict != type(function)) or not required_keys.issubset(function.keys()): - print '[1stEM] Data provided is not the correct type or required keys not provided' + if (dict != type(function)) or not required_keys.issubset(list(function.keys())): + print('[1stEM] Data provided is not the correct type or required keys not provided') return None dis = Disassembly(function['architecture'], function['opcodes']) @@ -270,7 +270,7 @@ def scan(self, user, opcodes, architecture, apis): engines = self._engines dis = Disassembly(architecture, opcodes) - for i in xrange(len(engines)): + for i in range(len(engines)): engine = engines[i] try: results = engine.scan(opcodes, architecture, apis, @@ -279,7 +279,7 @@ def scan(self, user, opcodes, architecture, apis): engine_results[i] = results except Exception as e: - print e + print(e) results = {} for i, hits in engine_results.iteritems(): diff --git a/server/first_core/engines/basic_masking.py b/server/first_core/engines/basic_masking.py index 75d8097..e1fd94d 100644 --- a/server/first_core/engines/basic_masking.py +++ b/server/first_core/engines/basic_masking.py @@ -79,7 +79,8 @@ def normalize(self, disassembly): normalized = [] original = [] for i in disassembly.instructions(): - original.append(str(i.bytes).encode('hex')) + import codecs + original.append(codecs.encode(i.bytes, 'hex')) instr = ''.join(chr(x) for x in i.opcode if x) # Special mnemonic masking (Call, Jmp, JCC) @@ -147,14 +148,14 @@ def normalize(self, disassembly): ''' if MIN_REQUIRED_INSTRUCTIONS > len(normalized): - print 145 + print(145) return (0, None) h_sha256 = sha256(''.join(normalized)).hexdigest() return (changed_bytes, h_sha256) except Exception as e: - print 160, e + print(160, e) return (0, None) @@ -248,4 +249,4 @@ def _install(self): execute_from_command_line(['manage.py', 'migrate', 'engines']) def _uninstall(self): - print 'Manually delete tables associated with {}'.format(self.engine_name) + print('Manually delete tables associated with {}'.format(self.engine_name)) diff --git a/server/first_core/engines/mnemonic_hash.py b/server/first_core/engines/mnemonic_hash.py index 63bc636..9ad0569 100644 --- a/server/first_core/engines/mnemonic_hash.py +++ b/server/first_core/engines/mnemonic_hash.py @@ -165,4 +165,4 @@ def _install(self): execute_from_command_line(['manage.py', 'migrate', 'engines']) def _uninstall(self): - print 'Manually delete tables associated with {}'.format(self.engine_name) + print('Manually delete tables associated with {}'.format(self.engine_name)) diff --git a/server/first_core/engines/results.py b/server/first_core/engines/results.py index 5e3a6db..79bffa1 100644 --- a/server/first_core/engines/results.py +++ b/server/first_core/engines/results.py @@ -27,8 +27,8 @@ def get_metadata(self, db): while True: data = self._get_metadata(db) - if ((type(data) != dict) or (None in data.values()) - or (not Result.required.issubset(data.keys()))): + if ((type(data) != dict) or (None in list(data.values())) + or (not Result.required.issubset(list(data.keys())))): return data['similarity'] = self.similarity diff --git a/server/first_core/util.py b/server/first_core/util.py index 5b49269..2a13a3a 100644 --- a/server/first_core/util.py +++ b/server/first_core/util.py @@ -44,7 +44,7 @@ def make_id(flags, metadata=0, engine=0): def parse_id(_id): - if type(_id) in [str, unicode]: + if type(_id) in [str]: if len(_id) != 26: return (None, None, None) diff --git a/server/rest/urls.py b/server/rest/urls.py index 013f6f4..04bcbf0 100644 --- a/server/rest/urls.py +++ b/server/rest/urls.py @@ -1,35 +1,35 @@ -from django.conf.urls import url +from django.urls import path, re_path, include from . import views app_name = 'rest' urlpatterns = [ - url(r'^test_connection/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^test_connection/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.test_connection, name='test_connection'), - url(r'^sample/architectures/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^sample/architectures/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.architectures, name='architectures'), - url(r'^sample/checkin/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^sample/checkin/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.checkin, name='checkin'), # Metadata related REST URIs - url(r'^metadata/history/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/history/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_history, name='metadata_history'), - url(r'^metadata/applied/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/applied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_applied, name='metadata_applied'), - url(r'^metadata/unapplied/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/unapplied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_unapplied, name='metadata_unapplied'), - url(r'^metadata/get/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/get/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_get, name='metadata_get'), - url(r'^metadata/delete/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?i)(?P<_id>[A-F\d]{26})$', + re_path(r'^metadata/delete/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P<_id>[A-F\d]{26})$', views.metadata_delete, name='metadata_delete'), - url(r'^metadata/created/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_created, name='metadata_created'), - url(r'^metadata/created/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P\d+)$', + re_path(r'^metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P\d+)$', views.metadata_created, name='metadata_created'), - url(r'^metadata/add/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/add/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_add, name='metadata_add'), - url(r'^metadata/scan/(?i)(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'^metadata/scan/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_scan, name='metadata_scan'), - url(r'^status$', views.status, name='status'), + path(r'status', views.status, name='status'), ] diff --git a/server/rest/views.py b/server/rest/views.py index f38fb36..56be9cc 100644 --- a/server/rest/views.py +++ b/server/rest/views.py @@ -163,12 +163,12 @@ def metadata_add(request, md5_hash, crc32, user): for client_key in functions: f = functions[client_key] - if not required_keys.issubset(f.keys()): + if not required_keys.issubset(list(f.keys())): return render(request, 'rest/error_json.html', {'msg' : 'Invalid function list'}) try: - f['opcodes'] = f['opcodes'].decode('base64') + f['opcodes'] = codecs.decode(f['opcodes'].encode(), 'base64') except binascii.Error as e: return render(request, 'rest/error_json.html', {'msg' : 'Unable to decode opcodes'}) @@ -235,7 +235,7 @@ def metadata_add(request, md5_hash, crc32, user): if not metadata_id: return render(request, 'rest/error_json.html', {'msg' : ('Unable to associate metadata with ' - 'function in FIRST')}) + 'function in FIRST')}) # The '0' indicated the metadata_id is from a user. _id = make_id(0, metadata=metadata_id) @@ -482,7 +482,7 @@ def metadata_scan(request, user): required_keys = {'opcodes', 'apis', 'architecture'} for client_id, details in functions.iteritems(): if ((dict != type(details)) - or (not required_keys.issubset(details.keys()))): + or (not required_keys.issubset(list(details.keys())))): return render(request, 'rest/error_json.html', {'msg' : 'Function details not provided'}) @@ -507,7 +507,7 @@ def metadata_scan(request, user): 'the submitted API valid is valid.')}) try: - opcodes = details['opcodes'].decode('base64') + opcodes = codecs.decode(details['opcodes'].encode(), 'base64') except binascii.Error as e: return render(request, 'rest/error_json.html', {'msg' : 'Unable to decode opcodes'}) diff --git a/server/utilities/engine_shell.py b/server/utilities/engine_shell.py index 80059a1..c2f0127 100644 --- a/server/utilities/engine_shell.py +++ b/server/utilities/engine_shell.py @@ -51,7 +51,7 @@ def emptyline(self): return def default(self, line): - print '"{}" is unknown command'.format(line) + print('"{}" is unknown command'.format(line)) def preloop(self): print ( '\n\n' @@ -90,30 +90,30 @@ def do_shell(self, line): class RootCmd(EngineCmd): def do_list(self, line): - print 'list - List all engines installed' + print('list - List all engines installed') if line in ['help', '?']: - print 'Usage: list \n' + print('Usage: list \n') return - print 'Engines currently installed\n' + print('Engines currently installed\n') if Engine.objects.count() == 0: - print 'No engines are currently installed' + print('No engines are currently installed') return for engine in Engine.objects.all(): name = engine.name description = engine.description - print '+{}+{}+'.format('-' * 18, '-' * 50) - for i in xrange(0, len(description), 48): - print '| {0:16} | {1:48} |'.format(name, description[i:i+48]) + print('+{}+{}+'.format('-' * 18, '-' * 50)) + for i in range(0, len(description), 48): + print('| {0:16} | {1:48} |'.format(name, description[i:i+48])) name = '' - print '+{}+{}+'.format('-' * 18, '-' * 50) + print('+{}+{}+'.format('-' * 18, '-' * 50)) def do_info(self, line): - print 'info - Displays details about an installed Engine' + print('info - Displays details about an installed Engine') if line in ['', 'help', '?']: - print 'Usage: info ' + print('Usage: info ') return engine = self._get_db_engine_obj(line) @@ -121,7 +121,7 @@ def do_info(self, line): return d = engine.description - d = [d[i:i+53] for i in xrange(0, len(d), 53)] + d = [d[i:i+53] for i in range(0, len(d), 53)] d = ' |\n| | '.join(['{:53}'.format(x) for x in d]) print ('+' + '-'*69 + '+\n' '| Name | {0.name:53} |\n' @@ -140,7 +140,7 @@ def do_info(self, line): '+' + '-'*69 + '+\n').format(engine, d) def do_install(self, line): - print 'install - Installs engine\n' + print('install - Installs engine\n') try: path, obj_name, email = line.split(' ') @@ -151,17 +151,17 @@ def do_install(self, line): # Skip module if the class name not located or is not a class if not hasattr(module, obj_name): - print '[Error] Unable to get {} from {}, exiting...'.format(obj_name, path) + print('[Error] Unable to get {} from {}, exiting...'.format(obj_name, path)) return obj = getattr(module, obj_name) if type(obj) != type: - print '[Error] {} is not the right type, exiting...'.format(obj_name) + print('[Error] {} is not the right type, exiting...'.format(obj_name)) return e = obj(DBManager, -1, -1) if not isinstance(e, AbstractEngine): - print '[Error] {} is not an AbstractEngine, exiting...'.format(obj_name) + print('[Error] {} is not an AbstractEngine, exiting...'.format(obj_name)) return e.install() @@ -170,14 +170,14 @@ def do_install(self, line): path=path, obj_name=obj_name, developer=developer, active=True) - print 'Engine added to FIRST' + print('Engine added to FIRST') return except ValueError as e: - print e + print(e) except ImportError as e: - print e + print(e) print ( 'Usage: \n' 'install \n' @@ -185,9 +185,9 @@ def do_install(self, line): 'Example of pythonic path: app.first.engines.exact_match\n') def do_delete(self, line): - print 'delete - Delete engine\n' + print('delete - Delete engine\n') if line in ['', 'help', '?']: - print 'Usage: delete ' + print('Usage: delete ') return engine, e = self._get_engine_by_name(line) @@ -198,9 +198,9 @@ def do_delete(self, line): engine.delete() def do_enable(self, line): - print 'enable - Enable engine \n' + print('enable - Enable engine \n') if line in ['', 'help', '?']: - print 'Usage: enable ' + print('Usage: enable ') return engine = self._get_db_engine_obj(line) @@ -209,12 +209,12 @@ def do_enable(self, line): engine.active = True engine.save() - print 'Engine "{}" enabled'.format(line) + print('Engine "{}" enabled'.format(line)) def do_disable(self, line): - print 'disable - Disable engine \n' + print('disable - Disable engine \n') if line in ['', 'help', '?']: - print 'Usage: disable ' + print('Usage: disable ') return engine = self._get_db_engine_obj(line) @@ -223,10 +223,10 @@ def do_disable(self, line): engine.active = False engine.save() - print 'Engine "{}" disabled'.format(line) + print('Engine "{}" disabled'.format(line)) def do_populate(self, line): - print 'populate - Populate engine by sending all functions to engine\n' + print('populate - Populate engine by sending all functions to engine\n') if line in ['', 'help', '?']: print ( 'Usage: populate ...\n\n' 'More than one engine name can be provided, separate with ' @@ -238,7 +238,7 @@ def do_populate(self, line): db = DBManager.first_db if not db: - print '[Error] Unable to connect to FIRST DB, exiting...' + print('[Error] Unable to connect to FIRST DB, exiting...') return # Get all engines the user entered @@ -246,16 +246,16 @@ def do_populate(self, line): engines = [] for engine_name in populate_engines: if engine_name not in all_engines: - print '[Error] Engine "{}" is not installed'.format(engine_name) + print('[Error] Engine "{}" is not installed'.format(engine_name)) continue engines.append(all_engines[engine_name]) if not engines: - print 'No engines to populate, exiting...' + print('No engines to populate, exiting...') return - print 'Starting to populate engines:\n-\t{}'.format('\n-\t'.join([e.name for e in engines])) + print('Starting to populate engines:\n-\t{}'.format('\n-\t'.join([e.name for e in engines]))) functions = db.get_all_functions().order_by('pk') total = functions.count() @@ -283,7 +283,7 @@ def do_populate(self, line): except Exception as e: msg = '[Error] Engine "{}": {}'.format(engine.name, e) errors.append(msg) - print msg + print(msg) i += 1 if 0 == (i % 50): @@ -292,20 +292,20 @@ def do_populate(self, line): sys.stdout.write('\n') sys.stdout.flush() - print 'Populating engines complete, exiting...' + print('Populating engines complete, exiting...') if errors: - print 'The below errors occured:\n{}'.format('\n '.join(errors)) + print('The below errors occured:\n{}'.format('\n '.join(errors))) def _get_db_engine_obj(self, name): engine = Engine.objects.filter(name=name) if not engine: - print 'Unable to locate Engine "{}"'.format(name) + print('Unable to locate Engine "{}"'.format(name)) return if len(engine) > 1: - print 'More than one engine "{}" exists'.format(name) + print('More than one engine "{}" exists'.format(name)) for e in engine: - print ' - {}: {}'.format(e.name, e.description) + print(' - {}: {}'.format(e.name, e.description)) return @@ -324,24 +324,24 @@ def _get_engine_by_name(self, name): # Skip module if the class name not located or is not a class if not hasattr(module, engine.obj_name): - print '[Error] Unable to get {0.obj_name} from {0.path}, exiting...'.format(engine) + print('[Error] Unable to get {0.obj_name} from {0.path}, exiting...'.format(engine)) return empty_result obj = getattr(module, engine.obj_name) if type(obj) != type: - print '[Error] {} is not the right type, exiting...'.format(engine.obj_name) + print('[Error] {} is not the right type, exiting...'.format(engine.obj_name)) return empty_result e = obj(DBManager, -1, -1) if not isinstance(e, AbstractEngine): - print '[Error] {} is not an AbstractEngine, exiting...'.format(engine.obj_name) + print('[Error] {} is not an AbstractEngine, exiting...'.format(engine.obj_name)) return empty_result except ValueError as e: - print e + print(e) except ImportError as e: - print e + print(e) return (engine, e) diff --git a/server/utilities/mongo_to_django_orm.py b/server/utilities/mongo_to_django_orm.py index 4139872..715a797 100644 --- a/server/utilities/mongo_to_django_orm.py +++ b/server/utilities/mongo_to_django_orm.py @@ -51,7 +51,7 @@ from django.core.paginator import Paginator, EmptyPage def info(): - print 'INFO: {} {}'.format(len(gc.get_objects()), sum([sys.getsizeof(o) for o in gc.get_objects()])) + print('INFO: {} {}'.format(len(gc.get_objects()), sum([sys.getsizeof(o) for o in gc.get_objects()]))) def migrate_users(): for u in User.objects.all(): @@ -90,14 +90,14 @@ def migrate_functions(skip, limit): i += 1 if 0 == (i % 1000): - print '---{}---'.format(i) + print('---{}---'.format(i)) info() gc.collect() info() def _mf(): - for i in xrange(0, Function.objects.count(), 1000): - print '--{}'.format(i) + for i in range(0, Function.objects.count(), 1000): + print('--{}'.format(i)) migrate_functions(i, 1000) if i % 20000 == 0: @@ -113,7 +113,7 @@ def migrate_apis(function, f): gc.collect() def migrate_metadata(function, f): - print 'Metadata: {} - {}'.format(f.sha256, len(f.metadata)) + print('Metadata: {} - {}'.format(f.sha256, len(f.metadata))) for m in f.metadata: creator = ORM.User.objects.get(email=m.user.email) metadata = ORM.Metadata.objects.create(user=creator) @@ -142,28 +142,28 @@ def main(args): user=args.mongo_user, password=getpass(pass_prompt)) # Convert User - print ' + Adding Users' + print(' + Adding Users') start = time.time() migrate_users() - print '[+] Users Added ({} s)'.format(time.time() - start) + print('[+] Users Added ({} s)'.format(time.time() - start)) # Convert Engine - print ' + Adding Engines' + print(' + Adding Engines') start = time.time() migrate_engines() - print '[+] Adding Engines ({} s)'.format(time.time() - start) + print('[+] Adding Engines ({} s)'.format(time.time() - start)) # Convert Samples - print ' + Adding Samples' + print(' + Adding Samples') start = time.time() migrate_samples() - print '[+] Adding Samples ({} s)'.format(time.time() - start) + print('[+] Adding Samples ({} s)'.format(time.time() - start)) # Convert Functions and their Metadata - print ' + Adding Functions & Metadata' + print(' + Adding Functions & Metadata') start = time.time() _mf() - print '[+] Adding Functions & Metadata ({} s)'.format(time.time() - start) + print('[+] Adding Functions & Metadata ({} s)'.format(time.time() - start)) @@ -239,7 +239,7 @@ def details(self): return [{'committed' : self.committed[i], 'name' : self.name[i], 'prototype' : self.prototype[i], - 'comment' : self.comment[i]} for i in xrange(len(self.name))] + 'comment' : self.comment[i]} for i in range(len(self.name))] # Use bson.Binary to insert binary data class Function(Document): diff --git a/server/utilities/user_shell.py b/server/utilities/user_shell.py index c472eab..2b268ea 100644 --- a/server/utilities/user_shell.py +++ b/server/utilities/user_shell.py @@ -51,7 +51,7 @@ def emptyline(self): return def default(self, line): - print '"{}" is unknown command'.format(line) + print('"{}" is unknown command'.format(line)) def preloop(self): print ( '\n\n' @@ -88,14 +88,14 @@ def do_shell(self, line): class RootCmd(UserCmd): def do_list(self, line): - print 'list - List all registered users' + print('list - List all registered users') if line in ['help', '?']: - print 'Usage: list \n' + print('Usage: list \n') return - print 'Registered Users\n' + print('Registered Users\n') if User.objects.count() == 0: - print 'No users are registered' + print('No users are registered') return header = ( '+{}+{}+\n'.format('-' * 39, '-' * 10) + @@ -105,21 +105,21 @@ def do_list(self, line): for user in User.objects.all(): handle = user.user_handle if (i % 15) == 0: - print header - print '| {0:37} | {1:^8} |'.format(handle, user.active) + print(header) + print('| {0:37} | {1:^8} |'.format(handle, user.active)) i += 1 - print '+{}+{}+'.format('-' * 39, '-' * 10) + print('+{}+{}+'.format('-' * 39, '-' * 10)) def do_adduser(self, line): - print 'info - Manually add user to FIRST' + print('info - Manually add user to FIRST') if line in ['', 'help', '?']: - print 'Usage: adduser ' + print('Usage: adduser ') return line = line.split(' ') if len(line) !=2: - print 'The correct arguments were not provided.' + print('The correct arguments were not provided.') return # Verify handle provided is valid @@ -128,26 +128,26 @@ def do_adduser(self, line): return if not re.match(r'^[a-zA-Z\d\._]+@[a-zA-Z\d\.\-_]+(?:\.[a-zA-Z]{2,4})+$', line[1]): - print 'Invalid email provided.' + print('Invalid email provided.') return email = line[1] user = self._get_db_user_obj(line[0]) if user: - print 'User {} already exists'.format(line[0]) + print('User {} already exists'.format(line[0])) return user = User(email=email, handle=handle, number=num, api_key=uuid4()) user.name = raw_input('Enter user name: ') user.save() - print 'User {0.user_handle} created (api key: {0.api_key})'.format(user) + print('User {0.user_handle} created (api key: {0.api_key})'.format(user)) def do_info(self, line): - print 'info - Displays details about a registered User' + print('info - Displays details about a registered User') if line in ['', 'help', '?']: - print 'Usage: info ' + print('Usage: info ') return user = self._get_db_user_obj(line) @@ -167,9 +167,9 @@ def do_info(self, line): '+' + '-'*65 + '+\n').format(user, str(user.created)) def do_enable(self, line): - print 'enable - Enable user \n' + print('enable - Enable user \n') if line in ['', 'help', '?']: - print 'Usage: enable ' + print('Usage: enable ') return user = self._get_db_user_obj(line) @@ -178,12 +178,12 @@ def do_enable(self, line): user.active = True user.save() - print 'User "{}" enabled'.format(line) + print('User "{}" enabled'.format(line)) def do_disable(self, line): - print 'disable - Disable user \n' + print('disable - Disable user \n') if line in ['', 'help', '?']: - print 'Usage: disable ' + print('Usage: disable ') return user = self._get_db_user_obj(line) @@ -192,12 +192,12 @@ def do_disable(self, line): user.active = False user.save() - print 'User "{}" disabled'.format(line) + print('User "{}" disabled'.format(line)) def _expand_user_handle(self, user_handle): matches = re.match('^([^#]+)#(\d{4})$', user_handle) if not matches: - print 'The provided handle is invalid' + print('The provided handle is invalid') return (None, None) handle, num = matches.groups() @@ -211,7 +211,7 @@ def _get_db_user_obj(self, line): user = User.objects.filter(handle=handle, number=int(num)) if not user: - print 'Unable to locate User handle "{}"'.format(line) + print('Unable to locate User handle "{}"'.format(line)) return return user.get() diff --git a/server/www/models.py b/server/www/models.py index a0efb4a..2407c0c 100644 --- a/server/www/models.py +++ b/server/www/models.py @@ -75,7 +75,7 @@ class Engine(models.Model): path = models.CharField(max_length=256) obj_name = models.CharField(max_length=32) - developer = models.ForeignKey('User') + developer = models.ForeignKey('User', on_delete=models.PROTECT) active = models.BooleanField(default=False) @property @@ -115,9 +115,9 @@ class Meta: # unique_together = ("sample_id", "user_id", "engine_metadata_id") class AppliedMetadata(models.Model): - metadata = models.ForeignKey('Metadata') - sample = models.ForeignKey('Sample') - user = models.ForeignKey('User') + metadata = models.ForeignKey('Metadata', on_delete = models.PROTECT) + sample = models.ForeignKey('Sample', on_delete = models.PROTECT) + user = models.ForeignKey('User', on_delete = models.PROTECT) class Meta: db_table = 'AppliedMetadata' @@ -137,7 +137,7 @@ class Meta: class Metadata(models.Model): id = models.BigAutoField(primary_key=True) - user = models.ForeignKey('User') + user = models.ForeignKey('User', on_delete = models.PROTECT) details = models.ManyToManyField('MetadataDetails') @property diff --git a/server/www/urls.py b/server/www/urls.py index 03d2c32..637367e 100644 --- a/server/www/urls.py +++ b/server/www/urls.py @@ -1,16 +1,16 @@ -from django.conf.urls import url +from django.urls import path, re_path, include from . import views app_name = 'www' urlpatterns = [ - url(r'^$', views.index, name='index'), - url(r'^profile$', views.profile, name='profile'), + path(r'', views.index, name='index'), + path(r'profile', views.profile, name='profile'), - url(r'^login$', views.login, name='login'), - url(r'^login/(?P[a-z]+)$', views.login, name='login'), - url(r'^oauth/(?P[a-z]+)$', views.oauth, name='oauth'), - url(r'^logout$', views.logout, name='logout'), + path(r'login', views.login, name='login'), + re_path(r'^login/(?P[a-z]+)$', views.login, name='login'), + re_path(r'^oauth/(?P[a-z]+)$', views.oauth, name='oauth'), + path(r'logout', views.logout, name='logout'), - url(r'^register$', views.register, name='register') + path(r'register', views.register, name='register') ] From 8ec5c27163eff4fedfc690fc5f3f6f7e6eb7ba0d Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Fri, 1 Mar 2019 08:32:52 -0500 Subject: [PATCH 02/17] Fix on URLs for Python3 support --- server/rest/urls.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/rest/urls.py b/server/rest/urls.py index 04bcbf0..b91fd45 100644 --- a/server/rest/urls.py +++ b/server/rest/urls.py @@ -4,31 +4,31 @@ app_name = 'rest' urlpatterns = [ - re_path(r'^test_connection/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'test_connection/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.test_connection, name='test_connection'), - re_path(r'^sample/architectures/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'sample/architectures/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.architectures, name='architectures'), - re_path(r'^sample/checkin/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'sample/checkin/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.checkin, name='checkin'), # Metadata related REST URIs - re_path(r'^metadata/history/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/history/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_history, name='metadata_history'), - re_path(r'^metadata/applied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/applied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_applied, name='metadata_applied'), - re_path(r'^metadata/unapplied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/unapplied/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_unapplied, name='metadata_unapplied'), - re_path(r'^metadata/get/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/get/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_get, name='metadata_get'), - re_path(r'^metadata/delete/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P<_id>[A-F\d]{26})$', + re_path(r'metadata/delete/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P<_id>[A-F\d]{26})$', views.metadata_delete, name='metadata_delete'), - re_path(r'^metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_created, name='metadata_created'), - re_path(r'^metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P\d+)$', + re_path(r'metadata/created/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})/(?P\d+)$', views.metadata_created, name='metadata_created'), - re_path(r'^metadata/add/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/add/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_add, name='metadata_add'), - re_path(r'^metadata/scan/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', + re_path(r'metadata/scan/(?P[A-F\d]{8}\-(?:[A-F\d]{4}\-){3}[A-F\d]{12})$', views.metadata_scan, name='metadata_scan'), path(r'status', views.status, name='status'), From 70637db07cc1f8ca7821de01db6fad188fed1ca8 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Fri, 1 Mar 2019 08:38:54 -0500 Subject: [PATCH 03/17] Add missing dependencies --- server/rest/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/rest/views.py b/server/rest/views.py index 56be9cc..643a53c 100644 --- a/server/rest/views.py +++ b/server/rest/views.py @@ -4,6 +4,7 @@ import json import binascii from functools import wraps +import codecs # Django Modules from django.shortcuts import render From 76ddf6eb49b76c5a241627f241e4bae79d33a2fb Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Fri, 1 Mar 2019 08:46:07 -0500 Subject: [PATCH 04/17] More Python3 fixes --- server/first_core/dbs/builtin_db.py | 4 ++-- server/first_core/engines/__init__.py | 2 +- server/rest/views.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/first_core/dbs/builtin_db.py b/server/first_core/dbs/builtin_db.py index 6c32a9a..d9cb974 100644 --- a/server/first_core/dbs/builtin_db.py +++ b/server/first_core/dbs/builtin_db.py @@ -241,7 +241,7 @@ def get_metadata_list(self, metadata): results = [] metadata_ids, engine_metadata = separate_metadata(metadata) - for _id, metadata in Metadata.objects.in_bulk(metadata_ids).iteritems(): + for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items(): data = metadata.dump() data['id'] = make_id(0, metadata=metadata.id) results.append(data) @@ -303,7 +303,7 @@ def metadata_history(self, metadata): e_comment = ('Generated by Engine: {0.name}\n{0.description}\n\n' 'Developer: {0.developer.user_handle}') - for _id, metadata in Metadata.objects.in_bulk(metadata_ids).iteritems(): + for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items(): data = metadata.dump(True) result_key = make_id(0, metadata=_id) results[result_key] = { 'creator' : data['creator'], diff --git a/server/first_core/engines/__init__.py b/server/first_core/engines/__init__.py index 3799dc2..f6c7c9f 100644 --- a/server/first_core/engines/__init__.py +++ b/server/first_core/engines/__init__.py @@ -282,7 +282,7 @@ def scan(self, user, opcodes, architecture, apis): print(e) results = {} - for i, hits in engine_results.iteritems(): + for i, hits in engine_results.items(): engine = engines[i] for result in hits: diff --git a/server/rest/views.py b/server/rest/views.py index 643a53c..152bd55 100644 --- a/server/rest/views.py +++ b/server/rest/views.py @@ -179,7 +179,7 @@ def metadata_add(request, md5_hash, crc32, user): # Ensure string lengths are enforced string_restrictions = { 'architecture' : 64, 'name' : 128, 'prototype' : 256, 'comment' : 512} - for key, max_length in string_restrictions.iteritems(): + for key, max_length in string_restrictions.items(): if max_length < len(f[key]): return render(request, 'rest/error_json.html', {'msg' : ('Data for "{}" exceeds the maximum ' @@ -481,7 +481,7 @@ def metadata_scan(request, user): # Validate input validated_input = {} required_keys = {'opcodes', 'apis', 'architecture'} - for client_id, details in functions.iteritems(): + for client_id, details in functions.items(): if ((dict != type(details)) or (not required_keys.issubset(list(details.keys())))): return render(request, 'rest/error_json.html', @@ -518,7 +518,7 @@ def metadata_scan(request, user): 'architecture' : architecture} data = {'engines' : {}, 'matches' : {}} - for client_id, details in validated_input.iteritems(): + for client_id, details in validated_input.items(): results = EngineManager.scan(user, **details) if (not results) or (results == ({}, [])): continue From 89e328014fcb013a711b74f29d0ee26040b7fa3a Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Mon, 4 Mar 2019 03:21:22 -0500 Subject: [PATCH 05/17] Removed long type --- server/first_core/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/first_core/util.py b/server/first_core/util.py index 2a13a3a..0c73186 100644 --- a/server/first_core/util.py +++ b/server/first_core/util.py @@ -33,7 +33,7 @@ def make_id(flags, metadata=0, engine=0): string: A 26 byte hex string ''' data = [flags, metadata, engine] - if (None in data) or (not all([type(x) in [int, long] for x in data])): + if (None in data) or (not all([type(x) in [int] for x in data])): return None if ((engine > (2**32 - 1)) or (metadata > (2**64 - 1)) @@ -50,7 +50,7 @@ def parse_id(_id): _id = int(_id, 16) - elif type(id) not in [int, long]: + elif type(id) not in [int]: return (None, None, None) flag = _id >> (8 * 12) From 0f9ab4357136fd7b174180a4b36ea984973d2156 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Mon, 4 Mar 2019 06:37:05 -0500 Subject: [PATCH 06/17] Fix on sorted function for Python3 support --- server/first_core/engines/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/first_core/engines/__init__.py b/server/first_core/engines/__init__.py index f6c7c9f..2926673 100644 --- a/server/first_core/engines/__init__.py +++ b/server/first_core/engines/__init__.py @@ -13,6 +13,7 @@ # Python Modules import re import sys +import functools # First Modules from first_core.error import FIRSTError @@ -298,7 +299,7 @@ def scan(self, user, opcodes, architecture, apis): # Order functions cmp_func = lambda x,y: cmp(y.similarity, x.similarity) - ordered_functions = sorted(results.values(), cmp_func) + ordered_functions = sorted(results.values(), key=functools.cmp_to_key(cmp_func)) # Create Metadata list # TODO: Narrow results to top 20 hits, use similarity and metadata rank From 823e1627db88d117fae5b440d8dde52678290a74 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Mon, 4 Mar 2019 07:03:47 -0500 Subject: [PATCH 07/17] Fixes on sha256 computation of strings (python3) --- server/first_core/engines/basic_masking.py | 2 +- server/first_core/engines/mnemonic_hash.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/first_core/engines/basic_masking.py b/server/first_core/engines/basic_masking.py index e1fd94d..4cb777c 100644 --- a/server/first_core/engines/basic_masking.py +++ b/server/first_core/engines/basic_masking.py @@ -151,7 +151,7 @@ def normalize(self, disassembly): print(145) return (0, None) - h_sha256 = sha256(''.join(normalized)).hexdigest() + h_sha256 = sha256(''.join(normalized).encode('utf-8')).hexdigest() return (changed_bytes, h_sha256) except Exception as e: diff --git a/server/first_core/engines/mnemonic_hash.py b/server/first_core/engines/mnemonic_hash.py index 9ad0569..90f3972 100644 --- a/server/first_core/engines/mnemonic_hash.py +++ b/server/first_core/engines/mnemonic_hash.py @@ -80,7 +80,7 @@ def mnemonic_hash(self, disassembly): if len(mnemonics) < MIN_REQUIRED_MNEMONICS: return (None, None) - return (mnemonics, sha256(''.join(mnemonics)).hexdigest()) + return (mnemonics, sha256(''.join(mnemonics).encode('utf-8')).hexdigest()) except Exception as e: raise e From 8e8d84c967aaf0feb9f91fc4ea73c8549c3701a2 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Mon, 4 Mar 2019 07:14:01 -0500 Subject: [PATCH 08/17] Substitute cmp function --- server/first_core/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/first_core/engines/__init__.py b/server/first_core/engines/__init__.py index 2926673..f242bc8 100644 --- a/server/first_core/engines/__init__.py +++ b/server/first_core/engines/__init__.py @@ -298,7 +298,7 @@ def scan(self, user, opcodes, architecture, apis): results[result.id].similarity = result.similarity # Order functions - cmp_func = lambda x,y: cmp(y.similarity, x.similarity) + cmp_func = lambda x,y: (y.similarity > x.similarity) - (y.similarity < x.similarity) ordered_functions = sorted(results.values(), key=functools.cmp_to_key(cmp_func)) # Create Metadata list From 015eee821934029345147c3dbe629c318ed8992c Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Mon, 4 Mar 2019 09:50:10 -0500 Subject: [PATCH 09/17] Fix on ORM query --- server/first_core/dbs/builtin_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/first_core/dbs/builtin_db.py b/server/first_core/dbs/builtin_db.py index d9cb974..a42029e 100644 --- a/server/first_core/dbs/builtin_db.py +++ b/server/first_core/dbs/builtin_db.py @@ -169,7 +169,7 @@ def get_function(self, opcodes, architecture, apis, create=False, **kwargs): opcodes=opcodes, architecture=architecture) - apis_ = [FunctionApis.objects.get_or_create(x)[0] for x in apis] + apis_ = [FunctionApis.objects.get_or_create(api=x)[0] for x in apis] for api in apis_: function.apis.add(api) From bb998ddfd37fc1dc93f7d5fba54ade8d95d74e0f Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 04:23:35 -0500 Subject: [PATCH 10/17] More Python3 migration --- server/first_core/engines/__init__.py | 4 ++-- server/first_core/engines/exact_match.py | 2 +- server/utilities/engine_shell.py | 2 +- server/utilities/user_shell.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/first_core/engines/__init__.py b/server/first_core/engines/__init__.py index f242bc8..a1c877c 100644 --- a/server/first_core/engines/__init__.py +++ b/server/first_core/engines/__init__.py @@ -114,7 +114,7 @@ def install(self): try: self._install() except FIRSTEngineError as e: - if e.message == 'Not Implemented': + if str(e) == 'Not Implemented': return raise e @@ -123,7 +123,7 @@ def uninstall(self): try: self._uninstall() except FIRSTEngineError as e: - if e.message == 'Not Implemented': + if str(e) == 'Not Implemented': return raise e diff --git a/server/first_core/engines/exact_match.py b/server/first_core/engines/exact_match.py index fb10857..fa6eded 100644 --- a/server/first_core/engines/exact_match.py +++ b/server/first_core/engines/exact_match.py @@ -50,7 +50,7 @@ def _scan(self, opcodes, architecture, apis, disassembly): return None similarity = 90.0 - if set(function.apis.values()) == set(apis): + if set([el['api'] for el in function.apis.values()]) == set(apis): similarity += 10.0 return [FunctionResult(str(function.id), similarity)] diff --git a/server/utilities/engine_shell.py b/server/utilities/engine_shell.py index c2f0127..983c208 100644 --- a/server/utilities/engine_shell.py +++ b/server/utilities/engine_shell.py @@ -86,7 +86,7 @@ def do_quit(self, line): def do_shell(self, line): '''Run line in python''' - exec line + exec(line) class RootCmd(EngineCmd): def do_list(self, line): diff --git a/server/utilities/user_shell.py b/server/utilities/user_shell.py index 2b268ea..c598904 100644 --- a/server/utilities/user_shell.py +++ b/server/utilities/user_shell.py @@ -84,7 +84,7 @@ def do_quit(self, line): def do_shell(self, line): '''Run line in python''' - exec line + exec(line) class RootCmd(UserCmd): def do_list(self, line): @@ -138,7 +138,7 @@ def do_adduser(self, line): return user = User(email=email, handle=handle, number=num, api_key=uuid4()) - user.name = raw_input('Enter user name: ') + user.name = input('Enter user name: ') user.save() print('User {0.user_handle} created (api key: {0.api_key})'.format(user)) From a1253d3f0eabfeee933e63b79798580a824a35c2 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 04:23:50 -0500 Subject: [PATCH 11/17] Changes in DB due to Python3 migration --- .../www/migrations/0002_auto_20190305_0329.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 server/www/migrations/0002_auto_20190305_0329.py diff --git a/server/www/migrations/0002_auto_20190305_0329.py b/server/www/migrations/0002_auto_20190305_0329.py new file mode 100644 index 0000000..e9cba09 --- /dev/null +++ b/server/www/migrations/0002_auto_20190305_0329.py @@ -0,0 +1,39 @@ +# Generated by Django 2.1.7 on 2019-03-05 08:29 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('www', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='appliedmetadata', + name='metadata', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.Metadata'), + ), + migrations.AlterField( + model_name='appliedmetadata', + name='sample', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.Sample'), + ), + migrations.AlterField( + model_name='appliedmetadata', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), + ), + migrations.AlterField( + model_name='engine', + name='developer', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), + ), + migrations.AlterField( + model_name='metadata', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), + ), + ] From df0feaf39bf02212f7b22eacbf7960502914774d Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 05:51:50 -0500 Subject: [PATCH 12/17] Reverted ON CASCADE policies on model --- .../www/migrations/0002_auto_20190305_0329.py | 39 ------------------- server/www/models.py | 10 ++--- 2 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 server/www/migrations/0002_auto_20190305_0329.py diff --git a/server/www/migrations/0002_auto_20190305_0329.py b/server/www/migrations/0002_auto_20190305_0329.py deleted file mode 100644 index e9cba09..0000000 --- a/server/www/migrations/0002_auto_20190305_0329.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 2.1.7 on 2019-03-05 08:29 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('www', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='appliedmetadata', - name='metadata', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.Metadata'), - ), - migrations.AlterField( - model_name='appliedmetadata', - name='sample', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.Sample'), - ), - migrations.AlterField( - model_name='appliedmetadata', - name='user', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), - ), - migrations.AlterField( - model_name='engine', - name='developer', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), - ), - migrations.AlterField( - model_name='metadata', - name='user', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='www.User'), - ), - ] diff --git a/server/www/models.py b/server/www/models.py index 2407c0c..7a3bca4 100644 --- a/server/www/models.py +++ b/server/www/models.py @@ -75,7 +75,7 @@ class Engine(models.Model): path = models.CharField(max_length=256) obj_name = models.CharField(max_length=32) - developer = models.ForeignKey('User', on_delete=models.PROTECT) + developer = models.ForeignKey('User', on_delete=models.CASCADE) active = models.BooleanField(default=False) @property @@ -115,9 +115,9 @@ class Meta: # unique_together = ("sample_id", "user_id", "engine_metadata_id") class AppliedMetadata(models.Model): - metadata = models.ForeignKey('Metadata', on_delete = models.PROTECT) - sample = models.ForeignKey('Sample', on_delete = models.PROTECT) - user = models.ForeignKey('User', on_delete = models.PROTECT) + metadata = models.ForeignKey('Metadata', on_delete = models.CASCADE) + sample = models.ForeignKey('Sample', on_delete = models.CASCADE) + user = models.ForeignKey('User', on_delete = models.CASCADE) class Meta: db_table = 'AppliedMetadata' @@ -137,7 +137,7 @@ class Meta: class Metadata(models.Model): id = models.BigAutoField(primary_key=True) - user = models.ForeignKey('User', on_delete = models.PROTECT) + user = models.ForeignKey('User', on_delete = models.CASCADE) details = models.ManyToManyField('MetadataDetails') @property From de90f88bf5b8ab1c791b30a1435cf8d98b2a0ca6 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 06:01:31 -0500 Subject: [PATCH 13/17] First version of REST automated tests --- server/rest/tests.py | 439 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 438 insertions(+), 1 deletion(-) diff --git a/server/rest/tests.py b/server/rest/tests.py index 7ce503c..6f7c356 100644 --- a/server/rest/tests.py +++ b/server/rest/tests.py @@ -1,3 +1,440 @@ from django.test import TestCase +from django.urls import reverse +from first_core.models import User +from first_core.models import Sample +from first_core.models import Engine -# Create your tests here. +import datetime +import json + +# NOTE: Before running these tests with "manage.py test rest", you +# will need to have all the migrations ready, both for the +# main database "manage.py makemigrations" and the following +# pluggable engines: ExactMatch, MnemomicHash, and BasicMasking. +# In order to prepare these migrations, run the engine_shell.py +# and install the engines (on the real database). That will +# create the corresponding rows in the database, and also +# invoke the makemigrations for each of those pluggable modules. +# (You can remove the entries from the database later, as the +# only important thing here are the migrations. The database +# entries for the test database are created in this test script). + +def create_user(**kwargs): + return User.objects.create(**kwargs) + +def create_engine(**kwargs): + return Engine.objects.create(**kwargs) + +class RestTests(TestCase): + def test_connection(self): + ''' + Connection test with correct and incorrect API keys + ''' + user1 = create_user(name = "user1", + email = "user1@noreply.cisco.com", + handle = "user1_h4x0r", + number = "1337", + api_key = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", + created = datetime.datetime.now(), + rank = 0, + active = True) + + # Test correct API key + response = self.client.get(reverse("rest:test_connection", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'})) + self.assertIs(response.status_code == 200 and \ + json.loads(str(response.content, encoding="utf-8")) == {"status": "connected"}, True) + + # Test incorrect API key + response = self.client.get(reverse("rest:test_connection", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'})) + self.assertIs(response.status_code == 401, True) + + def test_sample_architecture(self): + ''' + Test sample/architecures + ''' + user1 = create_user(name = "user1", + email = "user1@noreply.cisco.com", + handle = "user1_h4x0r", + number = "1337", + api_key = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", + created = datetime.datetime.now(), + rank = 0, + active = True) + + response = self.client.get(reverse("rest:architectures", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'})) + + passed = True + if response.status_code == 200: + d = json.loads(str(response.content, encoding="utf-8")) + if "failed" in d and d['failed'] == False: + if "architectures" in d: + for arch in ["intel64", "arm32", "sparc", "sysz", "intel32", "arm64", "intel16", "ppc", "mips"]: + if arch not in d["architectures"]: + passed = False + break + else: + passed = False + else: + passed = False + else: + passed = False + + self.assertIs(passed, True) + + def test_sample_checkin(self): + ''' + Test sample checkin + ''' + user1 = create_user(name = "user1", + email = "user1@noreply.cisco.com", + handle = "user1_h4x0r", + number = "1337", + api_key = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", + created = datetime.datetime.now(), + rank = 0, + active = True) + + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "sha1": "AA" * 40, "sha256": "AA" * 64} ) + + passed = True + if response.status_code == 200: + d = json.loads(str(response.content, encoding="utf-8")) + if "failed" not in d or d["failed"] is True: + passed = False + if "checkin" not in d or d["checkin"] is False: + passed = False + else: + passed = False + + self.assertIs(passed, True) + + # Get the last_seen date of the sample + s = Sample.objects.get(md5="AA" * 16) + self.assertIs(s is None, False) + last_seen_1 = s.last_seen + + # Run again, the last_seen data should be updated and a new relationship between the other user + # and the sample should be created + + user2 = create_user(name = "user2", + email = "user2@noreply.cisco.com", + handle = "user2_h4x0r", + number = "1338", + api_key = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAACC", + created = datetime.datetime.now(), + rank = 0, + active = True) + + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAACC'}), + {"md5": "AA" * 16, "crc32": 0, "sha1": "AA" * 40, "sha256": "AA" * 64} ) + + passed = True + if response.status_code == 200: + d = json.loads(str(response.content, encoding="utf-8")) + if "failed" not in d or d["failed"] is True: + passed = False + if "checkin" not in d or d["checkin"] is False: + passed = False + else: + passed = False + + # Now, get the last seen date of the sample + s = Sample.objects.get(md5="AA" * 16) + self.assertIs(s is None, False) + last_seen_2 = s.last_seen + + self.assertIs(last_seen_1 < last_seen_2, True) + + # Sample has been seen by 2 users + self.assertIs(len(s.seen_by.all()), 2) + + def test_metadata(self): + ''' + Big test for all the metadata related + functionality. It is used to test the + different metadata related APIs. + ''' + # User1 adds some metadata to a pair of functions + + # First, we need to create the user + user1 = create_user(name = "user1", + email = "user1@noreply.cisco.com", + handle = "user1_h4x0r", + number = "1337", + api_key = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", + created = datetime.datetime.now(), + rank = 0, + active = True) + + # Second, we need to create/activate the Engines + e1 = create_engine(name = "ExactMatch", + description = "Desc of ExactMatch", + path = "first_core.engines.exact_match", + obj_name = "ExactMatchEngine", + developer = user1, + active = True) + + e2 = create_engine(name = "MnemonicHash", + description = "Desc of MnemonicHash", + path = "first_core.engines.mnemonic_hash", + obj_name = "MnemonicHashEngine", + developer = user1, + active = True) + + e3 = create_engine(name = "BasicMasking", + description = "Desc of BasicMasking", + path = "first_core.engines.basic_masking", + obj_name = "BasicMaskingEngine", + developer = user1, + active = True) + + # And test that these engines have been correctly created + self.assertIs(e1 is not None and e2 is not None and e3 is not None, True) + + # Create the sample + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "sha1": "AA" * 40, "sha256": "AA" * 64} ) + + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("checkin" in d and d["checkin"] is True, True) + + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_0' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'name' : "my_function_0", + 'prototype' : "int my_function_0(int a)", + 'comment' : "This is a comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + }, + 'function_id_1' : + { + 'opcodes' : "VTHSieWLRQhWi3UMU41Y/w+2DBaITBMBg8IBhMl18VteXcM=", + 'architecture' : "intel32", + 'name' : "my_function_1", + 'prototype' : "int my_function_1(int b)", + 'comment' : "This is a comment for function 1", + 'apis' : ["CreateThread", "WriteProcessMemory"] + } + })}) + + self.assertIs(response.status_code, 200) + #b'{"failed": false, "results": {"function_id_0": "00000000000000000000000001", "function_id_1": "00000000000000000000000003"}}' + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d, True) + self.assertIs("function_id_0" in d["results"], True) + self.assertIs("function_id_1" in d["results"], True) + self.assertIs(d["results"]["function_id_0"] is not None, True) + self.assertIs(d["results"]["function_id_1"] is not None, True) + + # Retrieve metadata created by user + response = self.client.get(reverse("rest:metadata_created", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'})) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("page" in d and d["page"] == 1, True) + self.assertIs("pages" in d and d["pages"] == 1, True) + self.assertIs("results" in d and len(list(d["results"])) == 2, True) + + # We collect the metadata ids + metadata_ids = [] + for f in d["results"]: + if f["name"] == "my_function_0": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_0(int a)" , True) + self.assertIs(f["comment"] == "This is a comment for function 0" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["id"] is not None , True) + metadata_ids.append(f["id"]) + elif f["name"] == "my_function_1": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_1(int b)" , True) + self.assertIs(f["comment"] == "This is a comment for function 1" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["id"] is not None , True) + metadata_ids.append(f["id"]) + else: + self.assertIs("Incorrect function name...", True) + + # Test for metadata_get + response = self.client.post(reverse("rest:metadata_get", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": json.dumps(metadata_ids) } ) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d and len(list(d["results"])) == len(metadata_ids), True) + + for metadata_id in metadata_ids: + self.assertIs(metadata_id in d["results"], True) + f = d["results"][metadata_id] + if f["name"] == "my_function_0": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_0(int a)" , True) + self.assertIs(f["comment"] == "This is a comment for function 0" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["id"] is not None , True) + elif f["name"] == "my_function_1": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_1(int b)" , True) + self.assertIs(f["comment"] == "This is a comment for function 1" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["id"] is not None , True) + else: + self.assertIs("Incorrect function name...", True) + + # Test for metadata_scan + + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {'functions' : json.dumps( + { + 'function_id_0' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'apis' : ["ExitProcess", "CreateProcessA"] + }, + 'function_id_1' : + { + 'opcodes' : "VTHSieWLRQhWi3UMU41Y/w+2DBaITBMBg8IBhMl18VteXcM=", + 'architecture' : "intel32", + 'apis' : ["CreateThread", "WriteProcessMemory"] + }, + 'function_id_2' : + { + 'opcodes' : "VTHSieWLafrci3UMU41Y/w+2DBaITBMBg8IBhMl18VteXcM=", + 'architecture' : "intel32", + 'apis' : ["CreateThread", "WriteProcessMemory"] + } + + })}) + + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d and "engines" in d["results"] and "matches" in d["results"], True) + self.assertIs("ExactMatch" in d["results"]["engines"], True) + self.assertIs("MnemonicHash" in d["results"]["engines"], True) + self.assertIs("BasicMasking" in d["results"]["engines"], True) + + self.assertIs("function_id_0" in d["results"]["matches"], True) + self.assertIs("function_id_1" in d["results"]["matches"], True) + self.assertIs("function_id_2" not in d["results"]["matches"], True) + + for f_id in d["results"]["matches"]: + for f in d["results"]["matches"][f_id]: + if f["name"] == "my_function_0": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_0(int a)" , True) + self.assertIs(f["comment"] == "This is a comment for function 0" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["similarity"] == 100.0 , True) + self.assertIs(set(f["engines"]) == set(['MnemonicHash', 'ExactMatch', 'BasicMasking']) , True) + self.assertIs(f["id"] is not None , True) + elif f["name"] == "my_function_1": + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs(f["prototype"] == "int my_function_1(int b)" , True) + self.assertIs(f["comment"] == "This is a comment for function 1" , True) + self.assertIs(f["rank"] == 1 , True) + self.assertIs(f["similarity"] == 100.0 , True) + self.assertIs(set(f["engines"]) == set(['MnemonicHash', 'ExactMatch', 'BasicMasking']) , True) + self.assertIs(f["id"] is not None , True) + else: + self.assertIs("Incorrect function name...", True) + + # Test metadata applied / unapplied + + # Apply the metadata + for metadata_id in metadata_ids: + response = self.client.post(reverse("rest:metadata_applied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "id": metadata_id }) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d and d["results"] is True, True) + + # Unapply the metadata + for metadata_id in metadata_ids: + response = self.client.post(reverse("rest:metadata_unapplied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "id": metadata_id }) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d and d["results"] is True, True) + + + # Test metadata history + # First, we update the metadata, with a second version + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_0' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'name' : "my_function_0_v2", + 'prototype' : "int my_function_0_v2(int a)", + 'comment' : "This is a v2 comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + }, + 'function_id_1' : + { + 'opcodes' : "VTHSieWLRQhWi3UMU41Y/w+2DBaITBMBg8IBhMl18VteXcM=", + 'architecture' : "intel32", + 'name' : "my_function_1_v2", + 'prototype' : "int my_function_1_v2(int b)", + 'comment' : "This is a v2 comment for function 1", + 'apis' : ["CreateThread", "WriteProcessMemory"] + } + })}) + + self.assertIs(response.status_code, 200) + + # Then, we retrieve the history + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": json.dumps(metadata_ids) }) + + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is False, True) + self.assertIs("results" in d and isinstance(d["results"], dict), True) + for _id in d["results"].keys(): + f = d["results"][_id] + self.assertIs(f["creator"] == "user1_h4x0r#1337" , True) + self.assertIs("history" in f and isinstance(f["history"], list) , True) + for fh in f["history"]: + self.assertIs("committed" in fh and fh["committed"] is not None, True) + if fh["name"] == "my_function_0": + self.assertIs(fh["prototype"] == "int my_function_0(int a)" , True) + self.assertIs(fh["comment"] == "This is a comment for function 0" , True) + elif fh["name"] == "my_function_0_v2": + self.assertIs(fh["prototype"] == "int my_function_0_v2(int a)" , True) + self.assertIs(fh["comment"] == "This is a v2 comment for function 0" , True) + elif fh["name"] == "my_function_1": + self.assertIs(fh["prototype"] == "int my_function_1(int b)" , True) + self.assertIs(fh["comment"] == "This is a comment for function 1" , True) + elif fh["name"] == "my_function_1_v2": + self.assertIs(fh["prototype"] == "int my_function_1_v2(int b)" , True) + self.assertIs(fh["comment"] == "This is a v2 comment for function 1" , True) + else: + self.assertIs("Incorrect function name...", True) + + # Finally, test metadata deletion + for _id in metadata_ids: + response = self.client.get(reverse("rest:metadata_delete", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA', '_id': _id})) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs('failed' in d and d['failed'] is False, True) + self.assertIs('deleted' in d and d['deleted'] is True, True) + + print("Successfully finished tests!!") From 314f22a9f4cce9b0a273c3d0d8493a07c7172b91 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 06:12:13 -0500 Subject: [PATCH 14/17] Fix a pair of python3 print statements --- server/utilities/engine_shell.py | 4 ++-- server/utilities/user_shell.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/utilities/engine_shell.py b/server/utilities/engine_shell.py index 983c208..49dfe1b 100644 --- a/server/utilities/engine_shell.py +++ b/server/utilities/engine_shell.py @@ -123,7 +123,7 @@ def do_info(self, line): d = engine.description d = [d[i:i+53] for i in range(0, len(d), 53)] d = ' |\n| | '.join(['{:53}'.format(x) for x in d]) - print ('+' + '-'*69 + '+\n' + print(('+' + '-'*69 + '+\n' '| Name | {0.name:53} |\n' '+' + '-'*13 + '+' + '-'*55 + '\n' '| Description | {1:53} |\n' @@ -137,7 +137,7 @@ def do_info(self, line): '| Rank | {0.rank:<53} |\n' '+' + '-'*13 + '+' + '-'*55 + '\n' '| Active | {0.active:<53} |\n' - '+' + '-'*69 + '+\n').format(engine, d) + '+' + '-'*69 + '+\n').format(engine, d)) def do_install(self, line): print('install - Installs engine\n') diff --git a/server/utilities/user_shell.py b/server/utilities/user_shell.py index c598904..3dc2a0a 100644 --- a/server/utilities/user_shell.py +++ b/server/utilities/user_shell.py @@ -154,7 +154,7 @@ def do_info(self, line): if not user: return - print ('+' + '-'*65 + '+\n' + print(('+' + '-'*65 + '+\n' '| Name | {0.name:53} |\n' '+' + '-'*9 + '+' + '-'*55 + '\n' '| Email | {0.email:53} |\n' @@ -164,7 +164,7 @@ def do_info(self, line): '| Created | {1:53} |\n' '+' + '-'*9 + '+' + '-'*55 + '\n' '| Active | {0.active:53} |\n' - '+' + '-'*65 + '+\n').format(user, str(user.created)) + '+' + '-'*65 + '+\n').format(user, str(user.created))) def do_enable(self, line): print('enable - Enable user \n') From 7f38dc6a433367bd6fc090d1542c8e4af8a1f036 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 07:56:26 -0500 Subject: [PATCH 15/17] Added tests for negative test cases --- server/rest/tests.py | 357 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 350 insertions(+), 7 deletions(-) diff --git a/server/rest/tests.py b/server/rest/tests.py index 6f7c356..df55d05 100644 --- a/server/rest/tests.py +++ b/server/rest/tests.py @@ -9,15 +9,16 @@ # NOTE: Before running these tests with "manage.py test rest", you # will need to have all the migrations ready, both for the -# main database "manage.py makemigrations" and the following +# main database "manage.py makemigrations" and for the following # pluggable engines: ExactMatch, MnemomicHash, and BasicMasking. # In order to prepare these migrations, run the engine_shell.py # and install the engines (on the real database). That will -# create the corresponding rows in the database, and also +# create the corresponding rows in the database, and it will also # invoke the makemigrations for each of those pluggable modules. -# (You can remove the entries from the database later, as the -# only important thing here are the migrations. The database -# entries for the test database are created in this test script). +# Afterwards, if you want, you can remove the entries from the +# database later, as the only important thing here are the migrations. +# The database entries for the test database are created in this +# test script. def create_user(**kwargs): return User.objects.create(**kwargs) @@ -81,6 +82,11 @@ def test_sample_architecture(self): self.assertIs(passed, True) + # Test incorrect API key + response = self.client.get(reverse("rest:architectures", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'})) + self.assertIs(response.status_code == 401, True) + + def test_sample_checkin(self): ''' Test sample checkin @@ -109,6 +115,12 @@ def test_sample_checkin(self): self.assertIs(passed, True) + # Test for non-existing user + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"md5": "AA" * 16, "crc32": 0, "sha1": "AA" * 40, "sha256": "AA" * 64} ) + + self.assertIs(response.status_code == 401, True) + # Get the last_seen date of the sample s = Sample.objects.get(md5="AA" * 16) self.assertIs(s is None, False) @@ -149,13 +161,43 @@ def test_sample_checkin(self): # Sample has been seen by 2 users self.assertIs(len(s.seen_by.all()), 2) + # Try with incorrect parameters: missing crc32 + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16} ) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Sample info not provided", True) + # Try with incorrect parameters: missing md5 + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"crc32": 0} ) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Sample info not provided", True) + # Try without optional parameters + response = self.client.post(reverse("rest:checkin", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0} ) + passed = True + if response.status_code == 200: + d = json.loads(str(response.content, encoding="utf-8")) + if "failed" not in d or d["failed"] is True: + passed = False + if "checkin" not in d or d["checkin"] is False: + passed = False + else: + passed = False + + self.assertIs(passed, True) + def test_metadata(self): ''' Big test for all the metadata related functionality. It is used to test the different metadata related APIs. ''' - # User1 adds some metadata to a pair of functions + # Initial data + # ============ # First, we need to create the user user1 = create_user(name = "user1", @@ -200,6 +242,9 @@ def test_metadata(self): d = json.loads(str(response.content, encoding="utf-8")) self.assertIs("checkin" in d and d["checkin"] is True, True) + # Test metadata_add + # ================= + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), {"md5": "AA" * 16, "crc32": 0, 'functions' : json.dumps( @@ -225,7 +270,6 @@ def test_metadata(self): })}) self.assertIs(response.status_code, 200) - #b'{"failed": false, "results": {"function_id_0": "00000000000000000000000001", "function_id_1": "00000000000000000000000003"}}' d = json.loads(str(response.content, encoding="utf-8")) self.assertIs("failed" in d and d["failed"] is False, True) self.assertIs("results" in d, True) @@ -234,7 +278,89 @@ def test_metadata(self): self.assertIs(d["results"]["function_id_0"] is not None, True) self.assertIs(d["results"]["function_id_1"] is not None, True) + # Try and incorrect case: unauthorized + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_2' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'name' : "my_function_0", + 'prototype' : "int my_function_0(int a)", + 'comment' : "This is a comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 401, True) + + # Try and incorrect case: base64 with broken padding + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_2' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy=", + 'architecture' : "intel32", + 'name' : "my_function_0", + 'prototype' : "int my_function_0(int a)", + 'comment' : "This is a comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Unable to decode opcodes", True) + + # Try and incorrect case: invalid architecture parameter + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_2' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "my_custom_arch_this_is_super_long_and_should_not_be_valid" * 10, + 'name' : "my_function_0", + 'prototype' : "int my_function_0(int a)", + 'comment' : "This is a comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Data for "architecture" exceeds the maximum length (64)", True) + + # Try and incorrect case: missing parameter + response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, + 'functions' : json.dumps( + { + 'function_id_2' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'prototype' : "int my_function_0(int a)", + 'comment' : "This is a comment for function 0", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid function list", True) + # Retrieve metadata created by user + # ================================= + response = self.client.get(reverse("rest:metadata_created", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'})) self.assertIs(response.status_code, 200) d = json.loads(str(response.content, encoding="utf-8")) @@ -264,7 +390,12 @@ def test_metadata(self): else: self.assertIs("Incorrect function name...", True) + # Incorrect API key + response = self.client.get(reverse("rest:metadata_created", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'})) + self.assertIs(response.status_code == 401, True) + # Test for metadata_get + # ===================== response = self.client.post(reverse("rest:metadata_get", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), {"metadata": json.dumps(metadata_ids) } ) self.assertIs(response.status_code, 200) @@ -291,7 +422,31 @@ def test_metadata(self): else: self.assertIs("Incorrect function name...", True) + # Incorrect API key + response = self.client.post(reverse("rest:metadata_get", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"metadata": json.dumps(metadata_ids) } ) + self.assertIs(response.status_code == 401, True) + + # Missing or incorrect parameter + response = self.client.post(reverse("rest:metadata_get", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metdata": json.dumps(["blablabla", "blebleble"]) } ) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid metadata information", True) + + # Non existing id + response = self.client.post(reverse("rest:metadata_get", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": json.dumps(["blablabla", "blebleble"]) } ) + self.assertIs(response.status_code, 200) + d = json.loads(str(response.content, encoding="utf-8")) + + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid id value", True) + # Test for metadata_scan + # ====================== response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), {'functions' : json.dumps( @@ -351,7 +506,91 @@ def test_metadata(self): else: self.assertIs("Incorrect function name...", True) + # Incorrect api key + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {'functions' : json.dumps( + { + 'function_id_3' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 401, True) + + # Incorrect parameter + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {'function' : json.dumps( + { + 'function_id_3' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid function information", True) + + + # Incorrect values for function + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {'functions' : json.dumps( + { + 'function_id_3' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32", + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Unable to decode opcodes", True) + + # Incorrect values for function + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {'functions' : json.dumps( + { + 'function_id_3' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32" * 32, + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid architecture", True) + + #No APIs + response = self.client.post(reverse("rest:metadata_scan", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {'function' : json.dumps( + { + 'function_id_3' : + { + 'opcodes' : "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=", + 'architecture' : "intel32" * 32, + 'apis' : ["ExitProcess", "CreateProcessA"] + } + })}) + + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid function information", True) + # Test metadata applied / unapplied + # ================================= # Apply the metadata for metadata_id in metadata_ids: @@ -362,6 +601,27 @@ def test_metadata(self): self.assertIs("failed" in d and d["failed"] is False, True) self.assertIs("results" in d and d["results"] is True, True) + # Incorrect api key + response = self.client.post(reverse("rest:metadata_applied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"md5": "AA" * 16, "crc32": 0, "id": metadata_id }) + self.assertIs(response.status_code == 401, True) + + # Missing sample info + response = self.client.post(reverse("rest:metadata_applied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "id": metadata_id }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Sample info not provided", True) + + # Incorrect id + response = self.client.post(reverse("rest:metadata_applied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "id": "blablabla" }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid id value", True) + # Unapply the metadata for metadata_id in metadata_ids: response = self.client.post(reverse("rest:metadata_unapplied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), @@ -371,8 +631,29 @@ def test_metadata(self): self.assertIs("failed" in d and d["failed"] is False, True) self.assertIs("results" in d and d["results"] is True, True) + # Incorrect api key + response = self.client.post(reverse("rest:metadata_unapplied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"md5": "AA" * 16, "crc32": 0, "id": metadata_id }) + self.assertIs(response.status_code == 401, True) + + # Missing sample info + response = self.client.post(reverse("rest:metadata_unapplied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "id": metadata_id }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Sample info not provided", True) + + # Incorrect id + response = self.client.post(reverse("rest:metadata_unapplied", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"md5": "AA" * 16, "crc32": 0, "id": "blablabla" }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid id value", True) # Test metadata history + # ===================== # First, we update the metadata, with a second version response = self.client.post(reverse("rest:metadata_add", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), {"md5": "AA" * 16, "crc32": 0, @@ -429,7 +710,47 @@ def test_metadata(self): else: self.assertIs("Incorrect function name...", True) + # Incorrect ID + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB'}), + {"metadata": json.dumps(metadata_ids) }) + self.assertIs(response.status_code == 401, True) + + + # Missing or incorrect parameters + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadaa": json.dumps(metadata_ids) }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid metadata information", True) + + + # Incorrect json object + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": "blabla" }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid json object", True) + + # Incorrect metadata ids + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": json.dumps(["blabla"]) }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid metadata id", True) + + # Incorrect metadata ids + response = self.client.post(reverse("rest:metadata_history", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'}), + {"metadata": json.dumps("blabla") }) + self.assertIs(response.status_code == 200, True) + d = json.loads(str(response.content, encoding="utf-8")) + self.assertIs("failed" in d and d["failed"] is True, True) + self.assertIs("msg" in d and d["msg"] == "Invalid metadata id", True) + # Finally, test metadata deletion + # =============================== for _id in metadata_ids: response = self.client.get(reverse("rest:metadata_delete", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA', '_id': _id})) self.assertIs(response.status_code, 200) @@ -437,4 +758,26 @@ def test_metadata(self): self.assertIs('failed' in d and d['failed'] is False, True) self.assertIs('deleted' in d and d['deleted'] is True, True) + # Incorrect API key + response = self.client.get(reverse("rest:metadata_delete", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAABB', '_id': _id})) + self.assertIs(response.status_code == 401, True) + + # Incorrect parameter + import django.urls.exceptions + failed = False + try: + response = self.client.get(reverse("rest:metadata_delete", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA', '_ida': _id})) + except django.urls.exceptions.NoReverseMatch: + failed = True + self.assertIs(failed, True) + + # Incorrect id + import django.urls.exceptions + failed = False + try: + response = self.client.get(reverse("rest:metadata_delete", kwargs={'api_key' : 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA', '_id': "blabla"})) + except django.urls.exceptions.NoReverseMatch: + failed = True + self.assertIs(failed, True) + print("Successfully finished tests!!") From 1f9dd3a1d24c7512643e77991661077cec74612e Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 10:27:10 -0500 Subject: [PATCH 16/17] Updated docs to point to FreeBSD port --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 5c531e0..72bced1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -68,6 +68,10 @@ Once an engine is installed you can start using your FIRST installation to add a $ cd FIRST-server/server $ python manage.py runserver 0.0.0.0:1337 +.. note:: FreeBSD port + + FIRST also has a FreeBSD port available: https://www.freshports.org/security/py-first-server/ + .. _server-docs: .. toctree:: From a8d6c8fdaf695887acf5d8d750c3070cf8afbbd7 Mon Sep 17 00:00:00 2001 From: xabiugarte Date: Tue, 5 Mar 2019 10:27:24 -0500 Subject: [PATCH 17/17] Small update to example json configuration file --- server/example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/example_config.json b/server/example_config.json index 43ed855..bb19c86 100644 --- a/server/example_config.json +++ b/server/example_config.json @@ -11,5 +11,5 @@ "debug" : true, "allowed_hosts" : ["localhost", "testserver"], - "oauth_path" : "", + "oauth_path" : "/path/to/your/google_secret.json", }