Merge pull request #2 from demonduck/capstone_migration

Capstone migration

docs/engines/index.rst

@@ -8,12 +8,29 @@ Engines
 Engine Shell
 ============
 
+The Python script ``engine_shell.py`` provides you with some functionality to manage engines installed into FIRST. Below you will see the script's operations.
+
+.. code::
+
+    +========================================================+
+    |                 FIRST Engine Shell Menu                |
+    +========================================================+
+    | list     | List all engines currently installed        |
+    | info     | Get info on an engine                       |
+    | install  | Installs engine                             |
+    | delete   | Removes engine record but not other DB data |
+    | enable   | Enable engine (Engine will be enabled)      |
+    | populate | Sending all functions to engine             |
+    | disable  | Disable engine (Engine will be disabled)    |
+    +--------------------------------------------------------+
+
+
 
 Testing Engines
 ===============
 TODO
 
-.. autoclass:: first.engines.AbstractEngine
+.. autoclass:: first_core.engines.AbstractEngine
     :noindex:
     :members:
     :undoc-members:

docs/index.rst

@@ -17,7 +17,7 @@ Installing your own FIRST server can be quick and easy with an Ubuntu machine an
 
     **After cloning the Git repo**
 
-    Save your google auth json information to install/google_secret.json
+    Save your google auth json information to install/google_secret.json. To generate a google_secret.json file you will need to go to https://console.developers.google.com, create a project, select the project, select Credentials in the left set of links under APIs & services. Once selected, select the Create credentials drop down menu and click OAuth client ID. Select Web application, and fill out the details. Once created you will have the option to down the JSON file containing the generated secret.
 
     Optionally, you can add install/ssl/apache.crt and apache.key file if you have an SSL certificate you would prefer to use.
 
@@ -32,16 +32,23 @@ When the FIRST server is installed, no engines are installed. FIRST comes with t
 
 .. note::
 
-    Before engines can be installed, the developer must be registered with the system. Ensure the developer is registered before progressing.
+    Before engines can be installed, the developer must be registered with the system. This can be accomplished through the web UI if OAuth has been setup or manually by the user_shell.py located in the utilities folder.
+
+    .. code::
+
+        $ cd FIRST-server/server/utilities
+        $ python user_shell.py <user_handle: johndoe#0001> <user email: [email protected]>
+
+    Ensure the developer is registered before progressing.
 
 Python script ``engine_shell.py`` can be provided with command line arguments or used as a shell. To quickly install the three available engines run the below commands:
 
 .. code::
 
     $ cd FIRST-server/server/utilities
-    $ python engine_shell.py install first.engines.exact_match ExactMatchEngine <developer_email>
-    $ python engine_shell.py install first.engines.mnemonic_hash MnemonicHashEngine <developer_email>
-    $ python engine_shell.py install first.engines.basic_masking BasicMaskingEngine <developer_email>
+    $ python engine_shell.py install first_core.engines.exact_match ExactMatchEngine <developer_email>
+    $ python engine_shell.py install first_core.engines.mnemonic_hash MnemonicHashEngine <developer_email>
+    $ python engine_shell.py install first_core.engines.basic_masking BasicMaskingEngine <developer_email>
 
 Once an engine is installed you can start using your FIRST installation to add and/or query for annotations. Without engines FIRST will still be able to store annotations, but will never return any results for query operations.
 

server/engines/admin.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.contrib import admin
+
+# Register your models here.

server/engines/apps.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class EnginesConfig(AppConfig):
+    name = 'engines'

server/engines/models.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models
+
+# Create your models here.

server/engines/tests.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.test import TestCase
+
+# Create your tests here.

server/engines/views.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.shortcuts import render
+
+# Create your views here.

server/first/settings.py

@@ -48,6 +48,7 @@
 
 INSTALLED_APPS = [
     'www.apps.WwwConfig',
+    'engines.apps.EnginesConfig',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

server/first_core/auth.py

@@ -75,7 +75,7 @@ def decorated_function(*args, **kwargs):
         if key:
             user = verify_api_key(key)
             del kwargs['api_key']
-            if user:
+            if user and user.active:
                 kwargs['user'] = user
                 return view_function(*args, **kwargs)
 

server/first_core/dbs/builtin_db.py

@@ -224,12 +224,9 @@ def add_metadata_to_function(self, user, function, name, prototype, comment, **k
 
         #   Check to see if user already has metadata associated with the sample
         metadata = None
-        print function.id
-        print user.id
         if Function.objects.filter(pk=function.id, metadata__user=user).count():
             #   Metadata already exists
             metadata = Metadata.objects.get(function=function, user=user)
-
         else:
             metadata = Metadata.objects.create(user=user)
             function.metadata.add(metadata)

server/first_core/disassembly/__init__.py

@@ -0,0 +1,172 @@
+#   Third Party Modules
+from capstone import *
+from capstone.ppc import *
+from capstone.systemz import *
+from capstone.arm import *
+from capstone.arm64 import *
+from capstone.x86 import *
+from capstone.sparc import *
+from capstone.mips import *
+
+arch_mapping = {
+    'ppc' : (CS_ARCH_PPC, CS_MODE_32),
+    'ppc32' : (CS_ARCH_PPC, CS_MODE_32),
+    'ppc64' : (CS_ARCH_PPC, CS_MODE_64),
+    'intel16' : (CS_ARCH_X86, CS_MODE_16),
+    'sysz' : (CS_ARCH_SYSZ, None),
+    'arm32' : (CS_ARCH_ARM, CS_MODE_ARM),
+    'intel32' : (CS_ARCH_X86, CS_MODE_32),
+    'intel64' : (CS_ARCH_X86, CS_MODE_64),
+    'sparc' : (CS_ARCH_SPARC, None),
+    'arm64' : (CS_ARCH_ARM64, CS_MODE_ARM),
+    'mips' : (CS_ARCH_MIPS, CS_MODE_32),
+    'mips64' : (CS_ARCH_MIPS, CS_MODE_64)
+}
+
+reg_mapping = {
+    'ppc' :  PPC_OP_REG, 'ppc32' : PPC_OP_REG, 'ppc64' : PPC_OP_REG,
+    'sysz' : SYSZ_OP_REG,
+    'intel16' : X86_OP_REG, 'intel32' : X86_OP_REG, 'intel64' : X86_OP_REG,
+    'sparc' : SPARC_OP_REG,
+    'arm32' : ARM_OP_REG, 'arm64' : ARM64_OP_REG,
+    'mips' : MIPS_OP_REG, 'mips64' : MIPS_OP_REG
+}
+
+imm_mapping = {
+    'ppc' :  PPC_OP_IMM, 'ppc32' : PPC_OP_IMM, 'ppc64' : PPC_OP_IMM,
+    'sysz' : SYSZ_OP_IMM,
+    'intel16' : X86_OP_IMM, 'intel32' : X86_OP_IMM, 'intel64' : X86_OP_IMM,
+    'sparc' : SPARC_OP_IMM,
+    'arm32' : ARM_OP_IMM, 'arm64' : ARM64_OP_IMM,
+    'mips' : MIPS_OP_IMM, 'mips64' : MIPS_OP_IMM
+}
+
+mem_mapping = {
+    'ppc' :  PPC_OP_MEM, 'ppc32' : PPC_OP_MEM, 'ppc64' : PPC_OP_MEM,
+    'sysz' : SYSZ_OP_MEM,
+    'intel16' : X86_OP_MEM, 'intel32' : X86_OP_MEM, 'intel64' : X86_OP_MEM,
+    'sparc' : SPARC_OP_MEM,
+    'arm32' : ARM_OP_MEM, 'arm64' : ARM64_OP_MEM,
+    'mips' : MIPS_OP_MEM, 'mips64' : MIPS_OP_MEM
+}
+
+invalid_mapping = {
+    'ppc' :  PPC_OP_INVALID, 'ppc32' : PPC_OP_INVALID, 'ppc64' : PPC_OP_INVALID,
+    'sysz' : SYSZ_OP_INVALID,
+    'intel16' : X86_OP_INVALID, 'intel32' : X86_OP_INVALID, 'intel64' : X86_OP_INVALID,
+    'sparc' : SPARC_OP_INVALID,
+    'arm32' : ARM_OP_INVALID, 'arm64' : ARM64_OP_INVALID,
+    'mips' : MIPS_OP_INVALID, 'mips64' : MIPS_OP_INVALID
+}
+
+_call_mapping = {
+    'ppc' : [],
+    'sysz' : [],
+    'x86' : [X86_INS_CALL],
+    'sysz' : [],
+    'sparc' : [],
+    'arm' : [],
+    'arm64' : [],
+    'mips' : []
+}
+call_mapping = {
+    'ppc' :  _call_mapping['ppc'],
+    'ppc32' : _call_mapping['ppc'],
+    'ppc64' : _call_mapping['ppc'],
+    'sysz' : _call_mapping['sysz'],
+    'intel16' : _call_mapping['x86'],
+    'intel32' : _call_mapping['x86'],
+    'intel64' : _call_mapping['x86'],
+    'sparc' : _call_mapping['sparc'],
+    'arm32' : _call_mapping['arm'], 'arm64' : _call_mapping['arm64'],
+    'mips' : _call_mapping['mips'], 'mips64' : _call_mapping['mips']
+}
+
+_jump_mapping = {
+    'x86' : [   X86_INS_JA, X86_INS_JAE, X86_INS_JB, X86_INS_JBE, X86_INS_JCXZ,
+                X86_INS_JE, X86_INS_JECXZ, X86_INS_JG, X86_INS_JGE, X86_INS_JL,
+                X86_INS_JLE, X86_INS_JMP, X86_INS_JNE, X86_INS_JNO, X86_INS_JNP,
+                X86_INS_JNS, X86_INS_JO, X86_INS_JP, X86_INS_JRCXZ, X86_INS_JS,
+                X86_INS_LJMP]
+}
+jump_mapping = {
+    'intel16' : _jump_mapping['x86'],
+    'intel32' : _jump_mapping['x86'],
+    'intel64' : _jump_mapping['x86']
+}
+
+stack_offsets = {
+    'intel16' : [X86_REG_SP],
+    'intel32' : [X86_REG_EBP, X86_REG_ESP],
+    'intel64' : [X86_REG_RSP]
+}
+
+
+class Disassembly(object):
+    def __init__(self, architecture, code):
+        self.md = None
+        self.data = []
+        self.code = code
+        self.iterator = None
+        self.architecture = architecture
+
+        self.valid = False
+
+        if architecture in arch_mapping:
+            arch, mode = arch_mapping[architecture]
+            self.md = Cs(arch, mode)
+            self.md.detail = True
+            self.iterator = self.md.disasm(self.code, 0)
+            self.valid = True
+
+
+
+    def instructions(self):
+        #   When first called function will return cached instructions
+        for i in xrange(len(self.data)):
+            yield self.data[i]
+
+        #   Then iterate through non-cached instructions
+        if self.iterator:
+            for i in self.iterator:
+                self.data.append(i)
+                yield i
+
+            self.iterator = None
+
+
+    def _check_mapping(self, mapping, operand, attr='type', equal=True):
+        if ((not hasattr(operand, attr))
+        or (self.architecture not in mapping)):
+            False
+
+        if equal:
+            return getattr(operand, attr) == mapping[self.architecture]
+
+        return getattr(operand, attr) in mapping[self.architecture]
+
+    #   Operand Related Functionality
+    def is_op_reg(self, operand):
+        return self._check_mapping(reg_mapping, operand)
+
+    def is_op_mem(self, operand):
+        return self._check_mapping(mem_mapping, operand)
+
+    def is_op_imm(self, operand):
+        return self._check_mapping(imm_mapping, operand)
+
+    def is_op_invalid(self, operand):
+        return self._check_mapping(invalid_mapping, operand)
+
+    def is_stack_offset(self, operand):
+        if not hasattr(operand, 'mem'):
+            return False
+        return self._check_mapping(stack_offsets, operand.mem, 'base', False)
+
+
+    #   Instruction Related functionality
+    def is_call(self, instr):
+        return self._check_mapping(call_mapping, instr, 'id', False)
+
+    def is_jump(self, instr):
+        return self._check_mapping(jump_mapping, instr, 'id', False)

server/first_core/engines/__init__.py

@@ -18,9 +18,9 @@
 from first_core.error import FIRSTError
 from first_core.dbs import FIRSTDBManager
 from first_core.engines.results import Result
+from first_core.disassembly import Disassembly
 
 #   Third Party Modules
-from bson.objectid import ObjectId
 
 
 #   Class for FirstEngine related exceptions
@@ -96,9 +96,9 @@ def add(self, function):
 
         self._add(function)
 
-    def scan(self, opcodes, architecture, apis):
+    def scan(self, opcodes, architecture, apis, **kwargs):
         '''Returns a list of Result objects'''
-        results = self._scan(opcodes, architecture, apis)
+        results = self._scan(opcodes, architecture, apis, **kwargs)
 
         if isinstance(results, Result):
             return [results]
@@ -131,7 +131,7 @@ def _add(self, function):
         '''Returns nothing'''
         raise FIRSTEngineError('Not Implemented')
 
-    def _scan(self, opcodes, architecture, apis):
+    def _scan(self, opcodes, architecture, apis, **kwargs):
         '''Returns List of function IDs'''
         raise FIRSTEngineError('Not Implemented')
 
@@ -170,9 +170,7 @@ def _engines(self):
         #   Dynamically (re)load engines
         engines = []
         for e in active_engines:
-            if e.path in sys.modules:
-                reload(sys.modules[e.path])
-            else:
+            if e.path not in sys.modules:
                 __import__(e.path)
 
             module = sys.modules[e.path]
@@ -220,9 +218,13 @@ def add(self, function):
         '''
         required_keys = {'id', 'apis', 'opcodes', 'architecture', 'sha256'}
         if (dict != type(function)) or not required_keys.issubset(function.keys()):
-            print 'Data provided is not the correct type or required keys not provided'
+            print '[1stEM] Data provided is not the correct type or required keys not provided'
             return None
 
+        dis = Disassembly(function['architecture'], function['opcodes'])
+        if dis:
+            function['disassembly'] = dis
+
         #   Send function details to each registered engine
         errors = {}
         for engine in self._engines:
@@ -267,10 +269,12 @@ def scan(self, user, opcodes, architecture, apis):
         engine_results = {}
         engines = self._engines
 
+        dis = Disassembly(architecture, opcodes)
         for i in xrange(len(engines)):
             engine = engines[i]
             try:
-                results = engine.scan(opcodes, architecture, apis)
+                results = engine.scan(opcodes, architecture, apis,
+                                        disassembly=dis)
                 if results:
                     engine_results[i] = results