custom severity for sorting, mkdir_p dirs, better default handling in…

… config file (0.2.0)

README.md

@@ -52,25 +52,28 @@ user: admin
 password: password!
 ```
 
-Connection Section `[connection]` :
+Connection Detail Section `[connection]` :
 
 ```shell
 $(robust)> cat ~/.robust
 [connection]
-host: atd.localhost.localdomain
+ip: atd.localhost.localdomain
 skipssl: true
 maxthreads: 15
 ```
 
-Convict Section `[convict]`:
+Data Storage Section `[storage]`:
+
+Note: Datastorage locations will be created if they do not exist.
 
 ```shell
 $(robust)> cat ~/.robust
-[convict]
-cleandir: ./clean
-dirtydir: ./dirty
-reportdir: ./reports
-errordir: ./errors
+[storage]
+severity: 3
+cleandir: ~/robust/clean
+dirtydir: ~/robust/dirty
+reportdir: ~/robust/reports
+errordir: ~/robust/errors
 ```
 
 This file is expanded via the `os` module and maps to windows too.
@@ -326,10 +329,11 @@ Options
 
 ```
 usage: robust-convict.py [-h] [-u USER] [-p PASSWORD] [-i ATD IP] [-n] -a
-                         ANALYZER_PROFILE -d DIRECTORY [-e] -c CLEANDIR -x
-                         DIRTYDIR -r REPORTDIR -z ERRORDIR [-j MAXTHREADS]
+                         ANALYZER_PROFILE -d DIRECTORY [-e] [-y SEVERITY]
+                         [-c CLEANDIR] [-x DIRTYDIR] [-r REPORTDIR]
+                         [-z ERRORDIR]
                          [-t {html,txt,xml,zip,json,ioc,stix,pdf,sample}]
-                         [--version] [-v | -q]
+                         [-j MAXTHREADS] [--version] [-v | -q]
 
 Robust Intel Security ATD Python CLI tool
 
@@ -343,13 +347,13 @@ optional arguments:
 
 Authentication parameters:
   -u USER               (u)sername for the API of the ATD
-                        		(default: admin)
+                        		(default: robust)
   -p PASSWORD           (p)assword for username
-                        		(default: password!)
+                        		(default: ****<.robust>*****)
   -i ATD IP             (i)p or hostname address of ATD
                         		(default: atd.localhost.localdomain)
   -n                    do (n)ot verify the SSL certificate for the communications
-                        		(default: False)
+                        		(default: True)
 
 Watch parameters:
   -a ANALYZER_PROFILE   (a)nalyzer profile id to be used during analysis
@@ -358,18 +362,20 @@ Watch parameters:
                         		(default: None)
   -e                    (e)xisting files in directory will be submitted
                         		(default: False)
+  -j MAXTHREADS         (j) max number of threads
+                        		(default: 1)
 
 Convict parameters:
+  -y SEVERITY           (y) treat sample as dirty with this severity [0-5] or higher
+                        		(default: 3)
   -c CLEANDIR           (c) move clean files to this directory
-                        		(default: None)
+                        		(default: ~/robust/clean/)
   -x DIRTYDIR           (x) move processed dirty files to this directory
-                        		(default: None)
+                        		(default: ~/robust/malware/)
   -r REPORTDIR          (r) save reports to this directory
-                        		(default: None)
+                        		(default: ~/robust/reports/)
   -z ERRORDIR           (z) move error or skip files to this directory
-                        		(default: None)
-  -j MAXTHREADS         (j) max number of threads
-                        		(default: None)
+                        		(default: ~/robust/errors/)
   -t {html,txt,xml,zip,json,ioc,stix,pdf,sample}
                         (t)ype of report requested
                         		(default: None)
@@ -443,13 +449,13 @@ A modified Fork of `atdcli.py` (Carlos Munoz - 2014) is also included.
 
 ## VX Workshop Appliance Option
 
-There is a fully operational Xubuntu 14.04 liveCD that includes: 
+There is a fully operational Xubuntu 14.04 liveCD that includes:
 
 * robust - https://github.com/shadowbq/robust-atd
 * maltrieve - https://github.com/shadowbq/maltrieve
 * vxcage - https://github.com/shadowbq/vxcage
 
-It also includes 
+It also includes
 
 * hexeditors
 * static analysis tools

ratd/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.2.0"

ratd/api/__init__.py

@@ -65,8 +65,12 @@ def connect(self, user, password):
                 error_info = 'Connection unsucessful'
                 return (0, error_info)
         else:
-            error_info = 'Error conecting to ATD, Status Code: %d' % r.status_code
-            return(0, error_info)
+            if r.status_code == 401:
+                error_info = 'Error conecting to ATD, Username / Password combination not accepted.'
+                return(0, error_info)
+            else:
+                error_info = 'Error conecting to ATD, Status Code: %d' % r.status_code
+                return(0, error_info)
 
         return(1, 'Connection sucessful')
 

ratd/cliargs/__init__.py

@@ -14,6 +14,7 @@ def check_md5(value):
 def slash_dir(value):
     if value[len(value)-1] != "/":
         raise argparse.ArgumentTypeError("%s should end in a slash" % value)
+    value = os.path.expanduser(value)
     return value
 
 
@@ -32,6 +33,7 @@ def __init__(self, tool, explicit=None):
         self.arg_dict = {
             'user': '(u)sername for the API of the ATD\n\t\t(default: %(default)s)',
             'password': '(p)assword for username\n\t\t(default: %(default)s) ',
+            'password_secured': '(p)assword for username\n\t\t(default: ****<.robust>*****) ',
             'ip': '(i)p or hostname address of ATD\n\t\t(default: %(default)s) ',
             'sample': '(s)ample or file to be analyzed\n\t\t(default: %(default)s)',
             'skipssl': 'do (n)ot verify the SSL certificate for the communications\n\t\t(default: %(default)s)',
@@ -46,6 +48,7 @@ def __init__(self, tool, explicit=None):
             'dirtydir': '(x) move processed dirty files to this directory\n\t\t(default: %(default)s)',
             'reportdir': '(r) save reports to this directory\n\t\t(default: %(default)s)',
             'errordir': '(z) move error or skip files to this directory \n\t\t(default: %(default)s)',
+            'severity': '(y) treat sample as dirty with this severity [0-5] or higher\n\t\t(default: %(default)s)',
             'maxthreads': '(j) max number of threads\n\t\t(default: %(default)s)',
             'quiet': '(q)uiet all output\n\t\t(default: %(default)s)',
             'verbosity': 'increase output (v)erbosity\n\t\t(default: %(default)s)'
@@ -82,6 +85,11 @@ def __init__(self, tool, explicit=None):
 
             if tool == 'convict':
                 convict_group = self.parser.add_argument_group('Convict parameters')
+                if 'severity' in self.dot_robust:
+                    convict_group.add_argument('-y', required=False, action='store', default=self.dot_robust['severity'], dest='severity', help=self.arg_dict['severity'])
+                else:
+                    convict_group.add_argument('-y', required=False, action='store', dest='severity', help=self.arg_dict['severity'])
+
                 if 'cleandir' in self.dot_robust:
                     convict_group.add_argument('-c', required=False, action='store', type=slash_dir, default=self.dot_robust['cleandir'], dest='cleandir', help=self.arg_dict['cleandir'])
                 else:
@@ -108,6 +116,7 @@ def __init__(self, tool, explicit=None):
                 watch_group.add_argument('-j', required=False, action='store', default=self.dot_robust['maxthreads'], dest='maxthreads', help=self.arg_dict['maxthreads'])
             else:
                 watch_group.add_argument('-j', required=False, action='store', dest='maxthreads', help=self.arg_dict['maxthreads'])
+
         else:
             raise CliArgError(tool)
 
@@ -117,41 +126,72 @@ def __init__(self, tool, explicit=None):
         else:
             self.parser.parse_args(args=explicit, namespace=self)
 
+    def config_section_map(self, config, section, defaults):
+        dict1 = {}
+        options = config.options(section)
+
+        for option in options:
+            try:
+                dict1[option] = config.get(section, option)
+                if dict1[option] == -1:
+                    DebugPrint("skip: %s" % option)
+            except:
+                try:
+                    dict1[option] = defaults[option]
+                except:
+                    print("exception on %s!" % option)
+                    dict1[option] = None
+
+        for k,v in defaults.iteritems():
+            if not k in dict1:
+                dict1[k] = v
+
+        return dict1
+
     def dot_robust_helper(self):
-        config = ConfigParser.ConfigParser({'user': False, 'password': False, 'ip': False, 'skipssl': False, 'maxthreads': 1})
+        config = ConfigParser.ConfigParser({})
         fname = os.path.expanduser("~/.robust")
+
+        auth_defaults = {'user': False, 'password': False}
+        connection_defaults = {'ip': False, 'skipssl': False, 'maxthreads': 1}
+        storage_defaults = {'severity': 3, 'cleandir': '~/robust/clean/', 'dirtydir': '~/robust/malware/', 'reportdir': '~/robust/reports/', 'errordir': '~/robust/errors/'}
+
         if os.path.isfile(fname):
             config.read(fname)
             if config.has_section("auth"):
+                auth = self.config_section_map(config, "auth", auth_defaults)
                 dot_robust_auth = {
-                    'user': config.get("auth", "user"),
-                    'password': config.get("auth", "password")
+                    'user': auth["user"],
+                    'password':  auth["password"]
                 }
             else:
-                dot_robust_auth = {}
+                dot_robust_auth = auth_defaults
 
             if config.has_section("connection"):
+                connection = self.config_section_map(config, "connection", connection_defaults)
                 dot_robust_connection = {
-                    'ip': config.get("connection", "ip"),
-                    'skipssl': config.get("connection", "skipssl"),
-                    'maxthreads': config.get("connection", "maxthreads")
+                    'ip': connection["ip"],
+                    'skipssl': connection["skipssl"],
+                    'maxthreads': connection["maxthreads"]
                 }
             else:
-                dot_robust_connection = {}
-
-            if config.has_section("convict"):
-                dot_robust_convict = {
-                    'cleandir': config.get("convict", "cleandir"),
-                    'dirtydir': config.get("convict", "dirtydir"),
-                    'reportdir': config.get("convict", "reportdir"),
-                    'errordir': config.get("convict", "errordir")
+                dot_robust_connection = connection_defaults
+
+            if config.has_section("storage"):
+                storage = self.config_section_map(config, "storage", storage_defaults)
+                dot_robust_storage = {
+                    'severity': storage["severity"],
+                    'cleandir': storage["cleandir"],
+                    'dirtydir': storage["dirtydir"],
+                    'reportdir': storage["reportdir"],
+                    'errordir': storage["errordir"]
                 }
             else:
-                dot_robust_convict = {}
+                dot_robust_storage = storage_defaults
 
-            dot_robust_dict = utils.merge_dicts(dot_robust_auth, dot_robust_connection, dot_robust_convict)
+            dot_robust_dict = utils.merge_dicts(dot_robust_auth, dot_robust_connection, dot_robust_storage)
         else:
-            dot_robust_dict = {'user': False, 'password': False, 'ip': False, 'skipssl': False, 'maxthreads': 1}
+            dot_robust_dict = utils.merge_dicts(auth_defaults, connection_defaults, storage_defaults)
         return dot_robust_dict
 
     def common_args(self):
@@ -171,7 +211,7 @@ def auth_args(self):
             auth_group.add_argument('-u', required=True, action='store', dest='user', help=self.arg_dict['user'], metavar='USER')
 
         if self.dot_robust['password']:
-            auth_group.add_argument('-p', required=False, action='store', default=self.dot_robust['password'], dest='password', help=self.arg_dict['password'], metavar='PASSWORD')
+            auth_group.add_argument('-p', required=False, action='store', default=self.dot_robust['password'], dest='password', help=self.arg_dict['password_secured'], metavar='PASSWORD')
         else:
             auth_group.add_argument('-p', required=False, action='store', dest='password', help=self.arg_dict['password'], metavar='PASSWORD')
 
@@ -181,7 +221,7 @@ def auth_args(self):
             auth_group.add_argument('-i', required=True, action='store', dest='ip', help=self.arg_dict['ip'], metavar='ATD IP')
 
         if self.dot_robust['skipssl']:
-            auth_group.add_argument('-n', required=False, action='store_true', default=True, dest='skipssl', help=self.arg_dict['skipssl'])
+            auth_group.add_argument('-n', required=False, action='store_true', default=self.dot_robust['skipssl'], dest='skipssl', help=self.arg_dict['skipssl'])
         else:
             auth_group.add_argument('-n', required=False, action='store_true', dest='skipssl', help=self.arg_dict['skipssl'])
 

ratd/lib/__init__.py

@@ -6,7 +6,7 @@
 import tempfile
 import copy
 import threading
-
+import ratd.utils as utils
 
 from collections import namedtuple
 
@@ -271,7 +271,7 @@ def sort_file(self):
 
         try:
             if self.options.dirtydir:
-                if severity > 4:
+                if severity >= self.options.severity:
                     target = self.options.dirtydir+filename
                     if self.options.verbosity:
                         print('Move file {0}.. to dirty {1}'.format(filename, target))
@@ -295,7 +295,7 @@ def sort_file(self):
                 self.options.md5 = md5
                 # Report ouput filename
                 if 1 == 1:
-                    self.options.filename = self.options.reportdir + md5
+                    self.options.filename = self.options.reportdir + md5 + "." + self.options.rType
                 else:
                     self.options.filename = self.options.reportdir + filename
                 if self.options.verbosity:

ratd/scripts/robust-convict.py

@@ -18,6 +18,13 @@
 
     if options.password is None:
         options.password = getpass.getpass()
+    try:
+        print 'starting mkdirs'
+        mkdirs = utils.Mkdirs(options)
+        print 'running mkdirs'
+    except:
+        print 'Failed to create configured directories'
+        sys.exit(1)
 
     job = ratd.lib.ScanFolder(options)
 

ratd/scripts/robust-watchdog.py

@@ -22,6 +22,12 @@
     if options.verbosity:
         utils.copyleftnotice()
 
+    try:
+        mkdirs = utils.Mkdirs(options)
+    except:
+        print 'Failed to create configured directories'
+        sys.exit(1)    
+
     job = ratd.lib.ScanFolder(options)
 
     try:

ratd/utils/__init__.py

@@ -1,3 +1,7 @@
+import errno
+import os
+import sys
+
 def copyleftnotice():
     print '# Author - Shadowbq 2015 - www.github.com/shadowbq/robust-atd'
     print '# MIT LICENSE'
@@ -18,3 +22,51 @@ def merge_dicts(*dict_args):
     for dictionary in dict_args:
         result.update(dictionary)
     return result
+
+class Mkdirs:
+
+    '''Class defining the mkdir_p algorithm folder'''
+    def __init__(self, options):
+
+        print 'in mkdirs'
+        self.options = options
+        self.path = options.directory
+        print '1st mkdirs'
+        try:
+            print '1st mkdirs try'
+            if self.options.dirtydir:
+                print '1st mkdirs found'
+                self.mkdir_p(self.options.dirtydir)
+        except AttributeError:
+            pass
+
+        try:
+            if self.options.cleandir:
+                self.mkdir_p(self.options.cleandir)
+        except AttributeError:
+            pass
+
+        try:
+            if self.options.errordir:
+                self.mkdir_p(self.options.errordir)
+        except AttributeError:
+            pass
+
+        try:
+            if self.options.reportdir:
+                self.mkdir_p(self.options.reportdir)
+        except AttributeError:
+            pass
+
+    def mkdir_p(self, path):
+        try:
+            print 'mkdirs_p run'
+            os.makedirs(path)
+            if self.options.verbosity:
+                print ('mkdir_p %s' % path)
+                sys.stdout.flush()
+        except OSError as exc:  # Python >2.5
+            if exc.errno == errno.EEXIST and os.path.isdir(path):
+                pass
+            else:
+                raise