set the database dir as a config var

README.md

@@ -33,7 +33,12 @@ The source code is stored in thie repo, and the auxiary data (which compises the
 The data directory can be anywhere in the host file system and must have the following structure: <br>
 ```
 {data-dir}
-       data/   <---- each course has its own dir, named as the course ID 
+       data/   <---- each course has its own sub dirirectory, named as the course ID 
+           94219/
+              ex1/
+                data files
+              ex2/
+                data files
        matchers/
        runners/
        hw_config.json
@@ -43,12 +48,12 @@ The path to the data directory is passed to the server in environment variable *
 <br>
 <br>
 
- ## storing the database
- The database (job submissions) is stored in sqlite3 file in a directory that must be writable.
+## storing the database
+ The database (job submissions) is stored in sqlite3 file in a directory that must be writable in the host machine.
 ## storing the log output
 The logger output is written to a directory in the host file system that must be writable.
 
-All the above directories are mounted in the run_docker.sh script.
+All the above directories are mounted in the **run_docker.sh** script.
 
 # Instructions for the Tutor
 As the tutor, you have to prepare:
@@ -87,6 +92,7 @@ The config file json will look like:
      "id": 2,
      "matcher" : "tester_ex2.py",
      "runner" :"check_py.sh",
+     "data_dir": "relative/path/to/DATA_DIR", <<< optional
      "timeout" : 20
     }
  ]

TODO

@@ -0,0 +1,17 @@
+BUGS
+
+
+
+
+
+TESTS
+multi thread access to shared resources such as the database file (pickle)
+
+
+
+FEATURES
+- add support for LDAP
+
+- refactor the docker images so we don't have gcc/cmake/qemu if not needed (why?)
+
+- refactor the Docker etc. so no need to rebuild the image while debugging (e.g. runners are in hosts' file system)

scripts/run_docker.sh

@@ -15,6 +15,7 @@ set -e
  	       --mount type=bind,source=$CHECKER_DATA_DIR/logs,target=/logs\
 	       --mount type=bind,source=$CHECKER_DATA_DIR/db,target=/db\
 	       --env CHECKER_DATA_DIR=/data \
+	       --env CHECKER_LOG_DIR=/logs \
              -p80:8000 \
 	     --user nobody \
 	     --restart unless-stopped \

serverpkg/job_status.py

@@ -165,12 +165,28 @@ class JobStatusDB():
     # keep the db file in a place that will persist!
     # in Docker, every container restart, the local filesystem is cleaned
     # so '/db' is mounted on the host's file system
-    db_file_name = '/db/jobs.db'
+    # @param: db_dir_name: full path to the database directory. Can be non existent
+    def __init__(self, db_dir_name : str):
+        import os
+        self.db_dir_name = db_dir_name
+        self.db_file_name = os.path.join(db_dir_name, "jobs.db")
 
     def _create_tables(self):
         """Create the needed SQL table if they are not already created.
         :Note: must be multiprocess safe since this code can be called concurrently
         from (e.g) 3 processes"""
+        #db_creation_sync_lock.lock() #TODO: finish coding the multi process safe creation or make it redundant
+
+        # make sure the db dir is created
+        import os
+        logger.info("opening file %s" % self.db_file_name)
+        try:
+            os.mkdir(self.db_dir_name)
+        except FileExistsError:
+            pass
+        except PermissionError:
+            logger.error("Permission denied when trying to create the DB directory")
+            raise
         try:
             with sqlite3.connect(self.db_file_name) as conn:
                 conn.execute("CREATE TABLE IF NOT EXISTS jobs ( {} ); ".format(Job.JobSqlSchema()))

serverpkg/server.py

@@ -25,10 +25,6 @@
 logger = init_logger('server')
 
 # The following import is needed to prepare the admin endpoints
-
-_job_status_db = job_status.JobStatusDB()
-_job_status_db._create_tables()
-
 # regretably, the admin module uses _job_status_db so it can be imported only here
 from . import admin
 
@@ -46,6 +42,7 @@ def _configure_app():
     app.config['matcher_dir'] = data_path + '/matchers'
     app.config['runner_dir'] = data_path + '/runners'
     app.config['assignment_config_file'] = data_path + '/hw_settings.json'
+    app.config['database_dir'] = os.environ['CHECKER_LOG_DIR']
 
     app.config['LDAP_HOST'] = 'ldap://ccldap.technion.ac.il'
     app.config['LDAP_BASE_DN'] = 'ou=tx,dc=technion,dc=ac,dc=il'
@@ -76,6 +73,8 @@ def _running_on_dev_machine():
 # ---------------------
 
 _configure_app()
+_job_status_db = job_status.JobStatusDB(app.config['database_dir'])
+_job_status_db._create_tables()
 
 
 
@@ -160,6 +159,9 @@ def handle_submission(course,ex_type, number):
     except SanityError as ex:
         return "<H1>Message to Tutor</H1>There is something wrong in the config file for this exercise.<br>"\
                "Please fix and submit again. <br><strong>Error: " + str(ex) + '</strong>'
+    except FileNotFoundError:
+        return "<H1>Message to Tutor</H1>The config file is not found. Deleted or acces problems?<br>" , HTTPStatus.NOT_FOUND
+
     # check if the post request has the file part
     if 'file' not in request.files:
         flash('No file part')
@@ -383,7 +385,7 @@ def handle_file_async(package_under_test, course_number, ex_type, ex_number, ref
     new_job.set_handlers(os.path.join(app.config['matcher_dir'], config['matcher']),
                          os.path.join(app.config['runner_dir'], config['runner']))
 
-    data_path = os.path.join(app.config['data_dir'], config.get('data_path'))
+    data_path = os.path.join(app.config['data_dir'], config.get('data_path',""))
     async_task = AsyncChecker(_job_status_db, new_job, package_under_test,
                         reference_input, reference_output, completionCb, full_data_path=data_path, timeout_sec=config['timeout'])
     async_task.start()
@@ -430,6 +432,9 @@ def _purge_db_stale_jobs():
     _job_status_db.delete_jobs(Job.Status.pending)
 
 
+try:
+    course_id = _get_configured_course_ids()
+except FileNotFoundError:
+    logger.fatal("Configuration file not found. Check permissions and name")
 
-course_id = _get_configured_course_ids()
 _purge_db_stale_jobs()