CSS placeholders; Redis port + offset support

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## [0.4.1] - 2019-02-04
+### Added
+- Placeholder CSS classes for customization.
+- Support for alternative Redis ports and database offsets.
+
 ## [0.4.0] - 2019-02-04
 ### Added
 - The `DAPYTHONVERSION` Docker environment variable.

config.yml

@@ -0,0 +1,13 @@
+version: 2
+jobs:
+  build:
+    working_directory: /app
+    docker:
+      - image: docker:18.09
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build application Docker image
+          command: |
+            docker build --cache-from=app -t app .

docassemble_base/docassemble/base/config.py

@@ -145,12 +145,6 @@ def load(**kwargs):
         AZURE_ENABLED = True
     if 'db' not in daconfig:
         daconfig['db'] = dict(name="docassemble", user="docassemble", password="abc123")
-    # for key in [['DBPREFIX', 'prefix'], ['DBNAME', 'name'], ['DBUSER', 'user'], ['DBPASSWORD', 'password'], ['DBHOST', 'host'], ['DBPORT', 'port'], ['DBTABLEPREFIX', 'table prefix']]:
-    #     if key[0] in os.environ:
-    #         val = trenv(os.environ[key[0]])
-    #         if key[1] not in daconfig['db'] or daconfig['db'][key[1]] != val:
-    #             daconfig['db'][key[1]] = val
-    #             changed = True
     dbtableprefix = daconfig['db'].get('table prefix', None)
     if not dbtableprefix:
         dbtableprefix = ''

docassemble_base/docassemble/base/util.py

@@ -60,20 +60,13 @@
 # def TheSimpleTextMachineLearner(*pargs, **kwargs):
 #     return knn_machine_learner(*pargs, **kwargs)
 
-#redis_server = None
-
-#def redis_server():
-#    return re.sub(r'^redis://', r'', server.redis_host)
-
 class DARedis(DAObject):
     """A class used to interact with the redis server."""
     def key(self, keyname):
         """Returns a key that combines the interview name with the keyname."""
         return this_thread.current_info.get('yaml_filename', '') + ':' + str(keyname)
     def get_data(self, key):
         """Returns data from Redis and unpickles it."""
-        # if this_thread.redis is None:
-        #     this_thread.redis = redis.StrictRedis(host=redis_server, db=2)
         result = server.server_redis_user.get(key)
         if result is None:
             return None
@@ -85,8 +78,6 @@ def get_data(self, key):
         return result
     def set_data(self, key, data, expire=None):
         """Saves data in Redis after pickling it."""
-        # if this_thread.redis is None:
-        #     this_thread.redis = redis.StrictRedis(host=redis_server, db=2)
         pickled_data = pickle.dumps(data)
         if expire is not None:
             if not isinstance(expire, int):
@@ -98,14 +89,8 @@ def set_data(self, key, data, expire=None):
         else:
             server.server_redis_user.set(key, pickled_data)
     def __getattr__(self, funcname):
-        # if this_thread.redis is None:
-        #     this_thread.redis = redis.StrictRedis(host=redis_server, db=2)
         return getattr(server.server_redis_user, funcname)
 
-#def set_redis_server(redis_host):
-#    global redis_server
-#    redis_server = re.sub(r'^redis://', r'', redis_host)
-
 class DACloudStorage(DAObject):
     """Returns an object that can be used to interface with S3 or Azure."""
     def init(self, *pargs, **kwargs):

docassemble_webapp/docassemble/webapp/cleanup_sessions.py

@@ -7,11 +7,7 @@
 docassemble.base.config.load(arguments=sys.argv)
 import docassemble.base.util
 import docassemble.webapp.daredis
-#import redis
-#redis_host = daconfig.get('redis', None)
-#if redis_host is None:
-#    redis_host = 'redis://localhost'
-#docassemble.base.util.set_redis_server(redis_host)
+
 store = RedisStore(docassemble.webapp.daredis.r_store)
 kv_session = KVSessionExtension(store, app)
 with app.app_context():

docassemble_webapp/docassemble/webapp/daredis.py

@@ -5,11 +5,20 @@
 redis_host = daconfig.get('redis', None)
 if redis_host is None:
     redis_host = 'redis://localhost'
+redis_host = redis_host.strip()
 redis_host = re.sub(r'^redis://', r'', redis_host)
-
-r = redis.StrictRedis(host=redis_host, db=0)
-r_store = redis.StrictRedis(host=redis_host, db=1)
-r_user = redis.StrictRedis(host=redis_host, db=2)
+m = re.search(r':([0-9]+)$', redis_host)
+if m:
+    redis_port = m.group(1)
+    redis_host = re.sub(r':([0-9]+)$', '', redis_host)
+else:
+    redis_port = '6379'
+
+redis_offset = daconfig.get('redis database offset', 0)
+
+r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
+r_store = redis.StrictRedis(host=redis_host, port=redis_port, db=1 + redis_offset)
+r_user = redis.StrictRedis(host=redis_host, port=redis_port, db=2 + redis_offset)
 
 # def clear_user_cache(user_id=None):
 #     if user_id is None:

docassemble_webapp/docassemble/webapp/server.py

@@ -732,7 +732,6 @@ def password_validator(form, field):
 import json
 import base64
 import requests
-#import redis
 import yaml
 import inspect
 import pyotp
@@ -855,10 +854,9 @@ def wrapped_function(*args, **kwargs):
         return update_wrapper(wrapped_function, f)
     return decorator
 
-from docassemble.webapp.daredis import r_store#, clear_user_cache
+from docassemble.webapp.daredis import r_store
 
 store = RedisStore(r_store)
-#store = RedisStore(redis.StrictRedis(host=docassemble.base.util.redis_server, db=1))
 
 kv_session = KVSessionExtension(store, app)
 
@@ -8834,6 +8832,8 @@ def index():
         bodyclass="dasignature"
     else:
         bodyclass="dabody pad-for-navbar"
+    if hasattr(interview_status.question, 'id'):
+        bodyclass += ' question-' + re.sub(r'[^A-Za-z0-9]+', '-', interview_status.question.id.lower())
         # if not is_ajax:
         #     start_output = standard_header_start + '\n    <title>' + browser_title + '</title>\n  </head>\n  <body class="dasignature">\n'
         # output = make_navbar(interview_status, default_title, default_short_title, (steps - user_dict['_internal']['steps_offset']), SHOW_LOGIN, user_dict['_internal']['livehelp'], debug_mode)
@@ -19894,10 +19894,7 @@ def error_notification(err, message=None, history=None, trace=None, referer=None
     sys.exit("Unable to modify the timestamp of the WSGI file: " + WEBAPP_PATH)
 
 from docassemble.webapp.daredis import r, r_user
-#r = redis.StrictRedis(host=docassemble.base.util.redis_server, db=0)
-#docassemble.base.functions.set_server_redis(r)
 
-#docassemble.base.util.set_twilio_config(twilio_config)
 docassemble.base.functions.update_server(url_finder=get_url_from_file_reference,
                                          navigation_bar=navigation_bar,
                                          chat_partners_available=chat_partners_available,

docassemble_webapp/docassemble/webapp/socketserver.py

@@ -32,21 +32,14 @@
 from flask import session, request
 from flask_kvsession import KVSessionExtension
 
-#redis_host = daconfig.get('redis', None)
-#if redis_host is None:
-#    redis_host = 'redis://localhost'
-#docassemble.base.util.set_redis_server(redis_host)
 import docassemble.webapp.daredis
-
-from docassemble.webapp.daredis import redis_host
+from docassemble.webapp.daredis import redis_host, redis_port, redis_offset
 
 store = RedisStore(docassemble.webapp.daredis.r_store)
 kv_session = KVSessionExtension(store, app)
 
 from docassemble.webapp.daredis import r as rr
 
-#rr = redis.StrictRedis(host=docassemble.base.util.redis_server, db=0)
-
 threads = dict()
 secrets = dict()
 
@@ -93,7 +86,7 @@ def background_thread(sid=None, user_id=None, temp_user_id=None):
             the_timezone = pytz.timezone(person.timezone)
         else:
             the_timezone = pytz.timezone(get_default_timezone())
-        r = redis.StrictRedis(host=redis_host, db=0)
+        r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
 
         partners = set()
         pubsub = r.pubsub()
@@ -427,7 +420,7 @@ def monitor_thread(sid=None, user_id=None):
             the_timezone = pytz.timezone(person.timezone)
         else:
             the_timezone = pytz.timezone(get_default_timezone())
-        r = redis.StrictRedis(host=redis_host, db=0)
+        r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
         listening_sids = set()
         pubsub = r.pubsub()
         pubsub.subscribe(['da:monitor', sid])
@@ -869,7 +862,7 @@ def monitor_chat_log(data):
 def observer_thread(sid=None, key=None):
     with app.app_context():
         sys.stderr.write("Started observer thread for " + str(sid) + "\n")
-        r = redis.StrictRedis(host=redis_host, db=0)
+        r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
         pubsub = r.pubsub()
         pubsub.subscribe([key, sid])
         for item in pubsub.listen():

docassemble_webapp/docassemble/webapp/users/forms.py

@@ -22,11 +22,8 @@ def fix_nickname(form, field):
 
 class MySignInForm(LoginForm):
     def validate(self):
-        #import redis
         from docassemble.webapp.daredis import r
-        #import docassemble.base.util
         from flask import request, abort
-        #r = redis.StrictRedis(host=docassemble.base.util.redis_server, db=0)
         key = 'da:failedlogin:ip:' + str(request.remote_addr)
         failed_attempts = r.get(key)
         if failed_attempts is not None and int(failed_attempts) > daconfig['attempt limit']:
@@ -201,13 +198,10 @@ class PhoneLoginVerifyForm(FlaskForm):
     verification_code = StringField(word('Verification code'), [validators.Length(min=daconfig['verification code digits'], max=daconfig['verification code digits'])])
     submit = SubmitField(word('Verify'))
     def validate(self):
-        #import redis
-        #import docassemble.base.util
         from docassemble.webapp.daredis import r
         from docassemble.base.logger import logmessage
         from flask import request, abort
         result = True
-        #r = redis.StrictRedis(host=docassemble.base.util.redis_server, db=0)
         key = 'da:failedlogin:ip:' + str(request.remote_addr)
         failed_attempts = r.get(key)
         if failed_attempts is not None and int(failed_attempts) > daconfig['attempt limit']:

docassemble_webapp/docassemble/webapp/watchdog.py

@@ -4,31 +4,35 @@
 import sys
 
 busy_pids = set()
+index = 0
 
 while True:
     busy_now = set()
     no_longer_busy = set()
     for pid in psutil.pids():
         try:
             p = psutil.Process(pid)
-            if p.name() == 'apache2' and p.cpu_times().user > 90.0 and p.cpu_percent(interval=1.0) > 19.0:
+            if p.name() == 'apache2' and p.cpu_percent(interval=30.0) > 90.0:
                 busy_now.add(pid)
         except:
             continue
-    for pid in busy_pids:
-        if not pid in busy_now:
-            no_longer_busy.add(pid)
-    for pid in no_longer_busy:
-        busy_pids.discard(pid)
-    for pid in busy_now:
-        if pid in busy_pids:
-            try:
-                p = psutil.Process(pid)
-                p.kill()
-            except:
-                pass
-            sys.stderr.write("Killed " + text_type(pid))
+    index += 1
+    index = index % 6
+    if index == 5:
+        for pid in busy_pids:
+            if not pid in busy_now:
+                no_longer_busy.add(pid)
+        for pid in no_longer_busy:
             busy_pids.discard(pid)
-        else:
-            busy_pids.add(pid)
-    time.sleep(30)
+        for pid in busy_now:
+            if pid in busy_pids:
+                try:
+                    p = psutil.Process(pid)
+                    p.kill()
+                except:
+                    pass
+                sys.stderr.write("Killed " + text_type(pid) + "\n")
+                busy_pids.discard(pid)
+            else:
+                busy_pids.add(pid)
+    time.sleep(5)