Added Viz lab and automation

Added automated Viz setup - alpha
Added images for Lab 6 instructions
Added images for streaming lite labs
Change Lab 6 from Hue to CDP Data Visualization
Enabled unauthenticated app access. Added viz app status / health check.
Fixed a typo ("yum" -> "git")
Fixing Kudu timestamp column issue
Merging addional images for streaming lite workshop from zBrainiac/edge2ai-workshop
Merging streaming_flink and streaming_lite content from zBrainiac/edge2ai-workshop
Modified source topic name from iot_enriched to iot (for lite labs)
Resolving conflicts with upstream
Typo fix
Update README.adoc
Update streaming_lite.adoc
Updated Lab 6 instructions for CDV
Updated Lab 7 instructions for lite workshop
Updated images for Lab 7 instructions
Updated web forms to improve registration process, fixed small issue in cdsw setup script.

"This commit does not contain secrets"

.gitignore

@@ -1,3 +1,5 @@
 .DS_Store
 .idea/
 __pycache__/
+.vscode/
+edge2ai.code-workspace

sensor_enhanced.avsc

@@ -0,0 +1,115 @@
+{
+  "type": "record",
+  "name": "SensorReading",
+  "namespace": "com.cloudera.example",
+  "doc": "This is a sample sensor reading",
+  "fields": [
+    {
+      "name": "sensor_id",
+      "doc": "Sensor identification number.",
+      "type": "int"
+    },
+    {
+      "name": "sensor_ts",
+      "doc": "Timestamp of the collected readings.",
+      "type": "long"
+    },
+    {
+      "name": "is_healthy",
+      "doc": "Flag indicating health (healthy == 1)",
+      "type": "int"
+    },
+    {
+      "name": "response",
+      "doc": "response record",
+      "type": {
+        "type": "record",
+        "name": "CDSWResponse",
+        "namespace": "com.cloudera.example",
+        "doc": "This is a CDSW Model response",
+        "fields": [
+          {
+            "name": "result",
+            "doc": "Result",
+            "type": "int"
+          }
+        ]
+      }
+    },
+    {
+    "name": "sensor_0",
+    "doc": "Reading #0.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_1",
+    "doc": "Reading #1.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_2",
+    "doc": "Reading #2.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_3",
+    "doc": "Reading #3.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_4",
+    "doc": "Reading #4.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_5",
+    "doc": "Reading #5.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_6",
+    "doc": "Reading #6.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_7",
+    "doc": "Reading #7.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_8",
+    "doc": "Reading #8.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_9",
+    "doc": "Reading #9.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_10",
+    "doc": "Reading #10.",
+    "type": "int"
+    },
+    {
+    "name": "sensor_11",
+    "doc": "Reading #11.",
+    "type": "int"
+    },
+    {
+    "name": "city",
+    "doc": "closest city",
+    "type": "string"
+    },
+    {
+    "name": "lat",
+    "doc": "geo location",
+    "type": "double"
+    },
+    {
+    "name": "lon",
+    "doc": "geo location",
+    "type": "double"
+    }
+  ]
+}

setup/terraform/check-services.sh

@@ -31,7 +31,7 @@ fi
 NAMESPACE=$1
 load_env $NAMESPACE
 
-printf "%-30s %-30s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %s\n" "instance" "ip address" "WEB" "CM" "CEM" "NIFI" "NREG" "SREG" "SMM" "HUE" "CDSW" "Model Status"
+printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-14s %s\n" "instance" "ip address" "WEB" "CM" "CEM" "NIFI" "NREG" "SREG" "SMM" "HUE" "CDSW" "Model Status" "Viz Status" 
 ensure_tf_json_file
 if [ -s $TF_JSON_FILE ]; then
   cat $TF_JSON_FILE | \
@@ -50,7 +50,8 @@ if [ -s $TF_JSON_FILE ]; then
     (("${CURL[@]}" http://$host:8888/                ;  "${CURL[@]}" https://$host:8889/)                2>/dev/null | grep "<title>Hue" > /dev/null 2>&1 && echo Ok) > .curl.hue.$$ &
     (("${CURL[@]}" http://cdsw.$ip.nip.io/           || "${CURL[@]}" https://cdsw.$ip.nip.io/)           2>/dev/null | egrep "(Cloudera Machine Learning|Cloudera Data Science Workbench)" > /dev/null 2>&1 && echo Ok) > .curl.cml.$$ &
     (get_model_status $ip) > .curl.model.$$ &
+    ("${CURL[@]}" http://viz.cdsw.$ip.nip.io/arc/adminapi/users 2>/dev/null | grep 'user' > /dev/null 2>&1 && echo running) > .curl.viz.$$ &
     wait
-    printf "%-30s %-30s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)"
+    printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-14s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)" "$(cat .curl.viz.$$)"
   done | sort -t\[ -k1,1r -k2,2n
 fi

setup/terraform/resources/cdsw_setup.py

@@ -60,7 +60,7 @@
         else:
             print('Waiting for CDSW to be ready... (connection timed out)')
         time.sleep(10)
-
+    
     print('# Authenticate')
     r = s.post(CDSW_API + '/authenticate', json={'login': USERNAME, 'password': PASSWORD})
     s.headers.update({'Authorization': 'Bearer ' + r.json()['auth_token']})
@@ -71,8 +71,8 @@
     models = [m for m in r.json() if m['name'] == model_name]
     if models and models[0]['latestModelDeployment']['status'] == 'deployed':
         print('Model is already deployed!! Skipping.')
-        exit(0)
-
+        #exit(0)
+    
     print('# Add engine')
     r = s.post(CDSW_API + '/site/engine-profiles', json={'cpu': 1, 'memory': 4})
     engine_id = r.json()['id']
@@ -99,11 +99,11 @@
 
     print('# Upload setup script')
     setup_script = """!pip3 install --upgrade pip scikit-learn
-!HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /user/$HADOOP_USER_NAME
-!HADOOP_USER_NAME=hdfs hdfs dfs -chown $HADOOP_USER_NAME:$HADOOP_USER_NAME /user/$HADOOP_USER_NAME
-!hdfs dfs -put data/historical_iot.txt /user/$HADOOP_USER_NAME
-!hdfs dfs -ls -R /user/$HADOOP_USER_NAME
-"""
+    !HADOOP_USER_NAME=hdfs hdfs dfs -mkdir /user/$HADOOP_USER_NAME
+    !HADOOP_USER_NAME=hdfs hdfs dfs -chown $HADOOP_USER_NAME:$HADOOP_USER_NAME /user/$HADOOP_USER_NAME
+    !hdfs dfs -put data/historical_iot.txt /user/$HADOOP_USER_NAME
+    !hdfs dfs -ls -R /user/$HADOOP_USER_NAME
+    """
     r = s.put(CDSW_API + '/projects/admin/edge2ai-workshop/files/setup_workshop.py', files={'name': setup_script})
 
     print('# Upload model')
@@ -174,6 +174,71 @@
     model_id = r.json()['id']
     print('Model ID: %s' % (model_id,))
 
+    # ================================================================================
+
+    # See https://docs.cloudera.com/cdsw/latest/analytical-apps/topics/cdsw-application-limitations.html
+
+    print('# Allow applications to be configured with unauthenticated access')
+    r = s.patch(CDSW_API + '/site/config', json={"allow_unauthenticated_access_to_app": True})
+    print('Set unauthenticated access flag to: %s'% (r.json()["allow_unauthenticated_access_to_app"],))
+
+    print('# Add project for Data Visualization server')
+    project_name = 'viz'
+    viz_project_id = None
+    r = s.get(CDSW_API + '/users/admin/projects')
+    for project in r.json():
+        if project['name'] == project_name:
+            viz_project_id = project['id']
+            break
+    if not viz_project_id:
+        r = s.post(CDSW_API + '/users/admin/projects', json={'template': 'blank',
+                                                             'project_visibility': 'private',
+                                                             'name': project_name})
+        viz_project_id = r.json()['id']
+    print('Viz project ID: %s'% (viz_project_id,))
+
+    print('# Add custom engine for Data Visualization server')
+    params = { 
+      "engineImage": {
+        "description": "dataviz-623",
+        "repository": "docker.repository.cloudera.com/cloudera/cdv/cdswdataviz",
+        "tag": "6.2.3-b18"}
+    }
+    r = s.post(CDSW_API + '/engine-images', json=params)
+    engine_image_id = r.json()['id']
+    print('Engine Image ID: %s'% (engine_image_id,))
+
+    print('# Set new engine image as default for the viz project')
+    r = s.patch(CDSW_API + '/projects/admin/viz/engine-images',
+                json={'engineImageId': engine_image_id})
+    r = s.get(CDSW_API + '/projects/admin/viz/engine-images')
+    project_engine_image_id = r.json()['id']
+    print('Project image default engine Image ID set to: %s'% (project_engine_image_id))  
+
+    print('# Create application with Data Visualization server')
+    application_name = 'viz'
+
+    params = { 
+        'bypass_authentication': True,
+        'cpu': 1,
+        'environment': {},
+        'description': 'viz server app',
+        'kernel': 'python3',
+        'memory': 2,
+        'name': 'viz-server',
+        'nvidia_gpu': 0,
+        'script': '/opt/vizapps/tools/arcviz/startup_app.py',
+        'subdomain': 'viz',
+        'type': 'manual'
+    }
+
+    r = s.post(CDSW_API + '/projects/admin/viz/applications', json=params)
+    r = s.get(CDSW_API + '/projects/admin/viz/applications')
+    r.json()
+
+
+    # ================================================================================
+
     while True:
         r = s.post(CDSW_ALTUS_API + '/models/list-models', json={
             'project': str(project_id),
@@ -198,6 +263,23 @@
             elif build_status == 'failed' or deployment_status == 'failed':
                 raise RuntimeError('Model deployment failed')
         time.sleep(10)
+
+    while True:
+        r = s.get(CDSW_API + '/projects/admin/viz/applications')
+        app_status = r.json()[0]['status']
+        print('Data Visualization app status: %s' % (app_status))
+        if app_status == 'running':
+          print('# Viz server app is running. CDSW setup complete!')
+          break
+        elif build_status == 'stopped':
+            # Additional error handling - if the app exists and is stopped, start it?
+            break
+        elif build_status == 'failed':
+            raise RuntimeError('Application deployment failed')
+        time.sleep(10)
+
+    # Connect to Viz App API, validate status
+
 except Exception as e:
     if r:
         print(r.text)

setup/terraform/resources/common.sh

@@ -940,6 +940,7 @@ function get_service_urls() {
       fi
       if [[ ${HAS_CDSW:-0} == 1 ]]; then
         echo "CDSW=${protocol}://cdsw.{ip_address}.nip.io/"
+        echo "CDP Data Visualization=${protocol}://viz.cdsw.{ip_address}.nip.io/"
       fi
     ) | sort
   ) | tr "\n" "," | sed 's/,$//'

setup/terraform/web/app/forms.py

@@ -10,8 +10,8 @@ class LoginForm(FlaskForm):
     """Login form
     """
     email = StringField('Email', validators=[DataRequired(), Email()])
-    password = PasswordField('Password', validators=[DataRequired()])
-    login_submit = SubmitField('Sign In')
+    password = PasswordField('Registration code / Password', validators=[DataRequired()])
+    login_submit = SubmitField('Submit')
 
 class RegistrationForm(FlaskForm):
     """Registration form for new users

setup/terraform/web/app/templates/login.html

@@ -2,10 +2,11 @@
 {% import 'bootstrap/wtf.html' as wtf %}
 
 {% block app_content %}
-<h1>Sign In</h1>
+<h1>Register new user / Log in</h1>
 <div class="row">
     <div class="col-md-4">
         {{ wtf.quick_form(form, button_map={'login_submit': 'primary'}) }}
     </div>
 </div>
+
 {% endblock %}

setup/terraform/web/app/templates/register.html

@@ -2,7 +2,7 @@
 {% import 'bootstrap/wtf.html' as wtf %}
 
 {% block app_content %}
-    <h1>Register</h1>
+    <h1>Register new user</h1>
     <div class="row">
         <div class="col-md-4">
             {{ wtf.quick_form(form, button_map={'register_submit': 'primary'}) }}