Major updates to workshop content and automation

asdaraujo · Oct 27, 2021 · e8de299 · e8de299
1 parent f99d907
commit e8de299
Show file tree

Hide file tree

Showing 137 changed files with 818 additions and 1,519 deletions.
diff --git a/.build b/.build
@@ -1,2 +1,2 @@
-20211019201223
-Previous commit: 461eb0b Refactored test modules
+20211026133942
+Previous commit: 4abf0c0 Updated SSB lab to conform with CSA 1.4
diff --git a/README.adoc b/README.adoc
@@ -1,18 +1,20 @@
-= CDF Workshop
+= CDF Workshops
 
 == Introduction
 
 In this hands-on workshop, you will build a full OT to IT workflow for an **IoT Predictive Maintenance** use case. Below is the architecture diagram, showing all the components you will setup over the lab exercises. While the diagram divides the components according to their location (factory, regional or datacenter level) in this workshop all such components will reside in one single host.
 
 image::images/iot-architecture.png[width=800]
 
-=== Labs
+=== Workshops
 
 IMPORTANT: If this is your first time going through this content, please read the rest of this README introduction before jumping to the Labs.
 
 If you already familiar with the instructions in the README, time to start working and see some interesting stuff! Pick your lab and let's get started!
 
-* link:streaming.adoc[From Edge to Streams Processing]
+* link:workshop_edge.adoc[Ingesting data from the edge]
+* link:workshop_nifi.adoc[NiFi and Streams Processing]
+* link:workshop_dataviz.adoc[Creating Dashboards with Cloudera Data Viz]
 * link:streams_replication.adoc[Streams Replication]
 * link:sql_stream_builder.adoc[Querying streams with SQL]
 * link:spark_analytics.adoc[Spark and Fast Analytics with Kudu]

diff --git a/images/create_table.png b/images/create_table.png
diff --git a/images/dataviz/add-dataset.png b/images/dataviz/add-dataset.png
diff --git a/images/dataviz/add-expression.png b/images/dataviz/add-expression.png
diff --git a/images/dataviz/add-measures.png b/images/dataviz/add-measures.png
diff --git a/images/dataviz/add-visuals.png b/images/dataviz/add-visuals.png
diff --git a/images/dataviz/change-measure-aggregation-type.png b/images/dataviz/change-measure-aggregation-type.png
diff --git a/images/dataviz/clone-field.png b/images/dataviz/clone-field.png
diff --git a/images/dataviz/connection-explorer-table.png b/images/dataviz/connection-explorer-table.png
diff --git a/images/dataviz/create-dashboard.png b/images/dataviz/create-dashboard.png
diff --git a/images/dataviz/dashboard-add-dimensions.png b/images/dataviz/dashboard-add-dimensions.png
diff --git a/images/viz_dashboard_auto_refresh.png → images/dataviz/dashboard-auto-refresh.png b/images/viz_dashboard_auto_refresh.png → images/dataviz/dashboard-auto-refresh.png
diff --git a/images/viz_dashboard_view.png → images/dataviz/dashboard-view.png b/images/viz_dashboard_view.png → images/dataviz/dashboard-view.png
diff --git a/images/dataviz/dataset-fields.png b/images/dataviz/dataset-fields.png
diff --git a/images/dataviz/dataviz-home-page.png b/images/dataviz/dataviz-home-page.png
diff --git a/images/dataviz/dim-icon.png b/images/dataviz/dim-icon.png
diff --git a/images/dataviz/edit-dataset.png b/images/dataviz/edit-dataset.png
diff --git a/images/dataviz/edit-filter-expresion.png b/images/dataviz/edit-filter-expresion.png
diff --git a/images/dataviz/edit-measure.png b/images/dataviz/edit-measure.png
diff --git a/images/dataviz/explore-visuals-icon.png b/images/dataviz/explore-visuals-icon.png
diff --git a/images/dataviz/layout-button.png b/images/dataviz/layout-button.png
diff --git a/images/dataviz/layout.png b/images/dataviz/layout.png
diff --git a/images/dataviz/mes-icon.png b/images/dataviz/mes-icon.png
diff --git a/images/viz_new_connection_advanced.png → images/dataviz/new-connection-advanced.png b/images/viz_new_connection_advanced.png → images/dataviz/new-connection-advanced.png
diff --git a/images/dataviz/new-connection-basic.png b/images/dataviz/new-connection-basic.png
diff --git a/images/dataviz/new-dasboard.png b/images/dataviz/new-dasboard.png
diff --git a/images/dataviz/new-data-connection.png b/images/dataviz/new-data-connection.png
diff --git a/images/dataviz/new-dataset.png b/images/dataviz/new-dataset.png
diff --git a/images/dataviz/opening-dataviz.png b/images/dataviz/opening-dataviz.png
diff --git a/images/dataviz/real-time-dashboard.png b/images/dataviz/real-time-dashboard.png
diff --git a/images/dataviz/refreshed-fields.png b/images/dataviz/refreshed-fields.png
diff --git a/images/dataviz/updated-dataset.png b/images/dataviz/updated-dataset.png
diff --git a/images/dataviz/updated-field-category.png b/images/dataviz/updated-field-category.png
diff --git a/images/dataviz/visual-axes-settings.png b/images/dataviz/visual-axes-settings.png
diff --git a/images/dataviz/visual-legend-style.png b/images/dataviz/visual-legend-style.png
diff --git a/images/dataviz/visual-style.png b/images/dataviz/visual-style.png
diff --git a/images/NiFiDataProvenance.png → images/edge/NiFiDataProvenance.png b/images/NiFiDataProvenance.png → images/edge/NiFiDataProvenance.png
diff --git a/images/NiFiViewDataProvenance.png → images/edge/NiFiViewDataProvenance.png b/images/NiFiViewDataProvenance.png → images/edge/NiFiViewDataProvenance.png
diff --git a/images/add_consumer_mqtt.png → images/edge/add_consumer_mqtt.png b/images/add_consumer_mqtt.png → images/edge/add_consumer_mqtt.png
diff --git a/images/add_consumer_mqtt_2.png → images/edge/add_consumer_mqtt_2.png b/images/add_consumer_mqtt_2.png → images/edge/add_consumer_mqtt_2.png
diff --git a/images/add_funnel.png → images/edge/add_funnel.png b/images/add_funnel.png → images/edge/add_funnel.png
diff --git a/images/add_input_port.png → images/edge/add_input_port.png b/images/add_input_port.png → images/edge/add_input_port.png
diff --git a/images/add_rpg.png → images/edge/add_rpg.png b/images/add_rpg.png → images/edge/add_rpg.png
diff --git a/images/edge/bogus-readings.png b/images/edge/bogus-readings.png
diff --git a/images/cem_first_version.png → images/edge/cem_first_version.png b/images/cem_first_version.png → images/edge/cem_first_version.png
diff --git a/images/cem_heartbeats.png → images/edge/cem_heartbeats.png b/images/cem_heartbeats.png → images/edge/cem_heartbeats.png
diff --git a/images/connect_extract_to_route.png → images/edge/connect_extract_to_route.png b/images/connect_extract_to_route.png → images/edge/connect_extract_to_route.png
diff --git a/images/connect_filter_ro_rpg.png → images/edge/connect_filter_ro_rpg.png b/images/connect_filter_ro_rpg.png → images/edge/connect_filter_ro_rpg.png
diff --git a/images/connect_to_rpg.png → images/edge/connect_to_rpg.png b/images/connect_to_rpg.png → images/edge/connect_to_rpg.png
diff --git a/images/connecting.png → images/edge/connecting.png b/images/connecting.png → images/edge/connecting.png
diff --git a/images/create_bucket.png → images/edge/create_bucket.png b/images/create_bucket.png → images/edge/create_bucket.png
diff --git a/images/create_connection.png → images/edge/create_connection.png b/images/create_connection.png → images/edge/create_connection.png
diff --git a/images/create_last_connection.png → images/edge/create_last_connection.png b/images/create_last_connection.png → images/edge/create_last_connection.png
diff --git a/images/dashboard_icon.png → images/edge/dashboard_icon.png b/images/dashboard_icon.png → images/edge/dashboard_icon.png
diff --git a/images/efm_set_cloude_config.png → images/edge/efm_set_cloude_config.png b/images/efm_set_cloude_config.png → images/edge/efm_set_cloude_config.png
diff --git a/images/evaluate_json_path.png → images/edge/evaluate_json_path.png b/images/evaluate_json_path.png → images/edge/evaluate_json_path.png
diff --git a/images/extract_attributes.png → images/edge/extract_attributes.png b/images/extract_attributes.png → images/edge/extract_attributes.png
diff --git a/images/flow_designer_icon.png → images/edge/flow_designer_icon.png b/images/flow_designer_icon.png → images/edge/flow_designer_icon.png
diff --git a/images/flow_in_nifi_registry.png → images/edge/flow_in_nifi_registry.png b/images/flow_in_nifi_registry.png → images/edge/flow_in_nifi_registry.png
diff --git a/images/input_port_id.png → images/edge/input_port_id.png b/images/input_port_id.png → images/edge/input_port_id.png
diff --git a/images/edge/list-queue.png b/images/edge/list-queue.png
diff --git a/images/edge/message-queue-details.png b/images/edge/message-queue-details.png
diff --git a/images/nifiDemoStart.png → images/edge/nifiDemoStart.png b/images/nifiDemoStart.png → images/edge/nifiDemoStart.png
diff --git a/images/nifiTerminateRelationships.png → images/edge/nifiTerminateRelationships.png b/images/nifiTerminateRelationships.png → images/edge/nifiTerminateRelationships.png
diff --git a/images/operate_panel.png → images/edge/operate_panel.png b/images/operate_panel.png → images/edge/operate_panel.png
diff --git a/images/publish_flow.png → images/edge/publish_flow.png b/images/publish_flow.png → images/edge/publish_flow.png
diff --git a/images/queued_events.png → images/edge/queued_events.png b/images/queued_events.png → images/edge/queued_events.png
diff --git a/images/reconnect_consume_mqtt.png → images/edge/reconnect_consume_mqtt.png b/images/reconnect_consume_mqtt.png → images/edge/reconnect_consume_mqtt.png
diff --git a/images/route_on_attribute.png → images/edge/route_on_attribute.png b/images/route_on_attribute.png → images/edge/route_on_attribute.png
diff --git a/images/route_on_attribute_config.png → images/edge/route_on_attribute_config.png b/images/route_on_attribute_config.png → images/edge/route_on_attribute_config.png
diff --git a/images/runSimulator1or30.png → images/edge/runSimulator1or30.png b/images/runSimulator1or30.png → images/edge/runSimulator1or30.png
diff --git a/images/simulate1.png → images/edge/simulate1.png b/images/simulate1.png → images/edge/simulate1.png
diff --git a/images/simulate2.png → images/edge/simulate2.png b/images/simulate2.png → images/edge/simulate2.png
diff --git a/images/spanner_icon.png → images/edge/spanner_icon.png b/images/spanner_icon.png → images/edge/spanner_icon.png
diff --git a/images/terminate_errors.png → images/edge/terminate_errors.png b/images/terminate_errors.png → images/edge/terminate_errors.png
diff --git a/images/terminations.png → images/edge/terminations.png b/images/terminations.png → images/edge/terminations.png
diff --git a/images/access_key_variable.png → images/nifi/access_key_variable.png b/images/access_key_variable.png → images/nifi/access_key_variable.png
diff --git a/images/add_kafka_failure_connection.png → images/nifi/add_kafka_failure_connection.png b/images/add_kafka_failure_connection.png → images/nifi/add_kafka_failure_connection.png
diff --git a/images/add_registry_client.png → images/nifi/add_registry_client.png b/images/add_registry_client.png → images/nifi/add_registry_client.png
diff --git a/images/add_updateattribute.png → images/nifi/add_updateattribute.png b/images/add_updateattribute.png → images/nifi/add_updateattribute.png
diff --git a/images/added_hwx_sr_service.png → images/nifi/added_hwx_sr_service.png b/images/added_hwx_sr_service.png → images/nifi/added_hwx_sr_service.png
diff --git a/images/additional_controller_services.png → ...s/nifi/additional_controller_services.png b/images/additional_controller_services.png → ...s/nifi/additional_controller_services.png
diff --git a/images/breadcrumbs.png → images/nifi/breadcrumbs.png b/images/breadcrumbs.png → images/nifi/breadcrumbs.png
diff --git a/images/cog_icon.png → images/nifi/cog_icon.png b/images/cog_icon.png → images/nifi/cog_icon.png
diff --git a/images/connect_input.png → images/nifi/connect_input.png b/images/connect_input.png → images/nifi/connect_input.png
diff --git a/images/controller_services.png → images/nifi/controller_services.png b/images/controller_services.png → images/nifi/controller_services.png
diff --git a/images/create_pgroup.png → images/nifi/create_pgroup.png b/images/create_pgroup.png → images/nifi/create_pgroup.png
diff --git a/images/nifi/create_table.png b/images/nifi/create_table.png
diff --git a/images/enable_icon.png → images/nifi/enable_icon.png b/images/enable_icon.png → images/nifi/enable_icon.png
diff --git a/images/explore_partition.png → images/nifi/explore_partition.png b/images/explore_partition.png → images/nifi/explore_partition.png
diff --git a/images/from_kafka_to_kudu_flow.png → images/nifi/from_kafka_to_kudu_flow.png b/images/from_kafka_to_kudu_flow.png → images/nifi/from_kafka_to_kudu_flow.png
diff --git a/images/global_controller_settings.png → images/nifi/global_controller_settings.png b/images/global_controller_settings.png → images/nifi/global_controller_settings.png
diff --git a/images/impala_editor.png → images/nifi/impala_editor.png b/images/impala_editor.png → images/nifi/impala_editor.png
diff --git a/images/kafka_success.png → images/nifi/kafka_success.png b/images/kafka_success.png → images/nifi/kafka_success.png
diff --git a/images/kudu_success.png → images/nifi/kudu_success.png b/images/kudu_success.png → images/nifi/kudu_success.png
diff --git a/images/model_access_key.png → images/nifi/model_access_key.png b/images/model_access_key.png → images/nifi/model_access_key.png
diff --git a/images/operate_panel_cog.png → images/nifi/operate_panel_cog.png b/images/operate_panel_cog.png → images/nifi/operate_panel_cog.png
diff --git a/images/parse_failure_relationship.png → images/nifi/parse_failure_relationship.png b/images/parse_failure_relationship.png → images/nifi/parse_failure_relationship.png
diff --git a/images/plus_button.png → images/nifi/plus_button.png b/images/plus_button.png → images/nifi/plus_button.png
diff --git a/images/producers.png → images/nifi/producers.png b/images/producers.png → images/nifi/producers.png
diff --git a/images/publishKafka_flow.png → images/nifi/publishKafka_flow.png b/images/publishKafka_flow.png → images/nifi/publishKafka_flow.png
diff --git a/images/register_schema.png → images/nifi/register_schema.png b/images/register_schema.png → images/nifi/register_schema.png
diff --git a/images/sensor_flows_bucket.png → images/nifi/sensor_flows_bucket.png b/images/sensor_flows_bucket.png → images/nifi/sensor_flows_bucket.png
diff --git a/images/smm.png → images/nifi/smm.png b/images/smm.png → images/nifi/smm.png
diff --git a/images/nifi/table_select.png b/images/nifi/table_select.png
diff --git a/...s/terminate_publishkafka_relationship.png → ...i/terminate_publishkafka_relationship.png b/...s/terminate_publishkafka_relationship.png → ...i/terminate_publishkafka_relationship.png
diff --git a/images/to_input.png → images/nifi/to_input.png b/images/to_input.png → images/nifi/to_input.png
diff --git a/images/version_control_tick.png → images/nifi/version_control_tick.png b/images/version_control_tick.png → images/nifi/version_control_tick.png
diff --git a/images/table_select.png b/images/table_select.png
diff --git a/images/viz_add_expression.png b/images/viz_add_expression.png
diff --git a/images/viz_clone_field.png b/images/viz_clone_field.png
diff --git a/images/viz_connection_explorer.png b/images/viz_connection_explorer.png
diff --git a/images/viz_connection_explorer_table.png b/images/viz_connection_explorer_table.png
diff --git a/images/viz_create_dashboard.png b/images/viz_create_dashboard.png
diff --git a/images/viz_dashboard_add_dimensions.png b/images/viz_dashboard_add_dimensions.png
diff --git a/images/viz_dashboard_add_measures.png b/images/viz_dashboard_add_measures.png
diff --git a/images/viz_new_connection_basic.png b/images/viz_new_connection_basic.png
diff --git a/images/viz_new_dashboard.png b/images/viz_new_dashboard.png
diff --git a/images/viz_new_dataset.png b/images/viz_new_dataset.png
diff --git a/images/viz_save_dataset.png b/images/viz_save_dataset.png
diff --git a/sensor.avsc b/sensor.avsc
@@ -11,7 +11,7 @@
     },
     {
       "name": "sensor_ts",
-      "doc": "Timestamp of the collected readings.",
+      "doc": "Timestamp, in microseconds, of the collected readings.",
       "type": "long"
     },
     {

diff --git a/setup/README.adoc b/setup/README.adoc
@@ -185,7 +185,7 @@ git clone https://github.com/cloudera-labs/edge2ai-workshop.git
 +
 [source,shell]
 ----
-cp edge2ai-workshop/setup/scripts/stack.cdp716p.sh edge2ai-workshop/setup/scripts/stack.sh
+cp edge2ai-workshop/setup/terraform/resources/stack.cdp716p.sh edge2ai-workshop/setup/terraform/resources/stack.sh
 ----
 +
 The `stack.sh` file is the default stack definition file used by all the namespaces that lack _namespace-specific_ stacks. You can create namespace-specific stack by naming the file `stack.<namespace>.sh` instead.

diff --git a/setup/scripts b/setup/scripts
diff --git a/setup/terraform/check-services.sh b/setup/terraform/check-services.sh
@@ -15,9 +15,22 @@ function get_model_status() {
   CDSW_ALTUS_API="cdsw.$ip.nip.io/api/altus-ds-1"
   status=""
   for scheme in http https; do
-    token=$("${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" --data '{"_local":false,"login":"admin","password":"'"${THE_PWD}"'"}' "$scheme://$CDSW_API/authenticate" 2>/dev/null | jq -r '.auth_token' 2> /dev/null)
+    token=$("${CURL[@]}" -X POST --cookie-jar .curl.cj-model.$$ --cookie .curl.cj-model.$$ -H "Content-Type: application/json" --data '{"_local":false,"login":"admin","password":"'"${THE_PWD}"'"}' "$scheme://$CDSW_API/authenticate" 2>/dev/null | jq -r '.auth_token // empty' 2> /dev/null)
     [[ ! -n $token ]] && continue
-    status=$("${CURL[@]}" -X POST --cookie-jar .curl.cj.$$ --cookie .curl.cj.$$ -H "Content-Type: application/json" -H "Authorization: Bearer $token" --data '{"projectOwnerName":"admin","latestModelDeployment":true,"latestModelBuild":true}' "$scheme://$CDSW_ALTUS_API/models/list-models" 2>/dev/null | jq -r '.[].latestModelDeployment | select(.model.name == "IoT Prediction Model").status' 2>/dev/null)
+    status=$("${CURL[@]}" -X POST --cookie-jar .curl.cj-model.$$ --cookie .curl.cj-model.$$ -H "Content-Type: application/json" -H "Authorization: Bearer $token" --data '{"projectOwnerName":"admin","latestModelDeployment":true,"latestModelBuild":true}' "$scheme://$CDSW_ALTUS_API/models/list-models" 2>/dev/null | jq -r '.[].latestModelDeployment | select(.model.name == "IoT Prediction Model").status' 2>/dev/null)
+    [[ -n $status ]] && break
+  done
+  echo -n $status
+}
+
+function get_viz_status() {
+  local ip=$1
+  CDSW_API="cdsw.$ip.nip.io/api/v1"
+  status=""
+  for scheme in http https; do
+    token=$("${CURL[@]}" -X POST --cookie-jar .curl.cj-viz.$$ --cookie .curl.cj-viz.$$ -H "Content-Type: application/json" --data '{"_local":false,"login":"admin","password":"'"${THE_PWD}"'"}' "$scheme://$CDSW_API/authenticate" 2>/dev/null | jq -r '.auth_token // empty' 2> /dev/null)
+    [[ ! -n $token ]] && continue
+    status=$("${CURL[@]}" -X GET --cookie-jar .curl.cj-viz.$$ --cookie .curl.cj-viz.$$ -H "Content-Type: application/json" -H "Authorization: Bearer $token" "$scheme://$CDSW_API/projects/admin/vizapps-workshop/applications" 2>/dev/null | jq -r '.[0].status // empty' 2>/dev/null)
     [[ -n $status ]] && break
   done
   echo -n $status
@@ -50,8 +63,10 @@ if [ -s $TF_JSON_FILE ]; then
     (("${CURL[@]}" http://$host:8888/                ;  "${CURL[@]}" https://$host:8889/)                2>/dev/null | grep "<title>Hue" > /dev/null 2>&1 && echo Ok) > .curl.hue.$$ &
     (("${CURL[@]}" http://cdsw.$ip.nip.io/           || "${CURL[@]}" https://cdsw.$ip.nip.io/)           2>/dev/null | egrep "(Cloudera Machine Learning|Cloudera Data Science Workbench)" > /dev/null 2>&1 && echo Ok) > .curl.cml.$$ &
     (get_model_status $ip) > .curl.model.$$ &
-    ("${CURL[@]}" http://viz.cdsw.$ip.nip.io/arc/adminapi/users 2>/dev/null | grep 'user' > /dev/null 2>&1 && echo running) > .curl.viz.$$ &
+#    ("${CURL[@]}" http://viz.cdsw.$ip.nip.io/arc/adminapi/users 2>/dev/null | grep 'user' > /dev/null 2>&1 && echo running) > .curl.viz.$$ &
+    (get_viz_status $ip) > .curl.viz.$$ &
     wait
+#    printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)"
     printf "%-30s %-20s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-5s %-14s %s\n" "$instance" "$ip" "$(cat .curl.web.$$)" "$(cat .curl.cm.$$)" "$(cat .curl.cem.$$)" "$(cat .curl.nifi.$$)" "$(cat .curl.nifireg.$$)" "$(cat .curl.schreg.$$)" "$(cat .curl.smm.$$)" "$(cat .curl.hue.$$)" "$(cat .curl.cml.$$)" "$(cat .curl.model.$$)" "$(cat .curl.viz.$$)"
   done | sort -t\[ -k1,1r -k2,2n
 fi
diff --git a/setup/terraform/resources/cdsw_setup.py b/setup/terraform/resources/cdsw_setup.py
@@ -99,7 +99,6 @@ def add_vizapps_user(username, password, first_name, last_name):
     print('# Create user')
     while True:
         try:
-            print(CDSW_API)
             r = s.post(CDSW_API + '/users',
                     json={
                         'email': EMAIL,
@@ -134,7 +133,7 @@ def add_vizapps_user(username, password, first_name, last_name):
     models = [m for m in r.json() if m['name'] == model_name]
     if models and models[0]['latestModelDeployment']['status'] == 'deployed':
         model_id = models[0]['id']
-        project_id = get_project(name=PROJECT_NAME)['id']
+        project = get_project(name=PROJECT_NAME)
         print('Model is already deployed!! Skipping.')
     else:
         print('# Add engine')
@@ -153,8 +152,8 @@ def add_vizapps_user(username, password, first_name, last_name):
                                                                  'project_visibility': 'private',
                                                                  'name': PROJECT_NAME,
                                                                  'gitUrl': 'https://github.com/cloudera-labs/edge2ai-workshop'})
-            project_id = r.json()['id']
-        print('Project ID: %s'% (project_id,))
+            project = r.json()
+        print('Project ID: %s' % (project['id'],))
 
         print('# Upload setup script')
         setup_script = """!pip3 install --upgrade pip scikit-learn
@@ -166,7 +165,7 @@ def add_vizapps_user(username, password, first_name, last_name):
         r = s.put(CDSW_API + '/projects/admin/edge2ai-workshop/files/setup_workshop.py', files={'name': setup_script})
 
         print('# Upload model')
-        model_pkl = open(MODEL_PKL_FILE, 'r')
+        model_pkl = open(MODEL_PKL_FILE, 'rb')
         r = s.put(CDSW_API + '/projects/admin/edge2ai-workshop/files/iot_model.pkl', files={'name': model_pkl})
 
         print('# Create job to run the setup script')
@@ -216,7 +215,7 @@ def add_vizapps_user(username, password, first_name, last_name):
 
         print('# Deploy model')
         r = s.post(CDSW_ALTUS_API + '/models/create-model', json={
-                'projectId': project_id,
+                'projectId': project['id'],
                 'name': model_name,
                 'description': model_name,
                 'visibility': 'private',
@@ -272,7 +271,6 @@ def add_vizapps_user(username, password, first_name, last_name):
     print(viz_project_url + '/engine-images')
     r = s.patch(viz_project_url + '/engine-images',
                 json={'engineImageId': engine_image_id})
-    print(r)
     r = s.get(viz_project_url + '/engine-images')
     project_engine_image_id = r.json()['id']
     print('Project image default engine Image ID set to: %s'% (project_engine_image_id))  
@@ -302,7 +300,7 @@ def add_vizapps_user(username, password, first_name, last_name):
     print('# Wait for model to start')
     while True:
         r = s.post(CDSW_ALTUS_API + '/models/list-models', json={
-            'project': str(project_id),
+            'project': str(project['id']),
             'latestModelDeployment': True,
             'latestModelBuild': True,
         })

diff --git a/setup/terraform/resources/labs/__init__.py b/setup/terraform/resources/labs/__init__.py
@@ -112,17 +112,29 @@ def __init__(cls, name, bases, dct):
 
 
 class AbstractWorkshop(metaclass=AbstractWorkshopMeta):
-    def __init__(self, run_id=None):
+    def __init__(self, run_id=None, context=None):
         class _Context(object):
             pass
 
-        self.context = _Context()
+        self.context = context or _Context()
         self.run_id = run_id if run_id is not None else get_run_id()
 
     @classmethod
     @abstractmethod
     def workshop_id(cls):
-        """Return a short string to identify the CA type."""
+        """Return a short string to identify the workshop."""
+        pass
+
+    @classmethod
+    @abstractmethod
+    def prereqs(cls):
+        """
+        Return a list of prereqs for this workshop. The list can contain either:
+          - Strings identifying the name of other workshops that need to be setup before this one does. In
+            this case all the labs of the specified workshop will be setup.
+          - Tuples (String, Integer), where the String specifies the name of the workshop and Integer the number
+            of the last lab of that workshop to be executed/setup.
+        """
         pass
 
     def before_setup(self):
@@ -135,7 +147,20 @@ def after_setup(self):
     def teardown(self):
         pass
 
+    def _execute_prereqs(self):
+        global WORKSHOPS
+        for prereq in self.prereqs():
+            if isinstance(prereq, str):
+                workshop = prereq
+                lab = 99
+            else:
+                workshop, lab = prereq
+
+            LOG.info('Executing prereqs setup: Workshop {}, Lab < {}'.format(workshop, lab))
+            WORKSHOPS[workshop](self.run_id, self.context).setup(lab)
+
     def setup(self, target_lab=99):
+        self._execute_prereqs()
         self.before_setup()
         lab_setup_functions = [(n, f, _get_step_number(n)) for n, f in
                                getmembers(self.__class__) if _get_step_number(n) is not None]
@@ -147,7 +172,10 @@ def setup(self, target_lab=99):
             else:
                 LOG.debug("[{0}] is numbered higher than target [lab{1}], skipping".format(func_name, target_lab))
         self.after_setup()
+        return self.context
 
+    def get_artifacts_dir(self):
+        return os.path.join(os.path.dirname(__file__), 'artifacts', self.workshop_id())
 
 def _load_workshops():
     base_dir = get_base_dir()
@@ -158,22 +186,24 @@ def _load_workshops():
 
 
 def global_setup(target_workshop='base', target_lab=99, run_id=None):
-    LOG.info('Executing global setup for Lab {} in Workshop {}'.format(target_workshop, target_lab))
     _load_workshops()
     if target_workshop in WORKSHOPS:
+        LOG.info('Executing setup for Lab {} in Workshop {}'.format(target_workshop, target_lab))
         WORKSHOPS[target_workshop](run_id).setup(target_lab)
     else:
         raise RuntimeError("Workshop [{}] not found. Known workshops are: {}".format(target_workshop, WORKSHOPS))
     LOG.info('Global setup completed successfully!')
 
 
-def global_teardown(target_workshop='base', run_id=None):
-    LOG.info('Executing global teardown for Workshop {}'.format(target_workshop))
+def global_teardown(target_workshop=None, run_id=None):
     _load_workshops()
-    if target_workshop in WORKSHOPS:
+    if target_workshop is not None:
+        LOG.info('Executing teardown for Workshop {}'.format(target_workshop))
         WORKSHOPS[target_workshop](run_id).teardown()
     else:
-        raise RuntimeError("Workshop [{}] not found. Known workshops are: {}".format(target_workshop, WORKSHOPS))
+        for target_workshop in WORKSHOPS:
+            LOG.info('Executing teardown for Workshop {}'.format(target_workshop))
+            WORKSHOPS[target_workshop](run_id).teardown()
     LOG.info('Global teardown completed successfully!')
 
 

diff --git a/setup/terraform/resources/labs/artifacts/dataviz/visuals_dataset.json b/setup/terraform/resources/labs/artifacts/dataviz/visuals_dataset.json
@@ -0,0 +1,39 @@
+{
+  "reportimage": [],
+  "dashboards": [],
+  "segments": [],
+  "appgroups": [],
+  "visuals": [],
+  "appgroupmembership": [],
+  "events": [],
+  "colorpalette": [],
+  "dateranges": [],
+  "datasets": [
+    {
+      "model": "datasets.dataset",
+      "pk": 26,
+      "fields": {
+        "dataconnection": 22,
+        "dataset_name": "sensor data1",
+        "dataset_type": "singletable",
+        "dataset_detail": "default.sensors",
+        "dataset_description": "",
+        "dataset_info": "[{\"tablename\": \"default.sensors\", \"columns\": [{\"name\": \"sensor_id\", \"type\": \"INT\", \"isdim\": true, \"alias\": \"sensor_id\"}, {\"name\": \"\", \"type\": \"TIMESTAMP\", \"isdim\": true, \"alias\": \"sensor_timestamp\", \"basecol\": \"sensor_ts\", \"expr\": \"microseconds_add(to_timestamp(cast([sensor_ts]/1000000 as bigint)), [sensor_ts] % 1000000)\"}, {\"name\": \"sensor_ts\", \"type\": \"BIGINT\", \"isdim\": false, \"alias\": \"sensor_ts\"}, {\"name\": \"sensor_0\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_0\"}, {\"name\": \"sensor_1\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_1\"}, {\"name\": \"sensor_2\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_2\"}, {\"name\": \"sensor_3\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_3\"}, {\"name\": \"sensor_4\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_4\"}, {\"name\": \"sensor_5\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_5\"}, {\"name\": \"sensor_6\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_6\"}, {\"name\": \"sensor_7\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_7\"}, {\"name\": \"sensor_8\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_8\"}, {\"name\": \"sensor_9\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_9\"}, {\"name\": \"sensor_10\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_10\"}, {\"name\": \"sensor_11\", \"type\": \"DOUBLE\", \"isdim\": false, \"alias\": \"sensor_11\"}, {\"name\": \"is_healthy\", \"type\": \"INT\", \"isdim\": false, \"alias\": \"is_healthy\"}]}]",
+        "uuid": "73d9de65-3fdf-4ff3-a1a2-cc08c2a7701b",
+        "imported_uuid": null,
+        "cache_sequence": 0,
+        "dataset_settings": "{}",
+        "search_enabled": false
+      }
+    }
+  ],
+  "customcss": [],
+  "staticasset": [],
+  "version": {
+    "Arcviz Version": "6.2.3",
+    "Raw Timestamp of last commit": "1611943183",
+    "Timestamp of last commit": "Fri Jan 29 18:59:43 2021 +0100",
+    "SHA of last commit": "4e609f54bd9deb0e1e61658008174cafa75f7e3c",
+    "Description": "6.2.3-b18"
+  }
+}