Blockchain challenges as chart.

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+## Supported Versions
+
+Machinaris security patches will be made for the [current release](https://github.com/guydavis/machinaris/wiki/Releases#release-streams), aka `:latest`.
+
+## Reporting a Vulnerability
+
+Please create a new [Discussion](https://github.com/guydavis/machinaris/discussions) with full details of the vulnerability.  I will provide a response asap.  Thanks in advance!

api/commands/log_parser.py

@@ -24,8 +24,8 @@
 CHIA_LOG = '/root/.chia/mainnet/log/debug.log'
 FLAX_LOG = '/root/.flax/mainnet/log/debug.log'
 
-# Roughly 1 minutes worth of challenges
-CHALLENGES_TO_LOAD = 8
+# Roughly 2 minutes worth of challenges, sent 90 seconds, for overlap
+CHALLENGES_TO_LOAD = 16
 
 # Most recent partial proofs, actually double as 2 log lines per partial
 PARTIALS_TO_LOAD = 50

api/gunicorn.conf.py

@@ -24,7 +24,7 @@ def on_starting(server):
     #scheduler.add_job(func=stats_disk.collect, trigger='interval', seconds=10) # Test immediately
 
     # Status gathering - reported via API
-    scheduler.add_job(func=status_challenges.update, name="challenges", trigger='interval', seconds=5)
+    scheduler.add_job(func=status_challenges.update, name="challenges", trigger='interval', seconds=90, jitter=45)
     scheduler.add_job(func=status_worker.update, name="workers", trigger='interval', seconds=120, jitter=60) 
     scheduler.add_job(func=status_controller.update, name="controller", trigger='interval', seconds=120, jitter=60) 
     scheduler.add_job(func=status_farm.update, name="farms", trigger='interval', seconds=120, jitter=60) 

api/schedules/stats_disk.py

@@ -20,7 +20,7 @@
 TABLES = ['stat_plots_total_used', 'stat_plots_disk_used', 'stat_plots_disk_free',
           'stat_plotting_total_used', 'stat_plotting_disk_used', 'stat_plotting_disk_free']
 
-DELETE_OLD_STATS_AFTER_DAYS = 2
+DELETE_OLD_STATS_AFTER_DAYS = 1
 
 def get_db():
     db = getattr(g, '_stats_database', None)

api/schedules/status_alerts.py

@@ -38,7 +38,7 @@ def update():
                 first_run = False
             else: # On subsequent schedules, load only last 5 minutes.
                 since = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S.000")
-            alerts = db.session.query(a.Alert).filter(a.Alert.created_at >= since).order_by(a.Alert.created_at.desc()).limit(20).all()
+            alerts = db.session.query(a.Alert).filter(a.Alert.created_at >= since, a.Alert.hostname == hostname).order_by(a.Alert.created_at.desc()).limit(20).all()
             payload = []
             for alert in alerts:
                 payload.append({

api/schedules/status_challenges.py

@@ -2,6 +2,7 @@
 # Performs a REST call to controller (possibly localhost) of latest blockchain challenges.
 #
 
+import datetime
 import os
 import traceback
 
@@ -13,11 +14,25 @@
 from api.commands import log_parser
 from api import utils
 
+def delete_old_challenges(db):
+    try:
+        cutoff = datetime.datetime.now() - datetime.timedelta(hours=1)
+        cur = db.cursor()
+        cur.execute("DELETE FROM challenges WHERE created_at < {1}".format(
+                table, cutoff.strftime("%Y%m%d%H%M")))
+        db.commit()
+    except:
+        app.logger.info("Failed to delete old challenges.")
+        app.logger.info(traceback.format_exc())
+
 def update():
     if not globals.farming_enabled() and not globals.harvesting_enabled():
         #app.logger.info("Skipping recent challenges collection on plotting-only instance.")
         return
     with app.app_context():
+        from api import db
+        if globals.load()['is_controller']:
+            delete_old_challenges(db)
         try:
             hostname = utils.get_displayname()
             blockchains = ['chia']

api/views/certificates/resources.py

@@ -49,6 +49,6 @@ def allow_download(self):
         worker_setup_marker = "/root/.chia/machinaris/tmp/worker_launch.tmp"
         if os.path.exists(worker_setup_marker):
             last_modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(worker_setup_marker))
-            fifteen_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=15)
+            fifteen_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=30)
             return last_modified_date >= fifteen_minutes_ago
         return False

api/views/challenges/resources.py

@@ -35,12 +35,13 @@ def get(self, args):
     def post(self, new_items):
         if len(new_items) == 0:
             return "No challenges provided.", 400
-        db.session.query(Challenge).filter(Challenge.hostname==new_items[0]['hostname']).delete()
         items = []
         for new_item in new_items:
-            item = Challenge(**new_item)
-            items.append(item)
-            db.session.add(item)
+            item = Challenge.query.get(new_item['unique_id'])
+            if not item:  # Request contains previously received challenges, only add new
+                item = Challenge(**new_item)
+                items.append(item)
+                db.session.add(item)
         db.session.commit()
         return items
 
@@ -57,12 +58,13 @@ def get(self, hostname, blockchain):
     @blp.arguments(BatchOfChallengeSchema)
     @blp.response(200, ChallengeSchema(many=True))
     def put(self, new_items, hostname, blockchain):
-        db.session.query(Challenge).filter(Challenge.hostname==hostname, Challenge.blockchain==blockchain).delete()
         items = []
         for new_item in new_items:
-            item = Challenge(**new_item)
-            items.append(item)
-            db.session.add(item)
+            item = Challenge.query.get(new_item['unique_id'])
+            if not item:  # Request contains previously received challenges, only add new
+                item = Challenge(**new_item)
+                items.append(item)
+                db.session.add(item)
         db.session.commit()
         return items
 

common/utils/converters.py

@@ -32,6 +32,12 @@ def str_to_gibs(str):
         print(traceback.format_exc())
         return None
 
+def convert_date_for_luxon(datestr):
+    year = datestr[:4]
+    month = datestr[4:6]
+    day = datestr[6:8]
+    time = datestr[8:]
+    return "{0}-{1}-{2}T{3}".format(year, month, day, time)
 
 
 

scripts/chia_launch.sh

@@ -51,6 +51,8 @@ elif [[ ${mode} =~ ^harvester.* ]]; then
       response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=chia --output /tmp/certs.zip)
       if [ $response == '200' ]; then
         unzip /tmp/certs.zip -d /root/.chia/farmer_ca
+      else
+        echo "Certificates response of ${response} from http://${controller_host}:8927/certificates/?type=chia.  Try clicking 'New Worker' button on 'Workers' page first."
       fi
       rm -f /tmp/certs.zip 
     fi

scripts/flax_launch.sh

@@ -61,6 +61,8 @@ elif [[ ${mode} =~ ^harvester.* ]]; then
       response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=flax --output /tmp/certs.zip)
       if [ $response == '200' ]; then
         unzip /tmp/certs.zip -d /root/.flax/farmer_ca
+      else
+        echo "Certificates response of ${response} from http://${controller_host}:8927/certificates/?type=flax.  Try clicking 'New Worker' button on 'Workers' page first."
       fi
       rm -f /tmp/certs.zip 
     fi

web/actions/chia.py

@@ -28,7 +28,7 @@
     partials as pr
 from common.config import globals
 from web.models.chia import FarmSummary, FarmPlots, BlockchainChallenges, Wallets, \
-    Blockchains, Connections, Keys, Plotnfts, Pools, Partials
+    Blockchains, Connections, Keys, Plotnfts, Pools, Partials, ChallengesChartData
 from . import worker as wk
 
 CHIA_BINARY = '/chia-blockchain/venv/bin/chia'
@@ -51,6 +51,10 @@ def recent_challenges():
     challenges = db.session.query(c.Challenge).filter(c.Challenge.created_at >= five_minutes_ago).order_by(c.Challenge.created_at.desc()).limit(20)
     return BlockchainChallenges(challenges)
 
+def challenges_chart_data():
+    challenges = db.session.query(c.Challenge).order_by(c.Challenge.created_at.desc(), c.Challenge.hostname, c.Challenge.blockchain).all()
+    return ChallengesChartData(challenges)
+
 def load_partials():
     partials = db.session.query(pr.Partial).order_by(pr.Partial.created_at.desc()).limit(10)
     return Partials(partials)

web/actions/stats.py

@@ -156,12 +156,13 @@ def load_recent_disk_usage(disk_type):
         sql = "select path, value{0}, created_at from stat_{1}_disk_used where hostname = ? order by created_at, path".format(value_factor, disk_type)
         used_result = cur.execute(sql, [ wk['hostname'], ]).fetchall()
         for used_row in used_result:
-            if not used_row[2] in dates:
-                dates.append(used_row[2])
+            converted_date = converters.convert_date_for_luxon(used_row[2])
+            if not converted_date in dates:
+                dates.append(converted_date)
             if not used_row[0] in paths:
                 paths[used_row[0]] = {}
             values = paths[used_row[0]]
-            values[used_row[2]] = used_row[1]
+            values[converted_date] = used_row[1]
         if len(dates) > 0:
             summary_by_worker[hostname] = { "dates": dates, "paths": paths.keys(),  }
             for path in paths.keys():

web/models/chia.py

@@ -109,6 +109,32 @@ def __init__(self, challenges):
                 'created_at': challenge.created_at,
             })
 
+
+class ChallengesChartData:
+
+    def __init__(self, challenges):
+        self.labels = []
+        datasets = {}
+        for challenge in challenges:
+            created_at = challenge.created_at.replace(' ', 'T')
+            if not created_at in self.labels:
+                self.labels.append(created_at)
+            host_chain = challenge.hostname + '_' + challenge.blockchain
+            if not host_chain in datasets:
+                datasets[host_chain] = {}
+            dataset = datasets[host_chain]
+            dataset[created_at] = float(challenge.time_taken.split()[0]) # Drop off the 'secs'
+        # Now build a sparse array with null otherwise
+        self.data = {}
+        for key in datasets.keys():
+            self.data[key] = [] 
+        for label in self.labels:
+            for key in datasets.keys():
+                if label in datasets[key]:
+                    self.data[key].append(datasets[key][label])
+                else:
+                    self.data[key].append('null') # Javascript null
+
 class Wallets:
 
     def __init__(self, wallets):

web/routes.py

@@ -30,16 +30,13 @@ def index():
     farming = chia.load_farm_summary()
     plotting = plotman.load_plotting_summary()
     challenges = chia.recent_challenges()
+    challenges_chart_data = chia.challenges_chart_data()
     partials = chia.load_partials()
     daily_diff = stats.load_daily_diff()
     return render_template('index.html', reload_seconds=120, farming=farming.__dict__, \
         plotting=plotting.__dict__, challenges=challenges, workers=workers, 
-        daily_diff=daily_diff, partials=partials, global_config=gc)
-
-@app.route('/views/challenges')
-def views_challenges():
-    challenges = chia.recent_challenges()
-    return render_template('views/challenges.html', challenges=challenges)
+        daily_diff=daily_diff, partials=partials, challenges_chart_data=challenges_chart_data, 
+        global_config=gc)
 
 @app.route('/setup', methods=['GET', 'POST'])
 def setup():

web/templates/alerts.html

@@ -91,7 +91,7 @@
                     <tr>
                         <td class="text-center"><input type="checkbox" name="unique_id"
                                 value="{{ notification.unique_id }}" /></td>
-                        <td>{{notification.hostname}}</td>
+                        <td>{{notification.worker}}</td>
                         <td>{{notification.blockchain}}</td>
                         <td>{{notification.service}}</td>
                         <td style="white-space: pre-wrap;">{{notification.message}}</td>

web/templates/base.html

@@ -177,6 +177,10 @@
         src="https://cdn.datatables.net/1.10.25/js/dataTables.bootstrap5.js"></script>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.min.js"
         integrity="sha256-yz7K02nILYEeRDwEfzu/1zI9SpBKod/nLYMTFh7vszs=" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/build/global/luxon.min.js"
+        integrity="sha256-CnZmNCHHUMy22/PJclCIISZ5Ib4MnUu+7ee5YNxtsZQ=" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chartjs-adapter-luxon.min.js"
+        integrity="sha256-tOhXNe/Ue+TjR33s/CryFYOGMwNfkTjTuvM4LEOAHzc=" crossorigin="anonymous"></script>
     {% block scripts %}{% endblock %}
 </body>
 

web/templates/index.html

@@ -120,7 +120,7 @@ <h6 class="display-6 text-center">Flax Netspace</h6>
   {% if global_config.farming_enabled %}
   <div class="col-md-12" style="margin-top:5px; margin-bottom:5px;">
     <div class="h-100 p-2 text-white bg-dark rounded-3" id="challenges-table-div">
-      {% include 'views/challenges.html' %}
+      <canvas id="challenges_chart"></canvas>
     </div>
   </div>
   {% endif %}
@@ -160,20 +160,70 @@ <h6 class="display-6 text-center">Flax Netspace</h6>
   {% block scripts %}
   {% if global_config.farming_enabled %}
   <script>
+    const COLORS = [
+        '#3aac59',
+        '#4dc9f6',
+        '#f67019',
+        '#f53794',
+        '#537bc4',
+        '#acc236',
+        '#166a8f',
+        '#00a950',
+        '#58595b',
+        '#8549ba'
+    ];
+
+    function color(index) {
+        return COLORS[index % COLORS.length];
+    }
+
     $(document).ready(function () {
-      setInterval(function () {
-        $.ajax({
-          type: "GET",
-          url: "{{ url_for('views_challenges') }}",
-        })
-          .done(function (data) {
-            //console.log(data);
-            $("#challenges-table-div").html(data)
-          }).fail(function (jqXHR, textStatus, errorThrown) {
-            console.log(jqXHR, textStatus, errorThrown);
-          });
-      }, 1000 * 5);
       $('[data-toggle="tooltip"]').tooltip();
+
+      var ctx = document.getElementById('challenges_chart');
+      var myChart = new Chart(ctx, {
+          type: 'line',
+          data: {
+              labels: {{ challenges_chart_data.labels | safe }},
+              datasets: [
+              {% for key in challenges_chart_data.data %}
+                  {
+                      label: "{{key}}",
+                      data: {{ challenges_chart_data.data[key] | safe }},
+                      backgroundColor: color({{ loop.index - 1 }}),
+                  },
+              {% endfor %}
+              ],
+          },
+          borderWidth: 1,
+          options: {
+              plugins: {
+                title: {
+                  display: true,
+                  text: 'Blockchain Challenges',
+                }
+              },
+              scales: {
+                  x: {
+                      type: 'time',
+                      time: {
+                          tooltipFormat: 'DD T'
+                      },
+                      title: {
+                          display: true,
+                          text: 'Time - Last Hour'
+                      }
+                  },
+                  y: {
+                      beginAtZero: true,
+                      title: {
+                          display: true,
+                          text: "Time Taken (seconds)",
+                      }
+                  }
+              }
+          }
+        });
     })
   </script>
   {% endif %}