Overload Protection (#112)

* Highlight the suspend button in red when tracing is suspended.

* Configure the max queue length that the tracer process can accrue before suspending, with the maxTraceQueueLength property in configuration.

* Send the trace logs that overloaded the process, upto the configured maximum.

* Fixed a memory leak where the list of traces is duplicated, so when traces are deleted they are only deleted from the duplicate.

* Load shed trace logs if there are over a configurable amount.

src/main/java/erlyberly/DbgController.java

@@ -39,17 +39,44 @@
 
 public class DbgController implements Initializable {
 
-    public final ObservableList<TraceLog> traceLogs = FXCollections.observableArrayList();
+    /**
+     * The maximum trace logs that erlyberly can show in the table before it must
+     * load shed them e.g. start throwing them away so that the UI does not attempt
+     * to take an unlimited amount of memory, and fail!
+     *
+     * This is only checked at start up, so if it is changed it requires a restart.
+     */
+    private static final int MAX_TRACE_LOGS;
+
+    static {
+        Number maxTraceLogs = (Number) PrefBind.getOrDefault("maxTraceLogs", 1000);
+        MAX_TRACE_LOGS = maxTraceLogs.intValue();
+    }
+
+    private final ObservableList<TraceLog> traceLogs = FXCollections.observableArrayList();
 
     private final ObservableList<ModFunc> traces = FXCollections.observableArrayList();
 
     private final ObservableList<SeqTraceLog> seqTraces = FXCollections.observableArrayList();
 
     private volatile boolean collectingSeqTraces;
 
+    private TraceLog loadSheddingLog;
+
     @Override
     public void initialize(URL url, ResourceBundle r) {
         ErlyBerly.nodeAPI().setTraceLogCallback((traceLog) -> {
+            if(traceLogs.size() > MAX_TRACE_LOGS) {
+                if(loadSheddingLog == null) {
+                    loadSheddingLog = TraceLog.newLoadShedding();
+                    traceLogs.add(loadSheddingLog);
+                }
+                return;
+            }
+            if(loadSheddingLog != null) {
+                traceLogs.remove(loadSheddingLog);
+                loadSheddingLog = null;
+            }
             traceLogs.add(traceLog);
         });
         ErlyBerly.nodeAPI().suspendedProperty().addListener((o, oldv, suspended) -> {
@@ -58,6 +85,12 @@ public void initialize(URL url, ResourceBundle r) {
                 reapplyTraces();
             }
         });
+        ErlyBerly.nodeAPI().connectedProperty().addListener(
+            (o, oldV, newV) -> {
+                if(oldV && !newV) {
+                    traceLogs.add(TraceLog.newNodeDown());
+                }
+            });
         new SeqTraceCollectorThread((seqs) -> { seqTraces.addAll(seqs); }).start();
     }
 
@@ -82,7 +115,7 @@ public void toggleTraceModFunc(ModFunc function) {
 
     private void traceModFunc(ModFunc function) {
         try {
-            ErlyBerly.nodeAPI().startTrace(function);
+            ErlyBerly.nodeAPI().startTrace(function, getMaxTraceQueueLengthConfig());
 
             traces.add(function);
         }
@@ -91,6 +124,11 @@ private void traceModFunc(ModFunc function) {
         }
     }
 
+    private int getMaxTraceQueueLengthConfig() {
+        Number maxTraceQueueLength = (Number) PrefBind.getOrDefault("maxTraceQueueLength", 1000);
+        return maxTraceQueueLength.intValue();
+    }
+
     private void onRemoveTracer(ActionEvent e, ModFunc function) {
         try {
             ErlyBerly.nodeAPI().stopTrace(function);
@@ -107,7 +145,7 @@ public void reapplyTraces() {
 
         for (ModFunc function : tracesCopy) {
             try {
-                ErlyBerly.nodeAPI().startTrace(function);
+                ErlyBerly.nodeAPI().startTrace(function, getMaxTraceQueueLengthConfig());
             } catch (Exception e) {
                 e.printStackTrace();
 

src/main/java/erlyberly/DbgTraceView.java

@@ -24,9 +24,6 @@
 import javafx.application.Platform;
 import javafx.beans.value.ChangeListener;
 import javafx.beans.value.WeakChangeListener;
-import javafx.collections.FXCollections;
-import javafx.collections.ListChangeListener;
-import javafx.collections.ObservableList;
 import javafx.collections.transformation.FilteredList;
 import javafx.collections.transformation.SortedList;
 import javafx.css.PseudoClass;
@@ -53,27 +50,23 @@
 
 public class DbgTraceView extends VBox {
 
-    private final ObservableList<TraceLog> traceLogs = FXCollections.observableArrayList();
+    private final DbgController dbgController;
 
-    private final SortedList<TraceLog> sortedTtraces = new SortedList<TraceLog>(traceLogs);
+    private final SortedList<TraceLog> sortedTtraces;
 
-    private final FilteredList<TraceLog> filteredTraces = new FilteredList<TraceLog>(sortedTtraces);
+    private final FilteredList<TraceLog> filteredTraces;
 
     private final TableView<TraceLog> tracesBox;
 
-    /**
-     * Set insertTracesAtTop=true in the .erlyberly file in your home directory to
-     * make traces be inserted at the top of the list.
-     */
-    private boolean insertTracesAtTop;
+    public DbgTraceView(DbgController aDbgController) {
+        dbgController = aDbgController;
+        sortedTtraces = new SortedList<TraceLog>(dbgController.getTraceLogs());
+        filteredTraces = new FilteredList<TraceLog>(sortedTtraces);
 
-    public DbgTraceView(DbgController dbgController) {
         setSpacing(5d);
         setStyle("-fx-background-insets: 5;");
         setMaxHeight(Double.MAX_VALUE);
 
-        insertTracesAtTop = PrefBind.getOrDefaultBoolean("insertTracesAtTop", false);
-
         tracesBox = new TableView<TraceLog>();
         tracesBox.getStyleClass().add("trace-log-table");
         tracesBox.setOnMouseClicked(this::onTraceClicked);
@@ -95,15 +88,6 @@ public DbgTraceView(DbgController dbgController) {
         Parent p = traceLogFloatySearchControl();
 
         getChildren().addAll(p, tracesBox);
-
-        dbgController.getTraceLogs().addListener(this::traceLogsChanged);
-
-        ErlyBerly.nodeAPI().connectedProperty().addListener(
-            (o, oldV, newV) -> {
-                if(oldV && !newV) {
-                    traceLogs.add(TraceLog.newNodeDown());
-                }
-            });
     }
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
@@ -195,7 +179,7 @@ private void configureColumnWidth(String widthProperty, TableColumn<TraceLog, ?>
 
     private void putTraceContextMenu() {
         TraceContextMenu traceContextMenu = new TraceContextMenu();
-        traceContextMenu.setItems(traceLogs);
+        traceContextMenu.setItems(dbgController.getTraceLogs());
         traceContextMenu
                 .setSelectedItems(tracesBox.getSelectionModel().getSelectedItems());
 
@@ -307,15 +291,4 @@ private Region traceLogFloatySearchControl() {
 
         return fxmlNode;
     }
-
-    public void traceLogsChanged(ListChangeListener.Change<? extends TraceLog> e) {
-        while(e.next()) {
-            for (TraceLog trace : e.getAddedSubList()) {
-                if(insertTracesAtTop)
-                    traceLogs.add(0, trace);
-                else
-                    traceLogs.add(trace);
-            }
-        }
-    }
 }

src/main/java/erlyberly/TopBarView.java

@@ -258,11 +258,13 @@ public void initialize(URL url, ResourceBundle r) {
     private void onSuspendedStateChanged(Boolean suspended) {
         if(suspended) {
             suspendButton.setText("Unsuspend");
-            suspendButton.setGraphic(FAIcon.create().icon(AwesomeIcon.PLAY));
+            suspendButton.setGraphic(FAIcon.create().style("-fx-text-fill: white;").icon(AwesomeIcon.PLAY));
+            suspendButton.getStyleClass().add("button-suspended");
         }
         else {
             suspendButton.setText("Suspend");
             suspendButton.setGraphic(FAIcon.create().icon(AwesomeIcon.PAUSE));
+            suspendButton.getStyleClass().remove("button-suspended");
         }
     }
 

src/main/java/erlyberly/TraceContextMenu.java

@@ -85,10 +85,7 @@ private void copyToClipboard(StringBuilder sbuilder) {
 
     private void onDelete(ActionEvent e) {
         ArrayList<TraceLog> arrayList = new ArrayList<TraceLog>(selectedItems);
-
-        for (TraceLog traceLog : arrayList) {
-            items.remove(traceLog);
-        }
+        items.removeAll(arrayList);
     }
 
     private void onDeleteAll(ActionEvent e) {

src/main/java/erlyberly/TraceLog.java

@@ -310,6 +310,10 @@ public static TraceLog newNodeDown() {
         return new TraceLog("breaker-row", "NODE DOWN");
     }
 
+    public static TraceLog newLoadShedding() {
+        return new TraceLog("breaker-row", "LOAD SHEDDING");
+    }
+
     public OtpErlangList getStackTrace() {
         OtpErlangList stacktrace = (OtpErlangList) map.get(OtpUtil.atom("stack_trace"));
         if(stacktrace == null)

src/main/java/erlyberly/node/NodeAPI.java

@@ -69,6 +69,9 @@
 import javafx.collections.ObservableList;
 
 public class NodeAPI {
+
+    private static final OtpErlangAtom ERLYBERLY_TRACE_OVERLOAD_ATOM = atom("erlyberly_trace_overload");
+
     private static final String ERLYBERLY = "erlyberly";
 
     private static final String CANNOT_RUN_THIS_METHOD_FROM_THE_FX_THREAD = "cannot run this method from the FX thread";
@@ -85,6 +88,8 @@ public class NodeAPI {
 
     private static final OtpErlangAtom BET_SERVICES_MSG_ATOM = new OtpErlangAtom("add_locator");
 
+    private static final OtpErlangAtom REX_ATOM = atom("rex");
+
     public interface RpcCallback<T> {
         void callback(T result);
     }
@@ -408,7 +413,10 @@ else if(isTupleTagged(ERLYBERLY_MODULE_RELOADED_ATOM, receive)) {
             });
             return receiveRPC(timeout);
         }
-        else if(!isTupleTagged(atom("rex"), receive)) {
+        else if(isTupleTagged(ERLYBERLY_TRACE_OVERLOAD_ATOM, receive)) {
+            Platform.runLater(() -> { suspendedProperty.set(true); });
+        }
+        else if(!isTupleTagged(REX_ATOM, receive)) {
             throw new RuntimeException("Expected tuple tagged with atom rex but got " + receive);
         }
         OtpErlangObject result = receive.elementAt(1);
@@ -498,20 +506,18 @@ public synchronized OtpErlangList requestFunctions() throws Exception {
         return (OtpErlangList) receiveRPC();
     }
 
-    public synchronized void startTrace(ModFunc mf) throws Exception {
+    public synchronized void startTrace(ModFunc mf, int maxQueueLen) throws Exception {
         assert mf.getFuncName() != null : "function name cannot be null";
         // if tracing is suspended, we can't apply a new trace because that will
         // leave us in a state where some traces are active and others are not
         if(isSuspended())
             return;
-        sendRPC(ERLYBERLY, "start_trace", toTraceTuple(mf));
+        sendRPC(ERLYBERLY, "start_trace", toStartTraceFnArgs(mf, maxQueueLen));
 
         OtpErlangObject result = receiveRPC();
-
-        if(isTupleTagged(atom("error"), result)) {
+        if(!isTupleTagged(OK_ATOM, result)) {
             System.out.println(result);
 
-
             // TODO notify caller of failure!
             return;
         }
@@ -538,15 +544,15 @@ public synchronized void stopAllTraces() throws IOException, OtpErlangException
         receiveRPC();
     }
 
-    private OtpErlangList toTraceTuple(ModFunc mf) {
+    private OtpErlangList toStartTraceFnArgs(ModFunc mf, int maxQueueLen) {
         String node = self.node();
         OtpErlangPid self2 = mbox.self();
         return list(
             tuple(OtpUtil.atom(node), self2),
             atom(mf.getModuleName()),
             atom(mf.getFuncName()),
             mf.getArity(),
-            mf.isExported()
+            maxQueueLen
         );
     }
 

src/main/java/erlyberly/node/OtpUtil.java

@@ -83,11 +83,14 @@ private static OtpErlangObject[] toOtpElementArray(Object... elements) {
             if(e instanceof Integer) {
                 tuple[i] = new OtpErlangLong((Integer)e);
             }
+            else if(e instanceof Long) {
+                tuple[i] = new OtpErlangLong((Long)e);
+            }
             else if(e instanceof OtpErlangObject) {
-                tuple[i] = (OtpErlangObject) elements[i];
+                tuple[i] = (OtpErlangObject) e;
             }
             else if(e instanceof String) {
-                tuple[i] = new OtpErlangString((String)elements[i]);
+                tuple[i] = new OtpErlangString((String)e);
             }
             else if(e instanceof Boolean) {
                 if(Boolean.TRUE.equals(e))

src/main/java/ui/FAIcon.java

@@ -95,7 +95,7 @@ public FAIcon size(String iconSize) {
     }
 
     public FAIcon style(String style) {
-        setStyle(style);
+        setStyle(getStyle() + style);
         return this;
     }