Anomaly detector detection re running in inspect mode #757

server/spec/analytic_controller.jest.ts

@@ -3,92 +3,94 @@ import { queryByMetric } from 'grafana-datasource-kit';
 jest.mock('grafana-datasource-kit', () => (
   {
     ...(jest.requireActual('grafana-datasource-kit')),
-    queryByMetric: jest.fn((metric, url, from, to, apiKey) => {})
+    queryByMetric: jest.fn((metric, url, from, to, apiKey) => {
+      return { values:[], columns:[] }
+    })
   }
 ));
 
-import { saveAnalyticUnitFromObject, runDetect, onDetect } from '../src/controllers/analytics_controller';
+import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
 import * as AnalyticUnit from '../src/models/analytic_units';
 import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
 import * as Segment from '../src/models/segment_model';
 import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
 import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
-
 import { HASTIC_API_KEY } from '../src/config';
 
-
-describe('Check detection range', function() {
-  const analyticUnitObj = {
-    _id: 'test',
-    name: "test",
-    grafanaUrl: "http://127.0.0.1:3000",
-    panelId: "ZLc0KfNZk/2",
-    type: "GENERAL",
-    metric: {
-      datasource: {
-        url: "api/datasources/proxy/5/query",
-        method: "GET",
-        data: null,
-        params: {
-          db:"dbname",
-          q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
-          epoch: "ms"
-        },
-        type: "influxdb"
+import * as _ from 'lodash';
+
+
+const DEFAULT_ANALYTIC_UNIT_OBJECT = {
+  name: "test",
+  grafanaUrl: "http://127.0.0.1:3000",
+  panelId: "ZLc0KfNZk/2",
+  type: "GENERAL",
+  metric: {
+    datasource: {
+      url: "api/datasources/proxy/5/query",
+      method: "GET",
+      data: null,
+      params: {
+        db:"dbname",
+        q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
+        epoch: "ms"
       },
-      targets: [
-        {
-          groupBy: [
-            {
-              params: ["$__interval"],
-              type: "time"
-            },
-            {
-              params: ["null"],
-              type: "fill"
-            }
-          ],
-          measurement: "tcpconns_value",
-          orderByTime: "ASC",
-          policy: "autogen",
-          refId: "A",
-          resultFormat: "time_series",
-          select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
-        }
-      ]
+      type: "influxdb"
     },
-    alert: false,
-    labeledColor: "#FF99FF",
-    deletedColor: "#00f0ff",
-    detectorType: "pattern",
-    visible: true,
-    collapsed: false,
-    createdAt: {"$$date":1564476040880},
-    updatedAt: {"$$date":1564476040880}
-  }
-
-  const WINDOW_SIZE = 10;
-  const TIME_STEP = 1000;
-
-  async function addTestUnitToDB(): Promise<string> {
-    const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
-    await AnalyticUnit.update(analyticUnitId, {lastDetectionTime: 1000});
-    await AnalyticUnitCache.create(analyticUnitId);
-    await AnalyticUnitCache.setData(analyticUnitId, {
-      windowSize: WINDOW_SIZE,
-      timeStep: TIME_STEP
-    });
-    return analyticUnitId;
-  };
+    targets: [
+      {
+        groupBy: [
+          {
+            params: ["$__interval"],
+            type: "time"
+          },
+          {
+            params: ["null"],
+            type: "fill"
+          }
+        ],
+        measurement: "tcpconns_value",
+        orderByTime: "ASC",
+        policy: "autogen",
+        refId: "A",
+        resultFormat: "time_series",
+        select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
+      }
+    ]
+  },
+  alert: false,
+  labeledColor: "#FF99FF",
+  deletedColor: "#00f0ff",
+  detectorType: "pattern",
+  visible: true,
+  collapsed: false,
+  createdAt: {"$$date":1564476040880},
+  updatedAt: {"$$date":1564476040880}
+}
+
+const WINDOW_SIZE = 10;
+const TIME_STEP = 1000;
+
+async function addTestUnitToDB(analyticUnitObj: any): Promise<string> {
+  const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
+  await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 });
+  await AnalyticUnitCache.create(analyticUnitId);
+  await AnalyticUnitCache.setData(analyticUnitId, {
+    windowSize: WINDOW_SIZE,
+    timeStep: TIME_STEP
+  });
+  return analyticUnitId;
+};
 
+describe('Check detection range', function() {
   it('check range >= 2 * window size * timeStep', async () => {
     const from = 1500000000000;
     const to = 1500000000001;
     const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2;
 
-    const id = await addTestUnitToDB();
+    const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT);
     await runDetect(id, from, to);
-    expect(queryByMetric).toBeCalledWith(analyticUnitObj.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
+    expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
   });
 });
 
@@ -136,3 +138,15 @@ describe('onDetect', () => {
     expect(detectedRanges).toEqual([[7, 8]]);
   });
 });
+
+describe('getHSR', function() {
+  it('should return nothing if unit state is LEARNING', async () => {
+    let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT);
+    unitObj.detectorType = 'anomaly';
+    const analyticUnitId = await addTestUnitToDB(unitObj);
+    await AnalyticUnitCache.remove(analyticUnitId);
+    const unit = await AnalyticUnit.findById(analyticUnitId);
+    const result = await getHSR(unit, 9000, 100000);
+    expect(result).toEqual({"hsr": {"columns": [], "values": []}});
+  });
+});

server/src/controllers/analytics_controller.ts

@@ -640,34 +640,39 @@ export async function getHSR(
   analyticUnit: AnalyticUnit.AnalyticUnit,
   from: number,
   to: number
-): Promise<{
-  hsr: TableTimeSeries,
-  lowerBound?: TableTimeSeries,
-  upperBound?: TableTimeSeries
-}> {
+): Promise< HSRResult | undefined> {
   try {
     const grafanaUrl = getGrafanaUrl(analyticUnit.grafanaUrl);
-    const data = await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY);
-    let resultSeries: HSRResult = {
-      hsr: data
-    }
 
     if(analyticUnit.detectorType === AnalyticUnit.DetectorType.PATTERN) {
+      const resultSeries: HSRResult = {
+        hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
+      };
       return resultSeries;
     }
 
     let cache = await AnalyticUnitCache.findById(analyticUnit.id);
-    if(cache === null || cache.isCacheOutdated(analyticUnit)) {
-      await runLearning(analyticUnit.id, from, to);
-      cache = await AnalyticUnitCache.findById(analyticUnit.id);
+
+    const inLearningState = analyticUnit.status === AnalyticUnit.AnalyticUnitStatus.LEARNING;
+
+    //cache.data === null when learning started but not completed yet or first learning was failed
+    if(inLearningState || cache === null || cache.data === null) {
+      const resultSeries: HSRResult = {
+        hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
+      };
+      return resultSeries;
+      //TODO: send warning: can't show HSR before learning
     }
 
     cache = cache.data;
+    let resultSeries: HSRResult = {
+      hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
+    };
 
     const analyticUnitType = analyticUnit.type;
     const detector = analyticUnit.detectorType;
     const payload = {
-      data: data.values,
+      data: resultSeries.hsr.values,
       analyticUnitType,
       detector,
       cache
@@ -680,11 +685,11 @@ export async function getHSR(
     }
 
     if(result.payload.lowerBound !== undefined) {
-      resultSeries.lowerBound = { values: result.payload.lowerBound, columns: data.columns };
+      resultSeries.lowerBound = { values: result.payload.lowerBound, columns: resultSeries.hsr.columns };
     }
 
     if(result.payload.upperBound !== undefined) {
-      resultSeries.upperBound = { values: result.payload.upperBound, columns: data.columns };
+      resultSeries.upperBound = { values: result.payload.upperBound, columns: resultSeries.hsr.columns };
     }
 
     return resultSeries;

server/src/models/analytic_unit_cache_model.ts

@@ -52,12 +52,6 @@ export class AnalyticUnitCache {
   public getTimeStep(): number {
     return this.data.timeStep;
   }
-
-  public isCacheOutdated(analyticUnit: AnalyticUnit) {
-    return !_.every(
-      _.keys(analyticUnit.analyticProps).map(k => _.isEqual(analyticUnit.analyticProps[k], this.data[k]))
-    );
-  }
 }
 
 export async function findById(id: AnalyticUnitId): Promise<AnalyticUnitCache> {