Merge remote-tracking branch 'upstream/master'

cli_client/python/timesketch_cli_client/commands/sketch.py

@@ -158,3 +158,31 @@ def unarchive_sketch(ctx):
     if sketch.is_archived():
         sketch.unarchive()
         click.echo("Sketch unarchived")
+
+
+@sketch_group.command("add_label", help="Add a label to a sketch")
+@click.option("--label", required=True, help="Name of label to add.")
+@click.pass_context
+def add_label(ctx, label):
+    """Add a label to a sketch."""
+    sketch = ctx.obj.sketch
+    sketch.add_sketch_label(label)
+    click.echo("Label added")
+
+
+@sketch_group.command("list_label", help="List labels of sketch")
+@click.pass_context
+def list_label(ctx):
+    """List labels of a sketch."""
+    sketch = ctx.obj.sketch
+    click.echo(sketch.labels)
+
+
+@sketch_group.command("remove_label", help="Remove a label from a sketch")
+@click.option("--label", required=True, help="Name of label to remove.")
+@click.pass_context
+def remove_label(ctx, label):
+    """Remove a label from a sketch."""
+    sketch = ctx.obj.sketch
+    sketch.remove_sketch_label(label)
+    click.echo("Label removed.")

data/timesketch.conf

@@ -21,11 +21,11 @@ SECRET_KEY = '<KEY_GOES_HERE>'
 # production.
 SQLALCHEMY_DATABASE_URI = 'postgresql://<USERNAME>:<PASSWORD>@localhost/timesketch'
 
-# Configure where your Elasticsearch server is located.
+# Configure where your OpenSearch server is located.
 #
 # Make sure that the OpenSearch server is properly secured and not accessible
 # from the internet. See the following link for more information:
-# http://www.elasticsearch.org/blog/scripting-security/
+# https://opensearch.org/docs/latest/getting-started/security/
 OPENSEARCH_HOST = '127.0.0.1'
 OPENSEARCH_PORT = 9200
 OPENSEARCH_USER = None
@@ -34,6 +34,10 @@ OPENSEARCH_SSL = False
 OPENSEARCH_VERIFY_CERTS = True
 OPENSEARCH_TIMEOUT = 10
 OPENSEARCH_FLUSH_INTERVAL = 5000
+# Be careful when increasing the upper limit since this will impact your
+# OpenSearch clusters performance and storage requirements!
+OPENSEARCH_MAPPING_BUFFER = 0.1
+OPENSEARCH_MAPPING_UPPER_LIMIT = 1000
 
 # Define what labels should be defined that make it so that a sketch and
 # timelines will not be deleted. This can be used to add a list of different
@@ -366,7 +370,13 @@ LLM_PROVIDER_CONFIGS = {
     'vertexai': {
         'model': 'gemini-1.5-flash-001',
         'project_id': '',
-    }
+    },
+    # To use Google's AI Studio simply obtain an API key from https://aistudio.google.com/
+    # pip install google-generativeai
+    'aistudio': {
+        'api_key': '',
+        'model': 'gemini-2.0-flash-exp',
+    },
 }
 
 # LLM nl2q configuration

docs/guides/admin/admin-cli.md

@@ -150,6 +150,17 @@ bar
 dev (admin)
 ```
 
+Adding `--status` will add the status of the user to the output.
+
+Example
+```shell
+tsctl list-users --status
+dev (active: True)
+admin (active: True)
+foobar2 (active: True)
+foobar (active: False)
+```
+
 #### Make admin
 
 tsctl provides a subcommand for granting administrator privileges to a user in a Timesketch instance. This subcommand is called `make-admin`, and it allows you to specify the username of the user you want to grant administrator privileges to.
@@ -201,6 +212,18 @@ Not yet implemented.
 
 #### Managing group membership
 
+##### List groups
+
+To list groups, use `tsctl list-groups`. This can be extended  to show the members of a group using `tsctl list-groups --showmembershipt`
+
+Example:
+
+```bash
+tsctl list-groups --showmembership
+foobar:
+foobar-group:foobar2,foobar
+```
+
 ##### Add a suer to a group
 
 tsctl provides a subcommand for adding users to a group in a Timesketch instance. This subcommand is called `add-group-member`, and it allows you to specify the username of the user you want to add to the group, as well as the name of the group you want to add the user to.

docs/guides/user/cli-client.md

@@ -295,6 +295,38 @@ Running `sketch unarchive` will set the archive flag to the sketch.
 
 Running `sketch export` will export the complete Sketch to a file.
 
+### Labels
+
+#### list_labels
+
+Running `sketch list_labels` will give you a list of all labels of a sketch.
+
+Example:
+```bash
+timesketch --sketch 14 --output-format json sketch list_label
+['test', 'foobar']
+```
+
+#### add label
+
+Running `sketch add_label --label foobar` will add the label `foobar` to the sketch.
+
+Example:
+```bash
+timesketch --sketch 14 --output-format json sketch add_label --label=foobar
+Label added
+```
+
+#### Remove label
+
+Running `sketch remove_label --label foobar` will remove the label `foobar` from the sketch.
+
+Example:
+```bash
+timesketch --sketch 14 --output-format json sketch remove_label --label=foobar
+Label removed
+```
+
 ## Intelligence
 
 Intelligence is always sketch specific. The same can be achieved using 

end_to_end_tests/upload_test.py

@@ -14,6 +14,7 @@
 """End to end tests of Timesketch upload functionality."""
 import os
 import random
+import json
 
 from timesketch_api_client import search
 from . import interface
@@ -86,6 +87,61 @@ def test_large_upload_jsonl(self):
         events = sketch.explore("data_type:foobarjson", as_pandas=True)
         self.assertions.assertEqual(len(events), 4123)
 
+    def test_upload_jsonl_mapping_exceeds_limit(self):
+        """Test uploading a timeline with a jsonl events that exceeds the
+        default field mapping limit of 1000. The test will create a temporary
+        file with events that have many unique keys and then upload the file to
+        Timesketch. The test will then check that the correct error is triggered.
+        """
+
+        # create a new sketch
+        rand = random.randint(0, 10000)
+        sketch = self.api.create_sketch(
+            name=f"test_upload_jsonl_mapping_exceeds_limit {rand}"
+        )
+        self.sketch = sketch
+
+        file_path = "/tmp/mapping_over_1k.jsonl"
+        num_lines = 100
+        num_keys_per_line = 6
+        all_keys = set()
+
+        with open(file_path, "w", encoding="utf-8") as file_object:
+            for i in range(num_lines):
+                line_data = {
+                    "datetime": "2015-07-24T19:01:01+00:00",
+                    "message": f"Event {i} of {num_lines}",
+                    "timestamp_desc": "test",
+                    "data_type": "test:jsonl",
+                }
+                for j in range(num_keys_per_line):
+                    key = f"field_name_{j}_{random.randint(0, 100000)}"
+                    while key in all_keys:  # Avoid duplicate keys
+                        key = f"field_name_{random.randint(0, 100000)}"
+                    all_keys.add(key)
+                    line_data[key] = f"value_{j}_{random.randint(0, 10000)}"
+
+                json.dump(line_data, file_object)
+                file_object.write("\n")
+
+        try:
+            self.import_timeline(file_path, index_name=rand, sketch=sketch)
+        except RuntimeError:
+            print(
+                "Timeline import failing is expected. Checking for the correct "
+                "error message..."
+            )
+        os.remove(file_path)
+
+        timeline = sketch.list_timelines()[0]
+        # check that timeline threw the correct error
+        self.assertions.assertEqual(timeline.name, file_path)
+        self.assertions.assertEqual(timeline.index.name, str(rand))
+        self.assertions.assertEqual(timeline.index.status, "fail")
+        self.assertions.assertIn(
+            "OPENSEARCH_MAPPING_UPPER_LIMIT", timeline.data_sources[0]["error_message"]
+        )
+
     def test_very_large_upload_jsonl(self):
         """Test uploading a timeline with over 50 k events as jsonl. The test
         will create a temporary file and then

timesketch/frontend-ng/dist/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.f265108c.css rel=preload as=style><link href=/dist/css/index.008f2248.css rel=preload as=style><link href=/dist/js/chunk-vendors.eba39bd6.js rel=preload as=script><link href=/dist/js/index.1ff10a58.js rel=preload as=script><link href=/dist/css/chunk-vendors.f265108c.css rel=stylesheet><link href=/dist/css/index.008f2248.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.eba39bd6.js></script><script src=/dist/js/index.1ff10a58.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.26c8f211.css rel=preload as=style><link href=/dist/css/index.008f2248.css rel=preload as=style><link href=/dist/js/chunk-vendors.da6c04f5.js rel=preload as=script><link href=/dist/js/index.603e8c4f.js rel=preload as=script><link href=/dist/css/chunk-vendors.26c8f211.css rel=stylesheet><link href=/dist/css/index.008f2248.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.da6c04f5.js></script><script src=/dist/js/index.603e8c4f.js></script></body></html>

timesketch/frontend-ng/dist/login.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.f265108c.css rel=preload as=style><link href=/dist/css/login.5d50b378.css rel=preload as=style><link href=/dist/js/chunk-vendors.eba39bd6.js rel=preload as=script><link href=/dist/js/login.fc00c6ee.js rel=preload as=script><link href=/dist/css/chunk-vendors.f265108c.css rel=stylesheet><link href=/dist/css/login.5d50b378.css rel=stylesheet></head><body><div class=card><div class=logo-container><div class=logo-image-container><img class=logo-image src=/dist/timesketch-color.png></div><div class=logo-text-container><div class=logo-text-header>timesketch</div><div class=logo-text-subheader>Digital Forensics Timeline Analysis</div></div></div><form method=post><input type=text class=input name=username placeholder=Username> <input type=password class=input name=password placeholder=Password> <button type=submit class=login-button>Sign in</button> {{ form.csrf_token }}</form></div><script src=/dist/js/chunk-vendors.eba39bd6.js></script><script src=/dist/js/login.fc00c6ee.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.26c8f211.css rel=preload as=style><link href=/dist/css/login.5d50b378.css rel=preload as=style><link href=/dist/js/chunk-vendors.da6c04f5.js rel=preload as=script><link href=/dist/js/login.fc00c6ee.js rel=preload as=script><link href=/dist/css/chunk-vendors.26c8f211.css rel=stylesheet><link href=/dist/css/login.5d50b378.css rel=stylesheet></head><body><div class=card><div class=logo-container><div class=logo-image-container><img class=logo-image src=/dist/timesketch-color.png></div><div class=logo-text-container><div class=logo-text-header>timesketch</div><div class=logo-text-subheader>Digital Forensics Timeline Analysis</div></div></div><form method=post><input type=text class=input name=username placeholder=Username> <input type=password class=input name=password placeholder=Password> <button type=submit class=login-button>Sign in</button> {{ form.csrf_token }}</form></div><script src=/dist/js/chunk-vendors.da6c04f5.js></script><script src=/dist/js/login.fc00c6ee.js></script></body></html>

timesketch/frontend-ng/package.json

@@ -17,7 +17,7 @@
     "cytoscape-dagre": "^2.5.0",
     "cytoscape-spread": "^3.0.0",
     "dayjs": "^1.10.7",
-    "dompurify": "^3.0.1",
+    "dompurify": "^3.2.3",
     "lodash": "^4.17.11",
     "marked": ">=2.0.0 <5.0.0",
     "moment": "^2.24.0",
@@ -47,7 +47,7 @@
     "eslint": "^5.8.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-vue": "^5.0.0",
-    "happy-dom": "^15.10.1",
+    "happy-dom": "^15.10.2",
     "node-sass": "^8.0.0",
     "sass": "~1.32",
     "sass-loader": "^10.1.1",

timesketch/frontend-ng/src/components/Explore/Comments.vue

@@ -116,6 +116,7 @@ export default {
           this.comments.push(response.data.objects[0][0])
           this.event._source.comment.push(this.comment)
           this.comment = ''
+          this.$store.dispatch('updateEventLabels', { label: "__ts_comment", num: 1 })
         })
         .catch((e) => {})
     },
@@ -135,6 +136,7 @@ export default {
           .then((response) => {
             this.comments.splice(commentIndex, 1)
             this.event._source.comment.splice(commentIndex, 1)
+            this.$store.dispatch('updateEventLabels', { label: "__ts_comment", num: -1 })
           })
           .catch((e) => {
             console.error(e)

timesketch/frontend-ng/src/components/Explore/EventList.vue

@@ -986,27 +986,36 @@ export default {
       this.selectedFields.splice(index, 1)
     },
     toggleStar(event) {
+      let count = 0
       if (event._source.label.includes('__ts_star')) {
         event._source.label.splice(event._source.label.indexOf('__ts_star'), 1)
+        count = -1
       } else {
         event._source.label.push('__ts_star')
+        count = 1
       }
       ApiClient.saveEventAnnotation(this.sketch.id, 'label', '__ts_star', event, this.currentSearchNode)
-        .then((response) => {})
+        .then((response) => {
+          this.$store.dispatch('updateEventLabels', { label: "__ts_star", num: count })
+        })
         .catch((e) => {
           console.error(e)
         })
     },
     toggleMultipleStars: function () {
+      let netStarCountChange = 0
       this.selectedEvents.forEach((event) => {
         if (event._source.label.includes('__ts_star')) {
           event._source.label.splice(event._source.label.indexOf('__ts_star'), 1)
+          netStarCountChange--
         } else {
           event._source.label.push('__ts_star')
+          netStarCountChange++
         }
       })
       ApiClient.saveEventAnnotation(this.sketch.id, 'label', '__ts_star', this.selectedEvents, this.currentSearchNode)
         .then((response) => {
+          this.$store.dispatch('updateEventLabels',{ label: "__ts_star", num: netStarCountChange })
           this.selectedEvents = []
         })
         .catch((e) => {})