From aab4af9397f28822b5ef9491650791de78cfcdd8 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sun, 21 Jul 2019 09:35:39 +0200 Subject: [PATCH 01/38] [elastic_items] Change filter-raw parameter This code changes the filter-raw parameter to `search_terms`. This change is needed to use the new Perceval metadata attribute `search_terms` to filter raw data. Signed-off-by: Valerio Cosentino --- grimoire_elk/elastic_items.py | 2 +- tests/test_elastic_items.py | 68 +++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/grimoire_elk/elastic_items.py b/grimoire_elk/elastic_items.py index c563a4a13..615af5080 100644 --- a/grimoire_elk/elastic_items.py +++ b/grimoire_elk/elastic_items.py @@ -33,7 +33,7 @@ HEADER_JSON = {"Content-Type": "application/json"} MAX_BULK_UPDATE_SIZE = 1000 -FILTER_DATA_ATTR = 'data.' +FILTER_DATA_ATTR = 'search_fields.' FILTER_SEPARATOR = r",\s*%s" % FILTER_DATA_ATTR PROJECTS_JSON_LABELS_PATTERN = r".*(--labels=\[(.*)\]).*" diff --git a/tests/test_elastic_items.py b/tests/test_elastic_items.py index 7aabb19fd..24c513942 100644 --- a/tests/test_elastic_items.py +++ b/tests/test_elastic_items.py @@ -25,6 +25,8 @@ import os import unittest +from grimoire_elk.elastic_items import FILTER_DATA_ATTR + import requests from grimoire_elk.elastic import ElasticSearch @@ -166,112 +168,118 @@ def test_set_filter_raw(self): ei = ElasticItems(None) filter_raws = [ - "data.product:Firefox, for Android,data.component:Logins, Passwords and Form Fill", - "data.product:Add-on SDK", - "data.product:Add-on SDK, data.component:Documentation", - "data.product:Add-on SDK, data.component:General", - "data.product:addons.mozilla.org Graveyard, data.component:API", - "data.product:addons.mozilla.org Graveyard, data.component:Add-on Builder", - "data.product:Firefox for Android,data.component:Build Config & IDE Support", - "data.product:Firefox for Android,data.component:Logins, Passwords and Form Fill", - "data.product:Mozilla Localizations,data.component:nb-NO / Norwegian Bokm\u00e5l", - "data.product:addons.mozilla.org Graveyard,data.component:Add-on Validation" + "{}product:Firefox, for Android,{}component:Logins, Passwords and Form Fill".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), + "{}product:Add-on SDK".format(FILTER_DATA_ATTR), + "{}product:Add-on SDK, {}component:Documentation".format(FILTER_DATA_ATTR, FILTER_DATA_ATTR), + "{}product:Add-on SDK, {}component:General".format(FILTER_DATA_ATTR, FILTER_DATA_ATTR), + "{}product:addons.mozilla.org Graveyard, {}component:API".format(FILTER_DATA_ATTR, FILTER_DATA_ATTR), + "{}product:addons.mozilla.org Graveyard, {}component:Add-on Builder".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), + "{}product:Firefox for Android,{}component:Build Config & IDE Support".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), + "{}product:Firefox for Android,{}component:Logins, Passwords and Form Fill".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), + "{}product:Mozilla Localizations,{}component:nb-NO / Norwegian Bokm\u00e5l".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), + "{}product:addons.mozilla.org Graveyard,{}component:Add-on Validation".format(FILTER_DATA_ATTR, + FILTER_DATA_ATTR), ] expected = [ [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Firefox, for Android" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Logins, Passwords and Form Fill" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Add-on SDK" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Add-on SDK" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Documentation" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Add-on SDK" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "General" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "addons.mozilla.org Graveyard" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "API" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "addons.mozilla.org Graveyard" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Add-on Builder" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Firefox for Android" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Build Config & IDE Support" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Firefox for Android" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Logins, Passwords and Form Fill" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "Mozilla Localizations" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "nb-NO / Norwegian Bokm\u00e5l" } ], [ { - "name": "data.product", + "name": FILTER_DATA_ATTR + "product", "value": "addons.mozilla.org Graveyard" }, { - "name": "data.component", + "name": FILTER_DATA_ATTR + "component", "value": "Add-on Validation" } ] @@ -423,7 +431,7 @@ def test_fetch_filter_raw(self): ocean.feed_items(items) eitems = ElasticItems(perceval_backend) - eitems.set_filter_raw("data.commit:87783129c3f00d2c81a3a8e585eb86a47e39891a") + eitems.set_filter_raw("item_id:456a68ee1407a77f3e804a30dff245bb6c6b872f") eitems.elastic = elastic items = [ei for ei in eitems.fetch()] self.assertEqual(len(items), 1) From 5c989fe084ddf7f41f2c264311a5045d5da9273b Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sun, 21 Jul 2019 10:05:15 +0200 Subject: [PATCH 02/38] [raw-jira] Change mapping This code changes the mapping of the Jira raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/jira.py | 47 ++------------------------------ tests/data/projects-release.json | 2 +- tests/test_jira.py | 2 +- 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/grimoire_elk/raw/jira.py b/grimoire_elk/raw/jira.py index e1f3760da..3835331d3 100644 --- a/grimoire_elk/raw/jira.py +++ b/grimoire_elk/raw/jira.py @@ -29,57 +29,16 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.renderedFields.description - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, "properties": { "data": { - "properties": { - "renderedFields": { - "dynamic":false, - "properties": {} - }, - "operations": { - "dynamic":false, - "properties": {} - }, - "fields": { - "dynamic":true, - "properties": { - "description": { - "type": "text", - "index": true - }, - "environment": { - "type": "text", - "index": true - } - } - }, - "changelog": { - "properties": { - "histories": { - "dynamic":false, - "properties": {} - } - } - }, - "comments_data": { - "properties": { - "body": { - "type": "text", - "index": true - } - } - } - } + "dynamic":false, + "properties": {} } } } diff --git a/tests/data/projects-release.json b/tests/data/projects-release.json index 8eb474843..223c40f4b 100644 --- a/tests/data/projects-release.json +++ b/tests/data/projects-release.json @@ -57,7 +57,7 @@ "https://build.opnfv.org/ci" ], "jira": [ - "https://jira.opnfv.org --filter-raw=data.fields.project.key:PROJECT-KEY" + "https://jira.opnfv.org --filter-raw=search_fields.project_key:PROJECT-KEY" ], "kitsune": [""], "mattermost": [ diff --git a/tests/test_jira.py b/tests/test_jira.py index 79698c563..75b864f02 100644 --- a/tests/test_jira.py +++ b/tests/test_jira.py @@ -185,7 +185,7 @@ def test_get_p2o_params_from_url(self): with open("data/projects-release.json") as projects_filename: url = json.load(projects_filename)['grimoire']['jira'][0] - p2o_params = {'url': 'https://jira.opnfv.org', 'filter-raw': 'data.fields.project.key:PROJECT-KEY'} + p2o_params = {'url': 'https://jira.opnfv.org', 'filter-raw': 'search_fields.project_key:PROJECT-KEY'} self.assertDictEqual(p2o_params, JiraOcean.get_p2o_params_from_url(url)) From 3ce862fe3881e3f6886006217b25635e1bcbab9b Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:41:35 +0200 Subject: [PATCH 03/38] [raw-askbot] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/askbot.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/grimoire_elk/raw/askbot.py b/grimoire_elk/raw/askbot.py index 9b9788474..032df1a15 100644 --- a/grimoire_elk/raw/askbot.py +++ b/grimoire_elk/raw/askbot.py @@ -29,34 +29,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "answers": { - "dynamic":false, - "properties": {} - }, - "author": { - "dynamic":false, - "properties": {} - }, - "comments": { - "dynamic":false, - "properties": {} - }, - "summary": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 3ae3458a3c25b68e2dc9e87b1ea4a97e1425ee8a Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:42:28 +0200 Subject: [PATCH 04/38] [raw-bugzilla] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/bugzilla.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/grimoire_elk/raw/bugzilla.py b/grimoire_elk/raw/bugzilla.py index 79bfa19ba..d32a059ca 100644 --- a/grimoire_elk/raw/bugzilla.py +++ b/grimoire_elk/raw/bugzilla.py @@ -32,35 +32,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "long_desc": { - "dynamic": false, - "properties": {} - }, - "short_desc": { - "dynamic": false, - "properties": { - "__text__": { - "type": "text", - "index": true - } - } - }, - "activity": { - "dynamic": false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 256e4ccad94b5e8bafc12a0966b379f452e23863 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:42:47 +0200 Subject: [PATCH 05/38] [raw-bugzillarest] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/bugzillarest.py | 74 +++----------------------------- 1 file changed, 6 insertions(+), 68 deletions(-) diff --git a/grimoire_elk/raw/bugzillarest.py b/grimoire_elk/raw/bugzillarest.py index f55fa796b..5f3dd474e 100644 --- a/grimoire_elk/raw/bugzillarest.py +++ b/grimoire_elk/raw/bugzillarest.py @@ -32,79 +32,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "comments": { - "dynamic":false, - "properties": { - "raw_text": { - "type": "text", - "index": true - }, - "text": { - "type": "text", - "index": true - } - } - }, - "attachments": { - "properties": { - "description" : { - "type": "text", - "index": true - }, - "summary" : { - "type": "text", - "index": true - } - } - }, - "clones": { - "dynamic":false, - "properties": {} - }, - "description": { - "type": "text", - "index": true - }, - "history": { - "properties": { - "changes": { - "properties": { - "removed": { - "type": "text", - "index": true - }, - "added": { - "type": "text", - "index": true - }, - "comment": { - "properties": { - "thetext": { - "type": "text", - "index": true - } - } - } - } - } - } - }, - "summary": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 6c86f0e2edb1256dd04f78a56a403bd2ab65a1cd Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:43:05 +0200 Subject: [PATCH 06/38] [raw-confluence] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/confluence.py | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/grimoire_elk/raw/confluence.py b/grimoire_elk/raw/confluence.py index 8dcd74b00..089cab355 100644 --- a/grimoire_elk/raw/confluence.py +++ b/grimoire_elk/raw/confluence.py @@ -29,38 +29,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.extensions - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "extensions": { - "dynamic":false, - "properties": {} - }, - "ancestors": { - "properties": { - "extensions": { - "dynamic":false, - "properties": {} - } - } - }, - "body": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From fdce2e2a3a9b9e2b4975caf4f2ea0a691d20b40a Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:43:25 +0200 Subject: [PATCH 07/38] [raw-discourse] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/discourse.py | 55 ++++------------------------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/grimoire_elk/raw/discourse.py b/grimoire_elk/raw/discourse.py index f02ce8480..766253258 100644 --- a/grimoire_elk/raw/discourse.py +++ b/grimoire_elk/raw/discourse.py @@ -30,60 +30,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "details": { - "properties": { - "suggested_topics": { - "dynamic": false, - "properties": { - "slug": { - "type": "text", - "index": true - }, - "title": { - "type": "text", - "index": true - } - } - } - } - }, - "fancy_title": { - "type": "text", - "index": true - }, - "slug": { - "type": "text", - "index": true - }, - "title": { - "type": "text", - "index": true - }, - "post_stream": { - "dynamic": false, - "properties": { - "posts": { - "properties": { - "cooked": { - "type": "text", - "index": true - } - } - } - } - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 65f1c17105a573e4dd8ab0366d41abdb8b4a969c Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:43:42 +0200 Subject: [PATCH 08/38] [raw-dockerhub] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/dockerhub.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/grimoire_elk/raw/dockerhub.py b/grimoire_elk/raw/dockerhub.py index 34c9b0053..213e963da 100644 --- a/grimoire_elk/raw/dockerhub.py +++ b/grimoire_elk/raw/dockerhub.py @@ -30,26 +30,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "description": { - "type": "text", - "index": true - }, - "full_description": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From f16c193cd2f17d005f1b64df5d944bd9766da322 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:43:59 +0200 Subject: [PATCH 09/38] [raw-gerrit] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/gerrit.py | 43 +++----------------------------------- 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/grimoire_elk/raw/gerrit.py b/grimoire_elk/raw/gerrit.py index 9b1333f69..313217122 100644 --- a/grimoire_elk/raw/gerrit.py +++ b/grimoire_elk/raw/gerrit.py @@ -32,52 +32,15 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, "properties": { "data": { - "properties": { - "commitMessage": { - "type": "text", - "index": true - }, - "comments": { - "properties": { - "message": { - "type": "text", - "index": true - } - } - }, - "subject": { - "type": "text", - "index": true - }, - "patchSets": { - "properties": { - "approvals": { - "properties": { - "description": { - "type": "text", - "index": true - } - } - }, - "comments": { - "properties": { - "message": { - "type": "text", - "index": true - } - } - } - } - } - } + "dynamic":false, + "properties": {} } } } From 809f46a08e1268030bb213ce24bff69bbcef4104 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:44:18 +0200 Subject: [PATCH 10/38] [raw-git] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/git.py | 13 +++---------- tests/data/git.json | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/grimoire_elk/raw/git.py b/grimoire_elk/raw/git.py index a219dad45..1073d9515 100644 --- a/grimoire_elk/raw/git.py +++ b/grimoire_elk/raw/git.py @@ -30,23 +30,16 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Ensure data.message is string, since it can be very large - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, "properties": { "data": { - "properties": { - "message": { - "type": "text", - "index": true - } - } + "dynamic":false, + "properties": {} } } } diff --git a/tests/data/git.json b/tests/data/git.json index ea708e281..890470c52 100644 --- a/tests/data/git.json +++ b/tests/data/git.json @@ -2,6 +2,9 @@ "backend_name": "Git", "backend_version": "0.8.10", "category": "commit", + "search_fields": { + "item_id": "bc57a9209f096a130dcc5ba7089a8663f758a703" + }, "data": { "Author": "Eduardo Morais ", "AuthorDate": "Tue Aug 14 14:30:13 2012 -0300", @@ -65,6 +68,9 @@ },{ "backend_name": "Git", "backend_version": "0.8.10", + "search_fields": { + "item_id": "87783129c3f00d2c81a3a8e585eb86a47e39891a" + }, "category": "commit", "data": { "Author": "Eduardo Morais ", @@ -104,6 +110,9 @@ },{ "backend_name": "Git", "backend_version": "0.8.10", + "search_fields": { + "item_id": "7debcf8a2f57f86663809c58b5c07a398be7674c" + }, "category": "commit", "data": { "Author": "Eduardo Morais ", @@ -142,6 +151,9 @@ },{ "backend_name": "Git", "backend_version": "0.8.10", + "search_fields": { + "item_id": "c0d66f92a95e31c77be08dc9d0f11a16715d1885" + }, "category": "commit", "data": { "Author": "Eduardo Morais ", @@ -195,6 +207,9 @@ },{ "backend_name": "Git", "backend_version": "0.8.10", + "search_fields": { + "item_id": "c6ba8f7a1058db3e6b4bc6f1090e932b107605fb" + }, "category": "commit", "data": { "Author": "Eduardo Morais ", @@ -234,6 +249,9 @@ "backend_name": "Git", "backend_version": "0.8.10", "category": "commit", + "search_fields": { + "item_id": "589bb080f059834829a2a5955bebfd7c2baa110a" + }, "data": { "Author": "Eduardo Morais ", "AuthorDate": "Tue Aug 14 15:04:01 2012 -0300", @@ -272,6 +290,9 @@ "backend_name": "Git", "backend_version": "0.8.10", "category": "commit", + "search_fields": { + "item_id": "ce8e0b86a1e9877f42fe9453ede418519115f367" + }, "data": { "Author": "Zhongpeng Lin (\u6797\u4e2d\u9e4f) ", "AuthorDate": "Tue Feb 11 22:07:49 2014 -0800", @@ -311,6 +332,9 @@ "backend_name": "Git", "backend_version": "0.8.10", "category": "commit", + "search_fields": { + "item_id": "51a3b654f252210572297f47597b31527c475fb8" + }, "data": { "Author": "Zhongpeng Lin (\u6797\u4e2d\u9e4f) ", "AuthorDate": "Tue Feb 11 22:09:26 2014 -0800", @@ -351,6 +375,9 @@ "backend_name": "Git", "backend_version": "0.8.10", "category": "commit", + "search_fields": { + "item_id": "456a68ee1407a77f3e804a30dff245bb6c6b872f" + }, "data": { "Author": "Zhongpeng Lin (\u6797\u4e2d\u9e4f) ", "AuthorDate": "Tue Feb 11 22:10:39 2014 -0800", From b7bd993a032920ca765e48685dce0af0474a9585 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:44:37 +0200 Subject: [PATCH 11/38] [raw-github] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/github.py | 48 +++++--------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/grimoire_elk/raw/github.py b/grimoire_elk/raw/github.py index 687702bec..ec8d0abef 100644 --- a/grimoire_elk/raw/github.py +++ b/grimoire_elk/raw/github.py @@ -31,53 +31,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "comments_data": { - "dynamic":false, - "properties": { - "body": { - "type": "text", - "index": true - } - } - }, - "review_comments_data": { - "dynamic":false, - "properties": { - "body": { - "type": "text", - "index": true - }, - "diff_hunk": { - "type": "text", - "index": true - } - } - }, - "reviews_data": { - "dynamic":false, - "properties": { - "body": { - "type": "text", - "index": true - } - } - }, - "body": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 68950138842f987a6d06423a2cd0d527ad17e850 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Fri, 26 Jul 2019 23:44:59 +0200 Subject: [PATCH 12/38] [raw-gitlab] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/gitlab.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/grimoire_elk/raw/gitlab.py b/grimoire_elk/raw/gitlab.py index 377bcb72f..dd0957faa 100644 --- a/grimoire_elk/raw/gitlab.py +++ b/grimoire_elk/raw/gitlab.py @@ -31,35 +31,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "notes_data": { - "dynamic":false, - "properties": { - "body": { - "type": "text", - "index": true - } - } - }, - "description": { - "type": "text", - "index": true - }, - "versions_data": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From da494c06011b9127bf66d22658cd34932cda0e36 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:44:39 +0200 Subject: [PATCH 13/38] [raw-googlehits] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/google_hits.py | 27 ++++++++++++++++++++++++++- tests/test_google_hits.py | 5 +++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/raw/google_hits.py b/grimoire_elk/raw/google_hits.py index 8d52cf1bd..7ecb9c888 100644 --- a/grimoire_elk/raw/google_hits.py +++ b/grimoire_elk/raw/google_hits.py @@ -22,9 +22,34 @@ """GoogleHits Ocean feeder""" from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} class GoogleHitsOcean(ElasticOcean): """GoogleHits Ocean feeder""" - pass + mapping = Mapping diff --git a/tests/test_google_hits.py b/tests/test_google_hits.py index 2137968b9..c0d11b245 100644 --- a/tests/test_google_hits.py +++ b/tests/test_google_hits.py @@ -56,6 +56,11 @@ def test_raw_to_enrich(self): self.assertGreater(result['enrich'], 0) self.assertEqual(result['raw'], result['enrich']) + enrich_backend = self.connectors[self.connector][2]() + + item = self.items[0] + self.assertEqual(enrich_backend.get_identities(item), []) + def test_enrich_repo_labels(self): """Test whether the field REPO_LABELS is present in the enriched items""" From e5877cd20a8774a45601d5e9c3234c4ea78aaceb Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:46:27 +0200 Subject: [PATCH 14/38] [raw-hyperkitty] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/hyperkitty.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/grimoire_elk/raw/hyperkitty.py b/grimoire_elk/raw/hyperkitty.py index af8bb0b03..3d9363c19 100644 --- a/grimoire_elk/raw/hyperkitty.py +++ b/grimoire_elk/raw/hyperkitty.py @@ -34,26 +34,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.body (inmense field) - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "body": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From b33108453696864a650e391507a34b0acec74874 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:47:42 +0200 Subject: [PATCH 15/38] [raw-jenkins] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/jenkins.py | 88 +++---------------------------------- 1 file changed, 6 insertions(+), 82 deletions(-) diff --git a/grimoire_elk/raw/jenkins.py b/grimoire_elk/raw/jenkins.py index ca9467ce3..afcb1b175 100644 --- a/grimoire_elk/raw/jenkins.py +++ b/grimoire_elk/raw/jenkins.py @@ -29,94 +29,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.runs - * data.actions - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "_class" : { - "type" : "keyword" - }, - "runs": { - "dynamic":false, - "properties": {} - }, - "actions": { - "dynamic":false, - "properties": {} - }, - "building" : { - "type" : "boolean" - }, - "builtOn" : { - "type" : "keyword" - }, - "changeSet": { - "properties": { - "items": { - "properties": { - "comment": { - "type": "text", - "index": true - } - } - } - } - }, - "description" : { - "type" : "keyword" - }, - "displayName" : { - "type" : "keyword" - }, - "duration" : { - "type" : "long" - }, - "estimatedDuration" : { - "type" : "long" - }, - "executor" : { - "type" : "object" - }, - "fullDisplayName" : { - "type" : "keyword" - }, - "id" : { - "type" : "keyword" - }, - "keepLog" : { - "type" : "boolean" - }, - "number" : { - "type" : "long" - }, - "queueId" : { - "type" : "long" - }, - "result" : { - "type" : "keyword" - }, - "subBuilds" : { - "type" : "object" - }, - "timestamp" : { - "type" : "long" - }, - "url" : { - "type" : "keyword" - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 4adfc39f03ac66b0736e3f16a8684cd5b2511a43 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:49:16 +0200 Subject: [PATCH 16/38] [raw-launchpad] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/launchpad.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/raw/launchpad.py b/grimoire_elk/raw/launchpad.py index a393730ae..0e68c3db6 100644 --- a/grimoire_elk/raw/launchpad.py +++ b/grimoire_elk/raw/launchpad.py @@ -20,9 +20,34 @@ # from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} class LaunchpadOcean(ElasticOcean): """Launchpad Ocean feeder""" - pass + mapping = Mapping From 769a1c76ea2603f55c61876d167cf86d2fafeefe Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:50:14 +0200 Subject: [PATCH 17/38] [raw-mattermost] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/mattermost.py | 40 +++++++++------------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/grimoire_elk/raw/mattermost.py b/grimoire_elk/raw/mattermost.py index 9b83e8a0f..5daaa444c 100644 --- a/grimoire_elk/raw/mattermost.py +++ b/grimoire_elk/raw/mattermost.py @@ -29,40 +29,20 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.attachments.ts - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' - { - "dynamic":true, - "properties": { - "data": { - "properties": { - "attachments": { - "dynamic":false, - "properties": {} - }, - "channel_info": { - "properties": { - "latest": { - "dynamic": false, - "properties": {} - } - } - }, - "root": { - "dynamic":false, - "properties": {} - } - } - } - } + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } } - ''' + } + ''' return {"items": mapping} From 31f1a083e823d27c9327d1aae9d4363ce38ebfe0 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:51:48 +0200 Subject: [PATCH 18/38] [raw-mbox] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/mbox.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/grimoire_elk/raw/mbox.py b/grimoire_elk/raw/mbox.py index 479498e14..57f4606a8 100644 --- a/grimoire_elk/raw/mbox.py +++ b/grimoire_elk/raw/mbox.py @@ -34,26 +34,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.body (inmense field) - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "body": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 4f23dc5a06babdbd214b99a7d7abf9dc62ace87c Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:53:32 +0200 Subject: [PATCH 19/38] [raw-mediawiki] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/mediawiki.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/grimoire_elk/raw/mediawiki.py b/grimoire_elk/raw/mediawiki.py index 7b236b799..4f8fc0d6b 100644 --- a/grimoire_elk/raw/mediawiki.py +++ b/grimoire_elk/raw/mediawiki.py @@ -30,26 +30,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "revisions": { - "properties": { - "comment": { - "type": "text", - "index": true - } - } - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 738ee4dce829f94e0c81f895bc5dc4b901ae9e18 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Sat, 27 Jul 2019 16:55:03 +0200 Subject: [PATCH 20/38] [raw-meetup] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/meetup.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/grimoire_elk/raw/meetup.py b/grimoire_elk/raw/meetup.py index c6c482452..1f944636e 100644 --- a/grimoire_elk/raw/meetup.py +++ b/grimoire_elk/raw/meetup.py @@ -31,34 +31,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "comments": { - "properties": { - "comment": { - "type": "text", - "index": true - }, - "member": { - "properties": { - "bio": { - "type": "text", - "index": true - } - } - } - } - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 39d50635a0c557a6209971b7a5c9554b00db96d1 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:45:45 +0200 Subject: [PATCH 21/38] [raw-nntp] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/nntp.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/grimoire_elk/raw/nntp.py b/grimoire_elk/raw/nntp.py index 0136692bb..e6c0c6571 100644 --- a/grimoire_elk/raw/nntp.py +++ b/grimoire_elk/raw/nntp.py @@ -30,23 +30,19 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' - { - "dynamic":true, - "properties": { - "data": { - "dynamic":false, - "properties": {} - }, - "updated_on": { - "type": "long" - } - } + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } } - ''' + } + ''' return {"items": mapping} From dbd7341f08f5aaf5deba92709c6e48b0dc9e41a4 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:46:28 +0200 Subject: [PATCH 22/38] [raw-redmine] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/redmine.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/grimoire_elk/raw/redmine.py b/grimoire_elk/raw/redmine.py index c4980a0b5..5847f6b3a 100644 --- a/grimoire_elk/raw/redmine.py +++ b/grimoire_elk/raw/redmine.py @@ -29,34 +29,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.journals - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "journals": { - "dynamic": false, - "properties": {} - }, - "subject": { - "type": "text", - "index": true - }, - "description": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 77e685af7a4842483c9a0fb64b7737bfa9f8af32 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:46:44 +0200 Subject: [PATCH 23/38] [raw-phabricator] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/phabricator.py | 35 ++++++--------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/grimoire_elk/raw/phabricator.py b/grimoire_elk/raw/phabricator.py index b4a6b3ea0..4b2c22b1d 100644 --- a/grimoire_elk/raw/phabricator.py +++ b/grimoire_elk/raw/phabricator.py @@ -29,41 +29,18 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.transaction: has string arrays and dicts arrays - Specific type for: - * data.fields.priority.subpriority (float) - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' - { + { "dynamic":true, "properties": { "data": { - "properties": { - "transactions": { - "dynamic":false, - "properties": {} - }, - "fields": { - "properties": { - "priority" : { - "properties": { - "subpriority" : {"type": "float"} - } - }, - "description": { - "dynamic":false, - "properties": {} - } - } - } - } - } - } + "dynamic":false, + "properties": {} + } + } } ''' From 65a03577fd3779176c9319b51a247b5b3a1e9e2d Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:47:03 +0200 Subject: [PATCH 24/38] [raw-rss] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/rss.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/grimoire_elk/raw/rss.py b/grimoire_elk/raw/rss.py index 12688910e..ca6c5941c 100644 --- a/grimoire_elk/raw/rss.py +++ b/grimoire_elk/raw/rss.py @@ -30,30 +30,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "content": { - "dynamic":false, - "properties": {} - }, - "summary_detail": { - "dynamic":false, - "properties": {} - }, - "summary": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 6db84545bc59ef47b2f20b810bf27dbee3f24095 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:47:19 +0200 Subject: [PATCH 25/38] [raw-slack] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/slack.py | 44 +++++++++------------------------------ 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/grimoire_elk/raw/slack.py b/grimoire_elk/raw/slack.py index d73972b4e..165bb0044 100644 --- a/grimoire_elk/raw/slack.py +++ b/grimoire_elk/raw/slack.py @@ -29,44 +29,20 @@ class Mapping(BaseMapping): def get_elastic_mappings(es_major): """Get Elasticsearch mapping. - Non dynamic discovery of type for: - * data.attachments.ts - :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' - { - "dynamic":true, - "properties": { - "data": { - "properties": { - "attachments": { - "dynamic":false, - "properties": {} - }, - "blocks": { - "dynamic":false, - "properties": {} - }, - "channel_info": { - "properties": { - "latest": { - "dynamic": false, - "properties": {} - } - } - }, - "root": { - "dynamic":false, - "properties": {} - } - } - } - } + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } } - ''' + } + ''' return {"items": mapping} From 9d587fb508c2e9a6bf7f65bf549c42ea4e9c5bce Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:47:34 +0200 Subject: [PATCH 26/38] [raw-stackexchange] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/stackexchange.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/grimoire_elk/raw/stackexchange.py b/grimoire_elk/raw/stackexchange.py index 93fcaeca3..87f4c2d39 100644 --- a/grimoire_elk/raw/stackexchange.py +++ b/grimoire_elk/raw/stackexchange.py @@ -30,30 +30,17 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "body_markdown": { - "type": "text", - "index": true - }, - "answers": { - "properties": { - "body_markdown": { - "type": "text", - "index": true - } - } - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 90660233949eb5d904e392d4bb428a01eb6b8f50 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:47:51 +0200 Subject: [PATCH 27/38] [raw-supybot] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/supybot.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/grimoire_elk/raw/supybot.py b/grimoire_elk/raw/supybot.py index 2845847c1..7d8f684a6 100644 --- a/grimoire_elk/raw/supybot.py +++ b/grimoire_elk/raw/supybot.py @@ -22,9 +22,38 @@ from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} + + class SupybotOcean(ElasticOcean): """MediaWiki Ocean feeder""" + mapping = Mapping + @classmethod def get_perceval_params_from_url(cls, url): # In the url the uri and the data dir are included From dff989f5c9b6d1758a322ac2769c68fc4d6f8e38 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:48:08 +0200 Subject: [PATCH 28/38] [raw-telegram] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/telegram.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/raw/telegram.py b/grimoire_elk/raw/telegram.py index 247871776..2804517f4 100644 --- a/grimoire_elk/raw/telegram.py +++ b/grimoire_elk/raw/telegram.py @@ -20,9 +20,34 @@ # from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} class TelegramOcean(ElasticOcean): """Telegram Ocean feeder""" - pass + mapping = Mapping From cacb20e1c804ad33071f9c620ad8f9515b5317a5 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Mon, 29 Jul 2019 17:48:29 +0200 Subject: [PATCH 29/38] [raw-twitter] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/twitter.py | 63 ++----------------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/grimoire_elk/raw/twitter.py b/grimoire_elk/raw/twitter.py index 0e585b983..646e050ea 100644 --- a/grimoire_elk/raw/twitter.py +++ b/grimoire_elk/raw/twitter.py @@ -30,72 +30,15 @@ def get_elastic_mappings(es_major): """Get Elasticsearch mapping. :param es_major: major version of Elasticsearch, as string - :returns: dictionary with a key, 'items', with the mapping + :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, "properties": { "data": { - "properties": { - "user": { - "properties": { - "entities": { - "dynamic":false, - "properties": {} - } - } - }, - "hashtags": { - "properties": { - "indices": { - "dynamic":false, - "properties": {} - } - } - }, - "metadata": { - "dynamic":false, - "properties": {} - }, - "entities": { - "properties": { - "media": { - "dynamic":false, - "properties": {} - }, - "symbols": { - "dynamic":false, - "properties": {} - }, - "urls": { - "dynamic":false, - "properties": {} - }, - "user_mentions": { - "dynamic":false, - "properties": {} - } - } - }, - "extended_entities": { - "dynamic":false, - "properties": {} - }, - "extended_tweet": { - "dynamic":false, - "properties": {} - }, - "quoted_status": { - "dynamic":false, - "properties": {} - }, - "retweeted_status": { - "dynamic":false, - "properties": {} - } - } + "dynamic":false, + "properties": {} } } } From 25bcd9cb5b618a96442bcd8b33f04c097e46f0fb Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:54:55 +0200 Subject: [PATCH 30/38] [raw-remo] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/remo.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/grimoire_elk/raw/remo.py b/grimoire_elk/raw/remo.py index d8572736b..6d5543f60 100644 --- a/grimoire_elk/raw/remo.py +++ b/grimoire_elk/raw/remo.py @@ -36,20 +36,15 @@ def get_elastic_mappings(es_major): :param es_major: major version of Elasticsearch, as string :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "description": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 1cd233a16c0a3a5ae22d8961169632004493990e Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:52:31 +0200 Subject: [PATCH 31/38] [raw-finosmeetings] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/finosmeetings.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/grimoire_elk/raw/finosmeetings.py b/grimoire_elk/raw/finosmeetings.py index 433af6df3..18004cdfd 100644 --- a/grimoire_elk/raw/finosmeetings.py +++ b/grimoire_elk/raw/finosmeetings.py @@ -38,11 +38,8 @@ def get_elastic_mappings(es_major): "dynamic":true, "properties": { "data": { - "properties": { - "date": { - "type": "keyword" - } - } + "dynamic":false, + "properties": {} } } } From 060963750e910e01195d094c33467496c701e766 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:52:45 +0200 Subject: [PATCH 32/38] [raw-functest] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/functest.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/grimoire_elk/raw/functest.py b/grimoire_elk/raw/functest.py index 993387f92..85c582d9f 100644 --- a/grimoire_elk/raw/functest.py +++ b/grimoire_elk/raw/functest.py @@ -42,20 +42,12 @@ def get_elastic_mappings(es_major): mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "details": { - "dynamic":false, - "properties": {} - }, - "trust_indicator": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From f54381c65cd85afbb86ec4e0d917dc20ef894c3c Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:53:41 +0200 Subject: [PATCH 33/38] [raw-crates] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/crates.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/grimoire_elk/raw/crates.py b/grimoire_elk/raw/crates.py index 1158d8d25..0d206a2c6 100644 --- a/grimoire_elk/raw/crates.py +++ b/grimoire_elk/raw/crates.py @@ -36,16 +36,12 @@ def get_elastic_mappings(es_major): mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "versions_data": { - "dynamic":false, - "properties": {} - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 93843e9b3afe96b799e57e6cb568f0c6810d5bdc Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:54:13 +0200 Subject: [PATCH 34/38] [raw-graal] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/graal.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/grimoire_elk/raw/graal.py b/grimoire_elk/raw/graal.py index df998de38..d647acc5d 100644 --- a/grimoire_elk/raw/graal.py +++ b/grimoire_elk/raw/graal.py @@ -36,20 +36,12 @@ def get_elastic_mappings(es_major): """ mapping = ''' - { + { "dynamic":true, "properties": { "data": { - "properties": { - "message": { - "type": "text", - "index": true - }, - "analysis": { - "dynamic":false, - "properties": {} - } - } + "dynamic":false, + "properties": {} } } } From 7e55380ce9f8006136cfaf4c83d7edd28616247a Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:54:30 +0200 Subject: [PATCH 35/38] [raw-kitsune] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/kitsune.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/raw/kitsune.py b/grimoire_elk/raw/kitsune.py index 0c50bcf32..854eab718 100644 --- a/grimoire_elk/raw/kitsune.py +++ b/grimoire_elk/raw/kitsune.py @@ -20,9 +20,34 @@ # from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} class KitsuneOcean(ElasticOcean): """Kitsune Ocean feeder""" - pass + mapping = Mapping From f68c27340ec6279b8810e249686c81d7c62805eb Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:54:37 +0200 Subject: [PATCH 36/38] [raw-mozillaclub] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/mozillaclub.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/grimoire_elk/raw/mozillaclub.py b/grimoire_elk/raw/mozillaclub.py index c987ecb42..ef03cba3f 100644 --- a/grimoire_elk/raw/mozillaclub.py +++ b/grimoire_elk/raw/mozillaclub.py @@ -32,24 +32,15 @@ def get_elastic_mappings(es_major): :param es_major: major version of Elasticsearch, as string :returns: dictionary with a key, 'items', with the mapping """ - mapping = ''' { "dynamic":true, - "properties": { - "data": { - "properties": { - "Event Creations": { - "type": "text", - "index": true - }, - "Event Description": { - "type": "text", - "index": true - } - } - } + "properties": { + "data": { + "dynamic":false, + "properties": {} } + } } ''' From 8493671d2393278cc63dd736bf552266de4c9a8a Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 11:54:46 +0200 Subject: [PATCH 37/38] [raw-puppetforge] Change mapping This code changes the mapping of the raw items. The new mapping is the following: ``` { "dynamic":true, "properties": { "data": { "dynamic":false, "properties": {} } } } ``` Signed-off-by: Valerio Cosentino --- grimoire_elk/raw/puppetforge.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/raw/puppetforge.py b/grimoire_elk/raw/puppetforge.py index bd5965d55..eb09e7ee9 100644 --- a/grimoire_elk/raw/puppetforge.py +++ b/grimoire_elk/raw/puppetforge.py @@ -20,9 +20,34 @@ # from .elastic import ElasticOcean +from ..elastic_mapping import Mapping as BaseMapping + + +class Mapping(BaseMapping): + + @staticmethod + def get_elastic_mappings(es_major): + """Get Elasticsearch mapping. + + :param es_major: major version of Elasticsearch, as string + :returns: dictionary with a key, 'items', with the mapping + """ + mapping = ''' + { + "dynamic":true, + "properties": { + "data": { + "dynamic":false, + "properties": {} + } + } + } + ''' + + return {"items": mapping} class PuppetForgeOcean(ElasticOcean): """PuppetForge Ocean feeder""" - pass + mapping = Mapping From 04f2b3848833c525660e65233996570077517c7c Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 31 Mar 2020 13:04:06 +0200 Subject: [PATCH 38/38] [tests] Update test_all_properties in test_elastic This code updates the expected results of the test test_all_properties. This change is needed to match the new mappings, which doesn't index the `data` attribute. Signed-off-by: Valerio Cosentino --- tests/test_elastic.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_elastic.py b/tests/test_elastic.py index 799e8b064..c49ac85e5 100644 --- a/tests/test_elastic.py +++ b/tests/test_elastic.py @@ -552,11 +552,8 @@ def test_all_properties(self): expected_properties = { 'data': { - 'properties': { - 'message': { - 'type': 'text' - } - } + 'type': 'object', + 'dynamic': 'false' } } elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)