From 94082929f1cdce30f51346192cd4fe4bdfac50e8 Mon Sep 17 00:00:00 2001 From: Chris Sampson Date: Mon, 3 Feb 2025 21:35:55 +0000 Subject: [PATCH] NIFI-14220 allow use of Expression Language for PutElasticsearchJson and PutElasticsearchRecord boolean fields Signed-off-by: Pierre Villard This closes #9688. --- .../elasticsearch/PutElasticsearchJson.java | 4 ++-- .../elasticsearch/PutElasticsearchRecord.java | 3 --- .../PutElasticsearchJsonTest.java | 24 ++++++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java index ec7495c923fa..27e3b061d1fd 100644 --- a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java +++ b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java @@ -110,14 +110,14 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch { .name("put-es-json-scripted-upsert") .displayName("Scripted Upsert") .description("Whether to add the scripted_upsert flag to the Upsert Operation. " + - "Forces Elasticsearch to execute the Script whether or not the document exists, defaults to false. " + + "If true, forces Elasticsearch to execute the Script whether or not the document exists, defaults to false. " + "If the Upsert Document provided (from FlowFile content) will be empty, but sure to set the " + CLIENT_SERVICE.getDisplayName() + " controller service's " + ElasticSearchClientService.SUPPRESS_NULLS.getDisplayName() + " to " + ElasticSearchClientService.NEVER_SUPPRESS.getDisplayName() + " or no \"upsert\" doc will be, " + "included in the request to Elasticsearch and the operation will not create a new document for the script " + "to execute against, resulting in a \"not_found\" error") .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .defaultValue("false") .build(); diff --git a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java index 9e19965d0a4b..0c6aab182ee0 100644 --- a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java +++ b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java @@ -166,7 +166,6 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch { .displayName("Retain ID (Record Path)") .description("Whether to retain the existing field used as the ID Record Path.") .addValidator(StandardValidators.BOOLEAN_VALIDATOR) - .allowableValues("true", "false") .defaultValue("false") .required(false) .dependsOn(ID_RECORD_PATH) @@ -242,7 +241,6 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch { .displayName("Retain @timestamp (Record Path)") .description("Whether to retain the existing field used as the @timestamp Record Path.") .addValidator(StandardValidators.BOOLEAN_VALIDATOR) - .allowableValues("true", "false") .defaultValue("false") .required(false) .dependsOn(AT_TIMESTAMP_RECORD_PATH) @@ -268,7 +266,6 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch { "and the error related to the first record within the FlowFile added to the FlowFile as \"elasticsearch.bulk.error\". " + "If \"" + NOT_FOUND_IS_SUCCESSFUL.getDisplayName() + "\" is \"false\" then records associated with \"not_found\" " + "Elasticsearch document responses will also be send to the \"" + REL_ERRORS.getName() + "\" relationship.") - .allowableValues("true", "false") .defaultValue("false") .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .required(false) diff --git a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java index 30cdd6aafd10..621c223f31f9 100644 --- a/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java +++ b/nifi-extension-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.java @@ -111,8 +111,12 @@ void basicTest(final int failure, final int retry, final int successful) { } void basicTest(final int failure, final int retry, final int successful, final Consumer> consumer) { + basicTest(failure, retry, successful, consumer, Collections.emptyMap()); + } + + void basicTest(final int failure, final int retry, final int successful, final Consumer> consumer, final Map attr) { clientService.setEvalConsumer(consumer); - basicTest(failure, retry, successful, Collections.emptyMap()); + basicTest(failure, retry, successful, attr); } void basicTest(final int failure, final int retry, final int successful, final Map attr) { @@ -289,6 +293,24 @@ void simpleTestWithScriptedUpsert() { basicTest(0, 0, 1, consumer); } + @Test + void simpleTestWithScriptedUpsertEL() { + runner.setProperty(PutElasticsearchJson.SCRIPT, script); + runner.setProperty(PutElasticsearchJson.DYNAMIC_TEMPLATES, dynamicTemplates); + runner.setProperty(PutElasticsearchJson.INDEX_OP, IndexOperationRequest.Operation.Upsert.getValue().toLowerCase()); + runner.setProperty(PutElasticsearchJson.SCRIPTED_UPSERT, "${scripted}"); + final Consumer> consumer = (final List items) -> { + final long scriptCount = items.stream().filter(item -> item.getScript().equals(expectedScript)).count(); + final long trueScriptedUpsertCount = items.stream().filter(IndexOperationRequest::isScriptedUpsert).count(); + final long dynamicTemplatesCount = items.stream().filter(item -> item.getDynamicTemplates().equals(expectedDynamicTemplate)).count(); + + assertEquals(1L, scriptCount); + assertEquals(1L, trueScriptedUpsertCount); + assertEquals(1L, dynamicTemplatesCount); + }; + basicTest(0, 0, 1, consumer, Map.of("scripted", "true")); + } + @Test void testNonJsonScript() { runner.setProperty(PutElasticsearchJson.SCRIPT, "not-json");