From 6dbf37bb19fd3e49dc94a6c237ca940bfbe068b7 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 01/30] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/expression/json/JsonFunctions.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064..56ea5f10c3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,10 +5,20 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; From b8c6d68a4f340464c84c7c5afa2cb98b9b1c7537 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 02/30] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ 2 files changed, 77 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 0000000000..62e7868b41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 0000000000..b825254b11 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} From afb668c74bd1a2f140ec16c18ee2b38cf8fbf1ef Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 03/30] addressed pr comments Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b11..fb97836d5e 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From 54ef1835c6d9cc4766aa57c9bb05a017ae5fca6d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 04/30] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 ++ .../org/opensearch/sql/expression/datetime/ExtractTest.java | 3 +++ .../org/opensearch/sql/expression/datetime/YearweekTest.java | 3 +++ 3 files changed, 8 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ad15dadfb7..e8a4fc81fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1231,6 +1232,7 @@ public void testWeekFormats( } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index d7635de610..02645eca06 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,6 +11,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -91,6 +93,7 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index d944f7c85c..4de5d3a341 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,6 +16,8 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -100,6 +102,7 @@ public void testYearweekWithoutMode() { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From d84139489e107edb6c1a12cafc8872b5eb90af76 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 05/30] removed unused dependencies Signed-off-by: Kenrick Yap --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 56ea5f10c3..05d5187a14 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,9 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 25fb527882104b9083e0cf0c614b24a1586336f4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 06/30] linting Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 5 ++ .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 5 files changed, 50 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 05d5187a14..2f83262922 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -33,4 +33,9 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index e8a4fc81fb..eb5074f4f7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1232,7 +1232,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02645eca06..925c1f1b7c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -93,7 +92,8 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 4de5d3a341..daae8b1ff5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -102,7 +101,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41..f02750147d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From 4a20d0870a58c8273f43f837a5382aaeff80fabc Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 07/30] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index eb5074f4f7..ab391c6834 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1231,9 +1231,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 925c1f1b7c..1dfa3908e6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -92,8 +92,6 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index daae8b1ff5..266994c046 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -100,9 +100,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From fdc4729de3b1c8310245cec8b469b41b96ec95d6 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 08/30] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ab391c6834..115898e349 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 1dfa3908e6..d7635de610 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 266994c046..ee4df23be4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 707a0b9693b6293cf0261311b852ecb9f91f689f Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 09/30] nit Signed-off-by: Kenrick Yap --- .../indexDefinitions/json_test_index_mappping.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json deleted file mode 100644 index fb97836d5e..0000000000 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mappings": { - "properties": { - "test_name": { - "type": "keyword" - }, - "json_string": { - "type": "text" - } - } - } -} From 4f28211701c7ceb7a974abc0dd377b0acc66f500 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 10/30] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d..501ef9448e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 9ec633599d14814284b368d9294263b3760213c9 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 11/30] fixed integ test Signed-off-by: Kenrick Yap --- .../opensearch/sql/ppl/JsonFunctionIT.java | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java deleted file mode 100644 index 501ef9448e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -public class JsonFunctionsIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); - } -} From 3324e661953a855fc46dce474001796d4fdf33ac Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Jan 2025 22:24:54 -0800 Subject: [PATCH 12/30] SQL: adding error case unit tests for json_valid Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2..557a2ab9e1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -46,6 +46,9 @@ public void json_valid_returns_false() { public void json_valid_throws_ExpressionEvaluationException() { assertThrows( ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); + + // caught by nullMissingHandling and returns null + assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); } @Test From 7123c350f0164a0dda45f3daa8aade84a9b28abf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 13/30] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 2f83262922..b0554fd210 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -15,7 +15,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -33,9 +32,4 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); - } } From dbca9915557246f4f4728f8b92fd5046efecec0d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 14/30] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ast/expression/Cast.java | 2 + .../org/opensearch/sql/expression/DSL.java | 8 ++ .../function/BuiltinFunctionName.java | 6 +- .../operator/convert/TypeCastOperators.java | 9 ++ .../org/opensearch/sql/utils/JsonUtils.java | 68 ++++++++- .../expression/json/JsonFunctionsTest.java | 132 ++++++++++++++++++ .../convert/TypeCastOperatorTest.java | 103 ++++++++++++++ docs/user/ppl/functions/json.rst | 25 ++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 28 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 ++ 11 files changed, 386 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedead..854ba0ed69 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163..a50e3d1470 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -843,6 +843,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } @@ -973,6 +977,10 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb7..ef9a872974 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -235,6 +235,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ @@ -259,7 +260,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89a..d6518e47d9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,10 +17,12 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -183,6 +186,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e..9835db9193 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,24 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -23,9 +37,57 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** Converts a JSON encoded string to an Expression object. */ + public static ExprValue castJson(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 557a2ab9e1..809d82941b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,17 +7,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -64,4 +107,93 @@ private ExprValue execute(ExprValue jsonString) { FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); return exp.valueOf(); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb47..27bd806c11 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -389,4 +396,100 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + + String emptyObject = "{}"; + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), + DSL.castJson(DSL.literal(emptyObject)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fa704b6c65..bbc972ba73 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,28 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb..33c54ff60b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -52,4 +52,32 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval json=cast(json_string to json) | fields json", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ea7e060d9c..9d6707a872 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,6 +143,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 999c5d9c87..74b05dc28b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -409,6 +409,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +420,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +702,10 @@ positionFunctionName : POSITION ; +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -963,4 +969,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 7df87cbfbd1f94986bcdaacd022309f22287ffbf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 15/30] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 6 ++---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ef9a872974..cd309a712d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -260,10 +261,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9d6707a872..31a668be00 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,7 +143,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; @@ -335,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 74b05dc28b..5d1f29614d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -870,6 +870,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE @@ -969,6 +970,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From cd45fcc628f6ad57a1de1c7b479dde6fbe73cfc8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 16/30] Fix testes Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 8 ++++++ .../operator/convert/TypeCastOperators.java | 4 +-- .../org/opensearch/sql/utils/JsonUtils.java | 4 +-- .../opensearch/sql/ppl/JsonFunctionsIT.java | 25 ++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b0554fd210..b1380eb017 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -26,10 +27,17 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index d6518e47d9..cfd570ab89 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -44,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -189,7 +189,7 @@ private static DefaultFunctionResolver castToIp() { private static DefaultFunctionResolver castToJson() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_JSON.getName(), - impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } private static DefaultFunctionResolver castToDate() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 9835db9193..0579eda933 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -44,11 +44,11 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } /** Converts a JSON encoded string to an Expression object. */ - public static ExprValue castJson(String json) { + public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json); + jsonNode = objectMapper.readTree(json.stringValue()); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 33c54ff60b..465443c096 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -58,15 +60,15 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | eval json=cast(json_string to json) | fields json", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } @Test @@ -76,8 +78,13 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } } From 6f5dc07f749975bb60b6016baa25357633179e54 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 9 Jan 2025 09:33:34 -0800 Subject: [PATCH 17/30] spotless Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 465443c096..015333cdea 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -60,13 +60,14 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -78,12 +79,14 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } From 0aae36e94f765094ea66122197885124e4462153 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:13:01 -0800 Subject: [PATCH 18/30] Fix tests Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 13 +++++++++---- integ-test/src/test/resources/json_test.json | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 015333cdea..a636273107 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,8 +12,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -66,8 +68,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -83,10 +86,12 @@ public void test_json() throws IOException { + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e..dae01ea4ce 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From b225f2886513637cd8316471a56e800a303fa2b8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 19/30] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 176 +++++++++--------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 9 +- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 809d82941b..1d06e1b2bd 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -108,92 +108,92 @@ private ExprValue execute(ExprValue jsonString) { return exp.valueOf(); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index a636273107..eecf7fb338 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,7 +12,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -68,7 +67,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), @@ -89,7 +90,9 @@ public void test_json() throws IOException { JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), From 78af4f8f636d26ba05f1a32044fd942076ef94ba Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 20/30] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 1 + .../expression/json/JsonFunctionsTest.java | 30 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b1380eb017..64f84c44f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 0579eda933..c5f7031b13 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1d06e1b2bd..f3b17241fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,24 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,28 +27,10 @@ import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; From b84282aa2fbf0459a13591ea85041a7ee282ef4d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:52:02 -0800 Subject: [PATCH 21/30] clean up unit tests Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index f3b17241fb..4ba8b69e1c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -36,23 +36,16 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonNestedObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); - private static final ExprValue JsonObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = - ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); - assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); + assertEquals( + LITERAL_FALSE, + DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"))) + .valueOf()); + assertEquals( + LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf()); } @Test @@ -66,16 +59,35 @@ public void json_valid_throws_ExpressionEvaluationException() { @Test public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); - } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); } @Test From 1e2328606c375b166ef2a7335d7a89e0c37e80b7 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 11:07:48 -0800 Subject: [PATCH 22/30] Add casting from undefined Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index cfd570ab89..960f6743c3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -122,7 +122,9 @@ private static DefaultFunctionResolver castToInt() { impl( nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) + ); } private static DefaultFunctionResolver castToLong() { @@ -136,7 +138,9 @@ private static DefaultFunctionResolver castToLong() { impl( nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) + ); } private static DefaultFunctionResolver castToFloat() { @@ -150,7 +154,9 @@ private static DefaultFunctionResolver castToFloat() { impl( nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) + ); } private static DefaultFunctionResolver castToDouble() { @@ -164,7 +170,9 @@ private static DefaultFunctionResolver castToDouble() { impl( nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) + ); } private static DefaultFunctionResolver castToBoolean() { @@ -176,7 +184,9 @@ private static DefaultFunctionResolver castToBoolean() { STRING), impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), - impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN)); + impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) + ); } private static DefaultFunctionResolver castToIp() { From 343f5a2970948393caafd1b6b0b243fd7bf5640a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 14:48:51 -0800 Subject: [PATCH 23/30] Add cast to scalar from undefined expression Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 23 +++--- .../convert/TypeCastOperatorTest.java | 53 ++++++++++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 71 +++++++++++++++++++ integ-test/src/test/resources/json_test.json | 14 +++- 4 files changed, 145 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 960f6743c3..1a43ef14e1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,9 +92,8 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), - BYTE, - BOOLEAN)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), + impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); } private static DefaultFunctionResolver castToShort() { @@ -108,7 +107,8 @@ private static DefaultFunctionResolver castToShort() { impl( nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)), SHORT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED)); } private static DefaultFunctionResolver castToInt() { @@ -123,8 +123,7 @@ private static DefaultFunctionResolver castToInt() { nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, BOOLEAN), - impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED)); } private static DefaultFunctionResolver castToLong() { @@ -139,8 +138,7 @@ private static DefaultFunctionResolver castToLong() { nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, BOOLEAN), - impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED)); } private static DefaultFunctionResolver castToFloat() { @@ -155,8 +153,7 @@ private static DefaultFunctionResolver castToFloat() { nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN), - impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED)); } private static DefaultFunctionResolver castToDouble() { @@ -171,8 +168,7 @@ private static DefaultFunctionResolver castToDouble() { nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, BOOLEAN), - impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED)); } private static DefaultFunctionResolver castToBoolean() { @@ -185,8 +181,7 @@ private static DefaultFunctionResolver castToBoolean() { impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), - impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED)); } private static DefaultFunctionResolver castToIp() { diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 27bd806c11..4ad28d76d0 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -165,6 +165,15 @@ void castBooleanToShort() { assertEquals(new ExprShortValue(0), expression.valueOf()); } + @Test + void castUndefinedToShort() { + Short value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castShort(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(value), expression.valueOf()); + } + @Test void castBooleanToInt() { FunctionExpression expression = DSL.castInt(DSL.literal(true)); @@ -176,6 +185,15 @@ void castBooleanToInt() { assertEquals(new ExprIntegerValue(0), expression.valueOf()); } + @Test + void castUndefinedToInt() { + Integer value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castInt(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(INTEGER, expression.type()); + assertEquals(new ExprIntegerValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToLong({0})") @MethodSource({"numberData"}) void castToLong(ExprValue value) { @@ -208,6 +226,15 @@ void castBooleanToLong() { assertEquals(new ExprLongValue(0), expression.valueOf()); } + @Test + void castUndefinedToLong() { + Long value = 42l; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castLong(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(LONG, expression.type()); + assertEquals(new ExprLongValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToFloat({0})") @MethodSource({"numberData"}) void castToFloat(ExprValue value) { @@ -240,6 +267,15 @@ void castBooleanToFloat() { assertEquals(new ExprFloatValue(0), expression.valueOf()); } + @Test + void castUndefinedToFloat() { + Float value = 23.45f; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castFloat(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(FLOAT, expression.type()); + assertEquals(new ExprFloatValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToDouble({0})") @MethodSource({"numberData"}) void castToDouble(ExprValue value) { @@ -272,6 +308,15 @@ void castBooleanToDouble() { assertEquals(new ExprDoubleValue(0), expression.valueOf()); } + @Test + void castUndefinedToDouble() { + Double value = 23.45e5; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + assertEquals(DOUBLE, expression.type()); + assertEquals(new ExprDoubleValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToBoolean({0})") @MethodSource({"numberData"}) void castToBoolean(ExprValue value) { @@ -301,6 +346,14 @@ void castBooleanToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } + @Test + void castUndefinedToBoolean() { + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); + assertEquals(BOOLEAN, expression.type()); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); + } + @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index eecf7fb338..e837057b68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -73,6 +73,11 @@ public void test_cast_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar float", 12.34f), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } @@ -96,6 +101,72 @@ public void test_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar long", 42), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } + + @Test + public void test_cast_json_scalar_to_type() throws IOException { + // cast to integer + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as int) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "integer")); + verifyDataRows(result, rows("json scalar int", 1234)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as long) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "long")); + verifyDataRows(result, rows("json scalar int", 1234l)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar float' | " + + "eval casted=cast(json(json_string) as float) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "float")); + verifyDataRows(result, rows("json scalar float", 12.34f)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar double' | " + + "eval casted=cast(json(json_string) as double) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "double")); + verifyDataRows(result, rows("json scalar double", 2.99792458e8)); + + result = + executeQuery( + String.format( + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); + verifyDataRows( + result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); + } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index dae01ea4ce..acab339c01 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -7,8 +7,18 @@ {"index":{"_id":"3"}} {"test_name":"json scalar string", "json_string":"\"abc\""} {"index":{"_id":"4"}} -{"test_name":"json empty string","json_string":""} +{"test_name":"json scalar int", "json_string":"1234"} {"index":{"_id":"5"}} -{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"test_name":"json scalar float", "json_string":"12.34"} {"index":{"_id":"6"}} +{"test_name":"json scalar double", "json_string":"2.99792458e8"} +{"index":{"_id":"7"}} +{"test_name":"json scalar boolean true", "json_string":"true"} +{"index":{"_id":"8"}} +{"test_name":"json scalar boolean false", "json_string":"false"} +{"index":{"_id":"9"}} +{"test_name":"json empty string", "json_string":""} +{"index":{"_id":"10"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"11"}} {"test_name":"json null", "json_string":null} From e8b6df3e6ab0ce616b0ba1c9868dca19044cdfad Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 15:34:17 -0800 Subject: [PATCH 24/30] Add test for missing/null Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 4ba8b69e1c..1cbd71b806 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -74,6 +74,7 @@ public void json_valid_returns_true() { // test json scalars are valid "\"abc\"", "1234", + "12.34", "true", "false", "null", @@ -155,10 +156,16 @@ void json_returnsScalar() { assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + + assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); assertEquals( From ab9be7533c3d497b5e13c6cc36337312264247c4 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 09:58:42 -0800 Subject: [PATCH 25/30] Clean up merge conflicts Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 8 ++-- .../sql/expression/json/JsonFunctions.java | 6 --- .../operator/convert/TypeCastOperators.java | 5 ++- .../org/opensearch/sql/utils/JsonUtils.java | 20 +++++++++- .../expression/json/JsonFunctionsTest.java | 38 ++++++++++--------- .../convert/TypeCastOperatorTest.java | 14 ++++--- .../opensearch/sql/ppl/JsonFunctionsIT.java | 7 +++- 7 files changed, 59 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index a50e3d1470..966077c790 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression stringToJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } @@ -977,10 +981,6 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 64f84c44f7..75f134aa4e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,6 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 1a43ef14e1..c1391ac9ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,8 +92,9 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), - impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), + BYTE, + BOOLEAN)); } private static DefaultFunctionResolver castToShort() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c5f7031b13..63cbaf4a99 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -12,9 +12,11 @@ import java.util.List; import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -27,7 +29,7 @@ public class JsonUtils { * Checks if given JSON string can be parsed as valid JSON. * * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). - * @return true if the string can be parsed as valid JSON, else false. + * @return true if the string can be parsed as valid JSON, else false (including null or missing). */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); @@ -44,7 +46,21 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } } - /** Converts a JSON encoded string to an Expression object. */ + /** + * Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @return ExprValue returns an expression that best represents the provided JSON-encoded string. + *
    + *
  1. {@link ExprTupleValue} if the JSON is an object + *
  2. {@link ExprCollectionValue} if the JSON is an array + *
  3. {@link ExprDoubleValue} if the JSON is a floating-point number scalar + *
  4. {@link ExprIntegerValue} if the JSON is an integral number scalar + *
  5. {@link ExprStringValue} if the JSON is a string scalar + *
  6. {@link ExprBooleanValue} if the JSON is a boolean scalar + *
  7. {@link ExprNullValue} if the JSON is null, empty, or invalid + *
+ */ public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1cbd71b806..26b8ed89fd 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -51,10 +51,8 @@ public void json_valid_returns_false() { @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( - ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); - - // caught by nullMissingHandling and returns null - assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + ExpressionEvaluationException.class, + () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } @Test @@ -115,12 +113,16 @@ void json_returnsJsonObject() { ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); // exercise - exp = DSL.json_function(DSL.literal(objectJson)); + exp = DSL.stringToJson(DSL.literal(objectJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); } @Test @@ -141,35 +143,35 @@ void json_returnsJsonArray() { LITERAL_NULL)); // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); + exp = DSL.stringToJson(DSL.literal(arrayJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); } @Test void json_returnsScalar() { assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); } @Test @@ -181,7 +183,7 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 4ad28d76d0..ff0c8bcc01 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -312,7 +312,7 @@ void castBooleanToDouble() { void castUndefinedToDouble() { Double value = 23.45e5; // json cast is an UNDEFINED type expression - FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value.toString()))); assertEquals(DOUBLE, expression.type()); assertEquals(new ExprDoubleValue(value), expression.valueOf()); } @@ -480,6 +480,10 @@ void castJson_returnsJsonObject() { var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty-object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.castJson(DSL.literal("{}")).valueOf()); } @Test @@ -506,6 +510,9 @@ void castJson_returnsJsonArray() { var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.castJson(DSL.literal("[]")).valueOf()); } @Test @@ -525,11 +532,6 @@ void castJson_returnsScalar() { String empty = ""; assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); - - String emptyObject = "{}"; - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), - DSL.castJson(DSL.literal(emptyObject)).valueOf()); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index e837057b68..9ed61c33a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -40,6 +40,11 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), + rows("json scalar int"), + rows("json scalar float"), + rows("json scalar double"), + rows("json scalar boolean true"), + rows("json scalar boolean false"), rows("json empty string")); } @@ -102,7 +107,7 @@ public void test_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json scalar int", 1234), - rows("json scalar long", 42), + rows("json scalar float", 12.34), rows("json scalar double", 2.99792458e8), rows("json scalar boolean true", true), rows("json scalar boolean false", false), From 788be9d6e91e4c647cc14859e3b66fab40c5324d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:24:03 -0800 Subject: [PATCH 26/30] Fix jacoco coverage Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 26b8ed89fd..1f39b5be9b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -164,6 +164,7 @@ void json_returnsScalar() { assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); From a9721bfb2269524e83ba381eb32ed3e468c8fc5f Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:44:23 -0800 Subject: [PATCH 27/30] Move to Switch by json type Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/utils/JsonUtils.java | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 63cbaf4a99..6120d11bb5 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -75,36 +75,33 @@ public static ExprValue castJson(ExprValue json) { } private static ExprValue processJsonNode(JsonNode jsonNode) { - if (jsonNode.isFloatingPointNumber()) { - return new ExprDoubleValue(jsonNode.asDouble()); + switch (jsonNode.getNodeType()) { + case ARRAY: + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + case OBJECT: + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + case STRING: + return new ExprStringValue(jsonNode.asText()); + case NUMBER: + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + return new ExprIntegerValue(jsonNode.asLong()); + case BOOLEAN: + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; + default: + // in all other cases, return null + return LITERAL_NULL; } - if (jsonNode.isIntegralNumber()) { - return new ExprIntegerValue(jsonNode.asLong()); - } - if (jsonNode.isBoolean()) { - return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; - } - if (jsonNode.isTextual()) { - return new ExprStringValue(jsonNode.asText()); - } - if (jsonNode.isArray()) { - List elements = new LinkedList<>(); - for (var iter = jsonNode.iterator(); iter.hasNext(); ) { - jsonNode = iter.next(); - elements.add(processJsonNode(jsonNode)); - } - return new ExprCollectionValue(elements); - } - if (jsonNode.isObject()) { - Map values = new LinkedHashMap<>(); - for (var iter = jsonNode.fields(); iter.hasNext(); ) { - Map.Entry entry = iter.next(); - values.put(entry.getKey(), processJsonNode(entry.getValue())); - } - return ExprTupleValue.fromExprValueMap(values); - } - - // in all other cases, return null - return LITERAL_NULL; } } From 7eb5979b4aad7badfe03dceede7b0789176dac8a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 20 Jan 2025 10:02:13 -0800 Subject: [PATCH 28/30] Add JSON_OBJECT to ppl Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 4 + .../function/BuiltinFunctionName.java | 1 + .../sql/expression/json/JsonFunctions.java | 76 +++++++++++++++++++ .../expression/json/JsonFunctionsTest.java | 76 +++++++++++++++++++ docs/user/ppl/functions/json.rst | 40 ++++++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 36 ++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 +++ 8 files changed, 241 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 966077c790..720da0979e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -691,6 +691,10 @@ public static FunctionExpression stringToJson(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); } + public static FunctionExpression jsonObject(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_OBJECT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index cd309a712d..a6832b366f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 75f134aa4e..885380578c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,15 +7,33 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.expression.DSL.jsonObject; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.utils.JsonUtils; @UtilityClass @@ -23,6 +41,7 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); + repository.register(jsonObject()); } private DefaultFunctionResolver jsonValid() { @@ -35,4 +54,61 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + /** Creates a JSON Object/tuple expr from a given list of kv pairs. */ + private static FunctionResolver jsonObject() { + return new FunctionResolver() { + @Override + public FunctionName getFunctionName() { + return BuiltinFunctionName.JSON_OBJECT.getName(); + } + + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + List paramList = unresolvedSignature.getParamTypeList(); + // check that we got an even number of arguments + if (paramList.size() % 2 != 0) { + throw new SemanticCheckException( + String.format( + "Expected an even number of arguments but instead got %d arguments", + paramList.size())); + } + + // check that each "key" argument (of key-value pair) is a string + for (int i = 0; i < paramList.size(); i = i + 2) { + ExprType paramType = paramList.get(i); + if (!ExprCoreType.STRING.equals(paramType)) { + throw new SemanticCheckException( + String.format( + "Expected type %s instead of %s for parameter #%d", + ExprCoreType.STRING, paramType.typeName(), i + 1)); + } + } + + // return the unresolved signature and function builder + return Pair.of( + unresolvedSignature, + (functionProperties, arguments) -> + new FunctionExpression(getFunctionName(), arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + LinkedHashMap tupleValues = new LinkedHashMap<>(); + Iterator iter = getArguments().iterator(); + while (iter.hasNext()) { + tupleValues.put( + iter.next().valueOf(valueEnv).stringValue(), + iter.next().valueOf(valueEnv)); + } + return ExprTupleValue.fromExprValueMap(tupleValues); + } + + @Override + public ExprType type() { + return STRUCT; + } + }); + } + }; + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1f39b5be9b..0b1bd809a9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -188,4 +188,80 @@ void json_returnsSemanticCheckException() { assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + + @Test + public void json_object_returns_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + // TODO: requires json_array() + // objectMap.put( + // "arr", + // new ExprCollectionValue( + // List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), + // ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = + DSL.jsonObject( + DSL.literal("foo"), DSL.literal("foo"), + DSL.literal("fuzz"), DSL.literal(true), + DSL.literal("bar"), DSL.literal(1234), + DSL.literal("bar2"), DSL.literal(12.34), + DSL.literal("baz"), DSL.literal(LITERAL_NULL), + DSL.literal("obj"), DSL.jsonObject(DSL.literal("internal"), DSL.literal("value"))); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_returns_empty_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.jsonObject(); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_throws_SemanticCheckException() { + // wrong number of arguments + assertThrows( + SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("only one")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject(DSL.literal("one"), DSL.literal("two"), DSL.literal("three")).valueOf()); + + // key argument is not a string + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonObject(DSL.literal(1234), DSL.literal("two")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject( + DSL.literal("one"), DSL.literal(true), DSL.literal(true), DSL.literal("four")) + .valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index bbc972ba73..22ac2cd241 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -59,3 +59,43 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+------------------------------+---------------+ + +JSON_OBJECT +----------- + +Description +>>>>>>>>>>> + +Usage: `json_object(, [, , ]...)` returns a JSON object from key-value pairs. + +Argument type: +- A \ must be STRING. +- A \ can be a scalar, another json object, or json array type. Note: scalar fields will be treated as single-value. Use `json_array` to construct an array value from a multi-value. + +Return type: STRUCT + +Example: + + os> source=people | eval result = json_object('key', 123.45) | fields result + fetched rows / total rows = 1/1 + +------------------+ + | result | + +------------------+ + | {"key":123.45} | + +------------------+ + + os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result + fetched rows / total rows = 1/1 + +------------------------------+ + | result | + +------------------------------+ + | {"outer":{"inner":123.45}} | + +------------------------------+ + + os> source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + fetched rows / total rows = 1/1 + +------------------------------------------------+ + | result | + +------------------------------------------------+ + | {"array_doc":[123.45, "string", true, null]} | + +------------------------------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9ed61c33a4..ffff02409d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -166,12 +166,42 @@ public void test_cast_json_scalar_to_type() throws IOException { result = executeQuery( String.format( - "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" - + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" - + " test_name, casted", + "source=%s | " + + "where test_name='json scalar boolean true' OR test_name='json scalar boolean false' | " + + "eval casted=cast(json(json_string) as boolean) | " + + "fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); verifyDataRows( result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); } + + @Test + public void test_json_object() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject(Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), + rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), + rows("json scalar string", Map.of("key", "abc")), + rows("json scalar int", Map.of("key", 1234)), + rows("json scalar float", Map.of("key", 12.34)), + rows("json scalar double", Map.of("key", 2.99792458e8)), + rows("json scalar boolean true", Map.of("key", true)), + rows("json scalar boolean false", Map.of("key", false)), + rows("json empty string", Map.of()) + ); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 31a668be00..4e812ea9ca 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -335,6 +335,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 5d1f29614d..55b629a9cc 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -310,6 +310,7 @@ valueExpression | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | jsonObjectFunction # jsonObjectFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr ; @@ -324,6 +325,10 @@ positionFunction : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; +jsonObjectFunction + : jsonObjectFunctionName LT_PRTHS (functionArg COMMA functionArg (COMMA functionArg COMMA functionArg)*)? RT_PRTHS + ; + booleanExpression : booleanFunctionCall ; @@ -420,6 +425,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonObjectFunctionName | jsonFunctionName ; @@ -702,6 +708,10 @@ positionFunctionName : POSITION ; +jsonObjectFunctionName + : JSON_OBJECT + ; + jsonFunctionName : JSON ; From d29a53e4f91ccbe38fad55f8ad38cbab54e8628c Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 20 Jan 2025 10:47:17 -0800 Subject: [PATCH 29/30] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ppl/JsonFunctionsIT.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index ffff02409d..3d57edcb9a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -166,10 +166,9 @@ public void test_cast_json_scalar_to_type() throws IOException { result = executeQuery( String.format( - "source=%s | " + - "where test_name='json scalar boolean true' OR test_name='json scalar boolean false' | " + - "eval casted=cast(json(json_string) as boolean) | " + - "fields test_name, casted", + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); verifyDataRows( @@ -183,16 +182,17 @@ public void test_json_object() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | " + - "eval obj=json_object('key', json(json_string)) | " + - "fields test_name, obj", + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); verifyDataRows( result, rows( "json nested object", - new JSONObject(Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + new JSONObject( + Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), rows("json scalar string", Map.of("key", "abc")), @@ -201,7 +201,6 @@ public void test_json_object() throws IOException { rows("json scalar double", Map.of("key", 2.99792458e8)), rows("json scalar boolean true", Map.of("key", true)), rows("json scalar boolean false", Map.of("key", false)), - rows("json empty string", Map.of()) - ); + rows("json empty string", Map.of())); } } From 50449b0e19a9b1085ba32fd06082d78efdb669bd Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 10:41:07 -0800 Subject: [PATCH 30/30] Fix doctest Signed-off-by: Andrew Carbonetto --- docs/user/ppl/functions/json.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 22ac2cd241..9ee01f2406 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -78,21 +78,21 @@ Example: os> source=people | eval result = json_object('key', 123.45) | fields result fetched rows / total rows = 1/1 - +------------------+ - | result | - +------------------+ - | {"key":123.45} | - +------------------+ + +-----------------+ + | result | + |-----------------| + | {'key': 123.45} | + +-----------------+ os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result fetched rows / total rows = 1/1 +------------------------------+ | result | - +------------------------------+ - | {"outer":{"inner":123.45}} | + |------------------------------| + | {'outer': {'inner': 123.45}} | +------------------------------+ - os> source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result fetched rows / total rows = 1/1 +------------------------------------------------+ | result |