diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedead..854ba0ed69 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163..720da0979e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,14 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression stringToJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + + public static FunctionExpression jsonObject(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_OBJECT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } @@ -843,6 +851,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb7..a6832b366f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,8 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -235,6 +237,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064..885380578c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,23 +7,108 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.expression.DSL.jsonObject; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); + repository.register(jsonObject()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } + + /** Creates a JSON Object/tuple expr from a given list of kv pairs. */ + private static FunctionResolver jsonObject() { + return new FunctionResolver() { + @Override + public FunctionName getFunctionName() { + return BuiltinFunctionName.JSON_OBJECT.getName(); + } + + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + List paramList = unresolvedSignature.getParamTypeList(); + // check that we got an even number of arguments + if (paramList.size() % 2 != 0) { + throw new SemanticCheckException( + String.format( + "Expected an even number of arguments but instead got %d arguments", + paramList.size())); + } + + // check that each "key" argument (of key-value pair) is a string + for (int i = 0; i < paramList.size(); i = i + 2) { + ExprType paramType = paramList.get(i); + if (!ExprCoreType.STRING.equals(paramType)) { + throw new SemanticCheckException( + String.format( + "Expected type %s instead of %s for parameter #%d", + ExprCoreType.STRING, paramType.typeName(), i + 1)); + } + } + + // return the unresolved signature and function builder + return Pair.of( + unresolvedSignature, + (functionProperties, arguments) -> + new FunctionExpression(getFunctionName(), arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + LinkedHashMap tupleValues = new LinkedHashMap<>(); + Iterator iter = getArguments().iterator(); + while (iter.hasNext()) { + tupleValues.put( + iter.next().valueOf(valueEnv).stringValue(), + iter.next().valueOf(valueEnv)); + } + return ExprTupleValue.fromExprValueMap(tupleValues); + } + + @Override + public ExprType type() { + return STRUCT; + } + }); + } + }; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89a..c1391ac9ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,6 +17,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; @@ -42,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -105,7 +108,8 @@ private static DefaultFunctionResolver castToShort() { impl( nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)), SHORT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED)); } private static DefaultFunctionResolver castToInt() { @@ -119,7 +123,8 @@ private static DefaultFunctionResolver castToInt() { impl( nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED)); } private static DefaultFunctionResolver castToLong() { @@ -133,7 +138,8 @@ private static DefaultFunctionResolver castToLong() { impl( nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED)); } private static DefaultFunctionResolver castToFloat() { @@ -147,7 +153,8 @@ private static DefaultFunctionResolver castToFloat() { impl( nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED)); } private static DefaultFunctionResolver castToDouble() { @@ -161,7 +168,8 @@ private static DefaultFunctionResolver castToDouble() { impl( nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED)); } private static DefaultFunctionResolver castToBoolean() { @@ -173,7 +181,8 @@ private static DefaultFunctionResolver castToBoolean() { STRING), impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), - impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN)); + impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED)); } private static DefaultFunctionResolver castToIp() { @@ -183,6 +192,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e..6120d11bb5 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,27 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -12,7 +29,7 @@ public class JsonUtils { * Checks if given JSON string can be parsed as valid JSON. * * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). - * @return true if the string can be parsed as valid JSON, else false. + * @return true if the string can be parsed as valid JSON, else false (including null or missing). */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); @@ -23,9 +40,68 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** + * Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @return ExprValue returns an expression that best represents the provided JSON-encoded string. + *
    + *
  1. {@link ExprTupleValue} if the JSON is an object + *
  2. {@link ExprCollectionValue} if the JSON is an array + *
  3. {@link ExprDoubleValue} if the JSON is a floating-point number scalar + *
  4. {@link ExprIntegerValue} if the JSON is an integral number scalar + *
  5. {@link ExprStringValue} if the JSON is a string scalar + *
  6. {@link ExprBooleanValue} if the JSON is a boolean scalar + *
  7. {@link ExprNullValue} if the JSON is null, empty, or invalid + *
+ */ + public static ExprValue castJson(ExprValue json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json.stringValue()); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + switch (jsonNode.getNodeType()) { + case ARRAY: + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + case OBJECT: + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + case STRING: + return new ExprStringValue(jsonNode.asText()); + case NUMBER: + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + return new ExprIntegerValue(jsonNode.asLong()); + case BOOLEAN: + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; + default: + // in all other cases, return null + return LITERAL_NULL; } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ad15dadfb7..115898e349 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,6 +1230,8 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index d944f7c85c..ee4df23be4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -99,6 +99,8 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2..0b1bd809a9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,58 +7,261 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonNestedObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); - private static final ExprValue JsonObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = - ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); - assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); + assertEquals( + LITERAL_FALSE, + DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"))) + .valueOf()); + assertEquals( + LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf()); } @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( - ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); + ExpressionEvaluationException.class, + () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } @Test public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); + + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "12.34", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); + } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.stringToJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + + // also test the empty object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.stringToJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); + + assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); + + assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); + + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // missing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } + + @Test + public void json_object_returns_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + // TODO: requires json_array() + // objectMap.put( + // "arr", + // new ExprCollectionValue( + // List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), + // ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = + DSL.jsonObject( + DSL.literal("foo"), DSL.literal("foo"), + DSL.literal("fuzz"), DSL.literal(true), + DSL.literal("bar"), DSL.literal(1234), + DSL.literal("bar2"), DSL.literal(12.34), + DSL.literal("baz"), DSL.literal(LITERAL_NULL), + DSL.literal("obj"), DSL.jsonObject(DSL.literal("internal"), DSL.literal("value"))); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_returns_empty_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.jsonObject(); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); + @Test + public void json_object_throws_SemanticCheckException() { + // wrong number of arguments + assertThrows( + SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("only one")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject(DSL.literal("one"), DSL.literal("two"), DSL.literal("three")).valueOf()); + + // key argument is not a string + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonObject(DSL.literal(1234), DSL.literal("two")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject( + DSL.literal("one"), DSL.literal(true), DSL.literal(true), DSL.literal("four")) + .valueOf()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb47..ff0c8bcc01 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -158,6 +165,15 @@ void castBooleanToShort() { assertEquals(new ExprShortValue(0), expression.valueOf()); } + @Test + void castUndefinedToShort() { + Short value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castShort(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(value), expression.valueOf()); + } + @Test void castBooleanToInt() { FunctionExpression expression = DSL.castInt(DSL.literal(true)); @@ -169,6 +185,15 @@ void castBooleanToInt() { assertEquals(new ExprIntegerValue(0), expression.valueOf()); } + @Test + void castUndefinedToInt() { + Integer value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castInt(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(INTEGER, expression.type()); + assertEquals(new ExprIntegerValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToLong({0})") @MethodSource({"numberData"}) void castToLong(ExprValue value) { @@ -201,6 +226,15 @@ void castBooleanToLong() { assertEquals(new ExprLongValue(0), expression.valueOf()); } + @Test + void castUndefinedToLong() { + Long value = 42l; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castLong(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(LONG, expression.type()); + assertEquals(new ExprLongValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToFloat({0})") @MethodSource({"numberData"}) void castToFloat(ExprValue value) { @@ -233,6 +267,15 @@ void castBooleanToFloat() { assertEquals(new ExprFloatValue(0), expression.valueOf()); } + @Test + void castUndefinedToFloat() { + Float value = 23.45f; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castFloat(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(FLOAT, expression.type()); + assertEquals(new ExprFloatValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToDouble({0})") @MethodSource({"numberData"}) void castToDouble(ExprValue value) { @@ -265,6 +308,15 @@ void castBooleanToDouble() { assertEquals(new ExprDoubleValue(0), expression.valueOf()); } + @Test + void castUndefinedToDouble() { + Double value = 23.45e5; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(DOUBLE, expression.type()); + assertEquals(new ExprDoubleValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToBoolean({0})") @MethodSource({"numberData"}) void castToBoolean(ExprValue value) { @@ -294,6 +346,14 @@ void castBooleanToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } + @Test + void castUndefinedToBoolean() { + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); + assertEquals(BOOLEAN, expression.type()); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); + } + @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); @@ -389,4 +449,102 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + + // also test the empty-object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.castJson(DSL.literal("{}")).valueOf()); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.castJson(DSL.literal("[]")).valueOf()); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fa704b6c65..9ee01f2406 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,68 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ + +JSON_OBJECT +----------- + +Description +>>>>>>>>>>> + +Usage: `json_object(, [, , ]...)` returns a JSON object from key-value pairs. + +Argument type: +- A \ must be STRING. +- A \ can be a scalar, another json object, or json array type. Note: scalar fields will be treated as single-value. Use `json_array` to construct an array value from a multi-value. + +Return type: STRUCT + +Example: + + os> source=people | eval result = json_object('key', 123.45) | fields result + fetched rows / total rows = 1/1 + +-----------------+ + | result | + |-----------------| + | {'key': 123.45} | + +-----------------+ + + os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result + fetched rows / total rows = 1/1 + +------------------------------+ + | result | + |------------------------------| + | {'outer': {'inner': 123.45}} | + +------------------------------+ + + source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + fetched rows / total rows = 1/1 + +------------------------------------------------+ + | result | + +------------------------------------------------+ + | {"array_doc":[123.45, "string", true, null]} | + +------------------------------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb..3d57edcb9a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,9 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -37,6 +40,11 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), + rows("json scalar int"), + rows("json scalar float"), + rows("json scalar double"), + rows("json scalar boolean true"), + rows("json scalar boolean false"), rows("json empty string")); } @@ -52,4 +60,147 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), + rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar float", 12.34f), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), + rows("json empty string", null)); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), + rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar float", 12.34), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), + rows("json empty string", null)); + } + + @Test + public void test_cast_json_scalar_to_type() throws IOException { + // cast to integer + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as int) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "integer")); + verifyDataRows(result, rows("json scalar int", 1234)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as long) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "long")); + verifyDataRows(result, rows("json scalar int", 1234l)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar float' | " + + "eval casted=cast(json(json_string) as float) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "float")); + verifyDataRows(result, rows("json scalar float", 12.34f)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar double' | " + + "eval casted=cast(json(json_string) as double) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "double")); + verifyDataRows(result, rows("json scalar double", 2.99792458e8)); + + result = + executeQuery( + String.format( + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); + verifyDataRows( + result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); + } + + @Test + public void test_json_object() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject( + Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), + rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), + rows("json scalar string", Map.of("key", "abc")), + rows("json scalar int", Map.of("key", 1234)), + rows("json scalar float", Map.of("key", 12.34)), + rows("json scalar double", Map.of("key", 2.99792458e8)), + rows("json scalar boolean true", Map.of("key", true)), + rows("json scalar boolean false", Map.of("key", false)), + rows("json empty string", Map.of())); + } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e..acab339c01 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} @@ -7,8 +7,18 @@ {"index":{"_id":"3"}} {"test_name":"json scalar string", "json_string":"\"abc\""} {"index":{"_id":"4"}} -{"test_name":"json empty string","json_string":""} +{"test_name":"json scalar int", "json_string":"1234"} {"index":{"_id":"5"}} -{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"test_name":"json scalar float", "json_string":"12.34"} {"index":{"_id":"6"}} +{"test_name":"json scalar double", "json_string":"2.99792458e8"} +{"index":{"_id":"7"}} +{"test_name":"json scalar boolean true", "json_string":"true"} +{"index":{"_id":"8"}} +{"test_name":"json scalar boolean false", "json_string":"false"} +{"index":{"_id":"9"}} +{"test_name":"json empty string", "json_string":""} +{"index":{"_id":"10"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"11"}} {"test_name":"json null", "json_string":null} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index c484f34a2a..98ffb8c85f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -330,6 +330,8 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index acae54b7d9..dd1ae10b88 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -310,6 +310,7 @@ valueExpression | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | jsonObjectFunction # jsonObjectFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr ; @@ -324,6 +325,10 @@ positionFunction : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; +jsonObjectFunction + : jsonObjectFunctionName LT_PRTHS (functionArg COMMA functionArg (COMMA functionArg COMMA functionArg)*)? RT_PRTHS + ; + booleanExpression : booleanFunctionCall ; @@ -409,6 +414,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +425,8 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonObjectFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +708,14 @@ positionFunctionName : POSITION ; +jsonObjectFunctionName + : JSON_OBJECT + ; + +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -864,6 +880,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE