diff --git a/singer_sdk/_singerlib/schema.py b/singer_sdk/_singerlib/schema.py index 2cdc04f29..bb7851b2b 100644 --- a/singer_sdk/_singerlib/schema.py +++ b/singer_sdk/_singerlib/schema.py @@ -11,6 +11,11 @@ if t.TYPE_CHECKING: from referencing._core import Resolver +META_KEYS = [ + "id", + "schema", +] + # These are keys defined in the JSON Schema spec that do not themselves contain # schemas (or lists of schemas) STANDARD_KEYS = [ @@ -52,6 +57,9 @@ class Schema: This is because we wanted to expand it with extra STANDARD_KEYS. """ + id: str | None = None + schema: str | None = None + type: str | list[str] | None = None default: t.Any | None = None properties: dict | None = None @@ -94,6 +102,10 @@ def to_dict(self) -> dict[str, t.Any]: if self.__dict__.get(key) is not None: result[key] = self.__dict__[key] + for key in META_KEYS: + if self.__dict__.get(key) is not None: + result[f"${key}"] = self.__dict__[key] + return result @classmethod @@ -110,7 +122,40 @@ def from_dict( Returns: The initialized Schema object. - """ + + Example: + >>> data = { + ... "$id": "https://example.com/person.schema.json", + ... "$schema": "http://json-schema.org/draft/2020-12/schema", + ... "title": "Person", + ... "type": "object", + ... "properties": { + ... "firstName": { + ... "type": "string", + ... "description": "The person's first name.", + ... }, + ... "lastName": { + ... "type": "string", + ... "description": "The person's last name.", + ... }, + ... "age": { + ... "description": "Age in years which must be equal to or greater than zero.", + ... "type": "integer", + ... "minimum": 0, + ... }, + ... }, + ... "required": ["firstName", "lastName"], + ... } + >>> schema = Schema.from_dict(data) + >>> schema.title + 'Person' + >>> schema.properties["firstName"].description + "The person's first name." + >>> schema.properties["age"].minimum + 0 + >>> schema.schema + 'http://json-schema.org/draft/2020-12/schema' + """ # noqa: E501 kwargs = schema_defaults.copy() properties = data.get("properties") items = data.get("items") @@ -121,9 +166,15 @@ def from_dict( } if items is not None: kwargs["items"] = cls.from_dict(items, **schema_defaults) + for key in STANDARD_KEYS: if key in data: kwargs[key] = data[key] + + for key in META_KEYS: + if f"${key}" in data: + kwargs[key] = data[f"${key}"] + return cls(**kwargs) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index d1fe530d6..fb22f4e82 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -123,7 +123,7 @@ None, ] -DEFAULT_JSONSCHEMA_VALIDATOR: type[Validator] = validators.Draft7Validator # type: ignore[assignment] +DEFAULT_JSONSCHEMA_VALIDATOR: type[Validator] = validators.Draft202012Validator # type: ignore[assignment] T = t.TypeVar("T", bound=_JsonValue) P = t.TypeVar("P") @@ -1115,6 +1115,17 @@ def append(self, property: Property) -> None: # noqa: A002 """ self.wrapped[property.name] = property + @property + def type_dict(self) -> dict: # type: ignore[override] + """Get type dictionary. + + Returns: + A dictionary describing the type. + """ + d = super().type_dict + d["$schema"] = "https://json-schema.org/draft/2020-12/schema" + return d + def __iter__(self) -> t.Iterator[Property]: """Iterate all properties of the property list. diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 488030392..0e3d9bc49 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -161,7 +161,8 @@ def test_to_json(): "required": [ "test_property" ], - "additionalProperties": false + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema" }""", ) @@ -172,6 +173,7 @@ def test_any_type(caplog: pytest.LogCaptureFixture): ) with caplog.at_level(WARNING): assert schema.to_dict() == { + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "any_type": { diff --git a/tests/snapshots/countries_write_schemas/countries_write_schemas b/tests/snapshots/countries_write_schemas/countries_write_schemas index 71a900f1b..f9783e51d 100644 --- a/tests/snapshots/countries_write_schemas/countries_write_schemas +++ b/tests/snapshots/countries_write_schemas/countries_write_schemas @@ -1,2 +1,2 @@ {"type":"SCHEMA","stream":"continents","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"key_properties":["code"]} -{"type":"SCHEMA","stream":"countries","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]},"native":{"type":["string","null"]},"phone":{"type":["string","null"]},"capital":{"type":["string","null"]},"currency":{"type":["string","null"]},"emoji":{"type":["string","null"]},"continent":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":["object","null"]},"languages":{"items":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"type":["array","null"]}},"type":"object"},"key_properties":["code"]} +{"type":"SCHEMA","stream":"countries","schema":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]},"native":{"type":["string","null"]},"phone":{"type":["string","null"]},"capital":{"type":["string","null"]},"currency":{"type":["string","null"]},"emoji":{"type":["string","null"]},"continent":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":["object","null"]},"languages":{"items":{"properties":{"code":{"type":["string","null"]},"name":{"type":["string","null"]}},"type":"object"},"type":["array","null"]}},"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["code"]} diff --git a/tests/snapshots/mapped_stream/aliased_stream.jsonl b/tests/snapshots/mapped_stream/aliased_stream.jsonl index d4c4cc942..94f0a6b5b 100644 --- a/tests/snapshots/mapped_stream/aliased_stream.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl b/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl index 6b0dbe121..8aac11990 100644 --- a/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream_batch.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"BATCH","stream":"aliased_stream","encoding":{"format":"jsonl","compression":"gzip"},"manifest":["file:///tmp/stream.json.gz"]} {"type":"STATE","value":{}} {"type":"STATE","value":{"bookmarks":{"mystream":{}}}} diff --git a/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl b/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl index 8a341c09f..5a96a8c5e 100644 --- a/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream_not_expr.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl b/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl index bd3f0cb04..5855671d3 100644 --- a/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream_quoted.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/builtin_variable_self.jsonl b/tests/snapshots/mapped_stream/builtin_variable_self.jsonl index 4711ac95a..3338fa72e 100644 --- a/tests/snapshots/mapped_stream/builtin_variable_self.jsonl +++ b/tests/snapshots/mapped_stream/builtin_variable_self.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl index 3fd81e87a..ec27eb117 100644 --- a/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl +++ b/tests/snapshots/mapped_stream/builtin_variable_stream_name.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl index 8c56822cd..ee253c34d 100644 --- a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl +++ b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"aliased_mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"aliased_mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"aliased_mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased_mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"aliased_mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl index 8b4257369..9b54b4c83 100644 --- a/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl +++ b/tests/snapshots/mapped_stream/builtin_variable_stream_name_alias_expr.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"MYSTREAM","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"MYSTREAM","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"MYSTREAM","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"MYSTREAM","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl b/tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl index 4711ac95a..3338fa72e 100644 --- a/tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl +++ b/tests/snapshots/mapped_stream/builtin_variable_underscore.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/changed_key_properties.jsonl b/tests/snapshots/mapped_stream/changed_key_properties.jsonl index d90219138..eb5b9491f 100644 --- a/tests/snapshots/mapped_stream/changed_key_properties.jsonl +++ b/tests/snapshots/mapped_stream/changed_key_properties.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}}},"key_properties":["email_hash"]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["email_hash"]} {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/dates.jsonl b/tests/snapshots/mapped_stream/dates.jsonl index 43ae073ec..f022c8c60 100644 --- a/tests/snapshots/mapped_stream/dates.jsonl +++ b/tests/snapshots/mapped_stream/dates.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"joined_date":{"type":["string","null"],"format":"date"},"joined_timestamp":{"type":["number","null"]},"some_datetime":{"type":["string","null"],"format":"date-time"}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"joined_date":{"type":["string","null"],"format":"date"},"joined_timestamp":{"type":["number","null"]},"some_datetime":{"type":["string","null"],"format":"date-time"}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","joined_date":"2022-01-01","joined_timestamp":1640995200.0,"some_datetime":"2024-10-10T10:10:10+00:00"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","joined_date":"2022-01-01","joined_timestamp":1640995200.0,"some_datetime":"2024-10-10T10:10:10+00:00"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","joined_date":"2022-01-01","joined_timestamp":1640995200.0,"some_datetime":"2024-10-10T10:10:10+00:00"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/drop_property.jsonl b/tests/snapshots/mapped_stream/drop_property.jsonl index 84bfa899d..a7ba3e35b 100644 --- a/tests/snapshots/mapped_stream/drop_property.jsonl +++ b/tests/snapshots/mapped_stream/drop_property.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl index 84bfa899d..a7ba3e35b 100644 --- a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl +++ b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":[],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl b/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl index 6db048524..b816de5c1 100644 --- a/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl +++ b/tests/snapshots/mapped_stream/fake_credit_card_number.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"cc":{"type":["string","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"cc":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"cc":"4201040137208265027"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"cc":"675987782884"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"cc":"502011811259"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/fake_email_seed_class.jsonl b/tests/snapshots/mapped_stream/fake_email_seed_class.jsonl index e27c2e53f..74e528641 100644 --- a/tests/snapshots/mapped_stream/fake_email_seed_class.jsonl +++ b/tests/snapshots/mapped_stream/fake_email_seed_class.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"zwells@example.org"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"josephcunningham@example.com"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"lydia62@example.net"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/fake_email_seed_instance.jsonl b/tests/snapshots/mapped_stream/fake_email_seed_instance.jsonl index e27c2e53f..74e528641 100644 --- a/tests/snapshots/mapped_stream/fake_email_seed_instance.jsonl +++ b/tests/snapshots/mapped_stream/fake_email_seed_instance.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"zwells@example.org"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"josephcunningham@example.com"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"lydia62@example.net"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/flatten_all.jsonl b/tests/snapshots/mapped_stream/flatten_all.jsonl index cdc5ddd99..922f1934a 100644 --- a/tests/snapshots/mapped_stream/flatten_all.jsonl +++ b/tests/snapshots/mapped_stream/flatten_all.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[3.14,2.718]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__sub__custom_obj":"obj-world","user__some_numbers":"[10.32,1.618]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[1.414,1.732]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl index 99ecd5c17..b526731b5 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_0.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_0.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl index f3d54bfb0..efadf1c48 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub":"{\"num\":1,\"custom_obj\":\"obj-hello\"}","user__some_numbers":"[3.14,2.718]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub":"{\"num\":2,\"custom_obj\":\"obj-world\"}","user__some_numbers":"[10.32,1.618]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub":"{\"num\":3,\"custom_obj\":\"obj-hello\"}","user__some_numbers":"[1.414,1.732]","joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/keep_all_fields.jsonl b/tests/snapshots/mapped_stream/keep_all_fields.jsonl index 9176918f2..33df3a6f8 100644 --- a/tests/snapshots/mapped_stream/keep_all_fields.jsonl +++ b/tests/snapshots/mapped_stream/keep_all_fields.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/map_and_flatten.jsonl b/tests/snapshots/mapped_stream/map_and_flatten.jsonl index df06b7f1f..6be3ac645 100644 --- a/tests/snapshots/mapped_stream/map_and_flatten.jsonl +++ b/tests/snapshots/mapped_stream/map_and_flatten.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object","required":["email"]},"key_properties":["email_hash"]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user__id":{"type":["integer","null"]},"user__sub__num":{"type":["integer","null"]},"user__sub__custom_obj":{"type":["string","null"]},"user__some_numbers":{"type":["string","null"]},"joined_at":{"format":"date-time","type":["string","null"]},"email_hash":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":["email_hash"]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user__id":1,"user__sub__num":1,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[3.14,2.718]","joined_at":"2022-01-01T00:00:00Z","email_hash":"c160f8cc69a4f0bf2b0362752353d060"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user__id":2,"user__sub__num":2,"user__sub__custom_obj":"obj-world","user__some_numbers":"[10.32,1.618]","joined_at":"2022-01-01T00:00:00Z","email_hash":"4b9bb80620f03eb3719e0a061c14283d"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user__id":3,"user__sub__num":3,"user__sub__custom_obj":"obj-hello","user__some_numbers":"[1.414,1.732]","joined_at":"2022-01-01T00:00:00Z","email_hash":"426b189df1e2f359efe6ee90f2d2030f"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/no_map.jsonl b/tests/snapshots/mapped_stream/no_map.jsonl index 99ecd5c17..b526731b5 100644 --- a/tests/snapshots/mapped_stream/no_map.jsonl +++ b/tests/snapshots/mapped_stream/no_map.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl index 78d332dce..4d9439007 100644 --- a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl +++ b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"count":21},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":13},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"count":19},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl index df053aa52..ef70c9aac 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl index df053aa52..ef70c9aac 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email_hash":{"type":["string","null"]},"fixed_count":{"type":["integer","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"email_hash":"c160f8cc69a4f0bf2b0362752353d060","fixed_count":20},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"4b9bb80620f03eb3719e0a061c14283d","fixed_count":12},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"email_hash":"426b189df1e2f359efe6ee90f2d2030f","fixed_count":18},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/record_to_column.jsonl b/tests/snapshots/mapped_stream/record_to_column.jsonl index 9a2c450ba..599edbb9a 100644 --- a/tests/snapshots/mapped_stream/record_to_column.jsonl +++ b/tests/snapshots/mapped_stream/record_to_column.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"_data":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":["object","null"],"required":["email"]}}},"key_properties":[]} +{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"_data":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":["object","null"],"required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"mystream","record":{"_data":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"_data":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"}},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"mystream","record":{"_data":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"}},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl index 753e15da4..501a51ab6 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl index 753e15da4..501a51ab6 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"sourced_stream_1","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_1","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl index c677521cf..8226cb510 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl @@ -1,5 +1,5 @@ {"type":"STATE","value":{}} -{"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"]},"key_properties":[]} +{"type":"SCHEMA","stream":"sourced_stream_2","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"]},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"]},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]} {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"} {"type":"RECORD","stream":"sourced_stream_2","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}