Make SDK more backward and forward compatible with protocol

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .idea
+.run
 *.iml
 .env
 

atproto/xrpc_client/models/base.py

@@ -1,6 +1,8 @@
+import os
 import typing as t
+import warnings
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, model_validator
 
 from atproto.exceptions import ModelFieldNotFoundError
 
@@ -15,14 +17,29 @@ class ModelBase(BaseModel, AtProtocolBase):
     Provides square brackets [] notation to get attributes like in a dictionary.
     """
 
-    model_config = ConfigDict(extra='forbid', populate_by_name=True, strict=True)
+    model_config = ConfigDict(extra='allow', populate_by_name=True, strict=True)
 
     def __getitem__(self, item: str) -> t.Any:
         if hasattr(self, item):
             return getattr(self, item)
 
         raise ModelFieldNotFoundError(f"Can't find field '{item}' in the object of type {type(self)}.")
 
+    @model_validator(mode='after')
+    def __alert_about_extra_fields(self) -> 'ModelBase':
+        if self.model_extra and os.environ.get('ATPROTO_LEXICON_WARN', '1') == '1':
+            warnings.warn(
+                f'Extra fields found in the object of type {type(self)}: {self.model_extra}. '  # noqa: S608
+                f'Probably you are using the old version of SDK. '
+                f'Please update it using `pip install -U atproto`. '
+                f'In case you are working with custom lexicon ignore this warning. '
+                f'It is also possible that you are working with extended record. '
+                f'To disable this warning set `ATPROTO_LEXICON_WARN` to `0` in the environment variables.',
+                stacklevel=0,
+            )
+
+        return self
+
 
 class ParamsModelBase(ModelBase):
     pass

atproto/xrpc_client/models/blob_ref.py

@@ -16,7 +16,7 @@ class BlobRefLink(BaseModel):
 class BlobRef(BaseModel):
     """Blob reference."""
 
-    model_config = ConfigDict(extra='forbid', populate_by_name=True, strict=True)
+    model_config = ConfigDict(extra='allow', populate_by_name=True, strict=True)
 
     mime_type: str = Field(alias='mimeType')  #: Mime type.
     size: int  #: Size in bytes.

atproto/xrpc_client/models/utils.py

@@ -31,10 +31,13 @@ def get_or_create(
     """Get model instance from raw data.
 
     Note:
-        The record could have custom fields and be completely custom.
-        For example, custom bsky clients add a "via" field to indicate that it was posted using a not official client.
-        Such custom types can't be decoded into proper models,
-        and will be decoded to :obj:`atproto.xrpc_client.models.base.DotDict`.
+        The record could have additional fields and be completely custom.
+        For example, third-party bsky clients add a "via"
+        field to indicate that it was posted using a not official client.
+        Such records are corresponding to the lexicon, but have additional fields.
+        This is called "extended record".
+        Extended records will be decoded to proper models with extra, non-typehinted fields.
+        Unknown record types will be decoded to :obj:`atproto.xrpc_client.models.base.DotDict`.
 
     Note:
         By default, the method raises an exception on custom models.

tests/models/fetch_test_data.py

@@ -116,7 +116,7 @@ def main() -> None:
             params={'collection': 'app.bsky.feed.post', 'repo': 'test.marshal.dev', 'rkey': '3k2yihcrp6f2c'},
         ),
         Call(
-            name='custom_post_record',
+            name='extended_post_record',
             method='com.atproto.repo.get_record',
             params={'collection': 'app.bsky.feed.post', 'repo': 'test.marshal.dev', 'rkey': '3k2yinh52ne2x'},
         ),
@@ -126,7 +126,7 @@ def main() -> None:
             params={'collection': 'app.bsky.feed.like', 'repo': 'test.marshal.dev', 'rkey': '3k5u7c7j7a52v'},
         ),
         Call(
-            name='custom_like_record',
+            name='extended_like_record',
             method='com.atproto.repo.get_record',
             params={'collection': 'app.bsky.feed.like', 'repo': 'test.marshal.dev', 'rkey': '3k5u5ammyg72r'},
         ),

tests/models/test_data/custom_record.json

@@ -0,0 +1,12 @@
+{
+    "uri": "at://did:plc:kvwvcn5iqfooopmyzvb4qzba/app.bsky.feed.post/3k2yinh52ne2x",
+    "cid": "bafyreiaepzhlaf2hd4exhx5o74orvhtqloog4xwaptcrzsuyfkzr43va4e",
+    "value": {
+        "text": "foo",
+        "$type": "app.bsky.feed.pythonSdkCustomRecordPost",
+        "langs": [
+            "en"
+        ],
+        "createdAt": "2023-07-21T01:33:51.481951"
+    }
+}

tests/models/tests/test_changed_lexicon_compatability.py

@@ -0,0 +1,94 @@
+import pytest
+
+from atproto.exceptions import ModelError
+from atproto.xrpc_client import models
+from atproto.xrpc_client.models import get_or_create
+
+"""These tests are based on the following statements:
+
+Lexicons are allowed to change over time, within some bounds to ensure both forwards and backwards compatibility.
+The basic principle is that all old data must still be valid under the updated Lexicon,
+and new data must be valid under the old Lexicon.
+- Any new fields must be optional
+- Non-optional fields can not be removed
+- Types can not change
+- Fields can not be renamed
+
+Ref: https://atproto.com/specs/lexicon#lexicon-evolution
+"""
+
+
+def test_model_serialization() -> None:
+    expected_signing_key = 'blabla'
+    test_data = {
+        'signingKey': expected_signing_key,
+    }
+    model = get_or_create(test_data, models.ComAtprotoServerReserveSigningKey.Response)
+
+    assert isinstance(model, models.ComAtprotoServerReserveSigningKey.Response)
+    assert isinstance(model.signing_key, str)
+
+    assert model.signing_key == expected_signing_key
+    assert model['signing_key'] == expected_signing_key
+
+
+def test_added_new_fields_as_optional() -> None:
+    """New fields must be optional. It means that model must be created without validation errors."""
+
+    expected_signing_key = 'blabla'
+    test_data = {
+        'signingKey': expected_signing_key,
+        'brandNewBackendField': 'foo',
+    }
+    model = get_or_create(test_data, models.ComAtprotoServerReserveSigningKey.Response)
+
+    assert isinstance(model, models.ComAtprotoServerReserveSigningKey.Response)
+    assert isinstance(model.signing_key, str)
+
+    assert model.signing_key == expected_signing_key
+    assert model['signing_key'] == expected_signing_key
+
+    assert model.model_extra is not None
+
+    # also, we want to have access to new fields from SDK
+    # the problem here is that we can't access them via snake_case,
+    # could be fixed in the future
+    assert model.brandNewBackendField == 'foo'
+    assert model['brandNewBackendField'] == 'foo'
+
+
+def test_removed_non_optional_field() -> None:
+    """If protocol removed non-optional field, it breaks backward compatibility. We must throw an error."""
+
+    test_data = {
+        # for example, signingKey was removed
+        'brandNewBackendField': 'foo',
+    }
+
+    with pytest.raises(ModelError):
+        get_or_create(test_data, models.ComAtprotoServerReserveSigningKey.Response)
+
+
+def test_changed_field_type() -> None:
+    """If protocol changed a field type, it breaks backward compatibility. We must throw an error."""
+
+    test_data = {
+        # for example, signingKey now is an integer instead of string
+        'signingKey': 123,
+    }
+
+    with pytest.raises(ModelError):
+        get_or_create(test_data, models.ComAtprotoServerReserveSigningKey.Response)
+
+
+def test_renamed_field_type() -> None:
+    """If protocol changed a field type, it breaks backward compatibility. We must throw an error."""
+
+    expected_signing_key = 'blabla'
+    test_data = {
+        # for example, signingKey now signKey
+        'signKey': expected_signing_key,
+    }
+
+    with pytest.raises(ModelError):
+        get_or_create(test_data, models.ComAtprotoServerReserveSigningKey.Response)

tests/models/tests/test_custom_record.py

@@ -0,0 +1,48 @@
+from atproto.xrpc_client import models
+from atproto.xrpc_client.models import get_model_as_dict, get_or_create
+from atproto.xrpc_client.models.dot_dict import DotDict
+from tests.models.tests.utils import load_data_from_file
+
+
+def load_test_data() -> dict:
+    return load_data_from_file('custom_record')
+
+
+def test_custom_record_deserialization() -> None:
+    model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
+    expected_custom_record_id = 'app.bsky.feed.pythonSdkCustomRecordPost'
+
+    assert isinstance(model, models.ComAtprotoRepoGetRecord.Response)
+    assert isinstance(model.value, DotDict)
+
+    assert model.value['$type'] != models.ids.AppBskyFeedPost
+    assert model.value['$type'] == expected_custom_record_id
+
+    assert model.value.text == 'foo'
+    assert model.value['text'] == 'foo'
+
+    assert model.value.langs == ['en']
+    assert model.value['createdAt'] == '2023-07-21T01:33:51.481951'
+    assert model.value.createdAt == '2023-07-21T01:33:51.481951'
+    assert model.value['created_at'] == '2023-07-21T01:33:51.481951'
+    assert model.value.created_at == '2023-07-21T01:33:51.481951'
+
+
+def test_custom_record_serialization() -> None:
+    model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
+    expected_custom_record_id = 'app.bsky.feed.pythonSdkCustomRecordPost'
+
+    model_dict = get_model_as_dict(model)
+    restored_model = get_or_create(model_dict, models.ComAtprotoRepoGetRecord.Response)
+
+    assert isinstance(get_model_as_dict(model.value), dict)
+    assert model_dict == get_model_as_dict(restored_model)
+
+    assert restored_model.value['$type'] != models.ids.AppBskyFeedPost
+    assert restored_model.value['$type'] == expected_custom_record_id
+
+    assert restored_model.value.text == 'foo'
+    assert restored_model.value['text'] == 'foo'
+
+    assert model_dict['value']['$type'] == expected_custom_record_id
+    assert model_dict['value']['text'] == 'foo'

tests/models/tests/test_custom_like_record.py → tests/models/tests/test_extended_like_record.py

@@ -1,26 +1,25 @@
 from atproto.xrpc_client import models
 from atproto.xrpc_client.models import get_model_as_dict, get_or_create
-from atproto.xrpc_client.models.dot_dict import DotDict
 from tests.models.tests.utils import load_data_from_file
 
 
 def load_test_data() -> dict:
-    return load_data_from_file('custom_like_record')
+    return load_data_from_file('extended_like_record')
 
 
-def test_custom_like_record_deserialization() -> None:
+def test_extended_like_record_deserialization() -> None:
     model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
 
     assert isinstance(model, models.ComAtprotoRepoGetRecord.Response)
-    assert isinstance(model.value, DotDict)
+    assert isinstance(model.value, models.AppBskyFeedLike.Main)
 
-    assert model.value['$type'] == models.ids.AppBskyFeedLike
+    assert model.value.py_type == models.ids.AppBskyFeedLike
     # record_type is the custom field out of lexicon
     assert model.value.record_type == 'app.bsky.feed.like'
     assert model.value['record_type'] == 'app.bsky.feed.like'
 
 
-def test_custom_like_record_serialization() -> None:
+def test_extended_like_record_serialization() -> None:
     model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
 
     model_dict = get_model_as_dict(model)
@@ -29,8 +28,8 @@ def test_custom_like_record_serialization() -> None:
     assert isinstance(get_model_as_dict(model.value), dict)
     assert model_dict == get_model_as_dict(restored_model)
 
-    assert restored_model.value['$type'] == models.ids.AppBskyFeedLike
-    # record_type is the custom field out of lexicon
+    assert restored_model.value.py_type == models.ids.AppBskyFeedLike
+    # record_type is the additional field out of lexicon
     assert restored_model.value.record_type == 'app.bsky.feed.like'
     assert restored_model.value['record_type'] == 'app.bsky.feed.like'
 

tests/models/tests/test_custom_post_record.py → tests/models/tests/test_extended_post_record.py

@@ -1,26 +1,30 @@
 from atproto.xrpc_client import models
 from atproto.xrpc_client.models import get_model_as_dict, get_or_create
-from atproto.xrpc_client.models.dot_dict import DotDict
 from tests.models.tests.utils import load_data_from_file
 
 
 def load_test_data() -> dict:
-    return load_data_from_file('custom_post_record')
+    return load_data_from_file('extended_post_record')
 
 
-def test_custom_post_record_deserialization() -> None:
+def test_extended_post_record_deserialization() -> None:
     model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
 
     assert isinstance(model, models.ComAtprotoRepoGetRecord.Response)
-    assert isinstance(model.value, DotDict)
+    assert isinstance(model.value, models.AppBskyFeedPost.Main)
+
+    assert model.value['py_type'] == models.ids.AppBskyFeedPost
+    assert model.value.py_type == models.ids.AppBskyFeedPost
+
+    # lol is the additional field out of lexicon
+
+    assert 'lol' in model.value.model_extra
 
-    assert model.value['$type'] == models.ids.AppBskyFeedPost
-    # lol is the custom field out of lexicon
     assert model.value.lol == 'kek'
     assert model.value['lol'] == 'kek'
 
 
-def test_custom_post_record_serialization() -> None:
+def test_extended_post_record_serialization() -> None:
     model = get_or_create(load_test_data(), models.ComAtprotoRepoGetRecord.Response)
 
     model_dict = get_model_as_dict(model)
@@ -29,8 +33,12 @@ def test_custom_post_record_serialization() -> None:
     assert isinstance(get_model_as_dict(model.value), dict)
     assert model_dict == get_model_as_dict(restored_model)
 
-    assert restored_model.value['$type'] == models.ids.AppBskyFeedPost
-    # lol is the custom field out of lexicon
+    assert restored_model.value['py_type'] == models.ids.AppBskyFeedPost
+
+    # lol is the additional field out of lexicon
+
+    assert 'lol' in restored_model.value.model_extra
+
     assert restored_model.value.lol == 'kek'
     assert restored_model.value['lol'] == 'kek'
 

tests/models/tests/test_is_record_type.py

@@ -8,22 +8,22 @@ def load_test_correct_data() -> dict:
     return load_data_from_file('post_record')
 
 
-def load_test_extended_data() -> dict:
-    return load_data_from_file('custom_post_record')
+def load_test_custom_data() -> dict:
+    return load_data_from_file('custom_record')
 
 
 def test_is_record_type() -> None:
     lexicon_correct_post_record = get_or_create(load_test_correct_data(), models.ComAtprotoRepoGetRecord.Response)
-    extended_post_record = get_or_create(load_test_extended_data(), models.ComAtprotoRepoGetRecord.Response)
+    custom_record = get_or_create(load_test_custom_data(), models.ComAtprotoRepoGetRecord.Response)
+    expected_custom_record_id = 'app.bsky.feed.pythonSdkCustomRecordPost'
 
     assert isinstance(lexicon_correct_post_record.value, models.AppBskyFeedPost.Main)
     assert is_record_type(lexicon_correct_post_record.value, models.ids.AppBskyFeedPost) is True
     assert is_record_type(lexicon_correct_post_record.value, models.ids.AppBskyFeedGenerator) is False
     assert is_record_type(lexicon_correct_post_record.value, models.AppBskyFeedPost) is True
     assert is_record_type(lexicon_correct_post_record.value, models.AppBskyFeedGenerator) is False
 
-    assert isinstance(extended_post_record.value, DotDict)
-    assert is_record_type(extended_post_record.value, models.ids.AppBskyFeedPost) is True
-    assert is_record_type(extended_post_record.value, models.ids.AppBskyFeedGenerator) is False
-    assert is_record_type(extended_post_record.value, models.AppBskyFeedPost) is True
-    assert is_record_type(extended_post_record.value, models.AppBskyFeedGenerator) is False
+    assert isinstance(custom_record.value, DotDict)
+    assert is_record_type(custom_record.value, expected_custom_record_id) is True
+    assert is_record_type(custom_record.value, models.ids.AppBskyFeedPost) is False
+    assert is_record_type(custom_record.value, models.AppBskyFeedPost) is False