From 237a212e8c0443bb1f51d276718042c7664879ad Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Wed, 2 May 2018 09:13:13 +0200 Subject: [PATCH 1/6] improve excludes relationships and schema nodes can now be excluded by named overrides. speed up exclude handling in get_schema_from_column. --- colanderalchemy/schema.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/colanderalchemy/schema.py b/colanderalchemy/schema.py index 4925e76..7a56e24 100644 --- a/colanderalchemy/schema.py +++ b/colanderalchemy/schema.py @@ -182,6 +182,12 @@ def add_nodes(self, includes, excludes, overrides): log.debug('Attribute %s skipped imperatively', name) continue + key = 'exclude' + if overrides.get(name, {}).get(key, None): + log.debug('Attribute %s skipped due to imperative overrides', + name) + continue + name_overrides_copy = overrides.get(name, {}).copy() if (isinstance(prop, ColumnProperty) @@ -241,19 +247,14 @@ def get_schema_from_column(self, prop, overrides): key = 'exclude' - if key not in itertools.chain(declarative_overrides, overrides) \ - and typedecorator_overrides.pop(key, False): + if declarative_overrides.pop(key, False): log.debug('Column %s skipped due to TypeDecorator overrides', name) return None - if key not in overrides and declarative_overrides.pop(key, False): + if declarative_overrides.pop(key, False): log.debug('Column %s skipped due to declarative overrides', name) return None - if overrides.pop(key, False): - log.debug('Column %s skipped due to imperative overrides', name) - return None - self.check_overrides(name, 'name', typedecorator_overrides, declarative_overrides, overrides) From f3ccd8655abc6aab0d00add5062128b2c9711c3b Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Wed, 2 May 2018 10:09:27 +0200 Subject: [PATCH 2/6] allow relationship to be explicit excluded --- colanderalchemy/schema.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/colanderalchemy/schema.py b/colanderalchemy/schema.py index 7a56e24..01621a6 100644 --- a/colanderalchemy/schema.py +++ b/colanderalchemy/schema.py @@ -182,12 +182,6 @@ def add_nodes(self, includes, excludes, overrides): log.debug('Attribute %s skipped imperatively', name) continue - key = 'exclude' - if overrides.get(name, {}).get(key, None): - log.debug('Attribute %s skipped due to imperative overrides', - name) - continue - name_overrides_copy = overrides.get(name, {}).copy() if (isinstance(prop, ColumnProperty) @@ -247,14 +241,19 @@ def get_schema_from_column(self, prop, overrides): key = 'exclude' - if declarative_overrides.pop(key, False): + if key not in itertools.chain(declarative_overrides, overrides) \ + and typedecorator_overrides.pop(key, False): log.debug('Column %s skipped due to TypeDecorator overrides', name) return None - if declarative_overrides.pop(key, False): + if key not in overrides and declarative_overrides.pop(key, False): log.debug('Column %s skipped due to declarative overrides', name) return None + if overrides.pop(key, False): + log.debug('Column %s skipped due to imperative overrides', name) + return None + self.check_overrides(name, 'name', typedecorator_overrides, declarative_overrides, overrides) @@ -368,12 +367,12 @@ def get_schema_from_column(self, prop, overrides): 1 arg version takes ExecutionContext - set missing to 'drop' to allow SQLA to fill this in and make it an unrequired field - + if nullable, then missing = colander.null (this has to be the case since some colander types won't accept `None` as a value, but all accept `colander.null`) - - all values for server_default should result in 'drop' + + all values for server_default should result in 'drop' for Colander missing autoincrement results in drop @@ -460,11 +459,17 @@ def get_schema_from_relationship(self, prop, overrides): class_ = prop.mapper.class_ - if declarative_overrides.pop('exclude', False): + key = 'exclude' + if declarative_overrides.pop(key, False): log.debug('Relationship %s skipped due to declarative overrides', name) return None + if overrides.pop(key, False): + log.debug('Relationship %s skipped due to imperative overrides', + name) + return None + for key in ['name', 'typ']: self.check_overrides(name, key, {}, declarative_overrides, overrides) From c87c409ea287a96df792a1b811b9dd6171d3b25c Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Wed, 2 May 2018 11:07:51 +0200 Subject: [PATCH 3/6] tests for exclude relationship --- tests/test_schema.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index ba40cbf..e9de609 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1368,3 +1368,50 @@ def test_relationship_infinite_recursion(self): # Unpatched, creating a bar or baz schema node causes infinite recursion schema = SQLAlchemySchemaNode(Bar) schema = SQLAlchemySchemaNode(Baz) + + def test_exclude_relationship(self): + overrides = { + 'person': { + 'includes': ['name', 'surname', 'gender', 'addresses'], + 'overrides': { + 'addresses': { + 'exclude': True, + } + } + } + } + includes = ['email', 'enabled', 'created', 'timeout', 'person'] + account_schema = SQLAlchemySchemaNode( + Account, overrides=overrides, includes=includes) + address_args = dict(street='My Street', city='My City') + address = Address(**address_args) + + person_args = dict(name='My Name', surname='My Surname', + gender='M', addresses=[address]) + person = Person(**person_args) + + account_args = dict(email='mailbox@domain.tld', + enabled=True, + created=datetime.datetime.now(), + timeout=datetime.time(hour=1, minute=0), + person=person) + account = Account(**account_args) + + appstruct = account_schema.dictify(account) + self.maxDiff = None + + person_args.pop('addresses') + account_args['person'] = person_args + self.assertEqual(appstruct, account_args) + for account_key in account_args: + self.assertIn(account_key, appstruct) + if account_key == 'person': + for person_key in person_args: + self.assertIn(person_key, appstruct[account_key]) + cstruct = account_schema.serialize(appstruct=appstruct) + newappstruct = account_schema.deserialize(cstruct) + self.assertEqual(appstruct, newappstruct) + + cstruct = account_schema.serialize(appstruct=appstruct) + newappstruct = account_schema.deserialize(cstruct) + self.assertEqual(appstruct, newappstruct) From 5c4f27b93bc096dc77792a78ad3c63b556cec8bc Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Thu, 3 May 2018 11:42:13 +0200 Subject: [PATCH 4/6] allow to pass kwargs pass to none list mapping --- colanderalchemy/schema.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/colanderalchemy/schema.py b/colanderalchemy/schema.py index 01621a6..1aae816 100644 --- a/colanderalchemy/schema.py +++ b/colanderalchemy/schema.py @@ -560,16 +560,22 @@ def get_schema_from_relationship(self, prop, overrides): # xToOne relationships. return SchemaNode(Mapping(), *children, **kwargs) - node = SQLAlchemySchemaNode(class_, - name=name, - includes=includes, - excludes=excludes, - overrides=rel_overrides, - missing=missing, - parents_=self.parents_ + [self.class_]) - if prop.uselist: + node = SQLAlchemySchemaNode(class_, + name=name, + includes=includes, + excludes=excludes, + overrides=rel_overrides, + missing=missing, + parents_=self.parents_ + [self.class_]) node = SchemaNode(Sequence(), node, **kwargs) + else: + kwargs['name'] = name + kwargs['includes'] = includes + kwargs['excludes'] = excludes + kwargs['overrides'] = rel_overrides + kwargs['parents_'] = self.parents_ + [self.class_] + node = SQLAlchemySchemaNode(class_, **kwargs) node.name = name From b08a28fd4c2803f7e3db9ad102fbb2ac47593f95 Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Wed, 13 Jun 2018 15:25:28 +0200 Subject: [PATCH 5/6] don't objectify empty dictionaries --- colanderalchemy/schema.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/colanderalchemy/schema.py b/colanderalchemy/schema.py index 1aae816..c806f4e 100644 --- a/colanderalchemy/schema.py +++ b/colanderalchemy/schema.py @@ -707,7 +707,11 @@ def objectify(self, dict_, context=None): for obj in dict_[attr]] else: # Single object - value = self[attr].objectify(dict_[attr]) + sub_dict = dict_[attr] + if sub_dict: + value = self[attr].objectify(sub_dict) + else: + value = None else: value = dict_[attr] if value is colander.null: From 72c8834b82bcfc75ea25c1b73a71668715b2ce12 Mon Sep 17 00:00:00 2001 From: Karsten-Merkle Date: Wed, 13 Jun 2018 15:26:40 +0200 Subject: [PATCH 6/6] test empty relationships --- tests/test_schema.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index e9de609..479cbed 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1415,3 +1415,41 @@ def test_exclude_relationship(self): cstruct = account_schema.serialize(appstruct=appstruct) newappstruct = account_schema.deserialize(cstruct) self.assertEqual(appstruct, newappstruct) + + def test_empty_dict_relationship(self): + dict_ = { + 'person': {}, + 'enabled': True, + 'email': 'mailbox@domain.tld', + 'timeout': datetime.time(hour=0, minute=0), + 'created': datetime.datetime.now(), + 'foobar': 'a fake value', # Not present in schema + 'buzbaz': {} # Not present in schema + } + schema = self._prep_schema() + + objectified = schema.objectify(dict_) + self.assertIsInstance(objectified, Account) + self.assertEqual(objectified.email, 'mailbox@domain.tld') + self.assertIsNone(objectified.person) + self.assertFalse(hasattr(objectified, 'foobar')) + self.assertFalse(hasattr(objectified, 'buzbaz')) + + def test_none_relationship(self): + dict_ = { + 'person': None, + 'enabled': True, + 'email': 'mailbox@domain.tld', + 'timeout': datetime.time(hour=0, minute=0), + 'created': datetime.datetime.now(), + 'foobar': 'a fake value', # Not present in schema + 'buzbaz': None # Not present in schema + } + schema = self._prep_schema() + + objectified = schema.objectify(dict_) + self.assertIsInstance(objectified, Account) + self.assertEqual(objectified.email, 'mailbox@domain.tld') + self.assertIsNone(objectified.person) + self.assertFalse(hasattr(objectified, 'foobar')) + self.assertFalse(hasattr(objectified, 'buzbaz'))