From 56f03e1229c4148052ee061b8a5b1d0090934d9a Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Thu, 15 Aug 2013 16:56:30 +0300 Subject: [PATCH 1/8] Better class naming --- tests/batch_fetch/test_deep_relationships.py | 2 +- tests/batch_fetch/test_join_table_inheritance.py | 2 +- ...imple_relationships.py => test_one_to_many_relationships.py} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/batch_fetch/{test_simple_relationships.py => test_one_to_many_relationships.py} (97%) diff --git a/tests/batch_fetch/test_deep_relationships.py b/tests/batch_fetch/test_deep_relationships.py index b837c29..a4b6737 100644 --- a/tests/batch_fetch/test_deep_relationships.py +++ b/tests/batch_fetch/test_deep_relationships.py @@ -3,7 +3,7 @@ from sqlalchemy_utils import batch_fetch, with_backrefs from tests import TestCase -class TestBatchFetch(TestCase): +class TestBatchFetchDeepRelationships(TestCase): def create_models(self): class User(self.Base): __tablename__ = 'user' diff --git a/tests/batch_fetch/test_join_table_inheritance.py b/tests/batch_fetch/test_join_table_inheritance.py index dfab812..95abdec 100644 --- a/tests/batch_fetch/test_join_table_inheritance.py +++ b/tests/batch_fetch/test_join_table_inheritance.py @@ -3,7 +3,7 @@ from sqlalchemy_utils import batch_fetch from tests import TestCase -class TestBatchFetch(TestCase): +class TestBatchFetchAssociations(TestCase): def create_models(self): class Category(self.Base): __tablename__ = 'category' diff --git a/tests/batch_fetch/test_simple_relationships.py b/tests/batch_fetch/test_one_to_many_relationships.py similarity index 97% rename from tests/batch_fetch/test_simple_relationships.py rename to tests/batch_fetch/test_one_to_many_relationships.py index 03dfed7..6ae00ea 100644 --- a/tests/batch_fetch/test_simple_relationships.py +++ b/tests/batch_fetch/test_one_to_many_relationships.py @@ -4,7 +4,7 @@ from sqlalchemy_utils import batch_fetch from tests import TestCase -class TestBatchFetch(TestCase): +class TestBatchFetchOneToManyRelationships(TestCase): def create_models(self): class User(self.Base): __tablename__ = 'user' From e62d8c27a8fbd403754ce301df8d53fc4de12bb7 Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Thu, 15 Aug 2013 17:01:25 +0300 Subject: [PATCH 2/8] Added tests for many to one relationship fetching --- .../test_many_to_one_relationships.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/batch_fetch/test_many_to_one_relationships.py diff --git a/tests/batch_fetch/test_many_to_one_relationships.py b/tests/batch_fetch/test_many_to_one_relationships.py new file mode 100644 index 0000000..c8a0e6f --- /dev/null +++ b/tests/batch_fetch/test_many_to_one_relationships.py @@ -0,0 +1,47 @@ +import sqlalchemy as sa +from sqlalchemy_utils import batch_fetch +from tests import TestCase + + +class TestBatchFetchManyToOneRelationships(TestCase): + def create_models(self): + class User(self.Base): + __tablename__ = 'user' + id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) + name = sa.Column(sa.Unicode(255)) + + class Article(self.Base): + __tablename__ = 'article' + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column(sa.Unicode(255)) + author_id = sa.Column(sa.Integer, sa.ForeignKey(User.id)) + + author = sa.orm.relationship( + User, + backref=sa.orm.backref( + 'articles' + ) + ) + + self.User = User + self.Article = Article + + def setup_method(self, method): + TestCase.setup_method(self, method) + articles = [ + self.Article(name=u'Article 1', author=self.User(name=u'John')), + self.Article(name=u'Article 2', author=self.User(name=u'Matt')), + ] + self.session.add_all(articles) + self.session.commit() + + def test_supports_relationship_attributes(self): + articles = self.session.query(self.Article).all() + batch_fetch( + articles, + 'author' + ) + query_count = self.connection.query_count + assert articles[0].author # no lazy load should occur + assert articles[1].author # no lazy load should occur + assert self.connection.query_count == query_count From a7f9021fc624be5a95797e325bcf4d2efacf700c Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Fri, 16 Aug 2013 16:50:25 +0300 Subject: [PATCH 3/8] Bumped version --- CHANGES.rst | 7 ++++++- setup.py | 2 +- sqlalchemy_utils/__init__.py | 2 +- sqlalchemy_utils/functions/batch_fetch.py | 4 ++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 67c22ca..a8f98ae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,11 +3,16 @@ Changelog Here you can see the full list of changes between each SQLAlchemy-Utils release. +0.16.6 (2013-08-16) +^^^^^^^^^^^^^^^^^^^ + +- Rewritten batch_fetch schematics, new syntax for backref population + 0.16.5 (2013-08-08) ^^^^^^^^^^^^^^^^^^^ -- Initial backref population forcing for batch_fetch +- Initial backref population forcing support for batch_fetch 0.16.4 (2013-08-08) diff --git a/setup.py b/setup.py index a4f68d9..b302acc 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ for name, requirements in extras_require.items(): setup( name='SQLAlchemy-Utils', - version='0.16.5', + version='0.16.6', url='https://github.com/kvesteri/sqlalchemy-utils', license='BSD', author='Konsta Vesterinen', diff --git a/sqlalchemy_utils/__init__.py b/sqlalchemy_utils/__init__.py index 8cf3d49..bb56817 100644 --- a/sqlalchemy_utils/__init__.py +++ b/sqlalchemy_utils/__init__.py @@ -37,7 +37,7 @@ from .types import ( ) -__version__ = '0.16.5' +__version__ = '0.16.6' __all__ = ( diff --git a/sqlalchemy_utils/functions/batch_fetch.py b/sqlalchemy_utils/functions/batch_fetch.py index 2e99d62..e7fc2f7 100644 --- a/sqlalchemy_utils/functions/batch_fetch.py +++ b/sqlalchemy_utils/functions/batch_fetch.py @@ -5,6 +5,10 @@ from sqlalchemy.orm.session import object_session class with_backrefs(object): + """ + Marks given attribute path so that whenever its fetched with batch_fetch + the backref relations are force set too. + """ def __init__(self, attr_path): self.attr_path = attr_path From ee8b97f0f78b39073ac6da21eeeb2ff07d7c951e Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Sat, 17 Aug 2013 18:49:02 +0300 Subject: [PATCH 4/8] Added more tests and refactored batch fetch --- sqlalchemy_utils/functions/batch_fetch.py | 188 +++++++++++------- .../test_many_to_one_relationships.py | 12 +- 2 files changed, 130 insertions(+), 70 deletions(-) diff --git a/sqlalchemy_utils/functions/batch_fetch.py b/sqlalchemy_utils/functions/batch_fetch.py index e7fc2f7..7e67375 100644 --- a/sqlalchemy_utils/functions/batch_fetch.py +++ b/sqlalchemy_utils/functions/batch_fetch.py @@ -74,49 +74,17 @@ def batch_fetch(entities, *attr_paths): """ if entities: - fetcher = BatchFetcher(entities) + fetcher = FetchingCoordinator(entities) for attr_path in attr_paths: fetcher(attr_path) -class BatchFetcher(object): +class FetchingCoordinator(object): def __init__(self, entities): self.entities = entities self.first = entities[0] - self.parent_ids = [entity.id for entity in entities] self.session = object_session(self.first) - def populate_backrefs(self, related_entities): - """ - Populates backrefs for given related entities. - """ - - backref_dict = dict( - (entity.id, []) for entity, parent_id in related_entities - ) - for entity, parent_id in related_entities: - backref_dict[entity.id].append( - self.session.query(self.first.__class__).get(parent_id) - ) - for entity, parent_id in related_entities: - set_committed_value( - entity, self.prop.back_populates, backref_dict[entity.id] - ) - - def populate_entities(self): - """ - Populate batch fetched entities to parent objects. - """ - for entity in self.entities: - set_committed_value( - entity, - self.prop.key, - self.parent_dict[entity.id] - ) - - if self.should_populate_backrefs: - self.populate_backrefs(self.related_entities) - def parse_attr_path(self, attr_path, should_populate_backrefs): if isinstance(attr_path, six.string_types): attrs = attr_path.split('.') @@ -150,21 +118,97 @@ class BatchFetcher(object): 'are supported.' ) - column_name = list(self.prop.remote_side)[0].name - - self.related_entities = ( - self.session.query(self.model) - .filter( - getattr(self.model, column_name).in_(self.parent_ids) + def fetcher(self, property_): + if not isinstance(property_, RelationshipProperty): + raise Exception( + 'Given attribute is not a relationship property.' ) + + if property_.secondary is not None: + return ManyToManyFetcher(self, property_) + else: + if property_.direction.name == 'MANYTOONE': + return ManyToOneFetcher(self, property_) + else: + return OneToManyFetcher(self, property_) + + def __call__(self, attr_path): + if isinstance(attr_path, with_backrefs): + self.should_populate_backrefs = True + attr_path = attr_path.attr_path + else: + self.should_populate_backrefs = False + + attr = self.parse_attr_path(attr_path, self.should_populate_backrefs) + if not attr: + return + + fetcher = self.fetcher(attr.property) + fetcher.fetch() + fetcher.populate() + + +class Fetcher(object): + def __init__(self, coordinator, property_): + self.coordinator = coordinator + self.prop = property_ + self.model = self.prop.mapper.class_ + self.entities = coordinator.entities + self.first = self.entities[0] + self.session = object_session(self.first) + + for entity in self.entities: + self.parent_dict = dict( + (self.local_values(entity), []) + for entity in self.entities + ) + + @property + def local_values_list(self): + return [ + self.local_values(entity) + for entity in self.entities + ] + + def local_values(self, entity): + return getattr(entity, list(self.prop.local_columns)[0].name) + + def populate_backrefs(self, related_entities): + """ + Populates backrefs for given related entities. + """ + + backref_dict = dict( + (entity.id, []) for entity, parent_id in related_entities ) - - for entity in self.related_entities: - self.parent_dict[getattr(entity, column_name)].append( - entity + for entity, parent_id in related_entities: + backref_dict[entity.id].append( + self.session.query(self.first.__class__).get(parent_id) + ) + for entity, parent_id in related_entities: + set_committed_value( + entity, self.prop.back_populates, backref_dict[entity.id] ) - def fetch_association_entities(self): + def populate(self): + """ + Populate batch fetched entities to parent objects. + """ + for entity in self.entities: + set_committed_value( + entity, + self.prop.key, + self.parent_dict[self.local_values(entity)] + ) + + if self.coordinator.should_populate_backrefs: + self.populate_backrefs(self.related_entities) + + +class ManyToManyFetcher(Fetcher): + def fetch(self): + parent_ids = [entity.id for entity in self.entities] + column_name = None for column in self.prop.remote_side: for fk in column.foreign_keys: @@ -183,7 +227,7 @@ class BatchFetcher(object): ) .filter( getattr(self.prop.secondary.c, column_name).in_( - self.parent_ids + parent_ids ) ) ) @@ -192,30 +236,38 @@ class BatchFetcher(object): entity ) - def __call__(self, attr_path): - self.parent_dict = dict( - (entity.id, []) for entity in self.entities + +class ManyToOneFetcher(Fetcher): + def fetch(self): + column_name = list(self.prop.remote_side)[0].name + + self.related_entities = ( + self.session.query(self.model) + .filter( + getattr(self.model, column_name).in_(self.local_values_list) + ) ) - if isinstance(attr_path, with_backrefs): - self.should_populate_backrefs = True - attr_path = attr_path.attr_path - else: - self.should_populate_backrefs = False - attr = self.parse_attr_path(attr_path, self.should_populate_backrefs) - if not attr: - return - - self.prop = attr.property - if not isinstance(self.prop, RelationshipProperty): - raise Exception( - 'Given attribute is not a relationship property.' + for entity in self.related_entities: + self.parent_dict[getattr(entity, column_name)].append( + entity ) - self.model = self.prop.mapper.class_ - if self.prop.secondary is None: - self.fetch_relation_entities() - else: - self.fetch_association_entities() - self.populate_entities() +class OneToManyFetcher(Fetcher): + def fetch(self): + parent_ids = [entity.id for entity in self.entities] + + column_name = list(self.prop.remote_side)[0].name + + self.related_entities = ( + self.session.query(self.model) + .filter( + getattr(self.model, column_name).in_(parent_ids) + ) + ) + + for entity in self.related_entities: + self.parent_dict[getattr(entity, column_name)].append( + entity + ) diff --git a/tests/batch_fetch/test_many_to_one_relationships.py b/tests/batch_fetch/test_many_to_one_relationships.py index c8a0e6f..36b0af4 100644 --- a/tests/batch_fetch/test_many_to_one_relationships.py +++ b/tests/batch_fetch/test_many_to_one_relationships.py @@ -29,8 +29,16 @@ class TestBatchFetchManyToOneRelationships(TestCase): def setup_method(self, method): TestCase.setup_method(self, method) articles = [ - self.Article(name=u'Article 1', author=self.User(name=u'John')), - self.Article(name=u'Article 2', author=self.User(name=u'Matt')), + self.Article( + id=1, + name=u'Article 1', + author=self.User(id=333, name=u'John') + ), + self.Article( + id=2, + name=u'Article 2', + author=self.User(id=334, name=u'Matt') + ), ] self.session.add_all(articles) self.session.commit() From 32f84a5c5df2c7fa79c33b41747c687ca3fab518 Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Sat, 17 Aug 2013 19:04:06 +0300 Subject: [PATCH 5/8] Smarter local value checking for batch fetch --- sqlalchemy_utils/functions/batch_fetch.py | 27 ++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/sqlalchemy_utils/functions/batch_fetch.py b/sqlalchemy_utils/functions/batch_fetch.py index 7e67375..7c8c1cf 100644 --- a/sqlalchemy_utils/functions/batch_fetch.py +++ b/sqlalchemy_utils/functions/batch_fetch.py @@ -157,11 +157,10 @@ class Fetcher(object): self.first = self.entities[0] self.session = object_session(self.first) - for entity in self.entities: - self.parent_dict = dict( - (self.local_values(entity), []) - for entity in self.entities - ) + self.parent_dict = dict( + (self.local_values(entity), []) + for entity in self.entities + ) @property def local_values_list(self): @@ -177,17 +176,19 @@ class Fetcher(object): """ Populates backrefs for given related entities. """ - backref_dict = dict( - (entity.id, []) for entity, parent_id in related_entities + (self.local_values(entity), []) + for entity, parent_id in related_entities ) for entity, parent_id in related_entities: - backref_dict[entity.id].append( + backref_dict[self.local_values(entity)].append( self.session.query(self.first.__class__).get(parent_id) ) for entity, parent_id in related_entities: set_committed_value( - entity, self.prop.back_populates, backref_dict[entity.id] + entity, + self.prop.back_populates, + backref_dict[self.local_values(entity)] ) def populate(self): @@ -207,8 +208,6 @@ class Fetcher(object): class ManyToManyFetcher(Fetcher): def fetch(self): - parent_ids = [entity.id for entity in self.entities] - column_name = None for column in self.prop.remote_side: for fk in column.foreign_keys: @@ -227,7 +226,7 @@ class ManyToManyFetcher(Fetcher): ) .filter( getattr(self.prop.secondary.c, column_name).in_( - parent_ids + self.local_values_list ) ) ) @@ -256,14 +255,12 @@ class ManyToOneFetcher(Fetcher): class OneToManyFetcher(Fetcher): def fetch(self): - parent_ids = [entity.id for entity in self.entities] - column_name = list(self.prop.remote_side)[0].name self.related_entities = ( self.session.query(self.model) .filter( - getattr(self.model, column_name).in_(parent_ids) + getattr(self.model, column_name).in_(self.local_values_list) ) ) From 1cf6d2216be21e265b388fb3a053b9c2559af0bb Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Sun, 18 Aug 2013 11:50:07 +0300 Subject: [PATCH 6/8] pep8 fix --- sqlalchemy_utils/types/password.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_utils/types/password.py b/sqlalchemy_utils/types/password.py index ec6d1c6..6c48036 100644 --- a/sqlalchemy_utils/types/password.py +++ b/sqlalchemy_utils/types/password.py @@ -94,8 +94,11 @@ class PasswordType(types.TypeDecorator, ScalarCoercible): length = 4 + len(scheme.name) length += len(str(getattr(scheme, 'max_rounds', ''))) length += scheme.max_salt_size or 0 - length += getattr(scheme, 'encoded_checksum_size', - scheme.checksum_size) + length += getattr( + scheme, + 'encoded_checksum_size', + scheme.checksum_size + ) max_lengths.append(length) # Set the max_length to the maximum calculated max length. From 7751fb0035e0b9e92f15f4bef5bbb62a5d4d0fa6 Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Sun, 18 Aug 2013 12:04:25 +0300 Subject: [PATCH 7/8] PasswordType now falls back to default length (1024) if no schemes were found in crypt context --- sqlalchemy_utils/types/password.py | 2 +- tests/types/test_password.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlalchemy_utils/types/password.py b/sqlalchemy_utils/types/password.py index 6c48036..6c571df 100644 --- a/sqlalchemy_utils/types/password.py +++ b/sqlalchemy_utils/types/password.py @@ -88,7 +88,7 @@ class PasswordType(types.TypeDecorator, ScalarCoercible): if max_length is None: # Calculate the largest possible encoded password. # name + rounds + salt + hash + ($ * 4) of largest hash - max_lengths = [] + max_lengths = [1024] for name in self.context.schemes(): scheme = getattr(__import__('passlib.hash').hash, name) length = 4 + len(scheme.name) diff --git a/tests/types/test_password.py b/tests/types/test_password.py index da4209c..5b1907b 100644 --- a/tests/types/test_password.py +++ b/tests/types/test_password.py @@ -8,7 +8,6 @@ from sqlalchemy_utils import Password, PasswordType @mark.skipif('password.passlib is None') class TestPasswordType(TestCase): - def create_models(self): class User(self.Base): __tablename__ = 'user' @@ -86,3 +85,6 @@ class TestPasswordType(TestCase): expected_length += 4 assert impl.length == expected_length + + def test_without_schemes(self): + assert PasswordType(schemes=[]).length == 1024 From 65ec0fc182465a1ab2be0e067e63385796e171ba Mon Sep 17 00:00:00 2001 From: Konsta Vesterinen Date: Sun, 18 Aug 2013 12:13:29 +0300 Subject: [PATCH 8/8] Bumped version --- CHANGES.rst | 8 ++++++++ setup.py | 2 +- sqlalchemy_utils/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a8f98ae..8e85aaa 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,14 @@ Changelog Here you can see the full list of changes between each SQLAlchemy-Utils release. + +0.16.7 (2013-08-18) +^^^^^^^^^^^^^^^^^^^ + +- Added better handling of local column names in batch_fetch +- PasswordType gets default length even if no crypt context schemes provided + + 0.16.6 (2013-08-16) ^^^^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index b302acc..960b7a7 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ for name, requirements in extras_require.items(): setup( name='SQLAlchemy-Utils', - version='0.16.6', + version='0.16.7', url='https://github.com/kvesteri/sqlalchemy-utils', license='BSD', author='Konsta Vesterinen', diff --git a/sqlalchemy_utils/__init__.py b/sqlalchemy_utils/__init__.py index bb56817..42ca103 100644 --- a/sqlalchemy_utils/__init__.py +++ b/sqlalchemy_utils/__init__.py @@ -37,7 +37,7 @@ from .types import ( ) -__version__ = '0.16.6' +__version__ = '0.16.7' __all__ = (